From 838ff99f37d88214829018c6a7dd2a84fdb418b4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 1 Jan 2025 21:31:04 +0500 Subject: [PATCH 001/491] Enable RUF Ruff rules. --- pyproject.toml | 10 ++++++++++ scrapy/__init__.py | 12 ++++++------ scrapy/commands/crawl.py | 5 ++--- scrapy/commands/parse.py | 2 +- scrapy/commands/runspider.py | 2 +- scrapy/downloadermiddlewares/redirect.py | 12 +++++++----- scrapy/exporters.py | 8 ++++---- scrapy/extensions/debug.py | 2 +- scrapy/http/request/json_request.py | 2 +- scrapy/http/response/text.py | 2 +- scrapy/link.py | 2 +- scrapy/logformatter.py | 4 ++-- scrapy/pipelines/media.py | 3 +-- scrapy/spidermiddlewares/referer.py | 8 +++----- scrapy/utils/misc.py | 4 +++- scrapy/utils/reactor.py | 2 +- scrapy/utils/request.py | 3 ++- scrapy/utils/testproc.py | 2 +- scrapy/utils/url.py | 2 +- scrapy/utils/versions.py | 2 +- tests/test_cmdline/__init__.py | 2 +- tests/test_commands.py | 4 ++-- tests/test_crawler.py | 2 +- tests/test_downloadermiddleware_retry.py | 2 +- tests/test_downloaderslotssettings.py | 2 +- tests/test_exporters.py | 2 +- tests/test_http_response.py | 2 +- tests/test_utils_trackref.py | 8 ++++---- 28 files changed, 62 insertions(+), 51 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ad85e5c75..065382205 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -246,6 +246,8 @@ extend-select = [ "RET", # flake8-raise "RSE", + # Ruff-specific rules + "RUF", # flake8-bandit "S", # flake8-slots @@ -324,6 +326,14 @@ ignore = [ "PLR2004", # `for` loop variable overwritten by assignment target "PLW2901", + # String contains ambiguous {}. + "RUF001", + # Docstring contains ambiguous {}. + "RUF002", + # Comment contains ambiguous {}. + "RUF003", + # Mutable class attributes should be annotated with `typing.ClassVar` + "RUF012", # Use of `assert` detected; needed for mypy "S101", # FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180 diff --git a/scrapy/__init__.py b/scrapy/__init__.py index c19710a6a..256504c9c 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -13,14 +13,14 @@ from scrapy.selector import Selector from scrapy.spiders import Spider __all__ = [ + "Field", + "FormRequest", + "Item", + "Request", + "Selector", + "Spider", "__version__", "version_info", - "Spider", - "Request", - "FormRequest", - "Selector", - "Item", - "Field", ] diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 86d4cc41c..184bd5ca4 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -39,9 +39,8 @@ class Command(BaseRunSpiderCommand): else: self.crawler_process.start() - if ( - self.crawler_process.bootstrap_failed - or hasattr(self.crawler_process, "has_exception") + if self.crawler_process.bootstrap_failed or ( + hasattr(self.crawler_process, "has_exception") and self.crawler_process.has_exception ): self.exitcode = 1 diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index cc5c1350b..f996d1806 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -269,7 +269,7 @@ class Command(BaseRunSpiderCommand): assert self.crawler_process assert self.spidercls self.crawler_process.crawl(self.spidercls, **opts.spargs) - self.pcrawler = list(self.crawler_process.crawlers)[0] + self.pcrawler = next(iter(self.crawler_process.crawlers)) self.crawler_process.start() if not self.first_response: diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index bf8e41020..357ca8b37 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -20,7 +20,7 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType: if abspath.suffix not in (".py", ".pyw"): raise ValueError(f"Not a Python source file: {abspath}") dirname = str(abspath.parent) - sys.path = [dirname] + sys.path + sys.path = [dirname, *sys.path] try: module = import_module(abspath.stem) finally: diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 0b883b43a..612426371 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -101,12 +101,14 @@ class BaseRedirectMiddleware: if ttl and redirects <= self.max_redirect_times: redirected.meta["redirect_times"] = redirects redirected.meta["redirect_ttl"] = ttl - 1 - redirected.meta["redirect_urls"] = request.meta.get("redirect_urls", []) + [ - request.url + redirected.meta["redirect_urls"] = [ + *request.meta.get("redirect_urls", []), + request.url, + ] + redirected.meta["redirect_reasons"] = [ + *request.meta.get("redirect_reasons", []), + reason, ] - redirected.meta["redirect_reasons"] = request.meta.get( - "redirect_reasons", [] - ) + [reason] redirected.dont_filter = request.dont_filter redirected.priority = request.priority + self.priority_adjust logger.debug( diff --git a/scrapy/exporters.py b/scrapy/exporters.py index cdb7ac159..834a05ae9 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -25,13 +25,13 @@ if TYPE_CHECKING: __all__ = [ "BaseItemExporter", - "PprintItemExporter", - "PickleItemExporter", "CsvItemExporter", - "XmlItemExporter", - "JsonLinesItemExporter", "JsonItemExporter", + "JsonLinesItemExporter", "MarshalItemExporter", + "PickleItemExporter", + "PprintItemExporter", + "XmlItemExporter", ] diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 6948c394c..5ca07394f 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -77,4 +77,4 @@ class Debugger: def _enter_debugger(self, signum: int, frame: FrameType | None) -> None: assert frame - Pdb().set_trace(frame.f_back) # noqa: T100 + Pdb().set_trace(frame.f_back) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 289c60591..e5b63ef14 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -20,7 +20,7 @@ if TYPE_CHECKING: class JsonRequest(Request): - attributes: tuple[str, ...] = Request.attributes + ("dumps_kwargs",) + attributes: tuple[str, ...] = (*Request.attributes, "dumps_kwargs") def __init__( self, *args: Any, dumps_kwargs: dict[str, Any] | None = None, **kwargs: Any diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index f954b5e9e..476f1754e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -43,7 +43,7 @@ class TextResponse(Response): _DEFAULT_ENCODING = "ascii" _cached_decoded_json = _NONE - attributes: tuple[str, ...] = Response.attributes + ("encoding",) + attributes: tuple[str, ...] = (*Response.attributes, "encoding") def __init__(self, *args: Any, **kwargs: Any): self._encoding: str | None = kwargs.pop("encoding", None) diff --git a/scrapy/link.py b/scrapy/link.py index 1a569f892..9c272ab2f 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -24,7 +24,7 @@ class Link: of the anchor tag. """ - __slots__ = ["url", "text", "fragment", "nofollow"] + __slots__ = ["fragment", "nofollow", "text", "url"] def __init__( self, url: str, text: str = "", fragment: str = "", nofollow: bool = False diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 544f4adfe..76f9c7856 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -76,8 +76,8 @@ class LogFormatter: self, request: Request, response: Response, spider: Spider ) -> LogFormatterResult: """Logs a message when the crawler finds a webpage.""" - request_flags = f" {str(request.flags)}" if request.flags else "" - response_flags = f" {str(response.flags)}" if response.flags else "" + request_flags = f" {request.flags!s}" if request.flags else "" + response_flags = f" {response.flags!s}" if response.flags else "" return { "level": logging.DEBUG, "msg": CRAWLEDMSG, diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 5438b8522..0f3329db1 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -127,8 +127,7 @@ class MediaPipeline(ABC): if ( not base_class_name or class_name == base_class_name - or settings - and not settings.get(formatted_key) + or (settings and not settings.get(formatted_key)) ): return key return formatted_key diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 93b7fcf17..18cc991bf 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -195,8 +195,7 @@ class StrictOriginPolicy(ReferrerPolicy): if ( self.tls_protected(response_url) and self.potentially_trustworthy(request_url) - or not self.tls_protected(response_url) - ): + ) or not self.tls_protected(response_url): return self.origin_referrer(response_url) return None @@ -249,8 +248,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): if ( self.tls_protected(response_url) and self.potentially_trustworthy(request_url) - or not self.tls_protected(response_url) - ): + ) or not self.tls_protected(response_url): return self.origin_referrer(response_url) return None @@ -282,7 +280,7 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): using ``file://`` or ``s3://`` scheme. """ - NOREFERRER_SCHEMES: tuple[str, ...] = LOCAL_SCHEMES + ("file", "s3") + NOREFERRER_SCHEMES: tuple[str, ...] = (*LOCAL_SCHEMES, "file", "s3") name: str = POLICY_SCRAPY_DEFAULT diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 5ce4863f6..d319e7950 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -252,7 +252,9 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: def returns_none(return_node: ast.Return) -> bool: value = return_node.value - return value is None or isinstance(value, ast.Constant) and value.value is None + return value is None or ( + isinstance(value, ast.Constant) and value.value is None + ) if inspect.isgeneratorfunction(callable): func = callable diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 66a06a9f0..679e38206 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -100,7 +100,7 @@ def install_reactor(reactor_path: str, event_loop_path: str | None = None) -> No asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") - installer_path = module + ["install"] + installer_path = [*module, "install"] installer = load_object(".".join(installer_path)) with suppress(error.ReactorAlreadyInstalledError): installer() diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index ad811e804..7f2b178f5 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -229,7 +229,8 @@ def request_to_curl(request: Request) -> str: cookies = f"--cookie '{cookie}'" elif isinstance(request.cookies, list): cookie = "; ".join( - f"{list(c.keys())[0]}={list(c.values())[0]}" for c in request.cookies + f"{next(iter(c.keys()))}={next(iter(c.values()))}" + for c in request.cookies ) cookies = f"--cookie '{cookie}'" diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 05e04e2d1..3b1035eab 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -31,7 +31,7 @@ class ProcessTest: if settings is not None: env["SCRAPY_SETTINGS_MODULE"] = settings assert self.command - cmd = self.prefix + [self.command] + list(args) + cmd = [*self.prefix, self.command, *args] pp = TestProcessProtocol() pp.deferred.addCallback(self._process_finished, cmd, check_code) reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 3bf831c26..d487849bb 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -51,7 +51,7 @@ def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool: """Return True if the url belongs to the given spider""" return url_is_from_any_domain( - url, [spider.name] + list(getattr(spider, "allowed_domains", [])) + url, [spider.name, *getattr(spider, "allowed_domains", [])] ) diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index ff1f9b346..052321ae3 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -11,7 +11,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings.default_settings import LOG_VERSIONS from scrapy.utils.ssl import get_openssl_version -_DEFAULT_SOFTWARE = ["Scrapy"] + LOG_VERSIONS +_DEFAULT_SOFTWARE = ["Scrapy", *LOG_VERSIONS] def _version(item): diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 4835e936b..acd524ea4 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -21,7 +21,7 @@ class CmdlineTest(unittest.TestCase): def _execute(self, *new_args, **kwargs): encoding = sys.stdout.encoding or "utf-8" - args = (sys.executable, "-m", "scrapy.cmdline") + new_args + args = (sys.executable, "-m", "scrapy.cmdline", *new_args) proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs) comm = proc.communicate()[0].strip() return comm.decode(encoding) diff --git a/tests/test_commands.py b/tests/test_commands.py index 32b69de8a..9d5720b98 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -87,13 +87,13 @@ class ProjectTest(unittest.TestCase): def call(self, *new_args, **kwargs): with TemporaryFile() as out: - args = (sys.executable, "-m", "scrapy.cmdline") + new_args + args = (sys.executable, "-m", "scrapy.cmdline", *new_args) return subprocess.call( args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **kwargs ) def proc(self, *new_args, **popen_kwargs): - args = (sys.executable, "-m", "scrapy.cmdline") + new_args + args = (sys.executable, "-m", "scrapy.cmdline", *new_args) p = subprocess.Popen( args, cwd=popen_kwargs.pop("cwd", self.cwd), diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f3e5ebf5d..8b3a6eeca 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -647,7 +647,7 @@ class ScriptRunnerMixin: def get_script_args(self, script_name: str, *script_args: str) -> list[str]: script_path = self.script_dir / script_name - return [sys.executable, str(script_path)] + list(script_args) + return [sys.executable, str(script_path), *script_args] def run_script(self, script_name: str, *script_args: str) -> str: args = self.get_script_args(script_name, *script_args) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index c99f19b03..1eb7dcf9d 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -114,7 +114,7 @@ class RetryTest(unittest.TestCase): def test_exception_to_retry_added(self): exc = ValueError settings_dict = { - "RETRY_EXCEPTIONS": list(RETRY_EXCEPTIONS) + [exc], + "RETRY_EXCEPTIONS": [*RETRY_EXCEPTIONS, exc], } crawler = get_crawler(Spider, settings_dict=settings_dict) mw = RetryMiddleware.from_crawler(crawler) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 55f9ecac9..879bc8697 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -31,7 +31,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): def start_requests(self): self.times = {None: []} - slots = list(self.custom_settings.get("DOWNLOAD_SLOTS", {}).keys()) + [None] + slots = [*self.custom_settings.get("DOWNLOAD_SLOTS", {}), None] for slot in slots: url = self.mockserver.url(f"/?downloader_slot={slot}") diff --git a/tests/test_exporters.py b/tests/test_exporters.py index fa9389044..522c6638d 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -116,7 +116,7 @@ class BaseItemExporterTest(unittest.TestCase): ) ie = self._get_exporter(fields_to_export=["name"], encoding="latin-1") - _, name = list(ie._get_serialized_fields(self.i))[0] + _, name = next(iter(ie._get_serialized_fields(self.i))) assert isinstance(name, str) self.assertEqual(name, "John\xa3") diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 679cc8238..0730cff3a 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -960,7 +960,7 @@ class XmlResponseTest(TextResponseTest): class CustomResponse(TextResponse): - attributes = TextResponse.attributes + ("foo", "bar") + attributes = (*TextResponse.attributes, "foo", "bar") def __init__(self, *args, **kwargs) -> None: self.foo = kwargs.pop("foo", None) diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index ef07d625f..58efad585 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -61,11 +61,11 @@ Foo 1 oldest: 0s ago\n\n""", ) def test_get_oldest(self): - o1 = Foo() # noqa: F841 + o1 = Foo() o1_time = time() - o2 = Bar() # noqa: F841 + o2 = Bar() o3_time = time() if o3_time <= o1_time: @@ -80,9 +80,9 @@ Foo 1 oldest: 0s ago\n\n""", self.assertIsNone(trackref.get_oldest("XXX")) def test_iter_all(self): - o1 = Foo() # noqa: F841 + o1 = Foo() o2 = Bar() # noqa: F841 - o3 = Foo() # noqa: F841 + o3 = Foo() self.assertEqual( set(trackref.iter_all("Foo")), {o1, o3}, From f44ca39fa23f07c857a34309d79db6394c5faefb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 1 Jan 2025 21:50:02 +0500 Subject: [PATCH 002/491] Enable FLY Ruff rules. --- pyproject.toml | 2 ++ scrapy/pqueues.py | 2 +- tests/test_spidermiddleware_referer.py | 25 +++---------------------- 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 065382205..a0b37b966 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -222,6 +222,8 @@ extend-select = [ "D", # flake8-future-annotations "FA", + # flynt + "FLY", # refurb "FURB", # isort diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 5b2f81335..a04e0107b 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -34,7 +34,7 @@ def _path_safe(text: str) -> str: # as we replace some letters we can get collision for different slots # add we add unique part unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # noqa: S324 - return "-".join([pathable_slot, unique_slot]) + return f"{pathable_slot}-{unique_slot}" class QueueProtocol(Protocol): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 23b0c17c6..cefd33e4e 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -891,14 +891,7 @@ class TestSettingsPolicyByName(TestCase): # test parsing without space(s) after the comma settings1 = Settings( { - "REFERRER_POLICY": ",".join( - [ - "some-custom-unknown-policy", - POLICY_SAME_ORIGIN, - POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, - "another-custom-unknown-policy", - ] - ) + "REFERRER_POLICY": f"some-custom-unknown-policy,{POLICY_SAME_ORIGIN},{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN},another-custom-unknown-policy" } ) mw1 = RefererMiddleware(settings1) @@ -907,13 +900,7 @@ class TestSettingsPolicyByName(TestCase): # test parsing with space(s) after the comma settings2 = Settings( { - "REFERRER_POLICY": ", ".join( - [ - POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, - "another-custom-unknown-policy", - POLICY_UNSAFE_URL, - ] - ) + "REFERRER_POLICY": f"{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}, another-custom-unknown-policy, {POLICY_UNSAFE_URL}" } ) mw2 = RefererMiddleware(settings2) @@ -922,13 +909,7 @@ class TestSettingsPolicyByName(TestCase): def test_multiple_policy_tokens_all_invalid(self): settings = Settings( { - "REFERRER_POLICY": ",".join( - [ - "some-custom-unknown-policy", - "another-custom-unknown-policy", - "yet-another-custom-unknown-policy", - ] - ) + "REFERRER_POLICY": "some-custom-unknown-policy,another-custom-unknown-policy,yet-another-custom-unknown-policy" } ) with self.assertRaises(RuntimeError): From 273620488ced7dd1a3c8a1c5022d17f0cc7f9496 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 1 Jan 2025 22:03:42 +0500 Subject: [PATCH 003/491] Enable PTH Ruff rules. --- pyproject.toml | 2 ++ scrapy/commands/genspider.py | 2 +- scrapy/commands/startproject.py | 7 +++---- scrapy/core/downloader/handlers/ftp.py | 15 +++++++++------ scrapy/utils/testproc.py | 2 +- tests/test_crawler.py | 2 -- tests/test_downloader_handlers.py | 2 +- tests/test_utils_project.py | 2 +- 8 files changed, 18 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a0b37b966..08b4b09b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -240,6 +240,8 @@ extend-select = [ "PIE", # pylint "PL", + # flake8-use-pathlib + "PTH", # flake8-pyi "PYI", # flake8-quotes diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index d7dc104c2..2a1dea997 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -154,7 +154,7 @@ class Command(ScrapyCommand): spiders_dir = Path(spiders_module.__file__).parent.resolve() else: spiders_module = None - spiders_dir = Path(".") + spiders_dir = Path() spider_file = f"{spiders_dir / module}.py" shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 5cb73f0d2..e0c004580 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import re import string from importlib.util import find_spec @@ -28,9 +27,9 @@ TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = ( IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn") -def _make_writable(path: str | os.PathLike) -> None: - current_permissions = os.stat(path).st_mode - os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION) +def _make_writable(path: Path) -> None: + current_permissions = path.stat().st_mode + path.chmod(current_permissions | OWNER_WRITE_PERMISSION) class Command(ScrapyCommand): diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 598659b4d..0ad10baff 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -32,6 +32,7 @@ from __future__ import annotations import re from io import BytesIO +from pathlib import Path from typing import TYPE_CHECKING, Any, BinaryIO from urllib.parse import unquote @@ -56,9 +57,11 @@ if TYPE_CHECKING: class ReceivedDataProtocol(Protocol): - def __init__(self, filename: str | None = None): - self.__filename: str | None = filename - self.body: BinaryIO = open(filename, "wb") if filename else BytesIO() + def __init__(self, filename: bytes | None = None): + self.__filename: bytes | None = filename + self.body: BinaryIO = ( + Path(filename.decode()).open("wb") if filename else BytesIO() + ) self.size: int = 0 def dataReceived(self, data: bytes) -> None: @@ -66,7 +69,7 @@ class ReceivedDataProtocol(Protocol): self.size += len(data) @property - def filename(self) -> str | None: + def filename(self) -> bytes | None: return self.__filename def close(self) -> None: @@ -128,8 +131,8 @@ class FTPDownloadHandler: ) -> Response: self.result = result protocol.close() - headers = {"local filename": protocol.filename or "", "size": protocol.size} - body = to_bytes(protocol.filename or protocol.body.read()) + headers = {"local filename": protocol.filename or b"", "size": protocol.size} + body = protocol.filename or protocol.body.read() respcls = responsetypes.from_args(url=request.url, body=body) # hints for Headers-related types may need to be fixed to not use AnyStr return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type] diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 3b1035eab..85d7c940f 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -17,7 +17,7 @@ if TYPE_CHECKING: class ProcessTest: command: str | None = None prefix = [sys.executable, "-m", "scrapy.cmdline"] - cwd = os.getcwd() # trial chdirs to temp dir + cwd = os.getcwd() # trial chdirs to temp dir # noqa: PTH109 def execute( self, diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 8b3a6eeca..6c3fe96b0 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,5 +1,4 @@ import logging -import os import platform import re import signal @@ -643,7 +642,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase): class ScriptRunnerMixin: script_dir: Path - cwd = os.getcwd() def get_script_args(self, script_name: str, *script_args: str) -> list[str]: script_path = self.script_dir / script_name diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 8ecba41bf..05b64e704 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -116,7 +116,7 @@ class FileTestCase(unittest.TestCase): def tearDown(self): os.close(self.fd) - os.remove(self.tmpname) + Path(self.tmpname).unlink() def test_download(self): def _test(response): diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 3831f4c21..1d149d48d 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -12,7 +12,7 @@ from scrapy.utils.project import data_path, get_project_settings @contextlib.contextmanager def inside_a_project(): - prev_dir = os.getcwd() + prev_dir = Path.cwd() project_dir = tempfile.mkdtemp() try: From c87354cd46afdba35ca104a62b0c2bbfa6bd6f64 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 1 Jan 2025 23:05:07 +0500 Subject: [PATCH 004/491] Enable SIM Ruff rules. --- pyproject.toml | 8 +++++++ scrapy/cmdline.py | 4 +--- scrapy/commands/parse.py | 11 ++++----- scrapy/commands/startproject.py | 5 +--- scrapy/core/downloader/handlers/http11.py | 5 +--- scrapy/core/engine.py | 9 +++----- scrapy/core/spidermw.py | 5 +--- scrapy/crawler.py | 6 ++--- scrapy/downloadermiddlewares/httpauth.py | 9 +++++--- scrapy/downloadermiddlewares/httpproxy.py | 5 +--- scrapy/exporters.py | 5 +--- scrapy/extensions/debug.py | 7 +++--- scrapy/extensions/feedexport.py | 5 ++-- scrapy/extensions/httpcache.py | 5 +--- scrapy/extensions/periodic_log.py | 5 +--- scrapy/http/cookies.py | 5 ++-- scrapy/http/request/json_request.py | 4 ++-- scrapy/linkextractors/lxmlhtml.py | 13 ++++++----- scrapy/mail.py | 8 +++---- scrapy/pipelines/files.py | 6 ++--- scrapy/shell.py | 5 ++-- scrapy/spidermiddlewares/referer.py | 9 ++++---- scrapy/utils/_compression.py | 5 ++-- scrapy/utils/datatypes.py | 6 ++--- scrapy/utils/signal.py | 4 ++-- scrapy/utils/url.py | 14 ++++++++---- tests/test_addons.py | 28 ++++++++++++----------- tests/test_downloader_handlers.py | 7 +++--- tests/test_feedexport.py | 12 ++++------ tests/test_http_request.py | 5 +--- tests/test_pipeline_files.py | 28 ++++++++++++----------- tests/test_settings/__init__.py | 2 +- tests/test_spiderloader/__init__.py | 5 ++-- tests/test_utils_deprecate.py | 12 ++++++---- tests/test_utils_iterators.py | 2 +- 35 files changed, 128 insertions(+), 146 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 08b4b09b2..a75f3b6db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -254,6 +254,8 @@ extend-select = [ "RUF", # flake8-bandit "S", + # flake8-simplify + "SIM", # flake8-slots "SLOT", # flake8-debugger @@ -344,6 +346,12 @@ ignore = [ "S321", # Argument default set to insecure SSL protocol "S503", + # Use capitalized environment variable + "SIM112", + # Use a context manager for opening files + "SIM115", + # Yoda condition detected + "SIM300", ] [tool.ruff.lint.per-file-ignores] diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 9a24871de..48f462c65 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -90,12 +90,10 @@ def _get_commands_dict( def _pop_command_name(argv: list[str]) -> str | None: - i = 0 - for arg in argv[1:]: + for i, arg in enumerate(argv[1:]): if not arg.startswith("-"): del argv[i] return arg - i += 1 return None diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index f996d1806..61aea3ee4 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -174,13 +174,12 @@ class Command(BaseRunSpiderCommand): display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour) def print_requests(self, lvl: int | None = None, colour: bool = True) -> None: - if lvl is None: - if self.requests: - requests = self.requests[max(self.requests)] - else: - requests = [] - else: + if lvl is not None: requests = self.requests.get(lvl, []) + elif self.requests: + requests = self.requests[max(self.requests)] + else: + requests = [] print("# Requests ", "-" * 65) display.pprint(requests, colorize=colour) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index e0c004580..1adc1530f 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -95,10 +95,7 @@ class Command(ScrapyCommand): project_name = args[0] - if len(args) == 2: - project_dir = Path(args[1]) - else: - project_dir = Path(args[0]) + project_dir = Path(args[-1]) if (project_dir / "scrapy.cfg").exists(): self.exitcode = 1 diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 9f65794fe..aa8a1a2a4 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -424,10 +424,7 @@ class ScrapyAgent: headers = TxHeaders(request.headers) if isinstance(agent, self._TunnelingAgent): headers.removeHeader(b"Proxy-Authorization") - if request.body: - bodyproducer = _RequestBodyProducer(request.body) - else: - bodyproducer = None + bodyproducer = _RequestBodyProducer(request.body) if request.body else None start_time = time() d: Deferred[TxResponse] = agent.request( method, to_bytes(url, encoding="ascii"), headers, bodyproducer diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 5480df72c..61f444e31 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -291,9 +291,7 @@ class ExecutionEngine: return False if self.slot.start_requests is not None: # not all start requests are handled return False - if self.slot.scheduler.has_pending_requests(): - return False - return True + return not self.slot.scheduler.has_pending_requests() def crawl(self, request: Request) -> None: """Inject the request into the spider <-> downloader pipeline""" @@ -388,9 +386,8 @@ class ExecutionEngine: ) self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.spider = spider - if hasattr(scheduler, "open"): - if d := scheduler.open(spider): - yield d + if hasattr(scheduler, "open") and (d := scheduler.open(spider)): + yield d yield self.scraper.open_spider(spider) assert self.crawler.stats self.crawler.stats.open_spider(spider) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index a63ee40bf..4b2520aa1 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -198,10 +198,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # chain, they went through it already from the process_spider_exception method recovered: MutableChain[_T] | MutableAsyncChain[_T] last_result_is_async = isinstance(result, AsyncIterable) - if last_result_is_async: - recovered = MutableAsyncChain() - else: - recovered = MutableChain() + recovered = MutableAsyncChain() if last_result_is_async else MutableChain() # There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async. # 1. def foo. Sync iterables are passed as is, async ones are downgraded. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 05af1bf8a..0a28c4549 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextlib import logging import pprint import signal @@ -503,7 +504,6 @@ class CrawlerProcess(CrawlerRunner): def _stop_reactor(self, _: Any = None) -> None: from twisted.internet import reactor - try: + # raised if already stopped or in shutdown stage + with contextlib.suppress(RuntimeError): reactor.stop() - except RuntimeError: # raised if already stopped or in shutdown stage - pass diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index b74140ee1..80107261b 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -42,7 +42,10 @@ class HttpAuthMiddleware: self, request: Request, spider: Spider ) -> Request | Response | None: auth = getattr(self, "auth", None) - if auth and b"Authorization" not in request.headers: - if not self.domain or url_is_from_any_domain(request.url, [self.domain]): - request.headers[b"Authorization"] = auth + if ( + auth + and b"Authorization" not in request.headers + and (not self.domain or url_is_from_any_domain(request.url, [self.domain])) + ): + request.headers[b"Authorization"] = auth return None diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 2f3f2db47..cb7fa8c90 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -51,10 +51,7 @@ class HttpProxyMiddleware: proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", "")) - if user: - creds = self._basic_auth_header(user, password) - else: - creds = None + creds = self._basic_auth_header(user, password) if user else None return creds, proxy_url diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 834a05ae9..46c6aa3fa 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -81,10 +81,7 @@ class BaseItemExporter: include_empty = self.export_empty_fields if self.fields_to_export is None: - if include_empty: - field_iter = item.field_names() - else: - field_iter = item.keys() + field_iter = item.field_names() if include_empty else item.keys() elif isinstance(self.fields_to_export, Mapping): if include_empty: field_iter = self.fields_to_export.items() diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 5ca07394f..afaf81928 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -6,6 +6,7 @@ See documentation in docs/topics/extensions.rst from __future__ import annotations +import contextlib import logging import signal import sys @@ -69,11 +70,9 @@ class StackTraceDump: class Debugger: def __init__(self) -> None: - try: + # win32 platforms don't support SIGUSR signals + with contextlib.suppress(AttributeError): signal.signal(signal.SIGUSR2, self._enter_debugger) # type: ignore[attr-defined] - except AttributeError: - # win32 platforms don't support SIGUSR signals - pass def _enter_debugger(self, signum: int, frame: FrameType | None) -> None: assert frame diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index b6e6f55a6..8a3d607b0 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst from __future__ import annotations +import contextlib import logging import re import sys @@ -642,10 +643,8 @@ class FeedExporter: ) d = {} for k, v in conf.items(): - try: + with contextlib.suppress(NotConfigured): d[k] = load_object(v) - except NotConfigured: - pass return d def _exporter_supported(self, format: str) -> bool: diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 929807de8..fe2cbcb86 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -89,10 +89,7 @@ class RFC2616Policy: return False cc = self._parse_cachecontrol(request) # obey user-agent directive "Cache-Control: no-store" - if b"no-store" in cc: - return False - # Any other is eligible for caching - return True + return b"no-store" not in cc def should_cache_response(self, response: Response, request: Request) -> bool: # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 7cf08a1bb..f97577442 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -151,10 +151,7 @@ class PeriodicLog: return False if exclude and not include: return True - for p in include: - if p in stat_name: - return True - return False + return any(p in stat_name for p in include) def spider_closed(self, spider: Spider, reason: str) -> None: self.log() diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 60322fe6e..b7c3b9d37 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -64,9 +64,8 @@ class CookieJar: cookies += self.jar._cookies_for_domain(host, wreq) # type: ignore[attr-defined] attrs = self.jar._cookie_attrs(cookies) # type: ignore[attr-defined] - if attrs: - if not wreq.has_header("Cookie"): - wreq.add_unredirected_header("Cookie", "; ".join(attrs)) + if attrs and not wreq.has_header("Cookie"): + wreq.add_unredirected_header("Cookie", "; ".join(attrs)) self.processed += 1 if self.processed % self.check_expired_frequency == 0: diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index e5b63ef14..e26cbe05b 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -29,7 +29,7 @@ class JsonRequest(Request): dumps_kwargs.setdefault("sort_keys", True) self._dumps_kwargs: dict[str, Any] = dumps_kwargs - body_passed = kwargs.get("body", None) is not None + body_passed = kwargs.get("body") is not None data: Any = kwargs.pop("data", None) data_passed: bool = data is not None @@ -61,7 +61,7 @@ class JsonRequest(Request): def replace( self, *args: Any, cls: type[Request] | None = None, **kwargs: Any ) -> Request: - body_passed = kwargs.get("body", None) is not None + body_passed = kwargs.get("body") is not None data: Any = kwargs.pop("data", None) data_passed: bool = data is not None diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index f195dbdd7..4fd932b88 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -41,9 +41,12 @@ _collect_string_content = etree.XPath("string()") def _nons(tag: Any) -> Any: - if isinstance(tag, str): - if tag[0] == "{" and tag[1 : len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE: - return tag.split("}")[-1] + if ( + isinstance(tag, str) + and tag[0] == "{" + and tag[1 : len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE + ): + return tag.split("}")[-1] return tag @@ -230,9 +233,7 @@ class LxmlLinkExtractor: parsed_url, self.deny_extensions ): return False - if self.restrict_text and not _matches(link.text, self.restrict_text): - return False - return True + return not self.restrict_text or _matches(link.text, self.restrict_text) def matches(self, url: str) -> bool: if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): diff --git a/scrapy/mail.py b/scrapy/mail.py index a3c642401..be2423965 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -111,11 +111,9 @@ class MailSender: ) -> Deferred[None] | None: from twisted.internet import reactor - msg: MIMEBase - if attachs: - msg = MIMEMultipart() - else: - msg = MIMENonMultipart(*mimetype.split("/", 1)) + msg: MIMEBase = ( + MIMEMultipart() if attachs else MIMENonMultipart(*mimetype.split("/", 1)) + ) to = list(arg_to_iter(to)) cc = list(arg_to_iter(cc)) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 16bd45c00..a10117590 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -553,10 +553,8 @@ class FilesPipeline(MediaPipeline): ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE") def _get_store(self, uri: str) -> FilesStoreProtocol: - if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir - scheme = "file" - else: - scheme = urlparse(uri).scheme + # to support win32 paths like: C:\\some\dir + scheme = "file" if Path(uri).is_absolute() else urlparse(uri).scheme store_cls = self.STORE_SCHEMES[scheme] return store_cls(uri) diff --git a/scrapy/shell.py b/scrapy/shell.py index 5d0ab1e4d..4a5b9e9cf 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -6,6 +6,7 @@ See documentation in docs/topics/shell.rst from __future__ import annotations +import contextlib import os import signal from typing import TYPE_CHECKING, Any @@ -143,12 +144,10 @@ class Shell: else: request.meta["handle_httpstatus_all"] = True response = None - try: + with contextlib.suppress(IgnoreRequest): response, spider = threads.blockingCallFromThread( reactor, self._schedule, request, spider ) - except IgnoreRequest: - pass self.populate_vars(response, request, spider) def populate_vars( diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 18cc991bf..a3a1e5b92 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -360,11 +360,10 @@ class RefererMiddleware: - otherwise, the policy from settings is used. """ policy_name = request.meta.get("referrer_policy") - if policy_name is None: - if isinstance(resp_or_url, Response): - policy_header = resp_or_url.headers.get("Referrer-Policy") - if policy_header is not None: - policy_name = to_unicode(policy_header.decode("latin1")) + if policy_name is None and isinstance(resp_or_url, Response): + policy_header = resp_or_url.headers.get("Referrer-Policy") + if policy_header is not None: + policy_name = to_unicode(policy_header.decode("latin1")) if policy_name is None: return self.default_policy() diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 591737b8e..6b09f36ff 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -1,3 +1,4 @@ +import contextlib import zlib from io import BytesIO from warnings import warn @@ -37,10 +38,8 @@ else: return decompressor.process(data) -try: +with contextlib.suppress(ImportError): import zstandard -except ImportError: - pass _CHUNK_SIZE = 65536 # 64 KiB diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 98ecb2f02..3d0e0d3c7 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -8,6 +8,7 @@ This module must not depend on any module outside the Standard Library. from __future__ import annotations import collections +import contextlib import warnings import weakref from collections import OrderedDict @@ -173,10 +174,9 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): self.data: LocalCache = LocalCache(limit=limit) def __setitem__(self, key: _KT, value: _VT) -> None: - try: + # if raised, key is not weak-referenceable, skip caching + with contextlib.suppress(TypeError): super().__setitem__(key, value) - except TypeError: - pass # key is not weak-referenceable, skip caching def __getitem__(self, key: _KT) -> _VT | None: # type: ignore[override] try: diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index c1d3bfffb..5fd176a3f 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -36,7 +36,7 @@ def send_catch_log( dont_log = named.pop("dont_log", ()) dont_log = tuple(dont_log) if isinstance(dont_log, Sequence) else (dont_log,) dont_log += (StopDownload,) - spider = named.get("spider", None) + spider = named.get("spider") responses: list[tuple[TypingAny, TypingAny]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): result: TypingAny @@ -88,7 +88,7 @@ def send_catch_log_deferred( return failure dont_log = named.pop("dont_log", None) - spider = named.get("spider", None) + spider = named.get("spider") dfds: list[Deferred[tuple[TypingAny, TypingAny]]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): d: Deferred[TypingAny] = maybeDeferred_coro( diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index d487849bb..db2749d79 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -173,13 +173,19 @@ def strip_url( parsed_url.username or parsed_url.password ): netloc = netloc.split("@")[-1] - if strip_default_port and parsed_url.port: - if (parsed_url.scheme, parsed_url.port) in ( + + if ( + strip_default_port + and parsed_url.port + and (parsed_url.scheme, parsed_url.port) + in ( ("http", 80), ("https", 443), ("ftp", 21), - ): - netloc = netloc.replace(f":{parsed_url.port}", "") + ) + ): + netloc = netloc.replace(f":{parsed_url.port}", "") + return urlunparse( ( parsed_url.scheme, diff --git a/tests/test_addons.py b/tests/test_addons.py index 17949997c..a0caa3511 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -166,19 +166,21 @@ class AddonManagerTest(unittest.TestCase): def update_settings(self, settings): pass - with patch("scrapy.addons.logger") as logger_mock: - with patch("scrapy.addons.build_from_crawler") as build_from_crawler_mock: - settings_dict = { - "ADDONS": {LoggedAddon: 1}, - } - addon = LoggedAddon() - build_from_crawler_mock.return_value = addon - crawler = get_crawler(settings_dict=settings_dict) - logger_mock.info.assert_called_once_with( - "Enabled addons:\n%(addons)s", - {"addons": [addon]}, - extra={"crawler": crawler}, - ) + with ( + patch("scrapy.addons.logger") as logger_mock, + patch("scrapy.addons.build_from_crawler") as build_from_crawler_mock, + ): + settings_dict = { + "ADDONS": {LoggedAddon: 1}, + } + addon = LoggedAddon() + build_from_crawler_mock.return_value = addon + crawler = get_crawler(settings_dict=settings_dict) + logger_mock.info.assert_called_once_with( + "Enabled addons:\n%(addons)s", + {"addons": [addon]}, + extra={"crawler": crawler}, + ) @inlineCallbacks def test_enable_addon_in_spider(self): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 05b64e704..0dcbeaec1 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -530,9 +530,10 @@ class Http11TestCase(HttpTestCase): d = self.download_request(request, Spider("foo")) def checkDataLoss(failure): - if failure.check(ResponseFailed): - if any(r.check(_DataLoss) for r in failure.value.reasons): - return None + if failure.check(ResponseFailed) and any( + r.check(_DataLoss) for r in failure.value.reasons + ): + return None return failure d.addCallback(lambda _: self.fail("No DataLoss exception")) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index b087aaab1..0f149f172 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -756,7 +756,7 @@ class FeedExportTest(FeedExportTestBase): ) finally: - for file_path in FEEDS.keys(): + for file_path in FEEDS: if not Path(file_path).exists(): continue @@ -1229,15 +1229,13 @@ class FeedExportTest(FeedExportTestBase): class CustomFilter2(scrapy.extensions.feedexport.ItemFilter): def accepts(self, item): - if "foo" not in item.fields: - return False - return True + return "foo" in item.fields class CustomFilter3(scrapy.extensions.feedexport.ItemFilter): def accepts(self, item): - if isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1": - return True - return False + return ( + isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1" + ) formats = { "json": b'[\n{"foo": "bar1", "egg": "spam1"}\n]', diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 9997b7ab3..c5929c339 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1488,10 +1488,7 @@ def _buildresponse(body, **kwargs): def _qs(req, encoding="utf-8", to_unicode=False): - if req.method == "POST": - qs = req.body - else: - qs = req.url.partition("?")[2] + qs = req.body if req.method == "POST" else req.url.partition("?")[2] uqs = unquote_to_bytes(qs) if to_unicode: uqs = uqs.decode(encoding) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index a6c5f0a94..4c3fc36b6 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -634,19 +634,21 @@ class TestGCSFilesStore(unittest.TestCase): import google.cloud.storage # noqa: F401 except ModuleNotFoundError: raise unittest.SkipTest("google-cloud-storage is not installed") - with mock.patch("google.cloud.storage") as _: - with mock.patch("scrapy.pipelines.files.time") as _: - uri = "gs://my_bucket/my_prefix/" - store = GCSFilesStore(uri) - store.bucket = mock.Mock() - path = "full/my_data.txt" - yield store.persist_file( - path, mock.Mock(), info=None, meta=None, headers=None - ) - yield store.stat_file(path, info=None) - expected_blob_path = store.prefix + path - store.bucket.blob.assert_called_with(expected_blob_path) - store.bucket.get_blob.assert_called_with(expected_blob_path) + with ( + mock.patch("google.cloud.storage"), + mock.patch("scrapy.pipelines.files.time"), + ): + uri = "gs://my_bucket/my_prefix/" + store = GCSFilesStore(uri) + store.bucket = mock.Mock() + path = "full/my_data.txt" + yield store.persist_file( + path, mock.Mock(), info=None, meta=None, headers=None + ) + yield store.stat_file(path, info=None) + expected_blob_path = store.prefix + path + store.bucket.blob.assert_called_with(expected_blob_path) + store.bucket.get_blob.assert_called_with(expected_blob_path) class TestFTPFileStore(unittest.TestCase): diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 503c29e32..96d59c911 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -170,7 +170,7 @@ class BaseSettingsTest(unittest.TestCase): self.assertCountEqual(self.settings.attributes.keys(), ctrl_attributes.keys()) - for key in ctrl_attributes.keys(): + for key in ctrl_attributes: attr = self.settings.attributes[key] ctrl_attr = ctrl_attributes[key] self.assertEqual(attr.value, ctrl_attr.value) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 9b53b9b96..d5aac34eb 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,3 +1,4 @@ +import contextlib import shutil import sys import tempfile @@ -22,10 +23,8 @@ module_dir = Path(__file__).resolve().parent def _copytree(source: Path, target: Path): - try: + with contextlib.suppress(shutil.Error): shutil.copytree(source, target) - except shutil.Error: - pass class SpiderLoaderTest(unittest.TestCase): diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index eedb6f6af..dc5fbd3c3 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -259,12 +259,14 @@ class WarnWhenSubclassedTest(unittest.TestCase): self.assertIn("foo.Bar", str(w[1].message)) def test_inspect_stack(self): - with mock.patch("inspect.stack", side_effect=IndexError): - with warnings.catch_warnings(record=True) as w: - DeprecatedName = create_deprecated_class("DeprecatedName", NewName) + with ( + mock.patch("inspect.stack", side_effect=IndexError), + warnings.catch_warnings(record=True) as w, + ): + DeprecatedName = create_deprecated_class("DeprecatedName", NewName) - class SubClass(DeprecatedName): - pass + class SubClass(DeprecatedName): + pass self.assertIn("Error detecting parent module", str(w[0].message)) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 4c81e3a2f..12507c6a3 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -366,7 +366,7 @@ class UtilsCsvTestCase(unittest.TestCase): # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assertTrue(all(isinstance(k, str) for k in result_row.keys())) + self.assertTrue(all(isinstance(k, str) for k in result_row)) self.assertTrue(all(isinstance(v, str) for v in result_row.values())) def test_csviter_delimiter(self): From b70443f2d06b1b0ad8c474fc5e8a424e363bdd81 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 2 Jan 2025 00:31:26 +0500 Subject: [PATCH 005/491] Split ruff and pylint ignores into two categories, some pylint cleanup. --- docs/_ext/scrapydocs.py | 1 + docs/conf.py | 2 + pyproject.toml | 81 +++++++++++++++++---------------- scrapy/contracts/__init__.py | 2 +- scrapy/extensions/feedexport.py | 2 + scrapy/extensions/telnet.py | 4 +- scrapy/interfaces.py | 2 + scrapy/shell.py | 17 ++++--- scrapy/utils/console.py | 2 +- scrapy/utils/deprecate.py | 1 + scrapy/utils/display.py | 1 + scrapy/utils/engine.py | 2 +- tests/test_contracts.py | 4 +- tests/test_exporters.py | 4 +- tests/test_item.py | 2 +- tests/test_link.py | 2 +- tests/test_scrapy__getattr__.py | 2 +- tests/test_selector.py | 2 +- tests/test_squeues_request.py | 8 ++-- 19 files changed, 77 insertions(+), 64 deletions(-) diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index c23a89089..9b63f39f6 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,3 +1,4 @@ +# pylint: disable=import-error from operator import itemgetter from docutils import nodes diff --git a/docs/conf.py b/docs/conf.py index 7a5166053..d06828bcc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,6 +8,8 @@ # # All configuration values have a default; values that are commented out # serve to show the default. + +# pylint: disable=import-error import os import sys from pathlib import Path diff --git a/pyproject.toml b/pyproject.toml index a75f3b6db..88005ec4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,43 +125,30 @@ extension-pkg-allow-list=[ [tool.pylint."MESSAGES CONTROL"] disable = [ - "abstract-method", - "arguments-differ", - "arguments-renamed", + # Ones we want to ignore "attribute-defined-outside-init", "broad-exception-caught", "consider-using-with", "cyclic-import", - "dangerous-default-value", "disallowed-name", - "duplicate-code", # https://github.com/PyCQA/pylint/issues/214 - "eval-used", + "duplicate-code", # https://github.com/pylint-dev/pylint/issues/214 "fixme", - "import-error", "import-outside-toplevel", - "inherit-non-class", + "inherit-non-class", # false positives with create_deprecated_class() "invalid-name", "invalid-overridden-method", - "isinstance-second-argument-not-valid-type", - "keyword-arg-before-vararg", + "isinstance-second-argument-not-valid-type", # false positives with create_deprecated_class() "line-too-long", "logging-format-interpolation", "logging-fstring-interpolation", "logging-not-lazy", "missing-docstring", "no-member", - "no-method-argument", - "no-name-in-module", - "no-self-argument", - "no-value-for-parameter", # https://github.com/pylint-dev/pylint/issues/3268 + "no-value-for-parameter", # https://github.com/pylint-dev/pylint/issues/3268 "not-callable", - "pointless-statement", - "pointless-string-statement", "protected-access", - "raise-missing-from", "redefined-builtin", "redefined-outer-name", - "signature-differs", "too-few-public-methods", "too-many-ancestors", "too-many-arguments", @@ -173,14 +160,23 @@ disable = [ "too-many-positional-arguments", "too-many-public-methods", "too-many-return-statements", - "unbalanced-tuple-unpacking", - "unnecessary-dunder-call", "unused-argument", "unused-import", "unused-variable", - "used-before-assignment", - "useless-return", + "useless-return", # https://github.com/pylint-dev/pylint/issues/6530 "wrong-import-position", + + # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix") + "abstract-method", + "arguments-differ", + "arguments-renamed", + "dangerous-default-value", + "keyword-arg-before-vararg", + "pointless-statement", + "raise-missing-from", + "unbalanced-tuple-unpacking", + "unnecessary-dunder-call", + "used-before-assignment", ] [tool.pytest.ini_options] @@ -270,22 +266,8 @@ extend-select = [ "YTT", ] ignore = [ - # Assigning to `os.environ` doesn't clear the environment. - "B003", - # Do not use mutable data structures for argument defaults. - "B006", - # Loop control variable not used within the loop body. - "B007", - # Do not perform function calls in argument defaults. - "B008", - # Star-arg unpacking after a keyword argument is strongly discouraged. - "B026", - # Found useless expression. - "B018", - # No explicit stacklevel argument found. - "B028", - # Within an `except` clause, raise exceptions with `raise ... from` - "B904", + # Ones we want to ignore + # Missing docstring in public module "D100", # Missing docstring in public class @@ -346,12 +328,31 @@ ignore = [ "S321", # Argument default set to insecure SSL protocol "S503", - # Use capitalized environment variable - "SIM112", # Use a context manager for opening files "SIM115", # Yoda condition detected "SIM300", + + # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix") + + # Assigning to `os.environ` doesn't clear the environment. + "B003", + # Do not use mutable data structures for argument defaults. + "B006", + # Loop control variable not used within the loop body. + "B007", + # Do not perform function calls in argument defaults. + "B008", + # Found useless expression. + "B018", + # Star-arg unpacking after a keyword argument is strongly discouraged. + "B026", + # No explicit stacklevel argument found. + "B028", + # Within an `except` clause, raise exceptions with `raise ... from` + "B904", + # Use capitalized environment variable + "SIM112", ] [tool.ruff.lint.per-file-ignores] diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 3b4f932a0..bdb68c4ad 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -199,7 +199,7 @@ def _create_testcase(method: Callable, desc: str) -> TestCase: spider = method.__self__.name # type: ignore[attr-defined] class ContractTestCase(TestCase): - def __str__(_self) -> str: + def __str__(_self) -> str: # pylint: disable=no-self-argument return f"[{spider}] {method.__name__} ({desc})" name = f"{spider}_{method.__name__}" diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 8a3d607b0..c6e2aa0dd 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -110,6 +110,8 @@ class ItemFilter: class IFeedStorage(Interface): """Interface that all Feed Storages must implement""" + # pylint: disable=no-self-argument + def __init__(uri, *, feed_options=None): # pylint: disable=super-init-not-called """Initialize the storage with the parameters given in the URI and the feed-specific options (see :setting:`FEEDS`)""" diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index ee28d86ba..189b1953b 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -84,7 +84,9 @@ class TelnetConsole(protocol.ServerFactory): """An implementation of IPortal""" @defers - def login(self_, credentials, mind, *interfaces): + def login( + self_, credentials, mind, *interfaces + ): # pylint: disable=no-self-argument if not ( credentials.username == self.username.encode("utf8") and credentials.checkPassword(self.password.encode("utf8")) diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index 9a2c5f170..13a4d822d 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -1,3 +1,5 @@ +# pylint:disable=no-method-argument,no-self-argument + from zope.interface import Interface diff --git a/scrapy/shell.py b/scrapy/shell.py index 4a5b9e9cf..5e5e57a9a 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -71,17 +71,16 @@ class Shell: else: self.populate_vars() if self.code: + # pylint: disable-next=eval-used print(eval(self.code, globals(), self.vars)) # noqa: S307 else: - """ - Detect interactive shell setting in scrapy.cfg - e.g.: ~/.config/scrapy.cfg or ~/.scrapy.cfg - [settings] - # shell can be one of ipython, bpython or python; - # to be used as the interactive python console, if available. - # (default is ipython, fallbacks in the order listed above) - shell = python - """ + # Detect interactive shell setting in scrapy.cfg + # e.g.: ~/.config/scrapy.cfg or ~/.scrapy.cfg + # [settings] + # # shell can be one of ipython, bpython or python; + # # to be used as the interactive python console, if available. + # # (default is ipython, fallbacks in the order listed above) + # shell = python cfg = get_config() section, option = "settings", "shell" env = os.environ.get("SCRAPY_PYTHON_SHELL") diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 95844a48c..7425543ff 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -59,7 +59,7 @@ def _embed_ptpython_shell( namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start a ptpython shell""" - import ptpython.repl + import ptpython.repl # pylint: disable=import-error @wraps(_embed_ptpython_shell) def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 0a0acc742..20d03cae6 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -57,6 +57,7 @@ def create_deprecated_class( # https://github.com/python/mypy/issues/4177 class DeprecatedClass(new_class.__class__): # type: ignore[misc, name-defined] + # pylint: disable=no-self-argument deprecated_class: type | None = None warned_on_subclass: bool = False diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index 39f46270b..20744a604 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -30,6 +30,7 @@ def _tty_supports_color() -> bool: def _colorize(text: str, colorize: bool = True) -> str: + # pylint: disable=no-name-in-module if not colorize or not sys.stdout.isatty() or not _tty_supports_color(): return text try: diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 1948009e8..52f29e22c 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -32,7 +32,7 @@ def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]: checks: list[tuple[str, Any]] = [] for test in tests: try: - checks += [(test, eval(test))] # noqa: S307 + checks += [(test, eval(test))] # noqa: S307 # pylint: disable=eval-used except Exception as e: checks += [(test, f"{type(e).__name__} (exception)")] diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 743889234..f7581707b 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -517,8 +517,8 @@ class ContractsManagerTest(unittest.TestCase): super().__init__(*args, **kwargs) self.visited = 0 - def start_requests(s): - return self.conman.from_spider(s, self.results) + def start_requests(self_): # pylint: disable=no-self-argument + return self.conman.from_spider(self_, self.results) def parse_first(self, response): self.visited += 1 diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 522c6638d..970f8d2f5 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -216,7 +216,9 @@ class PprintItemExporterTest(BaseItemExporterTest): return PprintItemExporter(self.output, **kwargs) def _check_output(self): - self._assert_expected_item(eval(self.output.getvalue())) + self._assert_expected_item( + eval(self.output.getvalue()) # pylint: disable=eval-used + ) class PprintItemExporterDataclassTest(PprintItemExporterTest): diff --git a/tests/test_item.py b/tests/test_item.py index 13243b67f..35212c153 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -54,7 +54,7 @@ class ItemTest(unittest.TestCase): self.assertEqual(itemrepr, "{'name': 'John Doe', 'number': 123}") - i2 = eval(itemrepr) + i2 = eval(itemrepr) # pylint: disable=eval-used self.assertEqual(i2["name"], "John Doe") self.assertEqual(i2["number"], 123) diff --git a/tests/test_link.py b/tests/test_link.py index 7ba0851ae..35723bbd6 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -49,7 +49,7 @@ class LinkTest(unittest.TestCase): l1 = Link( "http://www.example.com", text="test", fragment="something", nofollow=True ) - l2 = eval(repr(l1)) + l2 = eval(repr(l1)) # pylint: disable=eval-used self._assert_same_links(l1, l2) def test_bytes_url(self): diff --git a/tests/test_scrapy__getattr__.py b/tests/test_scrapy__getattr__.py index 979c42267..443e26a3c 100644 --- a/tests/test_scrapy__getattr__.py +++ b/tests/test_scrapy__getattr__.py @@ -3,7 +3,7 @@ import warnings def test_deprecated_twisted_version(): with warnings.catch_warnings(record=True) as warns: - from scrapy import twisted_version + from scrapy import twisted_version # pylint: disable=no-name-in-module assert twisted_version is not None assert isinstance(twisted_version, tuple) diff --git a/tests/test_selector.py b/tests/test_selector.py index 1b5f3f018..857c7d626 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -264,7 +264,7 @@ class JMESPathTestCase(unittest.TestCase): ) @pytest.mark.skipif(PARSEL_18_PLUS, reason="parsel >= 1.8 supports jmespath") - def test_jmespath_not_available(my_json_page) -> None: + def test_jmespath_not_available(self) -> None: body = """ { "website": {"name": "Example"} diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 02ea8027f..04eeae4dc 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -1,3 +1,7 @@ +""" +Queues that handle requests +""" + import shutil import tempfile import unittest @@ -16,10 +20,6 @@ from scrapy.squeues import ( ) from scrapy.utils.test import get_crawler -""" -Queues that handle requests -""" - class BaseQueueTestCase(unittest.TestCase): def setUp(self): From dc706d4fc307f0b51d8122f10dee6ad8e2653629 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 2 Jan 2025 12:32:25 +0500 Subject: [PATCH 006/491] Remove useless pylint: disable lines. --- pyproject.toml | 3 +++ scrapy/interfaces.py | 2 +- tests/test_scheduler.py | 2 -- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 88005ec4a..8c985753f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -124,6 +124,9 @@ extension-pkg-allow-list=[ ] [tool.pylint."MESSAGES CONTROL"] +enable = [ + "useless-suppression", +] disable = [ # Ones we want to ignore "attribute-defined-outside-init", diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index 13a4d822d..b4f1d9394 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -1,4 +1,4 @@ -# pylint:disable=no-method-argument,no-self-argument +# pylint: disable=no-method-argument,no-self-argument from zope.interface import Interface diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 8bd1480ad..3ac330ae2 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -277,14 +277,12 @@ class DownloaderAwareSchedulerTestMixin: downloader = self.mock_crawler.engine.downloader while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() - # pylint: disable=protected-access slot = downloader.get_slot_key(request) dequeued_slots.append(slot) downloader.increment(slot) requests.append(request) for request in requests: - # pylint: disable=protected-access slot = downloader.get_slot_key(request) downloader.decrement(slot) From b10d46d280fbc84f7a1c50e116a1ed828aa286c9 Mon Sep 17 00:00:00 2001 From: Arthur <48801049+devfox-se@users.noreply.github.com> Date: Thu, 2 Jan 2025 15:36:28 +0400 Subject: [PATCH 007/491] Fix the calculate_final_stats method (#6599) --- scrapy/extensions/logstats.py | 9 ++++++--- tests/test_logstats.py | 11 ++++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index e829d8b92..f2e1f57b8 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -83,12 +83,15 @@ class LogStats: self, spider: Spider ) -> tuple[None, None] | tuple[float, float]: start_time = self.stats.get_value("start_time") - finished_time = self.stats.get_value("finished_time") + finish_time = self.stats.get_value("finish_time") - if not start_time or not finished_time: + if not start_time or not finish_time: return None, None - mins_elapsed = (finished_time - start_time).seconds / 60 + mins_elapsed = (finish_time - start_time).seconds / 60 + + if mins_elapsed == 0: + return None, None items = self.stats.get_value("item_scraped_count", 0) pages = self.stats.get_value("response_received_count", 0) diff --git a/tests/test_logstats.py b/tests/test_logstats.py index d87285df7..a4b002e34 100644 --- a/tests/test_logstats.py +++ b/tests/test_logstats.py @@ -47,7 +47,7 @@ class TestLogStats(unittest.TestCase): # Simulate when spider closes after running for 30 mins self.stats.set_value("start_time", datetime.fromtimestamp(1655100172)) - self.stats.set_value("finished_time", datetime.fromtimestamp(1655101972)) + self.stats.set_value("finish_time", datetime.fromtimestamp(1655101972)) logstats.spider_closed(self.spider, "test reason") self.assertEqual(self.stats.get_value("responses_per_minute"), 172.9) self.assertEqual(self.stats.get_value("items_per_minute"), 116.4) @@ -60,3 +60,12 @@ class TestLogStats(unittest.TestCase): logstats.spider_closed(self.spider, "test reason") self.assertIsNone(self.stats.get_value("responses_per_minute")) self.assertIsNone(self.stats.get_value("items_per_minute")) + + def test_stats_calculation_no_elapsed_time(self): + """The stat values should be None since the elapsed time is 0.""" + logstats = LogStats.from_crawler(self.crawler) + self.stats.set_value("start_time", datetime.fromtimestamp(1655100172)) + self.stats.set_value("finish_time", datetime.fromtimestamp(1655100172)) + logstats.spider_closed(self.spider, "test reason") + self.assertIsNone(self.stats.get_value("responses_per_minute")) + self.assertIsNone(self.stats.get_value("items_per_minute")) From 6ae5b9267145e5b01d282f766fa90d129eb40390 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 2 Jan 2025 15:45:04 +0400 Subject: [PATCH 008/491] Drop the remaining unittest.main() blocks. (#6602) --- tests/test_downloadermiddleware_httpcache.py | 4 ---- tests/test_downloadermiddleware_redirect.py | 4 ---- tests/test_downloadermiddleware_retry.py | 4 ---- tests/test_exporters.py | 4 ---- tests/test_http_request.py | 4 ---- tests/test_item.py | 4 ---- tests/test_loader.py | 4 ---- tests/test_loader_deprecated.py | 4 ---- tests/test_logformatter.py | 4 ---- tests/test_mail.py | 4 ---- tests/test_responsetypes.py | 4 ---- tests/test_settings/__init__.py | 4 ---- tests/test_utils_conf.py | 4 ---- tests/test_utils_console.py | 4 ---- tests/test_utils_datatypes.py | 4 ---- tests/test_utils_httpobj.py | 4 ---- tests/test_utils_misc/__init__.py | 4 ---- tests/test_utils_request.py | 4 ---- tests/test_utils_sitemap.py | 4 ---- tests/test_utils_spider.py | 4 ---- tests/test_utils_template.py | 4 ---- tests/test_utils_url.py | 4 ---- 22 files changed, 88 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index f80eff3e6..ec4e87ffb 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -566,7 +566,3 @@ class RFC2616PolicyTest(DefaultStorageTest): res2 = self._process_requestresponse(mw, req0, None) self.assertEqualResponse(res1, res2) assert "cached" in res2.flags - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index e37da9715..7b19ab781 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1314,7 +1314,3 @@ def test_meta_refresh_schemes(url, location, target): else: assert isinstance(redirect, Request) assert redirect.url == target - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index c99f19b03..9b39b84d9 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -643,7 +643,3 @@ class GetRetryRequestTest(unittest.TestCase): f"{stats_key}/reason_count/{expected_reason}", ): self.assertEqual(spider.crawler.stats.get_value(stat), 1) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_exporters.py b/tests/test_exporters.py index fa9389044..0f70887af 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -699,7 +699,3 @@ class CustomExporterItemTest(unittest.TestCase): class CustomExporterDataclassTest(CustomExporterItemTest): item_class = TestDataClass - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 9997b7ab3..d020a8911 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1717,7 +1717,3 @@ class JsonRequestTest(RequestTest): def tearDown(self): warnings.resetwarnings() super().tearDown() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_item.py b/tests/test_item.py index 13243b67f..3f10a724d 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -296,7 +296,3 @@ class ItemMetaClassCellRegression(unittest.TestCase): # TypeError: __class__ set to # defining 'MyItem' as super().__init__(*args, **kwargs) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_loader.py b/tests/test_loader.py index aca428bbe..824d7aecf 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -586,7 +586,3 @@ class FunctionProcessorTestCase(unittest.TestCase): lo.add_value("foo", " bar ") lo.add_value("foo", [" asdf ", " qwerty "]) self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]}) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 4bf22f6a0..8d4bd6bc1 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -715,7 +715,3 @@ class FunctionProcessorTestCase(unittest.TestCase): lo.add_value("foo", " bar ") lo.add_value("foo", [" asdf ", " qwerty "]) self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]}) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 5a92521cc..61a9f3f8d 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -234,7 +234,3 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): self.assertNotIn("Scraped from <200 http://127.0.0.1:", str(lc)) self.assertNotIn("Crawled (200) Date: Thu, 2 Jan 2025 18:14:18 +0500 Subject: [PATCH 009/491] Reformat long REFERRER_POLICY. --- tests/test_spidermiddleware_referer.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index cefd33e4e..4945ac25d 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -891,7 +891,12 @@ class TestSettingsPolicyByName(TestCase): # test parsing without space(s) after the comma settings1 = Settings( { - "REFERRER_POLICY": f"some-custom-unknown-policy,{POLICY_SAME_ORIGIN},{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN},another-custom-unknown-policy" + "REFERRER_POLICY": ( + f"some-custom-unknown-policy," + f"{POLICY_SAME_ORIGIN}," + f"{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}," + f"another-custom-unknown-policy" + ) } ) mw1 = RefererMiddleware(settings1) @@ -900,7 +905,11 @@ class TestSettingsPolicyByName(TestCase): # test parsing with space(s) after the comma settings2 = Settings( { - "REFERRER_POLICY": f"{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}, another-custom-unknown-policy, {POLICY_UNSAFE_URL}" + "REFERRER_POLICY": ( + f"{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}," + f" another-custom-unknown-policy," + f" {POLICY_UNSAFE_URL}" + ) } ) mw2 = RefererMiddleware(settings2) @@ -909,7 +918,11 @@ class TestSettingsPolicyByName(TestCase): def test_multiple_policy_tokens_all_invalid(self): settings = Settings( { - "REFERRER_POLICY": "some-custom-unknown-policy,another-custom-unknown-policy,yet-another-custom-unknown-policy" + "REFERRER_POLICY": ( + "some-custom-unknown-policy," + "another-custom-unknown-policy," + "yet-another-custom-unknown-policy" + ) } ) with self.assertRaises(RuntimeError): From 4d31277bc67169460dc2d8bca80946df8b355b8f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 3 Jan 2025 02:48:14 +0400 Subject: [PATCH 010/491] Explicitly mark re-exports. (#6579) --- pyproject.toml | 20 ++++++++++++-------- scrapy/core/downloader/handlers/http.py | 5 +++++ scrapy/http/__init__.py | 13 +++++++++++++ scrapy/linkextractors/__init__.py | 5 +++++ scrapy/selector/__init__.py | 5 +++++ scrapy/spiders/__init__.py | 9 +++++++++ tests/test_item.py | 3 ++- tests/test_utils_url.py | 2 +- 8 files changed, 52 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8c985753f..571a61f1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,9 +72,9 @@ version = {file = "./scrapy/VERSION"} [tool.mypy] ignore_missing_imports = true +implicit_reexport = false # Interface classes are hard to support - [[tool.mypy.overrides]] module = "twisted.internet.interfaces" follow_imports = "skip" @@ -92,6 +92,14 @@ follow_imports = "skip" module = "scrapy.settings.default_settings" ignore_errors = true +[[tool.mypy.overrides]] +module = "itemadapter" +implicit_reexport = true + +[[tool.mypy.overrides]] +module = "twisted" +implicit_reexport = true + [tool.bumpversion] current_version = "2.12.0" commit = true @@ -359,13 +367,9 @@ ignore = [ ] [tool.ruff.lint.per-file-ignores] -# Exclude files that are meant to provide top-level imports -"scrapy/__init__.py" = ["E402"] -"scrapy/core/downloader/handlers/http.py" = ["F401"] -"scrapy/http/__init__.py" = ["F401"] -"scrapy/linkextractors/__init__.py" = ["E402", "F401"] -"scrapy/selector/__init__.py" = ["F401"] -"scrapy/spiders/__init__.py" = ["E402", "F401"] +# Circular import workarounds +"scrapy/linkextractors/__init__.py" = ["E402"] +"scrapy/spiders/__init__.py" = ["E402"] # Skip bandit in tests "tests/**" = ["S"] diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 52535bd8b..93b96c779 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -2,3 +2,8 @@ from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import ( HTTP11DownloadHandler as HTTPDownloadHandler, ) + +__all__ = [ + "HTTP10DownloadHandler", + "HTTPDownloadHandler", +] diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py index d0b726bad..0e5c2b53b 100644 --- a/scrapy/http/__init__.py +++ b/scrapy/http/__init__.py @@ -15,3 +15,16 @@ from scrapy.http.response.html import HtmlResponse from scrapy.http.response.json import JsonResponse from scrapy.http.response.text import TextResponse from scrapy.http.response.xml import XmlResponse + +__all__ = [ + "FormRequest", + "Headers", + "HtmlResponse", + "JsonRequest", + "JsonResponse", + "Request", + "Response", + "TextResponse", + "XmlResponse", + "XmlRpcRequest", +] diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 1c7e96ae0..b39859f7b 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -126,3 +126,8 @@ def _is_valid_url(url: str) -> bool: # Top-level imports from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor + +__all__ = [ + "IGNORED_EXTENSIONS", + "LinkExtractor", +] diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index 85c500d66..7cfa3c364 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -4,3 +4,8 @@ Selectors # top-level imports from scrapy.selector.unified import Selector, SelectorList + +__all__ = [ + "Selector", + "SelectorList", +] diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 6136dabc7..e255e91cc 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -117,3 +117,12 @@ class Spider(object_ref): from scrapy.spiders.crawl import CrawlSpider, Rule from scrapy.spiders.feed import CSVFeedSpider, XMLFeedSpider from scrapy.spiders.sitemap import SitemapSpider + +__all__ = [ + "CSVFeedSpider", + "CrawlSpider", + "Rule", + "SitemapSpider", + "Spider", + "XMLFeedSpider", +] diff --git a/tests/test_item.py b/tests/test_item.py index 5a8ee095e..480412841 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,7 +1,8 @@ import unittest +from abc import ABCMeta from unittest import mock -from scrapy.item import ABCMeta, Field, Item, ItemMeta +from scrapy.item import Field, Item, ItemMeta class ItemTest(unittest.TestCase): diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 94a59f883..314082742 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -6,7 +6,7 @@ import pytest from scrapy.linkextractors import IGNORED_EXTENSIONS from scrapy.spiders import Spider from scrapy.utils.misc import arg_to_iter -from scrapy.utils.url import ( +from scrapy.utils.url import ( # type: ignore[attr-defined] _is_filesystem_path, _public_w3lib_objects, add_http_if_no_scheme, From f2234c5b96d4069d6881aacb7007ba3d305dfb0e Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Tue, 7 Jan 2025 06:40:49 -0300 Subject: [PATCH 011/491] Fix Crawler.request_fingerprinter typing (#6605) --- scrapy/crawler.py | 4 ++-- scrapy/extensions/httpcache.py | 6 ++++-- scrapy/pipelines/media.py | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 0a28c4549..f6dbe053a 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -44,7 +44,7 @@ if TYPE_CHECKING: from scrapy.logformatter import LogFormatter from scrapy.spiderloader import SpiderLoader from scrapy.statscollectors import StatsCollector - from scrapy.utils.request import RequestFingerprinter + from scrapy.utils.request import RequestFingerprinterProtocol logger = logging.getLogger(__name__) @@ -80,7 +80,7 @@ class Crawler: self.extensions: ExtensionManager | None = None self.stats: StatsCollector | None = None self.logformatter: LogFormatter | None = None - self.request_fingerprinter: RequestFingerprinter | None = None + self.request_fingerprinter: RequestFingerprinterProtocol | None = None self.spider: Spider | None = None self.engine: ExecutionEngine | None = None diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index fe2cbcb86..0cd16d737 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -29,7 +29,7 @@ if TYPE_CHECKING: from scrapy.http.request import Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider - from scrapy.utils.request import RequestFingerprinter + from scrapy.utils.request import RequestFingerprinterProtocol logger = logging.getLogger(__name__) @@ -265,7 +265,9 @@ class DbmCacheStorage: ) assert spider.crawler.request_fingerprinter - self._fingerprinter: RequestFingerprinter = spider.crawler.request_fingerprinter + self._fingerprinter: RequestFingerprinterProtocol = ( + spider.crawler.request_fingerprinter + ) def close_spider(self, spider: Spider) -> None: self.db.close() diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 0f3329db1..e66b86ce6 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -30,7 +30,7 @@ if TYPE_CHECKING: from scrapy import Spider from scrapy.crawler import Crawler from scrapy.http import Response - from scrapy.utils.request import RequestFingerprinter + from scrapy.utils.request import RequestFingerprinterProtocol class FileInfo(TypedDict): @@ -47,7 +47,7 @@ logger = logging.getLogger(__name__) class MediaPipeline(ABC): crawler: Crawler - _fingerprinter: RequestFingerprinter + _fingerprinter: RequestFingerprinterProtocol _modern_init = False LOG_FAILED_RESULTS: bool = True From 4869315d102e2e92f50582ebc3aebdb82266f5b3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 7 Jan 2025 13:46:12 +0400 Subject: [PATCH 012/491] Install libjpeg-dev on pinned envs to be able to install Pillow. (#6607) --- .github/workflows/tests-ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index b2a5681df..ab6794d3c 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -73,7 +73,7 @@ jobs: if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update - sudo apt-get install libxml2-dev libxslt-dev + sudo apt-get install libxml2-dev libxslt-dev libjpeg-dev - name: Run tests env: ${{ matrix.env }} From 5d3aa80ad1f1dcd67a185158c6a7ceb53855eefd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 7 Jan 2025 13:52:26 +0400 Subject: [PATCH 013/491] Switch CI to codecov/codecov-action and enable it on Windows. (#6609) --- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 2 +- .github/workflows/tests-windows.yml | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 9e78e26e3..c28a99982 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -27,4 +27,4 @@ jobs: tox -e py - name: Upload coverage report - run: bash <(curl -s https://codecov.io/bash) + uses: codecov/codecov-action@v5 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index ab6794d3c..89d1e70ac 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -82,4 +82,4 @@ jobs: tox - name: Upload coverage report - run: bash <(curl -s https://codecov.io/bash) + uses: codecov/codecov-action@v5 diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 67a32aac6..45e4ca157 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -44,3 +44,6 @@ jobs: run: | pip install -U tox tox + + - name: Upload coverage report + uses: codecov/codecov-action@v5 From 59fcb9b93c4971602b4a4afd4afdf08db7a7f2b7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 7 Jan 2025 15:18:18 +0400 Subject: [PATCH 014/491] Improve internal refs to scrapy.Request and scrapy.Selector (#6526) * Improve internal refs to scrapy.Selector. * Improve internal refs to scrapy.Request. * More scrapy.http fixes. * Fix FormRequest refs. * More fixes. * Simplifications. * Last fixes. * Add the parsel intersphinx. --- docs/conf.py | 1 + docs/news.rst | 139 +++++++++++++------------- docs/topics/downloader-middleware.rst | 2 +- docs/topics/dynamic-content.rst | 5 +- docs/topics/exceptions.rst | 2 +- docs/topics/request-response.rst | 137 ++++++++++++------------- docs/topics/selectors.rst | 16 +-- scrapy/core/scheduler.py | 4 +- scrapy/http/request/__init__.py | 12 +-- scrapy/http/response/__init__.py | 8 +- scrapy/http/response/text.py | 14 +-- scrapy/loader/__init__.py | 4 +- scrapy/utils/request.py | 10 +- 13 files changed, 176 insertions(+), 178 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index d06828bcc..fd8165db3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -284,6 +284,7 @@ intersphinx_mapping = { "cryptography": ("https://cryptography.io/en/latest/", None), "cssselect": ("https://cssselect.readthedocs.io/en/latest", None), "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None), + "parsel": ("https://parsel.readthedocs.io/en/latest/", None), "pytest": ("https://docs.pytest.org/en/latest", None), "python": ("https://docs.python.org/3", None), "sphinx": ("https://www.sphinx-doc.org/en/master", None), diff --git a/docs/news.rst b/docs/news.rst index 2bf65272f..924abb7a1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -635,7 +635,7 @@ Bug fixes exception if ``default`` is ``None``. (:issue:`6308`, :issue:`6310`) -- :class:`~scrapy.selector.Selector` now uses +- :class:`~scrapy.Selector` now uses :func:`scrapy.utils.response.get_base_url` to determine the base URL of a given :class:`~scrapy.http.Response`. (:issue:`6265`) @@ -653,7 +653,7 @@ Documentation - Add a FAQ entry about :ref:`creating blank requests `. (:issue:`6203`, :issue:`6208`) -- Document that :attr:`scrapy.selector.Selector.type` can be ``"json"``. +- Document that :attr:`scrapy.Selector.type` can be ``"json"``. (:issue:`6328`, :issue:`6334`) Quality assurance @@ -734,7 +734,7 @@ Documentation - Improved documentation for :class:`~scrapy.crawler.Crawler` initialization changes made in the 2.11.0 release. (:issue:`6057`, :issue:`6147`) -- Extended documentation for :attr:`Request.meta `. +- Extended documentation for :attr:`.Request.meta`. (:issue:`5565`) - Fixed the :reqmeta:`dont_merge_cookies` documentation. (:issue:`5936`, @@ -1095,7 +1095,7 @@ New features :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) -- Added :meth:`TextResponse.jmespath`, a shortcut for JMESPath selectors +- Added :meth:`.TextResponse.jmespath`, a shortcut for JMESPath selectors available since parsel_ 1.8.1. (:issue:`5894`, :issue:`5915`) - Added :signal:`feed_slot_closed` and :signal:`feed_exporter_closed` @@ -1275,7 +1275,7 @@ New features avoid confusion. (:issue:`5717`, :issue:`5722`, :issue:`5727`) -- The ``callback`` parameter of :class:`~scrapy.http.Request` can now be set +- The ``callback`` parameter of :class:`~scrapy.Request` can now be set to :func:`scrapy.http.request.NO_CALLBACK`, to distinguish it from ``None``, as the latter indicates that the default spider callback (:meth:`~scrapy.Spider.parse`) is to be used. @@ -1772,17 +1772,17 @@ Highlights: Security bug fixes ~~~~~~~~~~~~~~~~~~ -- When a :class:`~scrapy.http.Request` object with cookies defined gets a - redirect response causing a new :class:`~scrapy.http.Request` object to be +- When a :class:`~scrapy.Request` object with cookies defined gets a + redirect response causing a new :class:`~scrapy.Request` object to be scheduled, the cookies defined in the original - :class:`~scrapy.http.Request` object are no longer copied into the new - :class:`~scrapy.http.Request` object. + :class:`~scrapy.Request` object are no longer copied into the new + :class:`~scrapy.Request` object. If you manually set the ``Cookie`` header on a - :class:`~scrapy.http.Request` object and the domain name of the redirect + :class:`~scrapy.Request` object and the domain name of the redirect URL is not an exact match for the domain of the URL of the original - :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped - from the new :class:`~scrapy.http.Request` object. + :class:`~scrapy.Request` object, your ``Cookie`` header is now dropped + from the new :class:`~scrapy.Request` object. The old behavior could be exploited by an attacker to gain access to your cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more @@ -1795,10 +1795,10 @@ Security bug fixes ``example.com`` and any subdomain) by defining the shared domain suffix (e.g. ``example.com``) as the cookie domain when defining your cookies. See the documentation of the - :class:`~scrapy.http.Request` class for more information. + :class:`~scrapy.Request` class for more information. - When the domain of a cookie, either received in the ``Set-Cookie`` header - of a response or defined in a :class:`~scrapy.http.Request` object, is set + of a response or defined in a :class:`~scrapy.Request` object, is set to a `public suffix `_, the cookie is now ignored unless the cookie domain is the same as the request domain. @@ -1849,7 +1849,7 @@ Backward-incompatible changes meet expectations, :exc:`TypeError` is now raised at startup time. Before, other exceptions would be raised at run time. (:issue:`3559`) -- The ``_encoding`` field of serialized :class:`~scrapy.http.Request` objects +- The ``_encoding`` field of serialized :class:`~scrapy.Request` objects is now named ``encoding``, in line with all other fields (:issue:`5130`) @@ -1879,7 +1879,7 @@ Deprecations - :mod:`scrapy.utils.reqser` is deprecated. (:issue:`5130`) - Instead of :func:`~scrapy.utils.reqser.request_to_dict`, use the new - :meth:`Request.to_dict ` method. + :meth:`.Request.to_dict` method. - Instead of :func:`~scrapy.utils.reqser.request_from_dict`, use the new :func:`scrapy.utils.request.request_from_dict` function. @@ -1984,9 +1984,9 @@ New features using ``queuelib`` 1.6.1 or later), the ``peek`` method raises :exc:`NotImplementedError`. -- :class:`~scrapy.http.Request` and :class:`~scrapy.http.Response` now have +- :class:`~scrapy.Request` and :class:`~scrapy.http.Response` now have an ``attributes`` attribute that makes subclassing easier. For - :class:`~scrapy.http.Request`, it also allows subclasses to work with + :class:`~scrapy.Request`, it also allows subclasses to work with :func:`scrapy.utils.request.request_from_dict`. (:issue:`1877`, :issue:`5130`, :issue:`5218`) @@ -2452,14 +2452,13 @@ Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` once again - discards cookies defined in :attr:`Request.headers - `. + discards cookies defined in :attr:`.Request.headers`. We decided to revert this bug fix, introduced in Scrapy 2.2.0, because it was reported that the current implementation could break existing code. If you need to set cookies for a request, use the :class:`Request.cookies - ` parameter. + ` parameter. A future version of Scrapy will include a new, better implementation of the reverted bug fix. @@ -2580,16 +2579,16 @@ New features :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response` or :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception` - with a custom :class:`~scrapy.http.Request` object assigned to + with a custom :class:`~scrapy.Request` object assigned to :class:`response.request `: - The response is handled by the callback of that custom - :class:`~scrapy.http.Request` object, instead of being handled by the - callback of the original :class:`~scrapy.http.Request` object + :class:`~scrapy.Request` object, instead of being handled by the + callback of the original :class:`~scrapy.Request` object - - That custom :class:`~scrapy.http.Request` object is now sent as the + - That custom :class:`~scrapy.Request` object is now sent as the ``request`` argument to the :signal:`response_received` signal, instead - of the original :class:`~scrapy.http.Request` object + of the original :class:`~scrapy.Request` object (:issue:`4529`, :issue:`4632`) @@ -2760,7 +2759,7 @@ New features * The :command:`parse` command now allows specifying an output file (:issue:`4317`, :issue:`4377`) -* :meth:`Request.from_curl ` and +* :meth:`.Request.from_curl` and :func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support ``--data-raw`` (:issue:`4612`) @@ -2776,7 +2775,7 @@ Bug fixes :ref:`dataclass items ` and :ref:`attr.s items ` (:issue:`4667`, :issue:`4668`) -* :meth:`Request.from_curl ` and +* :meth:`.Request.from_curl` and :func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request method to ``POST`` when a request body is specified and no request method is specified (:issue:`4612`) @@ -2861,8 +2860,7 @@ Backward-incompatible changes Deprecations ~~~~~~~~~~~~ -* :meth:`TextResponse.body_as_unicode - ` is now deprecated, use +* ``TextResponse.body_as_unicode()`` is now deprecated, use :attr:`TextResponse.text ` instead (:issue:`4546`, :issue:`4555`, :issue:`4579`) @@ -2901,9 +2899,8 @@ New features * :ref:`Link extractors ` are now serializable, as long as you do not use :ref:`lambdas ` for parameters; for - example, you can now pass link extractors in :attr:`Request.cb_kwargs - ` or - :attr:`Request.meta ` when :ref:`persisting + example, you can now pass link extractors in :attr:`.Request.cb_kwargs` + or :attr:`.Request.meta` when :ref:`persisting scheduled requests ` (:issue:`4554`) * Upgraded the :ref:`pickle protocol ` that Scrapy uses @@ -2922,11 +2919,11 @@ Bug fixes * :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer discards cookies defined in :attr:`Request.headers - ` (:issue:`1992`, :issue:`2400`) + ` (:issue:`1992`, :issue:`2400`) * :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer re-encodes cookies defined as :class:`bytes` in the ``cookies`` parameter - of the ``__init__`` method of :class:`~scrapy.http.Request` + of the ``__init__`` method of :class:`~scrapy.Request` (:issue:`2400`, :issue:`3575`) * When :setting:`FEEDS` defines multiple URIs, :setting:`FEED_STORE_EMPTY` is @@ -2935,7 +2932,7 @@ Bug fixes * :class:`~scrapy.spiders.Spider` callbacks defined using :doc:`coroutine syntax ` no longer need to return an iterable, and may - instead return a :class:`~scrapy.http.Request` object, an + instead return a :class:`~scrapy.Request` object, an :ref:`item `, or ``None`` (:issue:`4609`) * The :command:`startproject` command now ensures that the generated project @@ -2976,8 +2973,8 @@ Documentation :issue:`4587`) * The display-on-hover behavior of internal documentation references now also - covers links to :ref:`commands `, :attr:`Request.meta - ` keys, :ref:`settings ` and + covers links to :ref:`commands `, :attr:`.Request.meta` + keys, :ref:`settings ` and :ref:`signals ` (:issue:`4495`, :issue:`4563`) * It is again possible to download the documentation for offline reading @@ -3262,7 +3259,7 @@ Deprecation removals ~~~~~~~~~~~~~~~~~~~~ * The :ref:`Scrapy shell ` no longer provides a `sel` proxy - object, use :meth:`response.selector ` + object, use :meth:`response.selector ` instead (:issue:`4347`) * LevelDB support has been removed (:issue:`4112`) @@ -3332,10 +3329,10 @@ New features * The new :attr:`Response.cb_kwargs ` attribute serves as a shortcut for :attr:`Response.request.cb_kwargs - ` (:issue:`4331`) + ` (:issue:`4331`) * :meth:`Response.follow ` now supports a - ``flags`` parameter, for consistency with :class:`~scrapy.http.Request` + ``flags`` parameter, for consistency with :class:`~scrapy.Request` (:issue:`4277`, :issue:`4279`) * :ref:`Item loader processors ` can now be @@ -3344,7 +3341,7 @@ New features * :class:`~scrapy.spiders.Rule` now accepts an ``errback`` parameter (:issue:`4000`) -* :class:`~scrapy.http.Request` no longer requires a ``callback`` parameter +* :class:`~scrapy.Request` no longer requires a ``callback`` parameter when an ``errback`` parameter is specified (:issue:`3586`, :issue:`4008`) * :class:`~scrapy.logformatter.LogFormatter` now supports some additional @@ -3416,7 +3413,7 @@ Bug fixes * Redirects to URLs starting with 3 slashes (``///``) are now supported (:issue:`4032`, :issue:`4042`) -* :class:`~scrapy.http.Request` no longer accepts strings as ``url`` simply +* :class:`~scrapy.Request` no longer accepts strings as ``url`` simply because they have a colon (:issue:`2552`, :issue:`4094`) * The correct encoding is now used for attach names in @@ -3462,7 +3459,7 @@ Documentation using :class:`~scrapy.crawler.CrawlerProcess` (:issue:`2149`, :issue:`2352`, :issue:`3146`, :issue:`3960`) -* Clarified the requirements for :class:`~scrapy.http.Request` objects +* Clarified the requirements for :class:`~scrapy.Request` objects :ref:`when using persistence ` (:issue:`4124`, :issue:`4139`) @@ -3731,17 +3728,17 @@ Scrapy 1.8.2 (2022-03-01) **Security bug fixes:** -- When a :class:`~scrapy.http.Request` object with cookies defined gets a - redirect response causing a new :class:`~scrapy.http.Request` object to be +- When a :class:`~scrapy.Request` object with cookies defined gets a + redirect response causing a new :class:`~scrapy.Request` object to be scheduled, the cookies defined in the original - :class:`~scrapy.http.Request` object are no longer copied into the new - :class:`~scrapy.http.Request` object. + :class:`~scrapy.Request` object are no longer copied into the new + :class:`~scrapy.Request` object. If you manually set the ``Cookie`` header on a - :class:`~scrapy.http.Request` object and the domain name of the redirect + :class:`~scrapy.Request` object and the domain name of the redirect URL is not an exact match for the domain of the URL of the original - :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped - from the new :class:`~scrapy.http.Request` object. + :class:`~scrapy.Request` object, your ``Cookie`` header is now dropped + from the new :class:`~scrapy.Request` object. The old behavior could be exploited by an attacker to gain access to your cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more @@ -3754,10 +3751,10 @@ Scrapy 1.8.2 (2022-03-01) ``example.com`` and any subdomain) by defining the shared domain suffix (e.g. ``example.com``) as the cookie domain when defining your cookies. See the documentation of the - :class:`~scrapy.http.Request` class for more information. + :class:`~scrapy.Request` class for more information. - When the domain of a cookie, either received in the ``Set-Cookie`` header - of a response or defined in a :class:`~scrapy.http.Request` object, is set + of a response or defined in a :class:`~scrapy.Request` object, is set to a `public suffix `_, the cookie is now ignored unless the cookie domain is the same as the request domain. @@ -3815,7 +3812,7 @@ Highlights: * Dropped Python 3.4 support and updated minimum requirements; made Python 3.8 support official -* New :meth:`Request.from_curl ` class method +* New :meth:`.Request.from_curl` class method * New :setting:`ROBOTSTXT_PARSER` and :setting:`ROBOTSTXT_USER_AGENT` settings * New :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` and :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` settings @@ -3869,7 +3866,7 @@ See also :ref:`1.8-deprecation-removals` below. New features ~~~~~~~~~~~~ -* A new :meth:`Request.from_curl ` class +* A new :meth:`Request.from_curl ` class method allows :ref:`creating a request from a cURL command ` (:issue:`2985`, :issue:`3862`) @@ -3898,9 +3895,8 @@ New features ``True`` to enable debug-level messages about TLS connection parameters after establishing HTTPS connections (:issue:`2111`, :issue:`3450`) -* Callbacks that receive keyword arguments - (see :attr:`Request.cb_kwargs `) can now be - tested using the new :class:`@cb_kwargs +* Callbacks that receive keyword arguments (see :attr:`.Request.cb_kwargs`) + can now be tested using the new :class:`@cb_kwargs ` :ref:`spider contract ` (:issue:`3985`, :issue:`3988`) @@ -4089,7 +4085,7 @@ Backward-incompatible changes * Non-default values for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting may stop working. Scheduler priority queue classes now need to handle - :class:`~scrapy.http.Request` objects instead of arbitrary Python data + :class:`~scrapy.Request` objects instead of arbitrary Python data structures. * An additional ``crawler`` parameter has been added to the ``__init__`` @@ -4111,7 +4107,7 @@ New features scheduling improvement on crawls targeting multiple web domains, at the cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`) -* A new :attr:`Request.cb_kwargs ` attribute +* A new :attr:`.Request.cb_kwargs` attribute provides a cleaner way to pass keyword arguments to callback methods (:issue:`1138`, :issue:`3563`) @@ -4192,7 +4188,7 @@ Bug fixes * Requests with private callbacks are now correctly unserialized from disk (:issue:`3790`) -* :meth:`FormRequest.from_response() ` +* :meth:`.FormRequest.from_response` now handles invalid methods like major web browsers (:issue:`3777`, :issue:`3794`) @@ -4272,13 +4268,13 @@ The following deprecated APIs have been removed (:issue:`3578`): * From both ``scrapy.selector`` and ``scrapy.selector.lxmlsel``: - * ``HtmlXPathSelector`` (use :class:`~scrapy.selector.Selector`) + * ``HtmlXPathSelector`` (use :class:`~scrapy.Selector`) - * ``XmlXPathSelector`` (use :class:`~scrapy.selector.Selector`) + * ``XmlXPathSelector`` (use :class:`~scrapy.Selector`) - * ``XPathSelector`` (use :class:`~scrapy.selector.Selector`) + * ``XPathSelector`` (use :class:`~scrapy.Selector`) - * ``XPathSelectorList`` (use :class:`~scrapy.selector.Selector`) + * ``XPathSelectorList`` (use :class:`~scrapy.Selector`) * From ``scrapy.selector.csstranslator``: @@ -4288,7 +4284,7 @@ The following deprecated APIs have been removed (:issue:`3578`): * ``ScrapyXPathExpr`` (use parsel.csstranslator.XPathExpr_) -* From :class:`~scrapy.selector.Selector`: +* From :class:`~scrapy.Selector`: * ``_root`` (both the ``__init__`` method argument and the object property, use ``root``) @@ -4818,7 +4814,7 @@ New Features (:issue:`2535`) - New :ref:`response.follow ` shortcut for creating requests (:issue:`1940`) -- Added ``flags`` argument and attribute to :class:`Request ` +- Added ``flags`` argument and attribute to :class:`~scrapy.Request` objects (:issue:`2047`) - Support Anonymous FTP (:issue:`2342`) - Added ``retry/count``, ``retry/max_reached`` and ``retry/reason_count/`` @@ -4860,7 +4856,7 @@ Bug fixes - LinkExtractor now strips leading and trailing whitespaces from attributes (:issue:`2547`, fixes :issue:`1614`) - Properly handle whitespaces in action attribute in - :class:`~scrapy.http.FormRequest` (:issue:`2548`) + :class:`~scrapy.FormRequest` (:issue:`2548`) - Buffer CONNECT response bytes from proxy until all HTTP headers are received (:issue:`2495`, fixes :issue:`2491`) - FTP downloader now works on Python 3, provided you use Twisted>=17.1 @@ -4902,8 +4898,7 @@ Documentation ~~~~~~~~~~~~~ - Binary mode is required for exporters (:issue:`2564`, fixes :issue:`2553`) -- Mention issue with :meth:`FormRequest.from_response - ` due to bug in lxml (:issue:`2572`) +- Mention issue with :meth:`.FormRequest.from_response` due to bug in lxml (:issue:`2572`) - Use single quotes uniformly in templates (:issue:`2596`) - Document :reqmeta:`ftp_user` and :reqmeta:`ftp_password` meta keys (:issue:`2587`) - Removed section on deprecated ``contrib/`` (:issue:`2636`) @@ -5442,7 +5437,7 @@ Bugfixes - Support empty password for http_proxy config (:issue:`1274`). - Interpret ``application/x-json`` as ``TextResponse`` (:issue:`1333`). - Support link rel attribute with multiple values (:issue:`1201`). -- Fixed ``scrapy.http.FormRequest.from_response`` when there is a ```` +- Fixed ``scrapy.FormRequest.from_response`` when there is a ```` tag (:issue:`1564`). - Fixed :setting:`TEMPLATES_DIR` handling (:issue:`1575`). - Various ``FormRequest`` fixes (:issue:`1595`, :issue:`1596`, :issue:`1597`). @@ -6369,7 +6364,7 @@ Scrapy 0.18.0 (released 2013-08-09) - Moved persistent (on disk) queues to a separate project (queuelib_) which Scrapy now depends on - Add Scrapy commands using external libraries (:issue:`260`) - Added ``--pdb`` option to ``scrapy`` command line tool -- Added :meth:`XPathSelector.remove_namespaces ` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`. +- Added :meth:`XPathSelector.remove_namespaces ` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`. - Several improvements to spider contracts - New default middleware named MetaRefreshMiddleware that handles meta-refresh html tag redirections, - MetaRefreshMiddleware and RedirectMiddleware have different priorities to address #62 diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 11a3fcb94..af7885a45 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -80,7 +80,7 @@ object gives you access, for example, to the :ref:`settings `. middleware. :meth:`process_request` should either: return ``None``, return a - :class:`~scrapy.Response` object, return a :class:`~scrapy.http.Request` + :class:`~scrapy.http.Response` object, return a :class:`~scrapy.Request` object, or raise :exc:`~scrapy.exceptions.IgnoreRequest`. If it returns ``None``, Scrapy will continue processing this request, executing all diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 75d980835..801f6d06d 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -117,7 +117,8 @@ data from it depends on the type of response: - If the response is HTML, XML or JSON, use :ref:`selectors ` as usual. -- If the response is JSON, use :func:`response.json()` to load the desired data: +- If the response is JSON, use :func:`response.json() + ` to load the desired data: .. code-block:: python @@ -143,7 +144,7 @@ data from it depends on the type of response: - If the response is an image or another format based on images (e.g. PDF), read the response as bytes from - :attr:`response.body ` and use an OCR + :attr:`response.body ` and use an OCR solution to extract the desired data as text. For example, you can use pytesseract_. To read a table from a PDF, diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index ea64edbe6..0b572ff95 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -105,7 +105,7 @@ response: In both cases, the response could have its body truncated: the body contains all bytes received up until the exception is raised, including the bytes received in the signal handler that raises the exception. Also, the response -object is marked with ``"download_stopped"`` in its :attr:`Response.flags` +object is marked with ``"download_stopped"`` in its :attr:`~scrapy.http.Response.flags` attribute. .. note:: ``fail`` is a keyword-only parameter, i.e. raising diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 710e2e131..1bb1a10a4 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -7,15 +7,15 @@ Requests and Responses .. module:: scrapy.http :synopsis: Request and Response classes -Scrapy uses :class:`Request` and :class:`Response` objects for crawling web +Scrapy uses :class:`~scrapy.Request` and :class:`Response` objects for crawling web sites. -Typically, :class:`Request` objects are generated in the spiders and pass +Typically, :class:`~scrapy.Request` objects are generated in the spiders and pass across the system until they reach the Downloader, which executes the request and returns a :class:`Response` object which travels back to the spider that issued the request. -Both :class:`Request` and :class:`Response` classes have subclasses which add +Both :class:`~scrapy.Request` and :class:`Response` classes have subclasses which add functionality not required in the base classes. These are described below in :ref:`topics-request-response-ref-request-subclasses` and :ref:`topics-request-response-ref-response-subclasses`. @@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and Request objects =============== -.. autoclass:: Request +.. autoclass:: scrapy.Request :param url: the URL of this request @@ -52,7 +52,7 @@ Request objects :param method: the HTTP method of this request. Defaults to ``'GET'``. :type method: str - :param meta: the initial values for the :attr:`Request.meta` attribute. If + :param meta: the initial values for the :attr:`.Request.meta` attribute. If given, the dict passed in this parameter will be shallow copied. :type meta: dict @@ -67,10 +67,10 @@ Request objects (for single valued headers) or lists (for multi-valued headers). If ``None`` is passed as value, the HTTP header will not be sent at all. - .. caution:: Cookies set via the ``Cookie`` header are not considered by the - :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known - current limitation that is being worked on. + .. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + ``cookies`` argument. This is a known current limitation that is being + worked on. :type headers: dict @@ -124,7 +124,7 @@ Request objects .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`scrapy.Request.cookies ` parameter. This is a known current limitation that is being worked on. .. versionadded:: 2.6.0 @@ -172,7 +172,7 @@ Request objects A string containing the URL of this request. Keep in mind that this attribute contains the escaped URL, so it can differ from the URL passed in - the ``__init__`` method. + the ``__init__()`` method. This attribute is read-only. To change the URL of a Request use :meth:`replace`. @@ -184,7 +184,8 @@ Request objects .. attribute:: Request.headers - A dictionary-like object which contains the request headers. + A dictionary-like (:class:`scrapy.http.headers.Headers`) object which contains + the request headers. .. attribute:: Request.body @@ -240,8 +241,8 @@ Request objects A dictionary that contains arbitrary metadata for this request. Its contents will be passed to the Request's callback as keyword arguments. It is empty - for new Requests, which means by default callbacks only get a :class:`Response` - object as argument. + for new Requests, which means by default callbacks only get a + :class:`~scrapy.http.Response` object as argument. This dict is :doc:`shallow copied ` when the request is cloned using the ``copy()`` or ``replace()`` methods, and can also be @@ -262,7 +263,7 @@ Request objects Return a Request object with the same members, except for those members given new values by whichever keyword arguments are specified. The - :attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are shallow + :attr:`~scrapy.Request.cb_kwargs` and :attr:`~scrapy.Request.meta` attributes are shallow copied by default (unless new values are given as arguments). See also :ref:`topics-request-response-ref-request-callback-arguments`. @@ -305,7 +306,7 @@ Example: In some cases you may be interested in passing arguments to those callback functions so you can receive the arguments later, in the second callback. The following example shows how to achieve this by using the -:attr:`Request.cb_kwargs` attribute: +:attr:`.Request.cb_kwargs` attribute: .. code-block:: python @@ -326,10 +327,10 @@ The following example shows how to achieve this by using the foo=foo, ) -.. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``. - Prior to that, using :attr:`Request.meta` was recommended for passing - information around callbacks. After ``1.7``, :attr:`Request.cb_kwargs` - became the preferred way for handling user information, leaving :attr:`Request.meta` +.. caution:: :attr:`.Request.cb_kwargs` was introduced in version ``1.7``. + Prior to that, using :attr:`.Request.meta` was recommended for passing + information around callbacks. After ``1.7``, :attr:`.Request.cb_kwargs` + became the preferred way for handling user information, leaving :attr:`.Request.meta` for communication with components like middlewares and extensions. .. _topics-request-response-ref-errbacks: @@ -441,7 +442,7 @@ Request fingerprints There are some aspects of scraping, such as filtering out duplicate requests (see :setting:`DUPEFILTER_CLASS`) or caching responses (see :setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short, -unique identifier from a :class:`~scrapy.http.Request` object: a request +unique identifier from a :class:`~scrapy.Request` object: a request fingerprint. You often do not need to worry about request fingerprints, the default request @@ -486,7 +487,7 @@ A request fingerprinter is a class that must implement the following method: See also :ref:`request-fingerprint-restrictions`. :param request: request to fingerprint - :type request: scrapy.http.Request + :type request: scrapy.Request Additionally, it may also implement the following method: @@ -566,7 +567,7 @@ URL canonicalization or taking the request method or body into account: If you need to be able to override the request fingerprinting for arbitrary requests from your spider callbacks, you may implement a request fingerprinter -that reads fingerprints from :attr:`request.meta ` +that reads fingerprints from :attr:`request.meta ` when available, and then falls back to :func:`scrapy.utils.request.fingerprint`. For example: @@ -581,10 +582,8 @@ when available, and then falls back to return request.meta["fingerprint"] return fingerprint(request) -If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6 -without using the deprecated ``'2.6'`` value of the -:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following -request fingerprinter: +If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6, use +the following request fingerprinter: .. code-block:: python @@ -628,7 +627,7 @@ The following built-in Scrapy components have such restrictions: :setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`, the following directory structure is created: - - :attr:`Spider.name ` + - :attr:`.Spider.name` - first byte of a request fingerprint as hexadecimal @@ -656,7 +655,7 @@ The following built-in Scrapy components have such restrictions: Request.meta special keys ========================= -The :attr:`Request.meta` attribute can contain any arbitrary data, but there +The :attr:`.Request.meta` attribute can contain any arbitrary data, but there are some special keys recognized by Scrapy and its built-in extensions. Those are: @@ -780,24 +779,25 @@ call their callback instead, like in this example, pass ``fail=False`` to the Request subclasses ================== -Here is the list of built-in :class:`Request` subclasses. You can also subclass +Here is the list of built-in :class:`~scrapy.Request` subclasses. You can also subclass it to implement your own custom functionality. FormRequest objects ------------------- -The FormRequest class extends the base :class:`Request` with functionality for +The FormRequest class extends the base :class:`~scrapy.Request` with functionality for dealing with HTML forms. It uses `lxml.html forms`_ to pre-populate form fields with form data from :class:`Response` objects. .. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms -.. class:: scrapy.http.request.form.FormRequest -.. class:: scrapy.http.FormRequest -.. class:: scrapy.FormRequest(url, [formdata, ...]) +.. currentmodule:: None - The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The - remaining arguments are the same as for the :class:`Request` class and are +.. class:: scrapy.FormRequest(url, [formdata, ...]) + :canonical: scrapy.http.request.form.FormRequest + + The :class:`~scrapy.FormRequest` class adds a new keyword parameter to the ``__init__()`` method. The + remaining arguments are the same as for the :class:`~scrapy.Request` class and are not documented here. :param formdata: is a dictionary (or iterable of (key, value) tuples) @@ -805,12 +805,12 @@ fields with form data from :class:`Response` objects. body of the request. :type formdata: dict or collections.abc.Iterable - The :class:`FormRequest` objects support the following class method in - addition to the standard :class:`Request` methods: + The :class:`~scrapy.FormRequest` objects support the following class method in + addition to the standard :class:`~scrapy.Request` methods: - .. classmethod:: FormRequest.from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...]) + .. classmethod:: from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...]) - Returns a new :class:`FormRequest` object with its form field values + Returns a new :class:`~scrapy.FormRequest` object with its form field values pre-populated with those found in the HTML ``
`` element contained in the given response. For an example see :ref:`topics-request-response-ref-request-userlogin`. @@ -832,7 +832,7 @@ fields with form data from :class:`Response` objects. :param response: the response containing a HTML form which will be used to pre-populate the form fields - :type response: :class:`Response` object + :type response: :class:`~scrapy.http.Response` object :param formname: if given, the form with name attribute set to this value will be used. :type formname: str @@ -869,7 +869,9 @@ fields with form data from :class:`Response` objects. :type dont_click: bool The other parameters of this class method are passed directly to the - :class:`FormRequest` ``__init__`` method. + :class:`~scrapy.FormRequest` ``__init__()`` method. + +.. currentmodule:: scrapy.http Request usage examples ---------------------- @@ -878,7 +880,7 @@ Using FormRequest to send data via HTTP POST ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you want to simulate a HTML Form POST in your spider and send a couple of -key-value fields, you can return a :class:`FormRequest` object (from your +key-value fields, you can return a :class:`~scrapy.FormRequest` object (from your spider) like this: .. skip: next @@ -901,7 +903,7 @@ It is usual for web sites to provide pre-populated form fields through ```` elements, such as session related data or authentication tokens (for login pages). When scraping, you'll want these fields to be automatically pre-populated and only override a couple of them, such as the -user name and password. You can use the :meth:`FormRequest.from_response` +user name and password. You can use the :meth:`.FormRequest.from_response()` method for this job. Here's an example spider which uses it: .. code-block:: python @@ -936,21 +938,22 @@ method for this job. Here's an example spider which uses it: JsonRequest ----------- -The JsonRequest class extends the base :class:`Request` class with functionality for +The JsonRequest class extends the base :class:`~scrapy.Request` class with functionality for dealing with JSON requests. .. class:: JsonRequest(url, [... data, dumps_kwargs]) - The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method. The - remaining arguments are the same as for the :class:`Request` class and are + The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__()`` method. The + remaining arguments are the same as for the :class:`~scrapy.Request` class and are not documented here. Using the :class:`JsonRequest` will set the ``Content-Type`` header to ``application/json`` and ``Accept`` header to ``application/json, text/javascript, */*; q=0.01`` :param data: is any JSON serializable object that needs to be JSON encoded and assigned to body. - if :attr:`Request.body` argument is provided this parameter will be ignored. - if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be + If the :attr:`~scrapy.Request.body` argument is provided this parameter will be ignored. + If the :attr:`~scrapy.Request.body` argument is not provided and the + ``data`` argument is provided the :attr:`~scrapy.Request.method` will be set to ``'POST'`` automatically. :type data: object @@ -1002,7 +1005,7 @@ Response objects :type flags: list :param request: the initial value of the :attr:`Response.request` attribute. - This represents the :class:`Request` that generated this response. + This represents the :class:`~scrapy.Request` that generated this response. :type request: scrapy.Request :param certificate: an object representing the server's SSL certificate. @@ -1038,11 +1041,12 @@ Response objects .. attribute:: Response.headers - A dictionary-like object which contains the response headers. Values can - be accessed using :meth:`get` to return the first header value with the - specified name or :meth:`getlist` to return all header values with the - specified name. For example, this call will give you all cookies in the - headers:: + A dictionary-like (:class:`scrapy.http.headers.Headers`) object which contains + the response headers. Values can be accessed using + :meth:`~scrapy.http.headers.Headers.get` to return the first header value with + the specified name or :meth:`~scrapy.http.headers.Headers.getlist` to return + all header values with the specified name. For example, this call will give you + all cookies in the headers:: response.headers.getlist('Set-Cookie') @@ -1058,7 +1062,7 @@ Response objects .. attribute:: Response.request - The :class:`Request` object that generated this response. This attribute is + The :class:`~scrapy.Request` object that generated this response. This attribute is assigned in the Scrapy engine, after the response and the request have passed through all :ref:`Downloader Middlewares `. In particular, this means that: @@ -1077,34 +1081,33 @@ Response objects .. attribute:: Response.meta - A shortcut to the :attr:`Request.meta` attribute of the + A shortcut to the :attr:`~scrapy.Request.meta` attribute of the :attr:`Response.request` object (i.e. ``self.request.meta``). Unlike the :attr:`Response.request` attribute, the :attr:`Response.meta` attribute is propagated along redirects and retries, so you will get - the original :attr:`Request.meta` sent from your spider. + the original :attr:`.Request.meta` sent from your spider. - .. seealso:: :attr:`Request.meta` attribute + .. seealso:: :attr:`.Request.meta` attribute .. attribute:: Response.cb_kwargs .. versionadded:: 2.0 - A shortcut to the :attr:`Request.cb_kwargs` attribute of the + A shortcut to the :attr:`~scrapy.Request.cb_kwargs` attribute of the :attr:`Response.request` object (i.e. ``self.request.cb_kwargs``). Unlike the :attr:`Response.request` attribute, the :attr:`Response.cb_kwargs` attribute is propagated along redirects and - retries, so you will get the original :attr:`Request.cb_kwargs` sent - from your spider. + retries, so you will get the original :attr:`.Request.cb_kwargs` sent from your spider. - .. seealso:: :attr:`Request.cb_kwargs` attribute + .. seealso:: :attr:`.Request.cb_kwargs` attribute .. attribute:: Response.flags A list that contains flags for this response. Flags are labels used for tagging Responses. For example: ``'cached'``, ``'redirected``', etc. And - they're shown on the string representation of the Response (`__str__` + they're shown on the string representation of the Response (``__str__()`` method) which is used by the engine for logging. .. attribute:: Response.certificate @@ -1181,7 +1184,7 @@ TextResponse objects :class:`Response` class, which is meant to be used only for binary data, such as images, sounds or any media file. - :class:`TextResponse` objects support a new ``__init__`` method argument, in + :class:`TextResponse` objects support a new ``__init__()`` method argument, in addition to the base :class:`Response` objects. The remaining functionality is the same as for the :class:`Response` class and is not documented here. @@ -1219,7 +1222,7 @@ TextResponse objects A string with the encoding of this response. The encoding is resolved by trying the following mechanisms, in order: - 1. the encoding passed in the ``__init__`` method ``encoding`` argument + 1. the encoding passed in the ``__init__()`` method ``encoding`` argument 2. the encoding declared in the Content-Type HTTP header. If this encoding is not valid (i.e. unknown), it is ignored and the next @@ -1273,7 +1276,7 @@ TextResponse objects Constructs an absolute url by combining the Response's base url with a possible relative url. The base url shall be extracted from the - ```` tag, or just the Response's :attr:`url` if there is no such + ```` tag, or just :attr:`Response.url` if there is no such tag. diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 202b0823a..b95e6eab3 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -777,7 +777,7 @@ Removing namespaces When dealing with scraping projects, it is often quite convenient to get rid of namespaces altogether and just work with element names, to write more simple/convenient XPaths. You can use the -:meth:`Selector.remove_namespaces` method for that. +:meth:`.Selector.remove_namespaces` method for that. Let's show an example that illustrates this with the Python Insider blog atom feed. @@ -814,7 +814,7 @@ doesn't work (because the Atom XML namespace is obfuscating those nodes): >>> response.xpath("//link") [] -But once we call the :meth:`Selector.remove_namespaces` method, all +But once we call the :meth:`.Selector.remove_namespaces` method, all nodes can be accessed directly by their names: .. code-block:: pycon @@ -1046,7 +1046,7 @@ Built-in Selectors reference Selector objects ---------------- -.. autoclass:: Selector +.. autoclass:: scrapy.Selector .. automethod:: xpath @@ -1126,8 +1126,8 @@ Examples Selector examples on HTML response ---------------------------------- -Here are some :class:`Selector` examples to illustrate several concepts. -In all cases, we assume there is already a :class:`Selector` instantiated with +Here are some :class:`~scrapy.Selector` examples to illustrate several concepts. +In all cases, we assume there is already a :class:`~scrapy.Selector` instantiated with a :class:`~scrapy.http.HtmlResponse` object like this: .. code-block:: python @@ -1135,7 +1135,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this: sel = Selector(html_response) 1. Select all ``

`` elements from an HTML response body, returning a list of - :class:`Selector` objects (i.e. a :class:`SelectorList` object): + :class:`~scrapy.Selector` objects (i.e. a :class:`SelectorList` object): .. code-block:: python @@ -1165,7 +1165,7 @@ Selector examples on XML response .. skip: start -Here are some examples to illustrate concepts for :class:`Selector` objects +Here are some examples to illustrate concepts for :class:`~scrapy.Selector` objects instantiated with an :class:`~scrapy.http.XmlResponse` object: .. code-block:: python @@ -1173,7 +1173,7 @@ instantiated with an :class:`~scrapy.http.XmlResponse` object: sel = Selector(xml_response) 1. Select all ```` elements from an XML response body, returning a list - of :class:`Selector` objects (i.e. a :class:`SelectorList` object): + of :class:`~scrapy.Selector` objects (i.e. a :class:`SelectorList` object): .. code-block:: python diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index fcc94879a..4bb143dfd 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -115,7 +115,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): @abstractmethod def next_request(self) -> Request | None: """ - Return the next :class:`~scrapy.http.Request` to be processed, or ``None`` + Return the next :class:`~scrapy.Request` to be processed, or ``None`` to indicate that there are no requests to be considered ready at the moment. Returning ``None`` implies that no request from the scheduler will be sent @@ -263,7 +263,7 @@ class Scheduler(BaseScheduler): def next_request(self) -> Request | None: """ - Return a :class:`~scrapy.http.Request` object from the memory queue, + Return a :class:`~scrapy.Request` object from the memory queue, falling back to the disk queue if the memory queue is empty. Return ``None`` if there are no more enqueued requests. diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 3d6cf4816..4eee5ffbb 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -59,7 +59,7 @@ RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn: """When assigned to the ``callback`` parameter of - :class:`~scrapy.http.Request`, it indicates that the request is not meant + :class:`~scrapy.Request`, it indicates that the request is not meant to have a spider callback at all. For example: @@ -83,7 +83,7 @@ def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn: class Request(object_ref): """Represents an HTTP request, which is usually generated in a Spider and - executed by the Downloader, thus generating a :class:`Response`. + executed by the Downloader, thus generating a :class:`~scrapy.http.Response`. """ attributes: tuple[str, ...] = ( @@ -103,9 +103,9 @@ class Request(object_ref): ) """A tuple of :class:`str` objects containing the name of all public attributes of the class that are also keyword parameters of the - ``__init__`` method. + ``__init__()`` method. - Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and + Currently used by :meth:`.Request.replace`, :meth:`.Request.to_dict` and :func:`~scrapy.utils.request.request_from_dict`. """ @@ -233,7 +233,7 @@ class Request(object_ref): finding unknown options call this method by passing ``ignore_unknown_options=False``. - .. caution:: Using :meth:`from_curl` from :class:`~scrapy.http.Request` + .. caution:: Using :meth:`from_curl` from :class:`~scrapy.Request` subclasses, such as :class:`~scrapy.http.JsonRequest`, or :class:`~scrapy.http.XmlRpcRequest`, as well as having :ref:`downloader middlewares ` @@ -244,7 +244,7 @@ class Request(object_ref): :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`, or :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`, - may modify the :class:`~scrapy.http.Request` object. + may modify the :class:`~scrapy.Request` object. To translate a cURL command into a Scrapy request, you may use `curl2scrapy `_. diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 387805f57..b84110b29 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -51,7 +51,7 @@ class Response(object_ref): ) """A tuple of :class:`str` objects containing the name of all public attributes of the class that are also keyword parameters of the - ``__init__`` method. + ``__init__()`` method. Currently used by :meth:`Response.replace`. """ @@ -199,8 +199,8 @@ class Response(object_ref): ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. - It accepts the same arguments as ``Request.__init__`` method, - but ``url`` can be a relative URL or a ``scrapy.link.Link`` object, + It accepts the same arguments as ``Request.__init__()`` method, + but ``url`` can be a relative URL or a :class:`~scrapy.link.Link` object, not only an absolute URL. :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow` @@ -254,7 +254,7 @@ class Response(object_ref): .. versionadded:: 2.0 Return an iterable of :class:`~.Request` instances to follow all links - in ``urls``. It accepts the same arguments as ``Request.__init__`` method, + in ``urls``. It accepts the same arguments as ``Request.__init__()`` method, but elements of ``urls`` can be relative URLs or :class:`~scrapy.link.Link` objects, not only absolute URLs. diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 476f1754e..081223882 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -185,15 +185,15 @@ class TextResponse(Response): ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. - It accepts the same arguments as ``Request.__init__`` method, + It accepts the same arguments as ``Request.__init__()`` method, but ``url`` can be not only an absolute URL, but also * a relative URL * a :class:`~scrapy.link.Link` object, e.g. the result of :ref:`topics-link-extractors` - * a :class:`~scrapy.selector.Selector` object for a ```` or ```` element, e.g. + * a :class:`~scrapy.Selector` object for a ```` or ```` element, e.g. ``response.css('a.my_link')[0]`` - * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g. + * an attribute :class:`~scrapy.Selector` (not SelectorList), e.g. ``response.css('a::attr(href)')[0]`` or ``response.xpath('//img/@src')[0]`` @@ -241,20 +241,20 @@ class TextResponse(Response): """ A generator that produces :class:`~.Request` instances to follow all links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s - ``__init__`` method, except that each ``urls`` element does not need to be + ``__init__()`` method, except that each ``urls`` element does not need to be an absolute URL, it can be any of the following: * a relative URL * a :class:`~scrapy.link.Link` object, e.g. the result of :ref:`topics-link-extractors` - * a :class:`~scrapy.selector.Selector` object for a ```` or ```` element, e.g. + * a :class:`~scrapy.Selector` object for a ```` or ```` element, e.g. ``response.css('a.my_link')[0]`` - * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g. + * an attribute :class:`~scrapy.Selector` (not SelectorList), e.g. ``response.css('a::attr(href)')[0]`` or ``response.xpath('//img/@src')[0]`` In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction - within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted). + within the ``follow_all()`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted). Note that when passing a ``SelectorList`` as argument for the ``urls`` parameter or using the ``css`` or ``xpath`` parameters, this method will not produce requests for diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index d35720a45..2f5c0343b 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -32,7 +32,7 @@ class ItemLoader(itemloaders.ItemLoader): :param selector: The selector to extract data from, when using the :meth:`add_xpath`, :meth:`add_css`, :meth:`replace_xpath`, or :meth:`replace_css` method. - :type selector: :class:`~scrapy.selector.Selector` object + :type selector: :class:`~scrapy.Selector` object :param response: The response used to construct the selector using the :attr:`default_selector_class`, unless the selector argument is given, @@ -79,7 +79,7 @@ class ItemLoader(itemloaders.ItemLoader): .. attribute:: selector - The :class:`~scrapy.selector.Selector` object to extract data from. + The :class:`~scrapy.Selector` object to extract data from. It's either the selector given in the ``__init__`` method or one created from the response given in the ``__init__`` method using the :attr:`default_selector_class`. This attribute is meant to be diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 7f2b178f5..9c1161968 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -1,6 +1,6 @@ """ This module provides some useful functions for working with -scrapy.http.Request objects +scrapy.Request objects """ from __future__ import annotations @@ -109,12 +109,10 @@ class RequestFingerprinter: It takes into account a canonical version (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url - ` and the values of :attr:`request.method - ` and :attr:`request.body - `. It then generates an `SHA1 + ` and the values of :attr:`request.method + ` and :attr:`request.body + `. It then generates an `SHA1 `_ hash. - - .. seealso:: :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION`. """ @classmethod From 7dfbecd3924a0d7a9e555e9cc3618cdb06b5415d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 7 Jan 2025 19:11:10 +0500 Subject: [PATCH 015/491] Fix tracking of coverage in subprocesses. --- pyproject.toml | 6 ++++++ tests/__init__.py | 7 ------- tox.ini | 10 +++++----- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 571a61f1c..29e26399f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,6 +120,12 @@ include = ["scrapy/*"] omit = ["tests/*"] disable_warnings = ["include-ignored"] +[tool.coverage.paths] +source = [ + "scrapy", + ".tox/**/site-packages/scrapy" +] + [tool.coverage.report] # https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185 exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"] diff --git a/tests/__init__.py b/tests/__init__.py index 5f0c0f7ad..cd52ade58 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -15,13 +15,6 @@ os.environ["http_proxy"] = "" os.environ["https_proxy"] = "" os.environ["ftp_proxy"] = "" -# Absolutize paths to coverage config and output file because tests that -# spawn subprocesses also changes current working directory. -_sourceroot = Path(__file__).resolve().parent.parent -if "COV_CORE_CONFIG" in os.environ: - os.environ["COVERAGE_FILE"] = str(_sourceroot / ".coverage") - os.environ["COV_CORE_CONFIG"] = str(_sourceroot / os.environ["COV_CORE_CONFIG"]) - tests_datadir = str(Path(__file__).parent.resolve() / "sample_data") diff --git a/tox.ini b/tox.ini index 39ab1ccd4..de91c8b04 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ deps = pyftpdlib >= 2.0.1 pygments pytest - pytest-cov==4.0.0 + pytest-cov >= 4.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures @@ -36,7 +36,7 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} --doctest-modules + pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} --doctest-modules install_command = python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} @@ -115,7 +115,7 @@ setenv = install_command = python -I -m pip install {opts} {packages} commands = - pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} + pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} [testenv:pinned] basepython = {[pinned]basepython} @@ -241,7 +241,7 @@ deps = {[testenv]deps} botocore>=1.4.87 commands = - pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore} + pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore} [testenv:botocore-pinned] basepython = {[pinned]basepython} @@ -252,4 +252,4 @@ install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands = - pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore} + pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore} From 3154b08e90d9777dfe2879b8686b6fc63a793c84 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 7 Jan 2025 19:40:25 +0500 Subject: [PATCH 016/491] Improve coverage speed on Python 3.12+. --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index de91c8b04..cf5e19a61 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ minversion = 1.7.0 [test-requirements] deps = attrs + coverage >= 7.4.0 pexpect >= 4.8.0 pyftpdlib >= 2.0.1 pygments @@ -26,6 +27,8 @@ deps = # mitmproxy does not support PyPy mitmproxy; implementation_name != 'pypy' +setenv = + COVERAGE_CORE=sysmon passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From 1fc91bb46262118c9ff7aa2b4719d880f727699f Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Thu, 9 Jan 2025 03:28:51 +1100 Subject: [PATCH 017/491] new `allow_offsite` parameter in OffsiteMiddleware (#6151) * new 'allow_offsite' parameter in OffsiteMiddleware * document deprecated dont_filter flag in OffsiteMiddleware * avoid deprecating dont_filter in OffsiteMiddleware * Copy the code to the downloader mw. * Add tests for allow_offsite in the downloader mw. * Mark allow_offsite with reqmeta. --------- Co-authored-by: Andrey Rakhmatullin --- docs/topics/downloader-middleware.rst | 9 ++++--- docs/topics/request-response.rst | 7 ++--- scrapy/downloadermiddlewares/offsite.py | 6 ++++- scrapy/spidermiddlewares/offsite.py | 6 ++++- tests/test_downloadermiddleware_offsite.py | 31 ++++++++++++++++++++++ tests/test_spidermiddleware_offsite.py | 1 + 6 files changed, 52 insertions(+), 8 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index af7885a45..1ab8f588f 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -797,9 +797,12 @@ OffsiteMiddleware :attr:`~scrapy.Spider.allowed_domains` attribute, or the attribute is empty, the offsite middleware will allow all requests. - If the request has the :attr:`~scrapy.Request.dont_filter` attribute - set, the offsite middleware will allow the request even if its domain is not - listed in allowed domains. + .. reqmeta:: allow_offsite + + If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to + ``True`` or :attr:`Request.meta` has ``allow_offsite`` set to ``True``, then + the OffsiteMiddleware will allow the request even if its domain is not listed + in allowed domains. RedirectMiddleware ------------------ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1bb1a10a4..b187f3aaf 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -145,9 +145,9 @@ Request objects :type priority: int :param dont_filter: indicates that this request should not be filtered by - the scheduler. This is used when you want to perform an identical - request multiple times, to ignore the duplicates filter. Use it with - care, or you will get into crawling loops. Default to ``False``. + the scheduler or some middlewares. This is used when you want to perform + an identical request multiple times, to ignore the duplicates filter. + Use it with care, or you will get into crawling loops. Default to ``False``. :type dont_filter: bool :param errback: a function that will be called if any exception was @@ -660,6 +660,7 @@ are some special keys recognized by Scrapy and its built-in extensions. Those are: +* :reqmeta:`allow_offsite` * :reqmeta:`autothrottle_dont_adjust_delay` * :reqmeta:`bindaddress` * :reqmeta:`cookiejar` diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index a69f531a7..a2cff65e7 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -40,7 +40,11 @@ class OffsiteMiddleware: self.process_request(request, spider) def process_request(self, request: Request, spider: Spider) -> None: - if request.dont_filter or self.should_follow(request, spider): + if ( + request.dont_filter + or request.meta.get("allow_offsite") + or self.should_follow(request, spider) + ): return domain = urlparse_cached(request).hostname if domain and domain not in self.domains_seen: diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index d3ed64ef5..95e753830 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -61,7 +61,11 @@ class OffsiteMiddleware: def _filter(self, request: Any, spider: Spider) -> bool: if not isinstance(request, Request): return True - if request.dont_filter or self.should_follow(request, spider): + if ( + request.dont_filter + or request.meta.get("allow_offsite") + or self.should_follow(request, spider) + ): return True domain = urlparse_cached(request).hostname if domain and domain not in self.domains_seen: diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index fec56a39f..23a1d06da 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -64,6 +64,37 @@ def test_process_request_dont_filter(value, filtered): assert mw.process_request(request, spider) is None +@pytest.mark.parametrize( + ("allow_offsite", "dont_filter", "filtered"), + ( + (True, UNSET, False), + (True, None, False), + (True, False, False), + (True, True, False), + (False, UNSET, True), + (False, None, True), + (False, False, True), + (False, True, False), + ), +) +def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): + crawler = get_crawler(Spider) + spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + kwargs = {"meta": {}} + if allow_offsite is not UNSET: + kwargs["meta"]["allow_offsite"] = allow_offsite + if dont_filter is not UNSET: + kwargs["dont_filter"] = dont_filter + request = Request("https://b.example", **kwargs) + if filtered: + with pytest.raises(IgnoreRequest): + mw.process_request(request, spider) + else: + assert mw.process_request(request, spider) is None + + @pytest.mark.parametrize( "value", ( diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 837f1c2c8..906928e01 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -29,6 +29,7 @@ class TestOffsiteMiddleware(TestCase): Request("http://scrapy.org/1"), Request("http://sub.scrapy.org/1"), Request("http://offsite.tld/letmepass", dont_filter=True), + Request("http://offsite-2.tld/allow", meta={"allow_offsite": True}), Request("http://scrapy.test.org/"), Request("http://scrapy.test.org:8000/"), ] From 402500b164efc01257679247d3dd1628a5f90f5e Mon Sep 17 00:00:00 2001 From: Ionut-Cezar Ciubotariu Date: Fri, 10 Jan 2025 20:08:27 +0200 Subject: [PATCH 018/491] Change unknown cmd message when outside project (#3426) * Change unknown cmd message when outside project * Simplification. * Move the import to the top level. * Reword the message. --------- Co-authored-by: Andrey Rakhmatullin --- scrapy/cmdline.py | 22 +++++++++++++++++++++- tests/test_commands.py | 22 +++++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 48f462c65..065adccfb 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -89,6 +89,12 @@ def _get_commands_dict( return cmds +def _get_project_only_cmds(settings: BaseSettings) -> set[str]: + return set(_get_commands_dict(settings, inproject=True)) - set( + _get_commands_dict(settings, inproject=False) + ) + + def _pop_command_name(argv: list[str]) -> str | None: for i, arg in enumerate(argv[1:]): if not arg.startswith("-"): @@ -121,11 +127,25 @@ def _print_commands(settings: BaseSettings, inproject: bool) -> None: print('Use "scrapy -h" to see more info about a command') +def _print_unknown_command_msg( + settings: BaseSettings, cmdname: str, inproject: bool +) -> None: + proj_only_cmds = _get_project_only_cmds(settings) + if cmdname in proj_only_cmds and not inproject: + cmd_list = ", ".join(sorted(proj_only_cmds)) + print( + f"The {cmdname} command is not available from this location.\n" + f"These commands are only available from within a project: {cmd_list}.\n" + ) + else: + print(f"Unknown command: {cmdname}\n") + + def _print_unknown_command( settings: BaseSettings, cmdname: str, inproject: bool ) -> None: _print_header(settings, inproject) - print(f"Unknown command: {cmdname}\n") + _print_unknown_command_msg(settings, cmdname, inproject) print('Use "scrapy" to see available commands') diff --git a/tests/test_commands.py b/tests/test_commands.py index 9d5720b98..1aae3222e 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -9,6 +9,7 @@ import re import subprocess import sys from contextlib import contextmanager +from io import StringIO from itertools import chain from pathlib import Path from shutil import copytree, rmtree @@ -16,12 +17,13 @@ from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from tempfile import TemporaryFile, mkdtemp from threading import Timer from typing import TYPE_CHECKING -from unittest import skipIf +from unittest import mock, skipIf from pytest import mark from twisted.trial import unittest import scrapy +from scrapy.cmdline import _print_unknown_command_msg from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings @@ -652,6 +654,24 @@ class MiscCommandsTest(CommandTest): def test_list(self): self.assertEqual(0, self.call("list")) + def test_command_not_found(self): + na_msg = """ +The list command is not available from this location. +These commands are only available from within a project: check, crawl, edit, list, parse. +""" + not_found_msg = """ +Unknown command: abc +""" + params = [ + ("list", 0, na_msg), + ("abc", 0, not_found_msg), + ("abc", 1, not_found_msg), + ] + for cmdname, inproject, message in params: + with mock.patch("sys.stdout", new=StringIO()) as out: + _print_unknown_command_msg(Settings(), cmdname, inproject) + self.assertEqual(out.getvalue().strip(), message.strip()) + class RunSpiderCommandTest(CommandTest): spider_filename = "myspider.py" From 98ba61256deceba7b04b938a97005258f4ef5c66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 14 Jan 2025 15:36:56 +0100 Subject: [PATCH 019/491] Deprecate BaseDupeFilter.log() and improve dupefilter docs (#4151) * Remove BaseDupeFilter.log() It is never called because request_seen() always returns False * Document the interface of DUPEFILTER_CLASS classes * Remove unnecessary BaseDupeFilter comments and add a short class description * Improve the documentation related to the DUPEFILTER_CLASS setting * Deprecate BaseDupeFilter.log * Update the docs * Fix the new code example * Remove typing to keep the example short Otherwise, it would have required yet another import line (from __future__ or typing). --------- Co-authored-by: Andrey Rakhmatullin --- docs/conf.py | 4 ++ docs/topics/settings.rst | 78 +++++++++++++++++++++++++++++++++++---- scrapy/dupefilters.py | 16 +++++++- tests/test_dupefilters.py | 19 +++++++++- 4 files changed, 108 insertions(+), 9 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index fd8165db3..8196b6934 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -258,6 +258,10 @@ coverage_ignore_pyobjects = [ # Base classes of downloader middlewares are implementation details that # are not meant for users. r"^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware", + # The interface methods of duplicate request filtering classes are already + # covered in the interface documentation part of the DUPEFILTER_CLASS + # setting documentation. + r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_settings|request_seen|open|close|log)$", # Private exception used by the command-line interface implementation. r"^scrapy\.exceptions\.UsageError", # Methods of BaseItemExporter subclasses are only documented in diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 76904a26e..06974f336 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -955,15 +955,79 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'`` The class used to detect and filter duplicate requests. -The default (``RFPDupeFilter``) filters based on the +The default, :class:`~scrapy.dupefilters.RFPDupeFilter`, filters based on the :setting:`REQUEST_FINGERPRINTER_CLASS` setting. -You can disable filtering of duplicate requests by setting -:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``. -Be very careful about this however, because you can get into crawling loops. -It's usually a better idea to set the ``dont_filter`` parameter to -``True`` on the specific :class:`~scrapy.Request` that should not be -filtered. +To change how duplicates are checked, you can point :setting:`DUPEFILTER_CLASS` +to a custom subclass of :class:`~scrapy.dupefilters.RFPDupeFilter` that +overrides its ``__init__`` method to use a :ref:`different request +fingerprinting class `. For example: + +.. code-block:: python + + from scrapy.dupefilters import RFPDupeFilter + from scrapy.utils.request import fingerprint + + + class CustomRequestFingerprinter: + def fingerprint(self, request): + return fingerprint(request, include_headers=["X-ID"]) + + + class CustomDupeFilter(RFPDupeFilter): + + def __init__(self, path=None, debug=False, *, fingerprinter=None): + super().__init__( + path=path, debug=debug, fingerprinter=CustomRequestFingerprinter() + ) + +To disable duplicate request filtering set :setting:`DUPEFILTER_CLASS` to +``'scrapy.dupefilters.BaseDupeFilter'``. Note that not filtering out duplicate +requests may cause crawling loops. It is usually better to set +the ``dont_filter`` parameter to ``True`` on the ``__init__`` method of a +specific :class:`~scrapy.Request` object that should not be filtered out. + +A class assigned to :setting:`DUPEFILTER_CLASS` must implement the following +interface:: + + class MyDupeFilter: + + @classmethod + def from_settings(cls, settings): + """Returns an instance of this duplicate request filtering class + based on the current crawl settings.""" + return cls() + + def request_seen(self, request): + """Returns ``True`` if *request* is a duplicate of another request + seen in a previous call to :meth:`request_seen`, or ``False`` + otherwise.""" + return False + + def open(self): + """Called before the spider opens. It may return a deferred.""" + pass + + def close(self, reason): + """Called before the spider closes. It may return a deferred.""" + pass + + def log(self, request, spider): + """Logs that a request has been filtered out. + + It is called right after a call to :meth:`request_seen` that + returns ``True``. + + If :meth:`request_seen` always returns ``False``, such as in the + case of :class:`~scrapy.dupefilters.BaseDupeFilter`, this method + may be omitted. + """ + pass + +.. autoclass:: scrapy.dupefilters.BaseDupeFilter + +.. autoclass:: scrapy.dupefilters.RFPDupeFilter + .. setting:: DUPEFILTER_DEBUG diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index caf69daf4..a3e2c5eb4 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -4,6 +4,7 @@ import logging import warnings from pathlib import Path from typing import TYPE_CHECKING +from warnings import warn from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.job import job_dir @@ -26,6 +27,9 @@ if TYPE_CHECKING: class BaseDupeFilter: + """Dummy duplicate request filtering class (:setting:`DUPEFILTER_CLASS`) + that does not filter out any request.""" + @classmethod def from_settings(cls, settings: BaseSettings) -> Self: warnings.warn( @@ -50,10 +54,19 @@ class BaseDupeFilter: def log(self, request: Request, spider: Spider) -> None: """Log that a request has been filtered""" + warn( + "Calling BaseDupeFilter.log() is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) class RFPDupeFilter(BaseDupeFilter): - """Request Fingerprint duplicates filter""" + """Duplicate request filtering class (:setting:`DUPEFILTER_CLASS`) that + filters out requests with the canonical + (:func:`w3lib.url.canonicalize_url`) :attr:`~scrapy.http.Request.url`, + :attr:`~scrapy.http.Request.method` and :attr:`~scrapy.http.Request.body`. + """ def __init__( self, @@ -117,6 +130,7 @@ class RFPDupeFilter(BaseDupeFilter): return False def request_fingerprint(self, request: Request) -> str: + """Returns a string that uniquely identifies the specified request.""" return self.fingerprinter.fingerprint(request).hex() def close(self, reason: str) -> None: diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 4fd648f48..703c23529 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -4,11 +4,13 @@ import sys import tempfile import unittest from pathlib import Path +from warnings import catch_warnings from testfixtures import LogCapture from scrapy.core.scheduler import Scheduler -from scrapy.dupefilters import RFPDupeFilter +from scrapy.dupefilters import BaseDupeFilter, RFPDupeFilter +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler @@ -252,3 +254,18 @@ class RFPDupeFilterTest(unittest.TestCase): ) dupefilter.close("finished") + + +class BaseDupeFilterTestCase(unittest.TestCase): + def test_log_deprecation(self): + dupefilter = _get_dupefilter( + settings={"DUPEFILTER_CLASS": BaseDupeFilter}, + ) + with catch_warnings(record=True) as warning_list: + dupefilter.log(None, None) + self.assertEqual(len(warning_list), 1) + self.assertEqual( + str(warning_list[0].message), + "Calling BaseDupeFilter.log() is deprecated.", + ) + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) From 1c1e83895c15dc491c6c133982cde22d778dcae6 Mon Sep 17 00:00:00 2001 From: anubhav Date: Tue, 14 Jan 2025 21:10:24 +0530 Subject: [PATCH 020/491] Fix _pop_command_name (#6606) --- scrapy/cmdline.py | 7 +++---- tests/test_commands.py | 28 +++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 065adccfb..b08fd3409 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -96,10 +96,9 @@ def _get_project_only_cmds(settings: BaseSettings) -> set[str]: def _pop_command_name(argv: list[str]) -> str | None: - for i, arg in enumerate(argv[1:]): - if not arg.startswith("-"): - del argv[i] - return arg + for i in range(1, len(argv)): + if not argv[i].startswith("-"): + return argv.pop(i) return None diff --git a/tests/test_commands.py b/tests/test_commands.py index 1aae3222e..50f093043 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -23,7 +23,7 @@ from pytest import mark from twisted.trial import unittest import scrapy -from scrapy.cmdline import _print_unknown_command_msg +from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings @@ -1163,3 +1163,29 @@ class HelpMessageTest(CommandTest): for command in self.commands: _, out, _ = self.proc(command, "-h") self.assertIn("Usage", out) + + +class PopCommandNameTest(unittest.TestCase): + def test_valid_command(self): + argv = ["scrapy", "crawl", "my_spider"] + command = _pop_command_name(argv) + self.assertEqual(command, "crawl") + self.assertEqual(argv, ["scrapy", "my_spider"]) + + def test_no_command(self): + argv = ["scrapy"] + command = _pop_command_name(argv) + self.assertIsNone(command) + self.assertEqual(argv, ["scrapy"]) + + def test_option_before_command(self): + argv = ["scrapy", "-h", "crawl"] + command = _pop_command_name(argv) + self.assertEqual(command, "crawl") + self.assertEqual(argv, ["scrapy", "-h"]) + + def test_option_after_command(self): + argv = ["scrapy", "crawl", "-h"] + command = _pop_command_name(argv) + self.assertEqual(command, "crawl") + self.assertEqual(argv, ["scrapy", "-h"]) From ca345a3b73904ffd6d2e8ffb17c45ebb69639d26 Mon Sep 17 00:00:00 2001 From: anubhav Date: Wed, 15 Jan 2025 15:38:18 +0530 Subject: [PATCH 021/491] Flexible severity of logging level when items are dropped (#6608) --- docs/topics/settings.rst | 32 +++++++++++++++ scrapy/exceptions.py | 6 +++ scrapy/logformatter.py | 6 ++- scrapy/settings/default_settings.py | 2 + tests/test_logformatter.py | 60 +++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 06974f336..8801434d8 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -418,6 +418,38 @@ This setting also affects :setting:`DOWNLOAD_DELAY` and :ref:`topics-autothrottle`: if :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, download delay is enforced per IP, not per domain. +.. setting:: DEFAULT_DROPITEM_LOG_LEVEL + +DEFAULT_DROPITEM_LOG_LEVEL +-------------------------- + +Default: ``"WARNING"`` + +Default :ref:`log level ` of messages about dropped items. + +When an item is dropped by raising :exc:`scrapy.exceptions.DropItem` from the +:func:`process_item` method of an :ref:`item pipeline `, +a message is logged, and by default its log level is the one configured in this +setting. + +You may specify this log level as an integer (e.g. ``20``), as a log level +constant (e.g. ``logging.INFO``) or as a string with the name of a log level +constant (e.g. ``"INFO"``). + +When writing an item pipeline, you can force a different log level by setting +:attr:`scrapy.exceptions.DropItem.log_level` in your +:exc:`scrapy.exceptions.DropItem` exception. For example: + +.. code-block:: python + + from scrapy.exceptions import DropItem + + + class MyPipeline: + def process_item(self, item, spider): + if not item.get("price"): + raise DropItem("Missing price data", log_level="INFO") + return item .. setting:: DEFAULT_ITEM_CLASS diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 96566ba86..f37f881a7 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -5,6 +5,8 @@ These exceptions are documented in docs/topics/exceptions.rst. Please don't add new exceptions here without documenting them there. """ +from __future__ import annotations + from typing import Any # Internal @@ -58,6 +60,10 @@ class StopDownload(Exception): class DropItem(Exception): """Drop item from the item pipeline""" + def __init__(self, message: str, log_level: str | None = None): + super().__init__(message) + self.log_level = log_level + class NotSupported(Exception): """Indicates a feature or method is not supported""" diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 76f9c7856..f10e91beb 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -120,8 +120,12 @@ class LogFormatter: spider: Spider, ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" + if (level := getattr(exception, "log_level", None)) is None: + level = spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] + if isinstance(level, str): + level = getattr(logging, level) return { - "level": logging.WARNING, + "level": level, "msg": DROPPEDMSG, "args": { "exception": exception, diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 0bbde118e..7ef365f68 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -49,6 +49,8 @@ CONCURRENT_REQUESTS_PER_IP = 0 COOKIES_ENABLED = True COOKIES_DEBUG = False +DEFAULT_DROPITEM_LOG_LEVEL = "WARNING" + DEFAULT_ITEM_CLASS = "scrapy.item.Item" DEFAULT_REQUEST_HEADERS = { diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 61a9f3f8d..e5d077858 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -1,5 +1,7 @@ +import logging import unittest +import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.python.failure import Failure @@ -26,6 +28,7 @@ class LogFormatterTestCase(unittest.TestCase): def setUp(self): self.formatter = LogFormatter() self.spider = Spider("default") + self.spider.crawler = get_crawler() def test_crawled_with_referer(self): req = Request("http://www.example.com") @@ -68,6 +71,62 @@ class LogFormatterTestCase(unittest.TestCase): assert all(isinstance(x, str) for x in lines) self.assertEqual(lines, ["Dropped: \u2018", "{}"]) + def test_dropitem_default_log_level(self): + item = {} + exception = DropItem("Test drop") + response = Response("http://www.example.com") + spider = Spider("foo") + spider.crawler = get_crawler(Spider) + + logkws = self.formatter.dropped(item, exception, response, spider) + self.assertEqual(logkws["level"], logging.WARNING) + + spider.crawler.settings.frozen = False + spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = logging.INFO + spider.crawler.settings.frozen = True + logkws = self.formatter.dropped(item, exception, response, spider) + self.assertEqual(logkws["level"], logging.INFO) + + spider.crawler.settings.frozen = False + spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = "INFO" + spider.crawler.settings.frozen = True + logkws = self.formatter.dropped(item, exception, response, spider) + self.assertEqual(logkws["level"], logging.INFO) + + spider.crawler.settings.frozen = False + spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = 10 + spider.crawler.settings.frozen = True + logkws = self.formatter.dropped(item, exception, response, spider) + self.assertEqual(logkws["level"], logging.DEBUG) + + spider.crawler.settings.frozen = False + spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = 0 + spider.crawler.settings.frozen = True + logkws = self.formatter.dropped(item, exception, response, spider) + self.assertEqual(logkws["level"], logging.NOTSET) + + unsupported_value = object() + spider.crawler.settings.frozen = False + spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = unsupported_value + spider.crawler.settings.frozen = True + logkws = self.formatter.dropped(item, exception, response, spider) + self.assertEqual(logkws["level"], unsupported_value) + + with pytest.raises(TypeError): + logging.log(logkws["level"], "message") + + def test_dropitem_custom_log_level(self): + item = {} + response = Response("http://www.example.com") + + exception = DropItem("Test drop", log_level="INFO") + logkws = self.formatter.dropped(item, exception, response, self.spider) + self.assertEqual(logkws["level"], logging.INFO) + + exception = DropItem("Test drop", log_level="ERROR") + logkws = self.formatter.dropped(item, exception, response, self.spider) + self.assertEqual(logkws["level"], logging.ERROR) + def test_item_error(self): # In practice, the complete traceback is shown by passing the # 'exc_info' argument to the logging function @@ -145,6 +204,7 @@ class LogformatterSubclassTest(LogFormatterTestCase): def setUp(self): self.formatter = LogFormatterSubclass() self.spider = Spider("default") + self.spider.crawler = get_crawler(Spider) def test_crawled_with_referer(self): req = Request("http://www.example.com") From d7168577b859d15a15c2bcbc6c4f607a62be7478 Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Sun, 19 Jan 2025 13:50:53 +0100 Subject: [PATCH 022/491] chore(docs): migrate to RTD template v3 notable change: Drop support for all versions of Internet Explorer. --- docs/conf.py | 7 ------- docs/requirements.txt | 6 +++--- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 8196b6934..a3475a323 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -117,13 +117,6 @@ html_theme = "sphinx_rtd_theme" # documentation. # html_theme_options = {} -# Add any paths that contain custom themes here, relative to this directory. -# Add path to the RTD explicitly to robustify builds (otherwise might -# fail in a clean Debian build env) -import sphinx_rtd_theme - -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - # The style sheet to use for HTML and HTML Help pages. A file of that name # must exist either in Sphinx' static/ path, or in one of the custom paths # given in html_static_path. diff --git a/docs/requirements.txt b/docs/requirements.txt index 5f683d34c..7ee897170 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ sphinx==6.2.1 -sphinx-hoverxref==1.3.0 -sphinx-notfound-page==1.0.0 -sphinx-rtd-theme==2.0.0 +sphinx-hoverxref==1.4.2 +sphinx-notfound-page==1.0.4 +sphinx-rtd-theme==3.0.2 From ee4f527f47111c18ddef0c2369c1dede7447a4e9 Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Sun, 19 Jan 2025 14:58:02 +0100 Subject: [PATCH 023/491] fix(docs): pillow domain is shut down permanently See https://github.com/python-pillow/Pillow/issues/8585 --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 924abb7a1..8230c3aef 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6862,7 +6862,7 @@ First release of Scrapy. .. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator .. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr .. _PEP 257: https://peps.python.org/pep-0257/ -.. _Pillow: https://python-pillow.org/ +.. _Pillow: https://github.com/python-pillow/Pillow .. _pyOpenSSL: https://www.pyopenssl.org/en/stable/ .. _queuelib: https://github.com/scrapy/queuelib .. _registered with IANA: https://www.iana.org/assignments/media-types/media-types.xhtml From e0c828b7f665d8c82e17787996634bc072e416ae Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Mon, 20 Jan 2025 12:18:30 +0100 Subject: [PATCH 024/491] chore(docs): refactor config (#6623) --- .readthedocs.yml | 2 +- docs/Makefile | 104 +++-------------- docs/_ext/scrapyfixautodoc.py | 18 +++ docs/conf.py | 211 ++++++---------------------------- 4 files changed, 65 insertions(+), 270 deletions(-) create mode 100644 docs/_ext/scrapyfixautodoc.py diff --git a/.readthedocs.yml b/.readthedocs.yml index 5ec6eafbb..23e4cabea 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,7 +5,7 @@ sphinx: fail_on_warning: true build: - os: ubuntu-20.04 + os: ubuntu-24.04 tools: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python diff --git a/docs/Makefile b/docs/Makefile index 48401bac8..ed8809902 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,96 +1,20 @@ -# -# Makefile for Scrapy documentation [based on Python documentation Makefile] -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Minimal makefile for Sphinx documentation # -# You can set these variables from the command line. -PYTHON = python -SPHINXOPTS = -PAPER = -SOURCES = -SHELL = /usr/bin/env bash - -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \ - -D latex_elements.papersize=$(PAPER) \ - $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES) - -.PHONY: help update build html htmlhelp clean +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = build +# Put it first so that "make" without argument is like "make help". help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " text to make plain text files" - @echo " changes to make an overview over all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " watch build HTML docs, open in browser and watch for changes" + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -build-dirs: - mkdir -p build/$(BUILDER) build/doctrees +.PHONY: help Makefile -build: build-dirs - sphinx-build $(ALLSPHINXOPTS) - @echo - -build-ignore-errors: build-dirs - -sphinx-build $(ALLSPHINXOPTS) - @echo - - -html: BUILDER = html -html: build - @echo "Build finished. The HTML pages are in build/html." - -htmlhelp: BUILDER = htmlhelp -htmlhelp: build - @echo "Build finished; now you can run HTML Help Workshop with the" \ - "build/htmlhelp/pydoc.hhp project file." - -latex: BUILDER = latex -latex: build - @echo "Build finished; the LaTeX files are in build/latex." - @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ - "run these through (pdf)latex." - -text: BUILDER = text -text: build - @echo "Build finished; the text files are in build/text." - -changes: BUILDER = changes -changes: build - @echo "The overview file is in build/changes." - -linkcheck: BUILDER = linkcheck -linkcheck: build - @echo "Link check complete; look for any errors in the above output " \ - "or in build/$(BUILDER)/output.txt" - -linkfix: BUILDER = linkcheck -linkfix: build-ignore-errors - $(PYTHON) utils/linkfix.py - @echo "Fixing redirecting links in docs has finished; check all " \ - "replacements before committing them" - -doctest: BUILDER = doctest -doctest: build - @echo "Testing of doctests in the sources finished, look at the " \ - "results in build/doctest/output.txt" - -pydoc-topics: BUILDER = pydoc-topics -pydoc-topics: build - @echo "Building finished; now copy build/pydoc-topics/pydoc_topics.py " \ - "into the Lib/ directory" - -coverage: BUILDER = coverage -coverage: build - -htmlview: html - $(PYTHON) -c "import webbrowser; from pathlib import Path; \ - webbrowser.open(Path('build/html/index.html').resolve().as_uri())" - -clean: - -rm -rf build/* - -watch: htmlview - watchmedo shell-command -p '*.rst' -c 'make html' -R -D +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_ext/scrapyfixautodoc.py b/docs/_ext/scrapyfixautodoc.py new file mode 100644 index 000000000..d7a3fb514 --- /dev/null +++ b/docs/_ext/scrapyfixautodoc.py @@ -0,0 +1,18 @@ +""" +Must be included after 'sphinx.ext.autodoc'. Fixes unwanted 'alias of' behavior. +https://github.com/sphinx-doc/sphinx/issues/4422 +""" + +# pylint: disable=import-error +from sphinx.application import Sphinx + + +def maybe_skip_member(app: Sphinx, what, name: str, obj, skip: bool, options) -> bool: + if not skip: + # autodocs was generating a text "alias of" for the following members + return name in {"default_item_class", "default_selector_class"} + return skip + + +def setup(app: Sphinx) -> None: + app.connect("autodoc-skip-member", maybe_skip_member) diff --git a/docs/conf.py b/docs/conf.py index a3475a323..be5e07195 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,17 +1,12 @@ -# Scrapy documentation build configuration file, created by -# sphinx-quickstart on Mon Nov 24 12:02:52 2008. +# Configuration file for the Sphinx documentation builder. # -# This file is execfile()d with the current directory set to its containing dir. -# -# The contents of this file are pickled, so don't put values in the namespace -# that aren't pickleable (module imports are okay, they're removed automatically). -# -# All configuration values have a default; values that are commented out -# serve to show the default. +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html # pylint: disable=import-error import os import sys +from collections.abc import Sequence from pathlib import Path # If your extensions are in another directory, add it here. If the directory @@ -20,36 +15,30 @@ sys.path.append(str(Path(__file__).parent / "_ext")) sys.path.insert(0, str(Path(__file__).parent.parent)) -# General configuration -# --------------------- +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "Scrapy" +project_copyright = "Scrapy developers" +author = "Scrapy developers" + + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ "hoverxref.extension", "notfound.extension", "scrapydocs", "sphinx.ext.autodoc", + "scrapyfixautodoc", # Must be after "sphinx.ext.autodoc" "sphinx.ext.coverage", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", ] -# Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8' - -# The master toctree document. -master_doc = "index" - -# General information about the project. -project = "Scrapy" -copyright = "Scrapy developers" +exclude_patterns = ["build", "Thumbs.db", ".DS_Store"] # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -65,118 +54,17 @@ except ImportError: version = "" release = "" -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -language = "en" - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of documents that shouldn't be included in the build. -# unused_docs = [] - -exclude_patterns = ["build"] - -# List of directories, relative to source directory, that shouldn't be searched -# for source files. -exclude_trees = [".build"] - -# The reST default role (used for this markup: `text`) to use for all documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# List of Sphinx warnings that will not be raised suppress_warnings = ["epub.unknown_project_files"] -# Options for HTML output -# ----------------------- +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. html_theme = "sphinx_rtd_theme" - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# html_theme_options = {} - -# The style sheet to use for HTML and HTML Help pages. A file of that name -# must exist either in Sphinx' static/ path, or in one of the custom paths -# given in html_static_path. -# html_style = 'scrapydoc.css' - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -# html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. html_last_updated_fmt = "%b %d, %Y" -# Custom sidebar templates, maps document names to template names. -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_use_modindex = True - -# If false, no index is generated. -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, the reST sources are included in the HTML build as _sources/. -html_copy_source = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = '' - -# Output file base name for HTML help builder. -htmlhelp_basename = "Scrapydoc" - html_css_files = [ "custom.css", ] @@ -184,14 +72,8 @@ html_css_files = [ # Set canonical URL from the Read the Docs Domain html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "") -# Options for LaTeX output -# ------------------------ - -# The paper size ('letter' or 'a4'). -# latex_paper_size = 'letter' - -# The font size ('10pt', '11pt' or '12pt'). -# latex_font_size = '10pt' +# -- Options for LaTeX output ------------------------------------------------ +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-latex-output # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). @@ -199,39 +81,22 @@ latex_documents = [ ("index", "Scrapy.tex", "Scrapy Documentation", "Scrapy developers", "manual"), ] -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# latex_logo = None -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False +# -- Options for the linkcheck builder --------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-the-linkcheck-builder -# Additional stuff for the LaTeX preamble. -# latex_preamble = '' - -# Documents to append as an appendix to all manuals. -# latex_appendices = [] - -# If false, no module index is generated. -# latex_use_modindex = True - - -# Options for the linkcheck builder -# --------------------------------- - -# A list of regular expressions that match URIs that should not be checked when -# doing a linkcheck build. linkcheck_ignore = [ r"http://localhost:\d+", "http://hg.scrapy.org", - "http://directory.google.com/", + r"https://github.com/scrapy/scrapy/commit/\w+", r"https://github.com/scrapy/scrapy/issues/\d+", ] +linkcheck_anchors_ignore_for_url = ["https://github.com/pyca/cryptography/issues/2692"] + +# -- Options for the Coverage extension -------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/coverage.html#configuration -# Options for the Coverage extension -# ---------------------------------- coverage_ignore_pyobjects = [ # Contract’s add_pre_hook and add_post_hook are not documented because # they should be transparent to contract developers, for whom pre_hook and @@ -272,8 +137,8 @@ coverage_ignore_pyobjects = [ ] -# Options for the InterSphinx extension -# ------------------------------------- +# -- Options for the InterSphinx extension ----------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration intersphinx_mapping = { "attrs": ("https://www.attrs.org/en/stable/", None), @@ -290,11 +155,11 @@ intersphinx_mapping = { "twistedapi": ("https://docs.twisted.org/en/stable/api/", None), "w3lib": ("https://w3lib.readthedocs.io/en/latest", None), } -intersphinx_disabled_reftypes = [] +intersphinx_disabled_reftypes: Sequence[str] = [] -# Options for sphinx-hoverxref options -# ------------------------------------ +# -- Options for sphinx-hoverxref extension ---------------------------------- +# https://sphinx-hoverxref.readthedocs.io/en/latest/configuration.html hoverxref_auto_ref = True hoverxref_role_types = { @@ -309,15 +174,3 @@ hoverxref_role_types = { "signal": "tooltip", } hoverxref_roles = ["command", "reqmeta", "setting", "signal"] - - -def setup(app): - app.connect("autodoc-skip-member", maybe_skip_member) - - -def maybe_skip_member(app, what, name, obj, skip, options): - if not skip: - # autodocs was generating a text "alias of" for the following members - # https://github.com/sphinx-doc/sphinx/issues/4422 - return name in {"default_item_class", "default_selector_class"} - return skip From 14219b1fca86f2229d2b69455bdbb4eb952cf504 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 22 Jan 2025 07:16:22 -0300 Subject: [PATCH 025/491] fix: test_s3_export fails with boto3 >= 1.36.0 --- tests/test_feedexport.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 0f149f172..81d05e2a3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2622,18 +2622,24 @@ class BatchDeliveriesTest(FeedExportTestBase): stubs = [] def open(self, *args, **kwargs): + from botocore import __version__ as botocore_version from botocore.stub import ANY, Stubber + from packaging.version import Version + + expected_params = { + "Body": ANY, + "Bucket": bucket, + "Key": ANY, + } + if Version(botocore_version) >= Version("1.36.0"): + expected_params["ChecksumAlgorithm"] = ANY stub = Stubber(self.s3_client) stub.activate() CustomS3FeedStorage.stubs.append(stub) stub.add_response( "put_object", - expected_params={ - "Body": ANY, - "Bucket": bucket, - "Key": ANY, - }, + expected_params=expected_params, service_response={}, ) return super().open(*args, **kwargs) From 9bc0029d27d8ed719e2cc2e9077a81450b995b96 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <95530227+guillermo-bondonno@users.noreply.github.com> Date: Wed, 22 Jan 2025 08:07:44 -0300 Subject: [PATCH 026/491] Allow updating pre-crawler settings from add-ons (#6568) --- docs/topics/addons.rst | 11 +- docs/topics/practices.rst | 22 +-- docs/topics/settings.rst | 151 +++++++++++++----- scrapy/addons.py | 22 ++- scrapy/crawler.py | 1 + tests/test_spiderloader/__init__.py | 19 +++ .../spiders_from_addons/__init__.py | 0 .../spiders_from_addons/spider0.py | 6 + 8 files changed, 171 insertions(+), 61 deletions(-) create mode 100644 tests/test_spiderloader/spiders_from_addons/__init__.py create mode 100644 tests/test_spiderloader/spiders_from_addons/spider0.py diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 14b4aa8ba..46cf1edbd 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -32,7 +32,7 @@ This is an example where two add-ons are enabled in a project's Writing your own add-ons ======================== -Add-ons are Python classes that include the following method: +Add-ons are Python classes that include one or both of the following methods: .. method:: update_settings(settings) @@ -45,6 +45,15 @@ Add-ons are Python classes that include the following method: :param settings: The settings object storing Scrapy/component configuration :type settings: :class:`~scrapy.settings.Settings` +.. classmethod:: update_pre_crawler_settings(cls, settings) + + Use this class method instead of the :meth:`update_settings` method to + update :ref:`pre-crawler settings ` whose value is + used before the :class:`~scrapy.crawler.Crawler` object is created. + + :param settings: The settings object storing Scrapy/component configuration + :type settings: :class:`~scrapy.settings.BaseSettings` + They can also have the following method: .. classmethod:: from_crawler(cls, crawler) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 1500011e7..5f6798601 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -246,24 +246,10 @@ Same example but running the spiders sequentially by chaining the deferreds: crawl() reactor.run() # the script will block here until the last crawl call is finished -Different spiders can set different values for the same setting, but when they -run in the same process it may be impossible, by design or because of some -limitations, to use these different values. What happens in practice is -different for different settings: - -* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value - (:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the - default one) cannot be read from the per-spider settings. These are applied - when the :class:`~scrapy.crawler.CrawlerRunner` or - :class:`~scrapy.crawler.CrawlerProcess` object is created. -* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first - available value is used, and if a spider requests a different reactor an - exception will be raised. These are applied when the reactor is installed. -* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the - ones used by the resolver (:setting:`DNSCACHE_ENABLED`, - :setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy) - the first available value is used. These are applied when the reactor is - started. +.. note:: When running multiple spiders in the same process, :ref:`reactor + settings ` should not have a different value per spider. + Also, :ref:`pre-crawler settings ` cannot be defined + per spider. .. seealso:: :ref:`run-from-script`. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8801434d8..a53e0806d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -33,42 +33,48 @@ Python :ref:`import search path `. Populating the settings ======================= -Settings can be populated using different mechanisms, each of which having a -different precedence. Here is the list of them in decreasing order of -precedence: +Settings can be populated using different mechanisms, each of which has a +different precedence: - 1. Command line options (most precedence) - 2. Settings per-spider - 3. Project settings module - 4. Settings set by add-ons - 5. Default settings per-command - 6. Default global settings (less precedence) + 1. :ref:`Command-line settings ` (highest precedence) + 2. :ref:`Spider settings ` + 3. :ref:`Project settings ` + 4. :ref:`Add-on settings ` + 5. :ref:`Command-specific default settings ` + 6. :ref:`Global default settings ` (lowest precedence) -The population of these settings sources is taken care of internally, but a -manual handling is possible using API calls. See the -:ref:`topics-api-settings` topic for reference. +.. _cli-settings: -These mechanisms are described in more detail below. +1. Command-line settings +------------------------ -1. Command line options ------------------------ +Settings set in the command line have the highest precedence, overriding any +other settings. -Arguments provided by the command line are the ones that take most precedence, -overriding any other options. You can explicitly override one (or more) -settings using the ``-s`` (or ``--set``) command line option. +You can explicitly override one or more settings using the ``-s`` (or +``--set``) command-line option. .. highlight:: sh Example:: - scrapy crawl myspider -s LOG_FILE=scrapy.log + scrapy crawl myspider -s LOG_LEVEL=INFO -s LOG_FILE=scrapy.log -2. Settings per-spider ----------------------- +.. _spider-settings: -Spiders (See the :ref:`topics-spiders` chapter for reference) can define their -own settings that will take precedence and override the project ones. One way -to do so is by setting their :attr:`~scrapy.Spider.custom_settings` attribute: +2. Spider settings +------------------ + +:ref:`Spiders ` can define their own settings that will take +precedence and override the project ones. + +.. note:: :ref:`Pre-crawler settings ` cannot be defined + per spider, and :ref:`reactor settings ` should not have + a different value per spider when :ref:`running multiple spiders in the + same process `. + +One way to do so is by setting their :attr:`~scrapy.Spider.custom_settings` +attribute: .. code-block:: python @@ -83,7 +89,7 @@ to do so is by setting their :attr:`~scrapy.Spider.custom_settings` attribute: } It's often better to implement :meth:`~scrapy.Spider.update_settings` instead, -and settings set there should use the "spider" priority explicitly: +and settings set there should use the ``"spider"`` priority explicitly: .. code-block:: python @@ -121,27 +127,37 @@ arguments ` or other logic: ) return spider -3. Project settings module --------------------------- +.. _project-settings: -The project settings module is the standard configuration file for your Scrapy -project, it's where most of your custom settings will be populated. For a -standard Scrapy project, this means you'll be adding or changing the settings -in the ``settings.py`` file created for your project. +3. Project settings +------------------- -4. Settings set by add-ons --------------------------- +Scrapy projects include a settings module, usually a file called +``settings.py``, where you should populate most settings that apply to all your +spiders. + +.. seealso:: :ref:`topics-settings-module-envvar` + +.. _addon-settings: + +4. Add-on settings +------------------ :ref:`Add-ons ` can modify settings. They should do this with -this priority, though this is not enforced. +``"addon"`` priority where possible. -5. Default settings per-command -------------------------------- +.. _cmd-default-settings: -Each :doc:`Scrapy tool ` command can have its own default -settings, which override the global default settings. Those custom command -settings are specified in the ``default_settings`` attribute of the command -class. +5. Command-specific default settings +------------------------------------ + +Each :ref:`Scrapy command ` can have its own default settings, +which override the :ref:`global default settings `. + +Those command-specific default settings are specified in the +``default_settings`` attribute of each command class. + +.. _default-settings: 6. Default global settings -------------------------- @@ -234,6 +250,61 @@ example, proper setting names for a fictional robots.txt extension would be ``ROBOTSTXT_ENABLED``, ``ROBOTSTXT_OBEY``, ``ROBOTSTXT_CACHEDIR``, etc. +Special settings +================ + +The following settings work slightly differently than all other settings. + +.. _pre-crawler-settings: + +Pre-crawler settings +-------------------- + +**Pre-crawler settings** are settings used before the +:class:`~scrapy.crawler.Crawler` object is created. + +These settings cannot be :ref:`set from a spider `. + +These settings are :setting:`SPIDER_LOADER_CLASS` and settings used by the +corresponding :ref:`component `, e.g. +:setting:`SPIDER_MODULES` and :setting:`SPIDER_LOADER_WARN_ONLY` for the +default component. + + +.. _reactor-settings: + +Reactor settings +---------------- + +**Reactor settings** are settings tied to the :doc:`Twisted reactor +`. + +These settings can be defined from a spider. However, because only 1 reactor +can be used per process, these settings cannot use a different value per spider +when :ref:`running multiple spiders in the same process +`. + +In general, if different spiders define different values, the first defined +value is used. However, if two spiders request a different reactor, an +exception is raised. + +These settings are: + +- :setting:`ASYNCIO_EVENT_LOOP` + +- :setting:`DNS_RESOLVER` and settings used by the corresponding + component, e.g. :setting:`DNSCACHE_ENABLED`, :setting:`DNSCACHE_SIZE` + and :setting:`DNS_TIMEOUT` for the default one. + +- :setting:`REACTOR_THREADPOOL_MAXSIZE` + +- :setting:`TWISTED_REACTOR` + +:setting:`ASYNCIO_EVENT_LOOP` and :setting:`TWISTED_REACTOR` are used upon +installing the reactor. The rest of the settings are applied when starting +the reactor. + + .. _topics-settings-ref: Built-in settings reference diff --git a/scrapy/addons.py b/scrapy/addons.py index 7a1da3afc..1024d2dcd 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -9,7 +9,7 @@ from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler - from scrapy.settings import Settings + from scrapy.settings import BaseSettings, Settings logger = logging.getLogger(__name__) @@ -36,7 +36,8 @@ class AddonManager: try: addoncls = load_object(clspath) addon = build_from_crawler(addoncls, self.crawler) - addon.update_settings(settings) + if hasattr(addon, "update_settings"): + addon.update_settings(settings) self.addons.append(addon) except NotConfigured as e: if e.args: @@ -52,3 +53,20 @@ class AddonManager: }, extra={"crawler": self.crawler}, ) + + @classmethod + def load_pre_crawler_settings(cls, settings: BaseSettings): + """Update early settings that do not require a crawler instance, such as SPIDER_MODULES. + + Similar to the load_settings method, this loads each add-on configured in the + ``ADDONS`` setting and calls their 'update_pre_crawler_settings' class method if present. + This method doesn't have access to the crawler instance or the addons list. + + :param settings: The :class:`~scrapy.settings.BaseSettings` object from \ + which to read the early add-on configuration + :type settings: :class:`~scrapy.settings.Settings` + """ + for clspath in build_component_list(settings["ADDONS"]): + addoncls = load_object(clspath) + if hasattr(addoncls, "update_pre_crawler_settings"): + addoncls.update_pre_crawler_settings(settings) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index f6dbe053a..1aa68cb00 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -292,6 +292,7 @@ class CrawlerRunner: def __init__(self, settings: dict[str, Any] | Settings | None = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) + AddonManager.load_pre_crawler_settings(settings) self.settings: Settings = settings self.spider_loader: SpiderLoader = self._get_spider_loader(settings) self._crawlers: set[Crawler] = set() diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index d5aac34eb..705f722b3 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -97,6 +97,25 @@ class SpiderLoaderTest(unittest.TestCase): self.spider_loader = SpiderLoader.from_settings(settings) assert len(self.spider_loader._spiders) == 0 + def test_load_spider_module_from_addons(self): + module = "tests.test_spiderloader.spiders_from_addons.spider0" + + class SpiderModuleAddon: + @classmethod + def update_pre_crawler_settings(cls, settings): + settings.set( + "SPIDER_MODULES", + [module], + "project", + ) + + runner = CrawlerRunner({"ADDONS": {SpiderModuleAddon: 1}}) + + crawler = runner.create_crawler("spider_from_addon") + self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) + self.assertEqual(crawler.spidercls.name, "spider_from_addon") + self.assertTrue(len(crawler.settings["SPIDER_MODULES"]) == 1) + def test_crawler_runner_loading(self): module = "tests.test_spiderloader.test_spiders.spider1" runner = CrawlerRunner( diff --git a/tests/test_spiderloader/spiders_from_addons/__init__.py b/tests/test_spiderloader/spiders_from_addons/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_spiderloader/spiders_from_addons/spider0.py b/tests/test_spiderloader/spiders_from_addons/spider0.py new file mode 100644 index 000000000..45c3f64a7 --- /dev/null +++ b/tests/test_spiderloader/spiders_from_addons/spider0.py @@ -0,0 +1,6 @@ +from scrapy.spiders import Spider + + +class SpiderFromAddon(Spider): + name = "spider_from_addon" + allowed_domains = ["scrapy1.org", "scrapy3.org"] From 7e61ff352439d3e5c85785fc26b5503e4fed67b8 Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Wed, 22 Jan 2025 18:09:42 +0100 Subject: [PATCH 027/491] Upgrade Sphinx (#6624) --- docs/_ext/scrapydocs.py | 130 +++++++++++++++++++++------------------- docs/requirements.txt | 2 +- 2 files changed, 71 insertions(+), 61 deletions(-) diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 9b63f39f6..4ceb003c7 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,63 +1,67 @@ # pylint: disable=import-error +from collections.abc import Sequence from operator import itemgetter +from typing import Any, TypedDict from docutils import nodes +from docutils.nodes import Element, General, Node, document from docutils.parsers.rst import Directive -from docutils.parsers.rst.roles import set_classes +from sphinx.application import Sphinx from sphinx.util.nodes import make_refnode -class settingslist_node(nodes.General, nodes.Element): +class SettingData(TypedDict): + docname: str + setting_name: str + refid: str + + +class SettingslistNode(General, Element): pass class SettingsListDirective(Directive): - def run(self): - return [settingslist_node("")] + def run(self) -> Sequence[Node]: + return [SettingslistNode()] -def is_setting_index(node): - if node.tagname == "index" and node["entries"]: +def is_setting_index(node: Node) -> bool: + if node.tagname == "index" and node["entries"]: # type: ignore[index,attr-defined] # index entries for setting directives look like: # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')] - entry_type, info, refid = node["entries"][0][:3] + entry_type, info, refid = node["entries"][0][:3] # type: ignore[index] return entry_type == "pair" and info.endswith("; setting") return False -def get_setting_target(node): - # target nodes are placed next to the node in the doc tree - return node.parent[node.parent.index(node) + 1] - - -def get_setting_name_and_refid(node): +def get_setting_name_and_refid(node: Node) -> tuple[str, str]: """Extract setting name from directive index node""" - entry_type, info, refid = node["entries"][0][:3] + entry_type, info, refid = node["entries"][0][:3] # type: ignore[index] return info.replace("; setting", ""), refid -def collect_scrapy_settings_refs(app, doctree): +def collect_scrapy_settings_refs(app: Sphinx, doctree: document) -> None: env = app.builder.env if not hasattr(env, "scrapy_all_settings"): - env.scrapy_all_settings = [] - - for node in doctree.traverse(is_setting_index): - targetnode = get_setting_target(node) - assert isinstance(targetnode, nodes.target), "Next node is not a target" + emptyList: list[SettingData] = [] + env.scrapy_all_settings = emptyList # type: ignore[attr-defined] + for node in doctree.findall(is_setting_index): setting_name, refid = get_setting_name_and_refid(node) - env.scrapy_all_settings.append( - { - "docname": env.docname, - "setting_name": setting_name, - "refid": refid, - } + env.scrapy_all_settings.append( # type: ignore[attr-defined] + SettingData( + docname=env.docname, + setting_name=setting_name, + refid=refid, + ) ) -def make_setting_element(setting_data, app, fromdocname): +def make_setting_element( + setting_data: SettingData, app: Sphinx, fromdocname: str +) -> Any: refnode = make_refnode( app.builder, fromdocname, @@ -73,22 +77,56 @@ def make_setting_element(setting_data, app, fromdocname): return item -def replace_settingslist_nodes(app, doctree, fromdocname): +def replace_settingslist_nodes( + app: Sphinx, doctree: document, fromdocname: str +) -> None: env = app.builder.env - for node in doctree.traverse(settingslist_node): + for node in doctree.findall(SettingslistNode): settings_list = nodes.bullet_list() settings_list.extend( [ make_setting_element(d, app, fromdocname) - for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) + for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) # type: ignore[attr-defined] if fromdocname != d["docname"] ] ) node.replace_self(settings_list) -def setup(app): +def source_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "https://github.com/scrapy/scrapy/blob/master/" + text + node = nodes.reference(rawtext, text, refuri=ref, **options) + return [node], [] + + +def issue_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "https://github.com/scrapy/scrapy/issues/" + text + node = nodes.reference(rawtext, "issue " + text, refuri=ref) + return [node], [] + + +def commit_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "https://github.com/scrapy/scrapy/commit/" + text + node = nodes.reference(rawtext, "commit " + text, refuri=ref) + return [node], [] + + +def rev_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "http://hg.scrapy.org/scrapy/changeset/" + text + node = nodes.reference(rawtext, "r" + text, refuri=ref) + return [node], [] + + +def setup(app: Sphinx) -> None: app.add_crossref_type( directivename="setting", rolename="setting", @@ -114,36 +152,8 @@ def setup(app): app.add_role("issue", issue_role) app.add_role("rev", rev_role) - app.add_node(settingslist_node) + app.add_node(SettingslistNode) app.add_directive("settingslist", SettingsListDirective) app.connect("doctree-read", collect_scrapy_settings_refs) app.connect("doctree-resolved", replace_settingslist_nodes) - - -def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "https://github.com/scrapy/scrapy/blob/master/" + text - set_classes(options) - node = nodes.reference(rawtext, text, refuri=ref, **options) - return [node], [] - - -def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "https://github.com/scrapy/scrapy/issues/" + text - set_classes(options) - node = nodes.reference(rawtext, "issue " + text, refuri=ref, **options) - return [node], [] - - -def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "https://github.com/scrapy/scrapy/commit/" + text - set_classes(options) - node = nodes.reference(rawtext, "commit " + text, refuri=ref, **options) - return [node], [] - - -def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "http://hg.scrapy.org/scrapy/changeset/" + text - set_classes(options) - node = nodes.reference(rawtext, "r" + text, refuri=ref, **options) - return [node], [] diff --git a/docs/requirements.txt b/docs/requirements.txt index 7ee897170..e2abe76d9 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -sphinx==6.2.1 +sphinx==8.1.3 sphinx-hoverxref==1.4.2 sphinx-notfound-page==1.0.4 sphinx-rtd-theme==3.0.2 From d4b152bbf64591317d2d7ec9dfed0746e7bdb8e1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Jan 2025 12:22:18 +0400 Subject: [PATCH 028/491] Drop PyPy 3.9, add a pypy3-extra-deps CI job. (#6613) --- .github/workflows/tests-ubuntu.yml | 8 ++++---- tox.ini | 28 +++++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 89d1e70ac..6c7842217 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -30,9 +30,6 @@ jobs: - python-version: "3.13" env: TOXENV: asyncio - - python-version: pypy3.9 - env: - TOXENV: pypy3 - python-version: pypy3.10 env: TOXENV: pypy3 @@ -44,7 +41,7 @@ jobs: - python-version: 3.9.19 env: TOXENV: asyncio-pinned - - python-version: pypy3.9 + - python-version: pypy3.10 env: TOXENV: pypy3-pinned - python-version: 3.9.19 @@ -57,6 +54,9 @@ jobs: - python-version: "3.13" env: TOXENV: extra-deps + - python-version: pypy3.10 + env: + TOXENV: pypy3-extra-deps - python-version: "3.13" env: TOXENV: botocore diff --git a/tox.ini b/tox.ini index cf5e19a61..0f91db19d 100644 --- a/tox.ini +++ b/tox.ini @@ -118,6 +118,7 @@ setenv = install_command = python -I -m pip install {opts} {packages} commands = + ; tests for docs fail with parsel < 1.8.0 pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} [testenv:pinned] @@ -191,14 +192,35 @@ setenv = [testenv:pypy3] basepython = pypy3 commands = + ; not enabling coverage as it significantly increases the run time pytest {posargs:--durations=10 docs scrapy tests} -[testenv:pypy3-pinned] -basepython = pypy3.9 +[testenv:pypy3-extra-deps] +basepython = pypy3 deps = - {[pinned]deps} + {[testenv:extra-deps]deps} +commands = {[testenv:pypy3]commands} + +[testenv:pypy3-pinned] +basepython = pypy3.10 +deps = + cryptography==41.0.5 + cssselect==0.9.1 + h2==3.1 + itemadapter==0.1.0 + parsel==1.5.0 + Protego==0.1.15 + pyOpenSSL==23.3.0 + queuelib==1.4.2 + service_identity==18.1.0 + Twisted[http2]==21.7.0 + w3lib==1.17.0 + zope.interface==5.1.0 + lxml==4.6.0 + {[test-requirements]deps} PyPyDispatcher==2.1.0 commands = + ; disabling both coverage and docs tests pytest {posargs:--durations=10 scrapy tests} install_command = {[pinned]install_command} setenv = From c03fb2abb8c354c56c4e8363fc602d49f956c280 Mon Sep 17 00:00:00 2001 From: anubhav Date: Thu, 23 Jan 2025 21:36:45 +0530 Subject: [PATCH 029/491] fix: added feed_options as a keyword argument to GCSFeedStorage. (#6628) --- scrapy/extensions/feedexport.py | 25 +++++++++++++++++++++++-- tests/test_feedexport.py | 15 +++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c6e2aa0dd..8bcd4e40d 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -276,7 +276,14 @@ class S3FeedStorage(BlockingFeedStorage): class GCSFeedStorage(BlockingFeedStorage): - def __init__(self, uri: str, project_id: str | None, acl: str | None): + def __init__( + self, + uri: str, + project_id: str | None, + acl: str | None, + *, + feed_options: dict[str, Any] | None = None, + ): self.project_id: str | None = project_id self.acl: str | None = acl u = urlparse(uri) @@ -284,12 +291,26 @@ class GCSFeedStorage(BlockingFeedStorage): self.bucket_name: str = u.hostname self.blob_name: str = u.path[1:] # remove first "/" + if feed_options and feed_options.get("overwrite", True) is False: + logger.warning( + "GCS does not support appending to files. To " + "suppress this warning, remove the overwrite " + "option from your FEEDS setting or set it to True." + ) + @classmethod - def from_crawler(cls, crawler: Crawler, uri: str) -> Self: + def from_crawler( + cls, + crawler: Crawler, + uri: str, + *, + feed_options: dict[str, Any] | None = None, + ) -> Self: return cls( uri, crawler.settings["GCS_PROJECT_ID"], crawler.settings["FEED_STORAGE_GCS_ACL"] or None, + feed_options=feed_options, ) def _store_in_thread(self, file: IO[bytes]) -> None: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 81d05e2a3..7edffa1f6 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -523,6 +523,21 @@ class GCSFeedStorageTest(unittest.TestCase): bucket_mock.blob.assert_called_once_with("export.csv") blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) + def test_overwrite_default(self): + with LogCapture() as log: + GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl") + self.assertNotIn("GCS does not support appending to files", str(log)) + + def test_overwrite_false(self): + with LogCapture() as log: + GCSFeedStorage( + "gs://mybucket/export.csv", + "myproject-123", + "custom-acl", + feed_options={"overwrite": False}, + ) + self.assertIn("GCS does not support appending to files", str(log)) + class StdoutFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks From cec0aeca58730b592bec50299414d4bf30fc9ec0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 27 Jan 2025 14:07:09 +0400 Subject: [PATCH 030/491] Bump ruff, switch from black to ruff-format (#6631) --- .pre-commit-config.yaml | 7 +- scrapy/commands/genspider.py | 6 +- scrapy/core/downloader/handlers/__init__.py | 13 ++-- scrapy/core/http2/stream.py | 8 +-- scrapy/downloadermiddlewares/cookies.py | 3 +- scrapy/downloadermiddlewares/offsite.py | 2 +- scrapy/downloadermiddlewares/robotstxt.py | 4 +- scrapy/downloadermiddlewares/stats.py | 2 +- scrapy/extensions/telnet.py | 4 +- scrapy/http/headers.py | 3 +- scrapy/http/request/form.py | 1 - scrapy/http/response/__init__.py | 3 +- scrapy/linkextractors/lxmlhtml.py | 1 - scrapy/pipelines/files.py | 4 +- scrapy/selector/unified.py | 3 +- scrapy/settings/__init__.py | 2 +- scrapy/settings/default_settings.py | 2 +- scrapy/spidermiddlewares/offsite.py | 2 +- scrapy/squeues.py | 16 +++-- scrapy/utils/curl.py | 5 +- scrapy/utils/defer.py | 2 +- scrapy/utils/iterators.py | 2 +- scrapy/utils/log.py | 1 - scrapy/utils/python.py | 7 +- tests/spiders.py | 6 +- tests/test_contracts.py | 3 +- tests/test_downloadermiddleware_httpproxy.py | 6 +- tests/test_downloadermiddleware_redirect.py | 24 +++---- tests/test_downloaderslotssettings.py | 6 +- tests/test_engine.py | 12 ++-- tests/test_engine_stop_download_headers.py | 6 +- tests/test_exporters.py | 4 +- tests/test_feedexport.py | 76 +++++++++----------- tests/test_http_request.py | 12 ++-- tests/test_http_response.py | 6 +- tests/test_item.py | 4 +- tests/test_pipeline_files.py | 2 +- tests/test_pipeline_media.py | 4 -- tests/test_robotstxt_interface.py | 7 +- tests/test_selector.py | 6 +- tests/test_utils_response.py | 30 ++++---- tests/test_utils_url.py | 6 +- 42 files changed, 151 insertions(+), 172 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c76c613d9..18402b908 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,10 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.4 + rev: v0.9.3 hooks: - id: ruff args: [ --fix ] -- repo: https://github.com/psf/black.git - rev: 24.10.0 - hooks: - - id: black + - id: ruff-format - repo: https://github.com/adamchainz/blacken-docs rev: 1.19.1 hooks: diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 2a1dea997..6d4aec3d8 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -188,9 +188,9 @@ class Command(ScrapyCommand): return True return False - assert ( - self.crawler_process is not None - ), "crawler_process must be set before calling run" + assert self.crawler_process is not None, ( + "crawler_process must be set before calling run" + ) try: spidercls = self.crawler_process.spider_loader.load(name) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 7f3da67eb..902f200b8 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -34,13 +34,12 @@ class DownloadHandlerProtocol(Protocol): class DownloadHandlers: def __init__(self, crawler: Crawler): self._crawler: Crawler = crawler - self._schemes: dict[str, str | Callable[..., Any]] = ( - {} - ) # stores acceptable schemes on instancing - self._handlers: dict[str, DownloadHandlerProtocol] = ( - {} - ) # stores instanced handlers for schemes - self._notconfigured: dict[str, str] = {} # remembers failed handlers + # stores acceptable schemes on instancing + self._schemes: dict[str, str | Callable[..., Any]] = {} + # stores instanced handlers for schemes + self._handlers: dict[str, DownloadHandlerProtocol] = {} + # remembers failed handlers + self._notconfigured: dict[str, str] = {} handlers: dict[str, str | Callable[..., Any]] = without_none_values( cast( "dict[str, str | Callable[..., Any]]", diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index a4dc89c18..afca99dcf 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -193,7 +193,7 @@ class Stream: url.netloc == str(self._protocol.metadata["uri"].host, "utf-8") or url.netloc == str(self._protocol.metadata["uri"].netloc, "utf-8") or url.netloc - == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' + == f"{self._protocol.metadata['ip_address']}:{self._protocol.metadata['uri'].port}" ) def _get_request_headers(self) -> list[tuple[str, str]]: @@ -339,7 +339,7 @@ class Stream: if self._log_warnsize: self.metadata["reached_warnsize"] = True warning_msg = ( - f'Received more ({self._response["flow_controlled_size"]}) bytes than download ' + f"Received more ({self._response['flow_controlled_size']}) bytes than download " f"warn size ({self._download_warnsize}) in request {self._request}" ) logger.warning(warning_msg) @@ -445,7 +445,7 @@ class Stream: ResponseFailed( [ Failure( - f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM', + f"Remote peer {self._protocol.metadata['ip_address']} sent RST_STREAM", ProtocolError, ) ] @@ -465,7 +465,7 @@ class Stream: InvalidHostname( self._request, str(self._protocol.metadata["uri"].host, "utf-8"), - f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}', + f"{self._protocol.metadata['ip_address']}:{self._protocol.metadata['uri'].port}", ) ) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 43348f632..9156b8c3a 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -54,8 +54,7 @@ class CookiesMiddleware: ) -> None: for cookie in cookies: cookie_domain = cookie.domain - if cookie_domain.startswith("."): - cookie_domain = cookie_domain[1:] + cookie_domain = cookie_domain.removeprefix(".") hostname = urlparse_cached(request).hostname assert hostname is not None diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index a2cff65e7..787c46a60 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -89,5 +89,5 @@ class OffsiteMiddleware: warnings.warn(message) else: domains.append(re.escape(domain)) - regex = rf'^(.*\.)?({"|".join(domains)})$' + regex = rf"^(.*\.)?({'|'.join(domains)})$" return re.compile(regex) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 9411cff14..aba455bdd 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -63,7 +63,9 @@ class RobotsTxtMiddleware: if request.url.startswith("data:") or request.url.startswith("file:"): return None d: Deferred[RobotParser | None] = maybeDeferred( - self.robot_parser, request, spider # type: ignore[call-overload] + self.robot_parser, + request, + spider, # type: ignore[call-overload] ) d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider) return d2 diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index fb0f30620..cb5887a6f 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -19,7 +19,7 @@ if TYPE_CHECKING: def get_header_size( - headers: dict[str, list[str | bytes] | tuple[str | bytes, ...]] + headers: dict[str, list[str | bytes] | tuple[str | bytes, ...]], ) -> int: size = 0 for key, value in headers.items(): diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 189b1953b..ac832e025 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -84,9 +84,7 @@ class TelnetConsole(protocol.ServerFactory): """An implementation of IPortal""" @defers - def login( - self_, credentials, mind, *interfaces - ): # pylint: disable=no-self-argument + def login(self_, credentials, mind, *interfaces): # pylint: disable=no-self-argument if not ( credentials.username == self.username.encode("utf8") and credentials.checkPassword(self.password.encode("utf8")) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 29ba9533b..60b04753b 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -105,7 +105,8 @@ class Headers(CaselessDict): def values(self) -> list[bytes | None]: # type: ignore[override] return [ - self[k] for k in self.keys() # pylint: disable=consider-using-dict-items + self[k] + for k in self.keys() # pylint: disable=consider-using-dict-items ] def to_string(self) -> bytes: diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index de3b24de0..7681419c4 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -24,7 +24,6 @@ from scrapy.http.request import Request from scrapy.utils.python import is_listlike, to_bytes if TYPE_CHECKING: - # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index b84110b29..de2188ceb 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -94,8 +94,7 @@ class Response(object_ref): return self.request.meta # type: ignore[union-attr] except AttributeError: raise AttributeError( - "Response.meta not available, this response " - "is not tied to any request" + "Response.meta not available, this response is not tied to any request" ) @property diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 4fd932b88..814e31fec 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -25,7 +25,6 @@ from scrapy.utils.response import get_base_url from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain if TYPE_CHECKING: - from lxml.html import HtmlElement from scrapy import Selector diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index a10117590..888be81c3 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -202,7 +202,9 @@ class S3FilesStore: return cast( "Deferred[dict[str, Any]]", deferToThread( - self.s3_client.head_object, Bucket=self.bucket, Key=key_name # type: ignore[attr-defined] + self.s3_client.head_object, # type: ignore[attr-defined] + Bucket=self.bucket, + Key=key_name, ), ) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index db9014b41..f8365a87b 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -81,8 +81,7 @@ class Selector(_ParselSelector, object_ref): ): if response is not None and text is not None: raise ValueError( - f"{self.__class__.__name__}.__init__() received " - "both response and text" + f"{self.__class__.__name__}.__init__() received both response and text" ) st = _st(response, type) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 3ebdb351a..f31f824a8 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -539,7 +539,7 @@ def iter_default_settings() -> Iterable[tuple[str, Any]]: def overridden_settings( - settings: Mapping[_SettingsKeyT, Any] + settings: Mapping[_SettingsKeyT, Any], ) -> Iterable[tuple[str, Any]]: """Return an iterable of the settings that have been overridden""" for name, defvalue in iter_default_settings(): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 7ef365f68..c473b369c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -333,7 +333,7 @@ TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve()) URLLENGTH_LIMIT = 2083 -USER_AGENT = f'Scrapy/{import_module("scrapy").__version__} (+https://scrapy.org)' +USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)" TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 95e753830..646beb911 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -110,7 +110,7 @@ class OffsiteMiddleware: warnings.warn(message, PortWarning) else: domains.append(re.escape(domain)) - regex = rf'^(.*\.)?({"|".join(domains)})$' + regex = rf"^(.*\.)?({'|'.join(domains)})$" return re.compile(regex) def spider_opened(self, spider: Spider) -> None: diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 80bb37e93..7007cd4b8 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -147,16 +147,24 @@ def _pickle_serialize(obj: Any) -> bytes: # queue.*Queue aren't subclasses of queue.BaseQueue _PickleFifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type] + _with_mkdir(queue.FifoDiskQueue), # type: ignore[arg-type] + _pickle_serialize, + pickle.loads, ) _PickleLifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type] + _with_mkdir(queue.LifoDiskQueue), # type: ignore[arg-type] + _pickle_serialize, + pickle.loads, ) _MarshalFifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type] + _with_mkdir(queue.FifoDiskQueue), # type: ignore[arg-type] + marshal.dumps, + marshal.loads, ) _MarshalLifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type] + _with_mkdir(queue.LifoDiskQueue), # type: ignore[arg-type] + marshal.dumps, + marshal.loads, ) # public queue classes diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index bfdd4dc8a..a563dc79a 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -22,8 +22,7 @@ class DataAction(argparse.Action): option_string: str | None = None, ) -> None: value = str(values) - if value.startswith("$"): - value = value[1:] + value = value.removeprefix("$") setattr(namespace, self.dest, value) @@ -96,7 +95,7 @@ def curl_to_request_kwargs( parsed_args, argv = curl_parser.parse_known_args(curl_args[1:]) if argv: - msg = f'Unrecognized options: {", ".join(argv)}' + msg = f"Unrecognized options: {', '.join(argv)}" if ignore_unknown_options: warnings.warn(msg) else: diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 000ab5c65..8f52836c4 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -377,7 +377,7 @@ def deferred_from_coro(o: _T) -> Deferred | _T: def deferred_f_from_coro_f( - coro_f: Callable[_P, Coroutine[Any, Any, _T]] + coro_f: Callable[_P, Coroutine[Any, Any, _T]], ) -> Callable[_P, Deferred[_T]]: """Converts a coroutine function into a function that returns a Deferred. diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index e8ed7b60a..c646fc218 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -71,7 +71,7 @@ def xmliter(obj: Response | str | bytes, nodename: str) -> Iterator[Selector]: nodetext = ( document_header + match.group().replace( - nodename, f'{nodename} {" ".join(namespaces.values())}', 1 + nodename, f"{nodename} {' '.join(namespaces.values())}", 1 ) + header_end ) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index d51231b82..b865cf48d 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -16,7 +16,6 @@ from scrapy.settings import Settings, _SettingsKeyT from scrapy.utils.versions import get_versions if TYPE_CHECKING: - from scrapy.crawler import Crawler from scrapy.logformatter import LogFormatterResult diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index e954b625c..fcf582082 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -119,8 +119,7 @@ def to_unicode( return text if not isinstance(text, (bytes, str)): raise TypeError( - "to_unicode must receive a bytes or str " - f"object, got {type(text).__name__}" + f"to_unicode must receive a bytes or str object, got {type(text).__name__}" ) if encoding is None: encoding = "utf-8" @@ -183,7 +182,7 @@ _SelfT = TypeVar("_SelfT") def memoizemethod_noargs( - method: Callable[Concatenate[_SelfT, _P], _T] + method: Callable[Concatenate[_SelfT, _P], _T], ) -> Callable[Concatenate[_SelfT, _P], _T]: """Decorator to cache the result of a method (without arguments) using a weak reference to its object @@ -313,7 +312,7 @@ def without_none_values(iterable: Iterable[_KT]) -> Iterable[_KT]: ... def without_none_values( - iterable: Mapping[_KT, _VT] | Iterable[_KT] + iterable: Mapping[_KT, _VT] | Iterable[_KT], ) -> dict[_KT, _VT] | Iterable[_KT]: """Return a copy of ``iterable`` with all ``None`` entries removed. diff --git a/tests/spiders.py b/tests/spiders.py index 3c44d7da5..da923de6e 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -338,9 +338,9 @@ class BrokenStartRequestsSpider(FollowAllSpider): if self.fail_yielding: 2 / 0 - assert ( - self.seedsseen - ), "All start requests consumed before any download happened" + assert self.seedsseen, ( + "All start requests consumed before any download happened" + ) def parse(self, response): self.seedsseen.append(response.meta.get("seed")) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index f7581707b..fb16140be 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -529,7 +529,7 @@ class ContractsManagerTest(unittest.TestCase): return TestItem() with MockServer() as mockserver: - contract_doc = f'@url {mockserver.url("/status?n=200")}' + contract_doc = f"@url {mockserver.url('/status?n=200')}" TestSameUrlSpider.parse_first.__doc__ = contract_doc TestSameUrlSpider.parse_second.__doc__ = contract_doc @@ -567,7 +567,6 @@ class CustomFailContractPostProcess(Contract): class CustomContractPrePostProcess(unittest.TestCase): - def setUp(self): self.results = TextTestResult(stream=None, descriptions=False, verbosity=0) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 0ea1ef5eb..97c276b48 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -94,7 +94,7 @@ class TestHttpProxyMiddleware(TestCase): def test_proxy_auth_encoding(self): # utf-8 encoding - os.environ["http_proxy"] = "https://m\u00E1n:pass@proxy:3128" + os.environ["http_proxy"] = "https://m\u00e1n:pass@proxy:3128" mw = HttpProxyMiddleware(auth_encoding="utf-8") req = Request("http://scrapytest.org") assert mw.process_request(req, spider) is None @@ -103,7 +103,7 @@ class TestHttpProxyMiddleware(TestCase): # proxy from request.meta req = Request( - "http://scrapytest.org", meta={"proxy": "https://\u00FCser:pass@proxy:3128"} + "http://scrapytest.org", meta={"proxy": "https://\u00fcser:pass@proxy:3128"} ) assert mw.process_request(req, spider) is None self.assertEqual(req.meta["proxy"], "https://proxy:3128") @@ -120,7 +120,7 @@ class TestHttpProxyMiddleware(TestCase): # proxy from request.meta, latin-1 encoding req = Request( - "http://scrapytest.org", meta={"proxy": "https://\u00FCser:pass@proxy:3128"} + "http://scrapytest.org", meta={"proxy": "https://\u00fcser:pass@proxy:3128"} ) assert mw.process_request(req, spider) is None self.assertEqual(req.meta["proxy"], "https://proxy:3128") diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 7b19ab781..eb3cdfc11 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -55,12 +55,12 @@ class Base: assert isinstance(req2, Request) self.assertEqual(req2.url, url2) self.assertEqual(req2.method, "GET") - assert ( - "Content-Type" not in req2.headers - ), "Content-Type header must not be present in redirected request" - assert ( - "Content-Length" not in req2.headers - ), "Content-Length header must not be present in redirected request" + assert "Content-Type" not in req2.headers, ( + "Content-Type header must not be present in redirected request" + ) + assert "Content-Length" not in req2.headers, ( + "Content-Length header must not be present in redirected request" + ) assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" def test_max_redirect_times(self): @@ -1243,12 +1243,12 @@ class MetaRefreshMiddlewareTest(Base.Test): assert isinstance(req2, Request) self.assertEqual(req2.url, "http://example.org/newpage") self.assertEqual(req2.method, "GET") - assert ( - "Content-Type" not in req2.headers - ), "Content-Type header must not be present in redirected request" - assert ( - "Content-Length" not in req2.headers - ), "Content-Length header must not be present in redirected request" + assert "Content-Type" not in req2.headers, ( + "Content-Type header must not be present in redirected request" + ) + assert "Content-Length" not in req2.headers, ( + "Content-Length header must not be present in redirected request" + ) assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" def test_ignore_tags_default(self): diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 879bc8697..0bb143f69 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -93,6 +93,6 @@ def test_params(): _, actual = downloader._get_slot(request, spider=None) expected = Slot(**params) for param in params: - assert getattr(expected, param) == getattr( - actual, param - ), f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}" + assert getattr(expected, param) == getattr(actual, param), ( + f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}" + ) diff --git a/tests/test_engine.py b/tests/test_engine.py index 8d645eada..95955f7be 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -294,9 +294,9 @@ class EngineTest(unittest.TestCase): ] urls_visited = {rp[0].url for rp in run.respplug} urls_expected = {run.geturl(p) for p in must_be_visited} - assert ( - urls_expected <= urls_visited - ), f"URLs not visited: {list(urls_expected - urls_visited)}" + assert urls_expected <= urls_visited, ( + f"URLs not visited: {list(urls_expected - urls_visited)}" + ) def _assert_scheduled_requests(self, run: CrawlerRun, count=None): self.assertEqual(count, len(run.reqplug)) @@ -496,9 +496,9 @@ def test_request_scheduled_signal(caplog): drop_request = Request("https://drop.example") caplog.set_level(DEBUG) engine._schedule_request(drop_request, spider) - assert scheduler.enqueued == [ - keep_request - ], f"{scheduler.enqueued!r} != [{keep_request!r}]" + assert scheduler.enqueued == [keep_request], ( + f"{scheduler.enqueued!r} != [{keep_request!r}]" + ) crawler.signals.disconnect(signal_handler, request_scheduled) diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index 0bad5ba55..db35bd81e 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -67,6 +67,6 @@ class HeadersReceivedEngineTest(EngineTest): must_be_visited = ["/", "/redirect", "/redirected"] urls_visited = {rp[0].url for rp in run.respplug} urls_expected = {run.geturl(p) for p in must_be_visited} - assert ( - urls_expected <= urls_visited - ), f"URLs not visited: {list(urls_expected - urls_visited)}" + assert urls_expected <= urls_visited, ( + f"URLs not visited: {list(urls_expected - urls_visited)}" + ) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 1fbacfdfc..c2cab9b2a 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -390,14 +390,14 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_errors_default(self): with self.assertRaises(UnicodeEncodeError): self.assertExportResult( - item={"text": "W\u0275\u200Brd"}, + item={"text": "W\u0275\u200brd"}, expected=None, encoding="windows-1251", ) def test_errors_xmlcharrefreplace(self): self.assertExportResult( - item={"text": "W\u0275\u200Brd"}, + item={"text": "W\u0275\u200brd"}, include_headers_line=False, expected="Wɵ​rd\r\n", encoding="windows-1251", diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 7edffa1f6..4f91795e4 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1190,8 +1190,7 @@ class FeedExportTest(FeedExportTestBase): "csv": b"baz,egg,foo\r\n,spam1,bar1\r\n", "json": b'[\n{"hello": "world2", "foo": "bar2"}\n]', "jsonlines": ( - b'{"foo": "bar1", "egg": "spam1"}\n' - b'{"hello": "world2", "foo": "bar2"}\n' + b'{"foo": "bar1", "egg": "spam1"}\n{"hello": "world2", "foo": "bar2"}\n' ), "xml": ( b'\n\n' @@ -2289,9 +2288,9 @@ class BatchDeliveriesTest(FeedExportTestBase): settings.update( { "FEEDS": { - self._random_temp_filename() - / "jl" - / self._file_mark: {"format": "jl"}, + self._random_temp_filename() / "jl" / self._file_mark: { + "format": "jl" + }, }, } ) @@ -2311,9 +2310,9 @@ class BatchDeliveriesTest(FeedExportTestBase): settings.update( { "FEEDS": { - self._random_temp_filename() - / "csv" - / self._file_mark: {"format": "csv"}, + self._random_temp_filename() / "csv" / self._file_mark: { + "format": "csv" + }, }, } ) @@ -2331,9 +2330,9 @@ class BatchDeliveriesTest(FeedExportTestBase): settings.update( { "FEEDS": { - self._random_temp_filename() - / "xml" - / self._file_mark: {"format": "xml"}, + self._random_temp_filename() / "xml" / self._file_mark: { + "format": "xml" + }, }, } ) @@ -2352,12 +2351,12 @@ class BatchDeliveriesTest(FeedExportTestBase): settings.update( { "FEEDS": { - self._random_temp_filename() - / "xml" - / self._file_mark: {"format": "xml"}, - self._random_temp_filename() - / "json" - / self._file_mark: {"format": "json"}, + self._random_temp_filename() / "xml" / self._file_mark: { + "format": "xml" + }, + self._random_temp_filename() / "json" / self._file_mark: { + "format": "json" + }, }, } ) @@ -2384,9 +2383,9 @@ class BatchDeliveriesTest(FeedExportTestBase): settings.update( { "FEEDS": { - self._random_temp_filename() - / "pickle" - / self._file_mark: {"format": "pickle"}, + self._random_temp_filename() / "pickle" / self._file_mark: { + "format": "pickle" + }, }, } ) @@ -2406,9 +2405,9 @@ class BatchDeliveriesTest(FeedExportTestBase): settings.update( { "FEEDS": { - self._random_temp_filename() - / "marshal" - / self._file_mark: {"format": "marshal"}, + self._random_temp_filename() / "marshal" / self._file_mark: { + "format": "marshal" + }, }, } ) @@ -2455,9 +2454,9 @@ class BatchDeliveriesTest(FeedExportTestBase): for fmt in ("json", "jsonlines", "xml", "csv"): settings = { "FEEDS": { - self._random_temp_filename() - / fmt - / self._file_mark: {"format": fmt}, + self._random_temp_filename() / fmt / self._file_mark: { + "format": fmt + }, }, "FEED_EXPORT_BATCH_ITEM_COUNT": 1, "FEED_STORE_EMPTY": False, @@ -2478,9 +2477,9 @@ class BatchDeliveriesTest(FeedExportTestBase): for fmt, expctd in formats: settings = { "FEEDS": { - self._random_temp_filename() - / fmt - / self._file_mark: {"format": fmt}, + self._random_temp_filename() / fmt / self._file_mark: { + "format": fmt + }, }, "FEED_STORE_EMPTY": True, "FEED_EXPORT_INDENT": None, @@ -2520,25 +2519,19 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = { "FEEDS": { - self._random_temp_filename() - / "json" - / self._file_mark: { + self._random_temp_filename() / "json" / self._file_mark: { "format": "json", "indent": 0, "fields": ["bar"], "encoding": "utf-8", }, - self._random_temp_filename() - / "xml" - / self._file_mark: { + self._random_temp_filename() / "xml" / self._file_mark: { "format": "xml", "indent": 2, "fields": ["foo"], "encoding": "latin-1", }, - self._random_temp_filename() - / "csv" - / self._file_mark: { + self._random_temp_filename() / "csv" / self._file_mark: { "format": "csv", "indent": None, "fields": ["foo", "bar"], @@ -2563,9 +2556,7 @@ class BatchDeliveriesTest(FeedExportTestBase): } settings = { "FEEDS": { - self._random_temp_filename() - / "json" - / self._file_mark: { + self._random_temp_filename() / "json" / self._file_mark: { "format": "json", "indent": None, "encoding": "utf-8", @@ -2591,8 +2582,7 @@ class BatchDeliveriesTest(FeedExportTestBase): ] settings = { "FEEDS": { - self._random_temp_filename() - / "%(batch_id)d": { + self._random_temp_filename() / "%(batch_id)d": { "format": "json", }, }, diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 34d3b25d5..9915aaca4 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -226,9 +226,9 @@ class RequestTest(unittest.TestCase): self.assertEqual(r1.flags, r2.flags) # make sure cb_kwargs dict is shallow copied - assert ( - r1.cb_kwargs is not r2.cb_kwargs - ), "cb_kwargs must be a shallow copy, not identical" + assert r1.cb_kwargs is not r2.cb_kwargs, ( + "cb_kwargs must be a shallow copy, not identical" + ) self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) # make sure meta dict is shallow copied @@ -236,9 +236,9 @@ class RequestTest(unittest.TestCase): self.assertEqual(r1.meta, r2.meta) # make sure headers attribute is shallow copied - assert ( - r1.headers is not r2.headers - ), "headers must be a shallow copy, not identical" + assert r1.headers is not r2.headers, ( + "headers must be a shallow copy, not identical" + ) self.assertEqual(r1.headers, r2.headers) self.assertEqual(r1.encoding, r2.encoding) self.assertEqual(r1.dont_filter, r2.dont_filter) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 0730cff3a..b157e9802 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -99,9 +99,9 @@ class BaseResponseTest(unittest.TestCase): self.assertEqual(r1.flags, r2.flags) # make sure headers attribute is shallow copied - assert ( - r1.headers is not r2.headers - ), "headers must be a shallow copy, not identical" + assert r1.headers is not r2.headers, ( + "headers must be a shallow copy, not identical" + ) self.assertEqual(r1.headers, r2.headers) def test_copy_meta(self): diff --git a/tests/test_item.py b/tests/test_item.py index 480412841..0399c8f8d 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -289,9 +289,7 @@ class ItemMetaTest(unittest.TestCase): class ItemMetaClassCellRegression(unittest.TestCase): def test_item_meta_classcell_regression(self): class MyItem(Item, metaclass=ItemMeta): - def __init__( - self, *args, **kwargs - ): # pylint: disable=useless-parent-delegation + def __init__(self, *args, **kwargs): # pylint: disable=useless-parent-delegation # This call to super() trigger the __classcell__ propagation # requirement. When not done properly raises an error: # TypeError: __class__ set to diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4c3fc36b6..4c59fcfb7 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -215,7 +215,7 @@ class FilesPipelineTestCase(unittest.TestCase): class CustomFilesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None, item=None): - return f'full/{item.get("path")}' + return f"full/{item.get('path')}" file_path = CustomFilesPipeline.from_crawler( get_crawler(None, {"FILES_STORE": self.tempdir}) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index dd8f1084a..c6fdd3767 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -35,7 +35,6 @@ def _mocked_download_func(request, info): class UserDefinedPipeline(MediaPipeline): - def media_to_download(self, request, info, *, item=None): pass @@ -376,7 +375,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): - def _assert_request_no3xx(self, pipeline_class, settings): pipe = pipeline_class(crawler=get_crawler(None, settings)) request = Request("http://url") @@ -403,11 +401,9 @@ class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): self.assertNotIn(status, request.meta["handle_httpstatus_list"]) def test_subclass_standard_setting(self): - self._assert_request_no3xx(UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True}) def test_subclass_specific_setting(self): - self._assert_request_no3xx( UserDefinedPipeline, {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True} ) diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index e127cc2e3..0d00ff660 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -27,10 +27,7 @@ class BaseRobotParserTest: def test_allowed(self): robotstxt_robotstxt_body = ( - b"User-agent: * \n" - b"Disallow: /disallowed \n" - b"Allow: /allowed \n" - b"Crawl-delay: 10" + b"User-agent: * \nDisallow: /disallowed \nAllow: /allowed \nCrawl-delay: 10" ) rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body @@ -140,7 +137,7 @@ class DecodeRobotsTxtTest(unittest.TestCase): self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") def test_decode_non_utf8(self): - robotstxt_body = b"User-agent: *\n\xFFDisallow: /\n" + robotstxt_body = b"User-agent: *\n\xffDisallow: /\n" decoded_content = decode_robotstxt(robotstxt_body, spider=None) self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") diff --git a/tests/test_selector.py b/tests/test_selector.py index 857c7d626..4eda0460f 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -107,9 +107,9 @@ class SelectorTestCase(unittest.TestCase): """Check that classes are using slots and are weak-referenceable""" x = Selector(text="") weakref.ref(x) - assert not hasattr( - x, "__dict__" - ), f"{x.__class__.__name__} does not use __slots__" + assert not hasattr(x, "__dict__"), ( + f"{x.__class__.__name__} does not use __slots__" + ) def test_selector_bad_args(self): with self.assertRaisesRegex(ValueError, "received both response and text"): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 7ad86127b..db6866571 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -158,18 +158,18 @@ class ResponseUtilsTest(unittest.TestCase): ) assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url" - assert open_in_browser( - r2, _openfunc=check_base_url - ), "Inject base url with argumented head" - assert open_in_browser( - r3, _openfunc=check_base_url - ), "Inject unique base url with misleading tag" - assert open_in_browser( - r4, _openfunc=check_base_url - ), "Inject unique base url with misleading comment" - assert open_in_browser( - r5, _openfunc=check_base_url - ), "Inject unique base url with conditional comment" + assert open_in_browser(r2, _openfunc=check_base_url), ( + "Inject base url with argumented head" + ) + assert open_in_browser(r3, _openfunc=check_base_url), ( + "Inject unique base url with misleading tag" + ) + assert open_in_browser(r4, _openfunc=check_base_url), ( + "Inject unique base url with misleading comment" + ) + assert open_in_browser(r5, _openfunc=check_base_url), ( + "Inject unique base url with conditional comment" + ) def test_open_in_browser_redos_comment(self): MAX_CPU_TIME = 0.02 @@ -240,6 +240,6 @@ class ResponseUtilsTest(unittest.TestCase): ), ) def test_remove_html_comments(input_body, output_body): - assert ( - _remove_html_comments(input_body) == output_body - ), f"{_remove_html_comments(input_body)=} == {output_body=}" + assert _remove_html_comments(input_body) == output_body, ( + f"{_remove_html_comments(input_body)=} == {output_body=}" + ) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 314082742..4b9a98d79 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -321,9 +321,9 @@ class GuessSchemeTest(unittest.TestCase): def create_guess_scheme_t(args): def do_expected(self): url = guess_scheme(args[0]) - assert url.startswith( - args[1] - ), f"Wrong scheme guessed: for `{args[0]}` got `{url}`, expected `{args[1]}...`" + assert url.startswith(args[1]), ( + f"Wrong scheme guessed: for `{args[0]}` got `{url}`, expected `{args[1]}...`" + ) return do_expected From 98a57e241879e1b56ef8bffeb8f85f868e91c1e9 Mon Sep 17 00:00:00 2001 From: Lidiane T Date: Mon, 27 Jan 2025 10:21:30 +0000 Subject: [PATCH 031/491] Fix error when running `scrapy bench` (#6633) --- scrapy/commands/bench.py | 2 +- tests/test_commands.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 714bc38da..16dae6ac4 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -67,6 +67,6 @@ class _BenchSpider(scrapy.Spider): return [scrapy.Request(url, dont_filter=True)] def parse(self, response: Response) -> Any: - assert isinstance(Response, TextResponse) + assert isinstance(response, TextResponse) for link in self.link_extractor.extract_links(response): yield scrapy.Request(link.url, callback=self.parse) diff --git a/tests/test_commands.py b/tests/test_commands.py index 50f093043..872b54d04 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1034,6 +1034,7 @@ class BenchCommandTest(CommandTest): ) self.assertIn("INFO: Crawled", log) self.assertNotIn("Unhandled Error", log) + self.assertNotIn("log_count/ERROR", log) class ViewCommandTest(CommandTest): From d27c6b46b11c2ceaa61b35372ad8a3c98185aa18 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 27 Jan 2025 21:25:47 +0500 Subject: [PATCH 032/491] Deprecate HTTP/1.0 support. --- scrapy/core/downloader/handlers/http10.py | 7 +++++++ scrapy/core/downloader/webclient.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 58f7ad577..0fbe5fc23 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -2,8 +2,10 @@ from __future__ import annotations +import warnings from typing import TYPE_CHECKING +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode @@ -26,6 +28,11 @@ class HTTP10DownloadHandler: lazy = False def __init__(self, settings: BaseSettings, crawler: Crawler): + warnings.warn( + "HTTP10DownloadHandler is deprecated and will be removed in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) self.HTTPClientFactory: type[ScrapyHTTPClientFactory] = load_object( settings["DOWNLOADER_HTTPCLIENTFACTORY"] ) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index ee10ae73b..aaaf68152 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,6 +1,7 @@ from __future__ import annotations import re +import warnings from time import time from typing import TYPE_CHECKING from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse @@ -9,6 +10,7 @@ from twisted.internet import defer from twisted.internet.protocol import ClientFactory from twisted.web.http import HTTPClient +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached @@ -49,6 +51,14 @@ def _parse(url: str) -> tuple[bytes, bytes, bytes, int, bytes]: class ScrapyHTTPPageGetter(HTTPClient): delimiter = b"\n" + def __init__(self): + warnings.warn( + "ScrapyHTTPPageGetter is deprecated and will be removed in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + super().__init__() + def connectionMade(self): self.headers = Headers() # bucket for response headers @@ -140,6 +150,12 @@ class ScrapyHTTPClientFactory(ClientFactory): self.path = self.url def __init__(self, request: Request, timeout: float = 180): + warnings.warn( + "ScrapyHTTPClientFactory is deprecated and will be removed in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self._url: str = urldefrag(request.url)[0] # converting to bytes to comply to Twisted interface self.url: bytes = to_bytes(self._url, encoding="ascii") From 16b998f9ca8b928b03f9f8be659ac01d4f2f623f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 28 Jan 2025 01:33:00 +0500 Subject: [PATCH 033/491] Sort out webclient tests. --- scrapy/core/downloader/contextfactory.py | 1 + scrapy/core/downloader/webclient.py | 2 + tests/test_core_downloader.py | 134 +++++++++++++++++++++++ tests/test_downloader_handlers.py | 2 + tests/test_webclient.py | 89 ++------------- 5 files changed, 146 insertions(+), 82 deletions(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index d44c663bb..b01ee97f3 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -121,6 +121,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context: + # FIXME ctx: SSL.Context = self.getCertificateOptions().getContext() ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT return ctx diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index aaaf68152..09751ea1a 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,3 +1,5 @@ +"""Deprecated HTTP/1.0 helper classes used by HTTP10DownloadHandler.""" + from __future__ import annotations import re diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index d929a9369..0a0c0a4f0 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -1,6 +1,31 @@ +from __future__ import annotations + +import shutil +from pathlib import Path +from tempfile import mkdtemp + +import OpenSSL.SSL +import pytest +from twisted.internet import reactor +from twisted.internet.defer import inlineCallbacks +from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest +from twisted.web import server, static +from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody +from twisted.web.client import Response as TxResponse from scrapy.core.downloader import Slot +from scrapy.core.downloader.contextfactory import ( + ScrapyClientContextFactory, + load_context_factory_from_settings, +) +from scrapy.core.downloader.handlers.http11 import _RequestBodyProducer +from scrapy.settings import Settings +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.python import to_bytes +from scrapy.utils.test import get_crawler +from tests.mockserver import PayloadResource, ssl_context_factory class SlotTest(unittest.TestCase): @@ -10,3 +35,112 @@ class SlotTest(unittest.TestCase): repr(slot), "Slot(concurrency=8, delay=0.10, randomize_delay=True)", ) + + +class ContextFactoryBaseTestCase(unittest.TestCase): + context_factory = None + + def _listen(self, site): + return reactor.listenSSL( + 0, + site, + contextFactory=self.context_factory or ssl_context_factory(), + interface="127.0.0.1", + ) + + def getURL(self, path): + return f"https://127.0.0.1:{self.portno}/{path}" + + def setUp(self): + self.tmpname = Path(mkdtemp()) + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) + r.putChild(b"payload", PayloadResource()) + self.site = server.Site(r, timeout=None) + self.wrapper = WrappingFactory(self.site) + self.port = self._listen(self.wrapper) + self.portno = self.port.getHost().port + + @inlineCallbacks + def tearDown(self): + yield self.port.stopListening() + shutil.rmtree(self.tmpname) + + @staticmethod + async def get_page( + url: str, + client_context_factory: BrowserLikePolicyForHTTPS, + body: str | None = None, + ) -> bytes: + agent = Agent(reactor, contextFactory=client_context_factory) + body_producer = _RequestBodyProducer(body.encode()) if body else None + response: TxResponse = await maybe_deferred_to_future( + agent.request(b"GET", url.encode(), bodyProducer=body_producer) + ) + return await maybe_deferred_to_future(readBody(response)) # type: ignore[arg-type] + + +class ContextFactoryTestCase(ContextFactoryBaseTestCase): + @deferred_f_from_coro_f + async def testPayload(self): + s = "0123456789" * 10 + crawler = get_crawler() + settings = Settings() + client_context_factory = load_context_factory_from_settings(settings, crawler) + body = await self.get_page( + self.getURL("payload"), client_context_factory, body=s + ) + self.assertEqual(body, to_bytes(s)) + + +class ContextFactoryTLSMethodTestCase(ContextFactoryBaseTestCase): + async def _assert_factory_works( + self, client_context_factory: ScrapyClientContextFactory + ) -> None: + s = "0123456789" * 10 + body = await self.get_page( + self.getURL("payload"), client_context_factory, body=s + ) + self.assertEqual(body, to_bytes(s)) + + @deferred_f_from_coro_f + async def test_setting_default(self): + crawler = get_crawler() + settings = Settings() + client_context_factory = load_context_factory_from_settings(settings, crawler) + assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD + await self._assert_factory_works(client_context_factory) + + def test_setting_none(self): + crawler = get_crawler() + settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": None}) + with pytest.raises(KeyError): + load_context_factory_from_settings(settings, crawler) + + def test_setting_bad(self): + crawler = get_crawler() + settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) + with pytest.raises(KeyError): + load_context_factory_from_settings(settings, crawler) + + @deferred_f_from_coro_f + async def test_setting_explicit(self): + crawler = get_crawler() + settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "TLSv1.2"}) + client_context_factory = load_context_factory_from_settings(settings, crawler) + assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD + await self._assert_factory_works(client_context_factory) + + @deferred_f_from_coro_f + async def test_direct_from_crawler(self): + # the setting is ignored + crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) + client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler) + assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD + await self._assert_factory_works(client_context_factory) + + @deferred_f_from_coro_f + async def test_direct_init(self): + client_context_factory = ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD) + assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD + await self._assert_factory_works(client_context_factory) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 0dcbeaec1..64f615bfe 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -422,6 +422,7 @@ class HttpTestCase(unittest.TestCase): return self.download_request(request, Spider("foo")).addCallback(_test) +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" @@ -780,6 +781,7 @@ class HttpProxyTestCase(unittest.TestCase): return self.download_request(request, Spider("foo")).addCallback(_test) +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class Http10ProxyTestCase(HttpProxyTestCase): download_handler_cls: type = HTTP10DownloadHandler diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 0a594aa7c..fa19b350b 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -8,12 +8,11 @@ from __future__ import annotations import shutil from pathlib import Path from tempfile import mkdtemp -from typing import Any import OpenSSL.SSL -from pytest import raises +import pytest from twisted.internet import defer, reactor -from twisted.internet.defer import Deferred, inlineCallbacks +from twisted.internet.defer import inlineCallbacks from twisted.internet.testing import StringTransport from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest @@ -22,10 +21,8 @@ from twisted.web import resource, server, static, util from scrapy.core.downloader import webclient as client from scrapy.core.downloader.contextfactory import ( ScrapyClientContextFactory, - load_context_factory_from_settings, ) from scrapy.http import Headers, Request -from scrapy.settings import Settings from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.test import get_crawler @@ -38,6 +35,7 @@ from tests.mockserver import ( PayloadResource, ssl_context_factory, ) +from tests.test_core_downloader import ContextFactoryBaseTestCase def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): @@ -129,6 +127,7 @@ class ParseUrlTestCase(unittest.TestCase): self.assertEqual(client._parse(url), test, url) +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class ScrapyHTTPPageGetterTests(unittest.TestCase): def test_earlyHeaders(self): # basic test stolen from twisted HTTPageGetter @@ -272,6 +271,7 @@ class EncodingResource(resource.Resource): return body.encode(self.out_encoding) +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class WebClientTestCase(unittest.TestCase): def _listen(self, site): return reactor.listenTCP(0, site, interface="127.0.0.1") @@ -427,35 +427,8 @@ class WebClientTestCase(unittest.TestCase): ) -class WebClientSSLTestCase(unittest.TestCase): - context_factory = None - - def _listen(self, site): - return reactor.listenSSL( - 0, - site, - contextFactory=self.context_factory or ssl_context_factory(), - interface="127.0.0.1", - ) - - def getURL(self, path): - return f"https://127.0.0.1:{self.portno}/{path}" - - def setUp(self): - self.tmpname = Path(mkdtemp()) - (self.tmpname / "file").write_bytes(b"0123456789") - r = static.File(str(self.tmpname)) - r.putChild(b"payload", PayloadResource()) - self.site = server.Site(r, timeout=None) - self.wrapper = WrappingFactory(self.site) - self.port = self._listen(self.wrapper) - self.portno = self.port.getHost().port - - @inlineCallbacks - def tearDown(self): - yield self.port.stopListening() - shutil.rmtree(self.tmpname) - +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class WebClientSSLTestCase(ContextFactoryBaseTestCase): def testPayload(self): s = "0123456789" * 10 return getPage(self.getURL("payload"), body=s).addCallback( @@ -490,51 +463,3 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): self.getURL("payload"), body=s, contextFactory=client_context_factory ) return self.assertFailure(d, OpenSSL.SSL.Error) - - -class WebClientTLSMethodTestCase(WebClientSSLTestCase): - def _assert_factory_works( - self, client_context_factory: ScrapyClientContextFactory - ) -> Deferred[Any]: - s = "0123456789" * 10 - return getPage( - self.getURL("payload"), body=s, contextFactory=client_context_factory - ).addCallback(self.assertEqual, to_bytes(s)) - - def test_setting_default(self): - crawler = get_crawler() - settings = Settings() - client_context_factory = load_context_factory_from_settings(settings, crawler) - assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD - return self._assert_factory_works(client_context_factory) - - def test_setting_none(self): - crawler = get_crawler() - settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": None}) - with raises(KeyError): - load_context_factory_from_settings(settings, crawler) - - def test_setting_bad(self): - crawler = get_crawler() - settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) - with raises(KeyError): - load_context_factory_from_settings(settings, crawler) - - def test_setting_explicit(self): - crawler = get_crawler() - settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "TLSv1.2"}) - client_context_factory = load_context_factory_from_settings(settings, crawler) - assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD - return self._assert_factory_works(client_context_factory) - - def test_direct_from_crawler(self): - # the setting is ignored - crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) - client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler) - assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD - return self._assert_factory_works(client_context_factory) - - def test_direct_init(self): - client_context_factory = ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD) - assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD - return self._assert_factory_works(client_context_factory) From bc1aeeefc970fbd123699e0cb6d8486141bf8418 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 28 Jan 2025 01:54:43 +0500 Subject: [PATCH 034/491] Deprecate overriding ScrapyClientContextFactory.getContext(). --- scrapy/core/downloader/contextfactory.py | 9 ++++++++- tests/test_core_downloader.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index b01ee97f3..d1ba6208a 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -22,6 +22,7 @@ from scrapy.core.downloader.tls import ( openssl_methods, ) from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.deprecate import method_is_overridden from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: @@ -62,6 +63,13 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers) else: self.tls_ciphers = DEFAULT_CIPHERS + if method_is_overridden(type(self), ScrapyClientContextFactory, "getContext"): + warnings.warn( + "Overriding ScrapyClientContextFactory.getContext() is deprecated and that method" + " will be removed in a future Scrapy version. Override creatorForNetloc() instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) @classmethod def from_settings( @@ -121,7 +129,6 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context: - # FIXME ctx: SSL.Context = self.getCertificateOptions().getContext() ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT return ctx diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 0a0c0a4f0..e67337fc7 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -1,8 +1,10 @@ from __future__ import annotations import shutil +import warnings from pathlib import Path from tempfile import mkdtemp +from typing import Any import OpenSSL.SSL import pytest @@ -92,6 +94,22 @@ class ContextFactoryTestCase(ContextFactoryBaseTestCase): ) self.assertEqual(body, to_bytes(s)) + def test_override_getContext(self): + class MyFactory(ScrapyClientContextFactory): + def getContext( + self, hostname: Any = None, port: Any = None + ) -> OpenSSL.SSL.Context: + ctx: OpenSSL.SSL.Context = super().getContext(hostname, port) + return ctx + + with warnings.catch_warnings(record=True) as w: + MyFactory() + self.assertEqual(len(w), 1) + self.assertIn( + "Overriding ScrapyClientContextFactory.getContext() is deprecated", + str(w[0].message), + ) + class ContextFactoryTLSMethodTestCase(ContextFactoryBaseTestCase): async def _assert_factory_works( From 0d2d2892badb2b6f47c9b597564510871c7f1518 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 28 Jan 2025 02:08:49 +0500 Subject: [PATCH 035/491] Silence the readBody warning. --- tests/test_core_downloader.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index e67337fc7..dffba303f 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -9,7 +9,7 @@ from typing import Any import OpenSSL.SSL import pytest from twisted.internet import reactor -from twisted.internet.defer import inlineCallbacks +from twisted.internet.defer import Deferred, inlineCallbacks from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest from twisted.web import server, static @@ -79,7 +79,15 @@ class ContextFactoryBaseTestCase(unittest.TestCase): response: TxResponse = await maybe_deferred_to_future( agent.request(b"GET", url.encode(), bodyProducer=body_producer) ) - return await maybe_deferred_to_future(readBody(response)) # type: ignore[arg-type] + with warnings.catch_warnings(): + # https://github.com/twisted/twisted/issues/8227 + warnings.filterwarnings( + "ignore", + category=DeprecationWarning, + message=r".*does not have an abortConnection method", + ) + d: Deferred[bytes] = readBody(response) # type: ignore[arg-type] + return await maybe_deferred_to_future(d) class ContextFactoryTestCase(ContextFactoryBaseTestCase): From 0a80871c3a2b353d870e337715bbbeedf6bc216e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 28 Jan 2025 22:22:09 +0500 Subject: [PATCH 036/491] Remove scrapy.core.downloader.webclient._parse(). --- scrapy/core/downloader/handlers/http11.py | 24 ++++---- scrapy/core/downloader/handlers/http2.py | 7 +-- scrapy/core/downloader/webclient.py | 56 +++++++----------- tests/test_downloader_handlers.py | 3 - tests/test_webclient.py | 71 +---------------------- 5 files changed, 38 insertions(+), 123 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index aa8a1a2a4..74a6e54ee 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -9,7 +9,7 @@ from contextlib import suppress from io import BytesIO from time import time from typing import TYPE_CHECKING, Any, TypedDict, TypeVar -from urllib.parse import urldefrag, urlunparse +from urllib.parse import urldefrag, urlparse from twisted.internet import ssl from twisted.internet.defer import CancelledError, Deferred, succeed @@ -32,11 +32,12 @@ from zope.interface import implementer from scrapy import Request, Spider, signals from scrapy.core.downloader.contextfactory import load_context_factory_from_settings -from scrapy.core.downloader.webclient import _parse from scrapy.exceptions import StopDownload from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.url import add_http_if_no_scheme if TYPE_CHECKING: from twisted.internet.base import ReactorBase @@ -378,12 +379,15 @@ class ScrapyAgent: bindaddress = request.meta.get("bindaddress") or self._bindAddress proxy = request.meta.get("proxy") if proxy: - proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy) - scheme = _parse(request.url)[0] - proxyHost_str = to_unicode(proxyHost) - if scheme == b"https": + proxy = add_http_if_no_scheme(proxy) + proxy_parsed = urlparse(proxy) + proxy_host = proxy_parsed.hostname + proxy_port = proxy_parsed.port + if not proxy_port: + proxy_port = 443 if proxy_parsed.scheme == "https" else 80 + if urlparse_cached(request).scheme == "https": proxyAuth = request.headers.get(b"Proxy-Authorization", None) - proxyConf = (proxyHost_str, proxyPort, proxyAuth) + proxyConf = (proxy_host, proxy_port, proxyAuth) return self._TunnelingAgent( reactor=reactor, proxyConf=proxyConf, @@ -392,13 +396,9 @@ class ScrapyAgent: bindAddress=bindaddress, pool=self._pool, ) - proxyScheme = proxyScheme or b"http" - proxyURI = urlunparse( - (proxyScheme, proxyNetloc, proxyParams, b"", b"", b"") - ) return self._ProxyAgent( reactor=reactor, - proxyURI=to_bytes(proxyURI, encoding="ascii"), + proxyURI=to_bytes(proxy, encoding="ascii"), connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool, diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index f0f9ceeb7..d0a95ee9d 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -8,8 +8,8 @@ from twisted.internet.error import TimeoutError from twisted.web.client import URI from scrapy.core.downloader.contextfactory import load_context_factory_from_settings -from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes if TYPE_CHECKING: @@ -75,10 +75,7 @@ class ScrapyH2Agent: bind_address = request.meta.get("bindaddress") or self._bind_address proxy = request.meta.get("proxy") if proxy: - _, _, proxy_host, proxy_port, proxy_params = _parse(proxy) - scheme = _parse(request.url)[0] - - if scheme == b"https": + if urlparse_cached(request).scheme == "https": # ToDo raise NotImplementedError( "Tunneling via CONNECT method using HTTP/2.0 is not yet supported" diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 09751ea1a..e5c2255af 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -2,11 +2,10 @@ from __future__ import annotations -import re import warnings from time import time from typing import TYPE_CHECKING -from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse +from urllib.parse import urldefrag, urlparse, urlunparse from twisted.internet import defer from twisted.internet.protocol import ClientFactory @@ -22,34 +21,6 @@ if TYPE_CHECKING: from scrapy import Request -def _parsed_url_args(parsed: ParseResult) -> tuple[bytes, bytes, bytes, int, bytes]: - # Assume parsed is urlparse-d from Request.url, - # which was passed via safe_url_string and is ascii-only. - path_str = urlunparse(("", "", parsed.path or "/", parsed.params, parsed.query, "")) - path = to_bytes(path_str, encoding="ascii") - assert parsed.hostname is not None - host = to_bytes(parsed.hostname, encoding="ascii") - port = parsed.port - scheme = to_bytes(parsed.scheme, encoding="ascii") - netloc = to_bytes(parsed.netloc, encoding="ascii") - if port is None: - port = 443 if scheme == b"https" else 80 - return scheme, netloc, host, port, path - - -def _parse(url: str) -> tuple[bytes, bytes, bytes, int, bytes]: - """Return tuple of (scheme, netloc, host, port, path), - all in bytes except for port which is int. - Assume url is from Request.url, which was passed via safe_url_string - and is ascii-only. - """ - url = url.strip() - if not re.match(r"^\w+://", url): - url = "//" + url - parsed = urlparse(url) - return _parsed_url_args(parsed) - - class ScrapyHTTPPageGetter(HTTPClient): delimiter = b"\n" @@ -142,14 +113,29 @@ class ScrapyHTTPClientFactory(ClientFactory): ) def _set_connection_attributes(self, request): - parsed = urlparse_cached(request) - self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args( - parsed - ) proxy = request.meta.get("proxy") if proxy: - self.scheme, _, self.host, self.port, _ = _parse(proxy) + proxy_parsed = urlparse(to_bytes(proxy, encoding="ascii")) + self.scheme = proxy_parsed.scheme + self.host = proxy_parsed.hostname + self.port = proxy_parsed.port + self.netloc = proxy_parsed.netloc + if self.port is None: + self.port = 443 if proxy_parsed.scheme == b"https" else 80 self.path = self.url + else: + parsed = urlparse_cached(request) + path_str = urlunparse( + ("", "", parsed.path or "/", parsed.params, parsed.query, "") + ) + self.path = to_bytes(path_str, encoding="ascii") + assert parsed.hostname is not None + self.host = to_bytes(parsed.hostname, encoding="ascii") + self.port = parsed.port + self.scheme = to_bytes(parsed.scheme, encoding="ascii") + self.netloc = to_bytes(parsed.netloc, encoding="ascii") + if self.port is None: + self.port = 443 if self.scheme == b"https" else 80 def __init__(self, request: Request, timeout: float = 180): warnings.warn( diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 64f615bfe..ae2030fe6 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -785,9 +785,6 @@ class HttpProxyTestCase(unittest.TestCase): class Http10ProxyTestCase(HttpProxyTestCase): download_handler_cls: type = HTTP10DownloadHandler - def test_download_with_proxy_https_noconnect(self): - raise unittest.SkipTest("noconnect is not supported in HTTP10DownloadHandler") - class Http11ProxyTestCase(HttpProxyTestCase): download_handler_cls: type = HTTP11DownloadHandler diff --git a/tests/test_webclient.py b/tests/test_webclient.py index fa19b350b..1b4ad2f2f 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -8,6 +8,7 @@ from __future__ import annotations import shutil from pathlib import Path from tempfile import mkdtemp +from urllib.parse import urlparse import OpenSSL.SSL import pytest @@ -61,72 +62,6 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): ).deferred -class ParseUrlTestCase(unittest.TestCase): - """Test URL parsing facility and defaults values.""" - - def _parse(self, url): - f = client.ScrapyHTTPClientFactory(Request(url)) - return (f.scheme, f.netloc, f.host, f.port, f.path) - - def testParse(self): - lip = "127.0.0.1" - tests = ( - ( - "http://127.0.0.1?c=v&c2=v2#fragment", - ("http", lip, lip, 80, "/?c=v&c2=v2"), - ), - ( - "http://127.0.0.1/?c=v&c2=v2#fragment", - ("http", lip, lip, 80, "/?c=v&c2=v2"), - ), - ( - "http://127.0.0.1/foo?c=v&c2=v2#frag", - ("http", lip, lip, 80, "/foo?c=v&c2=v2"), - ), - ( - "http://127.0.0.1:100?c=v&c2=v2#fragment", - ("http", lip + ":100", lip, 100, "/?c=v&c2=v2"), - ), - ( - "http://127.0.0.1:100/?c=v&c2=v2#frag", - ("http", lip + ":100", lip, 100, "/?c=v&c2=v2"), - ), - ( - "http://127.0.0.1:100/foo?c=v&c2=v2#frag", - ("http", lip + ":100", lip, 100, "/foo?c=v&c2=v2"), - ), - ("http://127.0.0.1", ("http", lip, lip, 80, "/")), - ("http://127.0.0.1/", ("http", lip, lip, 80, "/")), - ("http://127.0.0.1/foo", ("http", lip, lip, 80, "/foo")), - ("http://127.0.0.1?param=value", ("http", lip, lip, 80, "/?param=value")), - ("http://127.0.0.1/?param=value", ("http", lip, lip, 80, "/?param=value")), - ( - "http://127.0.0.1:12345/foo", - ("http", lip + ":12345", lip, 12345, "/foo"), - ), - ("http://spam:12345/foo", ("http", "spam:12345", "spam", 12345, "/foo")), - ( - "http://spam.test.org/foo", - ("http", "spam.test.org", "spam.test.org", 80, "/foo"), - ), - ("https://127.0.0.1/foo", ("https", lip, lip, 443, "/foo")), - ( - "https://127.0.0.1/?param=value", - ("https", lip, lip, 443, "/?param=value"), - ), - ("https://127.0.0.1:12345/", ("https", lip + ":12345", lip, 12345, "/")), - ( - "http://scrapytest.org/foo ", - ("http", "scrapytest.org", "scrapytest.org", 80, "/foo"), - ), - ("http://egg:7890 ", ("http", "egg:7890", "egg", 7890, "/")), - ) - - for url, test in tests: - test = tuple(to_bytes(x) if not isinstance(x, int) else x for x in test) - self.assertEqual(client._parse(url), test, url) - - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class ScrapyHTTPPageGetterTests(unittest.TestCase): def test_earlyHeaders(self): @@ -388,9 +323,9 @@ class WebClientTestCase(unittest.TestCase): def testFactoryInfo(self): url = self.getURL("file") - _, _, host, port, _ = client._parse(url) + parsed = urlparse(url) factory = client.ScrapyHTTPClientFactory(Request(url)) - reactor.connectTCP(to_unicode(host), port, factory) + reactor.connectTCP(parsed.hostname, parsed.port, factory) return factory.deferred.addCallback(self._cbFactoryInfo, factory) def _cbFactoryInfo(self, ignoredResult, factory): From 200d76afa96a78899faf9e2c30ef45273b71c600 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 1 Feb 2025 16:07:55 +0500 Subject: [PATCH 037/491] Refactor EngineTest tests. --- tests/test_engine.py | 82 +++++++++++----------- tests/test_engine_stop_download_bytes.py | 4 +- tests/test_engine_stop_download_headers.py | 4 +- 3 files changed, 46 insertions(+), 44 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index 95955f7be..91ce2c0de 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -243,46 +243,7 @@ class CrawlerRun: self.signals_caught[sig] = signalargs -class EngineTest(unittest.TestCase): - @defer.inlineCallbacks - def test_crawler(self): - for spider in ( - TestSpider, - DictItemsSpider, - AttrsItemsSpider, - DataClassItemsSpider, - ): - run = CrawlerRun(spider) - yield run.run() - self._assert_visited_urls(run) - self._assert_scheduled_requests(run, count=9) - self._assert_downloaded_responses(run, count=9) - self._assert_scraped_items(run) - self._assert_signals_caught(run) - self._assert_bytes_received(run) - - @defer.inlineCallbacks - def test_crawler_dupefilter(self): - run = CrawlerRun(TestDupeFilterSpider) - yield run.run() - self._assert_scheduled_requests(run, count=8) - self._assert_dropped_requests(run) - - @defer.inlineCallbacks - def test_crawler_itemerror(self): - run = CrawlerRun(ItemZeroDivisionErrorSpider) - yield run.run() - self._assert_items_error(run) - - @defer.inlineCallbacks - def test_crawler_change_close_reason_on_idle(self): - run = CrawlerRun(ChangeCloseReasonSpider) - yield run.run() - self.assertEqual( - {"spider": run.spider, "reason": "custom_reason"}, - run.signals_caught[signals.spider_closed], - ) - +class EngineTestBase(unittest.TestCase): def _assert_visited_urls(self, run: CrawlerRun): must_be_visited = [ "/", @@ -422,6 +383,47 @@ class EngineTest(unittest.TestCase): run.signals_caught[signals.spider_closed], ) + +class EngineTest(EngineTestBase): + @defer.inlineCallbacks + def test_crawler(self): + for spider in ( + TestSpider, + DictItemsSpider, + AttrsItemsSpider, + DataClassItemsSpider, + ): + run = CrawlerRun(spider) + yield run.run() + self._assert_visited_urls(run) + self._assert_scheduled_requests(run, count=9) + self._assert_downloaded_responses(run, count=9) + self._assert_scraped_items(run) + self._assert_signals_caught(run) + self._assert_bytes_received(run) + + @defer.inlineCallbacks + def test_crawler_dupefilter(self): + run = CrawlerRun(TestDupeFilterSpider) + yield run.run() + self._assert_scheduled_requests(run, count=8) + self._assert_dropped_requests(run) + + @defer.inlineCallbacks + def test_crawler_itemerror(self): + run = CrawlerRun(ItemZeroDivisionErrorSpider) + yield run.run() + self._assert_items_error(run) + + @defer.inlineCallbacks + def test_crawler_change_close_reason_on_idle(self): + run = CrawlerRun(ChangeCloseReasonSpider) + yield run.run() + self.assertEqual( + {"spider": run.spider, "reason": "custom_reason"}, + run.signals_caught[signals.spider_closed], + ) + @defer.inlineCallbacks def test_close_downloader(self): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index 8dbb5b7ea..8bf225ab1 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -7,7 +7,7 @@ from tests.test_engine import ( CrawlerRun, DataClassItemsSpider, DictItemsSpider, - EngineTest, + EngineTestBase, TestSpider, ) @@ -18,7 +18,7 @@ class BytesReceivedCrawlerRun(CrawlerRun): raise StopDownload(fail=False) -class BytesReceivedEngineTest(EngineTest): +class BytesReceivedEngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index db35bd81e..4efb6b7a8 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -7,7 +7,7 @@ from tests.test_engine import ( CrawlerRun, DataClassItemsSpider, DictItemsSpider, - EngineTest, + EngineTestBase, TestSpider, ) @@ -18,7 +18,7 @@ class HeadersReceivedCrawlerRun(CrawlerRun): raise StopDownload(fail=False) -class HeadersReceivedEngineTest(EngineTest): +class HeadersReceivedEngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( From 1a0dfbd32e8d96299c4f0f3d16cb2d52a73de339 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 2 Feb 2025 13:28:34 +0500 Subject: [PATCH 038/491] Reuse mockserver instances in test_feedexport.py. --- tests/test_feedexport.py | 80 ++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 49 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 4f91795e4..ae52a3e18 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -13,7 +13,6 @@ import tempfile import warnings from abc import ABC, abstractmethod from collections import defaultdict -from contextlib import ExitStack from io import BytesIO from logging import getLogger from pathlib import Path @@ -623,8 +622,6 @@ class LogOnStoreFileStorage: class FeedExportTestBase(ABC, unittest.TestCase): - __test__ = False - class MyItem(scrapy.Item): foo = scrapy.Field() egg = scrapy.Field() @@ -641,8 +638,11 @@ class FeedExportTestBase(ABC, unittest.TestCase): def setUp(self): self.temp_dir = tempfile.mkdtemp() + self.mockserver = MockServer() + self.mockserver.__enter__() def tearDown(self): + self.mockserver.__exit__(None, None, None) shutil.rmtree(self.temp_dir, ignore_errors=True) @defer.inlineCallbacks @@ -746,8 +746,6 @@ class ExceptionJsonItemExporter(JsonItemExporter): class FeedExportTest(FeedExportTestBase): - __test__ = True - @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): """Run spider with specified settings; return exported data.""" @@ -760,10 +758,9 @@ class FeedExportTest(FeedExportTestBase): content = {} try: - with MockServer() as s: - spider_cls.start_urls = [s.url("/")] - crawler = get_crawler(spider_cls, settings) - yield crawler.crawl() + spider_cls.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(spider_cls, settings) + yield crawler.crawl() for file_path, feed_options in FEEDS.items(): content[feed_options["format"]] = ( @@ -890,8 +887,7 @@ class FeedExportTest(FeedExportTestBase): }, } crawler = get_crawler(ItemSpider, settings) - with MockServer() as mockserver: - yield crawler.crawl(mockserver=mockserver) + yield crawler.crawl(mockserver=self.mockserver) self.assertIn( "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() ) @@ -909,15 +905,11 @@ class FeedExportTest(FeedExportTestBase): }, } crawler = get_crawler(ItemSpider, settings) - with ExitStack() as stack: - mockserver = stack.enter_context(MockServer()) - stack.enter_context( - mock.patch( - "scrapy.extensions.feedexport.FileFeedStorage.store", - side_effect=KeyError("foo"), - ) - ) - yield crawler.crawl(mockserver=mockserver) + with mock.patch( + "scrapy.extensions.feedexport.FileFeedStorage.store", + side_effect=KeyError("foo"), + ): + yield crawler.crawl(mockserver=self.mockserver) self.assertIn( "feedexport/failed_count/FileFeedStorage", crawler.stats.get_stats() ) @@ -938,8 +930,8 @@ class FeedExportTest(FeedExportTestBase): }, } crawler = get_crawler(ItemSpider, settings) - with MockServer() as mockserver, mock.patch.object(S3FeedStorage, "store"): - yield crawler.crawl(mockserver=mockserver) + with mock.patch.object(S3FeedStorage, "store"): + yield crawler.crawl(mockserver=self.mockserver) self.assertIn( "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() ) @@ -1730,8 +1722,6 @@ class FeedExportTest(FeedExportTestBase): class FeedPostProcessedExportsTest(FeedExportTestBase): - __test__ = True - items = [{"foo": "bar"}] expected = b"foo\r\nbar\r\n" @@ -1764,10 +1754,9 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): content = {} try: - with MockServer() as s: - spider_cls.start_urls = [s.url("/")] - crawler = get_crawler(spider_cls, settings) - yield crawler.crawl() + spider_cls.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(spider_cls, settings) + yield crawler.crawl() for file_path in FEEDS: content[str(file_path)] = ( @@ -2253,7 +2242,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): class BatchDeliveriesTest(FeedExportTestBase): - __test__ = True _file_mark = "_%(batch_time)s_#%(batch_id)02d_" @defer.inlineCallbacks @@ -2265,21 +2253,17 @@ class BatchDeliveriesTest(FeedExportTestBase): build_url(file_path): feed for file_path, feed in FEEDS.items() } content = defaultdict(list) - try: - with MockServer() as s: - spider_cls.start_urls = [s.url("/")] - crawler = get_crawler(spider_cls, settings) - yield crawler.crawl() + spider_cls.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(spider_cls, settings) + yield crawler.crawl() - for path, feed in FEEDS.items(): - dir_name = Path(path).parent - if not dir_name.exists(): - content[feed["format"]] = [] - continue - for file in sorted(dir_name.iterdir()): - content[feed["format"]].append(file.read_bytes()) - finally: - self.tearDown() + for path, feed in FEEDS.items(): + dir_name = Path(path).parent + if not dir_name.exists(): + content[feed["format"]] = [] + continue + for file in sorted(dir_name.iterdir()): + content[feed["format"]].append(file.read_bytes()) return content @defer.inlineCallbacks @@ -2604,8 +2588,7 @@ class BatchDeliveriesTest(FeedExportTestBase): "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } crawler = get_crawler(ItemSpider, settings) - with MockServer() as mockserver: - yield crawler.crawl(total=2, mockserver=mockserver) + yield crawler.crawl(total=2, mockserver=self.mockserver) self.assertIn( "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() ) @@ -2675,10 +2658,9 @@ class BatchDeliveriesTest(FeedExportTestBase): def parse(self, response): yield from items - with MockServer() as server: - TestSpider.start_urls = [server.url("/")] - crawler = get_crawler(TestSpider, settings) - yield crawler.crawl() + TestSpider.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(TestSpider, settings) + yield crawler.crawl() self.assertEqual(len(CustomS3FeedStorage.stubs), len(items)) for stub in CustomS3FeedStorage.stubs[:-1]: From 783b98dedaea65e0c6658d9d588328dd887c2c8a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 2 Feb 2025 14:10:09 +0500 Subject: [PATCH 039/491] Make mockserver instances per-class. --- tests/test_closespider.py | 12 +++++++----- tests/test_crawl.py | 24 ++++++++++++++---------- tests/test_downloader_handlers.py | 12 +++++++----- tests/test_downloaderslotssettings.py | 16 ++++++++++------ tests/test_feedexport.py | 12 +++++++++--- tests/test_logformatter.py | 14 +++++++++----- tests/test_pipeline_crawl.py | 13 +++++++++---- tests/test_pipelines.py | 12 +++++++----- tests/test_proxy_connect.py | 12 +++++++++--- tests/test_request_attribute_binding.py | 12 +++++++----- tests/test_request_cb_kwargs.py | 12 +++++++----- tests/test_request_left.py | 12 +++++++----- tests/test_signals.py | 16 ++++++++++------ tests/test_spidermiddleware_httperror.py | 12 +++++++----- 14 files changed, 119 insertions(+), 72 deletions(-) diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 9a837350f..ecde301d1 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -13,12 +13,14 @@ from tests.spiders import ( class TestCloseSpider(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_closespider_itemcount(self): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 1f81a6073..cd2a559a8 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -56,12 +56,14 @@ from tests.spiders import ( class CrawlTestCase(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_follow_all(self): @@ -448,12 +450,14 @@ with multiples lines class CrawlSpiderTestCase(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def _run_spider(self, spider_cls): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index ae2030fe6..1549059f0 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -693,12 +693,14 @@ class Http11MockServerTestCase(unittest.TestCase): settings_dict: dict | None = None - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_download_with_content_length(self): diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 0bb143f69..4f8b005d7 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -50,13 +50,17 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): class CrawlTestCase(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() - self.runner = CrawlerRunner() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + + def setUp(self): + self.runner = CrawlerRunner() @defer.inlineCallbacks def test_delay(self): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ae52a3e18..1620d2d41 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -636,13 +636,19 @@ class FeedExportTestBase(ABC, unittest.TestCase): filename = "".join(chars) return Path(self.temp_dir, inter_dir, filename) + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + def setUp(self): self.temp_dir = tempfile.mkdtemp() - self.mockserver = MockServer() - self.mockserver.__enter__() def tearDown(self): - self.mockserver.__exit__(None, None, None) shutil.rmtree(self.temp_dir, ignore_errors=True) @defer.inlineCallbacks diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index e5d077858..962692a31 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -262,9 +262,16 @@ class DropSomeItemsPipeline: class ShowOrSkipMessagesTestCase(TwistedTestCase): + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() self.base_settings = { "LOG_LEVEL": "DEBUG", "ITEM_PIPELINES": { @@ -272,9 +279,6 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): }, } - def tearDown(self): - self.mockserver.__exit__(None, None, None) - @defer.inlineCallbacks def test_show_messages(self): crawler = get_crawler(ItemSpider, self.base_settings) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 9e1b1ab5b..84d714e5c 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -64,10 +64,16 @@ class FileDownloadCrawlTestCase(TestCase): "ed3f6538dc15d4d9179dae57319edc5f", } - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + + def setUp(self): # prepare a directory for storing files self.tmpmediastore = Path(mkdtemp()) self.settings = { @@ -80,7 +86,6 @@ class FileDownloadCrawlTestCase(TestCase): def tearDown(self): shutil.rmtree(self.tmpmediastore) self.items = [] - self.mockserver.__exit__(None, None, None) def _on_item_scraped(self, item): self.items.append(item) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 5ab288c1a..222b19e7f 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -77,12 +77,14 @@ class ItemSpider(Spider): class PipelineTestCase(unittest.TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) def _on_item_scraped(self, item): self.assertIsInstance(item, dict) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 26bd6332c..6ed7e93a6 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -62,14 +62,21 @@ def _wrong_credentials(proxy_url): class ProxyConnectTestCase(TestCase): + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + def setUp(self): try: import mitmproxy # noqa: F401 except ImportError: self.skipTest("mitmproxy is not installed") - self.mockserver = MockServer() - self.mockserver.__enter__() self._oldenv = os.environ.copy() self._proxy = MitmProxy() @@ -78,7 +85,6 @@ class ProxyConnectTestCase(TestCase): os.environ["http_proxy"] = proxy_url def tearDown(self): - self.mockserver.__exit__(None, None, None) self._proxy.stop() os.environ = self._oldenv diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index d65d74206..0072660a7 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -57,12 +57,14 @@ class AlternativeCallbacksMiddleware: class CrawlTestCase(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_response_200(self): diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index b178c928b..a21cb43ff 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -154,12 +154,14 @@ class KeywordArgumentsSpider(MockServerSpider): class CallbackKeywordArgumentsTestCase(TestCase): maxDiff = None - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_callback_kwargs(self): diff --git a/tests/test_request_left.py b/tests/test_request_left.py index ba1b70695..cf4c8a2d5 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -25,12 +25,14 @@ class SignalCatcherSpider(Spider): class TestCatching(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_success(self): diff --git a/tests/test_signals.py b/tests/test_signals.py index 0df104600..1e693c094 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -21,13 +21,17 @@ class ItemSpider(Spider): class AsyncSignalTestCase(unittest.TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() - self.items = [] + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + + def setUp(self): + self.items = [] async def _on_item_scraped(self, item): item = await get_from_asyncio_queue(item) diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 01a2b4bb4..307054de7 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -172,12 +172,14 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): class TestHttpErrorMiddlewareIntegrational(TrialTestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_middleware_works(self): From df688910e0499b0a874d220fe00ed7355a70fce0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 2 Feb 2025 18:48:26 +0500 Subject: [PATCH 040/491] Remove a duplicate test. --- tests/test_downloadermiddleware_httpcache.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index ec4e87ffb..a0886d9e9 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -14,7 +14,7 @@ from scrapy.utils.test import get_crawler class _BaseTest(unittest.TestCase): - storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" + storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" policy_class = "scrapy.extensions.httpcache.RFC2616Policy" def setUp(self): @@ -161,11 +161,7 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): self.assertEqual(storage.dbmodule.__name__, self.dbm_module) -class FilesystemStorageTest(DefaultStorageTest): - storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" - - -class FilesystemStorageGzipTest(FilesystemStorageTest): +class FilesystemStorageGzipTest(DefaultStorageTest): def _get_settings(self, **new_settings): new_settings.setdefault("HTTPCACHE_GZIP", True) return super()._get_settings(**new_settings) From 393ff96e45ffcb738bc9f8d3240cd4d999bd07df Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 7 Nov 2024 21:21:17 +0500 Subject: [PATCH 041/491] Deprecate AjaxCrawlMiddleware. --- docs/topics/broad-crawls.rst | 24 ------------------- docs/topics/downloader-middleware.rst | 3 +-- scrapy/downloadermiddlewares/ajaxcrawl.py | 11 +++++++-- ...test_downloadermiddleware_ajaxcrawlable.py | 3 +++ 4 files changed, 13 insertions(+), 28 deletions(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 0286c3354..248e38b61 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -182,30 +182,6 @@ To disable redirects use: REDIRECT_ENABLED = False -Enable crawling of "Ajax Crawlable Pages" -========================================= - -Some pages (up to 1%, based on empirical data from year 2013) declare -themselves as ajax crawlable. This means they provide plain HTML -version of content that is usually available only via AJAX. -Pages can indicate it in two ways: - -1) by using ``#!`` in URL - this is the default way; -2) by using a special meta tag - this way is used on - "main", "index" website pages. - -Scrapy handles (1) automatically; to handle (2) enable -:ref:`AjaxCrawlMiddleware `: - -.. code-block:: python - - AJAXCRAWL_ENABLED = True - -When doing broad crawls it's common to crawl a lot of "index" web pages; -AjaxCrawlMiddleware helps to crawl them correctly. -It is turned OFF by default because it has some performance overhead, -and enabling it for focused crawls doesn't make much sense. - .. _broad-crawls-bfo: Crawl in BFO order diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1ab8f588f..ca597291f 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1249,8 +1249,7 @@ AJAXCRAWL_ENABLED Default: ``False`` -Whether the AjaxCrawlMiddleware will be enabled. You may want to -enable it for :ref:`broad crawls `. +Whether the AjaxCrawlMiddleware will be enabled. HttpProxyMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 166192b4f..c6a55732d 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -3,10 +3,11 @@ from __future__ import annotations import logging import re from typing import TYPE_CHECKING +from warnings import warn from w3lib import html -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Response if TYPE_CHECKING: @@ -30,6 +31,13 @@ class AjaxCrawlMiddleware: if not settings.getbool("AJAXCRAWL_ENABLED"): raise NotConfigured + warn( + "scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware is deprecated" + " and will be removed in a future Scrapy version.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + # XXX: Google parses at least first 100k bytes; scrapy's redirect # middleware parses first 4k. 4k turns out to be insufficient # for this middleware, and parsing 100k could be slow. @@ -75,7 +83,6 @@ class AjaxCrawlMiddleware: return _has_ajaxcrawlable_meta(body) -# XXX: move it to w3lib? _ajax_crawlable_re: re.Pattern[str] = re.compile( r'' ) diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 043dc0a12..63bd158f6 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -1,5 +1,7 @@ import unittest +import pytest + from scrapy.downloadermiddlewares.ajaxcrawl import AjaxCrawlMiddleware from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider @@ -8,6 +10,7 @@ from scrapy.utils.test import get_crawler __doctests__ = ["scrapy.downloadermiddlewares.ajaxcrawl"] +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class AjaxCrawlMiddlewareTest(unittest.TestCase): def setUp(self): crawler = get_crawler(Spider, {"AJAXCRAWL_ENABLED": True}) From 4842bcbf1da41029a604d1bf743d4dc893960d39 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 2 Feb 2025 23:23:51 +0500 Subject: [PATCH 042/491] Deprecate and disable escape_ajax(). --- scrapy/downloadermiddlewares/ajaxcrawl.py | 4 ++-- scrapy/http/request/__init__.py | 4 +--- scrapy/utils/url.py | 6 ++++++ tests/test_http_request.py | 12 ------------ 4 files changed, 9 insertions(+), 17 deletions(-) diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index c6a55732d..e7a8962a1 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -9,6 +9,7 @@ from w3lib import html from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Response +from scrapy.utils.url import escape_ajax if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -64,8 +65,7 @@ class AjaxCrawlMiddleware: if not self._has_ajax_crawlable_variant(response): return response - # scrapy already handles #! links properly - ajax_crawl_request = request.replace(url=request.url + "#!") + ajax_crawl_request = request.replace(url=escape_ajax(request.url + "#!")) logger.debug( "Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s", {"ajax_crawl_request": ajax_crawl_request, "request": request}, diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 4eee5ffbb..e24f6874d 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -27,7 +27,6 @@ from scrapy.http.headers import Headers from scrapy.utils.curl import curl_to_request_kwargs from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref -from scrapy.utils.url import escape_ajax if TYPE_CHECKING: from collections.abc import Callable, Iterable, Mapping @@ -170,8 +169,7 @@ class Request(object_ref): if not isinstance(url, str): raise TypeError(f"Request url must be str, got {type(url).__name__}") - s = safe_url_string(url, self.encoding) - self._url = escape_ajax(s) + self._url = safe_url_string(url, self.encoding) if ( "://" not in self._url diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index db2749d79..1348cc992 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -10,6 +10,7 @@ import warnings from importlib import import_module from typing import TYPE_CHECKING, Union from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse +from warnings import warn from w3lib.url import __all__ as _public_w3lib_objects from w3lib.url import add_or_replace_parameter as _add_or_replace_parameter @@ -83,6 +84,11 @@ def escape_ajax(url: str) -> str: >>> escape_ajax("www.example.com/ajax.html") 'www.example.com/ajax.html' """ + warn( + "escape_ajax() is deprecated and will be removed in a future Scrapy version.", + ScrapyDeprecationWarning, + stacklevel=2, + ) defrag, frag = urldefrag(url) if not frag.startswith("!"): return url diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 9915aaca4..a8ab8240f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -187,18 +187,6 @@ class RequestTest(unittest.TestCase): assert isinstance(r4.body, bytes) self.assertEqual(r4.body, b"Price: \xa3100") - def test_ajax_url(self): - # ascii url - r = self.request_class(url="http://www.example.com/ajax.html#!key=value") - self.assertEqual( - r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue" - ) - # unicode url - r = self.request_class(url="http://www.example.com/ajax.html#!key=value") - self.assertEqual( - r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue" - ) - def test_copy(self): """Test Request copy""" From 76a8badd24cea6509df24e070f7fc06f47ee9ac3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 3 Feb 2025 14:55:10 +0500 Subject: [PATCH 043/491] Add a deprecation notice to the AjaxCrawlMiddleware docs. --- docs/topics/downloader-middleware.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index ca597291f..33308940c 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1251,6 +1251,10 @@ Default: ``False`` Whether the AjaxCrawlMiddleware will be enabled. + .. note:: + + This middleware is deprecated and will be removed in a future Scrapy release. + HttpProxyMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From ba5df629a2004ca0d919d8b7f0a7f5725448e50a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 3 Feb 2025 19:11:47 +0400 Subject: [PATCH 044/491] Refactor downloader tests (#6647) * Make download handler test base classes abstract. * Small cleanup. * Don't run the full test suite for special HTTP cases. * Don't run tests in imported base classes. * Remove an obsolete service_identity check. * Move FTP imports back to the top level. * Simplify the H2DownloadHandler import. * Forbig pytest 8.2.x. * Revert "Simplify the H2DownloadHandler import." This reverts commit ed187046ac53c395c7423c0f5e6fb2bc7c27838f. --- pyproject.toml | 3 + tests/test_downloader_handlers.py | 150 +++++++++++++----------- tests/test_downloader_handlers_http2.py | 134 +++++++++++---------- tox.ini | 2 +- 4 files changed, 149 insertions(+), 140 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 29e26399f..1072730c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -180,6 +180,7 @@ disable = [ "unused-argument", "unused-import", "unused-variable", + "useless-import-alias", # used as a hint to mypy "useless-return", # https://github.com/pylint-dev/pylint/issues/6530 "wrong-import-position", @@ -319,6 +320,8 @@ ignore = [ "D403", # `try`-`except` within a loop incurs performance overhead "PERF203", + # Import alias does not rename original package + "PLC0414", # Too many return statements "PLR0911", # Too many branches diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 1549059f0..323a51002 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -4,6 +4,7 @@ import contextlib import os import shutil import sys +from abc import ABC, abstractmethod from pathlib import Path from tempfile import mkdtemp, mkstemp from unittest import SkipTest, mock @@ -12,17 +13,18 @@ import pytest from testfixtures import LogCapture from twisted.cred import checkers, credentials, portal from twisted.internet import defer, error, reactor +from twisted.protocols.ftp import FTPFactory, FTPRealm from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest from twisted.web import resource, server, static, util -from twisted.web._newclient import ResponseFailed +from twisted.web.client import ResponseFailed from twisted.web.http import _DataLoss from w3lib.url import path_to_file_uri -from scrapy.core.downloader.handlers import DownloadHandlers +from scrapy.core.downloader.handlers import DownloadHandlerProtocol, DownloadHandlers from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler -from scrapy.core.downloader.handlers.http import HTTPDownloadHandler +from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler @@ -183,10 +185,7 @@ class BrokenDownloadResource(resource.Resource): def closeConnection(request): # We have to force a disconnection for HTTP/1.1 clients. Otherwise # client keeps the connection open waiting for more data. - if hasattr(request.channel, "loseConnection"): # twisted >=16.3.0 - request.channel.loseConnection() - else: - request.channel.transport.loseConnection() + request.channel.loseConnection() request.finish() @@ -218,14 +217,18 @@ class DuplicateHeaderResource(resource.Resource): return b"" -class HttpTestCase(unittest.TestCase): +class HttpTestCase(unittest.TestCase, ABC): scheme = "http" - download_handler_cls: type = HTTPDownloadHandler # only used for HTTPS tests keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + def setUp(self): self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") @@ -426,7 +429,9 @@ class HttpTestCase(unittest.TestCase): class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" - download_handler_cls: type = HTTP10DownloadHandler + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP10DownloadHandler def test_protocol(self): request = Request(self.getURL("host"), method="GET") @@ -443,7 +448,9 @@ class Https10TestCase(Http10TestCase): class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" - download_handler_cls: type = HTTP11DownloadHandler + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP11DownloadHandler def test_download_without_maxsize_limit(self): request = Request(self.getURL("file")) @@ -604,50 +611,16 @@ class Https11TestCase(Http11TestCase): yield download_handler.close() -class Https11WrongHostnameTestCase(Http11TestCase): - scheme = "https" - - # above tests use a server certificate for "localhost", - # client connection to "localhost" too. - # here we test that even if the server certificate is for another domain, - # "www.example.com" in this case, - # the tests still pass - keyfile = "keys/example-com.key.pem" - certfile = "keys/example-com.cert.pem" - - -class Https11InvalidDNSId(Https11TestCase): - """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" - - def setUp(self): - super().setUp() - self.host = "127.0.0.1" - - -class Https11InvalidDNSPattern(Https11TestCase): - """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" - - keyfile = "keys/localhost.ip.key" - certfile = "keys/localhost.ip.crt" - - def setUp(self): - try: - from service_identity.exceptions import CertificateError # noqa: F401 - except ImportError: - raise unittest.SkipTest("cryptography lib is too old") - self.tls_log_message = ( - 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' - 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' - ) - super().setUp() - - -class Https11CustomCiphers(unittest.TestCase): - scheme = "https" - download_handler_cls: type = HTTP11DownloadHandler +class SimpleHttpsTest(unittest.TestCase): + """Base class for special cases tested with just one simple request""" keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" + cipher_string: str | None = None + + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP11DownloadHandler def setUp(self): self.tmpname = Path(mkdtemp()) @@ -659,14 +632,16 @@ class Https11CustomCiphers(unittest.TestCase): 0, self.site, ssl_context_factory( - self.keyfile, self.certfile, cipher_string="CAMELLIA256-SHA" + self.keyfile, self.certfile, cipher_string=self.cipher_string ), interface=self.host, ) self.portno = self.port.getHost().port - crawler = get_crawler( - settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": "CAMELLIA256-SHA"} - ) + if self.cipher_string is not None: + settings_dict = {"DOWNLOADER_CLIENT_TLS_CIPHERS": self.cipher_string} + else: + settings_dict = None + crawler = get_crawler(settings_dict=settings_dict) self.download_handler = build_from_crawler(self.download_handler_cls, crawler) self.download_request = self.download_handler.download_request @@ -678,7 +653,7 @@ class Https11CustomCiphers(unittest.TestCase): shutil.rmtree(self.tmpname) def getURL(self, path): - return f"{self.scheme}://{self.host}:{self.portno}/{path}" + return f"https://{self.host}:{self.portno}/{path}" def test_download(self): request = Request(self.getURL("file")) @@ -688,10 +663,40 @@ class Https11CustomCiphers(unittest.TestCase): return d +class Https11WrongHostnameTestCase(SimpleHttpsTest): + # above tests use a server certificate for "localhost", + # client connection to "localhost" too. + # here we test that even if the server certificate is for another domain, + # "www.example.com" in this case, + # the tests still pass + keyfile = "keys/example-com.key.pem" + certfile = "keys/example-com.cert.pem" + + +class Https11InvalidDNSId(SimpleHttpsTest): + """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" + + def setUp(self): + super().setUp() + self.host = "127.0.0.1" + + +class Https11InvalidDNSPattern(SimpleHttpsTest): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = "keys/localhost.ip.key" + certfile = "keys/localhost.ip.crt" + + +class Https11CustomCiphers(SimpleHttpsTest): + cipher_string = "CAMELLIA256-SHA" + + class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" settings_dict: dict | None = None + is_secure = False @classmethod def setUpClass(cls): @@ -709,7 +714,8 @@ class Http11MockServerTestCase(unittest.TestCase): # download it yield crawler.crawl( seed=Request( - url=self.mockserver.url("/partial"), meta={"download_maxsize": 1000} + url=self.mockserver.url("/partial", is_secure=self.is_secure), + meta={"download_maxsize": 1000}, ) ) failure = crawler.spider.meta["failure"] @@ -718,7 +724,9 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download(self): crawler = get_crawler(SingleRequestSpider, self.settings_dict) - yield crawler.crawl(seed=Request(url=self.mockserver.url(""))) + yield crawler.crawl( + seed=Request(url=self.mockserver.url("", is_secure=self.is_secure)) + ) failure = crawler.spider.meta.get("failure") self.assertTrue(failure is None) reason = crawler.spider.meta["close_reason"] @@ -740,10 +748,14 @@ class UriResource(resource.Resource): return b"" -class HttpProxyTestCase(unittest.TestCase): - download_handler_cls: type = HTTPDownloadHandler +class HttpProxyTestCase(unittest.TestCase, ABC): expected_http_proxy_request_body = b"http://example.com" + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + def setUp(self): site = server.Site(UriResource(), timeout=None) wrapper = WrappingFactory(site) @@ -785,11 +797,15 @@ class HttpProxyTestCase(unittest.TestCase): @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class Http10ProxyTestCase(HttpProxyTestCase): - download_handler_cls: type = HTTP10DownloadHandler + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP10DownloadHandler class Http11ProxyTestCase(HttpProxyTestCase): - download_handler_cls: type = HTTP11DownloadHandler + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP11DownloadHandler @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): @@ -1008,10 +1024,6 @@ class BaseFTPTestCase(unittest.TestCase): ) def setUp(self): - from twisted.protocols.ftp import FTPFactory, FTPRealm - - from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler - # setup dirs and test file self.directory = Path(mkdtemp()) userdir = self.directory / self.username @@ -1155,10 +1167,6 @@ class AnonymousFTPTestCase(BaseFTPTestCase): req_meta = {} def setUp(self): - from twisted.protocols.ftp import FTPFactory, FTPRealm - - from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler - # setup dir and test file self.directory = Path(mkdtemp()) for filename, content in self.test_files: diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 322075043..174bf841e 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -1,7 +1,7 @@ import json -from unittest import mock, skipIf +from unittest import mock -from pytest import mark +import pytest from testfixtures import LogCapture from twisted.internet import defer, error, reactor from twisted.trial import unittest @@ -9,30 +9,60 @@ from twisted.web import server from twisted.web.error import SchemeNotSupported from twisted.web.http import H2_ENABLED +from scrapy.core.downloader.handlers import DownloadHandlerProtocol from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler from tests.mockserver import ssl_context_factory from tests.test_downloader_handlers import ( - Http11MockServerTestCase, - Http11ProxyTestCase, - Https11CustomCiphers, - Https11TestCase, UriResource, ) +pytestmark = pytest.mark.skipif( + not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled" +) -@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") -class Https2TestCase(Https11TestCase): + +class BaseTestClasses: + # A hack to prevent tests from the imported classes to run here too. + # See https://stackoverflow.com/q/1323455/113586 for other ways. + from tests.test_downloader_handlers import ( + Http11MockServerTestCase as Http11MockServerTestCase, + ) + from tests.test_downloader_handlers import ( + Http11ProxyTestCase as Http11ProxyTestCase, + ) + from tests.test_downloader_handlers import ( + Https11CustomCiphers as Https11CustomCiphers, + ) + from tests.test_downloader_handlers import ( + Https11InvalidDNSId as Https11InvalidDNSId, + ) + from tests.test_downloader_handlers import ( + Https11InvalidDNSPattern as Https11InvalidDNSPattern, + ) + from tests.test_downloader_handlers import ( + Https11TestCase as Https11TestCase, + ) + from tests.test_downloader_handlers import ( + Https11WrongHostnameTestCase as Https11WrongHostnameTestCase, + ) + + +def _get_dh() -> type[DownloadHandlerProtocol]: + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + + return H2DownloadHandler + + +class Https2TestCase(BaseTestClasses.Https11TestCase): scheme = "https" HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" - @classmethod - def setUpClass(cls): - from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - - cls.download_handler_cls = H2DownloadHandler + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return _get_dh() def test_protocol(self): request = Request(self.getURL("host"), method="GET") @@ -99,7 +129,7 @@ class Https2TestCase(Https11TestCase): return defer.DeferredList([d1, d2]) - @mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247") + @pytest.mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247") def test_connect_request(self): request = Request(self.getURL("file"), method="CONNECT") d = self.download_request(request, Spider("foo")) @@ -150,61 +180,31 @@ class Https2TestCase(Https11TestCase): return d -class Https2WrongHostnameTestCase(Https2TestCase): - tls_log_message = ( - 'SSL connection certificate: issuer "/C=XW/ST=XW/L=The ' - 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", ' - 'subject "/C=XW/ST=XW/L=The ' - 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"' - ) - - # above tests use a server certificate for "localhost", - # client connection to "localhost" too. - # here we test that even if the server certificate is for another domain, - # "www.example.com" in this case, - # the tests still pass - keyfile = "keys/example-com.key.pem" - certfile = "keys/example-com.cert.pem" +class Https2WrongHostnameTestCase(BaseTestClasses.Https11WrongHostnameTestCase): + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return _get_dh() -class Https2InvalidDNSId(Https2TestCase): - """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" - - def setUp(self): - super().setUp() - self.host = "127.0.0.1" +class Https2InvalidDNSId(BaseTestClasses.Https11InvalidDNSId): + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return _get_dh() -class Https2InvalidDNSPattern(Https2TestCase): - """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" - - keyfile = "keys/localhost.ip.key" - certfile = "keys/localhost.ip.crt" - - def setUp(self): - try: - from service_identity.exceptions import CertificateError # noqa: F401 - except ImportError: - raise unittest.SkipTest("cryptography lib is too old") - self.tls_log_message = ( - 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' - 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' - ) - super().setUp() +class Https2InvalidDNSPattern(BaseTestClasses.Https11InvalidDNSPattern): + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return _get_dh() -@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") -class Https2CustomCiphers(Https11CustomCiphers): - scheme = "https" - - @classmethod - def setUpClass(cls): - from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - - cls.download_handler_cls = H2DownloadHandler +class Https2CustomCiphers(BaseTestClasses.Https11CustomCiphers): + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return _get_dh() -class Http2MockServerTestCase(Http11MockServerTestCase): +class Http2MockServerTestCase(BaseTestClasses.Http11MockServerTestCase): """HTTP 2.0 test case with MockServer""" settings_dict = { @@ -212,10 +212,10 @@ class Http2MockServerTestCase(Http11MockServerTestCase): "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler" } } + is_secure = True -@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") -class Https2ProxyTestCase(Http11ProxyTestCase): +class Https2ProxyTestCase(BaseTestClasses.Http11ProxyTestCase): # only used for HTTPS tests keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" @@ -225,11 +225,9 @@ class Https2ProxyTestCase(Http11ProxyTestCase): expected_http_proxy_request_body = b"/" - @classmethod - def setUpClass(cls): - from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - - cls.download_handler_cls = H2DownloadHandler + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return _get_dh() def setUp(self): site = server.Site(UriResource(), timeout=None) diff --git a/tox.ini b/tox.ini index 0f91db19d..82ad84c90 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ deps = pexpect >= 4.8.0 pyftpdlib >= 2.0.1 pygments - pytest + pytest != 8.2.* # https://github.com/pytest-dev/pytest/issues/12275 pytest-cov >= 4.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 From 9d35428770326a3e833a2720c4f641fa70b58d29 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 5 Feb 2025 06:48:56 -0300 Subject: [PATCH 045/491] Remove deprecated signals --- scrapy/signals.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scrapy/signals.py b/scrapy/signals.py index 0090f1c8b..8ef0f34f0 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -24,12 +24,3 @@ item_dropped = object() item_error = object() feed_slot_closed = object() feed_exporter_closed = object() - -# for backward compatibility -stats_spider_opened = spider_opened -stats_spider_closing = spider_closed -stats_spider_closed = spider_closed - -item_passed = item_scraped - -request_received = request_scheduled From 2eb3c75c697685af595b08023b4dc27d49403274 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 5 Feb 2025 13:16:51 -0300 Subject: [PATCH 046/491] Remove AjaxCrawlMiddleware mention from built-in downloader middleware --- docs/topics/downloader-middleware.rst | 76 +++++++-------------------- 1 file changed, 20 insertions(+), 56 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 33308940c..ab7e6a0ec 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -763,6 +763,26 @@ HttpProxyMiddleware Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` environment variables, and it will also ignore ``no_proxy`` environment variable. +HttpProxyMiddleware settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. setting:: HTTPPROXY_ENABLED +.. setting:: HTTPPROXY_AUTH_ENCODING + +HTTPPROXY_ENABLED +^^^^^^^^^^^^^^^^^ + +Default: ``True`` + +Whether or not to enable the :class:`HttpProxyMiddleware`. + +HTTPPROXY_AUTH_ENCODING +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``"latin-1"`` + +The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. + OffsiteMiddleware ----------------- @@ -1220,60 +1240,4 @@ UserAgentMiddleware In order for a spider to override the default user agent, its ``user_agent`` attribute must be set. -.. _ajaxcrawl-middleware: - -AjaxCrawlMiddleware -------------------- - -.. module:: scrapy.downloadermiddlewares.ajaxcrawl - -.. class:: AjaxCrawlMiddleware - - Middleware that finds 'AJAX crawlable' page variants based - on meta-fragment html tag. - - .. note:: - - Scrapy finds 'AJAX crawlable' pages for URLs like - ``'http://example.com/!#foo=bar'`` even without this middleware. - AjaxCrawlMiddleware is necessary when URL doesn't contain ``'!#'``. - This is often a case for 'index' or 'main' website pages. - -AjaxCrawlMiddleware Settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. setting:: AJAXCRAWL_ENABLED - -AJAXCRAWL_ENABLED -^^^^^^^^^^^^^^^^^ - -Default: ``False`` - -Whether the AjaxCrawlMiddleware will be enabled. - - .. note:: - - This middleware is deprecated and will be removed in a future Scrapy release. - -HttpProxyMiddleware settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. setting:: HTTPPROXY_ENABLED -.. setting:: HTTPPROXY_AUTH_ENCODING - -HTTPPROXY_ENABLED -^^^^^^^^^^^^^^^^^ - -Default: ``True`` - -Whether or not to enable the :class:`HttpProxyMiddleware`. - -HTTPPROXY_AUTH_ENCODING -^^^^^^^^^^^^^^^^^^^^^^^ - -Default: ``"latin-1"`` - -The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. - - .. _DBM: https://en.wikipedia.org/wiki/Dbm From 4e0a3087e4f4f2bc118d0f09b71e7440e78c42d7 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 6 Feb 2025 07:47:39 -0300 Subject: [PATCH 047/491] fix: Reactor info logged twice (#6657) * fix: Reactor info logged twice * Change condition syntax * Simplify logic * Format --- scrapy/crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1aa68cb00..1873c90d3 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -120,12 +120,12 @@ class Crawler: install_reactor(reactor_class, event_loop) else: from twisted.internet import reactor # noqa: F401 - log_reactor_info() if reactor_class: verify_installed_reactor(reactor_class) if is_asyncio_reactor_installed() and event_loop: verify_installed_asyncio_event_loop(event_loop) + if self._init_reactor or reactor_class: log_reactor_info() self.extensions = ExtensionManager.from_crawler(self) From f041f26a6ff636b764d2bf584ddbc9b9e4334d1b Mon Sep 17 00:00:00 2001 From: anubhav Date: Thu, 6 Feb 2025 22:37:07 +0530 Subject: [PATCH 048/491] Support dark mode in the documentation (#6653) --- docs/_static/custom.css | 48 ++++++++++++++++++++++++++++++++++++++++- docs/conf.py | 3 +++ docs/requirements.txt | 1 + 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 64f16939c..1c2859deb 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -7,4 +7,50 @@ } .rst-content dl p + ol, .rst-content dl p + ul { margin-top: -6px; /* Compensates margin-top: 12px of p */ -} \ No newline at end of file +} + +/*override some styles in +sphinx-rtd-dark-mode/static/dark_mode_css/general.css*/ +.theme-switcher { + right: 0.4em !important; + top: 0.6em !important; + -webkit-box-shadow: 0px 3px 14px 4px rgba(0, 0, 0, 0.30) !important; + box-shadow: 0px 3px 14px 4px rgba(0, 0, 0, 0.30) !important; + height: 2em !important; + width: 2em !important; +} + +/*place the toggle button for dark mode +at the bottom right corner on small screens*/ +@media (max-width: 768px) { + .theme-switcher { + right: 0.4em !important; + bottom: 2.6em !important; + top: auto !important; + } +} + +/*persist blue color at the top left used in +default rtd theme*/ +html[data-theme="dark"] .wy-side-nav-search, +html[data-theme="dark"] .wy-nav-top { + background-color: #1d577d !important; +} + +/*all the styles below used to present +API objects nicely in dark mode*/ +html[data-theme="dark"] .sig.sig-object { + border-left-color: #3e4446 !important; + background-color: #202325 !important +} + +html[data-theme="dark"] .sig-name, +html[data-theme="dark"] .sig-prename, +html[data-theme="dark"] .property, +html[data-theme="dark"] .sig-param, +html[data-theme="dark"] .sig-paren, +html[data-theme="dark"] .sig-return-icon, +html[data-theme="dark"] .sig-return-typehint, +html[data-theme="dark"] .optional { + color: #e8e6e3 !important +} diff --git a/docs/conf.py b/docs/conf.py index be5e07195..1167ce050 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -35,6 +35,7 @@ extensions = [ "sphinx.ext.coverage", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", + "sphinx_rtd_dark_mode", ] templates_path = ["_templates"] @@ -174,3 +175,5 @@ hoverxref_role_types = { "signal": "tooltip", } hoverxref_roles = ["command", "reqmeta", "setting", "signal"] + +default_dark_mode = False diff --git a/docs/requirements.txt b/docs/requirements.txt index e2abe76d9..103fb08d6 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,3 +2,4 @@ sphinx==8.1.3 sphinx-hoverxref==1.4.2 sphinx-notfound-page==1.0.4 sphinx-rtd-theme==3.0.2 +sphinx-rtd-dark-mode==1.3.0 From d8978d405c32ee63375e09bf0b66b1e803da3d08 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 8 Feb 2025 18:41:27 +0500 Subject: [PATCH 049/491] Improve diagnostics for sync-only spider middlewares. --- scrapy/core/spidermw.py | 30 ++++++++++++++++++------------ tests/test_spidermiddleware.py | 16 ++++++++++------ 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 4b2520aa1..c7706bb7b 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -27,7 +27,7 @@ from scrapy.utils.defer import ( maybe_deferred_to_future, mustbe_deferred, ) -from scrapy.utils.python import MutableAsyncChain, MutableChain +from scrapy.utils.python import MutableAsyncChain, MutableChain, global_object_name if TYPE_CHECKING: from collections.abc import Generator @@ -51,10 +51,6 @@ def _isiterable(o: Any) -> bool: class SpiderMiddlewareManager(MiddlewareManager): component_name = "spider middleware" - def __init__(self, *middlewares: Any): - super().__init__(*middlewares) - self.downgrade_warning_done = False - @classmethod def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES")) @@ -227,12 +223,13 @@ class SpiderMiddlewareManager(MiddlewareManager): # Iterable -> AsyncIterable result = as_async_generator(result) elif need_downgrade: - if not self.downgrade_warning_done: - logger.warning( - f"Async iterable passed to {method.__qualname__} " - f"was downgraded to a non-async one" - ) - self.downgrade_warning_done = True + logger.warning( + f"Async iterable passed to {method.__qualname__} was" + f" downgraded to a non-async one. This is deprecated and will" + f" stop working in a future version of Scrapy. Please see" + f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares" + f" for more information." + ) assert isinstance(result, AsyncIterable) # AsyncIterable -> Iterable result = yield deferred_from_coro(collect_asyncgen(result)) @@ -340,10 +337,19 @@ class SpiderMiddlewareManager(MiddlewareManager): methodname_async = methodname + "_async" async_method: Callable | None = getattr(mw, methodname_async, None) if not async_method: + if normal_method and not isasyncgenfunction(normal_method): + logger.warning( + f"Middleware {global_object_name(mw.__class__)} doesn't support" + f" asynchronous spider output, this is deprecated and will stop" + f" working in a future version of Scrapy. The middleware should" + f" be updated to support it. Please see" + f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares" + f" for more information." + ) return normal_method if not normal_method: logger.error( - f"Middleware {mw.__qualname__} has {methodname_async} " + f"Middleware {global_object_name(mw.__class__)} has {methodname_async} " f"without {methodname}, skipping this method." ) return None diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index ba64ba721..a8507c789 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -152,6 +152,10 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): self.assertEqual(len(result_list), self.RESULT_COUNT) self.assertIsInstance(result_list[0], self.ITEM_TYPE) self.assertEqual("downgraded to a non-async" in str(log), downgrade) + self.assertEqual( + "doesn't support asynchronous spider output" in str(log), + ProcessSpiderOutputSimpleMiddleware in mw_classes, + ) @defer.inlineCallbacks def _test_asyncgen_base( @@ -376,21 +380,21 @@ class UniversalMiddlewareManagerTest(TestCase): self.mwman = SpiderMiddlewareManager() def test_simple_mw(self): - mw = ProcessSpiderOutputSimpleMiddleware + mw = ProcessSpiderOutputSimpleMiddleware() self.mwman._add_middleware(mw) self.assertEqual( self.mwman.methods["process_spider_output"][0], mw.process_spider_output ) def test_async_mw(self): - mw = ProcessSpiderOutputAsyncGenMiddleware + mw = ProcessSpiderOutputAsyncGenMiddleware() self.mwman._add_middleware(mw) self.assertEqual( self.mwman.methods["process_spider_output"][0], mw.process_spider_output ) def test_universal_mw(self): - mw = ProcessSpiderOutputUniversalMiddleware + mw = ProcessSpiderOutputUniversalMiddleware() self.mwman._add_middleware(mw) self.assertEqual( self.mwman.methods["process_spider_output"][0], @@ -399,7 +403,7 @@ class UniversalMiddlewareManagerTest(TestCase): def test_universal_mw_no_sync(self): with LogCapture() as log: - self.mwman._add_middleware(UniversalMiddlewareNoSync) + self.mwman._add_middleware(UniversalMiddlewareNoSync()) self.assertIn( "UniversalMiddlewareNoSync has process_spider_output_async" " without process_spider_output", @@ -408,7 +412,7 @@ class UniversalMiddlewareManagerTest(TestCase): self.assertEqual(self.mwman.methods["process_spider_output"][0], None) def test_universal_mw_both_sync(self): - mw = UniversalMiddlewareBothSync + mw = UniversalMiddlewareBothSync() with LogCapture() as log: self.mwman._add_middleware(mw) self.assertIn( @@ -422,7 +426,7 @@ class UniversalMiddlewareManagerTest(TestCase): def test_universal_mw_both_async(self): with LogCapture() as log: - self.mwman._add_middleware(UniversalMiddlewareBothAsync) + self.mwman._add_middleware(UniversalMiddlewareBothAsync()) self.assertIn( "UniversalMiddlewareBothAsync.process_spider_output " "is an async generator function while process_spider_output_async exists", From ede9e9c3c3f9a9049fea2a6be0339b2c7434b8a1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 11 Feb 2025 23:07:25 +0500 Subject: [PATCH 050/491] Use full method names in all spidermw log messages. --- scrapy/core/spidermw.py | 16 ++++++++-------- scrapy/utils/python.py | 4 +++- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index c7706bb7b..86d11c0e0 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -79,7 +79,7 @@ class SpiderMiddlewareManager(MiddlewareManager): result = method(response=response, spider=spider) if result is not None: msg = ( - f"{method.__qualname__} must return None " + f"{global_object_name(method)} must return None " f"or raise an exception, got {type(result)}" ) raise _InvalidOutput(msg) @@ -168,12 +168,12 @@ class SpiderMiddlewareManager(MiddlewareManager): ) # we forbid waiting here because otherwise we would need to return a deferred from # _process_spider_exception too, which complicates the architecture - msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" + msg = f"Async iterable returned from {global_object_name(method)} cannot be downgraded" raise _InvalidOutput(msg) if result is None: continue msg = ( - f"{method.__qualname__} must return None " + f"{global_object_name(method)} must return None " f"or an iterable, got {type(result)}" ) raise _InvalidOutput(msg) @@ -224,7 +224,7 @@ class SpiderMiddlewareManager(MiddlewareManager): result = as_async_generator(result) elif need_downgrade: logger.warning( - f"Async iterable passed to {method.__qualname__} was" + f"Async iterable passed to {global_object_name(method)} was" f" downgraded to a non-async one. This is deprecated and will" f" stop working in a future version of Scrapy. Please see" f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares" @@ -257,12 +257,12 @@ class SpiderMiddlewareManager(MiddlewareManager): if iscoroutine(result): result.close() # Silence warning about not awaiting msg = ( - f"{method.__qualname__} must be an asynchronous " + f"{global_object_name(method)} must be an asynchronous " f"generator (i.e. use yield)" ) else: msg = ( - f"{method.__qualname__} must return an iterable, got " + f"{global_object_name(method)} must return an iterable, got " f"{type(result)}" ) raise _InvalidOutput(msg) @@ -355,13 +355,13 @@ class SpiderMiddlewareManager(MiddlewareManager): return None if not isasyncgenfunction(async_method): logger.error( - f"{async_method.__qualname__} is not " + f"{global_object_name(async_method)} is not " f"an async generator function, skipping this method." ) return normal_method if isasyncgenfunction(normal_method): logger.error( - f"{normal_method.__qualname__} is an async " + f"{global_object_name(normal_method)} is an async " f"generator function while {methodname_async} exists, " f"skipping both methods." ) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index fcf582082..2e6869779 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -326,11 +326,13 @@ def without_none_values( def global_object_name(obj: Any) -> str: - """Return the full import path of the given class. + """Return the full import path of the given object. >>> from scrapy import Request >>> global_object_name(Request) 'scrapy.http.request.Request' + >>> global_object_name(Request.replace) + 'scrapy.http.request.Request.replace' """ return f"{obj.__module__}.{obj.__qualname__}" From 7d5b189c1147e8aad632d4ef6759cc391d2017ac Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Feb 2025 19:40:06 +0400 Subject: [PATCH 051/491] Fix getting annotations for _parse_sitemap() at the runtime. (#6671) * Fix getting annotations for _parse_sitemap() at the runtime. * Split off the callback annotations test. --- scrapy/spiders/sitemap.py | 5 +++-- tests/test_poet.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 tests/test_poet.py diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 91c7e3be9..39033ac3c 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -2,6 +2,9 @@ from __future__ import annotations import logging import re + +# Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation +from collections.abc import Iterable, Sequence # noqa: TC003 from typing import TYPE_CHECKING, Any, cast from scrapy.http import Request, Response, XmlResponse @@ -11,8 +14,6 @@ from scrapy.utils.gz import gunzip, gzip_magic_number from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots if TYPE_CHECKING: - from collections.abc import Iterable, Sequence - # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/tests/test_poet.py b/tests/test_poet.py new file mode 100644 index 000000000..9601c75a1 --- /dev/null +++ b/tests/test_poet.py @@ -0,0 +1,20 @@ +"""Tests that make sure parts needed for the scrapy-poet stack work.""" + +from typing import get_type_hints + +from scrapy import Spider +from scrapy.spiders import CrawlSpider, CSVFeedSpider, SitemapSpider, XMLFeedSpider + + +def test_callbacks(): + """Making sure annotations on all non-abstract callbacks can be resolved.""" + + for cb in [ + Spider._parse, + CrawlSpider._parse, + CrawlSpider._callback, + XMLFeedSpider._parse, + CSVFeedSpider._parse, + SitemapSpider._parse_sitemap, + ]: + get_type_hints(cb) From a898331d14f889c1d4860cf1a364ba28285090a4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 16 Feb 2025 23:28:58 +0400 Subject: [PATCH 052/491] Preparations for switching to direct pytest. (#6678) --- conftest.py | 2 +- pyproject.toml | 11 +++- scrapy/utils/test.py | 9 +-- tests/test_commands.py | 4 +- tests/test_contracts.py | 50 ++++++++--------- tests/test_crawl.py | 26 +++++---- tests/test_crawler.py | 36 ++++++------ tests/test_downloadermiddleware.py | 6 +- ...test_downloadermiddleware_ajaxcrawlable.py | 2 - tests/test_downloadermiddleware_httpauth.py | 12 ++-- tests/test_downloadermiddleware_httpcache.py | 6 +- tests/test_downloadermiddleware_httpproxy.py | 3 +- tests/test_downloadermiddleware_offsite.py | 28 +++++----- tests/test_downloadermiddleware_redirect.py | 4 +- tests/test_engine.py | 30 +++++----- tests/test_engine_stop_download_bytes.py | 4 +- tests/test_engine_stop_download_headers.py | 4 +- tests/test_exporters.py | 40 +++++++------- tests/test_extension_periodic_log.py | 4 +- tests/test_extension_throttle.py | 55 +++++++++---------- tests/test_http2_client_protocol.py | 3 +- tests/test_http_response.py | 12 ++-- tests/test_linkextractors.py | 4 +- tests/test_loader.py | 54 +++++++++--------- tests/test_loader_deprecated.py | 46 ++++++++-------- tests/test_middleware.py | 8 +-- tests/test_pipelines.py | 6 +- tests/test_request_dict.py | 28 ++++++---- tests/test_scheduler_base.py | 6 +- tests/test_signals.py | 4 +- tests/test_squeues.py | 26 ++++----- tests/test_utils_asyncio.py | 6 +- tests/test_utils_datatypes.py | 2 - tests/test_utils_defer.py | 4 +- tests/test_utils_log.py | 4 +- tests/test_utils_misc/__init__.py | 2 - tests/test_utils_python.py | 2 - tests/test_utils_response.py | 8 +-- tests/test_utils_signal.py | 6 +- tests/test_utils_template.py | 2 - tests/test_utils_url.py | 2 - 41 files changed, 285 insertions(+), 286 deletions(-) diff --git a/conftest.py b/conftest.py index e9765962a..a08ad9d05 100644 --- a/conftest.py +++ b/conftest.py @@ -41,7 +41,7 @@ if not H2_ENABLED: ) -@pytest.fixture() +@pytest.fixture def chdir(tmpdir): """Change to pytest-provided temporary directory""" tmpdir.chdir() diff --git a/pyproject.toml b/pyproject.toml index 1072730c0..ad62ea212 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -200,8 +200,7 @@ disable = [ [tool.pytest.ini_options] xfail_strict = true usefixtures = "chdir" -python_files = ["test_*.py", "__init__.py"] -python_classes = [] +python_files = ["test_*.py", "test_*/__init__.py"] addopts = [ "--assert=plain", "--ignore=docs/_ext", @@ -254,6 +253,8 @@ extend-select = [ "PIE", # pylint "PL", + # flake8-pytest-style + "PT", # flake8-use-pathlib "PTH", # flake8-pyi @@ -373,6 +374,12 @@ ignore = [ "B904", # Use capitalized environment variable "SIM112", + + # Temporarily silenced PT rules + # Use a regular `assert` instead of unittest-style `assertEqual` + "PT009", + # Use `pytest.raises` instead of unittest-style `assertRaises` + "PT027", ] [tool.ruff.lint.per-file-ignores] diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index a7b84baef..e89786103 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -15,9 +15,10 @@ from unittest import TestCase, mock from twisted.trial.unittest import SkipTest -from scrapy import Spider from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.boto import is_botocore_available +from scrapy.utils.deprecate import create_deprecated_class +from scrapy.utils.spider import DefaultSpider if TYPE_CHECKING: from collections.abc import Awaitable @@ -25,6 +26,7 @@ if TYPE_CHECKING: from twisted.internet.defer import Deferred from twisted.web.client import Response as TxResponse + from scrapy import Spider from scrapy.crawler import Crawler @@ -82,8 +84,7 @@ def get_ftp_content_and_delete( return b"".join(ftp_data) -class TestSpider(Spider): - name = "test" +TestSpider = create_deprecated_class("TestSpider", DefaultSpider) def get_crawler( @@ -101,7 +102,7 @@ def get_crawler( settings: dict[str, Any] = {} settings.update(settings_dict or {}) runner = CrawlerRunner(settings) - crawler = runner.create_crawler(spidercls or TestSpider) + crawler = runner.create_crawler(spidercls or DefaultSpider) crawler._apply_settings() return crawler diff --git a/tests/test_commands.py b/tests/test_commands.py index 872b54d04..1a0db1e03 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -19,7 +19,7 @@ from threading import Timer from typing import TYPE_CHECKING from unittest import mock, skipIf -from pytest import mark +import pytest from twisted.trial import unittest import scrapy @@ -822,7 +822,7 @@ class MySpider(scrapy.Spider): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log ) - @mark.requires_uvloop + @pytest.mark.requires_uvloop def test_custom_asyncio_loop_enabled_true(self): log = self.get_log( self.debug_log_spider, diff --git a/tests/test_contracts.py b/tests/test_contracts.py index fb16140be..0f7d7b54c 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -21,7 +21,7 @@ from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -class TestItem(Item): +class DemoItem(Item): name = Field() url = Field() @@ -58,7 +58,7 @@ class CustomFormContract(Contract): return args -class TestSpider(Spider): +class DemoSpider(Spider): name = "demo_spider" def returns_request(self, response): @@ -80,7 +80,7 @@ class TestSpider(Spider): @url http://scrapy.org @returns items 1 1 """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def returns_request_cb_kwargs(self, response, url): """method which returns request @@ -96,7 +96,7 @@ class TestSpider(Spider): @cb_kwargs {"name": "Scrapy"} @returns items 1 1 """ - return TestItem(name=name, url=response.url) + return DemoItem(name=name, url=response.url) def returns_item_cb_kwargs_error_unexpected_keyword(self, response): """method which returns item @@ -104,14 +104,14 @@ class TestSpider(Spider): @cb_kwargs {"arg": "value"} @returns items 1 1 """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def returns_item_cb_kwargs_error_missing_argument(self, response, arg): """method which returns item @url http://scrapy.org @returns items 1 1 """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def returns_dict_item(self, response): """method which returns item @@ -125,7 +125,7 @@ class TestSpider(Spider): @url http://scrapy.org @returns items 0 0 """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def returns_dict_fail(self, response): """method which returns item @@ -140,7 +140,7 @@ class TestSpider(Spider): @returns items 1 1 @scrapes name url """ - return TestItem(name="test", url=response.url) + return DemoItem(name="test", url=response.url) def scrapes_dict_item_ok(self, response): """returns item with name and url @@ -156,7 +156,7 @@ class TestSpider(Spider): @returns items 1 1 @scrapes name url """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def scrapes_dict_item_fail(self, response): """returns item with no name @@ -212,7 +212,7 @@ class TestSpider(Spider): @meta {"key": "example"} @returns items 1 1 """ - return TestItem(name="example", url=response.url) + return DemoItem(name="example", url=response.url) def returns_error_missing_meta(self, response): """method which depends of metadata be defined @@ -242,7 +242,7 @@ class CustomContractFailSpider(Spider): """ -class InheritsTestSpider(TestSpider): +class InheritsDemoSpider(DemoSpider): name = "inherits_demo_spider" @@ -274,7 +274,7 @@ class ContractsManagerTest(unittest.TestCase): self.assertTrue(self.results.errors) def test_contracts(self): - spider = TestSpider() + spider = DemoSpider() # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request) @@ -293,7 +293,7 @@ class ContractsManagerTest(unittest.TestCase): self.assertEqual(request, None) def test_cb_kwargs(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() # extract contracts correctly @@ -356,7 +356,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_error() def test_meta(self): - spider = TestSpider() + spider = DemoSpider() # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request_meta) @@ -402,7 +402,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_error() def test_returns(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() # returns_item @@ -431,7 +431,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_fail() def test_returns_async(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() request = self.conman.from_method(spider.returns_request_async, self.results) @@ -439,7 +439,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_error() def test_scrapes(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() # scrapes_item_ok @@ -472,7 +472,7 @@ class ContractsManagerTest(unittest.TestCase): assert message in self.results.failures[-1][-1] def test_regex(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() # invalid regex @@ -494,7 +494,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_error() def test_errback(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() try: @@ -522,11 +522,11 @@ class ContractsManagerTest(unittest.TestCase): def parse_first(self, response): self.visited += 1 - return TestItem() + return DemoItem() def parse_second(self, response): self.visited += 1 - return TestItem() + return DemoItem() with MockServer() as mockserver: contract_doc = f"@url {mockserver.url('/status?n=200')}" @@ -540,13 +540,13 @@ class ContractsManagerTest(unittest.TestCase): self.assertEqual(crawler.spider.visited, 2) def test_form_contract(self): - spider = TestSpider() + spider = DemoSpider() request = self.conman.from_method(spider.custom_form, self.results) self.assertEqual(request.method, "POST") self.assertIsInstance(request, FormRequest) def test_inherited_contracts(self): - spider = InheritsTestSpider() + spider = InheritsDemoSpider() requests = self.conman.from_spider(spider, self.results) self.assertTrue(requests) @@ -571,7 +571,7 @@ class CustomContractPrePostProcess(unittest.TestCase): self.results = TextTestResult(stream=None, descriptions=False, verbosity=0) def test_pre_hook_keyboard_interrupt(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() contract = CustomFailContractPreProcess(spider.returns_request) conman = ContractsManager([contract]) @@ -590,7 +590,7 @@ class CustomContractPrePostProcess(unittest.TestCase): self.assertFalse(self.results.errors) def test_post_hook_keyboard_interrupt(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() contract = CustomFailContractPostProcess(spider.returns_request) conman = ContractsManager([contract]) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index cd2a559a8..3aca2bbce 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -6,7 +6,7 @@ from ipaddress import IPv4Address from socket import gethostbyname from urllib.parse import urlparse -from pytest import mark +import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.internet.ssl import Certificate @@ -536,7 +536,7 @@ class CrawlSpiderTestCase(TestCase): ) self.assertIn("Got response 200", str(log)) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncio_parse(self): crawler = get_crawler( @@ -551,7 +551,7 @@ class CrawlSpiderTestCase(TestCase): ) self.assertIn("Got response 200", str(log)) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncio_parse_items_list(self): log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider) @@ -559,7 +559,7 @@ class CrawlSpiderTestCase(TestCase): self.assertIn({"id": 1}, items) self.assertIn({"id": 2}, items) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncio_parse_items_single_element(self): items = [] @@ -576,7 +576,7 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("Got response 200", str(log)) self.assertIn({"foo": 42}, items) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse(self): log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider) @@ -584,7 +584,7 @@ class CrawlSpiderTestCase(TestCase): itemcount = stats.get_value("item_scraped_count") self.assertEqual(itemcount, 1) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse_loop(self): log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider) @@ -594,7 +594,7 @@ class CrawlSpiderTestCase(TestCase): for i in range(10): self.assertIn({"foo": i}, items) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse_exc(self): log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider) @@ -606,7 +606,7 @@ class CrawlSpiderTestCase(TestCase): for i in range(7): self.assertIn({"foo": i}, items) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse_complex(self): _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider) @@ -618,20 +618,20 @@ class CrawlSpiderTestCase(TestCase): for i in [10, 30, 122]: self.assertIn({"index2": i}, items) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncio_parse_reqs_list(self): log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider) for req_id in range(3): self.assertIn(f"Got response 200, req_id {req_id}", str(log)) - @mark.only_not_asyncio() + @pytest.mark.only_not_asyncio @defer.inlineCallbacks def test_async_def_deferred_direct(self): _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider) self.assertEqual(items, [{"code": 200}]) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_deferred_wrapped(self): log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider) @@ -659,7 +659,9 @@ class CrawlSpiderTestCase(TestCase): self.assertEqual(cert.getSubject().commonName, b"localhost") self.assertEqual(cert.getIssuer().commonName, b"localhost") - @mark.xfail(reason="Responses with no body return early and contain no certificate") + @pytest.mark.xfail( + reason="Responses with no body return early and contain no certificate" + ) @defer.inlineCallbacks def test_response_ssl_certificate_empty_response(self): crawler = get_crawler(SingleRequestSpider) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 6c3fe96b0..425188d32 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -8,9 +8,9 @@ import warnings from pathlib import Path from typing import Any +import pytest from packaging.version import parse as parse_version from pexpect.popen_spawn import PopenSpawn -from pytest import mark, raises from twisted.internet.defer import Deferred, inlineCallbacks from twisted.trial import unittest from w3lib import __version__ as w3lib_version @@ -77,14 +77,14 @@ class CrawlerTestCase(BaseCrawlerTest): self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") def test_crawler_rejects_spider_objects(self): - with raises(ValueError): + with pytest.raises(ValueError, match="spidercls argument must be a class"): Crawler(DefaultSpider()) @inlineCallbacks def test_crawler_crawl_twice_unsupported(self): crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) yield crawler.crawl() - with raises(RuntimeError, match="more than once on the same instance"): + with pytest.raises(RuntimeError, match="more than once on the same instance"): yield crawler.crawl() def test_get_addon(self): @@ -203,7 +203,7 @@ class CrawlerTestCase(BaseCrawlerTest): raise crawler = get_raw_crawler(MySpider, BASE_SETTINGS) - with raises(RuntimeError): + with pytest.raises(RuntimeError): yield crawler.crawl() @inlineCallbacks @@ -282,7 +282,7 @@ class CrawlerTestCase(BaseCrawlerTest): raise crawler = get_raw_crawler(MySpider, BASE_SETTINGS) - with raises(RuntimeError): + with pytest.raises(RuntimeError): yield crawler.crawl() @inlineCallbacks @@ -361,7 +361,7 @@ class CrawlerTestCase(BaseCrawlerTest): raise crawler = get_raw_crawler(MySpider, BASE_SETTINGS) - with raises(RuntimeError): + with pytest.raises(RuntimeError): yield crawler.crawl() @inlineCallbacks @@ -440,7 +440,7 @@ class CrawlerTestCase(BaseCrawlerTest): raise crawler = get_raw_crawler(MySpider, BASE_SETTINGS) - with raises(RuntimeError): + with pytest.raises(RuntimeError): yield crawler.crawl() @@ -575,7 +575,7 @@ class NoRequestsSpider(scrapy.Spider): return [] -@mark.usefixtures("reactor_pytest") +@pytest.mark.usefixtures("reactor_pytest") class CrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): return CrawlerRunner() @@ -744,7 +744,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log ) - @mark.skipif( + @pytest.mark.skipif( parse_version(w3lib_version) >= parse_version("2.0.0"), reason="w3lib 2.0.0 and later do not allow invalid domains.", ) @@ -781,7 +781,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "Using reactor: twisted.internet.selectreactor.SelectReactor", log ) - @mark.skipif( + @pytest.mark.skipif( platform.system() == "Windows", reason="PollReactor is not supported on Windows" ) def test_twisted_reactor_poll(self): @@ -820,7 +820,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): log, ) - @mark.requires_uvloop + @pytest.mark.requires_uvloop def test_custom_loop_asyncio(self): log = self.run_script("asyncio_custom_loop.py") self.assertIn("Spider closed (finished)", log) @@ -829,7 +829,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - @mark.requires_uvloop + @pytest.mark.requires_uvloop def test_custom_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") self.assertIn("Spider closed (finished)", log) @@ -839,7 +839,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) - @mark.requires_uvloop + @pytest.mark.requires_uvloop def test_asyncio_enabled_reactor_same_loop(self): log = self.run_script("asyncio_enabled_reactor_same_loop.py") self.assertIn("Spider closed (finished)", log) @@ -848,7 +848,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - @mark.requires_uvloop + @pytest.mark.requires_uvloop def test_asyncio_enabled_reactor_different_loop(self): log = self.run_script("asyncio_enabled_reactor_different_loop.py") self.assertNotIn("Spider closed (finished)", log) @@ -924,13 +924,13 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("DEBUG: Using asyncio event loop", log) -@mark.parametrize( - ["settings", "items"], - ( +@pytest.mark.parametrize( + ("settings", "items"), + [ ({}, default_settings.LOG_VERSIONS), ({"LOG_VERSIONS": ["itemadapter"]}, ["itemadapter"]), ({"LOG_VERSIONS": []}, None), - ), + ], ) def test_log_scrapy_info(settings, items, caplog): with caplog.at_level("INFO"): diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index c581e7596..42051042c 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,7 +1,7 @@ import asyncio from unittest import mock -from pytest import mark +import pytest from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.python.failure import Failure @@ -220,7 +220,7 @@ class MiddlewareUsingDeferreds(ManagerTestCase): self.assertFalse(download_func.called) -@mark.usefixtures("reactor_pytest") +@pytest.mark.usefixtures("reactor_pytest") class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" @@ -243,7 +243,7 @@ class MiddlewareUsingCoro(ManagerTestCase): self.assertIs(results[0], resp) self.assertFalse(download_func.called) - @mark.only_asyncio() + @pytest.mark.only_asyncio def test_asyncdef_asyncio(self): resp = Response("http://example.com/index.html") diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 63bd158f6..76fcece4f 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -7,8 +7,6 @@ from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -__doctests__ = ["scrapy.downloadermiddlewares.ajaxcrawl"] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class AjaxCrawlMiddlewareTest(unittest.TestCase): diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 500af6536..581fc1974 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -7,18 +7,18 @@ from scrapy.http import Request from scrapy.spiders import Spider -class TestSpiderLegacy(Spider): +class LegacySpider(Spider): http_user = "foo" http_pass = "bar" -class TestSpider(Spider): +class DomainSpider(Spider): http_user = "foo" http_pass = "bar" http_auth_domain = "example.com" -class TestSpiderAny(Spider): +class AnyDomainSpider(Spider): http_user = "foo" http_pass = "bar" http_auth_domain = None @@ -26,7 +26,7 @@ class TestSpiderAny(Spider): class HttpAuthMiddlewareLegacyTest(unittest.TestCase): def setUp(self): - self.spider = TestSpiderLegacy("foo") + self.spider = LegacySpider("foo") def test_auth(self): with self.assertRaises(AttributeError): @@ -37,7 +37,7 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase): class HttpAuthMiddlewareTest(unittest.TestCase): def setUp(self): self.mw = HttpAuthMiddleware() - self.spider = TestSpider("foo") + self.spider = DomainSpider("foo") self.mw.spider_opened(self.spider) def tearDown(self): @@ -67,7 +67,7 @@ class HttpAuthMiddlewareTest(unittest.TestCase): class HttpAuthAnyMiddlewareTest(unittest.TestCase): def setUp(self): self.mw = HttpAuthMiddleware() - self.spider = TestSpiderAny("foo") + self.spider = AnyDomainSpider("foo") self.mw.spider_opened(self.spider) def tearDown(self): diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index a0886d9e9..74db93f8a 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -353,7 +353,8 @@ class RFC2616PolicyTest(DefaultStorageTest): resc = mw.storage.retrieve_response(self.spider, req0) if shouldcache: self.assertEqualResponse(resc, res1) - assert "cached" in res2.flags and res2.status != 304 + assert "cached" in res2.flags + assert res2.status != 304 else: self.assertFalse(resc) assert "cached" not in res2.flags @@ -376,7 +377,8 @@ class RFC2616PolicyTest(DefaultStorageTest): resc = mw.storage.retrieve_response(self.spider, req0) if shouldcache: self.assertEqualResponse(resc, res1) - assert "cached" in res2.flags and res2.status != 304 + assert "cached" in res2.flags + assert res2.status != 304 else: self.assertFalse(resc) assert "cached" not in res2.flags diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 97c276b48..f0826ef5b 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -131,7 +131,8 @@ class TestHttpProxyMiddleware(TestCase): mw = HttpProxyMiddleware() req = Request("http://noproxy.com", meta={"proxy": None}) assert mw.process_request(req, spider) is None - assert "proxy" in req.meta and req.meta["proxy"] is None + assert "proxy" in req.meta + assert req.meta["proxy"] is None def test_no_proxy(self): os.environ["http_proxy"] = "https://proxy.for.http:3128" diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index 23a1d06da..cace52a27 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -12,7 +12,7 @@ UNSET = object() @pytest.mark.parametrize( ("allowed_domain", "url", "allowed"), - ( + [ ("example.com", "http://example.com/1", True), ("example.com", "http://example.org/1", False), ("example.com", "http://sub.example.com/1", True), @@ -24,7 +24,7 @@ UNSET = object() ("example.com", "http://example.com.example", False), ("a.example", "http://nota.example", False), ("b.a.example", "http://notb.a.example", False), - ), + ], ) def test_process_request_domain_filtering(allowed_domain, url, allowed): crawler = get_crawler(Spider) @@ -41,12 +41,12 @@ def test_process_request_domain_filtering(allowed_domain, url, allowed): @pytest.mark.parametrize( ("value", "filtered"), - ( + [ (UNSET, True), (None, True), (False, True), (True, False), - ), + ], ) def test_process_request_dont_filter(value, filtered): crawler = get_crawler(Spider) @@ -66,7 +66,7 @@ def test_process_request_dont_filter(value, filtered): @pytest.mark.parametrize( ("allow_offsite", "dont_filter", "filtered"), - ( + [ (True, UNSET, False), (True, None, False), (True, False, False), @@ -75,7 +75,7 @@ def test_process_request_dont_filter(value, filtered): (False, None, True), (False, False, True), (False, True, False), - ), + ], ) def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): crawler = get_crawler(Spider) @@ -97,11 +97,11 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): @pytest.mark.parametrize( "value", - ( + [ UNSET, None, [], - ), + ], ) def test_process_request_no_allowed_domains(value): crawler = get_crawler(Spider) @@ -133,7 +133,7 @@ def test_process_request_invalid_domains(): @pytest.mark.parametrize( ("allowed_domain", "url", "allowed"), - ( + [ ("example.com", "http://example.com/1", True), ("example.com", "http://example.org/1", False), ("example.com", "http://sub.example.com/1", True), @@ -145,7 +145,7 @@ def test_process_request_invalid_domains(): ("example.com", "http://example.com.example", False), ("a.example", "http://nota.example", False), ("b.a.example", "http://notb.a.example", False), - ), + ], ) def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): crawler = get_crawler(Spider) @@ -162,12 +162,12 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): @pytest.mark.parametrize( ("value", "filtered"), - ( + [ (UNSET, True), (None, True), (False, True), (True, False), - ), + ], ) def test_request_scheduled_dont_filter(value, filtered): crawler = get_crawler(Spider) @@ -187,11 +187,11 @@ def test_request_scheduled_dont_filter(value, filtered): @pytest.mark.parametrize( "value", - ( + [ UNSET, None, [], - ), + ], ) def test_request_scheduled_no_allowed_domains(value): crawler = get_crawler(Spider) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index eb3cdfc11..f950906e9 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1278,7 +1278,7 @@ class MetaRefreshMiddlewareTest(Base.Test): @pytest.mark.parametrize( SCHEME_PARAMS, - ( + [ *REDIRECT_SCHEME_CASES, # data/file/ftp/s3/foo → * does not redirect *( @@ -1300,7 +1300,7 @@ class MetaRefreshMiddlewareTest(Base.Test): for scheme in NON_HTTP_SCHEMES for location in ("//example.com/b", "/b") ), - ), + ], ) def test_meta_refresh_schemes(url, location, target): crawler = get_crawler(Spider) diff --git a/tests/test_engine.py b/tests/test_engine.py index 91ce2c0de..e9470493f 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -42,7 +42,7 @@ from scrapy.utils.test import get_crawler from tests import get_testdata, tests_datadir -class TestItem(Item): +class MyItem(Item): name = Field() url = Field() price = Field() @@ -62,7 +62,7 @@ class DataClassItem: price: int = 0 -class TestSpider(Spider): +class MySpider(Spider): name = "scrapytest.org" allowed_domains = ["scrapytest.org", "localhost"] @@ -70,7 +70,7 @@ class TestSpider(Spider): name_re = re.compile(r"

(.*?)

", re.MULTILINE) price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE) - item_cls: type = TestItem + item_cls: type = MyItem def parse(self, response): xlink = LinkExtractor() @@ -91,24 +91,24 @@ class TestSpider(Spider): return adapter.item -class TestDupeFilterSpider(TestSpider): +class DupeFilterSpider(MySpider): def start_requests(self): return (Request(url) for url in self.start_urls) # no dont_filter=True -class DictItemsSpider(TestSpider): +class DictItemsSpider(MySpider): item_cls = dict -class AttrsItemsSpider(TestSpider): +class AttrsItemsSpider(MySpider): item_cls = AttrsItem -class DataClassItemsSpider(TestSpider): +class DataClassItemsSpider(MySpider): item_cls = DataClassItem -class ItemZeroDivisionErrorSpider(TestSpider): +class ItemZeroDivisionErrorSpider(MySpider): custom_settings = { "ITEM_PIPELINES": { "tests.pipelines.ProcessWithZeroDivisionErrorPipeline": 300, @@ -116,7 +116,7 @@ class ItemZeroDivisionErrorSpider(TestSpider): } -class ChangeCloseReasonSpider(TestSpider): +class ChangeCloseReasonSpider(MySpider): @classmethod def from_crawler(cls, crawler, *args, **kwargs): spider = cls(*args, **kwargs) @@ -388,7 +388,7 @@ class EngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( - TestSpider, + MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider, @@ -404,7 +404,7 @@ class EngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler_dupefilter(self): - run = CrawlerRun(TestDupeFilterSpider) + run = CrawlerRun(DupeFilterSpider) yield run.run() self._assert_scheduled_requests(run, count=8) self._assert_dropped_requests(run) @@ -426,13 +426,13 @@ class EngineTest(EngineTestBase): @defer.inlineCallbacks def test_close_downloader(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + e = ExecutionEngine(get_crawler(MySpider), lambda _: None) yield e.close() @defer.inlineCallbacks def test_start_already_running_exception(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) + e = ExecutionEngine(get_crawler(MySpider), lambda _: None) + yield e.open_spider(MySpider(), []) e.start() try: yield self.assertFailure(e.start(), RuntimeError).addBoth( @@ -486,7 +486,7 @@ def test_request_scheduled_signal(caplog): if "drop" in request.url: raise IgnoreRequest - spider = TestSpider() + spider = MySpider() crawler = get_crawler(spider.__class__) engine = ExecutionEngine(crawler, lambda _: None) engine.downloader._slot_gc_loop.stop() diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index 8bf225ab1..5dd04c310 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -8,7 +8,7 @@ from tests.test_engine import ( DataClassItemsSpider, DictItemsSpider, EngineTestBase, - TestSpider, + MySpider, ) @@ -22,7 +22,7 @@ class BytesReceivedEngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( - TestSpider, + MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider, diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index 4efb6b7a8..06929d1e4 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -8,7 +8,7 @@ from tests.test_engine import ( DataClassItemsSpider, DictItemsSpider, EngineTestBase, - TestSpider, + MySpider, ) @@ -22,7 +22,7 @@ class HeadersReceivedEngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( - TestSpider, + MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider, diff --git a/tests/test_exporters.py b/tests/test_exporters.py index c2cab9b2a..eb8d309b6 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -31,7 +31,7 @@ def custom_serializer(value): return str(int(value) + 2) -class TestItem(Item): +class MyItem(Item): name = Field() age = Field() @@ -42,7 +42,7 @@ class CustomFieldItem(Item): @dataclasses.dataclass -class TestDataClass: +class MyDataClass: name: str age: int @@ -54,7 +54,7 @@ class CustomFieldDataclass: class BaseItemExporterTest(unittest.TestCase): - item_class: type = TestItem + item_class: type = MyItem custom_field_item_class: type = CustomFieldItem def setUp(self): @@ -138,7 +138,7 @@ class BaseItemExporterTest(unittest.TestCase): class BaseItemExporterDataclassTest(BaseItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -207,7 +207,7 @@ class PythonItemExporterTest(BaseItemExporterTest): class PythonItemExporterDataclassTest(PythonItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -222,7 +222,7 @@ class PprintItemExporterTest(BaseItemExporterTest): class PprintItemExporterDataclassTest(PprintItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -259,7 +259,7 @@ class PickleItemExporterTest(BaseItemExporterTest): class PickleItemExporterDataclassTest(PickleItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -286,7 +286,7 @@ class MarshalItemExporterTest(BaseItemExporterTest): class MarshalItemExporterDataclassTest(MarshalItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -406,7 +406,7 @@ class CsvItemExporterTest(BaseItemExporterTest): class CsvItemExporterDataclassTest(CsvItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -517,7 +517,7 @@ class XmlItemExporterTest(BaseItemExporterTest): class XmlItemExporterDataclassTest(XmlItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -563,7 +563,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -595,11 +595,11 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertTwoItemsExported(ItemAdapter(self.i).asdict()) def test_two_items_with_failure_between(self): - i1 = TestItem(name="Joseph\xa3", age="22") - i2 = TestItem( + i1 = MyItem(name="Joseph\xa3", age="22") + i2 = MyItem( name="Maria", age=1j ) # Invalid datetimes didn't consistently fail between Python versions - i3 = TestItem(name="Jesus", age="44") + i3 = MyItem(name="Jesus", age="44") self.ie.start_exporting() self.ie.export_item(i1) self.assertRaises(TypeError, self.ie.export_item, i2) @@ -652,9 +652,9 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest): return JsonItemExporter(self.output, **kwargs) def test_two_items_with_failure_between(self): - i1 = TestItem(name="Joseph", age="22") - i2 = TestItem(name="\u263a", age="11") - i3 = TestItem(name="Jesus", age="44") + i1 = MyItem(name="Joseph", age="22") + i2 = MyItem(name="\u263a", age="11") + i3 = MyItem(name="Jesus", age="44") self.ie.start_exporting() self.ie.export_item(i1) self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2) @@ -665,12 +665,12 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest): class JsonItemExporterDataclassTest(JsonItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass class CustomExporterItemTest(unittest.TestCase): - item_class: type = TestItem + item_class: type = MyItem def setUp(self): if self.item_class is None: @@ -700,4 +700,4 @@ class CustomExporterItemTest(unittest.TestCase): class CustomExporterDataclassTest(CustomExporterItemTest): - item_class = TestDataClass + item_class = MyDataClass diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 15129e31f..ca5ffdc26 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -51,7 +51,7 @@ stats_dump_2 = { } -class TestExtPeriodicLog(PeriodicLog): +class CustomPeriodicLog(PeriodicLog): def set_a(self): self.stats._stats = stats_dump_1 @@ -62,7 +62,7 @@ class TestExtPeriodicLog(PeriodicLog): def extension(settings=None): crawler = Crawler(MetaSpider, settings=settings) crawler._apply_settings() - return TestExtPeriodicLog.from_crawler(crawler) + return CustomPeriodicLog.from_crawler(crawler) class TestPeriodicLog(unittest.TestCase): diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index f2c9dc063..4874f284a 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -13,15 +13,12 @@ from scrapy.settings.default_settings import ( DOWNLOAD_DELAY, ) from scrapy.utils.misc import build_from_crawler +from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler as _get_crawler UNSET = object() -class TestSpider(Spider): - name = "test" - - def get_crawler(settings=None, spidercls=None): settings = settings or {} settings["AUTOTHROTTLE_ENABLED"] = True @@ -30,11 +27,11 @@ def get_crawler(settings=None, spidercls=None): @pytest.mark.parametrize( ("value", "expected"), - ( + [ (UNSET, False), (False, False), (True, True), - ), + ], ) def test_enabled(value, expected): settings = {} @@ -50,10 +47,10 @@ def test_enabled(value, expected): @pytest.mark.parametrize( "value", - ( + [ 0.0, -1.0, - ), + ], ) def test_target_concurrency_invalid(value): settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": value} @@ -64,13 +61,13 @@ def test_target_concurrency_invalid(value): @pytest.mark.parametrize( ("spider", "setting", "expected"), - ( + [ (UNSET, UNSET, DOWNLOAD_DELAY), (1.0, UNSET, 1.0), (UNSET, 1.0, 1.0), (1.0, 2.0, 1.0), (3.0, 2.0, 3.0), - ), + ], ) def test_mindelay_definition(spider, setting, expected): settings = {} @@ -91,10 +88,10 @@ def test_mindelay_definition(spider, setting, expected): @pytest.mark.parametrize( ("value", "expected"), - ( + [ (UNSET, AUTOTHROTTLE_MAX_DELAY), (1.0, 1.0), - ), + ], ) def test_maxdelay_definition(value, expected): settings = {} @@ -102,13 +99,13 @@ def test_maxdelay_definition(value, expected): settings["AUTOTHROTTLE_MAX_DELAY"] = value crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - at._spider_opened(TestSpider()) + at._spider_opened(DefaultSpider()) assert at.maxdelay == expected @pytest.mark.parametrize( ("min_spider", "min_setting", "start_setting", "expected"), - ( + [ (UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), @@ -134,7 +131,7 @@ def test_maxdelay_definition(value, expected): AUTOTHROTTLE_START_DELAY + 2.0, AUTOTHROTTLE_START_DELAY + 2.0, ), - ), + ], ) def test_startdelay_definition(min_spider, min_setting, start_setting, expected): settings = {} @@ -158,7 +155,7 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) @pytest.mark.parametrize( ("meta", "slot"), - ( + [ ({}, None), ({"download_latency": 1.0}, None), ({"download_slot": "foo"}, None), @@ -172,12 +169,12 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) }, "foo", ), - ), + ], ) def test_skipped(meta, slot): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) request = Request("https://example.com", meta=meta) @@ -193,7 +190,7 @@ def test_skipped(meta, slot): @pytest.mark.parametrize( ("download_latency", "target_concurrency", "slot_delay", "expected"), - ( + [ (2.0, 2.0, 1.0, 1.0), (1.0, 2.0, 1.0, 0.75), (4.0, 2.0, 1.0, 2.0), @@ -201,13 +198,13 @@ def test_skipped(meta, slot): (2.0, 4.0, 1.0, 0.75), (2.0, 2.0, 0.5, 1.0), (2.0, 2.0, 2.0, 1.5), - ), + ], ) def test_adjustment(download_latency, target_concurrency, slot_delay, expected): settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) @@ -227,11 +224,11 @@ def test_adjustment(download_latency, target_concurrency, slot_delay, expected): @pytest.mark.parametrize( ("mindelay", "maxdelay", "expected"), - ( + [ (0.5, 2.0, 1.0), (0.25, 0.5, 0.5), (2.0, 4.0, 2.0), - ), + ], ) def test_adjustment_limits(mindelay, maxdelay, expected): download_latency, target_concurrency, slot_delay = (2.0, 2.0, 1.0) @@ -243,7 +240,7 @@ def test_adjustment_limits(mindelay, maxdelay, expected): } crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) @@ -263,11 +260,11 @@ def test_adjustment_limits(mindelay, maxdelay, expected): @pytest.mark.parametrize( ("download_latency", "target_concurrency", "slot_delay", "expected"), - ( + [ (2.0, 2.0, 1.0, 1.0), (1.0, 2.0, 1.0, 1.0), # Instead of 0.75 (4.0, 2.0, 1.0, 2.0), - ), + ], ) def test_adjustment_bad_response( download_latency, target_concurrency, slot_delay, expected @@ -275,7 +272,7 @@ def test_adjustment_bad_response( settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) @@ -297,7 +294,7 @@ def test_debug(caplog): settings = {"AUTOTHROTTLE_DEBUG": True} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": 1.0, "download_slot": "foo"} request = Request("https://example.com", meta=meta) @@ -327,7 +324,7 @@ def test_debug(caplog): def test_debug_disabled(caplog): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": 1.0, "download_slot": "foo"} request = Request("https://example.com", meta=meta) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 1f998de1a..ddc772236 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -258,7 +258,8 @@ class Https2ClientProtocolTestCase(TestCase): :param path: Should have / at the starting compulsorily if not empty :return: Complete url """ - assert len(path) > 0 and (path[0] == "/" or path[0] == "&") + assert len(path) > 0 + assert path[0] == "/" or path[0] == "&" return f"{self.scheme}://{self.hostname}:{self.port_number}{path}" def make_request(self, request: Request) -> Deferred: diff --git a/tests/test_http_response.py b/tests/test_http_response.py index b157e9802..dde883451 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -2,8 +2,8 @@ import codecs import unittest from unittest import mock +import pytest from packaging.version import Version as parse_version -from pytest import mark from w3lib import __version__ as w3lib_version from w3lib.encoding import resolve_encoding @@ -218,7 +218,7 @@ class BaseResponseTest(unittest.TestCase): r = self.response_class("http://example.com") self.assertRaises(ValueError, r.follow, None) - @mark.xfail( + @pytest.mark.xfail( parse_version(w3lib_version) < parse_version("2.1.1"), reason="https://github.com/scrapy/w3lib/pull/207", strict=True, @@ -226,7 +226,7 @@ class BaseResponseTest(unittest.TestCase): def test_follow_whitespace_url(self): self._assert_followed_url("foo ", "http://example.com/foo") - @mark.xfail( + @pytest.mark.xfail( parse_version(w3lib_version) < parse_version("2.1.1"), reason="https://github.com/scrapy/w3lib/pull/207", strict=True, @@ -473,10 +473,8 @@ class TextResponseTest(BaseResponseTest): self._assert_response_encoding(r5, "utf-8") self._assert_response_encoding(r8, "utf-8") self._assert_response_encoding(r9, "cp1252") - assert ( - r4._body_inferred_encoding() is not None - and r4._body_inferred_encoding() != "ascii" - ) + assert r4._body_inferred_encoding() is not None + assert r4._body_inferred_encoding() != "ascii" self._assert_response_values(r1, "utf-8", "\xa3") self._assert_response_values(r2, "utf-8", "\xa3") self._assert_response_values(r3, "iso-8859-1", "\xa3") diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index a83cfb56c..e751e0a63 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -4,8 +4,8 @@ import pickle import re import unittest +import pytest from packaging.version import Version -from pytest import mark from w3lib import __version__ as w3lib_version from scrapy.http import HtmlResponse, XmlResponse @@ -930,7 +930,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): ], ) - @mark.skipif( + @pytest.mark.skipif( Version(w3lib_version) < Version("2.0.0"), reason=( "Before w3lib 2.0.0, w3lib.url.safe_url_string would not complain " diff --git a/tests/test_loader.py b/tests/test_loader.py index 824d7aecf..b52d5ea2e 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -18,12 +18,12 @@ class NameItem(Item): name = Field() -class TestItem(NameItem): +class SummaryItem(NameItem): url = Field() summary = Field() -class TestNestedItem(Item): +class NestedItem(Item): name = Field() name_div = Field() name_value = Field() @@ -38,20 +38,20 @@ class AttrsNameItem: @dataclasses.dataclass -class TestDataClass: +class NameDataClass: name: list = dataclasses.field(default_factory=list) # test item loaders class NameItemLoader(ItemLoader): - default_item_class = TestItem + default_item_class = SummaryItem class NestedItemLoader(ItemLoader): - default_item_class = TestNestedItem + default_item_class = NestedItem -class TestItemLoader(NameItemLoader): +class ProcessorItemLoader(NameItemLoader): name_in = MapCompose(lambda v: v.title()) @@ -68,11 +68,11 @@ def processor_with_args(value, other=None, loader_context=None): class BasicItemLoaderTest(unittest.TestCase): def test_add_value_on_unknown_field(self): - il = TestItemLoader() + il = ProcessorItemLoader() self.assertRaises(KeyError, il.add_value, "wrong_field", ["lala", "lolo"]) def test_load_item_using_default_loader(self): - i = TestItem() + i = SummaryItem() i["summary"] = "lala" il = ItemLoader(item=i) il.add_value("name", "marta") @@ -82,7 +82,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(item["name"], ["marta"]) def test_load_item_using_custom_loader(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", "marta") item = il.load_item() self.assertEqual(item["name"], ["Marta"]) @@ -194,7 +194,7 @@ class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase): - item_class = TestDataClass + item_class = NameDataClass class BaseNoInputReprocessingLoader(ItemLoader): @@ -289,11 +289,11 @@ class SelectortemLoaderTest(unittest.TestCase): ) def test_init_method(self): - l = TestItemLoader() + l = ProcessorItemLoader() self.assertEqual(l.selector, None) def test_init_method_errors(self): - l = TestItemLoader() + l = ProcessorItemLoader() self.assertRaises(RuntimeError, l.add_xpath, "url", "//a/@href") self.assertRaises(RuntimeError, l.replace_xpath, "url", "//a/@href") self.assertRaises(RuntimeError, l.get_xpath, "//a/@href") @@ -303,7 +303,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method_with_selector(self): sel = Selector(text="
marta
") - l = TestItemLoader(selector=sel) + l = ProcessorItemLoader(selector=sel) self.assertIs(l.selector, sel) l.add_xpath("name", "//div/text()") @@ -311,7 +311,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method_with_selector_css(self): sel = Selector(text="
marta
") - l = TestItemLoader(selector=sel) + l = ProcessorItemLoader(selector=sel) self.assertIs(l.selector, sel) l.add_css("name", "div::text") @@ -320,18 +320,18 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method_with_base_response(self): """Selector should be None after initialization""" response = Response("https://scrapy.org") - l = TestItemLoader(response=response) + l = ProcessorItemLoader(response=response) self.assertIs(l.selector, None) def test_init_method_with_response(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath("name", "//div/text()") self.assertEqual(l.get_output_value("name"), ["Marta"]) def test_init_method_with_response_css(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css("name", "div::text") @@ -350,12 +350,12 @@ class SelectortemLoaderTest(unittest.TestCase): ) def test_add_xpath_re(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) l.add_xpath("name", "//div/text()", re="ma") self.assertEqual(l.get_output_value("name"), ["Ma"]) def test_replace_xpath(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath("name", "//div/text()") self.assertEqual(l.get_output_value("name"), ["Marta"]) @@ -366,7 +366,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"]) def test_get_xpath(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertEqual(l.get_xpath("//p/text()"), ["paragraph"]) self.assertEqual(l.get_xpath("//p/text()", TakeFirst()), "paragraph") self.assertEqual(l.get_xpath("//p/text()", TakeFirst(), re="pa"), "pa") @@ -376,14 +376,14 @@ class SelectortemLoaderTest(unittest.TestCase): ) def test_replace_xpath_multi_fields(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) l.add_xpath(None, "//div/text()", TakeFirst(), lambda x: {"name": x}) self.assertEqual(l.get_output_value("name"), ["Marta"]) l.replace_xpath(None, "//p/text()", TakeFirst(), lambda x: {"name": x}) self.assertEqual(l.get_output_value("name"), ["Paragraph"]) def test_replace_xpath_re(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath("name", "//div/text()") self.assertEqual(l.get_output_value("name"), ["Marta"]) @@ -391,7 +391,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value("name"), ["Ma"]) def test_add_css_re(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) l.add_css("name", "div::text", re="ma") self.assertEqual(l.get_output_value("name"), ["Ma"]) @@ -399,7 +399,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"]) def test_replace_css(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css("name", "div::text") self.assertEqual(l.get_output_value("name"), ["Marta"]) @@ -415,7 +415,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) def test_get_css(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertEqual(l.get_css("p::text"), ["paragraph"]) self.assertEqual(l.get_css("p::text", TakeFirst()), "paragraph") self.assertEqual(l.get_css("p::text", TakeFirst(), re="pa"), "pa") @@ -427,7 +427,7 @@ class SelectortemLoaderTest(unittest.TestCase): ) def test_replace_css_multi_fields(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) l.add_css(None, "div::text", TakeFirst(), lambda x: {"name": x}) self.assertEqual(l.get_output_value("name"), ["Marta"]) l.replace_css(None, "p::text", TakeFirst(), lambda x: {"name": x}) @@ -439,7 +439,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) def test_replace_css_re(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css("url", "a::attr(href)") self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 8d4bd6bc1..1e504f539 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -24,17 +24,17 @@ class NameItem(Item): name = Field() -class TestItem(NameItem): +class SummaryItem(NameItem): url = Field() summary = Field() # test item loaders class NameItemLoader(ItemLoader): - default_item_class = TestItem + default_item_class = SummaryItem -class TestItemLoader(NameItemLoader): +class ProcessorItemLoader(NameItemLoader): name_in = MapCompose(lambda v: v.title()) @@ -51,7 +51,7 @@ def processor_with_args(value, other=None, loader_context=None): class BasicItemLoaderTest(unittest.TestCase): def test_load_item_using_default_loader(self): - i = TestItem() + i = SummaryItem() i["summary"] = "lala" il = ItemLoader(item=i) il.add_value("name", "marta") @@ -61,7 +61,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(item["name"], ["marta"]) def test_load_item_using_custom_loader(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", "marta") item = il.load_item() self.assertEqual(item["name"], ["Marta"]) @@ -125,7 +125,7 @@ class BasicItemLoaderTest(unittest.TestCase): ) def test_add_value(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", "marta") self.assertEqual(il.get_collected_values("name"), ["Marta"]) self.assertEqual(il.get_output_value("name"), ["Marta"]) @@ -146,7 +146,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_collected_values("name"), [0]) def test_replace_value(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.replace_value("name", "marta") self.assertEqual(il.get_collected_values("name"), ["Marta"]) self.assertEqual(il.get_output_value("name"), ["Marta"]) @@ -229,7 +229,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["mart"]) def test_input_processor_inheritance(self): - class ChildItemLoader(TestItemLoader): + class ChildItemLoader(ProcessorItemLoader): url_in = MapCompose(lambda v: v.lower()) il = ChildItemLoader() @@ -265,8 +265,8 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["marta"]) def test_extend_custom_input_processors(self): - class ChildItemLoader(TestItemLoader): - name_in = MapCompose(TestItemLoader.name_in, str.swapcase) + class ChildItemLoader(ProcessorItemLoader): + name_in = MapCompose(ProcessorItemLoader.name_in, str.swapcase) il = ChildItemLoader() il.add_value("name", "marta") @@ -283,11 +283,11 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["MART"]) def test_output_processor_using_function(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", ["mar", "ta"]) self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - class TakeFirstItemLoader(TestItemLoader): + class TakeFirstItemLoader(ProcessorItemLoader): name_out = " ".join il = TakeFirstItemLoader() @@ -296,7 +296,7 @@ class BasicItemLoaderTest(unittest.TestCase): def test_output_processor_error(self): class TestItemLoader(ItemLoader): - default_item_class = TestItem + default_item_class = SummaryItem name_out = MapCompose(float) il = TestItemLoader() @@ -319,18 +319,18 @@ class BasicItemLoaderTest(unittest.TestCase): assert expected_exc_str in s, s def test_output_processor_using_classes(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", ["mar", "ta"]) self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - class TakeFirstItemLoader(TestItemLoader): + class TakeFirstItemLoader(ProcessorItemLoader): name_out = Join() il = TakeFirstItemLoader() il.add_value("name", ["mar", "ta"]) self.assertEqual(il.get_output_value("name"), "Mar Ta") - class TakeFirstItemLoader2(TestItemLoader): + class TakeFirstItemLoader2(ProcessorItemLoader): name_out = Join("
") il = TakeFirstItemLoader2() @@ -338,11 +338,11 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), "Mar
Ta") def test_default_output_processor(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", ["mar", "ta"]) self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - class LalaItemLoader(TestItemLoader): + class LalaItemLoader(ProcessorItemLoader): default_output_processor = Identity() il = LalaItemLoader() @@ -350,7 +350,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) def test_loader_context_on_declaration(self): - class ChildItemLoader(TestItemLoader): + class ChildItemLoader(ProcessorItemLoader): url_in = MapCompose(processor_with_args, key="val") il = ChildItemLoader() @@ -360,7 +360,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("url"), ["val"]) def test_loader_context_on_instantiation(self): - class ChildItemLoader(TestItemLoader): + class ChildItemLoader(ProcessorItemLoader): url_in = MapCompose(processor_with_args) il = ChildItemLoader(key="val") @@ -370,7 +370,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("url"), ["val"]) def test_loader_context_on_assign(self): - class ChildItemLoader(TestItemLoader): + class ChildItemLoader(ProcessorItemLoader): url_in = MapCompose(processor_with_args) il = ChildItemLoader() @@ -384,10 +384,10 @@ class BasicItemLoaderTest(unittest.TestCase): def processor(value, loader_context): return loader_context["item"]["name"] - class ChildItemLoader(TestItemLoader): + class ChildItemLoader(ProcessorItemLoader): url_in = MapCompose(processor) - it = TestItem(name="marta") + it = SummaryItem(name="marta") il = ChildItemLoader(item=it) il.add_value("url", "text") self.assertEqual(il.get_output_value("url"), ["marta"]) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 3a1cf19ad..0cc532570 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -40,7 +40,7 @@ class MOff: raise NotConfigured("foo") -class TestMiddlewareManager(MiddlewareManager): +class MyMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): return [M1, MOff, M3] @@ -54,7 +54,7 @@ class TestMiddlewareManager(MiddlewareManager): class MiddlewareManagerTest(unittest.TestCase): def test_init(self): m1, m2, m3 = M1(), M2(), M3() - mwman = TestMiddlewareManager(m1, m2, m3) + mwman = MyMiddlewareManager(m1, m2, m3) self.assertEqual( list(mwman.methods["open_spider"]), [m1.open_spider, m2.open_spider] ) @@ -64,7 +64,7 @@ class MiddlewareManagerTest(unittest.TestCase): self.assertEqual(list(mwman.methods["process"]), [m1.process, m3.process]) def test_methods(self): - mwman = TestMiddlewareManager(M1(), M2(), M3()) + mwman = MyMiddlewareManager(M1(), M2(), M3()) self.assertEqual( [x.__self__.__class__ for x in mwman.methods["open_spider"]], [M1, M2] ) @@ -82,6 +82,6 @@ class MiddlewareManagerTest(unittest.TestCase): def test_enabled_from_settings(self): crawler = get_crawler() - mwman = TestMiddlewareManager.from_crawler(crawler) + mwman = MyMiddlewareManager.from_crawler(crawler) classes = [x.__class__ for x in mwman.middlewares] self.assertEqual(classes, [M1, M3]) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 222b19e7f..0ae86235c 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -1,6 +1,6 @@ import asyncio -from pytest import mark +import pytest from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.trial import unittest @@ -118,14 +118,14 @@ class PipelineTestCase(unittest.TestCase): yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(self.items), 1) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_asyncdef_asyncio_pipeline(self): crawler = self._create_crawler(AsyncDefAsyncioPipeline) yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(self.items), 1) - @mark.only_not_asyncio() + @pytest.mark.only_not_asyncio @defer.inlineCallbacks def test_asyncdef_not_asyncio_pipeline(self): crawler = self._create_crawler(AsyncDefNotAsyncioPipeline) diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 854805cf7..85133038a 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -11,7 +11,7 @@ class CustomRequest(Request): class RequestSerializationTest(unittest.TestCase): def setUp(self): - self.spider = TestSpider() + self.spider = MethodsSpider() def test_basic(self): r = Request("http://www.example.com") @@ -96,18 +96,22 @@ class RequestSerializationTest(unittest.TestCase): def test_private_reference_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._TestSpider__parse_item_reference, - errback=self.spider._TestSpider__handle_error_reference, + callback=self.spider._MethodsSpider__parse_item_reference, + errback=self.spider._MethodsSpider__handle_error_reference, ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) - self.assertEqual(request_dict["callback"], "_TestSpider__parse_item_reference") - self.assertEqual(request_dict["errback"], "_TestSpider__handle_error_reference") + self.assertEqual( + request_dict["callback"], "_MethodsSpider__parse_item_reference" + ) + self.assertEqual( + request_dict["errback"], "_MethodsSpider__handle_error_reference" + ) def test_private_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._TestSpider__parse_item_private, + callback=self.spider._MethodsSpider__parse_item_private, errback=self.spider.handle_error, ) self._assert_serializes_ok(r, spider=self.spider) @@ -115,7 +119,7 @@ class RequestSerializationTest(unittest.TestCase): def test_mixin_private_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._TestSpiderMixin__mixin_callback, + callback=self.spider._SpiderMixin__mixin_callback, errback=self.spider.handle_error, ) self._assert_serializes_ok(r, spider=self.spider) @@ -152,18 +156,18 @@ class RequestSerializationTest(unittest.TestCase): def test_callback_not_available(self): """Callback method is not available in the spider passed to from_dict""" - spider = TestSpiderDelegation() + spider = SpiderDelegation() r = Request("http://www.example.com", callback=spider.delegated_callback) d = r.to_dict(spider=spider) self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo")) -class TestSpiderMixin: +class SpiderMixin: def __mixin_callback(self, response): # pylint: disable=unused-private-member pass -class TestSpiderDelegation: +class SpiderDelegation: def delegated_callback(self, response): pass @@ -184,7 +188,7 @@ def private_handle_error(failure): pass -class TestSpider(Spider, TestSpiderMixin): +class MethodsSpider(Spider, SpiderMixin): name = "test" parse_item_reference = parse_item handle_error_reference = handle_error @@ -193,7 +197,7 @@ class TestSpider(Spider, TestSpiderMixin): def __init__(self, **kwargs): super().__init__(**kwargs) - self.delegated_callback = TestSpiderDelegation().delegated_callback + self.delegated_callback = SpiderDelegation().delegated_callback def parse_item(self, response): pass diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index b48a65e67..7c72805e2 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -51,8 +51,8 @@ class SimpleScheduler(MinimalScheduler): return len(self.requests) -class TestSpider(Spider): - name = "test" +class PathsSpider(Spider): + name = "paths" def __init__(self, mockserver, *args, **kwargs): super().__init__(*args, **kwargs) @@ -155,7 +155,7 @@ class MinimalSchedulerCrawlTest(TwistedTestCase): "SCHEDULER": self.scheduler_cls, } with LogCapture() as log: - crawler = get_crawler(TestSpider, settings) + crawler = get_crawler(PathsSpider, settings) yield crawler.crawl(mockserver) for path in PATHS: self.assertIn(f"{{'path': '{path}'}}", str(log)) diff --git a/tests/test_signals.py b/tests/test_signals.py index 1e693c094..a508eb41a 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -1,4 +1,4 @@ -from pytest import mark +import pytest from twisted.internet import defer from twisted.trial import unittest @@ -37,7 +37,7 @@ class AsyncSignalTestCase(unittest.TestCase): item = await get_from_asyncio_queue(item) self.items.append(item) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_simple_pipeline(self): crawler = get_crawler(ItemSpider) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 4ce7cc9a4..a2e7ae65d 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -15,7 +15,7 @@ from scrapy.squeues import ( ) -class TestItem(Item): +class MyItem(Item): name = Field() @@ -23,8 +23,8 @@ def _test_procesor(x): return x + x -class TestLoader(ItemLoader): - default_item_class = TestItem +class MyLoader(ItemLoader): + default_item_class = MyItem name_out = staticmethod(_test_procesor) @@ -80,19 +80,19 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): def test_serialize_item(self): q = self.queue() - i = TestItem(name="foo") + i = MyItem(name="foo") q.push(i) i2 = q.pop() - assert isinstance(i2, TestItem) + assert isinstance(i2, MyItem) self.assertEqual(i, i2) def test_serialize_loader(self): q = self.queue() - loader = TestLoader() + loader = MyLoader() q.push(loader) loader2 = q.pop() - assert isinstance(loader2, TestLoader) - assert loader2.default_item_class is TestItem + assert isinstance(loader2, MyLoader) + assert loader2.default_item_class is MyItem self.assertEqual(loader2.name_out("x"), "xx") def test_serialize_request_recursive(self): @@ -161,19 +161,19 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): def test_serialize_item(self): q = self.queue() - i = TestItem(name="foo") + i = MyItem(name="foo") q.push(i) i2 = q.pop() - assert isinstance(i2, TestItem) + assert isinstance(i2, MyItem) self.assertEqual(i, i2) def test_serialize_loader(self): q = self.queue() - loader = TestLoader() + loader = MyLoader() q.push(loader) loader2 = q.pop() - assert isinstance(loader2, TestLoader) - assert loader2.default_item_class is TestItem + assert isinstance(loader2, MyLoader) + assert loader2.default_item_class is MyItem self.assertEqual(loader2.name_out("x"), "xx") def test_serialize_request_recursive(self): diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index e00f69573..ecac0df9c 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,7 +1,7 @@ import asyncio import warnings -from pytest import mark +import pytest from twisted.trial.unittest import TestCase from scrapy.utils.defer import deferred_f_from_coro_f @@ -12,7 +12,7 @@ from scrapy.utils.reactor import ( ) -@mark.usefixtures("reactor_pytest") +@pytest.mark.usefixtures("reactor_pytest") class AsyncioTest(TestCase): def test_is_asyncio_reactor_installed(self): # the result should depend only on the pytest --reactor argument @@ -30,7 +30,7 @@ class AsyncioTest(TestCase): assert original_reactor == reactor - @mark.only_asyncio() + @pytest.mark.only_asyncio @deferred_f_from_coro_f async def test_set_asyncio_event_loop(self): install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index e80381671..fadbc6daa 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -16,8 +16,6 @@ from scrapy.utils.datatypes import ( ) from scrapy.utils.python import garbage_collect -__doctests__ = ["scrapy.utils.datatypes"] - class CaseInsensitiveDictMixin: def test_init_dict(self): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index e4ab97e5d..3a1030fcf 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -1,6 +1,6 @@ import random -from pytest import mark +import pytest from twisted.internet import defer, reactor from twisted.python.failure import Failure from twisted.trial import unittest @@ -150,7 +150,7 @@ class AsyncDefTestsuiteTest(unittest.TestCase): async def test_deferred_f_from_coro_f_generator(self): yield - @mark.xfail(reason="Checks that the test is actually executed", strict=True) + @pytest.mark.xfail(reason="Checks that the test is actually executed", strict=True) @deferred_f_from_coro_f async def test_deferred_f_from_coro_f_xfail(self): raise RuntimeError("This is expected to be raised") diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 76820eabf..06e88bd10 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -119,7 +119,7 @@ class StreamLoggerTest(unittest.TestCase): @pytest.mark.parametrize( ("base_extra", "log_extra", "expected_extra"), - ( + [ ( {"spider": "test"}, {"extra": {"log_extra": "info"}}, @@ -135,7 +135,7 @@ class StreamLoggerTest(unittest.TestCase): {"extra": {"spider": "test2"}}, {"extra": {"spider": "test"}}, ), - ), + ], ) def test_spider_logger_adapter_process( base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 478c1e73a..e25bdfe3f 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -17,8 +17,6 @@ from scrapy.utils.misc import ( walk_modules, ) -__doctests__ = ["scrapy.utils.misc"] - class UtilsMiscTestCase(unittest.TestCase): def test_load_object_class(self): diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 83004cec4..a693d6b53 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -20,8 +20,6 @@ from scrapy.utils.python import ( without_none_values, ) -__doctests__ = ["scrapy.utils.python"] - class MutableChainTest(unittest.TestCase): def test_mutablechain(self): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index db6866571..c6ba8cbbb 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -15,8 +15,6 @@ from scrapy.utils.response import ( response_status_message, ) -__doctests__ = ["scrapy.utils.response"] - class ResponseUtilsTest(unittest.TestCase): dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response") @@ -207,8 +205,8 @@ class ResponseUtilsTest(unittest.TestCase): @pytest.mark.parametrize( - "input_body,output_body", - ( + ("input_body", "output_body"), + [ ( b"ad", b"ad", ), - ), + ], ) def test_remove_html_comments(input_body, output_body): assert _remove_html_comments(input_body) == output_body, ( diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 60232f10b..858813e83 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -1,7 +1,7 @@ import asyncio +import pytest from pydispatch import dispatcher -from pytest import mark from testfixtures import LogCapture from twisted.internet import defer, reactor from twisted.python.failure import Failure @@ -67,7 +67,7 @@ class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): return d -@mark.usefixtures("reactor_pytest") +@pytest.mark.usefixtures("reactor_pytest") class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) @@ -76,7 +76,7 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): return "OK" -@mark.only_asyncio() +@pytest.mark.only_asyncio class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 5fbbd74da..fc6c33200 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -5,8 +5,6 @@ from tempfile import mkdtemp from scrapy.utils.template import render_templatefile -__doctests__ = ["scrapy.utils.template"] - class UtilsRenderTemplateFileTestCase(unittest.TestCase): def setUp(self): diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 4b9a98d79..e99ef40c4 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -17,8 +17,6 @@ from scrapy.utils.url import ( # type: ignore[attr-defined] url_is_from_spider, ) -__doctests__ = ["scrapy.utils.url"] - class UrlUtilsTest(unittest.TestCase): def test_url_is_from_any_domain(self): From 8c34e6d9a4994abda0059f76dbbdb64c2e8a9751 Mon Sep 17 00:00:00 2001 From: Matt Winter Date: Wed, 19 Feb 2025 04:17:37 -0500 Subject: [PATCH 053/491] curl: add support for parsing -b,--cookie (#6684) --- scrapy/utils/curl.py | 9 +++++++++ tests/test_utils_curl.py | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index a563dc79a..a40ee8997 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -36,6 +36,7 @@ curl_parser = CurlParser() curl_parser.add_argument("url") curl_parser.add_argument("-H", "--header", dest="headers", action="append") curl_parser.add_argument("-X", "--request", dest="method") +curl_parser.add_argument("-b", "--cookie", dest="cookies", action="append") curl_parser.add_argument("-d", "--data", "--data-raw", dest="data", action=DataAction) curl_parser.add_argument("-u", "--user", dest="auth") @@ -68,6 +69,14 @@ def _parse_headers_and_cookies( else: headers.append((name, val)) + for cookie_param in parsed_args.cookies or (): + # curl can treat this parameter as either "key=value; key2=value2" pairs, or a filename. + # Scrapy will only support key-value pairs. + if "=" not in cookie_param: + continue + for name, morsel in SimpleCookie(cookie_param).items(): + cookies[name] = morsel.value + if parsed_args.auth: user, password = parsed_args.auth.split(":", 1) headers.append(("Authorization", basic_auth_header(user, password))) diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index 1816db29b..5d99161bf 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -49,8 +49,8 @@ class CurlToRequestKwargsTest(unittest.TestCase): "ml,application/xhtml+xml,application/xml;q=0.9,image/webp,image/a" "png,*/*;q=0.8' -H 'Referer: http://httpbin.org/' -H 'Cookie: _gau" "ges_unique_year=1; _gauges_unique=1; _gauges_unique_month=1; _gau" - "ges_unique_hour=1; _gauges_unique_day=1' -H 'Connection: keep-ali" - "ve' --compressed" + "ges_unique_hour=1' -H 'Connection: keep-alive' --compressed -b '_" + "gauges_unique_day=1'" ) expected_result = { "method": "GET", From c200458f24a2e55dece5df5c22f39a0d1b4ff341 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 25 Feb 2025 04:16:23 +0500 Subject: [PATCH 054/491] Add more docs for updating sync spider middlewares. (#6688) --- docs/topics/coroutines.rst | 46 +++++++++++++++++++++++++++++++++----- scrapy/core/spidermw.py | 4 ++-- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index a65bab3ca..57aa3a62d 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -238,16 +238,52 @@ active spider middlewares must either have their ``process_spider_output`` method defined as an asynchronous generator or :ref:`define a process_spider_output_async method `. -.. note:: When using third-party spider middlewares that only define a - synchronous ``process_spider_output`` method, consider - :ref:`making them universal ` through - :ref:`subclassing `. +.. _sync-async-spider-middleware-users: +For middleware users +-------------------- + +If you have asynchronous callbacks or use asynchronous-only spider middlewares +you should make sure the asynchronous-to-synchronous conversions +:ref:`described above ` don't happen. To do this, +make sure all spider middlewares you use support asynchronous spider output. +Even if you don't have asynchronous callbacks and don't use asynchronous-only +spider middlewares in your project, it's still a good idea to make sure all +middlewares you use support asynchronous spider output, so that it will be easy +to start using asynchronous callbacks in the future. Because of this, Scrapy +logs a warning when it detects a synchronous-only spider middleware. + +If you want to update middlewares you wrote, see the :ref:`following section +`. If you have 3rd-party middlewares that +aren't yet updated by their authors, you can :ref:`subclass ` +them to make them :ref:`universal ` and use the +subclasses in your projects. + +.. _sync-async-spider-middleware-authors: + +For middleware authors +---------------------- + +If you have a spider middleware that defines a synchronous +``process_spider_output`` method, you should update it to support asynchronous +spider output for :ref:`better compatibility `, +even if you don't yet use it with asynchronous callbacks, especially if you +publish this middleware for other people to use. You have two options for this: + +1. Make the middleware asynchronous, by making the ``process_spider_output`` + method an :term:`asynchronous generator`. +2. Make the middleware universal, as described in the :ref:`next section + `. + +If your middleware won't be used in projects with synchronous-only middlewares, +e.g. because it's an internal middleware and you know that all other +middlewares in your projects are already updated, it's safe to choose the first +option. Otherwise, it's better to choose the second option. .. _universal-spider-middleware: Universal spider middlewares -============================ +---------------------------- .. versionadded:: 2.7 diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 86d11c0e0..85a3b5895 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -227,7 +227,7 @@ class SpiderMiddlewareManager(MiddlewareManager): f"Async iterable passed to {global_object_name(method)} was" f" downgraded to a non-async one. This is deprecated and will" f" stop working in a future version of Scrapy. Please see" - f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares" + f" https://docs.scrapy.org/en/latest/topics/coroutines.html#for-middleware-users" f" for more information." ) assert isinstance(result, AsyncIterable) @@ -343,7 +343,7 @@ class SpiderMiddlewareManager(MiddlewareManager): f" asynchronous spider output, this is deprecated and will stop" f" working in a future version of Scrapy. The middleware should" f" be updated to support it. Please see" - f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares" + f" https://docs.scrapy.org/en/latest/topics/coroutines.html#for-middleware-users" f" for more information." ) return normal_method From 391af6afcca232aed82eda516b035dbbd39cdcb1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 27 Feb 2025 22:37:01 +0500 Subject: [PATCH 055/491] Unknown encoding handling in HttpCompressionMiddleware, restore x-gzip support (#6618) * Unknown encoding handling in HttpCompressionMiddleware. * Implement the changes for unknown encoding handling. * Restore support for Content-Encoding: x-gzip. * Simplify the decoding logic. * Add tests for the unsupported encoding warning. * Add a test for the "no zstandard" warning. --- .../downloadermiddlewares/httpcompression.py | 32 +++++-- ...st_downloadermiddleware_httpcompression.py | 84 ++++++++++++++++++- 2 files changed, 108 insertions(+), 8 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index a65757972..58891b952 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -111,6 +111,8 @@ class HttpCompressionMiddleware: f"({len(decoded_body)} B) is larger than the " f"download warning size ({warn_size} B)." ) + if content_encoding: + self._warn_unknown_encoding(response, content_encoding) response.headers["Content-Encoding"] = content_encoding if self.stats: self.stats.inc_value( @@ -143,9 +145,11 @@ class HttpCompressionMiddleware: body = self._decode(body, encoding, max_size) return body, to_keep + @staticmethod def _split_encodings( - self, content_encoding: list[bytes] + content_encoding: list[bytes], ) -> tuple[list[bytes], list[bytes]]: + supported_encodings = {*ACCEPTED_ENCODINGS, b"x-gzip"} to_keep: list[bytes] = [ encoding.strip().lower() for encoding in chain.from_iterable( @@ -155,19 +159,35 @@ class HttpCompressionMiddleware: to_decode: list[bytes] = [] while to_keep: encoding = to_keep.pop() - if encoding not in ACCEPTED_ENCODINGS: + if encoding not in supported_encodings: to_keep.append(encoding) return to_decode, to_keep to_decode.append(encoding) return to_decode, to_keep - def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes: + @staticmethod + def _decode(body: bytes, encoding: bytes, max_size: int) -> bytes: if encoding in {b"gzip", b"x-gzip"}: return gunzip(body, max_size=max_size) if encoding == b"deflate": return _inflate(body, max_size=max_size) - if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS: + if encoding == b"br": return _unbrotli(body, max_size=max_size) - if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS: + if encoding == b"zstd": return _unzstd(body, max_size=max_size) - return body + # shouldn't be reached + return body # pragma: no cover + + def _warn_unknown_encoding( + self, response: Response, encodings: list[bytes] + ) -> None: + encodings_str = b",".join(encodings).decode() + msg = ( + f"{self.__class__.__name__} cannot decode the response for {response.url} " + f"from unsupported encoding(s) '{encodings_str}'." + ) + if b"br" in encodings: + msg += " You need to install brotli or brotlicffi to decode 'br'." + if b"zstd" in encodings: + msg += " You need to install zstandard to decode 'zstd'." + logger.warning(msg) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 78d0dd99d..a1c5883ec 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -23,7 +23,7 @@ SAMPLEDIR = Path(tests_datadir, "compressed") FORMAT = { "gzip": ("html-gzip.bin", "gzip"), - "x-gzip": ("html-gzip.bin", "gzip"), + "x-gzip": ("html-gzip.bin", "x-gzip"), "rawdeflate": ("html-rawdeflate.bin", "deflate"), "zlibdeflate": ("html-zlibdeflate.bin", "deflate"), "gzip-deflate": ("html-gzip-deflate.bin", "gzip, deflate"), @@ -145,6 +145,41 @@ class HttpCompressionTest(TestCase): self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_bytes", 74837) + def test_process_response_br_unsupported(self): + try: + try: + import brotli # noqa: F401 + + raise SkipTest("Requires not having brotli support") + except ImportError: + import brotlicffi # noqa: F401 + + raise SkipTest("Requires not having brotli support") + except ImportError: + pass + response = self._getresponse("br") + request = response.request + self.assertEqual(response.headers["Content-Encoding"], b"br") + with LogCapture( + "scrapy.downloadermiddlewares.httpcompression", + propagate=False, + level=WARNING, + ) as log: + newresponse = self.mw.process_response(request, response, self.spider) + log.check( + ( + "scrapy.downloadermiddlewares.httpcompression", + "WARNING", + ( + "HttpCompressionMiddleware cannot decode the response for" + " http://scrapytest.org/ from unsupported encoding(s) 'br'." + " You need to install brotli or brotlicffi to decode 'br'." + ), + ), + ) + assert newresponse is not response + self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"br"]) + def test_process_response_zstd(self): try: import zstandard # noqa: F401 @@ -166,6 +201,36 @@ class HttpCompressionTest(TestCase): assert newresponse.body.startswith(b" Date: Thu, 27 Feb 2025 23:13:04 +0500 Subject: [PATCH 056/491] Switch to pytest.raises(). (#6680) * Switch to pytest.raises(). * Add matches= to broad pytest.raises(). * Adjust the test_nonserializable_object() regex for Python <= 3.11. * Adjust the test_nonserializable_object() regex for PyPy. * Adjust other test exception regexes for PyPy. * Cleanup. --- pyproject.toml | 2 - scrapy/pqueues.py | 2 +- tests/test_crawler.py | 33 ++-- tests/test_downloader_handlers_http2.py | 2 +- tests/test_downloadermiddleware.py | 4 +- tests/test_downloadermiddleware_cookies.py | 9 +- tests/test_downloadermiddleware_httpauth.py | 5 +- tests/test_downloadermiddleware_httpcache.py | 7 +- ...st_downloadermiddleware_httpcompression.py | 37 ++--- tests/test_downloadermiddleware_redirect.py | 10 +- tests/test_downloadermiddleware_retry.py | 3 +- tests/test_downloadermiddleware_robotstxt.py | 4 +- tests/test_exporters.py | 14 +- tests/test_feedexport.py | 12 +- tests/test_http2_client_protocol.py | 5 +- tests/test_http_headers.py | 25 ++- tests/test_http_request.py | 149 ++++++++---------- tests/test_http_response.py | 113 +++++++------ tests/test_item.py | 29 ++-- tests/test_link.py | 4 +- tests/test_loader.py | 22 ++- tests/test_loader_deprecated.py | 53 +++++-- tests/test_logstats.py | 5 +- tests/test_pipeline_images.py | 7 +- tests/test_pqueues.py | 9 +- tests/test_request_dict.py | 18 ++- tests/test_scheduler.py | 10 +- tests/test_scheduler_base.py | 22 +-- tests/test_selector.py | 2 +- tests/test_settings/__init__.py | 37 +++-- tests/test_spider.py | 12 +- tests/test_spiderloader/__init__.py | 12 +- tests/test_spidermiddleware.py | 16 +- tests/test_spidermiddleware_httperror.py | 25 +-- tests/test_spidermiddleware_referer.py | 6 +- tests/test_spiderstate.py | 4 +- tests/test_squeues.py | 12 +- tests/test_squeues_request.py | 13 +- tests/test_utils_conf.py | 28 ++-- tests/test_utils_curl.py | 24 ++- tests/test_utils_datatypes.py | 13 +- tests/test_utils_deprecate.py | 5 +- tests/test_utils_gz.py | 7 +- tests/test_utils_iterators.py | 16 +- tests/test_utils_misc/__init__.py | 20 ++- tests/test_utils_python.py | 6 +- tests/test_utils_response.py | 3 +- 47 files changed, 482 insertions(+), 394 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ad62ea212..82d8056f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -378,8 +378,6 @@ ignore = [ # Temporarily silenced PT rules # Use a regular `assert` instead of unittest-style `assertEqual` "PT009", - # Use `pytest.raises` instead of unittest-style `assertRaises` - "PT027", ] [tool.ruff.lint.per-file-ignores] diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index a04e0107b..324a9b955 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -214,7 +214,7 @@ class DownloaderAwarePriorityQueue: "DownloaderAwarePriorityQueue accepts " "``slot_startprios`` as a dict; " f"{slot_startprios.__class__!r} instance " - "is passed. Most likely, it means the state is" + "is passed. Most likely, it means the state is " "created by an incompatible priority queue. " "Only a crawl started with the same priority " "queue class can be resumed." diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 425188d32..df5ebfa7b 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -185,9 +185,8 @@ class CrawlerTestCase(BaseCrawlerTest): def test_get_downloader_middleware_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) - self.assertRaises( - RuntimeError, crawler.get_downloader_middleware, DefaultSpider - ) + with pytest.raises(RuntimeError): + crawler.get_downloader_middleware(DefaultSpider) @inlineCallbacks def test_get_downloader_middleware_no_engine(self): @@ -266,7 +265,8 @@ class CrawlerTestCase(BaseCrawlerTest): def test_get_extension_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) - self.assertRaises(RuntimeError, crawler.get_extension, DefaultSpider) + with pytest.raises(RuntimeError): + crawler.get_extension(DefaultSpider) @inlineCallbacks def test_get_extension_no_engine(self): @@ -345,7 +345,8 @@ class CrawlerTestCase(BaseCrawlerTest): def test_get_item_pipeline_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) - self.assertRaises(RuntimeError, crawler.get_item_pipeline, DefaultSpider) + with pytest.raises(RuntimeError): + crawler.get_item_pipeline(DefaultSpider) @inlineCallbacks def test_get_item_pipeline_no_engine(self): @@ -424,7 +425,8 @@ class CrawlerTestCase(BaseCrawlerTest): def test_get_spider_middleware_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) - self.assertRaises(RuntimeError, crawler.get_spider_middleware, DefaultSpider) + with pytest.raises(RuntimeError): + crawler.get_spider_middleware(DefaultSpider) @inlineCallbacks def test_get_spider_middleware_no_engine(self): @@ -537,7 +539,8 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, } ) - self.assertRaises(MultipleInvalid, CrawlerRunner, settings) + with pytest.raises(MultipleInvalid): + CrawlerRunner(settings) def test_crawler_runner_accepts_dict(self): runner = CrawlerRunner({"foo": "bar"}) @@ -630,13 +633,15 @@ class CrawlerRunnerHasSpider(unittest.TestCase): } ) else: - msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" - with self.assertRaisesRegex(Exception, msg): - runner = CrawlerRunner( - settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - } - ) + runner = CrawlerRunner( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + ) + with pytest.raises( + Exception, + match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)", + ): yield runner.crawl(NoRequestsSpider) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 174bf841e..17d5c2d0a 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -248,5 +248,5 @@ class Https2ProxyTestCase(BaseTestClasses.Http11ProxyTestCase): @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): - with self.assertRaises(NotImplementedError): + with pytest.raises(NotImplementedError): yield super().test_download_with_proxy_https_timeout() diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 42051042c..49498375c 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,4 +1,5 @@ import asyncio +from gzip import BadGzipFile from unittest import mock import pytest @@ -106,7 +107,8 @@ class DefaultsTest(ManagerTestCase): "Location": "http://example.com/login", }, ) - self.assertRaises(OSError, self._download, request=req, response=resp) + with pytest.raises(BadGzipFile): + self._download(request=req, response=resp) class ResponseFromProcessRequestTest(ManagerTestCase): diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 772769690..694a669d4 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -83,11 +83,10 @@ class CookiesMiddlewareTest(TestCase): self.assertEqual(req2.headers.get("Cookie"), b"C1=value1") def test_setting_false_cookies_enabled(self): - self.assertRaises( - NotConfigured, - CookiesMiddleware.from_crawler, - get_crawler(settings_dict={"COOKIES_ENABLED": False}), - ) + with pytest.raises(NotConfigured): + CookiesMiddleware.from_crawler( + get_crawler(settings_dict={"COOKIES_ENABLED": False}) + ) def test_setting_default_cookies_enabled(self): self.assertIsInstance( diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 581fc1974..0f1489344 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,5 +1,6 @@ import unittest +import pytest from w3lib.http import basic_auth_header from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware @@ -29,8 +30,8 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase): self.spider = LegacySpider("foo") def test_auth(self): - with self.assertRaises(AttributeError): - mw = HttpAuthMiddleware() + mw = HttpAuthMiddleware() + with pytest.raises(AttributeError): mw.spider_opened(self.spider) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 74db93f8a..de3a9689b 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -5,6 +5,8 @@ import time import unittest from contextlib import contextmanager +import pytest + from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware from scrapy.exceptions import IgnoreRequest from scrapy.http import HtmlResponse, Request, Response @@ -192,9 +194,8 @@ class DummyPolicyTest(_BaseTest): def test_middleware_ignore_missing(self): with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw: - self.assertRaises( - IgnoreRequest, mw.process_request, self.request, self.spider - ) + with pytest.raises(IgnoreRequest): + mw.process_request(self.request, self.spider) mw.process_response(self.request, self.response, self.spider) response = mw.process_request(self.request, self.spider) assert isinstance(response, HtmlResponse) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index a1c5883ec..b3e3b98d7 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -4,6 +4,7 @@ from logging import WARNING from pathlib import Path from unittest import SkipTest, TestCase +import pytest from testfixtures import LogCapture from w3lib.encoding import resolve_encoding @@ -87,11 +88,10 @@ class HttpCompressionTest(TestCase): ) def test_setting_false_compression_enabled(self): - self.assertRaises( - NotConfigured, - HttpCompressionMiddleware.from_crawler, - get_crawler(settings_dict={"COMPRESSION_ENABLED": False}), - ) + with pytest.raises(NotConfigured): + HttpCompressionMiddleware.from_crawler( + get_crawler(settings_dict={"COMPRESSION_ENABLED": False}) + ) def test_setting_default_compression_enabled(self): self.assertIsInstance( @@ -520,13 +520,8 @@ class HttpCompressionTest(TestCase): mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") - self.assertRaises( - IgnoreRequest, - mw.process_response, - response.request, - response, - spider, - ) + with pytest.raises(IgnoreRequest): + mw.process_response(response.request, response, spider) def test_compression_bomb_setting_br(self): try: @@ -561,13 +556,8 @@ class HttpCompressionTest(TestCase): mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") - self.assertRaises( - IgnoreRequest, - mw.process_response, - response.request, - response, - spider, - ) + with pytest.raises(IgnoreRequest): + mw.process_response(response.request, response, spider) def test_compression_bomb_spider_attr_br(self): try: @@ -600,13 +590,8 @@ class HttpCompressionTest(TestCase): response = self._getresponse(f"bomb-{compression_id}") response.meta["download_maxsize"] = 10_000_000 - self.assertRaises( - IgnoreRequest, - mw.process_response, - response.request, - response, - spider, - ) + with pytest.raises(IgnoreRequest): + mw.process_response(response.request, response, spider) def test_compression_bomb_request_meta_br(self): try: diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index f950906e9..47abeee7a 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -72,9 +72,8 @@ class Base: assert isinstance(req, Request) assert "redirect_times" in req.meta self.assertEqual(req.meta["redirect_times"], 1) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp, self.spider) def test_ttl(self): self.mw.max_redirect_times = 100 @@ -83,9 +82,8 @@ class Base: req = self.mw.process_response(req, rsp, self.spider) assert isinstance(req, Request) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp, self.spider) def test_redirect_urls(self): req1 = Request("http://scrapytest.org/first") diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 6b9b39413..36f48db69 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,6 +1,7 @@ import logging import unittest +import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.internet.error import ( @@ -407,7 +408,7 @@ class GetRetryRequestTest(unittest.TestCase): def test_no_spider(self): request = Request("https://example.com") - with self.assertRaises(TypeError): + with pytest.raises(TypeError): get_retry_request(request) # pylint: disable=missing-kwoa def test_max_retry_times_setting(self): diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 535e07c1f..9b95400fd 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,5 +1,6 @@ from unittest import mock +import pytest from twisted.internet import error, reactor from twisted.internet.defer import Deferred, DeferredList, maybeDeferred from twisted.python import failure @@ -26,7 +27,8 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): def test_robotstxt_settings(self): self.crawler.settings = Settings() self.crawler.settings.set("USER_AGENT", "CustomAgent") - self.assertRaises(NotConfigured, RobotsTxtMiddleware, self.crawler) + with pytest.raises(NotConfigured): + RobotsTxtMiddleware(self.crawler) def _get_successful_crawler(self): crawler = self.crawler diff --git a/tests/test_exporters.py b/tests/test_exporters.py index eb8d309b6..48728e078 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -10,6 +10,7 @@ from io import BytesIO from typing import Any import lxml.etree +import pytest from itemadapter import ItemAdapter from scrapy.exporters import ( @@ -147,7 +148,7 @@ class PythonItemExporterTest(BaseItemExporterTest): return PythonItemExporter(**kwargs) def test_invalid_option(self): - with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"): + with pytest.raises(TypeError, match="Unexpected options: invalid_option"): PythonItemExporter(invalid_option="something") def test_nested_item(self): @@ -388,7 +389,7 @@ class CsvItemExporterTest(BaseItemExporterTest): ) def test_errors_default(self): - with self.assertRaises(UnicodeEncodeError): + with pytest.raises(UnicodeEncodeError): self.assertExportResult( item={"text": "W\u0275\u200brd"}, expected=None, @@ -549,7 +550,8 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie = self._get_exporter(sort_keys=True) self.test_export_item() self._check_output() - self.assertRaises(TypeError, self._get_exporter, foo_unknown_keyword_bar=True) + with pytest.raises(TypeError): + self._get_exporter(foo_unknown_keyword_bar=True) def test_nonstring_types_item(self): item = self._get_nonstring_types_item() @@ -602,7 +604,8 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): i3 = MyItem(name="Jesus", age="44") self.ie.start_exporting() self.ie.export_item(i1) - self.assertRaises(TypeError, self.ie.export_item, i2) + with pytest.raises(TypeError): + self.ie.export_item(i2) self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) @@ -657,7 +660,8 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest): i3 = MyItem(name="Jesus", age="44") self.ie.start_exporting() self.ie.export_item(i1) - self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2) + with pytest.raises(UnicodeEncodeError): + self.ie.export_item(i2) self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue(), encoding="latin")) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 1620d2d41..b4c1b9631 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -233,7 +233,8 @@ class BlockingFeedStorageTest(unittest.TestCase): invalid_path = tests_path / "invalid_path" spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)}) - self.assertRaises(OSError, b.open, spider=spider) + with pytest.raises(OSError, match="Not a Directory:"): + b.open(spider=spider) @pytest.mark.requires_boto3 @@ -2437,7 +2438,8 @@ class BatchDeliveriesTest(FeedExportTestBase): "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } crawler = get_crawler(settings_dict=settings) - self.assertRaises(NotConfigured, FeedExporter, crawler) + with pytest.raises(NotConfigured): + FeedExporter(crawler) @defer.inlineCallbacks def test_export_no_items_not_store_empty(self): @@ -2758,7 +2760,7 @@ class FeedExportInitTest(unittest.TestCase): }, } crawler = get_crawler(settings_dict=settings) - with self.assertRaises(NotConfigured): + with pytest.raises(NotConfigured): FeedExporter.from_crawler(crawler) def test_unsupported_format(self): @@ -2770,7 +2772,7 @@ class FeedExportInitTest(unittest.TestCase): }, } crawler = get_crawler(settings_dict=settings) - with self.assertRaises(NotConfigured): + with pytest.raises(NotConfigured): FeedExporter.from_crawler(crawler) def test_absolute_pathlib_as_uri(self): @@ -2863,7 +2865,7 @@ class URIParamsTest: with warnings.catch_warnings(): warnings.simplefilter("error", ScrapyDeprecationWarning) - with self.assertRaises(KeyError): + with pytest.raises(KeyError): feed_exporter.open_spider(spider) def test_params_as_is(self): diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index ddc772236..0881bbeca 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING from unittest import mock, skipIf from urllib.parse import urlencode +import pytest from twisted.internet import reactor from twisted.internet.defer import ( CancelledError, @@ -406,7 +407,7 @@ class Https2ClientProtocolTestCase(TestCase): "scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2" ): request = Request(url=self.get_url("/status?n=200")) - with self.assertRaises(ResponseFailed): + with pytest.raises(ResponseFailed): yield self.make_request(request) def test_cancel_request(self): @@ -560,7 +561,7 @@ class Https2ClientProtocolTestCase(TestCase): return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) def test_invalid_request_type(self): - with self.assertRaises(TypeError): + with pytest.raises(TypeError): self.make_request("https://InvalidDataTypePassed.com") def test_query_parameters(self): diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 7db1eb8c5..0bbbcda46 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -1,6 +1,8 @@ import copy import unittest +import pytest + from scrapy.http import Headers @@ -13,7 +15,8 @@ class HeadersTest(unittest.TestCase): assert h["Content-Type"] assert h["Content-Length"] - self.assertRaises(KeyError, h.__getitem__, "Accept") + with pytest.raises(KeyError): + h["Accept"] self.assertEqual(h.get("Accept"), None) self.assertEqual(h.getlist("Accept"), []) @@ -152,15 +155,11 @@ class HeadersTest(unittest.TestCase): self.assertEqual(h1.getlist("hey"), [b"5"]) def test_invalid_value(self): - self.assertRaisesRegex( - TypeError, "Unsupported value type", Headers, {"foo": object()} - ) - self.assertRaisesRegex( - TypeError, "Unsupported value type", Headers().__setitem__, "foo", object() - ) - self.assertRaisesRegex( - TypeError, "Unsupported value type", Headers().setdefault, "foo", object() - ) - self.assertRaisesRegex( - TypeError, "Unsupported value type", Headers().setlist, "foo", [object()] - ) + with pytest.raises(TypeError, match="Unsupported value type"): + Headers({"foo": object()}) + with pytest.raises(TypeError, match="Unsupported value type"): + Headers()["foo"] = object() + with pytest.raises(TypeError, match="Unsupported value type"): + Headers().setdefault("foo", object()) + with pytest.raises(TypeError, match="Unsupported value type"): + Headers().setlist("foo", [object()]) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index a8ab8240f..e5291157d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -7,6 +7,8 @@ from typing import Any from unittest import mock from urllib.parse import parse_qs, unquote_to_bytes +import pytest + from scrapy.http import ( FormRequest, Headers, @@ -28,10 +30,12 @@ class RequestTest(unittest.TestCase): def test_init(self): # Request requires url in the __init__ method - self.assertRaises(Exception, self.request_class) + with pytest.raises(TypeError): + self.request_class() # url argument must be basestring - self.assertRaises(TypeError, self.request_class, 123) + with pytest.raises(TypeError): + self.request_class(123) r = self.request_class("http://www.example.com") r = self.request_class("http://www.example.com") @@ -64,9 +68,13 @@ class RequestTest(unittest.TestCase): self.request_class("data:,Hello%2C%20World!") def test_url_no_scheme(self): - self.assertRaises(ValueError, self.request_class, "foo") - self.assertRaises(ValueError, self.request_class, "/foo/") - self.assertRaises(ValueError, self.request_class, "/foo:bar") + msg = "Missing scheme in request url:" + with pytest.raises(ValueError, match=msg): + self.request_class("foo") + with pytest.raises(ValueError, match=msg): + self.request_class("/foo/") + with pytest.raises(ValueError, match=msg): + self.request_class("/foo:bar") def test_headers(self): # Different ways of setting headers attribute @@ -273,8 +281,10 @@ class RequestTest(unittest.TestCase): def test_immutable_attributes(self): r = self.request_class("http://example.com") - self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com") - self.assertRaises(AttributeError, setattr, r, "body", "xxx") + with pytest.raises(AttributeError): + r.url = "http://example2.com" + with pytest.raises(AttributeError): + r.body = "xxx" def test_callback_and_errback(self): def a_function(): @@ -309,11 +319,11 @@ class RequestTest(unittest.TestCase): self.assertIs(r5.errback, NO_CALLBACK) def test_callback_and_errback_type(self): - with self.assertRaises(TypeError): + with pytest.raises(TypeError): self.request_class("http://example.com", callback="a_function") - with self.assertRaises(TypeError): + with pytest.raises(TypeError): self.request_class("http://example.com", errback="a_function") - with self.assertRaises(TypeError): + with pytest.raises(TypeError): self.request_class( url="http://example.com", callback="a_function", @@ -321,7 +331,7 @@ class RequestTest(unittest.TestCase): ) def test_no_callback(self): - with self.assertRaises(RuntimeError): + with pytest.raises(RuntimeError): NO_CALLBACK() def test_from_curl(self): @@ -403,13 +413,11 @@ class RequestTest(unittest.TestCase): # If `ignore_unknown_options` is set to `False` it raises an error with # the unknown options: --foo and -z - self.assertRaises( - ValueError, - lambda: self.request_class.from_curl( + with pytest.raises(ValueError, match="Unrecognized options:"): + self.request_class.from_curl( 'curl -X PATCH "http://example.org" --foo -z', ignore_unknown_options=False, - ), - ) + ) class FormRequestTest(RequestTest): @@ -428,7 +436,7 @@ class FormRequestTest(RequestTest): data = (("a", "one"), ("a", "two"), ("b", "2")) url = self.request_class( "http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data - ).url.split("#")[0] + ).url.split("#", maxsplit=1)[0] fs = _qs(self.request_class(url, method="GET", formdata=data)) self.assertEqual(set(fs[b"a"]), {b"one", b"two"}) self.assertEqual(fs[b"b"], [b"2"]) @@ -897,12 +905,11 @@ class FormRequestTest(RequestTest): """ ) - self.assertRaises( + with pytest.raises( ValueError, - self.request_class.from_response, - response, - clickdata={"type": "submit"}, - ) + match="Multiple elements found .* matching the criteria in clickdata", + ): + self.request_class.from_response(response, clickdata={"type": "submit"}) def test_from_response_non_matching_clickdata(self): response = _buildresponse( @@ -910,12 +917,12 @@ class FormRequestTest(RequestTest): """ ) - self.assertRaises( - ValueError, - self.request_class.from_response, - response, - clickdata={"nonexistent": "notme"}, - ) + with pytest.raises( + ValueError, match="No clickable element matching clickdata:" + ): + self.request_class.from_response( + response, clickdata={"nonexistent": "notme"} + ) def test_from_response_nr_index_clickdata(self): response = _buildresponse( @@ -937,13 +944,15 @@ class FormRequestTest(RequestTest): """ ) - self.assertRaises( - ValueError, self.request_class.from_response, response, clickdata={"nr": 1} - ) + with pytest.raises( + ValueError, match="No clickable element matching clickdata:" + ): + self.request_class.from_response(response, clickdata={"nr": 1}) def test_from_response_errors_noform(self): response = _buildresponse("""""") - self.assertRaises(ValueError, self.request_class.from_response, response) + with pytest.raises(ValueError, match="No
element found in"): + self.request_class.from_response(response) def test_from_response_invalid_html5(self): response = _buildresponse( @@ -963,9 +972,8 @@ class FormRequestTest(RequestTest):
""" ) - self.assertRaises( - IndexError, self.request_class.from_response, response, formnumber=1 - ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formnumber=1) def test_from_response_noformname(self): response = _buildresponse( @@ -1021,13 +1029,8 @@ class FormRequestTest(RequestTest): """ ) - self.assertRaises( - IndexError, - self.request_class.from_response, - response, - formname="form3", - formnumber=2, - ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formname="form3", formnumber=2) def test_from_response_formid_exists(self): response = _buildresponse( @@ -1086,13 +1089,8 @@ class FormRequestTest(RequestTest): """ ) - self.assertRaises( - IndexError, - self.request_class.from_response, - response, - formid="form3", - formnumber=2, - ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formid="form3", formnumber=2) def test_from_response_select(self): res = _buildresponse( @@ -1245,12 +1243,10 @@ class FormRequestTest(RequestTest): fs = _qs(r1) self.assertEqual(fs[b"three"], [b"3"]) - self.assertRaises( - ValueError, - self.request_class.from_response, - response, - formxpath="//form/input[@name='abc']", - ) + with pytest.raises(ValueError, match="No
element found with"): + self.request_class.from_response( + response, formxpath="//form/input[@name='abc']" + ) def test_from_response_unicode_xpath(self): response = _buildresponse(b'
') @@ -1261,13 +1257,8 @@ class FormRequestTest(RequestTest): self.assertEqual(fs, {}) xpath = "//form[@name='\u03b1']" - self.assertRaisesRegex( - ValueError, - re.escape(xpath), - self.request_class.from_response, - response, - formxpath=xpath, - ) + with pytest.raises(ValueError, match=re.escape(xpath)): + self.request_class.from_response(response, formxpath=xpath) def test_from_response_button_submit(self): response = _buildresponse( @@ -1393,12 +1384,8 @@ class FormRequestTest(RequestTest): fs = _qs(r1) self.assertEqual(fs[b"three"], [b"3"]) - self.assertRaises( - ValueError, - self.request_class.from_response, - response, - formcss="input[name='abc']", - ) + with pytest.raises(ValueError, match="No
element found with"): + self.request_class.from_response(response, formcss="input[name='abc']") def test_from_response_valid_form_methods(self): form_methods = [ @@ -1424,13 +1411,11 @@ class FormRequestTest(RequestTest):
""" ) - with self.assertRaises(ValueError) as context: + with pytest.raises( + ValueError, match="formdata should be a dict or iterable of tuples" + ): FormRequest.from_response(response, formdata=123) - self.assertIn( - "formdata should be a dict or iterable of tuples", str(context.exception) - ) - def test_form_response_with_custom_invalid_formdata_value_error(self): """Test that a ValueError is raised for fault-inducing iterable formdata input""" response = _buildresponse( @@ -1441,13 +1426,11 @@ class FormRequestTest(RequestTest): """ ) - with self.assertRaises(ValueError) as context: + with pytest.raises( + ValueError, match="formdata should be a dict or iterable of tuples" + ): FormRequest.from_response(response, formdata=("a",)) - self.assertIn( - "formdata should be a dict or iterable of tuples", str(context.exception) - ) - def test_get_form_with_xpath_no_form_parent(self): """Test that _get_from raised a ValueError when an XPath selects an element not nested within a
and no parent is found""" @@ -1462,11 +1445,9 @@ class FormRequestTest(RequestTest): """ ) - with self.assertRaises(ValueError) as context: + with pytest.raises(ValueError, match="No element found with"): FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') - self.assertIn("No element found with", str(context.exception)) - def _buildresponse(body, **kwargs): kwargs.setdefault("body", body) @@ -1507,8 +1488,10 @@ class XmlRpcRequestTest(RequestTest): self._test_request(params=("response",), methodresponse="login") self._test_request(params=("pas£",), encoding="utf-8") self._test_request(params=(None,), allow_none=1) - self.assertRaises(TypeError, self._test_request) - self.assertRaises(TypeError, self._test_request, params=(None,)) + with pytest.raises(TypeError): + self._test_request() + with pytest.raises(TypeError): + self._test_request(params=(None,)) def test_latin1(self): self._test_request(params=("pas£",), encoding="latin1") diff --git a/tests/test_http_response.py b/tests/test_http_response.py index dde883451..5a943f084 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -27,14 +27,15 @@ class BaseResponseTest(unittest.TestCase): def test_init(self): # Response requires url in the constructor - self.assertRaises(Exception, self.response_class) + with pytest.raises(TypeError): + self.response_class() self.assertTrue( isinstance(self.response_class("http://example.com/"), self.response_class) ) - self.assertRaises(TypeError, self.response_class, b"http://example.com") - self.assertRaises( - TypeError, self.response_class, url="http://example.com", body={} - ) + with pytest.raises(TypeError): + self.response_class(b"http://example.com") + with pytest.raises(TypeError): + self.response_class(url="http://example.com", body={}) # body can be str or None self.assertTrue( isinstance( @@ -77,12 +78,8 @@ class BaseResponseTest(unittest.TestCase): self.assertEqual(r.status, 301) r = self.response_class("http://www.example.com", status="301") self.assertEqual(r.status, 301) - self.assertRaises( - ValueError, - self.response_class, - "http://example.com", - status="lala200", - ) + with pytest.raises(ValueError, match=r"invalid literal for int\(\)"): + self.response_class("http://example.com", status="lala200") def test_copy(self): """Test Response copy""" @@ -122,14 +119,12 @@ class BaseResponseTest(unittest.TestCase): def test_unavailable_meta(self): r1 = self.response_class("http://www.example.com", body=b"Some body") - with self.assertRaisesRegex(AttributeError, r"Response\.meta not available"): + with pytest.raises(AttributeError, match=r"Response\.meta not available"): r1.meta def test_unavailable_cb_kwargs(self): r1 = self.response_class("http://www.example.com", body=b"Some body") - with self.assertRaisesRegex( - AttributeError, r"Response\.cb_kwargs not available" - ): + with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"): r1.cb_kwargs def test_copy_inherited_classes(self): @@ -179,8 +174,10 @@ class BaseResponseTest(unittest.TestCase): def test_immutable_attributes(self): r = self.response_class("http://example.com") - self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com") - self.assertRaises(AttributeError, setattr, r, "body", "xxx") + with pytest.raises(AttributeError): + r.url = "http://example2.com" + with pytest.raises(AttributeError): + r.body = "xxx" def test_urljoin(self): """Test urljoin shortcut (only for existence, since behavior equals urljoin)""" @@ -192,10 +189,14 @@ class BaseResponseTest(unittest.TestCase): r = self.response_class("http://example.com", body=b"hello") if self.response_class == Response: msg = "Response content isn't text" - self.assertRaisesRegex(AttributeError, msg, getattr, r, "text") - self.assertRaisesRegex(NotSupported, msg, r.css, "body") - self.assertRaisesRegex(NotSupported, msg, r.xpath, "//body") - self.assertRaisesRegex(NotSupported, msg, r.jmespath, "body") + with pytest.raises(AttributeError, match=msg): + r.text + with pytest.raises(NotSupported, match=msg): + r.css("body") + with pytest.raises(NotSupported, match=msg): + r.xpath("//body") + with pytest.raises(NotSupported, match=msg): + r.jmespath("body") else: r.text r.css("body") @@ -216,7 +217,8 @@ class BaseResponseTest(unittest.TestCase): def test_follow_None_url(self): r = self.response_class("http://example.com") - self.assertRaises(ValueError, r.follow, None) + with pytest.raises(ValueError, match="url can't be None"): + r.follow(None) @pytest.mark.xfail( parse_version(w3lib_version) < parse_version("2.1.1"), @@ -279,18 +281,20 @@ class BaseResponseTest(unittest.TestCase): def test_follow_all_invalid(self): r = self.response_class("http://example.com") if self.response_class == Response: - with self.assertRaises(TypeError): + with pytest.raises(TypeError): list(r.follow_all(urls=None)) - with self.assertRaises(TypeError): + with pytest.raises(TypeError): list(r.follow_all(urls=12345)) - with self.assertRaises(ValueError): + with pytest.raises(ValueError, match="url can't be None"): list(r.follow_all(urls=[None])) else: - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, match="Please supply exactly one of the following arguments" + ): list(r.follow_all(urls=None)) - with self.assertRaises(TypeError): + with pytest.raises(TypeError): list(r.follow_all(urls=12345)) - with self.assertRaises(ValueError): + with pytest.raises(ValueError, match="url can't be None"): list(r.follow_all(urls=[None])) def test_follow_all_whitespace(self): @@ -399,12 +403,8 @@ class TextResponseTest(BaseResponseTest): "\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 " "\u0442\u0435\u043a\u0441\u0442" ) - self.assertRaises( - TypeError, - self.response_class, - "http://www.example.com", - body="unicode body", - ) + with pytest.raises(TypeError): + self.response_class("http://www.example.com", body="unicode body") original_string = unicode_string.encode("cp1251") r1 = self.response_class( @@ -483,12 +483,8 @@ class TextResponseTest(BaseResponseTest): self._assert_response_values(r9, "cp1252", "€") # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies - self.assertRaises( - TypeError, - self.response_class, - "http://www.example.com", - body="\xa3", - ) + with pytest.raises(TypeError): + self.response_class("http://www.example.com", body="\xa3") def test_declared_encoding_invalid(self): """Check that unknown declared encodings are ignored""" @@ -679,20 +675,20 @@ class TextResponseTest(BaseResponseTest): self._assert_followed_url(sel, url, response=resp) # non-a elements are not supported - self.assertRaises(ValueError, resp.follow, resp.css("div")[0]) + with pytest.raises( + ValueError, match="Only and elements are supported" + ): + resp.follow(resp.css("div")[0]) def test_follow_selector_list(self): resp = self._links_response() - self.assertRaisesRegex(ValueError, "SelectorList", resp.follow, resp.css("a")) + with pytest.raises(ValueError, match="SelectorList"): + resp.follow(resp.css("a")) def test_follow_selector_invalid(self): resp = self._links_response() - self.assertRaisesRegex( - ValueError, - "Unsupported", - resp.follow, - resp.xpath("count(//div)")[0], - ) + with pytest.raises(ValueError, match="Unsupported"): + resp.follow(resp.xpath("count(//div)")[0]) def test_follow_selector_attribute(self): resp = self._links_response() @@ -704,7 +700,8 @@ class TextResponseTest(BaseResponseTest): url="http://example.com", body=b"click me", ) - self.assertRaisesRegex(ValueError, "no href", resp.follow, resp.css("a")[0]) + with pytest.raises(ValueError, match="no href"): + resp.follow(resp.css("a")[0]) def test_follow_whitespace_selector(self): resp = self.response_class( @@ -812,7 +809,9 @@ class TextResponseTest(BaseResponseTest): def test_follow_all_too_many_arguments(self): response = self._links_response() - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, match="Please supply exactly one of the following arguments" + ): response.follow_all( css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]', @@ -825,7 +824,9 @@ class TextResponseTest(BaseResponseTest): text_body = b"""text""" text_response = self.response_class("http://www.example.com", body=text_body) - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, match="(Expecting value|Unexpected '<'): line 1" + ): text_response.json() def test_cache_json_response(self): @@ -1023,10 +1024,8 @@ class CustomResponseTest(TextResponseTest): self.assertEqual(r4.bar, "bar") self.assertIsNone(r4.lost) - with self.assertRaises(TypeError) as ctx: + with pytest.raises( + TypeError, + match=r"__init__\(\) got an unexpected keyword argument 'unknown'", + ): r1.replace(unknown="unknown") - self.assertTrue( - str(ctx.exception).endswith( - "__init__() got an unexpected keyword argument 'unknown'" - ) - ) diff --git a/tests/test_item.py b/tests/test_item.py index 0399c8f8d..47c5c3db6 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -2,6 +2,8 @@ import unittest from abc import ABCMeta from unittest import mock +import pytest + from scrapy.item import Field, Item, ItemMeta @@ -22,7 +24,8 @@ class ItemTest(unittest.TestCase): name = Field() i = TestItem() - self.assertRaises(KeyError, i.__getitem__, "name") + with pytest.raises(KeyError): + i["name"] i2 = TestItem(name="john doe") self.assertEqual(i2["name"], "john doe") @@ -33,15 +36,18 @@ class ItemTest(unittest.TestCase): i4 = TestItem(i3) self.assertEqual(i4["name"], "john doe") - self.assertRaises(KeyError, TestItem, {"name": "john doe", "other": "foo"}) + with pytest.raises(KeyError): + TestItem({"name": "john doe", "other": "foo"}) def test_invalid_field(self): class TestItem(Item): pass i = TestItem() - self.assertRaises(KeyError, i.__setitem__, "field", "text") - self.assertRaises(KeyError, i.__getitem__, "field") + with pytest.raises(KeyError): + i["field"] = "text" + with pytest.raises(KeyError): + i["field"] def test_repr(self): class TestItem(Item): @@ -72,14 +78,16 @@ class ItemTest(unittest.TestCase): name = Field() i = TestItem() - self.assertRaises(AttributeError, getattr, i, "name") + with pytest.raises(AttributeError): + i.name def test_raise_setattr(self): class TestItem(Item): name = Field() i = TestItem() - self.assertRaises(AttributeError, setattr, i, "name", "john") + with pytest.raises(AttributeError): + i.name = "john" def test_custom_methods(self): class TestItem(Item): @@ -92,7 +100,8 @@ class ItemTest(unittest.TestCase): self["name"] = name i = TestItem() - self.assertRaises(KeyError, i.get_name) + with pytest.raises(KeyError): + i.get_name() i["name"] = "lala" self.assertEqual(i.get_name(), "lala") i.change_name("other") @@ -223,7 +232,8 @@ class ItemTest(unittest.TestCase): class D(B, C): pass - self.assertRaises(KeyError, D, not_allowed="value") + with pytest.raises(KeyError): + D(not_allowed="value") self.assertEqual(D(save="X")["save"], "X") self.assertEqual(D.fields, {"save": {"default": "A"}, "load": {"default": "A"}}) @@ -231,7 +241,8 @@ class ItemTest(unittest.TestCase): class E(C, B): pass - self.assertRaises(KeyError, E, not_allowed="value") + with pytest.raises(KeyError): + E(not_allowed="value") self.assertEqual(E(save="X")["save"], "X") self.assertEqual(E.fields, {"save": {"default": "A"}, "load": {"default": "A"}}) diff --git a/tests/test_link.py b/tests/test_link.py index 35723bbd6..ed9d27a37 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -1,5 +1,7 @@ import unittest +import pytest + from scrapy.link import Link @@ -53,5 +55,5 @@ class LinkTest(unittest.TestCase): self._assert_same_links(l1, l2) def test_bytes_url(self): - with self.assertRaises(TypeError): + with pytest.raises(TypeError): Link(b"http://www.example.com/\xc2\xa3") diff --git a/tests/test_loader.py b/tests/test_loader.py index b52d5ea2e..1a933bb8d 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -4,6 +4,7 @@ import dataclasses import unittest import attr +import pytest from itemadapter import ItemAdapter from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst @@ -69,7 +70,8 @@ def processor_with_args(value, other=None, loader_context=None): class BasicItemLoaderTest(unittest.TestCase): def test_add_value_on_unknown_field(self): il = ProcessorItemLoader() - self.assertRaises(KeyError, il.add_value, "wrong_field", ["lala", "lolo"]) + with pytest.raises(KeyError): + il.add_value("wrong_field", ["lala", "lolo"]) def test_load_item_using_default_loader(self): i = SummaryItem() @@ -294,12 +296,18 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method_errors(self): l = ProcessorItemLoader() - self.assertRaises(RuntimeError, l.add_xpath, "url", "//a/@href") - self.assertRaises(RuntimeError, l.replace_xpath, "url", "//a/@href") - self.assertRaises(RuntimeError, l.get_xpath, "//a/@href") - self.assertRaises(RuntimeError, l.add_css, "name", "#name::text") - self.assertRaises(RuntimeError, l.replace_css, "name", "#name::text") - self.assertRaises(RuntimeError, l.get_css, "#name::text") + with pytest.raises(RuntimeError): + l.add_xpath("url", "//a/@href") + with pytest.raises(RuntimeError): + l.replace_xpath("url", "//a/@href") + with pytest.raises(RuntimeError): + l.get_xpath("//a/@href") + with pytest.raises(RuntimeError): + l.add_css("name", "#name::text") + with pytest.raises(RuntimeError): + l.replace_css("name", "#name::text") + with pytest.raises(RuntimeError): + l.get_css("#name::text") def test_init_method_with_selector(self): sel = Selector(text="
marta
") diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 1e504f539..0d7921b1d 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -6,6 +6,7 @@ Once we remove the references from scrapy, we can remove these tests. import unittest from functools import partial +import pytest from itemloaders.processors import ( Compose, Identity, @@ -435,7 +436,13 @@ class BasicItemLoaderTest(unittest.TestCase): name_in = MapCompose(float) il = TestItemLoader() - self.assertRaises(ValueError, il.add_value, "name", ["marta", "other"]) + with pytest.raises( + ValueError, + match="Error with input processor MapCompose: .* " + "error='ValueError: Error in MapCompose .* " + "error='ValueError: could not convert", + ): + il.add_value("name", ["marta", "other"]) def test_error_output_processor(self): class TestItem(Item): @@ -447,7 +454,12 @@ class BasicItemLoaderTest(unittest.TestCase): il = TestItemLoader() il.add_value("name", "marta") - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, + match="Error with output processor: .* " + "error='ValueError: Error in Compose .* " + "error='ValueError: could not convert", + ): il.load_item() def test_error_processor_as_argument(self): @@ -458,9 +470,13 @@ class BasicItemLoaderTest(unittest.TestCase): default_item_class = TestItem il = TestItemLoader() - self.assertRaises( - ValueError, il.add_value, "name", ["marta", "other"], Compose(float) - ) + with pytest.raises( + ValueError, + match=r"Error with processor Compose .* " + r"error='ValueError: Error in Compose .* " + r"error='TypeError: float\(\) argument", + ): + il.add_value("name", ["marta", "other"], Compose(float)) class InitializationFromDictTest(unittest.TestCase): @@ -630,7 +646,8 @@ class ProcessorsTest(unittest.TestCase): def test_join(self): proc = Join() - self.assertRaises(TypeError, proc, [None, "", "hello", "world"]) + with pytest.raises(TypeError): + proc([None, "", "hello", "world"]) self.assertEqual(proc(["", "hello", "world"]), " hello world") self.assertEqual(proc(["hello", "world"]), "hello world") self.assertIsInstance(proc(["hello", "world"]), str) @@ -641,9 +658,17 @@ class ProcessorsTest(unittest.TestCase): proc = Compose(str.upper) self.assertEqual(proc(None), None) proc = Compose(str.upper, stop_on_none=False) - self.assertRaises(ValueError, proc, None) + with pytest.raises( + ValueError, + match="Error in Compose with .* error='TypeError: (descriptor 'upper'|'str' object expected)", + ): + proc(None) proc = Compose(str.upper, lambda x: x + 1) - self.assertRaises(ValueError, proc, "hello") + with pytest.raises( + ValueError, + match="Error in Compose with .* error='TypeError: (can only|unsupported operand)", + ): + proc("hello") def test_mapcompose(self): def filter_world(x): @@ -657,9 +682,17 @@ class ProcessorsTest(unittest.TestCase): proc = MapCompose(filter_world, str.upper) self.assertEqual(proc(None), []) proc = MapCompose(filter_world, str.upper) - self.assertRaises(ValueError, proc, [1]) + with pytest.raises( + ValueError, + match="Error in MapCompose with .* error='TypeError: (descriptor 'upper'|'str' object expected)", + ): + proc([1]) proc = MapCompose(filter_world, lambda x: x + 1) - self.assertRaises(ValueError, proc, "hello") + with pytest.raises( + ValueError, + match="Error in MapCompose with .* error='TypeError: (can only|unsupported operand)", + ): + proc("hello") class SelectJmesTestCase(unittest.TestCase): diff --git a/tests/test_logstats.py b/tests/test_logstats.py index a4b002e34..6bc5b6f1f 100644 --- a/tests/test_logstats.py +++ b/tests/test_logstats.py @@ -1,6 +1,8 @@ import unittest from datetime import datetime +import pytest + from scrapy.extensions.logstats import LogStats from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider @@ -18,8 +20,9 @@ class TestLogStats(unittest.TestCase): def test_stats_calculations(self): logstats = LogStats.from_crawler(self.crawler) - with self.assertRaises(AttributeError): + with pytest.raises(AttributeError): logstats.pagesprev + with pytest.raises(AttributeError): logstats.itemsprev logstats.spider_opened(self.spider) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 3d049843a..1d89e44ce 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -7,6 +7,7 @@ from shutil import rmtree from tempfile import mkdtemp import attr +import pytest from itemadapter import ItemAdapter from twisted.trial import unittest @@ -146,11 +147,11 @@ class ImagesPipelineTestCase(unittest.TestCase): resp3 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf3.getvalue()) req = Request(url="https://dev.mydeco.com/mydeco.gif") - with self.assertRaises(ImageException): + with pytest.raises(ImageException): next(self.pipeline.get_images(response=resp1, request=req, info=object())) - with self.assertRaises(ImageException): + with pytest.raises(ImageException): next(self.pipeline.get_images(response=resp2, request=req, info=object())) - with self.assertRaises(ImageException): + with pytest.raises(ImageException): next(self.pipeline.get_images(response=resp3, request=req, info=object())) def test_get_images(self): diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 1584014b8..c223c4562 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -1,6 +1,7 @@ import tempfile import unittest +import pytest import queuelib from scrapy.http.request import Request @@ -40,9 +41,9 @@ class PriorityQueueTest(unittest.TestCase): self.crawler, FifoMemoryQueue, temp_dir ) queue.push(Request("https://example.org")) - with self.assertRaises( + with pytest.raises( NotImplementedError, - msg="The underlying queue class does not implement 'peek'", + match="The underlying queue class does not implement 'peek'", ): queue.peek() queue.close() @@ -129,9 +130,9 @@ class DownloaderAwarePriorityQueueTest(unittest.TestCase): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") self.queue.push(Request("https://example.org")) - with self.assertRaises( + with pytest.raises( NotImplementedError, - msg="The underlying queue class does not implement 'peek'", + match="The underlying queue class does not implement 'peek'", ): self.queue.peek() diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 85133038a..2c605a015 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -1,5 +1,7 @@ import unittest +import pytest + from scrapy import Request, Spider from scrapy.http import FormRequest, JsonRequest from scrapy.utils.request import request_from_dict @@ -134,11 +136,15 @@ class RequestSerializationTest(unittest.TestCase): def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) - self.assertRaises(ValueError, r.to_dict, spider=self.spider) + with pytest.raises( + ValueError, match="is not an instance method in: " - ), + match=r"\.process_spider_output must return an iterable, got ", ): yield self._get_middleware_result( ProcessSpiderOutputNonIterableMiddleware, @@ -312,9 +310,9 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): @defer.inlineCallbacks def test_coroutine(self): - with self.assertRaisesRegex( + with pytest.raises( _InvalidOutput, - r"\.process_spider_output must be an asynchronous generator", + match=r"\.process_spider_output must be an asynchronous generator", ): yield self._get_middleware_result( ProcessSpiderOutputCoroutineMiddleware, @@ -518,8 +516,8 @@ class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase): @defer.inlineCallbacks def _test_asyncgen_nodowngrade(self, *mw_classes): - with self.assertRaisesRegex( - _InvalidOutput, "Async iterable returned from .+ cannot be downgraded" + with pytest.raises( + _InvalidOutput, match="Async iterable returned from .+ cannot be downgraded" ): yield self._get_middleware_result(*mw_classes) diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 307054de7..f9eb93d6b 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,6 +1,7 @@ import logging from unittest import TestCase +import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase as TrialTestCase @@ -68,9 +69,8 @@ class TestHttpErrorMiddleware(TestCase): def test_process_spider_input(self): self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises( - HttpError, self.mw.process_spider_input, self.res404, self.spider - ) + with pytest.raises(HttpError): + self.mw.process_spider_input(self.res404, self.spider) def test_process_spider_exception(self): self.assertEqual( @@ -105,9 +105,8 @@ class TestHttpErrorMiddlewareSettings(TestCase): def test_process_spider_input(self): self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises( - HttpError, self.mw.process_spider_input, self.res404, self.spider - ) + with pytest.raises(HttpError): + self.mw.process_spider_input(self.res404, self.spider) self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider)) def test_meta_overrides_settings(self): @@ -120,14 +119,14 @@ class TestHttpErrorMiddlewareSettings(TestCase): res402.request = request self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) - self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) + with pytest.raises(HttpError): + self.mw.process_spider_input(res402, self.spider) def test_spider_override_settings(self): self.spider.handle_httpstatus_list = [404] self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) - self.assertRaises( - HttpError, self.mw.process_spider_input, self.res402, self.spider - ) + with pytest.raises(HttpError): + self.mw.process_spider_input(self.res402, self.spider) class TestHttpErrorMiddlewareHandleAll(TestCase): @@ -151,7 +150,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): res402.request = request self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) - self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) + with pytest.raises(HttpError): + self.mw.process_spider_input(res402, self.spider) def test_httperror_allow_all_false(self): crawler = get_crawler(_HttpErrorSpider) @@ -167,7 +167,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): res402 = self.res402.copy() res402.request = request_httpstatus_true - self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider) + with pytest.raises(HttpError): + mw.process_spider_input(res404, self.spider) self.assertIsNone(mw.process_spider_input(res402, self.spider)) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 4945ac25d..01a87c645 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -5,6 +5,8 @@ from typing import Any from unittest import TestCase from urllib.parse import urlparse +import pytest + from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.http import Request, Response from scrapy.settings import Settings @@ -884,7 +886,7 @@ class TestSettingsPolicyByName(TestCase): def test_invalid_name(self): settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"}) - with self.assertRaises(RuntimeError): + with pytest.raises(RuntimeError): RefererMiddleware(settings) def test_multiple_policy_tokens(self): @@ -925,7 +927,7 @@ class TestSettingsPolicyByName(TestCase): ) } ) - with self.assertRaises(RuntimeError): + with pytest.raises(RuntimeError): RefererMiddleware(settings) diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index 59d18d92e..72692afab 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -2,6 +2,7 @@ import shutil from datetime import datetime, timezone from tempfile import mkdtemp +import pytest from twisted.trial import unittest from scrapy.exceptions import NotConfigured @@ -42,4 +43,5 @@ class SpiderStateTest(unittest.TestCase): def test_not_configured(self): crawler = get_crawler(Spider) - self.assertRaises(NotConfigured, SpiderState.from_crawler, crawler) + with pytest.raises(NotConfigured): + SpiderState.from_crawler(crawler) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index a2e7ae65d..8556b75dd 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,6 +1,7 @@ import pickle import sys +import pytest from queuelib.tests import test_queue as t from scrapy.http import Request @@ -30,10 +31,17 @@ class MyLoader(ItemLoader): def nonserializable_object_test(self): q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + with pytest.raises( + ValueError, + match="unmarshallable object|Can't (get|pickle) local object|Can't pickle .*: it's not found as", + ): + q.push(lambda x: x) # Selectors should fail (lxml.html.HtmlElement objects can't be pickled) sel = Selector(text="

some text

") - self.assertRaises(ValueError, q.push, sel) + with pytest.raises( + ValueError, match="unmarshallable object|can't pickle Selector objects" + ): + q.push(sel) class FifoDiskQueueTestMixin: diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 04eeae4dc..88f6657d8 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -6,6 +6,7 @@ import shutil import tempfile import unittest +import pytest import queuelib from scrapy.http import Request @@ -69,9 +70,9 @@ class RequestQueueTestMixin: req = Request("http://www.example.com") q.push(req) self.assertEqual(len(q), 1) - with self.assertRaises( + with pytest.raises( NotImplementedError, - msg="The underlying queue class does not implement 'peek'", + match="The underlying queue class does not implement 'peek'", ): q.peek() self.assertEqual(q.pop().url, req.url) @@ -120,9 +121,9 @@ class FifoQueueMixin(RequestQueueTestMixin): q.push(req1) q.push(req2) q.push(req3) - with self.assertRaises( + with pytest.raises( NotImplementedError, - msg="The underlying queue class does not implement 'peek'", + match="The underlying queue class does not implement 'peek'", ): q.peek() self.assertEqual(len(q), 3) @@ -176,9 +177,9 @@ class LifoQueueMixin(RequestQueueTestMixin): q.push(req1) q.push(req2) q.push(req3) - with self.assertRaises( + with pytest.raises( NotImplementedError, - msg="The underlying queue class does not implement 'peek'", + match="The underlying queue class does not implement 'peek'", ): q.peek() self.assertEqual(len(q), 3) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index cbea41129..e27bb7b74 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,5 +1,7 @@ import unittest +import pytest + from scrapy.exceptions import UsageError from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( @@ -32,7 +34,9 @@ class BuildComponentListTest(unittest.TestCase): ) # Same priority raises ValueError duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20) - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, match="Some paths in .* convert to the same object" + ): build_component_list(duplicate_bs, convert=lambda x: x.lower()) def test_valid_numbers(self): @@ -58,21 +62,13 @@ class UtilsConfTestCase(unittest.TestCase): class FeedExportConfigTestCase(unittest.TestCase): def test_feed_export_config_invalid_format(self): settings = Settings() - self.assertRaises( - UsageError, - feed_process_params_from_cli, - settings, - ["items.dat"], - ) + with pytest.raises(UsageError): + feed_process_params_from_cli(settings, ["items.dat"]) def test_feed_export_config_mismatch(self): settings = Settings() - self.assertRaises( - UsageError, - feed_process_params_from_cli, - settings, - ["items1.dat", "items2.dat"], - ) + with pytest.raises(UsageError): + feed_process_params_from_cli(settings, ["items1.dat", "items2.dat"]) def test_feed_export_config_explicit_formats(self): settings = Settings() @@ -117,11 +113,9 @@ class FeedExportConfigTestCase(unittest.TestCase): ) def test_output_and_overwrite_output(self): - with self.assertRaises(UsageError): + with pytest.raises(UsageError): feed_process_params_from_cli( - Settings(), - ["output1.json"], - overwrite_output=["output2.json"], + Settings(), ["output1.json"], overwrite_output=["output2.json"] ) def test_feed_complete_default_values_from_settings_empty(self): diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index 5d99161bf..a5b438645 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -1,6 +1,7 @@ import unittest import warnings +import pytest from w3lib.http import basic_auth_header from scrapy import Request @@ -205,11 +206,11 @@ class CurlToRequestKwargsTest(unittest.TestCase): self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) def test_too_few_arguments_error(self): - self.assertRaisesRegex( + with pytest.raises( ValueError, - r"too few arguments|the following arguments are required:\s*url", - lambda: curl_to_request_kwargs("curl"), - ) + match=r"too few arguments|the following arguments are required:\s*url", + ): + curl_to_request_kwargs("curl") def test_ignore_unknown_options(self): # case 1: ignore_unknown_options=True: @@ -220,16 +221,11 @@ class CurlToRequestKwargsTest(unittest.TestCase): self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) # case 2: ignore_unknown_options=False (raise exception): - self.assertRaisesRegex( - ValueError, - "Unrecognized options:.*--bar.*--baz", - lambda: curl_to_request_kwargs( + with pytest.raises(ValueError, match="Unrecognized options:.*--bar.*--baz"): + curl_to_request_kwargs( "curl --bar --baz http://www.example.com", ignore_unknown_options=False - ), - ) + ) def test_must_start_with_curl_error(self): - self.assertRaises( - ValueError, - lambda: curl_to_request_kwargs("carl -X POST http://example.org"), - ) + with pytest.raises(ValueError, match="A curl command must start"): + curl_to_request_kwargs("carl -X POST http://example.org") diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index fadbc6daa..2e35d339a 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -87,8 +87,10 @@ class CaseInsensitiveDictMixin: def test_delete(self): d = self.dict_class({"key_lower": 1}) del d["key_LOWER"] - self.assertRaises(KeyError, d.__getitem__, "key_LOWER") - self.assertRaises(KeyError, d.__getitem__, "key_lower") + with pytest.raises(KeyError): + d["key_LOWER"] + with pytest.raises(KeyError): + d["key_lower"] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_getdefault(self): @@ -138,7 +140,8 @@ class CaseInsensitiveDictMixin: d = self.dict_class() d["a"] = 1 self.assertEqual(d.pop("A"), 1) - self.assertRaises(KeyError, d.pop, "A") + with pytest.raises(KeyError): + d.pop("A") def test_normkey(self): class MyDict(self.dict_class): @@ -279,8 +282,8 @@ class SequenceExcludeTest(unittest.TestCase): self.assertIn(set("bar"), d) # supplied sequence is a set, so checking for list (non)inclusion fails - self.assertRaises(TypeError, (0, 1, 2) in d) - self.assertRaises(TypeError, d.__contains__, ["a", "b", "c"]) + with pytest.raises(TypeError): + ["a", "b", "c"] in d # noqa: B015 for v in [-3, "test", 1.1]: self.assertNotIn(v, d) diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index dc5fbd3c3..e917b6947 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -3,6 +3,8 @@ import unittest import warnings from unittest import mock +import pytest + from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.deprecate import create_deprecated_class, update_classpath @@ -181,7 +183,8 @@ class WarnWhenSubclassedTest(unittest.TestCase): assert not issubclass(OutdatedUserClass1, OutdatedUserClass1a) assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1) - self.assertRaises(TypeError, issubclass, object(), DeprecatedName) + with pytest.raises(TypeError): + issubclass(object(), DeprecatedName) def test_isinstance(self): with warnings.catch_warnings(): diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 7b7a25db8..d40cae9c7 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -1,6 +1,8 @@ import unittest +from gzip import BadGzipFile from pathlib import Path +import pytest from w3lib.encoding import html_to_unicode from scrapy.http import Response @@ -27,9 +29,8 @@ class GunzipTest(unittest.TestCase): assert text.endswith(b" Date: Fri, 28 Feb 2025 15:18:55 +0500 Subject: [PATCH 057/491] Add SpiderLoaderProtocol. (#6694) --- scrapy/crawler.py | 10 ++++++---- scrapy/spiderloader.py | 19 ++++++++++++++++++- scrapy/utils/spider.py | 10 +++++----- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1873c90d3..1ec1e31dc 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -42,7 +42,7 @@ if TYPE_CHECKING: from collections.abc import Generator, Iterable from scrapy.logformatter import LogFormatter - from scrapy.spiderloader import SpiderLoader + from scrapy.spiderloader import SpiderLoaderProtocol from scrapy.statscollectors import StatsCollector from scrapy.utils.request import RequestFingerprinterProtocol @@ -282,19 +282,21 @@ class CrawlerRunner: ) @staticmethod - def _get_spider_loader(settings: BaseSettings) -> SpiderLoader: + def _get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol: """Get SpiderLoader instance from settings""" cls_path = settings.get("SPIDER_LOADER_CLASS") loader_cls = load_object(cls_path) verifyClass(ISpiderLoader, loader_cls) - return cast("SpiderLoader", loader_cls.from_settings(settings.frozencopy())) + return cast( + "SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy()) + ) def __init__(self, settings: dict[str, Any] | Settings | None = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) AddonManager.load_pre_crawler_settings(settings) self.settings: Settings = settings - self.spider_loader: SpiderLoader = self._get_spider_loader(settings) + self.spider_loader: SpiderLoaderProtocol = self._get_spider_loader(settings) self._crawlers: set[Crawler] = set() self._active: set[Deferred[None]] = set() self.bootstrap_failed = False diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 210e729a1..f537e0593 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -3,7 +3,7 @@ from __future__ import annotations import traceback import warnings from collections import defaultdict -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Protocol from zope.interface import implementer @@ -21,6 +21,23 @@ if TYPE_CHECKING: from scrapy.settings import BaseSettings +class SpiderLoaderProtocol(Protocol): + @classmethod + def from_settings(cls, settings: BaseSettings) -> Self: + """Return an instance of the class for the given settings""" + + def load(self, spider_name: str) -> type[Spider]: + """Return the Spider class for the given spider name. If the spider + name is not found, it must raise a KeyError.""" + + def list(self) -> list[str]: + """Return a list with the names of all spiders available in the + project""" + + def find_by_request(self, request: Request) -> __builtins__.list[str]: + """Return the list of spiders names that can handle the given request""" + + @implementer(ISpiderLoader) class SpiderLoader: """ diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 5277a292c..74fd0e354 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: from twisted.internet.defer import Deferred from scrapy import Request - from scrapy.spiderloader import SpiderLoader + from scrapy.spiderloader import SpiderLoaderProtocol logger = logging.getLogger(__name__) @@ -64,7 +64,7 @@ def iter_spider_classes(module: ModuleType) -> Iterable[type[Spider]]: @overload def spidercls_for_request( - spider_loader: SpiderLoader, + spider_loader: SpiderLoaderProtocol, request: Request, default_spidercls: type[Spider], log_none: bool = ..., @@ -74,7 +74,7 @@ def spidercls_for_request( @overload def spidercls_for_request( - spider_loader: SpiderLoader, + spider_loader: SpiderLoaderProtocol, request: Request, default_spidercls: Literal[None], log_none: bool = ..., @@ -84,7 +84,7 @@ def spidercls_for_request( @overload def spidercls_for_request( - spider_loader: SpiderLoader, + spider_loader: SpiderLoaderProtocol, request: Request, *, log_none: bool = ..., @@ -93,7 +93,7 @@ def spidercls_for_request( def spidercls_for_request( - spider_loader: SpiderLoader, + spider_loader: SpiderLoaderProtocol, request: Request, default_spidercls: type[Spider] | None = None, log_none: bool = False, From a5731c1944d8aa7b1921c543b7ad616ad131853e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 2 Mar 2025 21:04:12 +0500 Subject: [PATCH 058/491] Move most of the test utils inside tests. --- scrapy/utils/test.py | 28 +++++++++++++++ scrapy/utils/testproc.py | 9 +++++ scrapy/utils/testsite.py | 10 +++++- tests/test_command_fetch.py | 4 +-- tests/test_command_parse.py | 4 +-- tests/test_command_shell.py | 4 +-- tests/test_command_version.py | 2 +- tests/test_feedexport.py | 21 +++++++++-- tests/test_pipeline_files.py | 51 +++++++++++++++++++++++--- tests/utils/__init__.py | 0 tests/utils/testproc.py | 67 +++++++++++++++++++++++++++++++++++ tests/utils/testsite.py | 47 ++++++++++++++++++++++++ 12 files changed, 232 insertions(+), 15 deletions(-) create mode 100644 tests/utils/__init__.py create mode 100644 tests/utils/testproc.py create mode 100644 tests/utils/testsite.py diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index e89786103..db1f5c419 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -34,11 +34,23 @@ _T = TypeVar("_T") def assert_gcs_environ() -> None: + warnings.warn( + "The assert_gcs_environ() function is deprecated and will be removed in a future version of Scrapy." + " Check GCS_PROJECT_ID directly.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) if "GCS_PROJECT_ID" not in os.environ: raise SkipTest("GCS_PROJECT_ID not found") def skip_if_no_boto() -> None: + warnings.warn( + "The skip_if_no_boto() function is deprecated and will be removed in a future version of Scrapy." + " Check scrapy.utils.boto.is_botocore_available() directly.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) if not is_botocore_available(): raise SkipTest("missing botocore library") @@ -48,6 +60,11 @@ def get_gcs_content_and_delete( ) -> tuple[bytes, list[dict[str, str]], Any]: from google.cloud import storage + warnings.warn( + "The get_gcs_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) bucket = client.get_bucket(bucket) blob = bucket.get_blob(path) @@ -67,6 +84,11 @@ def get_ftp_content_and_delete( ) -> bytes: from ftplib import FTP + warnings.warn( + "The get_ftp_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) ftp = FTP() ftp.connect(host, port) ftp.login(username, password) @@ -150,6 +172,12 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]: """ from google.cloud.storage import Blob, Bucket, Client + warnings.warn( + "The mock_google_cloud_storage() function is deprecated and will be removed in a future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + client_mock = mock.create_autospec(Client) bucket_mock = mock.create_autospec(Bucket) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 85d7c940f..10f764ab8 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -2,18 +2,27 @@ from __future__ import annotations import os import sys +import warnings from typing import TYPE_CHECKING, cast from twisted.internet.defer import Deferred from twisted.internet.error import ProcessTerminated from twisted.internet.protocol import ProcessProtocol +from scrapy.exceptions import ScrapyDeprecationWarning + if TYPE_CHECKING: from collections.abc import Iterable from twisted.python.failure import Failure +warnings.warn( + "The scrapy.utils.testproc module is deprecated.", + ScrapyDeprecationWarning, +) + + class ProcessTest: command: str | None = None prefix = [sys.executable, "-m", "scrapy.cmdline"] diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index ca1f68116..f12b301fd 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -1,7 +1,15 @@ +import warnings from urllib.parse import urljoin from twisted.web import resource, server, static, util +from scrapy.exceptions import ScrapyDeprecationWarning + +warnings.warn( + "The scrapy.utils.testsite module is deprecated.", + ScrapyDeprecationWarning, +) + class SiteTest: def setUp(self): @@ -48,7 +56,7 @@ def test_site(): if __name__ == "__main__": - from twisted.internet import reactor + from twisted.internet import reactor # pylint: disable=ungrouped-imports port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") print(f"http://localhost:{port.getHost().port}/") diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index d2027d1c2..a4d7fdd30 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -1,8 +1,8 @@ from twisted.internet import defer from twisted.trial import unittest -from scrapy.utils.testproc import ProcessTest -from scrapy.utils.testsite import SiteTest +from tests.utils.testproc import ProcessTest +from tests.utils.testsite import SiteTest class FetchTest(ProcessTest, SiteTest, unittest.TestCase): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 9356d6b79..9f2c7fa13 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -7,9 +7,9 @@ from twisted.internet import defer from scrapy.commands import parse from scrapy.settings import Settings from scrapy.utils.python import to_unicode -from scrapy.utils.testproc import ProcessTest -from scrapy.utils.testsite import SiteTest from tests.test_commands import CommandTest +from tests.utils.testproc import ProcessTest +from tests.utils.testsite import SiteTest def _textmode(bstr): diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 7918d94b2..9ca5e05dc 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -7,10 +7,10 @@ from pexpect.popen_spawn import PopenSpawn from twisted.internet import defer from twisted.trial import unittest -from scrapy.utils.testproc import ProcessTest -from scrapy.utils.testsite import SiteTest from tests import NON_EXISTING_RESOLVABLE, tests_datadir from tests.mockserver import MockServer +from tests.utils.testproc import ProcessTest +from tests.utils.testsite import SiteTest class ShellTest(ProcessTest, SiteTest, unittest.TestCase): diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 18c1c531c..917f457cb 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -4,7 +4,7 @@ from twisted.internet import defer from twisted.trial import unittest import scrapy -from scrapy.utils.testproc import ProcessTest +from tests.utils.testproc import ProcessTest class VersionTest(ProcessTest, unittest.TestCase): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index b4c1b9631..8e008ab98 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -17,7 +17,7 @@ from io import BytesIO from logging import getLogger from pathlib import Path from string import ascii_letters, digits -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from unittest import mock from urllib.parse import quote, urljoin from urllib.request import pathname2url @@ -48,7 +48,7 @@ from scrapy.extensions.feedexport import ( ) from scrapy.settings import Settings from scrapy.utils.python import to_unicode -from scrapy.utils.test import get_crawler, mock_google_cloud_storage +from scrapy.utils.test import get_crawler from tests.mockserver import MockFTPServer, MockServer from tests.spiders import ItemSpider @@ -71,6 +71,23 @@ def build_url(path: str | PathLike) -> str: return urljoin("file:", path_str) +def mock_google_cloud_storage() -> tuple[Any, Any, Any]: + """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob + classes and set their proper return values. + """ + from google.cloud.storage import Blob, Bucket, Client + + client_mock = mock.create_autospec(Client) + + bucket_mock = mock.create_autospec(Bucket) + client_mock.get_bucket.return_value = bucket_mock + + blob_mock = mock.create_autospec(Blob) + bucket_mock.blob.return_value = blob_mock + + return (client_mock, bucket_mock, blob_mock) + + class FileFeedStorageTest(unittest.TestCase): def test_store_file_uri(self): path = Path(self.mktemp()).resolve() diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4c59fcfb7..05fd17207 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -6,8 +6,10 @@ import warnings from datetime import datetime from io import BytesIO from pathlib import Path +from posixpath import split from shutil import rmtree from tempfile import mkdtemp +from typing import Any from unittest import mock from urllib.parse import urlparse @@ -27,16 +29,54 @@ from scrapy.pipelines.files import ( S3FilesStore, ) from scrapy.utils.test import ( - assert_gcs_environ, get_crawler, - get_ftp_content_and_delete, - get_gcs_content_and_delete, ) from tests.mockserver import MockFTPServer from .test_pipeline_media import _mocked_download_func +def get_gcs_content_and_delete( + bucket: Any, path: str +) -> tuple[bytes, list[dict[str, str]], Any]: + from google.cloud import storage + + client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) + bucket = client.get_bucket(bucket) + blob = bucket.get_blob(path) + content = blob.download_as_string() + acl = list(blob.acl) # loads acl before it will be deleted + bucket.delete_blob(path) + return content, acl, blob + + +def get_ftp_content_and_delete( + path: str, + host: str, + port: int, + username: str, + password: str, + use_active_mode: bool = False, +) -> bytes: + from ftplib import FTP + + ftp = FTP() + ftp.connect(host, port) + ftp.login(username, password) + if use_active_mode: + ftp.set_pasv(False) + ftp_data: list[bytes] = [] + + def buffer_data(data: bytes) -> None: + ftp_data.append(data) + + ftp.retrbinary(f"RETR {path}", buffer_data) + dirname, filename = split(path) + ftp.cwd(dirname) + ftp.delete(filename) + return b"".join(ftp_data) + + class FilesPipelineTestCase(unittest.TestCase): def setUp(self): self.tempdir = mkdtemp() @@ -597,10 +637,12 @@ class TestS3FilesStore(unittest.TestCase): stub.assert_no_pending_responses() +@pytest.mark.skipif( + "GCS_PROJECT_ID" not in os.environ, reason="GCS_PROJECT_ID not found" +) class TestGCSFilesStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): - assert_gcs_environ() uri = os.environ.get("GCS_TEST_FILE_URI") if not uri: raise unittest.SkipTest("No GCS URI available for testing") @@ -629,7 +671,6 @@ class TestGCSFilesStore(unittest.TestCase): """Test to make sure that paths used to store files is the same as the one used to get already uploaded files. """ - assert_gcs_environ() try: import google.cloud.storage # noqa: F401 except ModuleNotFoundError: diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/utils/testproc.py b/tests/utils/testproc.py new file mode 100644 index 000000000..85d7c940f --- /dev/null +++ b/tests/utils/testproc.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import os +import sys +from typing import TYPE_CHECKING, cast + +from twisted.internet.defer import Deferred +from twisted.internet.error import ProcessTerminated +from twisted.internet.protocol import ProcessProtocol + +if TYPE_CHECKING: + from collections.abc import Iterable + + from twisted.python.failure import Failure + + +class ProcessTest: + command: str | None = None + prefix = [sys.executable, "-m", "scrapy.cmdline"] + cwd = os.getcwd() # trial chdirs to temp dir # noqa: PTH109 + + def execute( + self, + args: Iterable[str], + check_code: bool = True, + settings: str | None = None, + ) -> Deferred[TestProcessProtocol]: + from twisted.internet import reactor + + env = os.environ.copy() + if settings is not None: + env["SCRAPY_SETTINGS_MODULE"] = settings + assert self.command + cmd = [*self.prefix, self.command, *args] + pp = TestProcessProtocol() + pp.deferred.addCallback(self._process_finished, cmd, check_code) + reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) + return pp.deferred + + def _process_finished( + self, pp: TestProcessProtocol, cmd: list[str], check_code: bool + ) -> tuple[int, bytes, bytes]: + if pp.exitcode and check_code: + msg = f"process {cmd} exit with code {pp.exitcode}" + msg += f"\n>>> stdout <<<\n{pp.out.decode()}" + msg += "\n" + msg += f"\n>>> stderr <<<\n{pp.err.decode()}" + raise RuntimeError(msg) + return cast(int, pp.exitcode), pp.out, pp.err + + +class TestProcessProtocol(ProcessProtocol): + def __init__(self) -> None: + self.deferred: Deferred[TestProcessProtocol] = Deferred() + self.out: bytes = b"" + self.err: bytes = b"" + self.exitcode: int | None = None + + def outReceived(self, data: bytes) -> None: + self.out += data + + def errReceived(self, data: bytes) -> None: + self.err += data + + def processEnded(self, status: Failure) -> None: + self.exitcode = cast(ProcessTerminated, status.value).exitCode + self.deferred.callback(self) diff --git a/tests/utils/testsite.py b/tests/utils/testsite.py new file mode 100644 index 000000000..473738773 --- /dev/null +++ b/tests/utils/testsite.py @@ -0,0 +1,47 @@ +from urllib.parse import urljoin + +from twisted.web import resource, server, static, util + + +class SiteTest: + def setUp(self): + from twisted.internet import reactor + + super().setUp() + self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") + self.baseurl = f"http://localhost:{self.site.getHost().port}/" + + def tearDown(self): + super().tearDown() + self.site.stopListening() + + def url(self, path: str) -> str: + return urljoin(self.baseurl, path) + + +class NoMetaRefreshRedirect(util.Redirect): + def render(self, request: server.Request) -> bytes: + content = util.Redirect.render(self, request) + return content.replace( + b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"' + ) + + +def test_site(): + r = resource.Resource() + r.putChild(b"text", static.Data(b"Works", "text/plain")) + r.putChild( + b"html", + static.Data( + b"

Works

World

", + "text/html", + ), + ) + r.putChild( + b"enc-gb18030", + static.Data(b"

gb18030 encoding

", "text/html; charset=gb18030"), + ) + r.putChild(b"redirect", util.Redirect(b"/redirected")) + r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")) + r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) + return server.Site(r) From 93c076047bf5e3169feb2c29aca24e71bab0f8f0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 2 Mar 2025 21:19:24 +0500 Subject: [PATCH 059/491] Add scrapy/utils/testproc.py to collect_ignore to silence a warning. --- conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conftest.py b/conftest.py index a08ad9d05..f33ffb1a4 100644 --- a/conftest.py +++ b/conftest.py @@ -13,6 +13,7 @@ def _py_files(folder): collect_ignore = [ # not a test, but looks like a test + "scrapy/utils/testproc.py", "scrapy/utils/testsite.py", "tests/ftpserver.py", "tests/mockserver.py", From d161d1d47d445272cba35ed81eec068ed4be8b1a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 4 Mar 2025 13:31:26 +0500 Subject: [PATCH 060/491] Convert tests/test_utils* to plain asserts. (#6695) --- tests/test_utils_asyncgen.py | 6 +- tests/test_utils_asyncio.py | 9 +- tests/test_utils_conf.py | 126 +++---- tests/test_utils_console.py | 20 +- tests/test_utils_curl.py | 11 +- tests/test_utils_datatypes.py | 207 ++++++----- tests/test_utils_defer.py | 46 ++- tests/test_utils_deprecate.py | 83 +++-- tests/test_utils_display.py | 20 +- tests/test_utils_gz.py | 19 +- tests/test_utils_httpobj.py | 3 +- tests/test_utils_iterators.py | 274 +++++++-------- tests/test_utils_log.py | 43 ++- tests/test_utils_misc/__init__.py | 51 ++- ...t_return_with_argument_inside_generator.py | 63 ++-- tests/test_utils_project.py | 13 +- tests/test_utils_python.py | 113 +++--- tests/test_utils_request.py | 106 +++--- tests/test_utils_response.py | 27 +- tests/test_utils_serialize.py | 27 +- tests/test_utils_signal.py | 22 +- tests/test_utils_sitemap.py | 178 +++++----- tests/test_utils_spider.py | 14 +- tests/test_utils_template.py | 11 +- tests/test_utils_trackref.py | 42 ++- tests/test_utils_url.py | 323 +++++++----------- 26 files changed, 835 insertions(+), 1022 deletions(-) diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index 8adeea5c0..9b5a25b3a 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -4,15 +4,15 @@ from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import deferred_f_from_coro_f -class AsyncgenUtilsTest(unittest.TestCase): +class TestAsyncgenUtils(unittest.TestCase): @deferred_f_from_coro_f async def test_as_async_generator(self): ag = as_async_generator(range(42)) results = [i async for i in ag] - self.assertEqual(results, list(range(42))) + assert results == list(range(42)) @deferred_f_from_coro_f async def test_collect_asyncgen(self): ag = as_async_generator(range(42)) results = await collect_asyncgen(ag) - self.assertEqual(results, list(range(42))) + assert results == list(range(42)) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index ecac0df9c..a65a36219 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -2,7 +2,6 @@ import asyncio import warnings import pytest -from twisted.trial.unittest import TestCase from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.reactor import ( @@ -13,19 +12,17 @@ from scrapy.utils.reactor import ( @pytest.mark.usefixtures("reactor_pytest") -class AsyncioTest(TestCase): +class TestAsyncio: def test_is_asyncio_reactor_installed(self): # the result should depend only on the pytest --reactor argument - self.assertEqual( - is_asyncio_reactor_installed(), self.reactor_pytest == "asyncio" - ) + assert is_asyncio_reactor_installed() == (self.reactor_pytest == "asyncio") def test_install_asyncio_reactor(self): from twisted.internet import reactor as original_reactor with warnings.catch_warnings(record=True) as w: install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") - self.assertEqual(len(w), 0) + assert len(w) == 0 from twisted.internet import reactor # pylint: disable=reimported assert original_reactor == reactor diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index e27bb7b74..26f158380 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,5 +1,3 @@ -import unittest - import pytest from scrapy.exceptions import UsageError @@ -12,26 +10,24 @@ from scrapy.utils.conf import ( ) -class BuildComponentListTest(unittest.TestCase): +class TestBuildComponentList: def test_build_dict(self): d = {"one": 1, "two": None, "three": 8, "four": 4} - self.assertEqual( - build_component_list(d, convert=lambda x: x), ["one", "four", "three"] - ) + assert build_component_list(d, convert=lambda x: x) == ["one", "four", "three"] def test_duplicate_components_in_basesettings(self): # Higher priority takes precedence duplicate_bs = BaseSettings({"one": 1, "two": 2}, priority=0) duplicate_bs.set("ONE", 4, priority=10) - self.assertEqual( - build_component_list(duplicate_bs, convert=lambda x: x.lower()), - ["two", "one"], - ) + assert build_component_list(duplicate_bs, convert=lambda x: x.lower()) == [ + "two", + "one", + ] duplicate_bs.set("one", duplicate_bs["one"], priority=20) - self.assertEqual( - build_component_list(duplicate_bs, convert=lambda x: x.lower()), - ["one", "two"], - ) + assert build_component_list(duplicate_bs, convert=lambda x: x.lower()) == [ + "one", + "two", + ] # Same priority raises ValueError duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20) with pytest.raises( @@ -42,24 +38,24 @@ class BuildComponentListTest(unittest.TestCase): def test_valid_numbers(self): # work well with None and numeric values d = {"a": 10, "b": None, "c": 15, "d": 5.0} - self.assertEqual(build_component_list(d, convert=lambda x: x), ["d", "a", "c"]) + assert build_component_list(d, convert=lambda x: x) == ["d", "a", "c"] d = { "a": 33333333333333333333, "b": 11111111111111111111, "c": 22222222222222222222, } - self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"]) + assert build_component_list(d, convert=lambda x: x) == ["b", "c", "a"] -class UtilsConfTestCase(unittest.TestCase): +class TestUtilsConf: def test_arglist_to_dict(self): - self.assertEqual( - arglist_to_dict(["arg1=val1", "arg2=val2"]), - {"arg1": "val1", "arg2": "val2"}, - ) + assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == { + "arg1": "val1", + "arg2": "val2", + } -class FeedExportConfigTestCase(unittest.TestCase): +class TestFeedExportConfig: def test_feed_export_config_invalid_format(self): settings = Settings() with pytest.raises(UsageError): @@ -72,44 +68,36 @@ class FeedExportConfigTestCase(unittest.TestCase): def test_feed_export_config_explicit_formats(self): settings = Settings() - self.assertEqual( - { - "items_1.dat": {"format": "json"}, - "items_2.dat": {"format": "xml"}, - "items_3.dat": {"format": "csv"}, - }, - feed_process_params_from_cli( - settings, ["items_1.dat:json", "items_2.dat:xml", "items_3.dat:csv"] - ), + assert { + "items_1.dat": {"format": "json"}, + "items_2.dat": {"format": "xml"}, + "items_3.dat": {"format": "csv"}, + } == feed_process_params_from_cli( + settings, ["items_1.dat:json", "items_2.dat:xml", "items_3.dat:csv"] ) def test_feed_export_config_implicit_formats(self): settings = Settings() - self.assertEqual( - { - "items_1.json": {"format": "json"}, - "items_2.xml": {"format": "xml"}, - "items_3.csv": {"format": "csv"}, - }, - feed_process_params_from_cli( - settings, ["items_1.json", "items_2.xml", "items_3.csv"] - ), + assert { + "items_1.json": {"format": "json"}, + "items_2.xml": {"format": "xml"}, + "items_3.csv": {"format": "csv"}, + } == feed_process_params_from_cli( + settings, ["items_1.json", "items_2.xml", "items_3.csv"] ) def test_feed_export_config_stdout(self): settings = Settings() - self.assertEqual( - {"stdout:": {"format": "pickle"}}, - feed_process_params_from_cli(settings, ["-:pickle"]), + assert {"stdout:": {"format": "pickle"}} == feed_process_params_from_cli( + settings, ["-:pickle"] ) def test_feed_export_config_overwrite(self): settings = Settings() - self.assertEqual( - {"output.json": {"format": "json", "overwrite": True}}, - feed_process_params_from_cli( - settings, [], overwrite_output=["output.json"] - ), + assert { + "output.json": {"format": "json", "overwrite": True} + } == feed_process_params_from_cli( + settings, [], overwrite_output=["output.json"] ) def test_output_and_overwrite_output(self): @@ -131,18 +119,15 @@ class FeedExportConfigTestCase(unittest.TestCase): } ) new_feed = feed_complete_default_values_from_settings(feed, settings) - self.assertEqual( - new_feed, - { - "encoding": "custom encoding", - "fields": ["f1", "f2", "f3"], - "indent": 42, - "store_empty": True, - "uri_params": (1, 2, 3, 4), - "batch_item_count": 2, - "item_export_kwargs": {}, - }, - ) + assert new_feed == { + "encoding": "custom encoding", + "fields": ["f1", "f2", "f3"], + "indent": 42, + "store_empty": True, + "uri_params": (1, 2, 3, 4), + "batch_item_count": 2, + "item_export_kwargs": {}, + } def test_feed_complete_default_values_from_settings_non_empty(self): feed = { @@ -159,15 +144,12 @@ class FeedExportConfigTestCase(unittest.TestCase): } ) new_feed = feed_complete_default_values_from_settings(feed, settings) - self.assertEqual( - new_feed, - { - "encoding": "other encoding", - "fields": None, - "indent": 42, - "store_empty": True, - "uri_params": None, - "batch_item_count": 2, - "item_export_kwargs": {}, - }, - ) + assert new_feed == { + "encoding": "other encoding", + "fields": None, + "indent": 42, + "store_empty": True, + "uri_params": None, + "batch_item_count": 2, + "item_export_kwargs": {}, + } diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index 0bc86e1b9..6598bdce7 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -1,4 +1,4 @@ -import unittest +import pytest from scrapy.utils.console import get_shell_embed_func @@ -18,23 +18,23 @@ except ImportError: ipy = False -class UtilsConsoleTestCase(unittest.TestCase): +class TestUtilsConsole: def test_get_shell_embed_func(self): shell = get_shell_embed_func(["invalid"]) - self.assertEqual(shell, None) + assert shell is None shell = get_shell_embed_func(["invalid", "python"]) - self.assertTrue(callable(shell)) - self.assertEqual(shell.__name__, "_embed_standard_shell") + assert callable(shell) + assert shell.__name__ == "_embed_standard_shell" - @unittest.skipIf(not bpy, "bpython not available in testenv") + @pytest.mark.skipif(not bpy, reason="bpython not available in testenv") def test_get_shell_embed_func2(self): shell = get_shell_embed_func(["bpython"]) - self.assertTrue(callable(shell)) - self.assertEqual(shell.__name__, "_embed_bpython_shell") + assert callable(shell) + assert shell.__name__ == "_embed_bpython_shell" - @unittest.skipIf(not ipy, "IPython not available in testenv") + @pytest.mark.skipif(not ipy, reason="IPython not available in testenv") def test_get_shell_embed_func3(self): # default shell should be 'ipython' shell = get_shell_embed_func() - self.assertEqual(shell.__name__, "_embed_ipython_shell") + assert shell.__name__ == "_embed_ipython_shell" diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index a5b438645..e8dd88049 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -1,4 +1,3 @@ -import unittest import warnings import pytest @@ -8,16 +7,16 @@ from scrapy import Request from scrapy.utils.curl import curl_to_request_kwargs -class CurlToRequestKwargsTest(unittest.TestCase): +class TestCurlToRequestKwargs: maxDiff = 5000 def _test_command(self, curl_command, expected_result): result = curl_to_request_kwargs(curl_command) - self.assertEqual(result, expected_result) + assert result == expected_result try: Request(**result) except TypeError as e: - self.fail(f"Request kwargs are not correct {e}") + pytest.fail(f"Request kwargs are not correct {e}") def test_get(self): curl_command = "curl http://example.org/" @@ -203,7 +202,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): def test_get_silent(self): curl_command = 'curl --silent "www.example.com"' expected_result = {"method": "GET", "url": "http://www.example.com"} - self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) + assert curl_to_request_kwargs(curl_command) == expected_result def test_too_few_arguments_error(self): with pytest.raises( @@ -218,7 +217,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): warnings.simplefilter("ignore") curl_command = "curl --bar --baz http://www.example.com" expected_result = {"method": "GET", "url": "http://www.example.com"} - self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) + assert curl_to_request_kwargs(curl_command) == expected_result # case 2: ignore_unknown_options=False (raise exception): with pytest.raises(ValueError, match="Unrecognized options:.*--bar.*--baz"): diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 2e35d339a..75b6b0e99 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,5 +1,4 @@ import copy -import unittest import warnings from collections.abc import Iterator, Mapping, MutableMapping @@ -17,18 +16,18 @@ from scrapy.utils.datatypes import ( from scrapy.utils.python import garbage_collect -class CaseInsensitiveDictMixin: +class CaseInsensitiveDictBase: def test_init_dict(self): seq = {"red": 1, "black": 3} d = self.dict_class(seq) - self.assertEqual(d["red"], 1) - self.assertEqual(d["black"], 3) + assert d["red"] == 1 + assert d["black"] == 3 def test_init_pair_sequence(self): seq = (("red", 1), ("black", 3)) d = self.dict_class(seq) - self.assertEqual(d["red"], 1) - self.assertEqual(d["black"], 3) + assert d["red"] == 1 + assert d["black"] == 3 def test_init_mapping(self): class MyMapping(Mapping): @@ -46,8 +45,8 @@ class CaseInsensitiveDictMixin: seq = MyMapping(red=1, black=3) d = self.dict_class(seq) - self.assertEqual(d["red"], 1) - self.assertEqual(d["black"], 3) + assert d["red"] == 1 + assert d["black"] == 3 def test_init_mutable_mapping(self): class MyMutableMapping(MutableMapping): @@ -71,18 +70,18 @@ class CaseInsensitiveDictMixin: seq = MyMutableMapping(red=1, black=3) d = self.dict_class(seq) - self.assertEqual(d["red"], 1) - self.assertEqual(d["black"], 3) + assert d["red"] == 1 + assert d["black"] == 3 def test_caseless(self): d = self.dict_class() d["key_Lower"] = 1 - self.assertEqual(d["KEy_loWer"], 1) - self.assertEqual(d.get("KEy_loWer"), 1) + assert d["KEy_loWer"] == 1 + assert d.get("KEy_loWer") == 1 d["KEY_LOWER"] = 3 - self.assertEqual(d["key_Lower"], 3) - self.assertEqual(d.get("key_Lower"), 3) + assert d["key_Lower"] == 3 + assert d.get("key_Lower") == 3 def test_delete(self): d = self.dict_class({"key_lower": 1}) @@ -95,41 +94,41 @@ class CaseInsensitiveDictMixin: @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_getdefault(self): d = CaselessDict() - self.assertEqual(d.get("c", 5), 5) + assert d.get("c", 5) == 5 d["c"] = 10 - self.assertEqual(d.get("c", 5), 10) + assert d.get("c", 5) == 10 @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_setdefault(self): d = CaselessDict({"a": 1, "b": 2}) r = d.setdefault("A", 5) - self.assertEqual(r, 1) - self.assertEqual(d["A"], 1) + assert r == 1 + assert d["A"] == 1 r = d.setdefault("c", 5) - self.assertEqual(r, 5) - self.assertEqual(d["C"], 5) + assert r == 5 + assert d["C"] == 5 def test_fromkeys(self): keys = ("a", "b") d = self.dict_class.fromkeys(keys) - self.assertEqual(d["A"], None) - self.assertEqual(d["B"], None) + assert d["A"] is None + assert d["B"] is None d = self.dict_class.fromkeys(keys, 1) - self.assertEqual(d["A"], 1) - self.assertEqual(d["B"], 1) + assert d["A"] == 1 + assert d["B"] == 1 instance = self.dict_class() d = instance.fromkeys(keys) - self.assertEqual(d["A"], None) - self.assertEqual(d["B"], None) + assert d["A"] is None + assert d["B"] is None d = instance.fromkeys(keys, 1) - self.assertEqual(d["A"], 1) - self.assertEqual(d["B"], 1) + assert d["A"] == 1 + assert d["B"] == 1 def test_contains(self): d = self.dict_class() @@ -139,7 +138,7 @@ class CaseInsensitiveDictMixin: def test_pop(self): d = self.dict_class() d["a"] = 1 - self.assertEqual(d.pop("A"), 1) + assert d.pop("A") == 1 with pytest.raises(KeyError): d.pop("A") @@ -152,7 +151,7 @@ class CaseInsensitiveDictMixin: d = MyDict() d["key-one"] = 2 - self.assertEqual(list(d.keys()), ["Key-One"]) + assert list(d.keys()) == ["Key-One"] def test_normvalue(self): class MyDict(self.dict_class): @@ -164,62 +163,60 @@ class CaseInsensitiveDictMixin: normvalue = _normvalue # deprecated CaselessDict class d = MyDict({"key": 1}) - self.assertEqual(d["key"], 2) - self.assertEqual(d.get("key"), 2) + assert d["key"] == 2 + assert d.get("key") == 2 d = MyDict() d["key"] = 1 - self.assertEqual(d["key"], 2) - self.assertEqual(d.get("key"), 2) + assert d["key"] == 2 + assert d.get("key") == 2 d = MyDict() d.setdefault("key", 1) - self.assertEqual(d["key"], 2) - self.assertEqual(d.get("key"), 2) + assert d["key"] == 2 + assert d.get("key") == 2 d = MyDict() d.update({"key": 1}) - self.assertEqual(d["key"], 2) - self.assertEqual(d.get("key"), 2) + assert d["key"] == 2 + assert d.get("key") == 2 d = MyDict.fromkeys(("key",), 1) - self.assertEqual(d["key"], 2) - self.assertEqual(d.get("key"), 2) + assert d["key"] == 2 + assert d.get("key") == 2 def test_copy(self): h1 = self.dict_class({"header1": "value"}) h2 = copy.copy(h1) assert isinstance(h2, self.dict_class) - self.assertEqual(h1, h2) - self.assertEqual(h1.get("header1"), h2.get("header1")) - self.assertEqual(h1.get("header1"), h2.get("HEADER1")) + assert h1 == h2 + assert h1.get("header1") == h2.get("header1") + assert h1.get("header1") == h2.get("HEADER1") h3 = h1.copy() assert isinstance(h3, self.dict_class) - self.assertEqual(h1, h3) - self.assertEqual(h1.get("header1"), h3.get("header1")) - self.assertEqual(h1.get("header1"), h3.get("HEADER1")) + assert h1 == h3 + assert h1.get("header1") == h3.get("header1") + assert h1.get("header1") == h3.get("HEADER1") -class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase): +class TestCaseInsensitiveDict(CaseInsensitiveDictBase): dict_class = CaseInsensitiveDict def test_repr(self): d1 = self.dict_class({"foo": "bar"}) - self.assertEqual(repr(d1), "") + assert repr(d1) == "" d2 = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"}) - self.assertEqual( - repr(d2), "" - ) + assert repr(d2) == "" def test_iter(self): d = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"}) iterkeys = iter(d) - self.assertIsInstance(iterkeys, Iterator) - self.assertEqual(list(iterkeys), ["AsDf", "FoO"]) + assert isinstance(iterkeys, Iterator) + assert list(iterkeys) == ["AsDf", "FoO"] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase): +class TestCaselessDict(CaseInsensitiveDictBase): dict_class = CaselessDict def test_deprecation_message(self): @@ -227,93 +224,93 @@ class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase): warnings.filterwarnings("always", category=ScrapyDeprecationWarning) self.dict_class({"foo": "bar"}) - self.assertEqual(len(caught), 1) - self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning)) - self.assertEqual( - "scrapy.utils.datatypes.CaselessDict is deprecated," - " please use scrapy.utils.datatypes.CaseInsensitiveDict instead", - str(caught[0].message), + assert len(caught) == 1 + assert issubclass(caught[0].category, ScrapyDeprecationWarning) + assert ( + str(caught[0].message) + == "scrapy.utils.datatypes.CaselessDict is deprecated," + " please use scrapy.utils.datatypes.CaseInsensitiveDict instead" ) -class SequenceExcludeTest(unittest.TestCase): +class TestSequenceExclude: def test_list(self): seq = [1, 2, 3] d = SequenceExclude(seq) - self.assertIn(0, d) - self.assertIn(4, d) - self.assertNotIn(2, d) + assert 0 in d + assert 4 in d + assert 2 not in d def test_range(self): seq = range(10, 20) d = SequenceExclude(seq) - self.assertIn(5, d) - self.assertIn(20, d) - self.assertNotIn(15, d) + assert 5 in d + assert 20 in d + assert 15 not in d def test_range_step(self): seq = range(10, 20, 3) d = SequenceExclude(seq) are_not_in = [v for v in range(10, 20, 3) if v in d] - self.assertEqual([], are_not_in) + assert are_not_in == [] are_not_in = [v for v in range(10, 20) if v in d] - self.assertEqual([11, 12, 14, 15, 17, 18], are_not_in) + assert are_not_in == [11, 12, 14, 15, 17, 18] def test_string_seq(self): seq = "cde" d = SequenceExclude(seq) chars = "".join(v for v in "abcdefg" if v in d) - self.assertEqual("abfg", chars) + assert chars == "abfg" def test_stringset_seq(self): seq = set("cde") d = SequenceExclude(seq) chars = "".join(v for v in "abcdefg" if v in d) - self.assertEqual("abfg", chars) + assert chars == "abfg" def test_set(self): """Anything that is not in the supplied sequence will evaluate as 'in' the container.""" seq = {-3, "test", 1.1} d = SequenceExclude(seq) - self.assertIn(0, d) - self.assertIn("foo", d) - self.assertIn(3.14, d) - self.assertIn(set("bar"), d) + assert 0 in d + assert "foo" in d + assert 3.14 in d + assert set("bar") in d # supplied sequence is a set, so checking for list (non)inclusion fails with pytest.raises(TypeError): ["a", "b", "c"] in d # noqa: B015 for v in [-3, "test", 1.1]: - self.assertNotIn(v, d) + assert v not in d -class LocalCacheTest(unittest.TestCase): +class TestLocalCache: def test_cache_with_limit(self): cache = LocalCache(limit=2) cache["a"] = 1 cache["b"] = 2 cache["c"] = 3 - self.assertEqual(len(cache), 2) - self.assertNotIn("a", cache) - self.assertIn("b", cache) - self.assertIn("c", cache) - self.assertEqual(cache["b"], 2) - self.assertEqual(cache["c"], 3) + assert len(cache) == 2 + assert "a" not in cache + assert "b" in cache + assert "c" in cache + assert cache["b"] == 2 + assert cache["c"] == 3 def test_cache_without_limit(self): maximum = 10**4 cache = LocalCache() for x in range(maximum): cache[str(x)] = x - self.assertEqual(len(cache), maximum) + assert len(cache) == maximum for x in range(maximum): - self.assertIn(str(x), cache) - self.assertEqual(cache[str(x)], x) + assert str(x) in cache + assert cache[str(x)] == x -class LocalWeakReferencedCacheTest(unittest.TestCase): +class TestLocalWeakReferencedCache: def test_cache_with_limit(self): cache = LocalWeakReferencedCache(limit=2) r1 = Request("https://example.org") @@ -322,19 +319,19 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): cache[r1] = 1 cache[r2] = 2 cache[r3] = 3 - self.assertEqual(len(cache), 2) - self.assertNotIn(r1, cache) - self.assertIn(r2, cache) - self.assertIn(r3, cache) - self.assertEqual(cache[r1], None) - self.assertEqual(cache[r2], 2) - self.assertEqual(cache[r3], 3) + assert len(cache) == 2 + assert r1 not in cache + assert r2 in cache + assert r3 in cache + assert cache[r1] is None + assert cache[r2] == 2 + assert cache[r3] == 3 del r2 # PyPy takes longer to collect dead references garbage_collect() - self.assertEqual(len(cache), 1) + assert len(cache) == 1 def test_cache_non_weak_referenceable_objects(self): cache = LocalWeakReferencedCache() @@ -344,10 +341,10 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): cache[k1] = 1 cache[k2] = 2 cache[k3] = 3 - self.assertNotIn(k1, cache) - self.assertNotIn(k2, cache) - self.assertNotIn(k3, cache) - self.assertEqual(len(cache), 0) + assert k1 not in cache + assert k2 not in cache + assert k3 not in cache + assert len(cache) == 0 def test_cache_without_limit(self): max = 10**4 @@ -356,10 +353,10 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): for x in range(max): refs.append(Request(f"https://example.org/{x}")) cache[refs[-1]] = x - self.assertEqual(len(cache), max) + assert len(cache) == max for i, r in enumerate(refs): - self.assertIn(r, cache) - self.assertEqual(cache[r], i) + assert r in cache + assert cache[r] == i del r # delete reference to the last object in the list # pylint: disable=undefined-loop-variable # delete half of the objects, make sure that is reflected in the cache @@ -369,7 +366,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): # PyPy takes longer to collect dead references garbage_collect() - self.assertEqual(len(cache), max // 2) + assert len(cache) == max // 2 for i, r in enumerate(refs): - self.assertIn(r, cache) - self.assertEqual(cache[r], i) + assert r in cache + assert cache[r] == i diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 3a1030fcf..36bd8ced9 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -18,7 +18,7 @@ from scrapy.utils.defer import ( ) -class MustbeDeferredTest(unittest.TestCase): +class TestMustbeDeferred(unittest.TestCase): def test_success_function(self): steps = [] @@ -66,23 +66,19 @@ def eb1(failure, arg1, arg2): return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})" -class DeferUtilsTest(unittest.TestCase): +class TestDeferUtils(unittest.TestCase): @defer.inlineCallbacks def test_process_chain(self): x = yield process_chain([cb1, cb2, cb3], "res", "v1", "v2") - self.assertEqual(x, "(cb3 (cb2 (cb1 res v1 v2) v1 v2) v1 v2)") + assert x == "(cb3 (cb2 (cb1 res v1 v2) v1 v2) v1 v2)" - gotexc = False - try: + with pytest.raises(TypeError): yield process_chain([cb1, cb_fail, cb3], "res", "v1", "v2") - except TypeError: - gotexc = True - self.assertTrue(gotexc) @defer.inlineCallbacks def test_process_parallel(self): x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2") - self.assertEqual(x, ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"]) + assert x == ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"] def test_process_parallel_failure(self): d = process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2") @@ -90,15 +86,15 @@ class DeferUtilsTest(unittest.TestCase): return d -class IterErrbackTest(unittest.TestCase): +class TestIterErrback: def test_iter_errback_good(self): def itergood(): yield from range(10) errors = [] out = list(iter_errback(itergood(), errors.append)) - self.assertEqual(out, list(range(10))) - self.assertFalse(errors) + assert out == list(range(10)) + assert not errors def test_iter_errback_bad(self): def iterbad(): @@ -109,12 +105,12 @@ class IterErrbackTest(unittest.TestCase): errors = [] out = list(iter_errback(iterbad(), errors.append)) - self.assertEqual(out, [0, 1, 2, 3, 4]) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0].value, ZeroDivisionError) + assert out == [0, 1, 2, 3, 4] + assert len(errors) == 1 + assert isinstance(errors[0].value, ZeroDivisionError) -class AiterErrbackTest(unittest.TestCase): +class TestAiterErrback(unittest.TestCase): @deferred_f_from_coro_f async def test_aiter_errback_good(self): async def itergood(): @@ -123,8 +119,8 @@ class AiterErrbackTest(unittest.TestCase): errors = [] out = await collect_asyncgen(aiter_errback(itergood(), errors.append)) - self.assertEqual(out, list(range(10))) - self.assertFalse(errors) + assert out == list(range(10)) + assert not errors @deferred_f_from_coro_f async def test_iter_errback_bad(self): @@ -136,12 +132,12 @@ class AiterErrbackTest(unittest.TestCase): errors = [] out = await collect_asyncgen(aiter_errback(iterbad(), errors.append)) - self.assertEqual(out, [0, 1, 2, 3, 4]) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0].value, ZeroDivisionError) + assert out == [0, 1, 2, 3, 4] + assert len(errors) == 1 + assert isinstance(errors[0].value, ZeroDivisionError) -class AsyncDefTestsuiteTest(unittest.TestCase): +class TestAsyncDefTestsuite(unittest.TestCase): @deferred_f_from_coro_f async def test_deferred_f_from_coro_f(self): pass @@ -156,7 +152,7 @@ class AsyncDefTestsuiteTest(unittest.TestCase): raise RuntimeError("This is expected to be raised") -class AsyncCooperatorTest(unittest.TestCase): +class TestAsyncCooperator(unittest.TestCase): """This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage. parallel_async is called with the results of a callback (so an iterable of items, requests and None, @@ -207,7 +203,7 @@ class AsyncCooperatorTest(unittest.TestCase): ait = self.get_async_iterable(length) dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results) yield dl - self.assertEqual(list(range(length)), sorted(results)) + assert list(range(length)) == sorted(results) @defer.inlineCallbacks def test_delays(self): @@ -216,4 +212,4 @@ class AsyncCooperatorTest(unittest.TestCase): ait = self.get_async_iterable_with_delays(length) dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results) yield dl - self.assertEqual(list(range(length)), sorted(results)) + assert list(range(length)) == sorted(results) diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index e917b6947..52c165bb4 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,5 +1,4 @@ import inspect -import unittest import warnings from unittest import mock @@ -21,7 +20,7 @@ class NewName(SomeBaseClass): pass -class WarnWhenSubclassedTest(unittest.TestCase): +class TestWarnWhenSubclassed: def _mywarnings(self, w, category=MyWarning): return [x for x in w if x.category is MyWarning] @@ -30,7 +29,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): create_deprecated_class("Deprecated", NewName) w = self._mywarnings(w) - self.assertEqual(w, []) + assert w == [] def test_subclassing_warning_message(self): Deprecated = create_deprecated_class( @@ -43,15 +42,14 @@ class WarnWhenSubclassedTest(unittest.TestCase): pass w = self._mywarnings(w) - self.assertEqual(len(w), 1) - self.assertEqual( - str(w[0].message), - "tests.test_utils_deprecate.UserClass inherits from " + assert len(w) == 1 + assert ( + str(w[0].message) == "tests.test_utils_deprecate.UserClass inherits from " "deprecated class tests.test_utils_deprecate.Deprecated, " "please inherit from tests.test_utils_deprecate.NewName." - " (warning only on first subclass, there may be others)", + " (warning only on first subclass, there may be others)" ) - self.assertEqual(w[0].lineno, inspect.getsourcelines(UserClass)[1]) + assert w[0].lineno == inspect.getsourcelines(UserClass)[1] def test_custom_class_paths(self): Deprecated = create_deprecated_class( @@ -70,11 +68,11 @@ class WarnWhenSubclassedTest(unittest.TestCase): _ = Deprecated() w = self._mywarnings(w) - self.assertEqual(len(w), 2) - self.assertIn("foo.NewClass", str(w[0].message)) - self.assertIn("bar.OldClass", str(w[0].message)) - self.assertIn("foo.NewClass", str(w[1].message)) - self.assertIn("bar.OldClass", str(w[1].message)) + assert len(w) == 2 + assert "foo.NewClass" in str(w[0].message) + assert "bar.OldClass" in str(w[0].message) + assert "foo.NewClass" in str(w[1].message) + assert "bar.OldClass" in str(w[1].message) def test_subclassing_warns_only_on_direct_children(self): Deprecated = create_deprecated_class( @@ -90,8 +88,8 @@ class WarnWhenSubclassedTest(unittest.TestCase): pass w = self._mywarnings(w) - self.assertEqual(len(w), 1) - self.assertIn("UserClass", str(w[0].message)) + assert len(w) == 1 + assert "UserClass" in str(w[0].message) def test_subclassing_warns_once_by_default(self): Deprecated = create_deprecated_class( @@ -110,8 +108,8 @@ class WarnWhenSubclassedTest(unittest.TestCase): pass w = self._mywarnings(w) - self.assertEqual(len(w), 1) - self.assertIn("UserClass", str(w[0].message)) + assert len(w) == 1 + assert "UserClass" in str(w[0].message) def test_warning_on_instance(self): Deprecated = create_deprecated_class( @@ -130,13 +128,12 @@ class WarnWhenSubclassedTest(unittest.TestCase): _ = UserClass() # subclass instances don't warn w = self._mywarnings(w) - self.assertEqual(len(w), 1) - self.assertEqual( - str(w[0].message), - "tests.test_utils_deprecate.Deprecated is deprecated, " - "instantiate tests.test_utils_deprecate.NewName instead.", + assert len(w) == 1 + assert ( + str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, " + "instantiate tests.test_utils_deprecate.NewName instead." ) - self.assertEqual(w[0].lineno, lineno) + assert w[0].lineno == lineno def test_warning_auto_message(self): with warnings.catch_warnings(record=True) as w: @@ -146,8 +143,8 @@ class WarnWhenSubclassedTest(unittest.TestCase): pass msg = str(w[0].message) - self.assertIn("tests.test_utils_deprecate.NewName", msg) - self.assertIn("tests.test_utils_deprecate.Deprecated", msg) + assert "tests.test_utils_deprecate.NewName" in msg + assert "tests.test_utils_deprecate.Deprecated" in msg def test_issubclass(self): with warnings.catch_warnings(): @@ -225,7 +222,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): warnings.simplefilter("ignore", ScrapyDeprecationWarning) Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"}) - self.assertEqual(Deprecated.foo, "bar") + assert Deprecated.foo == "bar" def test_deprecate_a_class_with_custom_metaclass(self): Meta1 = type("Meta1", (type,), {}) @@ -246,7 +243,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): ) w = self._mywarnings(w) - self.assertEqual(len(w), 0, str(map(str, w))) + assert len(w) == 0, str(map(str, w)) with warnings.catch_warnings(record=True) as w: AlsoDeprecated() @@ -255,11 +252,11 @@ class WarnWhenSubclassedTest(unittest.TestCase): pass w = self._mywarnings(w) - self.assertEqual(len(w), 2) - self.assertIn("AlsoDeprecated", str(w[0].message)) - self.assertIn("foo.Bar", str(w[0].message)) - self.assertIn("AlsoDeprecated", str(w[1].message)) - self.assertIn("foo.Bar", str(w[1].message)) + assert len(w) == 2 + assert "AlsoDeprecated" in str(w[0].message) + assert "foo.Bar" in str(w[0].message) + assert "AlsoDeprecated" in str(w[1].message) + assert "foo.Bar" in str(w[1].message) def test_inspect_stack(self): with ( @@ -271,7 +268,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): class SubClass(DeprecatedName): pass - self.assertIn("Error detecting parent module", str(w[0].message)) + assert "Error detecting parent module" in str(w[0].message) @mock.patch( @@ -281,27 +278,27 @@ class WarnWhenSubclassedTest(unittest.TestCase): ("scrapy.contrib.", "scrapy.extensions."), ], ) -class UpdateClassPathTest(unittest.TestCase): +class TestUpdateClassPath: def test_old_path_gets_fixed(self): with warnings.catch_warnings(record=True) as w: output = update_classpath("scrapy.contrib.debug.Debug") - self.assertEqual(output, "scrapy.extensions.debug.Debug") - self.assertEqual(len(w), 1) - self.assertIn("scrapy.contrib.debug.Debug", str(w[0].message)) - self.assertIn("scrapy.extensions.debug.Debug", str(w[0].message)) + assert output == "scrapy.extensions.debug.Debug" + assert len(w) == 1 + assert "scrapy.contrib.debug.Debug" in str(w[0].message) + assert "scrapy.extensions.debug.Debug" in str(w[0].message) def test_sorted_replacement(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", ScrapyDeprecationWarning) output = update_classpath("scrapy.contrib.pipeline.Pipeline") - self.assertEqual(output, "scrapy.pipelines.Pipeline") + assert output == "scrapy.pipelines.Pipeline" def test_unmatched_path_stays_the_same(self): with warnings.catch_warnings(record=True) as w: output = update_classpath("scrapy.unmatched.Path") - self.assertEqual(output, "scrapy.unmatched.Path") - self.assertEqual(len(w), 0) + assert output == "scrapy.unmatched.Path" + assert len(w) == 0 def test_returns_nonstring(self): for notastring in [None, True, [1, 2, 3], object()]: - self.assertEqual(update_classpath(notastring), notastring) + assert update_classpath(notastring) == notastring diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index d1bf64828..cea564653 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -1,10 +1,10 @@ from io import StringIO -from unittest import TestCase, mock +from unittest import mock from scrapy.utils.display import pformat, pprint -class TestDisplay(TestCase): +class TestDisplay: object = {"a": 1} colorized_strings = { ( @@ -26,15 +26,15 @@ class TestDisplay(TestCase): @mock.patch("sys.stdout.isatty") def test_pformat(self, isatty): isatty.return_value = True - self.assertIn(pformat(self.object), self.colorized_strings) + assert pformat(self.object) in self.colorized_strings @mock.patch("sys.stdout.isatty") def test_pformat_dont_colorize(self, isatty): isatty.return_value = True - self.assertEqual(pformat(self.object, colorize=False), self.plain_string) + assert pformat(self.object, colorize=False) == self.plain_string def test_pformat_not_tty(self): - self.assertEqual(pformat(self.object), self.plain_string) + assert pformat(self.object) == self.plain_string @mock.patch("sys.platform", "win32") @mock.patch("platform.version") @@ -42,7 +42,7 @@ class TestDisplay(TestCase): def test_pformat_old_windows(self, isatty, version): isatty.return_value = True version.return_value = "10.0.14392" - self.assertIn(pformat(self.object), self.colorized_strings) + assert pformat(self.object) in self.colorized_strings @mock.patch("sys.platform", "win32") @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") @@ -54,7 +54,7 @@ class TestDisplay(TestCase): isatty.return_value = True version.return_value = "10.0.14393" terminal_processing.return_value = False - self.assertEqual(pformat(self.object), self.plain_string) + assert pformat(self.object) == self.plain_string @mock.patch("sys.platform", "win32") @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") @@ -64,7 +64,7 @@ class TestDisplay(TestCase): isatty.return_value = True version.return_value = "10.0.14393" terminal_processing.return_value = True - self.assertIn(pformat(self.object), self.colorized_strings) + assert pformat(self.object) in self.colorized_strings @mock.patch("sys.platform", "linux") @mock.patch("sys.stdout.isatty") @@ -81,10 +81,10 @@ class TestDisplay(TestCase): return real_import(name, globals, locals, fromlist, level) builtins.__import__ = mock_import - self.assertEqual(pformat(self.object), self.plain_string) + assert pformat(self.object) == self.plain_string builtins.__import__ = real_import def test_pprint(self): with mock.patch("sys.stdout", new=StringIO()) as mock_out: pprint(self.object) - self.assertEqual(mock_out.getvalue(), "{'a': 1}\n") + assert mock_out.getvalue() == "{'a': 1}\n" diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index d40cae9c7..c43ed152b 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -1,4 +1,3 @@ -import unittest from gzip import BadGzipFile from pathlib import Path @@ -12,17 +11,17 @@ from tests import tests_datadir SAMPLEDIR = Path(tests_datadir, "compressed") -class GunzipTest(unittest.TestCase): +class TestGunzip: def test_gunzip_basic(self): r1 = Response( "http://www.example.com", body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(), ) - self.assertTrue(gzip_magic_number(r1)) + assert gzip_magic_number(r1) r2 = Response("http://www.example.com", body=gunzip(r1.body)) - self.assertFalse(gzip_magic_number(r2)) - self.assertEqual(len(r2.body), 9950) + assert not gzip_magic_number(r2) + assert len(r2.body) == 9950 def test_gunzip_truncated(self): text = gunzip((SAMPLEDIR / "truncated-crc-error.gz").read_bytes()) @@ -37,15 +36,15 @@ class GunzipTest(unittest.TestCase): "http://www.example.com", body=(SAMPLEDIR / "truncated-crc-error-short.gz").read_bytes(), ) - self.assertTrue(gzip_magic_number(r1)) + assert gzip_magic_number(r1) r2 = Response("http://www.example.com", body=gunzip(r1.body)) assert r2.body.endswith(b"") - self.assertFalse(gzip_magic_number(r2)) + assert not gzip_magic_number(r2) def test_is_gzipped_empty(self): r1 = Response("http://www.example.com") - self.assertFalse(gzip_magic_number(r1)) + assert not gzip_magic_number(r1) def test_gunzip_illegal_eof(self): text = html_to_unicode( @@ -54,5 +53,5 @@ class GunzipTest(unittest.TestCase): expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text( encoding="utf-8" ) - self.assertEqual(len(text), len(expected_text)) - self.assertEqual(text, expected_text) + assert len(text) == len(expected_text) + assert text == expected_text diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index 741e69559..0c05ef7d6 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -1,11 +1,10 @@ -import unittest from urllib.parse import urlparse from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached -class HttpobjUtilsTest(unittest.TestCase): +class TestHttpobjUtils: def test_urlparse_cached(self): url = "http://www.example.com/index.html" request1 = Request(url) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 9ad30617a..fa0d37866 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,5 +1,4 @@ import pytest -from twisted.trial import unittest from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse, XmlResponse @@ -7,7 +6,7 @@ from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml from tests import get_testdata -class XmliterBaseTestCase: +class XmliterBase: @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter(self): body = b""" @@ -35,9 +34,10 @@ class XmliterBaseTestCase: for x in self.xmliter(response, "product") ] - self.assertEqual( - attrs, [("001", ["Name 1"], ["Type 1"]), ("002", ["Name 2"], ["Type 2"])] - ) + assert attrs == [ + ("001", ["Name 1"], ["Type 1"]), + ("002", ["Name 2"], ["Type 2"]), + ] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unusual_node(self): @@ -51,7 +51,7 @@ class XmliterBaseTestCase: nodenames = [ e.xpath("name()").getall() for e in self.xmliter(response, "matchme...") ] - self.assertEqual(nodenames, [["matchme..."]]) + assert nodenames == [["matchme..."]] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unicode(self): @@ -107,10 +107,11 @@ class XmliterBaseTestCase: for x in self.xmliter(r, "þingflokkur") ] - self.assertEqual( - attrs, - [("26", ["-"], ["80"]), ("21", ["Ab"], ["76"]), ("27", ["A"], ["27"])], - ) + assert attrs == [ + ("26", ["-"], ["80"]), + ("21", ["Ab"], ["76"]), + ("27", ["A"], ["27"]), + ] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_text(self): @@ -119,10 +120,10 @@ class XmliterBaseTestCase: "onetwo" ) - self.assertEqual( - [x.xpath("text()").getall() for x in self.xmliter(body, "product")], - [["one"], ["two"]], - ) + assert [x.xpath("text()").getall() for x in self.xmliter(body, "product")] == [ + ["one"], + ["two"], + ] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaces(self): @@ -148,21 +149,19 @@ class XmliterBaseTestCase: my_iter = self.xmliter(response, "item") node = next(my_iter) node.register_namespace("g", "http://base.google.com/ns/1.0") - self.assertEqual(node.xpath("title/text()").getall(), ["Item 1"]) - self.assertEqual(node.xpath("description/text()").getall(), ["This is item 1"]) - self.assertEqual( - node.xpath("link/text()").getall(), - ["http://www.mydummycompany.com/items/1"], - ) - self.assertEqual( - node.xpath("g:image_link/text()").getall(), - ["http://www.mydummycompany.com/images/item1.jpg"], - ) - self.assertEqual(node.xpath("g:id/text()").getall(), ["ITEM_1"]) - self.assertEqual(node.xpath("g:price/text()").getall(), ["400"]) - self.assertEqual(node.xpath("image_link/text()").getall(), []) - self.assertEqual(node.xpath("id/text()").getall(), []) - self.assertEqual(node.xpath("price/text()").getall(), []) + assert node.xpath("title/text()").getall() == ["Item 1"] + assert node.xpath("description/text()").getall() == ["This is item 1"] + assert node.xpath("link/text()").getall() == [ + "http://www.mydummycompany.com/items/1" + ] + assert node.xpath("g:image_link/text()").getall() == [ + "http://www.mydummycompany.com/images/item1.jpg" + ] + assert node.xpath("g:id/text()").getall() == ["ITEM_1"] + assert node.xpath("g:price/text()").getall() == ["400"] + assert node.xpath("image_link/text()").getall() == [] + assert node.xpath("id/text()").getall() == [] + assert node.xpath("price/text()").getall() == [] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename(self): @@ -188,10 +187,9 @@ class XmliterBaseTestCase: my_iter = self.xmliter(response, "g:image_link") node = next(my_iter) node.register_namespace("g", "http://base.google.com/ns/1.0") - self.assertEqual( - node.xpath("text()").extract(), - ["http://www.mydummycompany.com/images/item1.jpg"], - ) + assert node.xpath("text()").extract() == [ + "http://www.mydummycompany.com/images/item1.jpg" + ] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename_missing(self): @@ -246,13 +244,13 @@ class XmliterBaseTestCase: b"\n\n" ) response = XmlResponse("http://www.example.com", body=body) - self.assertEqual( - next(self.xmliter(response, "item")).get(), - "Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6", + assert ( + next(self.xmliter(response, "item")).get() + == "Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6" ) -class XmliterTestCase(XmliterBaseTestCase, unittest.TestCase): +class TestXmliter(XmliterBase): xmliter = staticmethod(xmliter) def test_deprecation(self): @@ -269,7 +267,7 @@ class XmliterTestCase(XmliterBaseTestCase, unittest.TestCase): next(self.xmliter(body, "product")) -class LxmlXmliterTestCase(XmliterBaseTestCase, unittest.TestCase): +class TestLxmlXmliter(XmliterBase): xmliter = staticmethod(xmliter_lxml) def test_xmliter_iterate_namespace(self): @@ -293,21 +291,19 @@ class LxmlXmliterTestCase(XmliterBaseTestCase, unittest.TestCase): response = XmlResponse(url="http://mydummycompany.com", body=body) no_namespace_iter = self.xmliter(response, "image_link") - self.assertEqual(len(list(no_namespace_iter)), 0) + assert len(list(no_namespace_iter)) == 0 namespace_iter = self.xmliter( response, "image_link", "http://base.google.com/ns/1.0" ) node = next(namespace_iter) - self.assertEqual( - node.xpath("text()").getall(), - ["http://www.mydummycompany.com/images/item1.jpg"], - ) + assert node.xpath("text()").getall() == [ + "http://www.mydummycompany.com/images/item1.jpg" + ] node = next(namespace_iter) - self.assertEqual( - node.xpath("text()").getall(), - ["http://www.mydummycompany.com/images/item2.jpg"], - ) + assert node.xpath("text()").getall() == [ + "http://www.mydummycompany.com/images/item2.jpg" + ] def test_xmliter_namespaces_prefix(self): body = b""" @@ -332,16 +328,16 @@ class LxmlXmliterTestCase(XmliterBaseTestCase, unittest.TestCase): my_iter = self.xmliter(response, "table", "http://www.w3.org/TR/html4/", "h") node = next(my_iter) - self.assertEqual(len(node.xpath("h:tr/h:td").getall()), 2) - self.assertEqual(node.xpath("h:tr/h:td[1]/text()").getall(), ["Apples"]) - self.assertEqual(node.xpath("h:tr/h:td[2]/text()").getall(), ["Bananas"]) + assert len(node.xpath("h:tr/h:td").getall()) == 2 + assert node.xpath("h:tr/h:td[1]/text()").getall() == ["Apples"] + assert node.xpath("h:tr/h:td[2]/text()").getall() == ["Bananas"] my_iter = self.xmliter( response, "table", "http://www.w3schools.com/furniture", "f" ) node = next(my_iter) - self.assertEqual(node.xpath("f:name/text()").getall(), ["African Coffee Table"]) + assert node.xpath("f:name/text()").getall() == ["African Coffee Table"] def test_xmliter_objtype_exception(self): i = self.xmliter(42, "product") @@ -349,42 +345,36 @@ class LxmlXmliterTestCase(XmliterBaseTestCase, unittest.TestCase): next(i) -class UtilsCsvTestCase(unittest.TestCase): +class TestUtilsCsv: def test_csviter_defaults(self): body = get_testdata("feeds", "feed-sample3.csv") response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) result = list(csv) - self.assertEqual( - result, - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert result == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assertTrue(all(isinstance(k, str) for k in result_row)) - self.assertTrue(all(isinstance(v, str) for v in result_row.values())) + assert all(isinstance(k, str) for k in result_row) + assert all(isinstance(v, str) for v in result_row.values()) def test_csviter_delimiter(self): body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t") response = TextResponse(url="http://example.com/", body=body) csv = csviter(response, delimiter="\t") - self.assertEqual( - list(csv), - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] def test_csviter_quotechar(self): body1 = get_testdata("feeds", "feed-sample6.csv") @@ -393,62 +383,50 @@ class UtilsCsvTestCase(unittest.TestCase): response1 = TextResponse(url="http://example.com/", body=body1) csv1 = csviter(response1, quotechar="'") - self.assertEqual( - list(csv1), - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv1) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] response2 = TextResponse(url="http://example.com/", body=body2) csv2 = csviter(response2, delimiter="|", quotechar="'") - self.assertEqual( - list(csv2), - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv2) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] def test_csviter_wrong_quotechar(self): body = get_testdata("feeds", "feed-sample6.csv") response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) - self.assertEqual( - list(csv), - [ - {"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"}, - { - "'id'": "2", - "'name'": "'unicode'", - "'value'": "'\xfan\xedc\xf3d\xe9\u203d'", - }, - {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"}, - {"'id'": "4", "'name'": "'empty'", "'value'": ""}, - ], - ) + assert list(csv) == [ + {"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"}, + { + "'id'": "2", + "'name'": "'unicode'", + "'value'": "'\xfan\xedc\xf3d\xe9\u203d'", + }, + {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"}, + {"'id'": "4", "'name'": "'empty'", "'value'": ""}, + ] def test_csviter_delimiter_binary_response_assume_utf8_encoding(self): body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t") response = Response(url="http://example.com/", body=body) csv = csviter(response, delimiter="\t") - self.assertEqual( - list(csv), - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] def test_csviter_headers(self): sample = get_testdata("feeds", "feed-sample3.csv").splitlines() @@ -457,15 +435,12 @@ class UtilsCsvTestCase(unittest.TestCase): response = TextResponse(url="http://example.com/", body=body) csv = csviter(response, headers=[h.decode("utf-8") for h in headers]) - self.assertEqual( - list(csv), - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] def test_csviter_falserow(self): body = get_testdata("feeds", "feed-sample3.csv") @@ -474,15 +449,12 @@ class UtilsCsvTestCase(unittest.TestCase): response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) - self.assertEqual( - list(csv), - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] def test_csviter_exception(self): body = get_testdata("feeds", "feed-sample3.csv") @@ -504,30 +476,24 @@ class UtilsCsvTestCase(unittest.TestCase): url="http://example.com/", body=body1, encoding="latin1" ) csv = csviter(response) - self.assertEqual( - list(csv), - [ - {"id": "1", "name": "latin1", "value": "test"}, - {"id": "2", "name": "something", "value": "\xf1\xe1\xe9\xf3"}, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "latin1", "value": "test"}, + {"id": "2", "name": "something", "value": "\xf1\xe1\xe9\xf3"}, + ] response = TextResponse(url="http://example.com/", body=body2, encoding="cp852") csv = csviter(response) - self.assertEqual( - list(csv), - [ - {"id": "1", "name": "cp852", "value": "test"}, - { - "id": "2", - "name": "something", - "value": "\u255a\u2569\u2569\u2569\u2550\u2550\u2557", - }, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "cp852", "value": "test"}, + { + "id": "2", + "name": "something", + "value": "\u255a\u2569\u2569\u2569\u2550\u2550\u2557", + }, + ] -class TestHelper(unittest.TestCase): +class TestHelper: bbody = b"utf8-body" ubody = bbody.decode("utf8") txtresponse = TextResponse(url="http://example.org/", body=bbody, encoding="utf-8") @@ -541,11 +507,9 @@ class TestHelper(unittest.TestCase): self._assert_type_and_value(r2, self.ubody, obj) r3 = _body_or_str(obj, unicode=False) self._assert_type_and_value(r3, self.bbody, obj) - self.assertTrue(type(r1) is type(r2)) - self.assertTrue(type(r1) is not type(r3)) + assert type(r1) is type(r2) + assert type(r1) is not type(r3) def _assert_type_and_value(self, a, b, obj): - self.assertTrue( - type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}" - ) - self.assertEqual(a, b) + assert type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}" + assert a == b diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 06e88bd10..af50fed7a 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -7,7 +7,6 @@ import sys import unittest from io import StringIO from typing import TYPE_CHECKING, Any -from unittest import TestCase import pytest from testfixtures import LogCapture @@ -27,7 +26,7 @@ if TYPE_CHECKING: from collections.abc import Mapping, MutableMapping -class FailureToExcInfoTest(unittest.TestCase): +class TestFailureToExcInfo: def test_failure(self): try: 0 / 0 @@ -35,14 +34,14 @@ class FailureToExcInfoTest(unittest.TestCase): exc_info = sys.exc_info() failure = Failure() - self.assertTupleEqual(exc_info, failure_to_exc_info(failure)) + assert exc_info == failure_to_exc_info(failure) def test_non_failure(self): - self.assertIsNone(failure_to_exc_info("test")) + assert failure_to_exc_info("test") is None -class TopLevelFormatterTest(unittest.TestCase): - def setUp(self): +class TestTopLevelFormatter: + def setup_method(self): self.handler = LogCapture() self.handler.addFilter(TopLevelFormatter(["test"])) @@ -71,8 +70,8 @@ class TopLevelFormatterTest(unittest.TestCase): log.check(("different", "WARNING", "test log msg")) -class LogCounterHandlerTest(unittest.TestCase): - def setUp(self): +class TestLogCounterHandler: + def setup_method(self): settings = {"LOG_LEVEL": "WARNING"} self.logger = logging.getLogger("test") self.logger.setLevel(logging.NOTSET) @@ -81,24 +80,24 @@ class LogCounterHandlerTest(unittest.TestCase): self.handler = LogCounterHandler(self.crawler) self.logger.addHandler(self.handler) - def tearDown(self): + def teardown_method(self): self.logger.propagate = True self.logger.removeHandler(self.handler) def test_init(self): - self.assertIsNone(self.crawler.stats.get_value("log_count/DEBUG")) - self.assertIsNone(self.crawler.stats.get_value("log_count/INFO")) - self.assertIsNone(self.crawler.stats.get_value("log_count/WARNING")) - self.assertIsNone(self.crawler.stats.get_value("log_count/ERROR")) - self.assertIsNone(self.crawler.stats.get_value("log_count/CRITICAL")) + assert self.crawler.stats.get_value("log_count/DEBUG") is None + assert self.crawler.stats.get_value("log_count/INFO") is None + assert self.crawler.stats.get_value("log_count/WARNING") is None + assert self.crawler.stats.get_value("log_count/ERROR") is None + assert self.crawler.stats.get_value("log_count/CRITICAL") is None def test_accepted_level(self): self.logger.error("test log msg") - self.assertEqual(self.crawler.stats.get_value("log_count/ERROR"), 1) + assert self.crawler.stats.get_value("log_count/ERROR") == 1 def test_filtered_out_level(self): self.logger.debug("test log msg") - self.assertIsNone(self.crawler.stats.get_value("log_count/INFO")) + assert self.crawler.stats.get_value("log_count/INFO") is None class StreamLoggerTest(unittest.TestCase): @@ -152,8 +151,8 @@ def test_spider_logger_adapter_process( assert result_kwargs == expected_extra -class LoggingTestCase(TestCase): - def setUp(self): +class TestLogging: + def setup_method(self): self.log_stream = StringIO() handler = logging.StreamHandler(self.log_stream) logger = logging.getLogger("log_spider") @@ -163,7 +162,7 @@ class LoggingTestCase(TestCase): self.logger = logger self.spider = LogSpider() - def tearDown(self): + def teardown_method(self): self.logger.removeHandler(self.handler) def test_debug_logging(self): @@ -202,8 +201,8 @@ class LoggingTestCase(TestCase): assert log_contents == f"{log_message}\n" -class LoggingWithExtraTestCase(TestCase): - def setUp(self): +class TestLoggingWithExtra: + def setup_method(self): self.log_stream = StringIO() handler = logging.StreamHandler(self.log_stream) formatter = logging.Formatter( @@ -218,7 +217,7 @@ class LoggingWithExtraTestCase(TestCase): self.spider = LogSpider() self.regex_pattern = re.compile(r"^]+>$") - def tearDown(self): + def teardown_method(self): self.logger.removeHandler(self.handler) def test_debug_logging(self): diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index a67e16962..b330819d9 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -1,6 +1,5 @@ import os import sys -import unittest from pathlib import Path from unittest import mock @@ -18,18 +17,18 @@ from scrapy.utils.misc import ( ) -class UtilsMiscTestCase(unittest.TestCase): +class TestUtilsMisc: def test_load_object_class(self): obj = load_object(Field) - self.assertIs(obj, Field) + assert obj is Field obj = load_object("scrapy.item.Field") - self.assertIs(obj, Field) + assert obj is Field def test_load_object_function(self): obj = load_object(load_object) - self.assertIs(obj, load_object) + assert obj is load_object obj = load_object("scrapy.utils.misc.load_object") - self.assertIs(obj, load_object) + assert obj is load_object def test_load_object_exceptions(self): with pytest.raises(ImportError): @@ -47,20 +46,20 @@ class UtilsMiscTestCase(unittest.TestCase): "tests.test_utils_misc.test_walk_modules.mod.mod0", "tests.test_utils_misc.test_walk_modules.mod1", ] - self.assertEqual({m.__name__ for m in mods}, set(expected)) + assert {m.__name__ for m in mods} == set(expected) mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod") expected = [ "tests.test_utils_misc.test_walk_modules.mod", "tests.test_utils_misc.test_walk_modules.mod.mod0", ] - self.assertEqual({m.__name__ for m in mods}, set(expected)) + assert {m.__name__ for m in mods} == set(expected) mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod1") expected = [ "tests.test_utils_misc.test_walk_modules.mod1", ] - self.assertEqual({m.__name__ for m in mods}, set(expected)) + assert {m.__name__ for m in mods} == set(expected) with pytest.raises(ImportError): walk_modules("nomodule999") @@ -76,7 +75,7 @@ class UtilsMiscTestCase(unittest.TestCase): "testegg.spiders.b", "testegg", ] - self.assertEqual({m.__name__ for m in mods}, set(expected)) + assert {m.__name__ for m in mods} == set(expected) finally: sys.path.remove(egg) @@ -90,15 +89,13 @@ class UtilsMiscTestCase(unittest.TestCase): assert hasattr(arg_to_iter([1, 2, 3]), "__iter__") assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__") - self.assertEqual(list(arg_to_iter(None)), []) - self.assertEqual(list(arg_to_iter("lala")), ["lala"]) - self.assertEqual(list(arg_to_iter(100)), [100]) - self.assertEqual(list(arg_to_iter(c for c in "abc")), ["a", "b", "c"]) - self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3]) - self.assertEqual(list(arg_to_iter({"a": 1})), [{"a": 1}]) - self.assertEqual( - list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")] - ) + assert not list(arg_to_iter(None)) + assert list(arg_to_iter("lala")) == ["lala"] + assert list(arg_to_iter(100)) == [100] + assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"] + assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3] + assert list(arg_to_iter({"a": 1})) == [{"a": 1}] + assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_create_instance(self): @@ -110,10 +107,10 @@ class UtilsMiscTestCase(unittest.TestCase): def _test_with_settings(mock, settings): create_instance(mock, settings, None, *args, **kwargs) if hasattr(mock, "from_crawler"): - self.assertEqual(mock.from_crawler.call_count, 0) + assert mock.from_crawler.call_count == 0 if hasattr(mock, "from_settings"): mock.from_settings.assert_called_once_with(settings, *args, **kwargs) - self.assertEqual(mock.call_count, 0) + assert mock.call_count == 0 else: mock.assert_called_once_with(*args, **kwargs) @@ -122,11 +119,11 @@ class UtilsMiscTestCase(unittest.TestCase): if hasattr(mock, "from_crawler"): mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) if hasattr(mock, "from_settings"): - self.assertEqual(mock.from_settings.call_count, 0) - self.assertEqual(mock.call_count, 0) + assert mock.from_settings.call_count == 0 + assert mock.call_count == 0 elif hasattr(mock, "from_settings"): mock.from_settings.assert_called_once_with(settings, *args, **kwargs) - self.assertEqual(mock.call_count, 0) + assert mock.call_count == 0 else: mock.assert_called_once_with(*args, **kwargs) @@ -172,11 +169,11 @@ class UtilsMiscTestCase(unittest.TestCase): if hasattr(mock, "from_crawler"): mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) if hasattr(mock, "from_settings"): - self.assertEqual(mock.from_settings.call_count, 0) - self.assertEqual(mock.call_count, 0) + assert mock.from_settings.call_count == 0 + assert mock.call_count == 0 elif hasattr(mock, "from_settings"): mock.from_settings.assert_called_once_with(settings, *args, **kwargs) - self.assertEqual(mock.call_count, 0) + assert mock.call_count == 0 else: mock.assert_called_once_with(*args, **kwargs) diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 480729d11..81a83c3d7 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -1,4 +1,3 @@ -import unittest import warnings from functools import partial from unittest import mock @@ -40,7 +39,7 @@ def generator_that_returns_stuff(): return 3 -class UtilsMiscPy3TestCase(unittest.TestCase): +class TestUtilsMisc: def test_generators_return_something(self): def f1(): yield 1 @@ -77,27 +76,27 @@ https://example.org with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, top_level_return_something) - self.assertEqual(len(w), 1) - self.assertIn( - 'The "NoneType.top_level_return_something" method is a generator', - str(w[0].message), + assert len(w) == 1 + assert ( + 'The "NoneType.top_level_return_something" method is a generator' + in str(w[0].message) ) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, f1) - self.assertEqual(len(w), 1) - self.assertIn('The "NoneType.f1" method is a generator', str(w[0].message)) + assert len(w) == 1 + assert 'The "NoneType.f1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, g1) - self.assertEqual(len(w), 1) - self.assertIn('The "NoneType.g1" method is a generator', str(w[0].message)) + assert len(w) == 1 + assert 'The "NoneType.g1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, h1) - self.assertEqual(len(w), 1) - self.assertIn('The "NoneType.h1" method is a generator', str(w[0].message)) + assert len(w) == 1 + assert 'The "NoneType.h1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, i1) - self.assertEqual(len(w), 1) - self.assertIn('The "NoneType.i1" method is a generator', str(w[0].message)) + assert len(w) == 1 + assert 'The "NoneType.i1" method is a generator' in str(w[0].message) def test_generators_return_none(self): def f2(): @@ -144,28 +143,28 @@ https://example.org with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, top_level_return_none) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, f2) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, g2) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, h2) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, i2) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, j2) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, k2) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, l2) - self.assertEqual(len(w), 0) + assert len(w) == 0 def test_generators_return_none_with_decorator(self): def decorator(func): @@ -225,28 +224,28 @@ https://example.org with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, top_level_return_none) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, f3) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, g3) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, h3) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, i3) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, j3) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, k3) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, l3) - self.assertEqual(len(w), 0) + assert len(w) == 0 @mock.patch( "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error @@ -254,8 +253,8 @@ https://example.org def test_indentation_error(self): with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, top_level_return_none) - self.assertEqual(len(w), 1) - self.assertIn("Unable to determine", str(w[0].message)) + assert len(w) == 1 + assert "Unable to determine" in str(w[0].message) def test_partial(self): def cb(arg1, arg2): diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 1d149d48d..aa250be69 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -2,7 +2,6 @@ import contextlib import os import shutil import tempfile -import unittest import warnings from pathlib import Path @@ -25,21 +24,21 @@ def inside_a_project(): shutil.rmtree(project_dir) -class ProjectUtilsTest(unittest.TestCase): +class TestProjectUtils: def test_data_path_outside_project(self): - self.assertEqual(str(Path(".scrapy", "somepath")), data_path("somepath")) + assert str(Path(".scrapy", "somepath")) == data_path("somepath") abspath = str(Path(os.path.sep, "absolute", "path")) - self.assertEqual(abspath, data_path(abspath)) + assert abspath == data_path(abspath) def test_data_path_inside_project(self): with inside_a_project() as proj_path: expected = Path(proj_path, ".scrapy", "somepath") - self.assertEqual(expected.resolve(), Path(data_path("somepath")).resolve()) + assert expected.resolve() == Path(data_path("somepath")).resolve() abspath = str(Path(os.path.sep, "absolute", "path").resolve()) - self.assertEqual(abspath, data_path(abspath)) + assert abspath == data_path(abspath) -class GetProjectSettingsTestCase(unittest.TestCase): +class TestGetProjectSettings: def test_valid_envvar(self): value = "tests.test_cmdline.settings" envvars = { diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3b0739276..291646ad7 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -21,18 +21,18 @@ from scrapy.utils.python import ( ) -class MutableChainTest(unittest.TestCase): +class TestMutableChain: def test_mutablechain(self): m = MutableChain(range(2), [2, 3], (4, 5)) m.extend(range(6, 7)) m.extend([7, 8]) m.extend([9, 10], (11, 12)) - self.assertEqual(next(m), 0) - self.assertEqual(m.__next__(), 1) - self.assertEqual(list(m), list(range(2, 13))) + assert next(m) == 0 + assert m.__next__() == 1 + assert list(m) == list(range(2, 13)) -class MutableAsyncChainTest(unittest.TestCase): +class TestMutableAsyncChain(unittest.TestCase): @staticmethod async def g1(): for i in range(3): @@ -62,9 +62,9 @@ class MutableAsyncChainTest(unittest.TestCase): m.extend(self.g2()) m.extend(self.g3()) - self.assertEqual(await m.__anext__(), 0) + assert await m.__anext__() == 0 results = await collect_asyncgen(m) - self.assertEqual(results, list(range(1, 10))) + assert results == list(range(1, 10)) @deferred_f_from_coro_f async def test_mutableasyncchain_exc(self): @@ -73,46 +73,46 @@ class MutableAsyncChainTest(unittest.TestCase): m.extend(self.g3()) results = await collect_asyncgen(aiter_errback(m, lambda _: None)) - self.assertEqual(results, list(range(5))) + assert results == list(range(5)) -class ToUnicodeTest(unittest.TestCase): +class TestToUnicode: def test_converting_an_utf8_encoded_string_to_unicode(self): - self.assertEqual(to_unicode(b"lel\xc3\xb1e"), "lel\xf1e") + assert to_unicode(b"lel\xc3\xb1e") == "lel\xf1e" def test_converting_a_latin_1_encoded_string_to_unicode(self): - self.assertEqual(to_unicode(b"lel\xf1e", "latin-1"), "lel\xf1e") + assert to_unicode(b"lel\xf1e", "latin-1") == "lel\xf1e" def test_converting_a_unicode_to_unicode_should_return_the_same_object(self): - self.assertEqual(to_unicode("\xf1e\xf1e\xf1e"), "\xf1e\xf1e\xf1e") + assert to_unicode("\xf1e\xf1e\xf1e") == "\xf1e\xf1e\xf1e" - def test_converting_a_strange_object_should_raise_TypeError(self): + def test_converting_a_strange_object_should_raise_type_error(self): with pytest.raises(TypeError): to_unicode(423) def test_errors_argument(self): - self.assertEqual(to_unicode(b"a\xedb", "utf-8", errors="replace"), "a\ufffdb") + assert to_unicode(b"a\xedb", "utf-8", errors="replace") == "a\ufffdb" -class ToBytesTest(unittest.TestCase): +class TestToBytes: def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self): - self.assertEqual(to_bytes("\xa3 49"), b"\xc2\xa3 49") + assert to_bytes("\xa3 49") == b"\xc2\xa3 49" def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self): - self.assertEqual(to_bytes("\xa3 49", "latin-1"), b"\xa3 49") + assert to_bytes("\xa3 49", "latin-1") == b"\xa3 49" def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self): - self.assertEqual(to_bytes(b"lel\xf1e"), b"lel\xf1e") + assert to_bytes(b"lel\xf1e") == b"lel\xf1e" - def test_converting_a_strange_object_should_raise_TypeError(self): + def test_converting_a_strange_object_should_raise_type_error(self): with pytest.raises(TypeError): to_bytes(pytest) def test_errors_argument(self): - self.assertEqual(to_bytes("a\ufffdb", "latin-1", errors="replace"), b"a?b") + assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b" -class MemoizedMethodTest(unittest.TestCase): +class TestMemoizedMethod: def test_memoizemethod_noargs(self): class A: @memoizemethod_noargs @@ -130,7 +130,7 @@ class MemoizedMethodTest(unittest.TestCase): assert one is not three -class BinaryIsTextTest(unittest.TestCase): +class TestBinaryIsText: def test_binaryistext(self): assert binary_is_text(b"hello") @@ -144,7 +144,7 @@ class BinaryIsTextTest(unittest.TestCase): assert not binary_is_text(b"\x02\xa3") -class UtilsPythonTestCase(unittest.TestCase): +class TestUtilsPython: @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_equal_attributes(self): class Obj: @@ -153,31 +153,31 @@ class UtilsPythonTestCase(unittest.TestCase): a = Obj() b = Obj() # no attributes given return False - self.assertFalse(equal_attributes(a, b, [])) + assert not equal_attributes(a, b, []) # nonexistent attributes - self.assertFalse(equal_attributes(a, b, ["x", "y"])) + assert not equal_attributes(a, b, ["x", "y"]) a.x = 1 b.x = 1 # equal attribute - self.assertTrue(equal_attributes(a, b, ["x"])) + assert equal_attributes(a, b, ["x"]) b.y = 2 # obj1 has no attribute y - self.assertFalse(equal_attributes(a, b, ["x", "y"])) + assert not equal_attributes(a, b, ["x", "y"]) a.y = 2 # equal attributes - self.assertTrue(equal_attributes(a, b, ["x", "y"])) + assert equal_attributes(a, b, ["x", "y"]) a.y = 1 # different attributes - self.assertFalse(equal_attributes(a, b, ["x", "y"])) + assert not equal_attributes(a, b, ["x", "y"]) # test callable a.meta = {} b.meta = {} - self.assertTrue(equal_attributes(a, b, ["meta"])) + assert equal_attributes(a, b, ["meta"]) # compare ['meta']['a'] a.meta["z"] = 1 @@ -189,10 +189,10 @@ class UtilsPythonTestCase(unittest.TestCase): def compare_z(obj): return get_z(get_meta(obj)) - self.assertTrue(equal_attributes(a, b, [compare_z, "x"])) + assert equal_attributes(a, b, [compare_z, "x"]) # fail z equality a.meta["z"] = 2 - self.assertFalse(equal_attributes(a, b, [compare_z, "x"])) + assert not equal_attributes(a, b, [compare_z, "x"]) def test_get_func_args(self): def f1(a, b, c): @@ -221,36 +221,35 @@ class UtilsPythonTestCase(unittest.TestCase): partial_f2 = functools.partial(f1, b=None) partial_f3 = functools.partial(partial_f2, None) - self.assertEqual(get_func_args(f1), ["a", "b", "c"]) - self.assertEqual(get_func_args(f2), ["a", "b", "c"]) - self.assertEqual(get_func_args(f3), ["a", "b", "c"]) - self.assertEqual(get_func_args(A), ["a", "b", "c"]) - self.assertEqual(get_func_args(a.method), ["a", "b", "c"]) - self.assertEqual(get_func_args(partial_f1), ["b", "c"]) - self.assertEqual(get_func_args(partial_f2), ["a", "c"]) - self.assertEqual(get_func_args(partial_f3), ["c"]) - self.assertEqual(get_func_args(cal), ["a", "b", "c"]) - self.assertEqual(get_func_args(object), []) - self.assertEqual(get_func_args(str.split, stripself=True), ["sep", "maxsplit"]) - self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"]) + assert get_func_args(f1) == ["a", "b", "c"] + assert get_func_args(f2) == ["a", "b", "c"] + assert get_func_args(f3) == ["a", "b", "c"] + assert get_func_args(A) == ["a", "b", "c"] + assert get_func_args(a.method) == ["a", "b", "c"] + assert get_func_args(partial_f1) == ["b", "c"] + assert get_func_args(partial_f2) == ["a", "c"] + assert get_func_args(partial_f3) == ["c"] + assert get_func_args(cal) == ["a", "b", "c"] + assert get_func_args(object) == [] + assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"] + assert get_func_args(" ".join, stripself=True) == ["iterable"] if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy": # the correct and correctly extracted signature - self.assertEqual( - get_func_args(operator.itemgetter(2), stripself=True), ["obj"] - ) + assert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"] elif platform.python_implementation() == "CPython": # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951) - self.assertIn( - get_func_args(operator.itemgetter(2), stripself=True), - [[], ["args", "kwargs"]], - ) + assert get_func_args(operator.itemgetter(2), stripself=True) in [ + [], + ["args", "kwargs"], + ] def test_without_none_values(self): - self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) - self.assertEqual(without_none_values((1, None, 3, 4)), (1, 3, 4)) - self.assertEqual( - without_none_values({"one": 1, "none": None, "three": 3, "four": 4}), - {"one": 1, "three": 3, "four": 4}, - ) + assert without_none_values([1, None, 3, 4]) == [1, 3, 4] + assert without_none_values((1, None, 3, 4)) == (1, 3, 4) + assert without_none_values({"one": 1, "none": None, "three": 3, "four": 4}) == { + "one": 1, + "three": 3, + "four": 4, + } diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 51bca9a31..5b8509753 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -import unittest import warnings from hashlib import sha1 from weakref import WeakKeyDictionary @@ -21,23 +20,23 @@ from scrapy.utils.request import ( from scrapy.utils.test import get_crawler -class UtilsRequestTest(unittest.TestCase): +class TestUtilsRequest: @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_request_authenticate(self): r = Request("http://www.example.com") request_authenticate(r, "someuser", "somepass") - self.assertEqual(r.headers["Authorization"], b"Basic c29tZXVzZXI6c29tZXBhc3M=") + assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M=" def test_request_httprepr(self): r1 = Request("http://www.example.com") - self.assertEqual( - request_httprepr(r1), b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n" + assert ( + request_httprepr(r1) == b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n" ) r1 = Request("http://www.example.com/some/page.html?arg=1") - self.assertEqual( - request_httprepr(r1), - b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + assert ( + request_httprepr(r1) + == b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n" ) r1 = Request( @@ -46,9 +45,9 @@ class UtilsRequestTest(unittest.TestCase): headers={"Content-type": b"text/html"}, body=b"Some body", ) - self.assertEqual( - request_httprepr(r1), - b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body", + assert ( + request_httprepr(r1) + == b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body" ) def test_request_httprepr_for_non_http_request(self): @@ -57,7 +56,7 @@ class UtilsRequestTest(unittest.TestCase): request_httprepr(Request("ftp://localhost/tmp/foo.txt")) -class FingerprintTest(unittest.TestCase): +class TestFingerprint: maxDiff = None function: staticmethod = staticmethod(fingerprint) @@ -147,23 +146,23 @@ class FingerprintTest(unittest.TestCase): def test_query_string_key_order(self): r1 = Request("http://www.example.com/query?id=111&cat=222") r2 = Request("http://www.example.com/query?cat=222&id=111") - self.assertEqual(self.function(r1), self.function(r1)) - self.assertEqual(self.function(r1), self.function(r2)) + assert self.function(r1) == self.function(r1) + assert self.function(r1) == self.function(r2) def test_query_string_key_without_value(self): r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78132,199") r2 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199") - self.assertNotEqual(self.function(r1), self.function(r2)) + assert self.function(r1) != self.function(r2) def test_caching(self): r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199") - self.assertEqual(self.function(r1), self.cache[r1][self.default_cache_key]) + assert self.function(r1) == self.cache[r1][self.default_cache_key] def test_header(self): r1 = Request("http://www.example.com/members/offers.html") r2 = Request("http://www.example.com/members/offers.html") r2.headers["SESSIONID"] = b"somehash" - self.assertEqual(self.function(r1), self.function(r2)) + assert self.function(r1) == self.function(r2) def test_headers(self): r1 = Request("http://www.example.com/") @@ -173,36 +172,35 @@ class FingerprintTest(unittest.TestCase): r3.headers["Accept-Language"] = b"en" r3.headers["SESSIONID"] = b"somehash" - self.assertEqual(self.function(r1), self.function(r2), self.function(r3)) + assert self.function(r1) == self.function(r2) == self.function(r3) - self.assertEqual( - self.function(r1), self.function(r1, include_headers=["Accept-Language"]) + assert self.function(r1) == self.function( + r1, include_headers=["Accept-Language"] ) - self.assertNotEqual( - self.function(r1), self.function(r2, include_headers=["Accept-Language"]) + assert self.function(r1) != self.function( + r2, include_headers=["Accept-Language"] ) - self.assertEqual( - self.function(r3, include_headers=["accept-language", "sessionid"]), - self.function(r3, include_headers=["SESSIONID", "Accept-Language"]), - ) + assert self.function( + r3, include_headers=["accept-language", "sessionid"] + ) == self.function(r3, include_headers=["SESSIONID", "Accept-Language"]) def test_fragment(self): r1 = Request("http://www.example.com/test.html") r2 = Request("http://www.example.com/test.html#fragment") - self.assertEqual(self.function(r1), self.function(r2)) - self.assertEqual(self.function(r1), self.function(r1, keep_fragments=True)) - self.assertNotEqual(self.function(r2), self.function(r2, keep_fragments=True)) - self.assertNotEqual(self.function(r1), self.function(r2, keep_fragments=True)) + assert self.function(r1) == self.function(r2) + assert self.function(r1) == self.function(r1, keep_fragments=True) + assert self.function(r2) != self.function(r2, keep_fragments=True) + assert self.function(r1) != self.function(r2, keep_fragments=True) def test_method_and_body(self): r1 = Request("http://www.example.com") r2 = Request("http://www.example.com", method="POST") r3 = Request("http://www.example.com", method="POST", body=b"request body") - self.assertNotEqual(self.function(r1), self.function(r2)) - self.assertNotEqual(self.function(r2), self.function(r3)) + assert self.function(r1) != self.function(r2) + assert self.function(r2) != self.function(r3) def test_request_replace(self): # cached fingerprint must be cleared on request copy @@ -210,7 +208,7 @@ class FingerprintTest(unittest.TestCase): fp1 = self.function(r1) r2 = r1.replace(url="http://www.example.com/other") fp2 = self.function(r2) - self.assertNotEqual(fp1, fp2) + assert fp1 != fp2 def test_part_separation(self): # An old implementation used to serialize request data in a way that @@ -219,7 +217,7 @@ class FingerprintTest(unittest.TestCase): fp1 = self.function(r1) r2 = Request("http://www.example.com/f", body=b"oo") fp2 = self.function(r2) - self.assertNotEqual(fp1, fp2) + assert fp1 != fp2 def test_hashes(self): """Test hardcoded hashes, to make sure future changes to not introduce @@ -228,7 +226,7 @@ class FingerprintTest(unittest.TestCase): self.function(request, **kwargs) for request, _, kwargs in self.known_hashes ] expected = [_fingerprint for _, _fingerprint, _ in self.known_hashes] - self.assertEqual(actual, expected) + assert actual == expected REQUEST_OBJECTS_TO_TEST = ( @@ -260,13 +258,12 @@ REQUEST_OBJECTS_TO_TEST = ( ) -class RequestFingerprinterTestCase(unittest.TestCase): +class TestRequestFingerprinter: def test_default_implementation(self): crawler = get_crawler() request = Request("https://example.com") - self.assertEqual( - crawler.request_fingerprinter.fingerprint(request), - fingerprint(request), + assert crawler.request_fingerprinter.fingerprint(request) == fingerprint( + request ) def test_deprecated_implementation(self): @@ -276,14 +273,13 @@ class RequestFingerprinterTestCase(unittest.TestCase): with warnings.catch_warnings(record=True) as logged_warnings: crawler = get_crawler(settings_dict=settings) request = Request("https://example.com") - self.assertEqual( - crawler.request_fingerprinter.fingerprint(request), - fingerprint(request), + assert crawler.request_fingerprinter.fingerprint(request) == fingerprint( + request ) - self.assertTrue(logged_warnings) + assert logged_warnings -class CustomRequestFingerprinterTestCase(unittest.TestCase): +class TestCustomRequestFingerprinter: def test_include_headers(self): class RequestFingerprinter: def fingerprint(self, request): @@ -298,7 +294,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com", headers={"X-ID": "2"}) fp2 = crawler.request_fingerprinter.fingerprint(r2) - self.assertNotEqual(fp1, fp2) + assert fp1 != fp2 def test_dont_canonicalize(self): class RequestFingerprinter: @@ -320,7 +316,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com?a=2&a=1") fp2 = crawler.request_fingerprinter.fingerprint(r2) - self.assertNotEqual(fp1, fp2) + assert fp1 != fp2 def test_meta(self): class RequestFingerprinter: @@ -342,10 +338,10 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): fp3 = crawler.request_fingerprinter.fingerprint(r3) r4 = Request("http://www.example.com", meta={"fingerprint": "b"}) fp4 = crawler.request_fingerprinter.fingerprint(r4) - self.assertNotEqual(fp1, fp2) - self.assertNotEqual(fp1, fp4) - self.assertNotEqual(fp2, fp4) - self.assertEqual(fp2, fp3) + assert fp1 != fp2 + assert fp1 != fp4 + assert fp2 != fp4 + assert fp2 == fp3 def test_from_crawler(self): class RequestFingerprinter: @@ -367,7 +363,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) - self.assertEqual(fingerprint, settings["FINGERPRINT"]) + assert fingerprint == settings["FINGERPRINT"] def test_from_settings(self): class RequestFingerprinter: @@ -391,7 +387,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) - self.assertEqual(fingerprint, settings["FINGERPRINT"]) + assert fingerprint == settings["FINGERPRINT"] def test_from_crawler_and_settings(self): class RequestFingerprinter: @@ -418,13 +414,13 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) - self.assertEqual(fingerprint, settings["FINGERPRINT"]) + assert fingerprint == settings["FINGERPRINT"] -class RequestToCurlTest(unittest.TestCase): +class TestRequestToCurl: def _test_request(self, request_object, expected_curl_command): curl_command = request_to_curl(request_object) - self.assertEqual(curl_command, expected_curl_command) + assert curl_command == expected_curl_command def test_get(self): request_object = Request("https://www.example.com") diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index af7906781..80f2f25d5 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,4 +1,3 @@ -import unittest from pathlib import Path from time import process_time from urllib.parse import urlparse @@ -16,7 +15,7 @@ from scrapy.utils.response import ( ) -class ResponseUtilsTest(unittest.TestCase): +class TestResponseUtils: dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response") def test_open_in_browser(self): @@ -28,7 +27,7 @@ class ResponseUtilsTest(unittest.TestCase): if not path or not Path(path).exists(): path = burl.replace("file://", "") bbody = Path(path).read_bytes() - self.assertIn(b'', bbody) + assert b'' in bbody return True response = HtmlResponse(url, body=body) @@ -68,9 +67,9 @@ class ResponseUtilsTest(unittest.TestCase): """, ) - self.assertEqual(get_meta_refresh(r1), (5.0, "http://example.org/newpage")) - self.assertEqual(get_meta_refresh(r2), (None, None)) - self.assertEqual(get_meta_refresh(r3), (None, None)) + assert get_meta_refresh(r1) == (5.0, "http://example.org/newpage") + assert get_meta_refresh(r2) == (None, None) + assert get_meta_refresh(r3) == (None, None) def test_get_base_url(self): resp = HtmlResponse( @@ -81,19 +80,19 @@ class ResponseUtilsTest(unittest.TestCase): blahablsdfsal& """, ) - self.assertEqual(get_base_url(resp), "http://www.example.com/img/") + assert get_base_url(resp) == "http://www.example.com/img/" resp2 = HtmlResponse( "http://www.example.com", body=b""" blahablsdfsal&""", ) - self.assertEqual(get_base_url(resp2), "http://www.example.com") + assert get_base_url(resp2) == "http://www.example.com" def test_response_status_message(self): - self.assertEqual(response_status_message(200), "200 OK") - self.assertEqual(response_status_message(404), "404 Not Found") - self.assertEqual(response_status_message(573), "573 Unknown Status") + assert response_status_message(200) == "200 OK" + assert response_status_message(404) == "404 Not Found" + assert response_status_message(573) == "573 Unknown Status" def test_inject_base_url(self): url = "http://www.example.com" @@ -103,7 +102,7 @@ class ResponseUtilsTest(unittest.TestCase): if not path or not Path(path).exists(): path = burl.replace("file://", "") bbody = Path(path).read_bytes() - self.assertEqual(bbody.count(b''), 1) + assert bbody.count(b'') == 1 return True r1 = HtmlResponse( @@ -185,7 +184,7 @@ class ResponseUtilsTest(unittest.TestCase): open_in_browser(response, lambda url: True) end_time = process_time() - self.assertLess(end_time - start_time, MAX_CPU_TIME) + assert end_time - start_time < MAX_CPU_TIME def test_open_in_browser_redos_head(self): MAX_CPU_TIME = 0.02 @@ -202,7 +201,7 @@ class ResponseUtilsTest(unittest.TestCase): open_in_browser(response, lambda url: True) end_time = process_time() - self.assertLess(end_time - start_time, MAX_CPU_TIME) + assert end_time - start_time < MAX_CPU_TIME @pytest.mark.parametrize( diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py index 055db4e5b..2ee3850b0 100644 --- a/tests/test_utils_serialize.py +++ b/tests/test_utils_serialize.py @@ -1,7 +1,6 @@ import dataclasses import datetime import json -import unittest from decimal import Decimal import attr @@ -11,8 +10,8 @@ from scrapy.http import Request, Response from scrapy.utils.serialize import ScrapyJSONEncoder -class JsonEncoderTestCase(unittest.TestCase): - def setUp(self): +class TestJsonEncoder: + def setup_method(self): self.encoder = ScrapyJSONEncoder(sort_keys=True) def test_encode_decode(self): @@ -39,24 +38,22 @@ class JsonEncoderTestCase(unittest.TestCase): (s, ss), (dt_set, dt_sets), ]: - self.assertEqual( - self.encoder.encode(input), json.dumps(output, sort_keys=True) - ) + assert self.encoder.encode(input) == json.dumps(output, sort_keys=True) def test_encode_deferred(self): - self.assertIn("Deferred", self.encoder.encode(defer.Deferred())) + assert "Deferred" in self.encoder.encode(defer.Deferred()) def test_encode_request(self): r = Request("http://www.example.com/lala") rs = self.encoder.encode(r) - self.assertIn(r.method, rs) - self.assertIn(r.url, rs) + assert r.method in rs + assert r.url in rs def test_encode_response(self): r = Response("http://www.example.com/lala") rs = self.encoder.encode(r) - self.assertIn(r.url, rs) - self.assertIn(str(r.status), rs) + assert r.url in rs + assert str(r.status) in rs def test_encode_dataclass_item(self) -> None: @dataclasses.dataclass @@ -67,9 +64,7 @@ class JsonEncoderTestCase(unittest.TestCase): item = TestDataClass(name="Product", url="http://product.org", price=1) encoded = self.encoder.encode(item) - self.assertEqual( - encoded, '{"name": "Product", "price": 1, "url": "http://product.org"}' - ) + assert encoded == '{"name": "Product", "price": 1, "url": "http://product.org"}' def test_encode_attrs_item(self): @attr.s @@ -80,6 +75,4 @@ class JsonEncoderTestCase(unittest.TestCase): item = AttrsItem(name="Product", url="http://product.org", price=1) encoded = self.encoder.encode(item) - self.assertEqual( - encoded, '{"name": "Product", "price": 1, "url": "http://product.org"}' - ) + assert encoded == '{"name": "Product", "price": 1, "url": "http://product.org"}' diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 858813e83..751a77031 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -11,7 +11,7 @@ from scrapy.utils.signal import send_catch_log, send_catch_log_deferred from scrapy.utils.test import get_from_asyncio_queue -class SendCatchLogTest(unittest.TestCase): +class TestSendCatchLog(unittest.TestCase): @defer.inlineCallbacks def test_send_catch_log(self): test_signal = object() @@ -29,13 +29,13 @@ class SendCatchLogTest(unittest.TestCase): assert self.error_handler in handlers_called assert self.ok_handler in handlers_called - self.assertEqual(len(log.records), 1) + assert len(log.records) == 1 record = log.records[0] - self.assertIn("error_handler", record.getMessage()) - self.assertEqual(record.levelname, "ERROR") - self.assertEqual(result[0][0], self.error_handler) - self.assertIsInstance(result[0][1], Failure) - self.assertEqual(result[1], (self.ok_handler, "OK")) + assert "error_handler" in record.getMessage() + assert record.levelname == "ERROR" + assert result[0][0] == self.error_handler # pylint: disable=comparison-with-callable + assert isinstance(result[0][1], Failure) + assert result[1] == (self.ok_handler, "OK") dispatcher.disconnect(self.error_handler, signal=test_signal) dispatcher.disconnect(self.ok_handler, signal=test_signal) @@ -53,7 +53,7 @@ class SendCatchLogTest(unittest.TestCase): return "OK" -class SendCatchLogDeferredTest(SendCatchLogTest): +class SendCatchLogDeferredTest(TestSendCatchLog): def _get_result(self, signal, *a, **kw): return send_catch_log_deferred(signal, *a, **kw) @@ -85,7 +85,7 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): return await get_from_asyncio_queue("OK") -class SendCatchLogTest2(unittest.TestCase): +class TestSendCatchLog2: def test_error_logged_if_deferred_not_supported(self): def test_handler(): return defer.Deferred() @@ -94,6 +94,6 @@ class SendCatchLogTest2(unittest.TestCase): dispatcher.connect(test_handler, test_signal) with LogCapture() as log: send_catch_log(test_signal) - self.assertEqual(len(log.records), 1) - self.assertIn("Cannot return deferreds from signal handler", str(log)) + assert len(log.records) == 1 + assert "Cannot return deferreds from signal handler" in str(log) dispatcher.disconnect(test_handler, test_signal) diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index 69a459d8b..36d612009 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -1,9 +1,7 @@ -import unittest - from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots -class SitemapTest(unittest.TestCase): +class TestSitemap: def test_sitemap(self): s = Sitemap( b""" @@ -23,23 +21,20 @@ class SitemapTest(unittest.TestCase): """ ) assert s.type == "urlset" - self.assertEqual( - list(s), - [ - { - "priority": "1", - "loc": "http://www.example.com/", - "lastmod": "2009-08-16", - "changefreq": "daily", - }, - { - "priority": "0.8", - "loc": "http://www.example.com/Special-Offers.html", - "lastmod": "2009-08-16", - "changefreq": "weekly", - }, - ], - ) + assert list(s) == [ + { + "priority": "1", + "loc": "http://www.example.com/", + "lastmod": "2009-08-16", + "changefreq": "daily", + }, + { + "priority": "0.8", + "loc": "http://www.example.com/Special-Offers.html", + "lastmod": "2009-08-16", + "changefreq": "weekly", + }, + ] def test_sitemap_index(self): s = Sitemap( @@ -56,19 +51,16 @@ class SitemapTest(unittest.TestCase): """ ) assert s.type == "sitemapindex" - self.assertEqual( - list(s), - [ - { - "loc": "http://www.example.com/sitemap1.xml.gz", - "lastmod": "2004-10-01T18:23:17+00:00", - }, - { - "loc": "http://www.example.com/sitemap2.xml.gz", - "lastmod": "2005-01-01", - }, - ], - ) + assert list(s) == [ + { + "loc": "http://www.example.com/sitemap1.xml.gz", + "lastmod": "2004-10-01T18:23:17+00:00", + }, + { + "loc": "http://www.example.com/sitemap2.xml.gz", + "lastmod": "2005-01-01", + }, + ] def test_sitemap_strip(self): """Assert we can deal with trailing spaces inside tags - we've @@ -90,18 +82,15 @@ class SitemapTest(unittest.TestCase): """ ) - self.assertEqual( - list(s), - [ - { - "priority": "1", - "loc": "http://www.example.com/", - "lastmod": "2009-08-16", - "changefreq": "daily", - }, - {"loc": "http://www.example.com/2", "lastmod": ""}, - ], - ) + assert list(s) == [ + { + "priority": "1", + "loc": "http://www.example.com/", + "lastmod": "2009-08-16", + "changefreq": "daily", + }, + {"loc": "http://www.example.com/2", "lastmod": ""}, + ] def test_sitemap_wrong_ns(self): """We have seen sitemaps with wrongs ns. Presumably, Google still works @@ -122,18 +111,15 @@ class SitemapTest(unittest.TestCase): """ ) - self.assertEqual( - list(s), - [ - { - "priority": "1", - "loc": "http://www.example.com/", - "lastmod": "2009-08-16", - "changefreq": "daily", - }, - {"loc": "http://www.example.com/2", "lastmod": ""}, - ], - ) + assert list(s) == [ + { + "priority": "1", + "loc": "http://www.example.com/", + "lastmod": "2009-08-16", + "changefreq": "daily", + }, + {"loc": "http://www.example.com/2", "lastmod": ""}, + ] def test_sitemap_wrong_ns2(self): """We have seen sitemaps with wrongs ns. Presumably, Google still works @@ -155,18 +141,15 @@ class SitemapTest(unittest.TestCase): """ ) assert s.type == "urlset" - self.assertEqual( - list(s), - [ - { - "priority": "1", - "loc": "http://www.example.com/", - "lastmod": "2009-08-16", - "changefreq": "daily", - }, - {"loc": "http://www.example.com/2", "lastmod": ""}, - ], - ) + assert list(s) == [ + { + "priority": "1", + "loc": "http://www.example.com/", + "lastmod": "2009-08-16", + "changefreq": "daily", + }, + {"loc": "http://www.example.com/2", "lastmod": ""}, + ] def test_sitemap_urls_from_robots(self): robots = """User-agent: * @@ -187,15 +170,14 @@ Sitemap: /sitemap-relative-url.xml Disallow: /forum/search/ Disallow: /forum/active/ """ - self.assertEqual( - list(sitemap_urls_from_robots(robots, base_url="http://example.com")), - [ - "http://example.com/sitemap.xml", - "http://example.com/sitemap-product-index.xml", - "http://example.com/sitemap-uppercase.xml", - "http://example.com/sitemap-relative-url.xml", - ], - ) + assert list( + sitemap_urls_from_robots(robots, base_url="http://example.com") + ) == [ + "http://example.com/sitemap.xml", + "http://example.com/sitemap-product-index.xml", + "http://example.com/sitemap-uppercase.xml", + "http://example.com/sitemap-relative-url.xml", + ] def test_sitemap_blanklines(self): """Assert we can deal with starting blank lines before tag""" @@ -224,14 +206,11 @@ Disallow: /forum/active/ """ ) - self.assertEqual( - list(s), - [ - {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap1.xml"}, - {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap2.xml"}, - {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap3.xml"}, - ], - ) + assert list(s) == [ + {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap1.xml"}, + {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap2.xml"}, + {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap3.xml"}, + ] def test_comment(self): s = Sitemap( @@ -245,7 +224,7 @@ Disallow: /forum/active/ """ ) - self.assertEqual(list(s), [{"loc": "http://www.example.com/"}]) + assert list(s) == [{"loc": "http://www.example.com/"}] def test_alternate(self): s = Sitemap( @@ -265,19 +244,16 @@ Disallow: /forum/active/ """ ) - self.assertEqual( - list(s), - [ - { - "loc": "http://www.example.com/english/", - "alternate": [ - "http://www.example.com/deutsch/", - "http://www.example.com/schweiz-deutsch/", - "http://www.example.com/english/", - ], - } - ], - ) + assert list(s) == [ + { + "loc": "http://www.example.com/english/", + "alternate": [ + "http://www.example.com/deutsch/", + "http://www.example.com/schweiz-deutsch/", + "http://www.example.com/english/", + ], + } + ] def test_xml_entity_expansion(self): s = Sitemap( @@ -294,4 +270,4 @@ Disallow: /forum/active/ """ ) - self.assertEqual(list(s), [{"loc": "http://127.0.0.1:8000/"}]) + assert list(s) == [{"loc": "http://127.0.0.1:8000/"}] diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index df8f37103..43e603f6c 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -1,5 +1,3 @@ -import unittest - from scrapy import Spider from scrapy.http import Request from scrapy.item import Item @@ -14,19 +12,19 @@ class MySpider2(Spider): name = "myspider2" -class UtilsSpidersTestCase(unittest.TestCase): +class TestUtilsSpiders: def test_iterate_spider_output(self): i = Item() r = Request("http://scrapytest.org") o = object() - self.assertEqual(list(iterate_spider_output(i)), [i]) - self.assertEqual(list(iterate_spider_output(r)), [r]) - self.assertEqual(list(iterate_spider_output(o)), [o]) - self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o]) + assert list(iterate_spider_output(i)) == [i] + assert list(iterate_spider_output(r)) == [r] + assert list(iterate_spider_output(o)) == [o] + assert list(iterate_spider_output([r, i, o])) == [r, i, o] def test_iter_spider_classes(self): import tests.test_utils_spider # noqa: PLW0406 # pylint: disable=import-self it = iter_spider_classes(tests.test_utils_spider) - self.assertEqual(set(it), {MySpider1, MySpider2}) + assert set(it) == {MySpider1, MySpider2} diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index fc6c33200..0b845fdb0 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -1,4 +1,3 @@ -import unittest from pathlib import Path from shutil import rmtree from tempfile import mkdtemp @@ -6,11 +5,11 @@ from tempfile import mkdtemp from scrapy.utils.template import render_templatefile -class UtilsRenderTemplateFileTestCase(unittest.TestCase): - def setUp(self): +class TestUtilsRenderTemplateFile: + def setup_method(self): self.tmp_path = mkdtemp() - def tearDown(self): + def teardown_method(self): rmtree(self.tmp_path) def test_simple_render(self): @@ -26,8 +25,8 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): render_templatefile(template_path, **context) - self.assertFalse(template_path.exists()) - self.assertEqual(render_path.read_text(encoding="utf8"), rendered) + assert not template_path.exists() + assert render_path.read_text(encoding="utf8") == rendered render_path.unlink() assert not render_path.exists() # Failure of test itself diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 58efad585..a945163ef 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -1,4 +1,3 @@ -import unittest from io import StringIO from time import sleep, time from unittest import mock @@ -16,48 +15,48 @@ class Bar(trackref.object_ref): pass -class TrackrefTestCase(unittest.TestCase): - def setUp(self): +class TestTrackref: + def setup_method(self): trackref.live_refs.clear() def test_format_live_refs(self): o1 = Foo() # noqa: F841 o2 = Bar() # noqa: F841 o3 = Foo() # noqa: F841 - self.assertEqual( - trackref.format_live_refs(), - """\ + assert ( + trackref.format_live_refs() + == """\ Live References Bar 1 oldest: 0s ago Foo 2 oldest: 0s ago -""", +""" ) - self.assertEqual( - trackref.format_live_refs(ignore=Foo), - """\ + assert ( + trackref.format_live_refs(ignore=Foo) + == """\ Live References Bar 1 oldest: 0s ago -""", +""" ) @mock.patch("sys.stdout", new_callable=StringIO) def test_print_live_refs_empty(self, stdout): trackref.print_live_refs() - self.assertEqual(stdout.getvalue(), "Live References\n\n\n") + assert stdout.getvalue() == "Live References\n\n\n" @mock.patch("sys.stdout", new_callable=StringIO) def test_print_live_refs_with_objects(self, stdout): o1 = Foo() # noqa: F841 trackref.print_live_refs() - self.assertEqual( - stdout.getvalue(), - """\ + assert ( + stdout.getvalue() + == """\ Live References -Foo 1 oldest: 0s ago\n\n""", +Foo 1 oldest: 0s ago\n\n""" ) def test_get_oldest(self): @@ -75,15 +74,12 @@ Foo 1 oldest: 0s ago\n\n""", raise SkipTest("time.time is not precise enough") o3 = Foo() # noqa: F841 - self.assertIs(trackref.get_oldest("Foo"), o1) - self.assertIs(trackref.get_oldest("Bar"), o2) - self.assertIsNone(trackref.get_oldest("XXX")) + assert trackref.get_oldest("Foo") is o1 + assert trackref.get_oldest("Bar") is o2 + assert trackref.get_oldest("XXX") is None def test_iter_all(self): o1 = Foo() o2 = Bar() # noqa: F841 o3 = Foo() - self.assertEqual( - set(trackref.iter_all("Foo")), - {o1, o3}, - ) + assert set(trackref.iter_all("Foo")) == {o1, o3} diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index e99ef40c4..5841d6866 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -18,301 +18,240 @@ from scrapy.utils.url import ( # type: ignore[attr-defined] ) -class UrlUtilsTest(unittest.TestCase): +class TestUrlUtils: def test_url_is_from_any_domain(self): url = "http://www.wheele-bin-art.co.uk/get/product/123" - self.assertTrue(url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])) - self.assertFalse(url_is_from_any_domain(url, ["art.co.uk"])) + assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"]) + assert not url_is_from_any_domain(url, ["art.co.uk"]) url = "http://wheele-bin-art.co.uk/get/product/123" - self.assertTrue(url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])) - self.assertFalse(url_is_from_any_domain(url, ["art.co.uk"])) + assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"]) + assert not url_is_from_any_domain(url, ["art.co.uk"]) url = "http://www.Wheele-Bin-Art.co.uk/get/product/123" - self.assertTrue(url_is_from_any_domain(url, ["wheele-bin-art.CO.UK"])) - self.assertTrue(url_is_from_any_domain(url, ["WHEELE-BIN-ART.CO.UK"])) + assert url_is_from_any_domain(url, ["wheele-bin-art.CO.UK"]) + assert url_is_from_any_domain(url, ["WHEELE-BIN-ART.CO.UK"]) url = "http://192.169.0.15:8080/mypage.html" - self.assertTrue(url_is_from_any_domain(url, ["192.169.0.15:8080"])) - self.assertFalse(url_is_from_any_domain(url, ["192.169.0.15"])) + assert url_is_from_any_domain(url, ["192.169.0.15:8080"]) + assert not url_is_from_any_domain(url, ["192.169.0.15"]) url = ( "javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20" "javascript:%20document.orderform_2581_1190810811.submit%28%29" ) - self.assertFalse(url_is_from_any_domain(url, ["testdomain.com"])) - self.assertFalse( - url_is_from_any_domain(url + ".testdomain.com", ["testdomain.com"]) - ) + assert not url_is_from_any_domain(url, ["testdomain.com"]) + assert not url_is_from_any_domain(url + ".testdomain.com", ["testdomain.com"]) def test_url_is_from_spider(self): spider = Spider(name="example.com") - self.assertTrue( - url_is_from_spider("http://www.example.com/some/page.html", spider) - ) - self.assertTrue( - url_is_from_spider("http://sub.example.com/some/page.html", spider) - ) - self.assertFalse( - url_is_from_spider("http://www.example.org/some/page.html", spider) - ) - self.assertFalse( - url_is_from_spider("http://www.example.net/some/page.html", spider) - ) + assert url_is_from_spider("http://www.example.com/some/page.html", spider) + assert url_is_from_spider("http://sub.example.com/some/page.html", spider) + assert not url_is_from_spider("http://www.example.org/some/page.html", spider) + assert not url_is_from_spider("http://www.example.net/some/page.html", spider) def test_url_is_from_spider_class_attributes(self): class MySpider(Spider): name = "example.com" - self.assertTrue( - url_is_from_spider("http://www.example.com/some/page.html", MySpider) - ) - self.assertTrue( - url_is_from_spider("http://sub.example.com/some/page.html", MySpider) - ) - self.assertFalse( - url_is_from_spider("http://www.example.org/some/page.html", MySpider) - ) - self.assertFalse( - url_is_from_spider("http://www.example.net/some/page.html", MySpider) - ) + assert url_is_from_spider("http://www.example.com/some/page.html", MySpider) + assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider) + assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider) + assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider) def test_url_is_from_spider_with_allowed_domains(self): spider = Spider( name="example.com", allowed_domains=["example.org", "example.net"] ) - self.assertTrue( - url_is_from_spider("http://www.example.com/some/page.html", spider) - ) - self.assertTrue( - url_is_from_spider("http://sub.example.com/some/page.html", spider) - ) - self.assertTrue(url_is_from_spider("http://example.com/some/page.html", spider)) - self.assertTrue( - url_is_from_spider("http://www.example.org/some/page.html", spider) - ) - self.assertTrue( - url_is_from_spider("http://www.example.net/some/page.html", spider) - ) - self.assertFalse( - url_is_from_spider("http://www.example.us/some/page.html", spider) - ) + assert url_is_from_spider("http://www.example.com/some/page.html", spider) + assert url_is_from_spider("http://sub.example.com/some/page.html", spider) + assert url_is_from_spider("http://example.com/some/page.html", spider) + assert url_is_from_spider("http://www.example.org/some/page.html", spider) + assert url_is_from_spider("http://www.example.net/some/page.html", spider) + assert not url_is_from_spider("http://www.example.us/some/page.html", spider) spider = Spider( name="example.com", allowed_domains={"example.com", "example.net"} ) - self.assertTrue( - url_is_from_spider("http://www.example.com/some/page.html", spider) - ) + assert url_is_from_spider("http://www.example.com/some/page.html", spider) spider = Spider( name="example.com", allowed_domains=("example.com", "example.net") ) - self.assertTrue( - url_is_from_spider("http://www.example.com/some/page.html", spider) - ) + assert url_is_from_spider("http://www.example.com/some/page.html", spider) def test_url_is_from_spider_with_allowed_domains_class_attributes(self): class MySpider(Spider): name = "example.com" allowed_domains = ("example.org", "example.net") - self.assertTrue( - url_is_from_spider("http://www.example.com/some/page.html", MySpider) - ) - self.assertTrue( - url_is_from_spider("http://sub.example.com/some/page.html", MySpider) - ) - self.assertTrue( - url_is_from_spider("http://example.com/some/page.html", MySpider) - ) - self.assertTrue( - url_is_from_spider("http://www.example.org/some/page.html", MySpider) - ) - self.assertTrue( - url_is_from_spider("http://www.example.net/some/page.html", MySpider) - ) - self.assertFalse( - url_is_from_spider("http://www.example.us/some/page.html", MySpider) - ) + assert url_is_from_spider("http://www.example.com/some/page.html", MySpider) + assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider) + assert url_is_from_spider("http://example.com/some/page.html", MySpider) + assert url_is_from_spider("http://www.example.org/some/page.html", MySpider) + assert url_is_from_spider("http://www.example.net/some/page.html", MySpider) + assert not url_is_from_spider("http://www.example.us/some/page.html", MySpider) def test_url_has_any_extension(self): deny_extensions = {"." + e for e in arg_to_iter(IGNORED_EXTENSIONS)} - self.assertTrue( - url_has_any_extension( - "http://www.example.com/archive.tar.gz", deny_extensions - ) + assert url_has_any_extension( + "http://www.example.com/archive.tar.gz", deny_extensions ) - self.assertTrue( - url_has_any_extension("http://www.example.com/page.doc", deny_extensions) + assert url_has_any_extension("http://www.example.com/page.doc", deny_extensions) + assert url_has_any_extension("http://www.example.com/page.pdf", deny_extensions) + assert not url_has_any_extension( + "http://www.example.com/page.htm", deny_extensions ) - self.assertTrue( - url_has_any_extension("http://www.example.com/page.pdf", deny_extensions) - ) - self.assertFalse( - url_has_any_extension("http://www.example.com/page.htm", deny_extensions) - ) - self.assertFalse( - url_has_any_extension("http://www.example.com/", deny_extensions) - ) - self.assertFalse( - url_has_any_extension( - "http://www.example.com/page.doc.html", deny_extensions - ) + assert not url_has_any_extension("http://www.example.com/", deny_extensions) + assert not url_has_any_extension( + "http://www.example.com/page.doc.html", deny_extensions ) -class AddHttpIfNoScheme(unittest.TestCase): +class TestAddHttpIfNoScheme: def test_add_scheme(self): - self.assertEqual( - add_http_if_no_scheme("www.example.com"), "http://www.example.com" - ) + assert add_http_if_no_scheme("www.example.com") == "http://www.example.com" def test_without_subdomain(self): - self.assertEqual(add_http_if_no_scheme("example.com"), "http://example.com") + assert add_http_if_no_scheme("example.com") == "http://example.com" def test_path(self): - self.assertEqual( - add_http_if_no_scheme("www.example.com/some/page.html"), - "http://www.example.com/some/page.html", + assert ( + add_http_if_no_scheme("www.example.com/some/page.html") + == "http://www.example.com/some/page.html" ) def test_port(self): - self.assertEqual( - add_http_if_no_scheme("www.example.com:80"), "http://www.example.com:80" + assert ( + add_http_if_no_scheme("www.example.com:80") == "http://www.example.com:80" ) def test_fragment(self): - self.assertEqual( - add_http_if_no_scheme("www.example.com/some/page#frag"), - "http://www.example.com/some/page#frag", + assert ( + add_http_if_no_scheme("www.example.com/some/page#frag") + == "http://www.example.com/some/page#frag" ) def test_query(self): - self.assertEqual( - add_http_if_no_scheme("www.example.com/do?a=1&b=2&c=3"), - "http://www.example.com/do?a=1&b=2&c=3", + assert ( + add_http_if_no_scheme("www.example.com/do?a=1&b=2&c=3") + == "http://www.example.com/do?a=1&b=2&c=3" ) def test_username_password(self): - self.assertEqual( - add_http_if_no_scheme("username:password@www.example.com"), - "http://username:password@www.example.com", + assert ( + add_http_if_no_scheme("username:password@www.example.com") + == "http://username:password@www.example.com" ) def test_complete_url(self): - self.assertEqual( + assert ( add_http_if_no_scheme( "username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag" - ), - "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag", + ) + == "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag" ) def test_preserve_http(self): - self.assertEqual( - add_http_if_no_scheme("http://www.example.com"), "http://www.example.com" + assert ( + add_http_if_no_scheme("http://www.example.com") == "http://www.example.com" ) def test_preserve_http_without_subdomain(self): - self.assertEqual( - add_http_if_no_scheme("http://example.com"), "http://example.com" - ) + assert add_http_if_no_scheme("http://example.com") == "http://example.com" def test_preserve_http_path(self): - self.assertEqual( - add_http_if_no_scheme("http://www.example.com/some/page.html"), - "http://www.example.com/some/page.html", + assert ( + add_http_if_no_scheme("http://www.example.com/some/page.html") + == "http://www.example.com/some/page.html" ) def test_preserve_http_port(self): - self.assertEqual( - add_http_if_no_scheme("http://www.example.com:80"), - "http://www.example.com:80", + assert ( + add_http_if_no_scheme("http://www.example.com:80") + == "http://www.example.com:80" ) def test_preserve_http_fragment(self): - self.assertEqual( - add_http_if_no_scheme("http://www.example.com/some/page#frag"), - "http://www.example.com/some/page#frag", + assert ( + add_http_if_no_scheme("http://www.example.com/some/page#frag") + == "http://www.example.com/some/page#frag" ) def test_preserve_http_query(self): - self.assertEqual( - add_http_if_no_scheme("http://www.example.com/do?a=1&b=2&c=3"), - "http://www.example.com/do?a=1&b=2&c=3", + assert ( + add_http_if_no_scheme("http://www.example.com/do?a=1&b=2&c=3") + == "http://www.example.com/do?a=1&b=2&c=3" ) def test_preserve_http_username_password(self): - self.assertEqual( - add_http_if_no_scheme("http://username:password@www.example.com"), - "http://username:password@www.example.com", + assert ( + add_http_if_no_scheme("http://username:password@www.example.com") + == "http://username:password@www.example.com" ) def test_preserve_http_complete_url(self): - self.assertEqual( + assert ( add_http_if_no_scheme( "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag" - ), - "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag", + ) + == "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag" ) def test_protocol_relative(self): - self.assertEqual( - add_http_if_no_scheme("//www.example.com"), "http://www.example.com" - ) + assert add_http_if_no_scheme("//www.example.com") == "http://www.example.com" def test_protocol_relative_without_subdomain(self): - self.assertEqual(add_http_if_no_scheme("//example.com"), "http://example.com") + assert add_http_if_no_scheme("//example.com") == "http://example.com" def test_protocol_relative_path(self): - self.assertEqual( - add_http_if_no_scheme("//www.example.com/some/page.html"), - "http://www.example.com/some/page.html", + assert ( + add_http_if_no_scheme("//www.example.com/some/page.html") + == "http://www.example.com/some/page.html" ) def test_protocol_relative_port(self): - self.assertEqual( - add_http_if_no_scheme("//www.example.com:80"), "http://www.example.com:80" + assert ( + add_http_if_no_scheme("//www.example.com:80") == "http://www.example.com:80" ) def test_protocol_relative_fragment(self): - self.assertEqual( - add_http_if_no_scheme("//www.example.com/some/page#frag"), - "http://www.example.com/some/page#frag", + assert ( + add_http_if_no_scheme("//www.example.com/some/page#frag") + == "http://www.example.com/some/page#frag" ) def test_protocol_relative_query(self): - self.assertEqual( - add_http_if_no_scheme("//www.example.com/do?a=1&b=2&c=3"), - "http://www.example.com/do?a=1&b=2&c=3", + assert ( + add_http_if_no_scheme("//www.example.com/do?a=1&b=2&c=3") + == "http://www.example.com/do?a=1&b=2&c=3" ) def test_protocol_relative_username_password(self): - self.assertEqual( - add_http_if_no_scheme("//username:password@www.example.com"), - "http://username:password@www.example.com", + assert ( + add_http_if_no_scheme("//username:password@www.example.com") + == "http://username:password@www.example.com" ) def test_protocol_relative_complete_url(self): - self.assertEqual( + assert ( add_http_if_no_scheme( "//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag" - ), - "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag", + ) + == "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag" ) def test_preserve_https(self): - self.assertEqual( - add_http_if_no_scheme("https://www.example.com"), "https://www.example.com" + assert ( + add_http_if_no_scheme("https://www.example.com") + == "https://www.example.com" ) def test_preserve_ftp(self): - self.assertEqual( - add_http_if_no_scheme("ftp://www.example.com"), "ftp://www.example.com" - ) + assert add_http_if_no_scheme("ftp://www.example.com") == "ftp://www.example.com" -class GuessSchemeTest(unittest.TestCase): +class TestGuessScheme: pass @@ -361,7 +300,7 @@ for k, args in enumerate( ): t_method = create_guess_scheme_t(args) t_method.__name__ = f"test_uri_{k:03}" - setattr(GuessSchemeTest, t_method.__name__, t_method) + setattr(TestGuessScheme, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation for k, skip_args in enumerate( @@ -376,29 +315,29 @@ for k, skip_args in enumerate( ): t_method = create_skipped_scheme_t(skip_args) t_method.__name__ = f"test_uri_skipped_{k:03}" - setattr(GuessSchemeTest, t_method.__name__, t_method) + setattr(TestGuessScheme, t_method.__name__, t_method) -class StripUrl(unittest.TestCase): +class TestStripUrl: def test_noop(self): - self.assertEqual( - strip_url("http://www.example.com/index.html"), - "http://www.example.com/index.html", + assert ( + strip_url("http://www.example.com/index.html") + == "http://www.example.com/index.html" ) def test_noop_query_string(self): - self.assertEqual( - strip_url("http://www.example.com/index.html?somekey=somevalue"), - "http://www.example.com/index.html?somekey=somevalue", + assert ( + strip_url("http://www.example.com/index.html?somekey=somevalue") + == "http://www.example.com/index.html?somekey=somevalue" ) def test_fragments(self): - self.assertEqual( + assert ( strip_url( "http://www.example.com/index.html?somekey=somevalue#section", strip_fragment=False, - ), - "http://www.example.com/index.html?somekey=somevalue#section", + ) + == "http://www.example.com/index.html?somekey=somevalue#section" ) def test_path(self): @@ -407,7 +346,7 @@ class StripUrl(unittest.TestCase): ("http://www.example.com", False, "http://www.example.com"), ("http://www.example.com", True, "http://www.example.com/"), ]: - self.assertEqual(strip_url(input_url, origin_only=origin), output_url) + assert strip_url(input_url, origin_only=origin) == output_url def test_credentials(self): for i, o in [ @@ -424,7 +363,7 @@ class StripUrl(unittest.TestCase): "ftp://www.example.com/index.html?somekey=somevalue", ), ]: - self.assertEqual(strip_url(i, strip_credentials=True), o) + assert strip_url(i, strip_credentials=True) == o def test_credentials_encoded_delims(self): for i, o in [ @@ -447,7 +386,7 @@ class StripUrl(unittest.TestCase): "ftp://www.example.com/index.html?somekey=somevalue", ), ]: - self.assertEqual(strip_url(i, strip_credentials=True), o) + assert strip_url(i, strip_credentials=True) == o def test_default_ports_creds_off(self): for i, o in [ @@ -484,7 +423,7 @@ class StripUrl(unittest.TestCase): "ftp://www.example.com:221/file.txt", ), ]: - self.assertEqual(strip_url(i), o) + assert strip_url(i) == o def test_default_ports(self): for i, o in [ @@ -521,9 +460,7 @@ class StripUrl(unittest.TestCase): "ftp://username:password@www.example.com:221/file.txt", ), ]: - self.assertEqual( - strip_url(i, strip_default_port=True, strip_credentials=False), o - ) + assert strip_url(i, strip_default_port=True, strip_credentials=False) == o def test_default_ports_keep(self): for i, o in [ @@ -560,9 +497,7 @@ class StripUrl(unittest.TestCase): "ftp://username:password@www.example.com:221/file.txt", ), ]: - self.assertEqual( - strip_url(i, strip_default_port=False, strip_credentials=False), o - ) + assert strip_url(i, strip_default_port=False, strip_credentials=False) == o def test_origin_only(self): for i, o in [ @@ -583,10 +518,10 @@ class StripUrl(unittest.TestCase): "https://www.example.com/", ), ]: - self.assertEqual(strip_url(i, origin_only=True), o) + assert strip_url(i, origin_only=True) == o -class IsPathTestCase(unittest.TestCase): +class TestIsPath: def test_path(self): for input_value, output_value in ( # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell @@ -604,9 +539,7 @@ class IsPathTestCase(unittest.TestCase): (r"C:\user\docs\somefile.ext:alternate_stream_name", True), (r"https://example.com", False), ): - self.assertEqual( - _is_filesystem_path(input_value), output_value, input_value - ) + assert _is_filesystem_path(input_value) == output_value, input_value @pytest.mark.parametrize( From 0c9200094e0764023cd34e72d4012f1c0450d8ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 5 Mar 2025 10:31:59 +0100 Subject: [PATCH 061/491] Extend BaseSettings with utils for add-ons (#6614) --- docs/topics/addons.rst | 30 +- docs/topics/settings.rst | 51 +++ scrapy/settings/__init__.py | 116 ++++++- scrapy/utils/conf.py | 3 +- tests/test_settings/__init__.py | 590 ++++++++++++++++++++++++++++++++ tox.ini | 2 +- 6 files changed, 779 insertions(+), 13 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 46cf1edbd..8ec7b0295 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -76,15 +76,11 @@ The settings set by the add-on should use the ``addon`` priority (see settings.set("DNSCACHE_ENABLED", True, "addon") This allows users to override these settings in the project or spider -configuration. This is not possible with settings that are mutable objects, -such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases -you can provide an add-on-specific setting that governs whether the add-on will -modify :setting:`ITEM_PIPELINES`:: +configuration. - class MyAddon: - def update_settings(self, settings): - if settings.getbool("MYADDON_ENABLE_PIPELINE"): - settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 +When editing the value of a setting instead of overriding it entirely, it is +usually best to leave its priority unchanged. For example, when editing a +:ref:`component priority dictionary `. If the ``update_settings`` method raises :exc:`scrapy.exceptions.NotConfigured`, the add-on will be skipped. This makes @@ -127,12 +123,28 @@ Add-on examples Set some basic configuration: +.. skip: next .. code-block:: python + from myproject.pipelines import MyPipeline + + class MyAddon: def update_settings(self, settings): - settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 settings.set("DNSCACHE_ENABLED", True, "addon") + settings.remove_from_list("METAREFRESH_IGNORE_TAGS", "noscript") + settings.setdefault_in_component_priority_dict( + "ITEM_PIPELINES", MyPipeline, 200 + ) + +.. tip:: When editing a :ref:`component priority dictionary + ` setting, like :setting:`ITEM_PIPELINES`, + consider using setting methods like + :meth:`~scrapy.settings.BaseSettings.replace_in_component_priority_dict`, + :meth:`~scrapy.settings.BaseSettings.set_in_component_priority_dict` + and + :meth:`~scrapy.settings.BaseSettings.setdefault_in_component_priority_dict` + to avoid mistakes. Check dependencies: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a53e0806d..7646aca4f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -250,6 +250,57 @@ example, proper setting names for a fictional robots.txt extension would be ``ROBOTSTXT_ENABLED``, ``ROBOTSTXT_OBEY``, ``ROBOTSTXT_CACHEDIR``, etc. +.. _component-priority-dictionaries: + +Component priority dictionaries +=============================== + +A **component priority dictionary** is a :class:`dict` where keys are +:ref:`components ` and values are component priorities. For +example: + +.. skip: next +.. code-block:: python + + { + "path.to.ComponentA": None, + ComponentB: 100, + } + +A component can be specified either as a class object or through an import +path. + +.. warning:: Component priority dictionaries are regular :class:`dict` objects. + Be careful not to define the same component more than once, e.g. with + different import path strings or defining both an import path and a + :class:`type` object. + +A priority can be an :class:`int` or :data:`None`. + +A component with priority 1 goes *before* a component with priority 2. What +going before entails, however, depends on the corresponding setting. For +example, in the :setting:`DOWNLOADER_MIDDLEWARES` setting, components have +their +:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request` +method executed before that of later components, but have their +:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response` +method executed after that of later components. + +A component with priority :data:`None` is disabled. + +Some component priority dictionaries get merged with some built-in value. For +example, :setting:`DOWNLOADER_MIDDLEWARES` is merged with +:setting:`DOWNLOADER_MIDDLEWARES_BASE`. This is where :data:`None` comes in +handy, allowing you to disable a component from the base setting in the regular +setting: + +.. code-block:: python + + DOWNLOADER_MIDDLEWARES = { + "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": None, + } + + Special settings ================ diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index f31f824a8..cc4853c8f 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -8,6 +8,7 @@ from pprint import pformat from typing import TYPE_CHECKING, Any, Union, cast from scrapy.settings import default_settings +from scrapy.utils.misc import load_object # The key types are restricted in BaseSettings._get_key() to ones supported by JSON, # see https://github.com/scrapy/scrapy/issues/5383. @@ -111,6 +112,31 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def __contains__(self, name: Any) -> bool: return name in self.attributes + def add_to_list(self, name: _SettingsKeyT, item: Any) -> None: + """Append *item* to the :class:`list` setting with the specified *name* + if *item* is not already in that list. + + This change is applied regardless of the priority of the *name* + setting. The setting priority is not affected by this change either. + """ + value: list[str] = self.getlist(name) + if item not in value: + self.set(name, [*value, item], self.getpriority(name) or 0) + + def remove_from_list(self, name: _SettingsKeyT, item: Any) -> None: + """Remove *item* from the :class:`list` setting with the specified + *name*. + + If *item* is missing, raise :exc:`ValueError`. + + This change is applied regardless of the priority of the *name* + setting. The setting priority is not affected by this change either. + """ + value: list[str] = self.getlist(name) + if item not in value: + raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).") + self.set(name, [v for v in value if v != item], self.getpriority(name) or 0) + def get(self, name: _SettingsKeyT, default: Any = None) -> Any: """ Get a setting value without affecting its original type. @@ -181,8 +207,9 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): self, name: _SettingsKeyT, default: list[Any] | None = None ) -> list[Any]: """ - Get a setting value as a list. If the setting original type is a list, a - copy of it will be returned. If it's a string it will be split by ",". + Get a setting value as a list. If the setting original type is a list, + a copy of it will be returned. If it's a string it will be split by + ",". If it is an empty string, an empty list will be returned. For example, settings populated through environment variables set to ``'one,two'`` will return a list ['one', 'two'] when using this method. @@ -194,6 +221,8 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): :type default: object """ value = self.get(name, default or []) + if not value: + return [] if isinstance(value, str): value = value.split(",") return list(value) @@ -299,6 +328,47 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return max(cast(int, self.getpriority(name)) for name in self) return get_settings_priority("default") + def replace_in_component_priority_dict( + self, + name: _SettingsKeyT, + old_cls: type, + new_cls: type, + priority: int | None = None, + ) -> None: + """Replace *old_cls* with *new_cls* in the *name* :ref:`component + priority dictionary `. + + If *old_cls* is missing, or has :data:`None` as value, :exc:`KeyError` + is raised. + + If *old_cls* was present as an import string, even more than once, + those keys are dropped and replaced by *new_cls*. + + If *priority* is specified, that is the value assigned to *new_cls* in + the component priority dictionary. Otherwise, the value of *old_cls* is + used. If *old_cls* was present multiple times (possible with import + strings) with different values, the value assigned to *new_cls* is one + of them, with no guarantee about which one it is. + + This change is applied regardless of the priority of the *name* + setting. The setting priority is not affected by this change either. + """ + component_priority_dict = self.getdict(name) + old_priority = None + for cls_or_path in tuple(component_priority_dict): + if load_object(cls_or_path) != old_cls: + continue + if (old_priority := component_priority_dict.pop(cls_or_path)) is None: + break + if old_priority is None: + raise KeyError( + f"{old_cls} not found in the {name} setting ({component_priority_dict!r})." + ) + component_priority_dict[new_cls] = ( + old_priority if priority is None else priority + ) + self.set(name, component_priority_dict, priority=self.getpriority(name) or 0) + def __setitem__(self, name: _SettingsKeyT, value: Any) -> None: self.set(name, value) @@ -332,6 +402,30 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): else: self.attributes[name].set(value, priority) + def set_in_component_priority_dict( + self, name: _SettingsKeyT, cls: type, priority: int | None + ) -> None: + """Set the *cls* component in the *name* :ref:`component priority + dictionary ` setting with *priority*. + + If *cls* already exists, its value is updated. + + If *cls* was present as an import string, even more than once, those + keys are dropped and replaced by *cls*. + + This change is applied regardless of the priority of the *name* + setting. The setting priority is not affected by this change either. + """ + component_priority_dict = self.getdict(name) + for cls_or_path in tuple(component_priority_dict): + if not isinstance(cls_or_path, str): + continue + _cls = load_object(cls_or_path) + if _cls == cls: + del component_priority_dict[cls_or_path] + component_priority_dict[cls] = priority + self.set(name, component_priority_dict, self.getpriority(name) or 0) + def setdefault( self, name: _SettingsKeyT, @@ -344,6 +438,24 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return self.attributes[name].value + def setdefault_in_component_priority_dict( + self, name: _SettingsKeyT, cls: type, priority: int | None + ) -> None: + """Set the *cls* component in the *name* :ref:`component priority + dictionary ` setting with *priority* + if not already defined (even as an import string). + + If *cls* is not already defined, it is set regardless of the priority + of the *name* setting. The setting priority is not affected by this + change either. + """ + component_priority_dict = self.getdict(name) + for cls_or_path in tuple(component_priority_dict): + if load_object(cls_or_path) == cls: + return + component_priority_dict[cls] = priority + self.set(name, component_priority_dict, self.getpriority(name) or 0) + def setdict(self, values: _SettingsInputT, priority: int | str = "project") -> None: self.update(values, priority) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index a86aad51c..891cbb485 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -22,7 +22,8 @@ def build_component_list( *, convert: Callable[[Any], Any] = update_classpath, ) -> list[Any]: - """Compose a component list from a { class: order } dictionary.""" + """Compose a component list from a :ref:`component priority dictionary + `.""" def _check_components(complist: Collection[Any]) -> None: if len({convert(c) for c in complist}) != len(complist): diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index b7a316eee..909b365a9 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -260,6 +260,7 @@ class BaseSettingsTest(unittest.TestCase): "TEST_FLOAT2": "123.45", "TEST_LIST1": ["one", "two"], "TEST_LIST2": "one,two", + "TEST_LIST3": "", "TEST_STR": "value", "TEST_DICT1": {"key1": "val1", "ke2": 3}, "TEST_DICT2": '{"key1": "val1", "ke2": 3}', @@ -292,6 +293,7 @@ class BaseSettingsTest(unittest.TestCase): self.assertEqual(settings.getfloat("TEST_FLOATx", 55.0), 55.0) self.assertEqual(settings.getlist("TEST_LIST1"), ["one", "two"]) self.assertEqual(settings.getlist("TEST_LIST2"), ["one", "two"]) + self.assertEqual(settings.getlist("TEST_LIST3"), []) self.assertEqual(settings.getlist("TEST_LISTx"), []) self.assertEqual(settings.getlist("TEST_LISTx", ["default"]), ["default"]) self.assertEqual(settings["TEST_STR"], "value") @@ -504,3 +506,591 @@ class SettingsTest(unittest.TestCase): TypeError, match="Trying to modify an immutable Settings object" ): settings.pop("OTHER_DUMMY_CONFIG") + + +@pytest.mark.parametrize( + ("before", "name", "item", "after"), + [ + ({}, "FOO", "BAR", {"FOO": ["BAR"]}), + ({"FOO": []}, "FOO", "BAR", {"FOO": ["BAR"]}), + ({"FOO": ["BAR"]}, "FOO", "BAZ", {"FOO": ["BAR", "BAZ"]}), + ({"FOO": ["BAR"]}, "FOO", "BAR", {"FOO": ["BAR"]}), + ({"FOO": ""}, "FOO", "BAR", {"FOO": ["BAR"]}), + ({"FOO": "BAR"}, "FOO", "BAR", {"FOO": "BAR"}), + ({"FOO": "BAR"}, "FOO", "BAZ", {"FOO": ["BAR", "BAZ"]}), + ({"FOO": "BAR,BAZ"}, "FOO", "BAZ", {"FOO": "BAR,BAZ"}), + ({"FOO": "BAR,BAZ"}, "FOO", "QUX", {"FOO": ["BAR", "BAZ", "QUX"]}), + ], +) +def test_add_to_list(before, name, item, after): + settings = BaseSettings(before, priority=0) + settings.add_to_list(name, item) + expected_priority = settings.getpriority(name) or 0 + expected_settings = BaseSettings(after, priority=expected_priority) + assert settings == expected_settings, ( + f"{settings[name]=} != {expected_settings[name]=}" + ) + assert settings.getpriority(name) == expected_settings.getpriority(name) + + +@pytest.mark.parametrize( + ("before", "name", "item", "after"), + [ + ({}, "FOO", "BAR", ValueError), + ({"FOO": ["BAR"]}, "FOO", "BAR", {"FOO": []}), + ({"FOO": ["BAR"]}, "FOO", "BAZ", ValueError), + ({"FOO": ["BAR", "BAZ"]}, "FOO", "BAR", {"FOO": ["BAZ"]}), + ({"FOO": ""}, "FOO", "BAR", ValueError), + ({"FOO": "[]"}, "FOO", "BAR", ValueError), + ({"FOO": "BAR"}, "FOO", "BAR", {"FOO": []}), + ({"FOO": "BAR"}, "FOO", "BAZ", ValueError), + ({"FOO": "BAR,BAZ"}, "FOO", "BAR", {"FOO": ["BAZ"]}), + ], +) +def test_remove_from_list(before, name, item, after): + settings = BaseSettings(before, priority=0) + + if isinstance(after, type) and issubclass(after, Exception): + with pytest.raises(after): + settings.remove_from_list(name, item) + return + + settings.remove_from_list(name, item) + expected_priority = settings.getpriority(name) or 0 + expected_settings = BaseSettings(after, priority=expected_priority) + assert settings == expected_settings, ( + f"{settings[name]=} != {expected_settings[name]=}" + ) + assert settings.getpriority(name) == expected_settings.getpriority(name) + + +class Component1: + pass + + +Component1Alias = Component1 + + +class Component1Subclass(Component1): + pass + + +Component1SubclassAlias = Component1Subclass + + +class Component2: + pass + + +class Component3: + pass + + +class Component4: + pass + + +@pytest.mark.parametrize( + ("before", "name", "old_cls", "new_cls", "priority", "after"), + [ + ({}, "FOO", Component1, Component2, None, KeyError), + ( + {"FOO": {Component1: 1}}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": {Component1: 1}}, + "FOO", + Component1, + Component2, + 2, + {"FOO": {Component2: 2}}, + ), + ( + {"FOO": {"tests.test_settings.Component1": 1}}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": {Component1Alias: 1}}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": {Component1Alias: 1}}, + "FOO", + Component1, + Component2, + 2, + {"FOO": {Component2: 2}}, + ), + ( + {"FOO": {"tests.test_settings.Component1Alias": 1}}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": {"tests.test_settings.Component1Alias": 1}}, + "FOO", + Component1, + Component2, + 2, + {"FOO": {Component2: 2}}, + ), + ( + { + "FOO": { + "tests.test_settings.Component1": 1, + "tests.test_settings.Component1Alias": 2, + } + }, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 2}}, + ), + ( + { + "FOO": { + "tests.test_settings.Component1": 1, + "tests.test_settings.Component1Alias": 2, + } + }, + "FOO", + Component1, + Component2, + 3, + {"FOO": {Component2: 3}}, + ), + ( + {"FOO": '{"tests.test_settings.Component1": 1}'}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": '{"tests.test_settings.Component1": 1}'}, + "FOO", + Component1, + Component2, + 2, + {"FOO": {Component2: 2}}, + ), + ( + {"FOO": '{"tests.test_settings.Component1Alias": 1}'}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": '{"tests.test_settings.Component1Alias": 1}'}, + "FOO", + Component1, + Component2, + 2, + {"FOO": {Component2: 2}}, + ), + ( + { + "FOO": '{"tests.test_settings.Component1": 1, "tests.test_settings.Component1Alias": 2}' + }, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 2}}, + ), + ( + { + "FOO": '{"tests.test_settings.Component1": 1, "tests.test_settings.Component1Alias": 2}' + }, + "FOO", + Component1, + Component2, + 3, + {"FOO": {Component2: 3}}, + ), + # If old_cls has None as value, raise KeyError. + ( + {"FOO": {Component1: None}}, + "FOO", + Component1, + Component2, + None, + KeyError, + ), + ( + {"FOO": '{"tests.test_settings.Component1": null}'}, + "FOO", + Component1, + Component2, + None, + KeyError, + ), + ( + {"FOO": {Component1: None, "tests.test_settings.Component1": None}}, + "FOO", + Component1, + Component2, + None, + KeyError, + ), + ( + {"FOO": {Component1: 1, "tests.test_settings.Component1": None}}, + "FOO", + Component1, + Component2, + None, + KeyError, + ), + ( + {"FOO": {Component1: None, "tests.test_settings.Component1": 1}}, + "FOO", + Component1, + Component2, + None, + KeyError, + ), + # Unrelated components are kept as is, as expected. + ( + { + "FOO": { + Component1: 1, + "tests.test_settings.Component2": 2, + Component3: 3, + } + }, + "FOO", + Component3, + Component4, + None, + { + "FOO": { + Component1: 1, + "tests.test_settings.Component2": 2, + Component4: 3, + } + }, + ), + ], +) +def test_replace_in_component_priority_dict( + before, name, old_cls, new_cls, priority, after +): + settings = BaseSettings(before, priority=0) + + if isinstance(after, type) and issubclass(after, Exception): + with pytest.raises(after): + settings.replace_in_component_priority_dict( + name, old_cls, new_cls, priority + ) + return + + expected_priority = settings.getpriority(name) or 0 + settings.replace_in_component_priority_dict(name, old_cls, new_cls, priority) + expected_settings = BaseSettings(after, priority=expected_priority) + assert settings == expected_settings + assert settings.getpriority(name) == expected_settings.getpriority(name) + + +@pytest.mark.parametrize( + ("before", "name", "cls", "priority", "after"), + [ + # Set + ({}, "FOO", Component1, None, {"FOO": {Component1: None}}), + ({}, "FOO", Component1, 0, {"FOO": {Component1: 0}}), + ({}, "FOO", Component1, 1, {"FOO": {Component1: 1}}), + # Add + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + None, + {"FOO": {Component1: 0, Component2: None}}, + ), + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + 0, + {"FOO": {Component1: 0, Component2: 0}}, + ), + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + 1, + {"FOO": {Component1: 0, Component2: 1}}, + ), + # Replace + ( + { + "FOO": { + Component1: None, + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + Component1Subclass: None, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1, + } + }, + "FOO", + Component1, + None, + { + "FOO": { + Component1: None, + Component1Subclass: None, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1, + } + }, + ), + ( + { + "FOO": { + Component1: 0, + "tests.test_settings.Component1": 1, + "tests.test_settings.Component1Alias": None, + Component1Subclass: 0, + "tests.test_settings.Component1Subclass": 1, + "tests.test_settings.Component1SubclassAlias": None, + } + }, + "FOO", + Component1, + 0, + { + "FOO": { + Component1: 0, + Component1Subclass: 0, + "tests.test_settings.Component1Subclass": 1, + "tests.test_settings.Component1SubclassAlias": None, + } + }, + ), + ( + { + "FOO": { + Component1: 1, + "tests.test_settings.Component1": None, + "tests.test_settings.Component1Alias": 0, + Component1Subclass: 1, + "tests.test_settings.Component1Subclass": None, + "tests.test_settings.Component1SubclassAlias": 0, + } + }, + "FOO", + Component1, + 1, + { + "FOO": { + Component1: 1, + Component1Subclass: 1, + "tests.test_settings.Component1Subclass": None, + "tests.test_settings.Component1SubclassAlias": 0, + } + }, + ), + # String-based setting values + ( + {"FOO": '{"tests.test_settings.Component1": 0}'}, + "FOO", + Component2, + None, + {"FOO": {"tests.test_settings.Component1": 0, Component2: None}}, + ), + ( + { + "FOO": """{ + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1 + }""" + }, + "FOO", + Component1, + None, + { + "FOO": { + Component1: None, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1, + } + }, + ), + ], +) +def test_set_in_component_priority_dict(before, name, cls, priority, after): + settings = BaseSettings(before, priority=0) + expected_priority = settings.getpriority(name) or 0 + settings.set_in_component_priority_dict(name, cls, priority) + expected_settings = BaseSettings(after, priority=expected_priority) + assert settings == expected_settings + assert settings.getpriority(name) == expected_settings.getpriority(name), ( + f"{settings.getpriority(name)=} != {expected_settings.getpriority(name)=}" + ) + + +@pytest.mark.parametrize( + ("before", "name", "cls", "priority", "after"), + [ + # Set + ({}, "FOO", Component1, None, {"FOO": {Component1: None}}), + ({}, "FOO", Component1, 0, {"FOO": {Component1: 0}}), + ({}, "FOO", Component1, 1, {"FOO": {Component1: 1}}), + # Add + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + None, + {"FOO": {Component1: 0, Component2: None}}, + ), + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + 0, + {"FOO": {Component1: 0, Component2: 0}}, + ), + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + 1, + {"FOO": {Component1: 0, Component2: 1}}, + ), + # Keep + ( + { + "FOO": { + Component1: None, + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + Component1Subclass: None, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1, + } + }, + "FOO", + Component1, + None, + { + "FOO": { + Component1: None, + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + Component1Subclass: None, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1, + } + }, + ), + ( + { + "FOO": { + Component1: 0, + "tests.test_settings.Component1": 1, + "tests.test_settings.Component1Alias": None, + Component1Subclass: 0, + "tests.test_settings.Component1Subclass": 1, + "tests.test_settings.Component1SubclassAlias": None, + } + }, + "FOO", + Component1, + 0, + { + "FOO": { + Component1: 0, + "tests.test_settings.Component1": 1, + "tests.test_settings.Component1Alias": None, + Component1Subclass: 0, + "tests.test_settings.Component1Subclass": 1, + "tests.test_settings.Component1SubclassAlias": None, + } + }, + ), + ( + { + "FOO": { + Component1: 1, + "tests.test_settings.Component1": None, + "tests.test_settings.Component1Alias": 0, + Component1Subclass: 1, + "tests.test_settings.Component1Subclass": None, + "tests.test_settings.Component1SubclassAlias": 0, + } + }, + "FOO", + Component1, + 1, + { + "FOO": { + Component1: 1, + "tests.test_settings.Component1": None, + "tests.test_settings.Component1Alias": 0, + Component1Subclass: 1, + "tests.test_settings.Component1Subclass": None, + "tests.test_settings.Component1SubclassAlias": 0, + } + }, + ), + # String-based setting values + ( + {"FOO": '{"tests.test_settings.Component1": 0}'}, + "FOO", + Component2, + None, + {"FOO": {"tests.test_settings.Component1": 0, Component2: None}}, + ), + ( + { + "FOO": """{ + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1 + }""" + }, + "FOO", + Component1, + None, + { + "FOO": """{ + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1 + }""" + }, + ), + ], +) +def test_setdefault_in_component_priority_dict(before, name, cls, priority, after): + settings = BaseSettings(before, priority=0) + expected_priority = settings.getpriority(name) or 0 + settings.setdefault_in_component_priority_dict(name, cls, priority) + expected_settings = BaseSettings(after, priority=expected_priority) + assert settings == expected_settings + assert settings.getpriority(name) == expected_settings.getpriority(name) diff --git a/tox.ini b/tox.ini index 82ad84c90..041fcffca 100644 --- a/tox.ini +++ b/tox.ini @@ -39,7 +39,7 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} --doctest-modules + pytest --cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml {posargs:--durations=10 docs scrapy tests} --doctest-modules install_command = python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} From 1843a4f75358a76fe8e4624f8f4dc26084d19b85 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 6 Mar 2025 23:50:14 +0400 Subject: [PATCH 062/491] Converting tests to plain asserts, part 3. (#6700) --- tests/test_downloadermiddleware.py | 54 +- ...test_downloadermiddleware_ajaxcrawlable.py | 18 +- tests/test_downloadermiddleware_cookies.py | 78 +- ...est_downloadermiddleware_defaultheaders.py | 10 +- ...st_downloadermiddleware_downloadtimeout.py | 12 +- tests/test_downloadermiddleware_httpauth.py | 30 +- tests/test_downloadermiddleware_httpcache.py | 51 +- ...st_downloadermiddleware_httpcompression.py | 88 +-- tests/test_downloadermiddleware_httpproxy.py | 166 ++-- tests/test_downloadermiddleware_redirect.py | 732 +++++++++--------- tests/test_downloadermiddleware_retry.py | 87 +-- tests/test_downloadermiddleware_robotstxt.py | 8 +- tests/test_downloadermiddleware_stats.py | 14 +- tests/test_downloadermiddleware_useragent.py | 10 +- 14 files changed, 638 insertions(+), 720 deletions(-) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 49498375c..8e718ad5b 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -16,7 +16,7 @@ from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, get_from_asyncio_queue -class ManagerTestCase(TestCase): +class TestManagerBase(TestCase): settings_dict = None def setUp(self): @@ -51,14 +51,14 @@ class ManagerTestCase(TestCase): return ret -class DefaultsTest(ManagerTestCase): +class TestDefaults(TestManagerBase): """Tests default behavior with default settings""" def test_request_response(self): req = Request("http://example.com/index.html") resp = Response(req.url, status=200) ret = self._download(req, resp) - self.assertTrue(isinstance(ret, Response), "Non-response returned") + assert isinstance(ret, Response), "Non-response returned" def test_3xx_and_invalid_gzipped_body_must_redirect(self): """Regression test for a failure when redirecting a compressed @@ -86,11 +86,9 @@ class DefaultsTest(ManagerTestCase): }, ) ret = self._download(request=req, response=resp) - self.assertTrue(isinstance(ret, Request), f"Not redirected: {ret!r}") - self.assertEqual( - to_bytes(ret.url), - resp.headers["Location"], - "Not redirected to location header", + assert isinstance(ret, Request), f"Not redirected: {ret!r}" + assert to_bytes(ret.url) == resp.headers["Location"], ( + "Not redirected to location header" ) def test_200_and_invalid_gzipped_body_must_fail(self): @@ -111,7 +109,7 @@ class DefaultsTest(ManagerTestCase): self._download(request=req, response=resp) -class ResponseFromProcessRequestTest(ManagerTestCase): +class TestResponseFromProcessRequest(TestManagerBase): """Tests middleware returning a response from process_request.""" def test_download_func_not_called(self): @@ -130,11 +128,11 @@ class ResponseFromProcessRequestTest(ManagerTestCase): dfd.addBoth(results.append) self._wait(dfd) - self.assertIs(results[0], resp) - self.assertFalse(download_func.called) + assert results[0] is resp + assert not download_func.called -class ProcessRequestInvalidOutput(ManagerTestCase): +class TestProcessRequestInvalidOutput(TestManagerBase): """Invalid return value for process_request method should raise an exception""" def test_invalid_process_request(self): @@ -149,11 +147,11 @@ class ProcessRequestInvalidOutput(ManagerTestCase): dfd = self.mwman.download(download_func, req, self.spider) results = [] dfd.addBoth(results.append) - self.assertIsInstance(results[0], Failure) - self.assertIsInstance(results[0].value, _InvalidOutput) + assert isinstance(results[0], Failure) + assert isinstance(results[0].value, _InvalidOutput) -class ProcessResponseInvalidOutput(ManagerTestCase): +class TestProcessResponseInvalidOutput(TestManagerBase): """Invalid return value for process_response method should raise an exception""" def test_invalid_process_response(self): @@ -168,11 +166,11 @@ class ProcessResponseInvalidOutput(ManagerTestCase): dfd = self.mwman.download(download_func, req, self.spider) results = [] dfd.addBoth(results.append) - self.assertIsInstance(results[0], Failure) - self.assertIsInstance(results[0].value, _InvalidOutput) + assert isinstance(results[0], Failure) + assert isinstance(results[0].value, _InvalidOutput) -class ProcessExceptionInvalidOutput(ManagerTestCase): +class TestProcessExceptionInvalidOutput(TestManagerBase): """Invalid return value for process_exception method should raise an exception""" def test_invalid_process_exception(self): @@ -190,11 +188,11 @@ class ProcessExceptionInvalidOutput(ManagerTestCase): dfd = self.mwman.download(download_func, req, self.spider) results = [] dfd.addBoth(results.append) - self.assertIsInstance(results[0], Failure) - self.assertIsInstance(results[0].value, _InvalidOutput) + assert isinstance(results[0], Failure) + assert isinstance(results[0].value, _InvalidOutput) -class MiddlewareUsingDeferreds(ManagerTestCase): +class TestMiddlewareUsingDeferreds(TestManagerBase): """Middlewares using Deferreds should work""" def test_deferred(self): @@ -218,12 +216,12 @@ class MiddlewareUsingDeferreds(ManagerTestCase): dfd.addBoth(results.append) self._wait(dfd) - self.assertIs(results[0], resp) - self.assertFalse(download_func.called) + assert results[0] is resp + assert not download_func.called @pytest.mark.usefixtures("reactor_pytest") -class MiddlewareUsingCoro(ManagerTestCase): +class TestMiddlewareUsingCoro(TestManagerBase): """Middlewares using asyncio coroutines should work""" def test_asyncdef(self): @@ -242,8 +240,8 @@ class MiddlewareUsingCoro(ManagerTestCase): dfd.addBoth(results.append) self._wait(dfd) - self.assertIs(results[0], resp) - self.assertFalse(download_func.called) + assert results[0] is resp + assert not download_func.called @pytest.mark.only_asyncio def test_asyncdef_asyncio(self): @@ -262,5 +260,5 @@ class MiddlewareUsingCoro(ManagerTestCase): dfd.addBoth(results.append) self._wait(dfd) - self.assertIs(results[0], resp) - self.assertFalse(download_func.called) + assert results[0] is resp + assert not download_func.called diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 76fcece4f..44084f1e8 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -1,5 +1,3 @@ -import unittest - import pytest from scrapy.downloadermiddlewares.ajaxcrawl import AjaxCrawlMiddleware @@ -9,8 +7,8 @@ from scrapy.utils.test import get_crawler @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class AjaxCrawlMiddlewareTest(unittest.TestCase): - def setUp(self): +class TestAjaxCrawlMiddleware: + def setup_method(self): crawler = get_crawler(Spider, {"AJAXCRAWL_ENABLED": True}) self.spider = crawler._create_spider("foo") self.mw = AjaxCrawlMiddleware.from_crawler(crawler) @@ -26,13 +24,13 @@ class AjaxCrawlMiddlewareTest(unittest.TestCase): def test_non_get(self): req, resp = self._req_resp("http://example.com/", {"method": "HEAD"}) resp2 = self.mw.process_response(req, resp, self.spider) - self.assertEqual(resp, resp2) + assert resp == resp2 def test_binary_response(self): req = Request("http://example.com/") resp = Response("http://example.com/", body=b"foobar\x00\x01\x02", request=req) resp2 = self.mw.process_response(req, resp, self.spider) - self.assertIs(resp, resp2) + assert resp is resp2 def test_ajaxcrawl(self): req, resp = self._req_resp( @@ -41,8 +39,8 @@ class AjaxCrawlMiddlewareTest(unittest.TestCase): {"body": self._ajaxcrawlable_body()}, ) req2 = self.mw.process_response(req, resp, self.spider) - self.assertEqual(req2.url, "http://example.com/?_escaped_fragment_=") - self.assertEqual(req2.meta["foo"], "bar") + assert req2.url == "http://example.com/?_escaped_fragment_=" + assert req2.meta["foo"] == "bar" def test_ajaxcrawl_loop(self): req, resp = self._req_resp( @@ -53,7 +51,7 @@ class AjaxCrawlMiddlewareTest(unittest.TestCase): resp3 = self.mw.process_response(req2, resp2, self.spider) assert isinstance(resp3, HtmlResponse), (resp3.__class__, resp3) - self.assertEqual(resp3.request.url, "http://example.com/?_escaped_fragment_=") + assert resp3.request.url == "http://example.com/?_escaped_fragment_=" assert resp3 is resp2 def test_noncrawlable_body(self): @@ -61,4 +59,4 @@ class AjaxCrawlMiddlewareTest(unittest.TestCase): "http://example.com/", {}, {"body": b""} ) resp2 = self.mw.process_response(req, resp, self.spider) - self.assertIs(resp, resp2) + assert resp is resp2 diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 694a669d4..8bf3a1f09 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -1,5 +1,4 @@ import logging -from unittest import TestCase import pytest from testfixtures import LogCapture @@ -53,19 +52,19 @@ def _cookies_to_set_cookie_list(cookies): return filter(None, (_cookie_to_set_cookie_value(cookie) for cookie in cookies)) -class CookiesMiddlewareTest(TestCase): +class TestCookiesMiddleware: def assertCookieValEqual(self, first, second, msg=None): def split_cookies(cookies): return sorted([s.strip() for s in to_bytes(cookies).split(b";")]) - return self.assertEqual(split_cookies(first), split_cookies(second), msg=msg) + assert split_cookies(first) == split_cookies(second), msg - def setUp(self): + def setup_method(self): self.spider = Spider("foo") self.mw = CookiesMiddleware() self.redirect_middleware = RedirectMiddleware(settings=Settings()) - def tearDown(self): + def teardown_method(self): del self.mw del self.redirect_middleware @@ -80,7 +79,7 @@ class CookiesMiddlewareTest(TestCase): req2 = Request("http://scrapytest.org/sub1/") assert self.mw.process_request(req2, self.spider) is None - self.assertEqual(req2.headers.get("Cookie"), b"C1=value1") + assert req2.headers.get("Cookie") == b"C1=value1" def test_setting_false_cookies_enabled(self): with pytest.raises(NotConfigured): @@ -89,12 +88,12 @@ class CookiesMiddlewareTest(TestCase): ) def test_setting_default_cookies_enabled(self): - self.assertIsInstance( + assert isinstance( CookiesMiddleware.from_crawler(get_crawler()), CookiesMiddleware ) def test_setting_true_cookies_enabled(self): - self.assertIsInstance( + assert isinstance( CookiesMiddleware.from_crawler( get_crawler(settings_dict={"COOKIES_ENABLED": True}) ), @@ -161,7 +160,7 @@ class CookiesMiddlewareTest(TestCase): req2 = Request("http://scrapytest.org/sub1/") assert self.mw.process_request(req2, self.spider) is None - self.assertIn("Cookie", req2.headers) + assert "Cookie" in req2.headers def test_dont_merge_cookies(self): # merge some cookies into jar @@ -185,12 +184,12 @@ class CookiesMiddlewareTest(TestCase): # check that cookies are merged back req = Request("http://scrapytest.org/mergeme") assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers.get("Cookie"), b"C1=value1") + assert req.headers.get("Cookie") == b"C1=value1" # check that cookies are merged when dont_merge_cookies is passed as 0 req = Request("http://scrapytest.org/mergeme", meta={"dont_merge_cookies": 0}) assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers.get("Cookie"), b"C1=value1") + assert req.headers.get("Cookie") == b"C1=value1" def test_complex_cookies(self): # merge some cookies into jar @@ -230,7 +229,7 @@ class CookiesMiddlewareTest(TestCase): # embed C2 for scrapytest.org/bar req = Request("http://scrapytest.org/bar") self.mw.process_request(req, self.spider) - self.assertEqual(req.headers.get("Cookie"), b"C2=value2") + assert req.headers.get("Cookie") == b"C2=value2" # embed nothing for scrapytest.org/baz req = Request("http://scrapytest.org/baz") @@ -240,7 +239,7 @@ class CookiesMiddlewareTest(TestCase): def test_merge_request_cookies(self): req = Request("http://scrapytest.org/", cookies={"galleta": "salada"}) assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers.get("Cookie"), b"galleta=salada") + assert req.headers.get("Cookie") == b"galleta=salada" headers = {"Set-Cookie": "C1=value1; path=/"} res = Response("http://scrapytest.org/", headers=headers) @@ -260,7 +259,7 @@ class CookiesMiddlewareTest(TestCase): meta={"cookiejar": "store1"}, ) assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers.get("Cookie"), b"galleta=salada") + assert req.headers.get("Cookie") == b"galleta=salada" headers = {"Set-Cookie": "C1=value1; path=/"} res = Response("http://scrapytest.org/", headers=headers, request=req) @@ -278,7 +277,7 @@ class CookiesMiddlewareTest(TestCase): meta={"cookiejar": "store2"}, ) assert self.mw.process_request(req3, self.spider) is None - self.assertEqual(req3.headers.get("Cookie"), b"galleta=dulce") + assert req3.headers.get("Cookie") == b"galleta=dulce" headers = {"Set-Cookie": "C2=value2; path=/"} res2 = Response("http://scrapytest.org/", headers=headers, request=req3) @@ -302,22 +301,22 @@ class CookiesMiddlewareTest(TestCase): req5_2 = Request("http://scrapytest.org:1104/some-redirected-path") assert self.mw.process_request(req5_2, self.spider) is None - self.assertEqual(req5_2.headers.get("Cookie"), b"C1=value1") + assert req5_2.headers.get("Cookie") == b"C1=value1" req5_3 = Request("http://scrapytest.org/some-redirected-path") assert self.mw.process_request(req5_3, self.spider) is None - self.assertEqual(req5_3.headers.get("Cookie"), b"C1=value1") + assert req5_3.headers.get("Cookie") == b"C1=value1" # skip cookie retrieval for not http request req6 = Request("file:///scrapy/sometempfile") assert self.mw.process_request(req6, self.spider) is None - self.assertEqual(req6.headers.get("Cookie"), None) + assert req6.headers.get("Cookie") is None def test_local_domain(self): request = Request("http://example-host/", cookies={"currencyCookie": "USD"}) assert self.mw.process_request(request, self.spider) is None - self.assertIn("Cookie", request.headers) - self.assertEqual(b"currencyCookie=USD", request.headers["Cookie"]) + assert "Cookie" in request.headers + assert request.headers["Cookie"] == b"currencyCookie=USD" @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_from_default_request_headers_middleware(self): @@ -474,7 +473,7 @@ class CookiesMiddlewareTest(TestCase): request1 = Request(cookies=input_cookies, **source) self.mw.process_request(request1, self.spider) cookies = request1.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies1 else None) + assert cookies == (b"a=b" if cookies1 else None) response = Response( headers={ @@ -482,21 +481,18 @@ class CookiesMiddlewareTest(TestCase): }, **target, ) - self.assertEqual( - self.mw.process_response(request1, response, self.spider), - response, - ) + assert self.mw.process_response(request1, response, self.spider) == response request2 = self.redirect_middleware.process_response( request1, response, self.spider, ) - self.assertIsInstance(request2, Request) + assert isinstance(request2, Request) self.mw.process_request(request2, self.spider) cookies = request2.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies2 else None) + assert cookies == (b"a=b" if cookies2 else None) def test_cookie_redirect_same_domain(self): self._test_cookie_redirect( @@ -573,10 +569,10 @@ class CookiesMiddlewareTest(TestCase): response, self.spider, ) - self.assertIsInstance(request2, Request) + assert isinstance(request2, Request) cookies = request2.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies2 else None) + assert cookies == (b"a=b" if cookies2 else None) def test_cookie_header_redirect_same_domain(self): self._test_cookie_header_redirect( @@ -626,12 +622,12 @@ class CookiesMiddlewareTest(TestCase): request1 = Request(url1, cookies=input_cookies) self.mw.process_request(request1, self.spider) cookies = request1.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies1 else None) + assert cookies == (b"a=b" if cookies1 else None) request2 = Request(url2) self.mw.process_request(request2, self.spider) cookies = request2.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies2 else None) + assert cookies == (b"a=b" if cookies2 else None) def test_user_set_cookie_domain_suffix_private(self): self._test_user_set_cookie_domain_followup( @@ -692,15 +688,12 @@ class CookiesMiddlewareTest(TestCase): "Set-Cookie": _cookies_to_set_cookie_list(input_cookies), } response = Response(url1, status=200, headers=headers) - self.assertEqual( - self.mw.process_response(request1, response, self.spider), - response, - ) + assert self.mw.process_response(request1, response, self.spider) == response request2 = Request(url2) self.mw.process_request(request2, self.spider) actual_cookies = request2.headers.get("Cookie") - self.assertEqual(actual_cookies, b"a=b" if cookies else None) + assert actual_cookies == (b"a=b" if cookies else None) def test_server_set_cookie_domain_suffix_private(self): self._test_server_set_cookie_domain_followup( @@ -752,30 +745,27 @@ class CookiesMiddlewareTest(TestCase): request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies) self.mw.process_request(request1, self.spider) cookies = request1.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies1 else None) + assert cookies == (b"a=b" if cookies1 else None) response = Response( f"{from_scheme}://a.example", headers={"Location": f"{to_scheme}://a.example"}, status=301, ) - self.assertEqual( - self.mw.process_response(request1, response, self.spider), - response, - ) + assert self.mw.process_response(request1, response, self.spider) == response request2 = self.redirect_middleware.process_response( request1, response, self.spider, ) - self.assertIsInstance(request2, Request) + assert isinstance(request2, Request) cookies = request2.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies2 else None) + assert cookies == (b"a=b" if cookies2 else None) self.mw.process_request(request2, self.spider) cookies = request2.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies3 else None) + assert cookies == (b"a=b" if cookies3 else None) def test_cookie_redirect_secure_undefined_downgrade(self): self._test_cookie_redirect_scheme_change( diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index 27d6224b4..5716e3631 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -1,5 +1,3 @@ -from unittest import TestCase - from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.http import Request from scrapy.spiders import Spider @@ -7,7 +5,7 @@ from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler -class TestDefaultHeadersMiddleware(TestCase): +class TestDefaultHeadersMiddleware: def get_defaults_spider_mw(self): crawler = get_crawler(Spider) spider = crawler._create_spider("foo") @@ -21,15 +19,15 @@ class TestDefaultHeadersMiddleware(TestCase): defaults, spider, mw = self.get_defaults_spider_mw() req = Request("http://www.scrapytest.org") mw.process_request(req, spider) - self.assertEqual(req.headers, defaults) + assert req.headers == defaults def test_update_headers(self): defaults, spider, mw = self.get_defaults_spider_mw() headers = {"Accept-Language": ["es"], "Test-Header": ["test"]} bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]} req = Request("http://www.scrapytest.org", headers=headers) - self.assertEqual(req.headers, bytes_headers) + assert req.headers == bytes_headers mw.process_request(req, spider) defaults.update(bytes_headers) - self.assertEqual(req.headers, defaults) + assert req.headers == defaults diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index 44458ade8..31323c8fa 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -1,12 +1,10 @@ -import unittest - from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class DownloadTimeoutMiddlewareTest(unittest.TestCase): +class TestDownloadTimeoutMiddleware: def get_request_spider_mw(self, settings=None): crawler = get_crawler(Spider, settings) spider = crawler._create_spider("foo") @@ -17,20 +15,20 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase): req, spider, mw = self.get_request_spider_mw() mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta.get("download_timeout"), 180) + assert req.meta.get("download_timeout") == 180 def test_string_download_timeout(self): req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"}) mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta.get("download_timeout"), 20.1) + assert req.meta.get("download_timeout") == 20.1 def test_spider_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() spider.download_timeout = 2 mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta.get("download_timeout"), 2) + assert req.meta.get("download_timeout") == 2 def test_request_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() @@ -38,4 +36,4 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase): mw.spider_opened(spider) req.meta["download_timeout"] = 1 assert mw.process_request(req, spider) is None - self.assertEqual(req.meta.get("download_timeout"), 1) + assert req.meta.get("download_timeout") == 1 diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 0f1489344..9154e1850 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,5 +1,3 @@ -import unittest - import pytest from w3lib.http import basic_auth_header @@ -25,8 +23,8 @@ class AnyDomainSpider(Spider): http_auth_domain = None -class HttpAuthMiddlewareLegacyTest(unittest.TestCase): - def setUp(self): +class TestHttpAuthMiddlewareLegacy: + def setup_method(self): self.spider = LegacySpider("foo") def test_auth(self): @@ -35,51 +33,51 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase): mw.spider_opened(self.spider) -class HttpAuthMiddlewareTest(unittest.TestCase): - def setUp(self): +class TestHttpAuthMiddleware: + def setup_method(self): self.mw = HttpAuthMiddleware() self.spider = DomainSpider("foo") self.mw.spider_opened(self.spider) - def tearDown(self): + def teardown_method(self): del self.mw def test_no_auth(self): req = Request("http://example-noauth.com/") assert self.mw.process_request(req, self.spider) is None - self.assertNotIn("Authorization", req.headers) + assert "Authorization" not in req.headers def test_auth_domain(self): req = Request("http://example.com/") assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) + assert req.headers["Authorization"] == basic_auth_header("foo", "bar") def test_auth_subdomain(self): req = Request("http://foo.example.com/") assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) + assert req.headers["Authorization"] == basic_auth_header("foo", "bar") def test_auth_already_set(self): req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], b"Digest 123") + assert req.headers["Authorization"] == b"Digest 123" -class HttpAuthAnyMiddlewareTest(unittest.TestCase): - def setUp(self): +class TestHttpAuthAnyMiddleware: + def setup_method(self): self.mw = HttpAuthMiddleware() self.spider = AnyDomainSpider("foo") self.mw.spider_opened(self.spider) - def tearDown(self): + def teardown_method(self): del self.mw def test_auth(self): req = Request("http://example.com/") assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) + assert req.headers["Authorization"] == basic_auth_header("foo", "bar") def test_auth_already_set(self): req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], b"Digest 123") + assert req.headers["Authorization"] == b"Digest 123" diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index de3a9689b..5fac88ed7 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -2,7 +2,6 @@ import email.utils import shutil import tempfile import time -import unittest from contextlib import contextmanager import pytest @@ -15,11 +14,11 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class _BaseTest(unittest.TestCase): +class TestBase: storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" policy_class = "scrapy.extensions.httpcache.RFC2616Policy" - def setUp(self): + def setup_method(self): self.yesterday = email.utils.formatdate(time.time() - 86400) self.today = email.utils.formatdate() self.tomorrow = email.utils.formatdate(time.time() + 86400) @@ -35,7 +34,7 @@ class _BaseTest(unittest.TestCase): ) self.crawler.stats.open_spider(self.spider) - def tearDown(self): + def teardown_method(self): self.crawler.stats.close_spider(self.spider, "") shutil.rmtree(self.tmpdir) @@ -72,44 +71,42 @@ class _BaseTest(unittest.TestCase): mw.spider_closed(self.spider) def assertEqualResponse(self, response1, response2): - self.assertEqual(response1.url, response2.url) - self.assertEqual(response1.status, response2.status) - self.assertEqual(response1.headers, response2.headers) - self.assertEqual(response1.body, response2.body) + assert response1.url == response2.url + assert response1.status == response2.status + assert response1.headers == response2.headers + assert response1.body == response2.body def assertEqualRequest(self, request1, request2): - self.assertEqual(request1.url, request2.url) - self.assertEqual(request1.headers, request2.headers) - self.assertEqual(request1.body, request2.body) + assert request1.url == request2.url + assert request1.headers == request2.headers + assert request1.body == request2.body def assertEqualRequestButWithCacheValidators(self, request1, request2): - self.assertEqual(request1.url, request2.url) + assert request1.url == request2.url assert b"If-None-Match" not in request1.headers assert b"If-Modified-Since" not in request1.headers assert any( h in request2.headers for h in (b"If-None-Match", b"If-Modified-Since") ) - self.assertEqual(request1.body, request2.body) + assert request1.body == request2.body def test_dont_cache(self): with self._middleware() as mw: self.request.meta["dont_cache"] = True mw.process_response(self.request, self.response, self.spider) - self.assertEqual( - mw.storage.retrieve_response(self.spider, self.request), None - ) + assert mw.storage.retrieve_response(self.spider, self.request) is None with self._middleware() as mw: self.request.meta["dont_cache"] = False mw.process_response(self.request, self.response, self.spider) if mw.policy.should_cache_response(self.response, self.request): - self.assertIsInstance( + assert isinstance( mw.storage.retrieve_response(self.spider, self.request), self.response.__class__, ) -class DefaultStorageTest(_BaseTest): +class TestDefaultStorage(TestBase): def test_storage(self): with self._storage() as storage: request2 = self.request.copy() @@ -142,15 +139,15 @@ class DefaultStorageTest(_BaseTest): ) storage.store_response(self.spider, self.request, response) cached_response = storage.retrieve_response(self.spider, self.request) - self.assertIsInstance(cached_response, HtmlResponse) + assert isinstance(cached_response, HtmlResponse) self.assertEqualResponse(response, cached_response) -class DbmStorageTest(DefaultStorageTest): +class TestDbmStorage(TestDefaultStorage): storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" -class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): +class TestDbmStorageWithCustomDbmModule(TestDbmStorage): dbm_module = "tests.mocks.dummydbm" def _get_settings(self, **new_settings): @@ -160,16 +157,16 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): def test_custom_dbm_module_loaded(self): # make sure our dbm module has been loaded with self._storage() as storage: - self.assertEqual(storage.dbmodule.__name__, self.dbm_module) + assert storage.dbmodule.__name__ == self.dbm_module -class FilesystemStorageGzipTest(DefaultStorageTest): +class TestFilesystemStorageGzip(TestDefaultStorage): def _get_settings(self, **new_settings): new_settings.setdefault("HTTPCACHE_GZIP", True) return super()._get_settings(**new_settings) -class DummyPolicyTest(_BaseTest): +class TestDummyPolicy(TestBase): policy_class = "scrapy.extensions.httpcache.DummyPolicy" def test_middleware(self): @@ -261,7 +258,7 @@ class DummyPolicyTest(_BaseTest): assert "cached" in response.flags -class RFC2616PolicyTest(DefaultStorageTest): +class TestRFC2616Policy(TestDefaultStorage): policy_class = "scrapy.extensions.httpcache.RFC2616Policy" def _process_requestresponse(self, mw, request, response): @@ -357,7 +354,7 @@ class RFC2616PolicyTest(DefaultStorageTest): assert "cached" in res2.flags assert res2.status != 304 else: - self.assertFalse(resc) + assert not resc assert "cached" not in res2.flags # cache unconditionally unless response contains no-store or is a 304 @@ -381,7 +378,7 @@ class RFC2616PolicyTest(DefaultStorageTest): assert "cached" in res2.flags assert res2.status != 304 else: - self.assertFalse(resc) + assert not resc assert "cached" not in res2.flags def test_cached_and_fresh(self): diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index b3e3b98d7..e7427c5ac 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -2,7 +2,7 @@ from gzip import GzipFile from io import BytesIO from logging import WARNING from pathlib import Path -from unittest import SkipTest, TestCase +from unittest import SkipTest import pytest from testfixtures import LogCapture @@ -51,8 +51,8 @@ FORMAT = { } -class HttpCompressionTest(TestCase): - def setUp(self): +class TestHttpCompression: + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("scrapytest.org") self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) @@ -81,10 +81,8 @@ class HttpCompressionTest(TestCase): return response def assertStatsEqual(self, key, value): - self.assertEqual( - self.crawler.stats.get_value(key, spider=self.spider), - value, - str(self.crawler.stats.get_stats(self.spider)), + assert self.crawler.stats.get_value(key, spider=self.spider) == value, str( + self.crawler.stats.get_stats(self.spider) ) def test_setting_false_compression_enabled(self): @@ -94,13 +92,13 @@ class HttpCompressionTest(TestCase): ) def test_setting_default_compression_enabled(self): - self.assertIsInstance( + assert isinstance( HttpCompressionMiddleware.from_crawler(get_crawler()), HttpCompressionMiddleware, ) def test_setting_true_compression_enabled(self): - self.assertIsInstance( + assert isinstance( HttpCompressionMiddleware.from_crawler( get_crawler(settings_dict={"COMPRESSION_ENABLED": True}) ), @@ -111,15 +109,13 @@ class HttpCompressionTest(TestCase): request = Request("http://scrapytest.org") assert "Accept-Encoding" not in request.headers self.mw.process_request(request, self.spider) - self.assertEqual( - request.headers.get("Accept-Encoding"), b", ".join(ACCEPTED_ENCODINGS) - ) + assert request.headers.get("Accept-Encoding") == b", ".join(ACCEPTED_ENCODINGS) def test_process_response_gzip(self): response = self._getresponse("gzip") request = response.request - self.assertEqual(response.headers["Content-Encoding"], b"gzip") + assert response.headers["Content-Encoding"] == b"gzip" newresponse = self.mw.process_response(request, response, self.spider) assert newresponse is not response assert newresponse.body.startswith(b" req.priority def test_dont_redirect(self): url = "http://www.example.com/301" @@ -53,8 +52,8 @@ class Base: req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) - self.assertEqual(req2.url, url2) - self.assertEqual(req2.method, "GET") + assert req2.url == url2 + assert req2.method == "GET" assert "Content-Type" not in req2.headers, ( "Content-Type header must not be present in redirected request" ) @@ -71,7 +70,7 @@ class Base: req = self.mw.process_response(req, rsp, self.spider) assert isinstance(req, Request) assert "redirect_times" in req.meta - self.assertEqual(req.meta["redirect_times"], 1) + assert req.meta["redirect_times"] == 1 with pytest.raises(IgnoreRequest): self.mw.process_response(req, rsp, self.spider) @@ -92,15 +91,13 @@ class Base: rsp2 = self.get_response(req1, "/redirected2") req3 = self.mw.process_response(req2, rsp2, self.spider) - self.assertEqual(req2.url, "http://scrapytest.org/redirected") - self.assertEqual( - req2.meta["redirect_urls"], ["http://scrapytest.org/first"] - ) - self.assertEqual(req3.url, "http://scrapytest.org/redirected2") - self.assertEqual( - req3.meta["redirect_urls"], - ["http://scrapytest.org/first", "http://scrapytest.org/redirected"], - ) + assert req2.url == "http://scrapytest.org/redirected" + assert req2.meta["redirect_urls"] == ["http://scrapytest.org/first"] + assert req3.url == "http://scrapytest.org/redirected2" + assert req3.meta["redirect_urls"] == [ + "http://scrapytest.org/first", + "http://scrapytest.org/redirected", + ] def test_redirect_reasons(self): req1 = Request("http://scrapytest.org/first") @@ -108,8 +105,8 @@ class Base: req2 = self.mw.process_response(req1, rsp1, self.spider) rsp2 = self.get_response(req2, "/redirected2") req3 = self.mw.process_response(req2, rsp2, self.spider) - self.assertEqual(req2.meta["redirect_reasons"], [self.reason]) - self.assertEqual(req3.meta["redirect_reasons"], [self.reason, self.reason]) + assert req2.meta["redirect_reasons"] == [self.reason] + assert req3.meta["redirect_reasons"] == [self.reason, self.reason] def test_cross_origin_header_dropping(self): safe_headers = {"A": "B"} @@ -129,10 +126,8 @@ class Base: internal_redirect_request = self.mw.process_response( original_request, internal_response, self.spider ) - self.assertIsInstance(internal_redirect_request, Request) - self.assertEqual( - original_request.headers, internal_redirect_request.headers - ) + assert isinstance(internal_redirect_request, Request) + assert original_request.headers == internal_redirect_request.headers # Redirects to the same origin (same scheme, same domain, same port) # keep all headers also when the scheme is http. @@ -144,8 +139,8 @@ class Base: http_redirect_request = self.mw.process_response( http_request, http_response, self.spider ) - self.assertIsInstance(http_redirect_request, Request) - self.assertEqual(http_request.headers, http_redirect_request.headers) + assert isinstance(http_redirect_request, Request) + assert http_request.headers == http_redirect_request.headers # For default ports, whether the port is explicit or implicit does not # affect the outcome, it is still the same origin. @@ -155,10 +150,8 @@ class Base: to_explicit_port_redirect_request = self.mw.process_response( original_request, to_explicit_port_response, self.spider ) - self.assertIsInstance(to_explicit_port_redirect_request, Request) - self.assertEqual( - original_request.headers, to_explicit_port_redirect_request.headers - ) + assert isinstance(to_explicit_port_redirect_request, Request) + assert original_request.headers == to_explicit_port_redirect_request.headers # For default ports, whether the port is explicit or implicit does not # affect the outcome, it is still the same origin. @@ -168,10 +161,8 @@ class Base: to_implicit_port_redirect_request = self.mw.process_response( original_request, to_implicit_port_response, self.spider ) - self.assertIsInstance(to_implicit_port_redirect_request, Request) - self.assertEqual( - original_request.headers, to_implicit_port_redirect_request.headers - ) + assert isinstance(to_implicit_port_redirect_request, Request) + assert original_request.headers == to_implicit_port_redirect_request.headers # A port change drops the Authorization header because the origin # changes, but keeps the Cookie header because the domain remains the @@ -182,11 +173,11 @@ class Base: different_port_redirect_request = self.mw.process_response( original_request, different_port_response, self.spider ) - self.assertIsInstance(different_port_redirect_request, Request) - self.assertEqual( - {**safe_headers, **cookie_header}, - different_port_redirect_request.headers.to_unicode_dict(), - ) + assert isinstance(different_port_redirect_request, Request) + assert { + **safe_headers, + **cookie_header, + } == different_port_redirect_request.headers.to_unicode_dict() # A domain change drops both the Authorization and the Cookie header. external_response = self.get_response( @@ -195,10 +186,8 @@ class Base: external_redirect_request = self.mw.process_response( original_request, external_response, self.spider ) - self.assertIsInstance(external_redirect_request, Request) - self.assertEqual( - safe_headers, external_redirect_request.headers.to_unicode_dict() - ) + assert isinstance(external_redirect_request, Request) + assert safe_headers == external_redirect_request.headers.to_unicode_dict() # A scheme upgrade (http → https) drops the Authorization header # because the origin changes, but keeps the Cookie header because the @@ -207,11 +196,11 @@ class Base: upgrade_redirect_request = self.mw.process_response( http_request, upgrade_response, self.spider ) - self.assertIsInstance(upgrade_redirect_request, Request) - self.assertEqual( - {**safe_headers, **cookie_header}, - upgrade_redirect_request.headers.to_unicode_dict(), - ) + assert isinstance(upgrade_redirect_request, Request) + assert { + **safe_headers, + **cookie_header, + } == upgrade_redirect_request.headers.to_unicode_dict() # A scheme downgrade (https → http) drops the Authorization header # because the origin changes, and the Cookie header because its value @@ -228,11 +217,8 @@ class Base: downgrade_redirect_request = self.mw.process_response( original_request, downgrade_response, self.spider ) - self.assertIsInstance(downgrade_redirect_request, Request) - self.assertEqual( - safe_headers, - downgrade_redirect_request.headers.to_unicode_dict(), - ) + assert isinstance(downgrade_redirect_request, Request) + assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict() def test_meta_proxy_http_absolute(self): crawler = get_crawler() @@ -244,37 +230,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "http://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "http://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_meta_proxy_http_relative(self): crawler = get_crawler() @@ -286,37 +272,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "/a") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "/a") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_meta_proxy_https_absolute(self): crawler = get_crawler() @@ -328,37 +314,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "https://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "https://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_meta_proxy_https_relative(self): crawler = get_crawler() @@ -370,37 +356,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "/a") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "/a") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_meta_proxy_http_to_https(self): crawler = get_crawler() @@ -412,37 +398,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "https://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "http://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_meta_proxy_https_to_http(self): crawler = get_crawler() @@ -454,37 +440,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "http://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "https://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_system_proxy_http_absolute(self): crawler = get_crawler() @@ -499,37 +485,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "http://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "http://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_system_proxy_http_relative(self): crawler = get_crawler() @@ -544,37 +530,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "/a") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "/a") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_system_proxy_https_absolute(self): crawler = get_crawler() @@ -589,37 +575,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "https://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "https://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_system_proxy_https_relative(self): crawler = get_crawler() @@ -634,37 +620,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "/a") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "/a") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_system_proxy_proxied_http_to_proxied_https(self): crawler = get_crawler() @@ -680,37 +666,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "https://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic Yjo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://b.example") - self.assertEqual(request2.meta["proxy"], "https://b.example") + assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request2.meta["_auth_proxy"] == "https://b.example" + assert request2.meta["proxy"] == "https://b.example" response2 = self.get_response(request2, "http://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_system_proxy_proxied_http_to_unproxied_https(self): crawler = get_crawler() @@ -725,37 +711,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "https://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta proxy_mw.process_request(request2, spider) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta response2 = self.get_response(request2, "http://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_system_proxy_unproxied_http_to_proxied_https(self): crawler = get_crawler() @@ -770,37 +756,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertNotIn("Proxy-Authorization", request1.headers) - self.assertNotIn("_auth_proxy", request1.meta) - self.assertNotIn("proxy", request1.meta) + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta response1 = self.get_response(request1, "https://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic Yjo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://b.example") - self.assertEqual(request2.meta["proxy"], "https://b.example") + assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request2.meta["_auth_proxy"] == "https://b.example" + assert request2.meta["proxy"] == "https://b.example" response2 = self.get_response(request2, "http://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta proxy_mw.process_request(request3, spider) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta def test_system_proxy_unproxied_http_to_unproxied_https(self): crawler = get_crawler() @@ -811,37 +797,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertNotIn("Proxy-Authorization", request1.headers) - self.assertNotIn("_auth_proxy", request1.meta) - self.assertNotIn("proxy", request1.meta) + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta response1 = self.get_response(request1, "https://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta proxy_mw.process_request(request2, spider) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta response2 = self.get_response(request2, "http://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta proxy_mw.process_request(request3, spider) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta def test_system_proxy_proxied_https_to_proxied_http(self): crawler = get_crawler() @@ -857,37 +843,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic Yjo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://b.example") - self.assertEqual(request1.meta["proxy"], "https://b.example") + assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request1.meta["_auth_proxy"] == "https://b.example" + assert request1.meta["proxy"] == "https://b.example" response1 = self.get_response(request1, "http://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "https://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic Yjo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://b.example") - self.assertEqual(request3.meta["proxy"], "https://b.example") + assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request3.meta["_auth_proxy"] == "https://b.example" + assert request3.meta["proxy"] == "https://b.example" def test_system_proxy_proxied_https_to_unproxied_http(self): crawler = get_crawler() @@ -902,37 +888,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic Yjo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://b.example") - self.assertEqual(request1.meta["proxy"], "https://b.example") + assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request1.meta["_auth_proxy"] == "https://b.example" + assert request1.meta["proxy"] == "https://b.example" response1 = self.get_response(request1, "http://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta proxy_mw.process_request(request2, spider) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta response2 = self.get_response(request2, "https://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic Yjo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://b.example") - self.assertEqual(request3.meta["proxy"], "https://b.example") + assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request3.meta["_auth_proxy"] == "https://b.example" + assert request3.meta["proxy"] == "https://b.example" def test_system_proxy_unproxied_https_to_proxied_http(self): crawler = get_crawler() @@ -947,37 +933,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertNotIn("Proxy-Authorization", request1.headers) - self.assertNotIn("_auth_proxy", request1.meta) - self.assertNotIn("proxy", request1.meta) + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta response1 = self.get_response(request1, "http://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "https://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta proxy_mw.process_request(request3, spider) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta def test_system_proxy_unproxied_https_to_unproxied_http(self): crawler = get_crawler() @@ -988,44 +974,44 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertNotIn("Proxy-Authorization", request1.headers) - self.assertNotIn("_auth_proxy", request1.meta) - self.assertNotIn("proxy", request1.meta) + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta response1 = self.get_response(request1, "http://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta proxy_mw.process_request(request2, spider) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta response2 = self.get_response(request2, "https://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta proxy_mw.process_request(request3, spider) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta -class RedirectMiddlewareTest(Base.Test): +class TestRedirectMiddleware(Base.Test): mwcls = RedirectMiddleware reason = 302 - def setUp(self): + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") self.mw = self.mwcls.from_crawler(self.crawler) @@ -1043,8 +1029,8 @@ class RedirectMiddlewareTest(Base.Test): req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) - self.assertEqual(req2.url, url2) - self.assertEqual(req2.method, method) + assert req2.url == url2 + assert req2.method == method # response without Location header but with status code is 3XX should be ignored del rsp.headers["Location"] @@ -1070,8 +1056,8 @@ class RedirectMiddlewareTest(Base.Test): req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) - self.assertEqual(req2.url, url2) - self.assertEqual(req2.method, "HEAD") + assert req2.url == url2 + assert req2.method == "HEAD" def test_redirect_302_relative(self): url = "http://www.example.com/302" @@ -1082,8 +1068,8 @@ class RedirectMiddlewareTest(Base.Test): req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) - self.assertEqual(req2.url, url3) - self.assertEqual(req2.method, "HEAD") + assert req2.url == url3 + assert req2.method == "HEAD" def test_spider_handling(self): smartspider = self.crawler._create_spider("smarty") @@ -1093,7 +1079,7 @@ class RedirectMiddlewareTest(Base.Test): req = Request(url) rsp = Response(url, headers={"Location": url2}, status=301) r = self.mw.process_response(req, rsp, smartspider) - self.assertIs(r, rsp) + assert r is rsp def test_request_meta_handling(self): url = "http://www.example.com/301" @@ -1102,7 +1088,7 @@ class RedirectMiddlewareTest(Base.Test): def _test_passthrough(req): rsp = Response(url, headers={"Location": url2}, status=301, request=req) r = self.mw.process_response(req, rsp, self.spider) - self.assertIs(r, rsp) + assert r is rsp _test_passthrough( Request(url, meta={"handle_httpstatus_list": [404, 301, 302]}) @@ -1119,7 +1105,7 @@ class RedirectMiddlewareTest(Base.Test): ) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = "http://scrapytest.org/a%E7%E3o" - self.assertEqual(perc_encoded_utf8_url, req_result.url) + assert perc_encoded_utf8_url == req_result.url def test_utf8_location(self): req = Request("http://scrapytest.org/first") @@ -1131,7 +1117,7 @@ class RedirectMiddlewareTest(Base.Test): ) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = "http://scrapytest.org/a%C3%A7%C3%A3o" - self.assertEqual(perc_encoded_utf8_url, req_result.url) + assert perc_encoded_utf8_url == req_result.url def test_no_location(self): request = Request("https://example.com") @@ -1197,11 +1183,11 @@ def meta_refresh_body(url, interval=5): return html.encode("utf-8") -class MetaRefreshMiddlewareTest(Base.Test): +class TestMetaRefreshMiddleware(Base.Test): mwcls = MetaRefreshMiddleware reason = "meta refresh" - def setUp(self): + def setup_method(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider("foo") self.mw = self.mwcls.from_crawler(crawler) @@ -1217,7 +1203,7 @@ class MetaRefreshMiddlewareTest(Base.Test): rsp = HtmlResponse(req.url, body=self._body()) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) - self.assertEqual(req2.url, "http://example.org/newpage") + assert req2.url == "http://example.org/newpage" def test_meta_refresh_with_high_interval(self): # meta-refresh with high intervals don't trigger redirects @@ -1239,8 +1225,8 @@ class MetaRefreshMiddlewareTest(Base.Test): req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) - self.assertEqual(req2.url, "http://example.org/newpage") - self.assertEqual(req2.method, "GET") + assert req2.url == "http://example.org/newpage" + assert req2.method == "GET" assert "Content-Type" not in req2.headers, ( "Content-Type header must not be present in redirected request" ) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 36f48db69..ffdcdf49e 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,5 +1,4 @@ import logging -import unittest import pytest from testfixtures import LogCapture @@ -21,8 +20,8 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class RetryTest(unittest.TestCase): - def setUp(self): +class TestRetry: + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") self.mw = RetryMiddleware.from_crawler(self.crawler) @@ -70,12 +69,12 @@ class RetryTest(unittest.TestCase): # first retry req = self.mw.process_response(req, rsp, self.spider) assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 1) + assert req.meta["retry_times"] == 1 # second retry req = self.mw.process_response(req, rsp, self.spider) assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 2) + assert req.meta["retry_times"] == 2 # discard it assert self.mw.process_response(req, rsp, self.spider) is rsp @@ -129,19 +128,19 @@ class RetryTest(unittest.TestCase): # first retry req = mw.process_exception(req, exception, self.spider) assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 1) + assert req.meta["retry_times"] == 1 # second retry req = mw.process_exception(req, exception, self.spider) assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 2) + assert req.meta["retry_times"] == 2 # discard it req = mw.process_exception(req, exception, self.spider) - self.assertEqual(req, None) + assert req is None -class MaxRetryTimesTest(unittest.TestCase): +class TestMaxRetryTimes: invalid_url = "http://www.scrapytest.org/invalid_url" def get_spider_and_middleware(self, settings=None): @@ -272,10 +271,10 @@ class MaxRetryTimesTest(unittest.TestCase): # discard it req = middleware.process_exception(req, exception, spider) - self.assertEqual(req, None) + assert req is None -class GetRetryRequestTest(unittest.TestCase): +class TestGetRetryRequest: def get_spider(self, settings=None): crawler = get_crawler(Spider, settings or {}) return crawler._create_spider("foo") @@ -288,15 +287,15 @@ class GetRetryRequestTest(unittest.TestCase): request, spider=spider, ) - self.assertIsInstance(new_request, Request) - self.assertNotEqual(new_request, request) - self.assertEqual(new_request.dont_filter, True) + assert isinstance(new_request, Request) + assert new_request != request + assert new_request.dont_filter expected_retry_times = 1 - self.assertEqual(new_request.meta["retry_times"], expected_retry_times) - self.assertEqual(new_request.priority, -1) + assert new_request.meta["retry_times"] == expected_retry_times + assert new_request.priority == -1 expected_reason = "unspecified" for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): - self.assertEqual(spider.crawler.stats.get_value(stat), 1) + assert spider.crawler.stats.get_value(stat) == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -316,8 +315,8 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, max_retry_times=max_retry_times, ) - self.assertEqual(new_request, None) - self.assertEqual(spider.crawler.stats.get_value("retry/max_reached"), 1) + assert new_request is None + assert spider.crawler.stats.get_value("retry/max_reached") == 1 failure_count = max_retry_times + 1 expected_reason = "unspecified" log.check_present( @@ -338,15 +337,15 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, max_retry_times=1, ) - self.assertIsInstance(new_request, Request) - self.assertNotEqual(new_request, request) - self.assertEqual(new_request.dont_filter, True) + assert isinstance(new_request, Request) + assert new_request != request + assert new_request.dont_filter expected_retry_times = 1 - self.assertEqual(new_request.meta["retry_times"], expected_retry_times) - self.assertEqual(new_request.priority, -1) + assert new_request.meta["retry_times"] == expected_retry_times + assert new_request.priority == -1 expected_reason = "unspecified" for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): - self.assertEqual(spider.crawler.stats.get_value(stat), 1) + assert spider.crawler.stats.get_value(stat) == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -368,16 +367,16 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, max_retry_times=max_retry_times, ) - self.assertIsInstance(new_request, Request) - self.assertNotEqual(new_request, request) - self.assertEqual(new_request.dont_filter, True) + assert isinstance(new_request, Request) + assert new_request != request + assert new_request.dont_filter expected_retry_times = index + 1 - self.assertEqual(new_request.meta["retry_times"], expected_retry_times) - self.assertEqual(new_request.priority, -expected_retry_times) + assert new_request.meta["retry_times"] == expected_retry_times + assert new_request.priority == -expected_retry_times expected_reason = "unspecified" for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): value = spider.crawler.stats.get_value(stat) - self.assertEqual(value, expected_retry_times) + assert value == expected_retry_times log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -393,8 +392,8 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, max_retry_times=max_retry_times, ) - self.assertEqual(new_request, None) - self.assertEqual(spider.crawler.stats.get_value("retry/max_reached"), 1) + assert new_request is None + assert spider.crawler.stats.get_value("retry/max_reached") == 1 failure_count = max_retry_times + 1 expected_reason = "unspecified" log.check_present( @@ -419,7 +418,7 @@ class GetRetryRequestTest(unittest.TestCase): request, spider=spider, ) - self.assertEqual(new_request, None) + assert new_request is None def test_max_retry_times_meta(self): max_retry_times = 0 @@ -430,7 +429,7 @@ class GetRetryRequestTest(unittest.TestCase): request, spider=spider, ) - self.assertEqual(new_request, None) + assert new_request is None def test_max_retry_times_argument(self): max_retry_times = 0 @@ -442,7 +441,7 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, max_retry_times=max_retry_times, ) - self.assertEqual(new_request, None) + assert new_request is None def test_priority_adjust_setting(self): priority_adjust = 1 @@ -452,7 +451,7 @@ class GetRetryRequestTest(unittest.TestCase): request, spider=spider, ) - self.assertEqual(new_request.priority, priority_adjust) + assert new_request.priority == priority_adjust def test_priority_adjust_argument(self): priority_adjust = 1 @@ -463,7 +462,7 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, priority_adjust=priority_adjust, ) - self.assertEqual(new_request.priority, priority_adjust) + assert new_request.priority == priority_adjust def test_log_extra_retry_success(self): request = Request("https://example.com") @@ -498,7 +497,7 @@ class GetRetryRequestTest(unittest.TestCase): ) expected_retry_times = 1 for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): - self.assertEqual(spider.crawler.stats.get_value(stat), 1) + assert spider.crawler.stats.get_value(stat) == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -523,7 +522,7 @@ class GetRetryRequestTest(unittest.TestCase): stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) - self.assertEqual(stat, 1) + assert stat == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -548,7 +547,7 @@ class GetRetryRequestTest(unittest.TestCase): stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) - self.assertEqual(stat, 1) + assert stat == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -573,7 +572,7 @@ class GetRetryRequestTest(unittest.TestCase): stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) - self.assertEqual(stat, 1) + assert stat == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -598,7 +597,7 @@ class GetRetryRequestTest(unittest.TestCase): stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) - self.assertEqual(stat, 1) + assert stat == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -643,4 +642,4 @@ class GetRetryRequestTest(unittest.TestCase): f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}", ): - self.assertEqual(spider.crawler.stats.get_value(stat), 1) + assert spider.crawler.stats.get_value(stat) == 1 diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 9b95400fd..38f0333bb 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -15,7 +15,7 @@ from scrapy.settings import Settings from tests.test_robotstxt_interface import rerp_available -class RobotsTxtMiddlewareTest(unittest.TestCase): +class TestRobotsTxtMiddleware(unittest.TestCase): def setUp(self): self.crawler = mock.MagicMock() self.crawler.settings = Settings() @@ -242,11 +242,11 @@ Disallow: /some/randome/page.html def assertRobotsTxtRequested(self, base_url): calls = self.crawler.engine.download.call_args_list request = calls[0][0][0] - self.assertEqual(request.url, f"{base_url}/robots.txt") - self.assertEqual(request.callback, NO_CALLBACK) + assert request.url == f"{base_url}/robots.txt" + assert request.callback == NO_CALLBACK -class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): +class TestRobotsTxtMiddlewareWithRerp(TestRobotsTxtMiddleware): if not rerp_available(): skip = "Rerp parser is not installed" diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 5b7181848..748ef7d76 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,5 +1,3 @@ -from unittest import TestCase - from scrapy.downloadermiddlewares.stats import DownloaderStats from scrapy.http import Request, Response from scrapy.spiders import Spider @@ -10,8 +8,8 @@ class MyException(Exception): pass -class TestDownloaderStats(TestCase): - def setUp(self): +class TestDownloaderStats: + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("scrapytest.org") self.mw = DownloaderStats(self.crawler.stats) @@ -22,10 +20,8 @@ class TestDownloaderStats(TestCase): self.res = Response("scrapytest.org", status=400) def assertStatsEqual(self, key, value): - self.assertEqual( - self.crawler.stats.get_value(key, spider=self.spider), - value, - str(self.crawler.stats.get_stats(self.spider)), + assert self.crawler.stats.get_value(key, spider=self.spider) == value, str( + self.crawler.stats.get_stats(self.spider) ) def test_process_request(self): @@ -44,5 +40,5 @@ class TestDownloaderStats(TestCase): 1, ) - def tearDown(self): + def teardown_method(self): self.crawler.stats.close_spider(self.spider, "") diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index cad3dea5c..1497f8c67 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -1,12 +1,10 @@ -from unittest import TestCase - from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class UserAgentMiddlewareTest(TestCase): +class TestUserAgentMiddleware: def get_spider_and_mw(self, default_useragent): crawler = get_crawler(Spider, {"USER_AGENT": default_useragent}) spider = crawler._create_spider("foo") @@ -16,7 +14,7 @@ class UserAgentMiddlewareTest(TestCase): spider, mw = self.get_spider_and_mw("default_useragent") req = Request("http://scrapytest.org/") assert mw.process_request(req, spider) is None - self.assertEqual(req.headers["User-Agent"], b"default_useragent") + assert req.headers["User-Agent"] == b"default_useragent" def test_remove_agent(self): # settings USER_AGENT to None should remove the user agent @@ -33,7 +31,7 @@ class UserAgentMiddlewareTest(TestCase): mw.spider_opened(spider) req = Request("http://scrapytest.org/") assert mw.process_request(req, spider) is None - self.assertEqual(req.headers["User-Agent"], b"spider_useragent") + assert req.headers["User-Agent"] == b"spider_useragent" def test_header_agent(self): spider, mw = self.get_spider_and_mw("default_useragent") @@ -43,7 +41,7 @@ class UserAgentMiddlewareTest(TestCase): "http://scrapytest.org/", headers={"User-Agent": "header_useragent"} ) assert mw.process_request(req, spider) is None - self.assertEqual(req.headers["User-Agent"], b"header_useragent") + assert req.headers["User-Agent"] == b"header_useragent" def test_no_agent(self): spider, mw = self.get_spider_and_mw(None) From 5a605969bdc102e0193ad15ccc571dc6164e5d26 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 6 Mar 2025 23:52:41 +0400 Subject: [PATCH 063/491] Converting tests to plain asserts, part 2. (#6699) --- tests/test_addons.py | 48 +-- tests/test_closespider.py | 28 +- tests/test_cmdline/__init__.py | 30 +- .../__init__.py | 7 +- tests/test_command_check.py | 14 +- tests/test_command_fetch.py | 10 +- tests/test_command_parse.py | 81 ++-- tests/test_command_shell.py | 28 +- tests/test_command_version.py | 36 +- tests/test_commands.py | 371 +++++++++--------- tests/test_contracts.py | 112 +++--- tests/test_core_downloader.py | 25 +- tests/test_crawl.py | 288 +++++++------- tests/test_crawler.py | 351 +++++++++-------- tests/test_dependencies.py | 7 +- tests/test_downloaderslotssettings.py | 2 +- tests/test_dupefilters.py | 21 +- tests/test_engine.py | 137 ++++--- tests/test_engine_stop_download_bytes.py | 13 +- tests/test_engine_stop_download_headers.py | 12 +- tests/test_webclient.py | 40 +- 21 files changed, 796 insertions(+), 865 deletions(-) diff --git a/tests/test_addons.py b/tests/test_addons.py index a0caa3511..686bf9952 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -39,7 +39,7 @@ class CreateInstanceAddon: settings.update(self.config, "addon") -class AddonTest(unittest.TestCase): +class TestAddon: def test_update_settings(self): settings = BaseSettings() settings.set("KEY1", "default", priority="default") @@ -47,19 +47,19 @@ class AddonTest(unittest.TestCase): addon_config = {"KEY1": "addon", "KEY2": "addon", "KEY3": "addon"} testaddon = get_addon_cls(addon_config)() testaddon.update_settings(settings) - self.assertEqual(settings["KEY1"], "addon") - self.assertEqual(settings["KEY2"], "project") - self.assertEqual(settings["KEY3"], "addon") + assert settings["KEY1"] == "addon" + assert settings["KEY2"] == "project" + assert settings["KEY3"] == "addon" -class AddonManagerTest(unittest.TestCase): +class TestAddonManager(unittest.TestCase): def test_load_settings(self): settings_dict = { "ADDONS": {"tests.test_addons.SimpleAddon": 0}, } crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons - self.assertIsInstance(manager.addons[0], SimpleAddon) + assert isinstance(manager.addons[0], SimpleAddon) def test_notconfigured(self): class NotConfiguredAddon: @@ -71,7 +71,7 @@ class AddonManagerTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons - self.assertFalse(manager.addons) + assert not manager.addons def test_load_settings_order(self): # Get three addons with different settings @@ -86,8 +86,8 @@ class AddonManagerTest(unittest.TestCase): settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} crawler = get_crawler(settings_dict=settings) manager = crawler.addons - self.assertEqual([a.number for a in manager.addons], expected_order) - self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1]) + assert [a.number for a in manager.addons] == expected_order + assert crawler.settings.getint("KEY1") == expected_order[-1] def test_build_from_crawler(self): settings_dict = { @@ -96,8 +96,8 @@ class AddonManagerTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons - self.assertIsInstance(manager.addons[0], CreateInstanceAddon) - self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val") + assert isinstance(manager.addons[0], CreateInstanceAddon) + assert crawler.settings.get("MYADDON_KEY") == "val" def test_settings_priority(self): config = { @@ -107,14 +107,14 @@ class AddonManagerTest(unittest.TestCase): "ADDONS": {get_addon_cls(config): 1}, } crawler = get_crawler(settings_dict=settings_dict) - self.assertEqual(crawler.settings.getint("KEY"), 15) + assert crawler.settings.getint("KEY") == 15 settings = Settings(settings_dict) settings.set("KEY", 0, priority="default") runner = CrawlerRunner(settings) crawler = runner.create_crawler(Spider) crawler._apply_settings() - self.assertEqual(crawler.settings.getint("KEY"), 15) + assert crawler.settings.getint("KEY") == 15 settings_dict = { "KEY": 20, # priority=project @@ -124,7 +124,7 @@ class AddonManagerTest(unittest.TestCase): settings.set("KEY", 0, priority="default") runner = CrawlerRunner(settings) crawler = runner.create_crawler(Spider) - self.assertEqual(crawler.settings.getint("KEY"), 20) + assert crawler.settings.getint("KEY") == 20 def test_fallback_workflow(self): FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" @@ -143,12 +143,12 @@ class AddonManagerTest(unittest.TestCase): "ADDONS": {AddonWithFallback: 1}, } crawler = get_crawler(settings_dict=settings_dict) - self.assertEqual( - crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" + assert ( + crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"] == "AddonHandler" ) - self.assertEqual( - crawler.settings.get(FALLBACK_SETTING), - "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", + assert ( + crawler.settings.get(FALLBACK_SETTING) + == "scrapy.core.downloader.handlers.http.HTTPDownloadHandler" ) settings_dict = { @@ -156,10 +156,10 @@ class AddonManagerTest(unittest.TestCase): "DOWNLOAD_HANDLERS": {"https": "UserHandler"}, } crawler = get_crawler(settings_dict=settings_dict) - self.assertEqual( - crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" + assert ( + crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"] == "AddonHandler" ) - self.assertEqual(crawler.settings.get(FALLBACK_SETTING), "UserHandler") + assert crawler.settings.get(FALLBACK_SETTING) == "UserHandler" def test_logging_message(self): class LoggedAddon: @@ -199,6 +199,6 @@ class AddonManagerTest(unittest.TestCase): settings.set("KEY", "default", priority="default") runner = CrawlerRunner(settings) crawler = runner.create_crawler(MySpider) - self.assertEqual(crawler.settings.get("KEY"), "default") + assert crawler.settings.get("KEY") == "default" yield crawler.crawl() - self.assertEqual(crawler.settings.get("KEY"), "addon") + assert crawler.settings.get("KEY") == "addon" diff --git a/tests/test_closespider.py b/tests/test_closespider.py index ecde301d1..476662789 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -28,9 +28,9 @@ class TestCloseSpider(TestCase): crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on}) yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_itemcount") + assert reason == "closespider_itemcount" itemcount = crawler.stats.get_value("item_scraped_count") - self.assertTrue(itemcount >= close_on) + assert itemcount >= close_on @defer.inlineCallbacks def test_closespider_pagecount(self): @@ -38,9 +38,9 @@ class TestCloseSpider(TestCase): crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on}) yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_pagecount") + assert reason == "closespider_pagecount" pagecount = crawler.stats.get_value("response_received_count") - self.assertTrue(pagecount >= close_on) + assert pagecount >= close_on @defer.inlineCallbacks def test_closespider_pagecount_no_item(self): @@ -57,10 +57,10 @@ class TestCloseSpider(TestCase): max_items=max_items, max_requests=max_requests, mockserver=self.mockserver ) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_pagecount_no_item") + assert reason == "closespider_pagecount_no_item" pagecount = crawler.stats.get_value("response_received_count") itemcount = crawler.stats.get_value("item_scraped_count") - self.assertLessEqual(pagecount, close_on + itemcount) + assert pagecount <= close_on + itemcount @defer.inlineCallbacks def test_closespider_pagecount_no_item_with_pagecount(self): @@ -75,9 +75,9 @@ class TestCloseSpider(TestCase): ) yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_pagecount_no_item") + assert reason == "closespider_pagecount_no_item" pagecount = crawler.stats.get_value("response_received_count") - self.assertLess(pagecount, close_on_pagecount) + assert pagecount < close_on_pagecount @defer.inlineCallbacks def test_closespider_errorcount(self): @@ -85,10 +85,10 @@ class TestCloseSpider(TestCase): crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on}) yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_errorcount") + assert reason == "closespider_errorcount" key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}" errorcount = crawler.stats.get_value(key) - self.assertTrue(errorcount >= close_on) + assert errorcount >= close_on @defer.inlineCallbacks def test_closespider_timeout(self): @@ -96,9 +96,9 @@ class TestCloseSpider(TestCase): crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on}) yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_timeout") + assert reason == "closespider_timeout" total_seconds = crawler.stats.get_value("elapsed_time_seconds") - self.assertTrue(total_seconds >= close_on) + assert total_seconds >= close_on @defer.inlineCallbacks def test_closespider_timeout_no_item(self): @@ -106,6 +106,6 @@ class TestCloseSpider(TestCase): crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) yield crawler.crawl(n=3, mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_timeout_no_item") + assert reason == "closespider_timeout_no_item" total_seconds = crawler.stats.get_value("elapsed_time_seconds") - self.assertTrue(total_seconds >= timeout) + assert total_seconds >= timeout diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index acd524ea4..98a85bc17 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -4,7 +4,6 @@ import pstats import shutil import sys import tempfile -import unittest from io import StringIO from pathlib import Path from subprocess import PIPE, Popen @@ -12,8 +11,8 @@ from subprocess import PIPE, Popen from scrapy.utils.test import get_testenv -class CmdlineTest(unittest.TestCase): - def setUp(self): +class TestCmdline: + def setup_method(self): self.env = get_testenv() tests_path = Path(__file__).parent.parent self.env["PYTHONPATH"] += os.pathsep + str(tests_path.parent) @@ -27,12 +26,12 @@ class CmdlineTest(unittest.TestCase): return comm.decode(encoding) def test_default_settings(self): - self.assertEqual(self._execute("settings", "--get", "TEST1"), "default") + assert self._execute("settings", "--get", "TEST1") == "default" def test_override_settings_using_set_arg(self): - self.assertEqual( - self._execute("settings", "--get", "TEST1", "-s", "TEST1=override"), - "override", + assert ( + self._execute("settings", "--get", "TEST1", "-s", "TEST1=override") + == "override" ) def test_profiling(self): @@ -40,14 +39,14 @@ class CmdlineTest(unittest.TestCase): filename = path / "res.prof" try: self._execute("version", "--profile", str(filename)) - self.assertTrue(filename.exists()) + assert filename.exists() out = StringIO() stats = pstats.Stats(str(filename), stream=out) stats.print_stats() out.seek(0) stats = out.read() - self.assertIn(str(Path("scrapy", "commands", "version.py")), stats) - self.assertIn("tottime", stats) + assert str(Path("scrapy", "commands", "version.py")) in stats + assert "tottime" in stats finally: shutil.rmtree(path) @@ -62,15 +61,14 @@ class CmdlineTest(unittest.TestCase): "EXTENSIONS=" + json.dumps(EXTENSIONS), ) # XXX: There's gotta be a smarter way to do this... - self.assertNotIn("...", settingsstr) + assert "..." not in settingsstr for char in ("'", "<", ">"): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) - self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys()) - self.assertEqual(200, settingsdict[EXT_PATH]) + assert set(settingsdict.keys()) == set(EXTENSIONS.keys()) + assert settingsdict[EXT_PATH] == 200 def test_pathlib_path_as_feeds_key(self): - self.assertEqual( - self._execute("settings", "--get", "FEEDS"), - json.dumps({"items.csv": {"format": "csv", "fields": ["price", "name"]}}), + assert self._execute("settings", "--get", "FEEDS") == json.dumps( + {"items.csv": {"format": "csv", "fields": ["price", "name"]}} ) diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py index 5cb09b5c0..5228f6abd 100644 --- a/tests/test_cmdline_crawl_with_pipeline/__init__.py +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -1,10 +1,9 @@ import sys -import unittest from pathlib import Path from subprocess import PIPE, Popen -class CmdlineCrawlPipelineTest(unittest.TestCase): +class TestCmdlineCrawlPipeline: def _execute(self, spname): args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname) cwd = Path(__file__).resolve().parent @@ -13,7 +12,7 @@ class CmdlineCrawlPipelineTest(unittest.TestCase): return proc.returncode def test_open_spider_normally_in_pipeline(self): - self.assertEqual(self._execute("normal"), 0) + assert self._execute("normal") == 0 def test_exception_at_open_spider_in_pipeline(self): - self.assertEqual(self._execute("exception"), 1) + assert self._execute("exception") == 1 diff --git a/tests/test_command_check.py b/tests/test_command_check.py index b0f1cd38a..975f31dfe 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -3,10 +3,10 @@ from io import StringIO from unittest.mock import Mock, PropertyMock, call, patch from scrapy.commands.check import Command, TextTestResult -from tests.test_commands import CommandTest +from tests.test_commands import TestCommandBase -class CheckCommandTest(CommandTest): +class TestCheckCommand(TestCommandBase): command = "check" def setUp(self): @@ -36,9 +36,9 @@ class CheckSpider(scrapy.Spider): def _test_contract(self, contracts="", parse_def="pass"): self._write_contract(contracts, parse_def) p, out, err = self.proc("check") - self.assertNotIn("F", out) - self.assertIn("OK", err) - self.assertEqual(p.returncode, 0) + assert "F" not in out + assert "OK" in err + assert p.returncode == 0 def test_check_returns_requests_contract(self): contracts = """ @@ -171,9 +171,7 @@ class CheckSpider(scrapy.Spider): cmd.run([spider_name], Mock(list=True)) - self.assertEqual( - "FakeSpider\n * fakeMethod1\n * fakeMethod2\n", output.getvalue() - ) + assert output.getvalue() == "FakeSpider\n * fakeMethod1\n * fakeMethod2\n" sys.stdout = sys.__stdout__ @patch("scrapy.commands.check.ContractsManager") diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index a4d7fdd30..a31cada85 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -5,18 +5,18 @@ from tests.utils.testproc import ProcessTest from tests.utils.testsite import SiteTest -class FetchTest(ProcessTest, SiteTest, unittest.TestCase): +class TestFetchCommand(ProcessTest, SiteTest, unittest.TestCase): command = "fetch" @defer.inlineCallbacks def test_output(self): _, out, _ = yield self.execute([self.url("/text")]) - self.assertEqual(out.strip(), b"Works") + assert out.strip() == b"Works" @defer.inlineCallbacks def test_redirect_default(self): _, out, _ = yield self.execute([self.url("/redirect")]) - self.assertEqual(out.strip(), b"Redirected here") + assert out.strip() == b"Redirected here" @defer.inlineCallbacks def test_redirect_disabled(self): @@ -24,8 +24,8 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase): ["--no-redirect", self.url("/redirect-no-meta-refresh")] ) err = err.strip() - self.assertIn(b"downloader/response_status_count/302", err, err) - self.assertNotIn(b"downloader/response_status_count/200", err, err) + assert b"downloader/response_status_count/302" in err, err + assert b"downloader/response_status_count/200" not in err, err @defer.inlineCallbacks def test_headers(self): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 9f2c7fa13..9e66d319c 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,5 +1,6 @@ import argparse import os +import re from pathlib import Path from twisted.internet import defer @@ -7,18 +8,18 @@ from twisted.internet import defer from scrapy.commands import parse from scrapy.settings import Settings from scrapy.utils.python import to_unicode -from tests.test_commands import CommandTest +from tests.test_commands import TestCommandBase from tests.utils.testproc import ProcessTest from tests.utils.testsite import SiteTest -def _textmode(bstr): +def _textmode(bstr: bytes) -> str: """Normalize input the same as writing to a file and reading from it in text mode""" return to_unicode(bstr).replace(os.linesep, "\n") -class ParseCommandTest(ProcessTest, SiteTest, CommandTest): +class TestParseCommand(ProcessTest, SiteTest, TestCommandBase): command = "parse" def setUp(self): @@ -184,7 +185,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("DEBUG: It Works!", _textmode(stderr)) + assert "DEBUG: It Works!" in _textmode(stderr) @defer.inlineCallbacks def test_request_with_meta(self): @@ -201,7 +202,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("DEBUG: It Works!", _textmode(stderr)) + assert "DEBUG: It Works!" in _textmode(stderr) _, _, stderr = yield self.execute( [ @@ -215,7 +216,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("DEBUG: It Works!", _textmode(stderr)) + assert "DEBUG: It Works!" in _textmode(stderr) @defer.inlineCallbacks def test_request_with_cb_kwargs(self): @@ -233,9 +234,9 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ] ) log = _textmode(stderr) - self.assertIn("DEBUG: It Works!", log) - self.assertIn( - "DEBUG: request.callback signature: (response, foo=None, key=None)", log + assert "DEBUG: It Works!" in log + assert ( + "DEBUG: request.callback signature: (response, foo=None, key=None)" in log ) @defer.inlineCallbacks @@ -250,7 +251,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("DEBUG: It Works!", _textmode(stderr)) + assert "DEBUG: It Works!" in _textmode(stderr) @defer.inlineCallbacks def test_pipelines(self): @@ -265,7 +266,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("INFO: It Works!", _textmode(stderr)) + assert "INFO: It Works!" in _textmode(stderr) @defer.inlineCallbacks def test_async_def_asyncio_parse_items_list(self): @@ -278,9 +279,9 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("INFO: Got response 200", _textmode(stderr)) - self.assertIn("{'id': 1}", _textmode(out)) - self.assertIn("{'id': 2}", _textmode(out)) + assert "INFO: Got response 200" in _textmode(stderr) + assert "{'id': 1}" in _textmode(out) + assert "{'id': 2}" in _textmode(out) @defer.inlineCallbacks def test_async_def_asyncio_parse_items_single_element(self): @@ -293,8 +294,8 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("INFO: Got response 200", _textmode(stderr)) - self.assertIn("{'foo': 42}", _textmode(out)) + assert "INFO: Got response 200" in _textmode(stderr) + assert "{'foo': 42}" in _textmode(out) @defer.inlineCallbacks def test_async_def_asyncgen_parse_loop(self): @@ -307,9 +308,9 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("INFO: Got response 200", _textmode(stderr)) + assert "INFO: Got response 200" in _textmode(stderr) for i in range(10): - self.assertIn(f"{{'foo': {i}}}", _textmode(out)) + assert f"{{'foo': {i}}}" in _textmode(out) @defer.inlineCallbacks def test_async_def_asyncgen_parse_exc(self): @@ -322,9 +323,9 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("ValueError", _textmode(stderr)) + assert "ValueError" in _textmode(stderr) for i in range(7): - self.assertIn(f"{{'foo': {i}}}", _textmode(out)) + assert f"{{'foo': {i}}}" in _textmode(out) @defer.inlineCallbacks def test_async_def_asyncio_parse(self): @@ -337,29 +338,29 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("DEBUG: Got response 200", _textmode(stderr)) + assert "DEBUG: Got response 200" in _textmode(stderr) @defer.inlineCallbacks def test_parse_items(self): status, out, stderr = yield self.execute( ["--spider", self.spider_name, "-c", "parse", self.url("/html")] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) + assert "[{}, {'foo': 'bar'}]" in _textmode(out) @defer.inlineCallbacks def test_parse_items_no_callback_passed(self): status, out, stderr = yield self.execute( ["--spider", self.spider_name, self.url("/html")] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) + assert "[{}, {'foo': 'bar'}]" in _textmode(out) @defer.inlineCallbacks def test_wrong_callback_passed(self): status, out, stderr = yield self.execute( ["--spider", self.spider_name, "-c", "dummy", self.url("/html")] ) - self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") - self.assertIn("""Cannot find callback""", _textmode(stderr)) + assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) + assert "Cannot find callback" in _textmode(stderr) @defer.inlineCallbacks def test_crawlspider_matching_rule_callback_set(self): @@ -367,7 +368,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} status, out, stderr = yield self.execute( ["--spider", "goodcrawl" + self.spider_name, "-r", self.url("/html")] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) + assert "[{}, {'foo': 'bar'}]" in _textmode(out) @defer.inlineCallbacks def test_crawlspider_matching_rule_default_callback(self): @@ -375,7 +376,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} status, out, stderr = yield self.execute( ["--spider", "goodcrawl" + self.spider_name, "-r", self.url("/text")] ) - self.assertIn("""[{}, {'nomatch': 'default'}]""", _textmode(out)) + assert "[{}, {'nomatch': 'default'}]" in _textmode(out) @defer.inlineCallbacks def test_spider_with_no_rules_attribute(self): @@ -383,15 +384,15 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} status, out, stderr = yield self.execute( ["--spider", self.spider_name, "-r", self.url("/html")] ) - self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") - self.assertIn("""No CrawlSpider rules found""", _textmode(stderr)) + assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) + assert "No CrawlSpider rules found" in _textmode(stderr) @defer.inlineCallbacks def test_crawlspider_missing_callback(self): status, out, stderr = yield self.execute( ["--spider", "badcrawl" + self.spider_name, "-r", self.url("/html")] ) - self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") + assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) @defer.inlineCallbacks def test_crawlspider_no_matching_rule(self): @@ -399,13 +400,13 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} status, out, stderr = yield self.execute( ["--spider", "badcrawl" + self.spider_name, "-r", self.url("/enc-gb18030")] ) - self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") - self.assertIn("""Cannot find a rule that matches""", _textmode(stderr)) + assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) + assert "Cannot find a rule that matches" in _textmode(stderr) @defer.inlineCallbacks def test_crawlspider_not_exists_with_not_matched_url(self): status, out, stderr = yield self.execute([self.url("/invalid_url")]) - self.assertEqual(status, 0) + assert status == 0 @defer.inlineCallbacks def test_output_flag(self): @@ -426,11 +427,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ] ) - self.assertTrue(file_path.exists()) - self.assertTrue(file_path.is_file()) + assert file_path.exists() + assert file_path.is_file() content = '[\n{},\n{"foo": "bar"}\n]' - self.assertEqual(file_path.read_text(encoding="utf-8"), content) + assert file_path.read_text(encoding="utf-8") == content def test_parse_add_options(self): command = parse.Command() @@ -445,7 +446,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} namespace = parser.parse_args( ["--verbose", "--nolinks", "-d", "2", "--spider", self.spider_name] ) - self.assertTrue(namespace.nolinks) - self.assertEqual(namespace.depth, 2) - self.assertEqual(namespace.spider, self.spider_name) - self.assertTrue(namespace.verbose) + assert namespace.nolinks + assert namespace.depth == 2 + assert namespace.spider == self.spider_name + assert namespace.verbose diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 9ca5e05dc..0f45a7ee8 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -13,7 +13,7 @@ from tests.utils.testproc import ProcessTest from tests.utils.testsite import SiteTest -class ShellTest(ProcessTest, SiteTest, unittest.TestCase): +class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): command = "shell" @defer.inlineCallbacks @@ -40,14 +40,14 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): def test_response_selector_html(self): xpath = "response.xpath(\"//p[@class='one']/text()\").get()" _, out, _ = yield self.execute([self.url("/html"), "-c", xpath]) - self.assertEqual(out.strip(), b"Works") + assert out.strip() == b"Works" @defer.inlineCallbacks def test_response_encoding_gb18030(self): _, out, _ = yield self.execute( [self.url("/enc-gb18030"), "-c", "response.encoding"] ) - self.assertEqual(out.strip(), b"gb18030") + assert out.strip() == b"gb18030" @defer.inlineCallbacks def test_redirect(self): @@ -79,7 +79,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): url = self.url("/redirect-no-meta-refresh") code = f"fetch('{url}')" errcode, out, errout = yield self.execute(["-c", code]) - self.assertEqual(errcode, 0, out) + assert errcode == 0, out assert b"Redirecting (302)" in errout assert b"Crawled (200)" in errout @@ -89,7 +89,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): url = self.url("/redirect-no-meta-refresh") code = f"fetch('{url}', redirect=False)" errcode, out, errout = yield self.execute(["-c", code]) - self.assertEqual(errcode, 0, out) + assert errcode == 0, out assert b"Crawled (302)" in errout @defer.inlineCallbacks @@ -97,14 +97,14 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): url = self.url("/text") code = f"fetch('{url}') or fetch(response.request.replace(method='POST'))" errcode, out, _ = yield self.execute(["-c", code]) - self.assertEqual(errcode, 0, out) + assert errcode == 0, out @defer.inlineCallbacks def test_scrapy_import(self): url = self.url("/text") code = f"fetch(scrapy.Request('{url}'))" errcode, out, _ = yield self.execute(["-c", code]) - self.assertEqual(errcode, 0, out) + assert errcode == 0, out @defer.inlineCallbacks def test_local_file(self): @@ -118,8 +118,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): errcode, out, err = yield self.execute( [filepath, "-c", "item"], check_code=False ) - self.assertEqual(errcode, 1, out or err) - self.assertIn(b"No such file or directory", err) + assert errcode == 1, out or err + assert b"No such file or directory" in err @defer.inlineCallbacks def test_dns_failures(self): @@ -127,8 +127,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): raise unittest.SkipTest("Non-existing hosts are resolvable") url = "www.somedomainthatdoesntexi.st" errcode, out, err = yield self.execute([url, "-c", "item"], check_code=False) - self.assertEqual(errcode, 1, out or err) - self.assertIn(b"DNS lookup failed", err) + assert errcode == 1, out or err + assert b"DNS lookup failed" in err @defer.inlineCallbacks def test_shell_fetch_async(self): @@ -137,10 +137,10 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): code = f"fetch('{url}')" args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"] _, _, err = yield self.execute(args, check_code=True) - self.assertNotIn(b"RuntimeError: There is no current event loop in thread", err) + assert b"RuntimeError: There is no current event loop in thread" not in err -class InteractiveShellTest(unittest.TestCase): +class TestInteractiveShell: def test_fetch(self): args = ( sys.executable, @@ -161,4 +161,4 @@ class InteractiveShellTest(unittest.TestCase): p.sendeof() p.wait() logfile.seek(0) - self.assertNotIn("Traceback", logfile.read().decode()) + assert "Traceback" not in logfile.read().decode() diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 917f457cb..a61a6a32b 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -7,17 +7,14 @@ import scrapy from tests.utils.testproc import ProcessTest -class VersionTest(ProcessTest, unittest.TestCase): +class TestVersionCommand(ProcessTest, unittest.TestCase): command = "version" @defer.inlineCallbacks def test_output(self): encoding = sys.stdout.encoding or "utf-8" _, out, _ = yield self.execute([]) - self.assertEqual( - out.strip().decode(encoding), - f"Scrapy {scrapy.__version__}", - ) + assert out.strip().decode(encoding) == f"Scrapy {scrapy.__version__}" @defer.inlineCallbacks def test_verbose_output(self): @@ -27,19 +24,16 @@ class VersionTest(ProcessTest, unittest.TestCase): line.partition(":")[0].strip() for line in out.strip().decode(encoding).splitlines() ] - self.assertEqual( - headers, - [ - "Scrapy", - "lxml", - "libxml2", - "cssselect", - "parsel", - "w3lib", - "Twisted", - "Python", - "pyOpenSSL", - "cryptography", - "Platform", - ], - ) + assert headers == [ + "Scrapy", + "lxml", + "libxml2", + "cssselect", + "parsel", + "w3lib", + "Twisted", + "Python", + "pyOpenSSL", + "cryptography", + "Platform", + ] diff --git a/tests/test_commands.py b/tests/test_commands.py index 1a0db1e03..f63e05628 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -14,7 +14,7 @@ from itertools import chain from pathlib import Path from shutil import copytree, rmtree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION -from tempfile import TemporaryFile, mkdtemp +from tempfile import TemporaryDirectory, TemporaryFile, mkdtemp from threading import Timer from typing import TYPE_CHECKING from unittest import mock, skipIf @@ -35,8 +35,8 @@ if TYPE_CHECKING: from collections.abc import Iterator -class CommandSettings(unittest.TestCase): - def setUp(self): +class TestCommandSettings: + def setup_method(self): self.command = ScrapyCommand() self.command.settings = Settings() self.parser = argparse.ArgumentParser( @@ -50,10 +50,8 @@ class CommandSettings(unittest.TestCase): args=["-s", f"FEEDS={feeds_json}", "spider.py"] ) self.command.process_options(args, opts) - self.assertIsInstance( - self.command.settings["FEEDS"], scrapy.settings.BaseSettings - ) - self.assertEqual(dict(self.command.settings["FEEDS"]), json.loads(feeds_json)) + assert isinstance(self.command.settings["FEEDS"], scrapy.settings.BaseSettings) + assert dict(self.command.settings["FEEDS"]) == json.loads(feeds_json) def test_help_formatter(self): formatter = ScrapyHelpFormatter(prog="scrapy") @@ -64,17 +62,14 @@ class CommandSettings(unittest.TestCase): "\n", "Global Options:\n", ] - self.assertEqual( - formatter._join_parts(part_strings), - ( - "Usage\n=====\n scrapy genspider [options] \n\n\n" - "Optional Arguments\n==================\n\n" - "Global Options\n--------------\n" - ), + assert formatter._join_parts(part_strings) == ( + "Usage\n=====\n scrapy genspider [options] \n\n\n" + "Optional Arguments\n==================\n\n" + "Global Options\n--------------\n" ) -class ProjectTest(unittest.TestCase): +class TestProjectBase(unittest.TestCase): project_name = "testproject" def setUp(self): @@ -130,12 +125,12 @@ class ProjectTest(unittest.TestCase): return None -class StartprojectTest(ProjectTest): +class TestStartprojectCommand(TestProjectBase): def test_startproject(self): p, out, err = self.proc("startproject", self.project_name) print(out) print(err, file=sys.stderr) - self.assertEqual(p.returncode, 0) + assert p.returncode == 0 assert Path(self.proj_path, "scrapy.cfg").exists() assert Path(self.proj_path, "testproject").exists() @@ -145,13 +140,13 @@ class StartprojectTest(ProjectTest): assert Path(self.proj_mod_path, "settings.py").exists() assert Path(self.proj_mod_path, "spiders", "__init__.py").exists() - self.assertEqual(1, self.call("startproject", self.project_name)) - self.assertEqual(1, self.call("startproject", "wrong---project---name")) - self.assertEqual(1, self.call("startproject", "sys")) + assert self.call("startproject", self.project_name) == 1 + assert self.call("startproject", "wrong---project---name") == 1 + assert self.call("startproject", "sys") == 1 def test_startproject_with_project_dir(self): project_dir = mkdtemp() - self.assertEqual(0, self.call("startproject", self.project_name, project_dir)) + assert self.call("startproject", self.project_name, project_dir) == 0 assert Path(project_dir, "scrapy.cfg").exists() assert Path(project_dir, "testproject").exists() @@ -161,20 +156,16 @@ class StartprojectTest(ProjectTest): assert Path(project_dir, self.project_name, "settings.py").exists() assert Path(project_dir, self.project_name, "spiders", "__init__.py").exists() - self.assertEqual( - 0, self.call("startproject", self.project_name, project_dir + "2") - ) + assert self.call("startproject", self.project_name, project_dir + "2") == 0 - self.assertEqual(1, self.call("startproject", self.project_name, project_dir)) - self.assertEqual( - 1, self.call("startproject", self.project_name + "2", project_dir) - ) - self.assertEqual(1, self.call("startproject", "wrong---project---name")) - self.assertEqual(1, self.call("startproject", "sys")) - self.assertEqual(2, self.call("startproject")) - self.assertEqual( - 2, - self.call("startproject", self.project_name, project_dir, "another_params"), + assert self.call("startproject", self.project_name, project_dir) == 1 + assert self.call("startproject", self.project_name + "2", project_dir) == 1 + assert self.call("startproject", "wrong---project---name") == 1 + assert self.call("startproject", "sys") == 1 + assert self.call("startproject") == 2 + assert ( + self.call("startproject", self.project_name, project_dir, "another_params") + == 2 ) def test_existing_project_dir(self): @@ -186,7 +177,7 @@ class StartprojectTest(ProjectTest): p, out, err = self.proc("startproject", project_name, cwd=project_dir) print(out) print(err, file=sys.stderr) - self.assertEqual(p.returncode, 0) + assert p.returncode == 0 assert Path(project_path, "scrapy.cfg").exists() assert Path(project_path, project_name).exists() @@ -224,7 +215,7 @@ def get_permissions_dict( return permissions_dict -class StartprojectTemplatesTest(ProjectTest): +class TestStartprojectTemplates(TestProjectBase): maxDiff = None def setUp(self): @@ -239,11 +230,10 @@ class StartprojectTemplatesTest(ProjectTest): args = ["--set", f"TEMPLATES_DIR={self.tmpl}"] p, out, err = self.proc("startproject", self.project_name, *args) - self.assertIn( - f"New Scrapy project '{self.project_name}', using template directory", - out, + assert ( + f"New Scrapy project '{self.project_name}', using template directory" in out ) - self.assertIn(self.tmpl_proj, out) + assert self.tmpl_proj in out assert Path(self.proj_path, "root_template").exists() def test_startproject_permissions_from_writable(self): @@ -280,7 +270,7 @@ class StartprojectTemplatesTest(ProjectTest): project_dir = Path(destination, project_name) actual_permissions = get_permissions_dict(project_dir) - self.assertEqual(actual_permissions, expected_permissions) + assert actual_permissions == expected_permissions def test_startproject_permissions_from_read_only(self): """Check that generated files have the right permissions when the @@ -333,7 +323,7 @@ class StartprojectTemplatesTest(ProjectTest): project_dir = Path(destination, project_name) actual_permissions = get_permissions_dict(project_dir) - self.assertEqual(actual_permissions, expected_permissions) + assert actual_permissions == expected_permissions def test_startproject_permissions_unchanged_in_destination(self): """Check that preexisting folders and files in the destination folder @@ -391,7 +381,7 @@ class StartprojectTemplatesTest(ProjectTest): actual_permissions = get_permissions_dict(project_dir) - self.assertEqual(actual_permissions, expected_permissions) + assert actual_permissions == expected_permissions def test_startproject_permissions_umask_022(self): """Check that generated files have the right permissions when the @@ -435,10 +425,10 @@ class StartprojectTemplatesTest(ProjectTest): project_dir = Path(destination, project_name) actual_permissions = get_permissions_dict(project_dir) - self.assertEqual(actual_permissions, expected_permissions) + assert actual_permissions == expected_permissions -class CommandTest(ProjectTest): +class TestCommandBase(TestProjectBase): def setUp(self): super().setUp() self.call("startproject", self.project_name) @@ -446,13 +436,13 @@ class CommandTest(ProjectTest): self.env["SCRAPY_SETTINGS_MODULE"] = f"{self.project_name}.settings" -class GenspiderCommandTest(CommandTest): +class TestGenspiderCommand(TestCommandBase): def test_arguments(self): # only pass one argument. spider script shouldn't be created - self.assertEqual(2, self.call("genspider", "test_name")) + assert self.call("genspider", "test_name") == 2 assert not Path(self.proj_mod_path, "spiders", "test_name.py").exists() # pass two arguments . spider script should be created - self.assertEqual(0, self.call("genspider", "test_name", "test.com")) + assert self.call("genspider", "test_name", "test.com") == 0 assert Path(self.proj_mod_path, "spiders", "test_name.py").exists() def test_template(self, tplname="crawl"): @@ -460,20 +450,20 @@ class GenspiderCommandTest(CommandTest): spname = "test_spider" spmodule = f"{self.project_name}.spiders.{spname}" p, out, err = self.proc("genspider", spname, "test.com", *args) - self.assertIn( - f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}", - out, + assert ( + f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}" + in out ) - self.assertTrue(Path(self.proj_mod_path, "spiders", "test_spider.py").exists()) + assert Path(self.proj_mod_path, "spiders", "test_spider.py").exists() modify_time_before = ( Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime ) p, out, err = self.proc("genspider", spname, "test.com", *args) - self.assertIn(f"Spider {spname!r} already exists in module", out) + assert f"Spider {spname!r} already exists in module" in out modify_time_after = ( Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime ) - self.assertEqual(modify_time_after, modify_time_before) + assert modify_time_after == modify_time_before def test_template_basic(self): self.test_template("basic") @@ -485,14 +475,14 @@ class GenspiderCommandTest(CommandTest): self.test_template("xmlfeed") def test_list(self): - self.assertEqual(0, self.call("genspider", "--list")) + assert self.call("genspider", "--list") == 0 def test_dump(self): - self.assertEqual(0, self.call("genspider", "--dump=basic")) - self.assertEqual(0, self.call("genspider", "-d", "basic")) + assert self.call("genspider", "--dump=basic") == 0 + assert self.call("genspider", "-d", "basic") == 0 def test_same_name_as_project(self): - self.assertEqual(2, self.call("genspider", self.project_name)) + assert self.call("genspider", self.project_name) == 2 assert not Path( self.proj_mod_path, "spiders", f"{self.project_name}.py" ).exists() @@ -500,7 +490,7 @@ class GenspiderCommandTest(CommandTest): def test_same_filename_as_existing_spider(self, force=False): file_name = "example" file_path = Path(self.proj_mod_path, "spiders", f"{file_name}.py") - self.assertEqual(0, self.call("genspider", file_name, "example.com")) + assert self.call("genspider", file_name, "example.com") == 0 assert file_path.exists() # change name of spider but not its file name @@ -515,39 +505,39 @@ class GenspiderCommandTest(CommandTest): if force: p, out, err = self.proc("genspider", "--force", file_name, "example.com") - self.assertIn( - f"Created spider {file_name!r} using template 'basic' in module", out + assert ( + f"Created spider {file_name!r} using template 'basic' in module" in out ) modify_time_after = file_path.stat().st_mtime - self.assertNotEqual(modify_time_after, modify_time_before) + assert modify_time_after != modify_time_before file_contents_after = file_path.read_text(encoding="utf-8") - self.assertNotEqual(file_contents_after, file_contents_before) + assert file_contents_after != file_contents_before else: p, out, err = self.proc("genspider", file_name, "example.com") - self.assertIn(f"{file_path.resolve()} already exists", out) + assert f"{file_path.resolve()} already exists" in out modify_time_after = file_path.stat().st_mtime - self.assertEqual(modify_time_after, modify_time_before) + assert modify_time_after == modify_time_before file_contents_after = file_path.read_text(encoding="utf-8") - self.assertEqual(file_contents_after, file_contents_before) + assert file_contents_after == file_contents_before def test_same_filename_as_existing_spider_force(self): self.test_same_filename_as_existing_spider(force=True) def test_url(self, url="test.com", domain="test.com"): - self.assertEqual(0, self.call("genspider", "--force", "test_name", url)) - self.assertEqual( - domain, + assert self.call("genspider", "--force", "test_name", url) == 0 + assert ( self.find_in_file( Path(self.proj_mod_path, "spiders", "test_name.py"), r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]", - ).group(1), + ).group(1) + == domain ) - self.assertEqual( - f"https://{domain}", + assert ( self.find_in_file( Path(self.proj_mod_path, "spiders", "test_name.py"), r"start_urls\s*=\s*\[['\"](.+)['\"]\]", - ).group(1), + ).group(1) + == f"https://{domain}" ) def test_url_schema(self): @@ -556,15 +546,13 @@ class GenspiderCommandTest(CommandTest): def test_template_start_urls( self, url="test.com", expected="https://test.com", template="basic" ): - self.assertEqual( - 0, self.call("genspider", "-t", template, "--force", "test_name", url) - ) - self.assertEqual( - expected, + assert self.call("genspider", "-t", template, "--force", "test_name", url) == 0 + assert ( self.find_in_file( Path(self.proj_mod_path, "spiders", "test_name.py"), r"start_urls\s*=\s*\[['\"](.+)['\"]\]", - ).group(1), + ).group(1) + == expected ) def test_genspider_basic_start_urls(self): @@ -611,7 +599,7 @@ class GenspiderCommandTest(CommandTest): ) -class GenspiderStandaloneCommandTest(ProjectTest): +class TestGenspiderStandaloneCommand(TestProjectBase): def test_generate_standalone_spider(self): self.call("genspider", "example", "example.com") assert Path(self.temp_path, "example.py").exists() @@ -620,7 +608,7 @@ class GenspiderStandaloneCommandTest(ProjectTest): file_name = "example" file_path = Path(self.temp_path, file_name + ".py") p, out, err = self.proc("genspider", file_name, "example.com") - self.assertIn(f"Created spider {file_name!r} using template 'basic' ", out) + assert f"Created spider {file_name!r} using template 'basic' " in out assert file_path.exists() modify_time_before = file_path.stat().st_mtime file_contents_before = file_path.read_text(encoding="utf-8") @@ -630,29 +618,29 @@ class GenspiderStandaloneCommandTest(ProjectTest): p, out, err = self.proc( "genspider", "--force", "-t", "crawl", file_name, "example.com" ) - self.assertIn(f"Created spider {file_name!r} using template 'crawl' ", out) + assert f"Created spider {file_name!r} using template 'crawl' " in out modify_time_after = file_path.stat().st_mtime - self.assertNotEqual(modify_time_after, modify_time_before) + assert modify_time_after != modify_time_before file_contents_after = file_path.read_text(encoding="utf-8") - self.assertNotEqual(file_contents_after, file_contents_before) + assert file_contents_after != file_contents_before else: p, out, err = self.proc("genspider", file_name, "example.com") - self.assertIn( - f"{Path(self.temp_path, file_name + '.py').resolve()} already exists", - out, + assert ( + f"{Path(self.temp_path, file_name + '.py').resolve()} already exists" + in out ) modify_time_after = file_path.stat().st_mtime - self.assertEqual(modify_time_after, modify_time_before) + assert modify_time_after == modify_time_before file_contents_after = file_path.read_text(encoding="utf-8") - self.assertEqual(file_contents_after, file_contents_before) + assert file_contents_after == file_contents_before def test_same_name_as_existing_file_force(self): self.test_same_name_as_existing_file(force=True) -class MiscCommandsTest(CommandTest): +class TestMiscCommands(TestCommandBase): def test_list(self): - self.assertEqual(0, self.call("list")) + assert self.call("list") == 0 def test_command_not_found(self): na_msg = """ @@ -670,10 +658,10 @@ Unknown command: abc for cmdname, inproject, message in params: with mock.patch("sys.stdout", new=StringIO()) as out: _print_unknown_command_msg(Settings(), cmdname, inproject) - self.assertEqual(out.getvalue().strip(), message.strip()) + assert out.getvalue().strip() == message.strip() -class RunSpiderCommandTest(CommandTest): +class TestRunSpiderCommand(TestCommandBase): spider_filename = "myspider.py" debug_log_spider = """ @@ -697,18 +685,14 @@ class BadSpider(scrapy.Spider): """ @contextmanager - def _create_file(self, content, name=None) -> Iterator[str]: - tmpdir = Path(self.mktemp()) - tmpdir.mkdir() - if name: - fname = (tmpdir / name).resolve() - else: - fname = (tmpdir / self.spider_filename).resolve() - fname.write_text(content, encoding="utf-8") - try: + def _create_file(self, content: str, name: str | None = None) -> Iterator[str]: + with TemporaryDirectory() as tmpdir: + if name: + fname = Path(tmpdir, name).resolve() + else: + fname = Path(tmpdir, self.spider_filename).resolve() + fname.write_text(content, encoding="utf-8") yield str(fname) - finally: - rmtree(tmpdir) def runspider(self, code, name=None, args=()): with self._create_file(code, name) as fname: @@ -720,29 +704,29 @@ class BadSpider(scrapy.Spider): def test_runspider(self): log = self.get_log(self.debug_log_spider) - self.assertIn("DEBUG: It Works!", log) - self.assertIn("INFO: Spider opened", log) - self.assertIn("INFO: Closing spider (finished)", log) - self.assertIn("INFO: Spider closed (finished)", log) + assert "DEBUG: It Works!" in log + assert "INFO: Spider opened" in log + assert "INFO: Closing spider (finished)" in log + assert "INFO: Spider closed (finished)" in log def test_run_fail_spider(self): proc, _, _ = self.runspider( "import scrapy\n" + inspect.getsource(ExceptionSpider) ) ret = proc.returncode - self.assertNotEqual(ret, 0) + assert ret != 0 def test_run_good_spider(self): proc, _, _ = self.runspider( "import scrapy\n" + inspect.getsource(NoRequestsSpider) ) ret = proc.returncode - self.assertEqual(ret, 0) + assert ret == 0 def test_runspider_log_level(self): log = self.get_log(self.debug_log_spider, args=("-s", "LOG_LEVEL=INFO")) - self.assertNotIn("DEBUG: It Works!", log) - self.assertIn("INFO: Spider opened", log) + assert "DEBUG: It Works!" not in log + assert "INFO: Spider opened" in log def test_runspider_dnscache_disabled(self): # see https://github.com/scrapy/scrapy/issues/2811 @@ -761,36 +745,36 @@ class MySpider(scrapy.Spider): return {'test': 'value'} """ log = self.get_log(dnscache_spider, args=("-s", "DNSCACHE_ENABLED=False")) - self.assertNotIn("DNSLookupError", log) - self.assertIn("INFO: Spider opened", log) + assert "DNSLookupError" not in log + assert "INFO: Spider opened" in log def test_runspider_log_short_names(self): log1 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=1")) - self.assertIn("[myspider] DEBUG: It Works!", log1) - self.assertIn("[scrapy]", log1) - self.assertNotIn("[scrapy.core.engine]", log1) + assert "[myspider] DEBUG: It Works!" in log1 + assert "[scrapy]" in log1 + assert "[scrapy.core.engine]" not in log1 log2 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=0")) - self.assertIn("[myspider] DEBUG: It Works!", log2) - self.assertNotIn("[scrapy]", log2) - self.assertIn("[scrapy.core.engine]", log2) + assert "[myspider] DEBUG: It Works!" in log2 + assert "[scrapy]" not in log2 + assert "[scrapy.core.engine]" in log2 def test_runspider_no_spider_found(self): log = self.get_log("from scrapy.spiders import Spider\n") - self.assertIn("No spider found in file", log) + assert "No spider found in file" in log def test_runspider_file_not_found(self): _, _, log = self.proc("runspider", "some_non_existent_file") - self.assertIn("File not found: some_non_existent_file", log) + assert "File not found: some_non_existent_file" in log def test_runspider_unable_to_load(self): log = self.get_log("", name="myspider.txt") - self.assertIn("Unable to load", log) + assert "Unable to load" in log def test_start_requests_errors(self): log = self.get_log(self.badspider, name="badspider.py") - self.assertIn("start_requests", log) - self.assertIn("badspider.py", log) + assert "start_requests" in log + assert "badspider.py" in log def test_asyncio_enabled_true(self): log = self.get_log( @@ -800,14 +784,16 @@ class MySpider(scrapy.Spider): "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", ], ) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) def test_asyncio_enabled_default(self): log = self.get_log(self.debug_log_spider, args=[]) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) def test_asyncio_enabled_false(self): @@ -815,11 +801,10 @@ class MySpider(scrapy.Spider): self.debug_log_spider, args=["-s", "TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor"], ) - self.assertIn( - "Using reactor: twisted.internet.selectreactor.SelectReactor", log - ) - self.assertNotIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log ) @pytest.mark.requires_uvloop @@ -833,7 +818,7 @@ class MySpider(scrapy.Spider): "ASYNCIO_EVENT_LOOP=uvloop.Loop", ], ) - self.assertIn("Using asyncio event loop: uvloop.Loop", log) + assert "Using asyncio event loop: uvloop.Loop" in log def test_custom_asyncio_loop_enabled_false(self): log = self.get_log( @@ -849,9 +834,9 @@ class MySpider(scrapy.Spider): loop = asyncio.new_event_loop() else: loop = asyncio.SelectorEventLoop() - self.assertIn( - f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", - log, + assert ( + f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}" + in log ) def test_output(self): @@ -867,9 +852,7 @@ class MySpider(scrapy.Spider): """ args = ["-o", "example.json"] log = self.get_log(spider_code, args=args) - self.assertIn( - "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log - ) + assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log def test_overwrite_output(self): spider_code = """ @@ -890,13 +873,13 @@ class MySpider(scrapy.Spider): Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ["-O", "example.json"] log = self.get_log(spider_code, args=args) - self.assertIn( - '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', - log, + assert ( + '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' + in log ) with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: first_line = f2.readline() - self.assertNotEqual(first_line, "not empty") + assert first_line != "not empty" def test_output_and_overwrite_output(self): spider_code = """ @@ -910,8 +893,8 @@ class MySpider(scrapy.Spider): """ args = ["-o", "example1.json", "-O", "example2.json"] log = self.get_log(spider_code, args=args) - self.assertIn( - "error: Please use only one of -o/--output and -O/--overwrite-output", log + assert ( + "error: Please use only one of -o/--output and -O/--overwrite-output" in log ) def test_output_stdout(self): @@ -927,7 +910,7 @@ class MySpider(scrapy.Spider): """ args = ["-o", "-:json"] log = self.get_log(spider_code, args=args) - self.assertIn("[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}", log) + assert "[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}" in log @skipIf(platform.system() == "Windows", reason="Linux only") def test_absolute_path_linux(self): @@ -946,16 +929,16 @@ class MySpider(scrapy.Spider): args = ["-o", f"{temp_dir}/output1.json:json"] log = self.get_log(spider_code, args=args) - self.assertIn( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json", - log, + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json" + in log ) args = ["-o", f"{temp_dir}/output2.json"] log = self.get_log(spider_code, args=args) - self.assertIn( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json", - log, + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json" + in log ) @skipIf(platform.system() != "Windows", reason="Windows only") @@ -975,16 +958,16 @@ class MySpider(scrapy.Spider): args = ["-o", f"{temp_dir}\\output1.json:json"] log = self.get_log(spider_code, args=args) - self.assertIn( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json", - log, + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json" + in log ) args = ["-o", f"{temp_dir}\\output2.json"] log = self.get_log(spider_code, args=args) - self.assertIn( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json", - log, + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json" + in log ) def test_args_change_settings(self): @@ -1006,11 +989,11 @@ class MySpider(scrapy.Spider): """ args = ["-a", "foo=42"] log = self.get_log(spider_code, args=args) - self.assertIn("Spider closed (finished)", log) - self.assertIn("The value of FOO is 42", log) + assert "Spider closed (finished)" in log + assert "The value of FOO is 42" in log -class WindowsRunSpiderCommandTest(RunSpiderCommandTest): +class TestWindowsRunSpiderCommand(TestRunSpiderCommand): spider_filename = "myspider.pyw" def setUp(self): @@ -1020,24 +1003,24 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): def test_start_requests_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") - self.assertIn("start_requests", log) - self.assertIn("badspider.pyw", log) + assert "start_requests" in log + assert "badspider.pyw" in log def test_runspider_unable_to_load(self): raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") -class BenchCommandTest(CommandTest): +class TestBenchCommand(TestCommandBase): def test_run(self): _, _, log = self.proc( "bench", "-s", "LOGSTATS_INTERVAL=0.001", "-s", "CLOSESPIDER_TIMEOUT=0.01" ) - self.assertIn("INFO: Crawled", log) - self.assertNotIn("Unhandled Error", log) - self.assertNotIn("log_count/ERROR", log) + assert "INFO: Crawled" in log + assert "Unhandled Error" not in log + assert "log_count/ERROR" not in log -class ViewCommandTest(CommandTest): +class TestViewCommand(TestCommandBase): def test_methods(self): command = view.Command() command.settings = Settings() @@ -1048,13 +1031,11 @@ class ViewCommandTest(CommandTest): conflict_handler="resolve", ) command.add_options(parser) - self.assertEqual(command.short_desc(), "Open URL in browser, as seen by Scrapy") - self.assertIn( - "URL using the Scrapy downloader and show its", command.long_desc() - ) + assert command.short_desc() == "Open URL in browser, as seen by Scrapy" + assert "URL using the Scrapy downloader and show its" in command.long_desc() -class CrawlCommandTest(CommandTest): +class TestCrawlCommand(TestCommandBase): def crawl(self, code, args=()): Path(self.proj_mod_path, "spiders", "myspider.py").write_text( code, encoding="utf-8" @@ -1077,7 +1058,7 @@ class MySpider(scrapy.Spider): return [] """ log = self.get_log(spider_code) - self.assertIn("[myspider] DEBUG: It works!", log) + assert "[myspider] DEBUG: It works!" in log def test_output(self): spider_code = """ @@ -1092,9 +1073,7 @@ class MySpider(scrapy.Spider): """ args = ["-o", "example.json"] log = self.get_log(spider_code, args=args) - self.assertIn( - "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log - ) + assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log def test_overwrite_output(self): spider_code = """ @@ -1115,13 +1094,13 @@ class MySpider(scrapy.Spider): Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ["-O", "example.json"] log = self.get_log(spider_code, args=args) - self.assertIn( - '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', - log, + assert ( + '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' + in log ) with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: first_line = f2.readline() - self.assertNotEqual(first_line, "not empty") + assert first_line != "not empty" def test_output_and_overwrite_output(self): spider_code = """ @@ -1135,12 +1114,12 @@ class MySpider(scrapy.Spider): """ args = ["-o", "example1.json", "-O", "example2.json"] log = self.get_log(spider_code, args=args) - self.assertIn( - "error: Please use only one of -o/--output and -O/--overwrite-output", log + assert ( + "error: Please use only one of -o/--output and -O/--overwrite-output" in log ) -class HelpMessageTest(CommandTest): +class TestHelpMessage(TestCommandBase): def setUp(self): super().setUp() self.commands = [ @@ -1163,30 +1142,30 @@ class HelpMessageTest(CommandTest): def test_help_messages(self): for command in self.commands: _, out, _ = self.proc(command, "-h") - self.assertIn("Usage", out) + assert "Usage" in out -class PopCommandNameTest(unittest.TestCase): +class TestPopCommandName: def test_valid_command(self): argv = ["scrapy", "crawl", "my_spider"] command = _pop_command_name(argv) - self.assertEqual(command, "crawl") - self.assertEqual(argv, ["scrapy", "my_spider"]) + assert command == "crawl" + assert argv == ["scrapy", "my_spider"] def test_no_command(self): argv = ["scrapy"] command = _pop_command_name(argv) - self.assertIsNone(command) - self.assertEqual(argv, ["scrapy"]) + assert command is None + assert argv == ["scrapy"] def test_option_before_command(self): argv = ["scrapy", "-h", "crawl"] command = _pop_command_name(argv) - self.assertEqual(command, "crawl") - self.assertEqual(argv, ["scrapy", "-h"]) + assert command == "crawl" + assert argv == ["scrapy", "-h"] def test_option_after_command(self): argv = ["scrapy", "crawl", "-h"] command = _pop_command_name(argv) - self.assertEqual(command, "crawl") - self.assertEqual(argv, ["scrapy", "-h"]) + assert command == "crawl" + assert argv == ["scrapy", "-h"] diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 0f7d7b54c..fb961ace2 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,5 +1,6 @@ from unittest import TextTestResult +import pytest from twisted.internet import defer from twisted.python import failure from twisted.trial import unittest @@ -246,7 +247,7 @@ class InheritsDemoSpider(DemoSpider): name = "inherits_demo_spider" -class ContractsManagerTest(unittest.TestCase): +class TestContractsManager(unittest.TestCase): contracts = [ UrlContract, CallbackKeywordArgumentsContract, @@ -263,34 +264,33 @@ class ContractsManagerTest(unittest.TestCase): self.results = TextTestResult(stream=None, descriptions=False, verbosity=0) def should_succeed(self): - self.assertFalse(self.results.failures) - self.assertFalse(self.results.errors) + assert not self.results.failures + assert not self.results.errors def should_fail(self): - self.assertTrue(self.results.failures) - self.assertFalse(self.results.errors) + assert self.results.failures + assert not self.results.errors def should_error(self): - self.assertTrue(self.results.errors) + assert self.results.errors def test_contracts(self): spider = DemoSpider() # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request) - self.assertEqual(len(contracts), 2) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, ReturnsContract]), + assert len(contracts) == 2 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, ReturnsContract] ) # returns request for valid method request = self.conman.from_method(spider.returns_request, self.results) - self.assertNotEqual(request, None) + assert request is not None # no request for missing url request = self.conman.from_method(spider.parse_no_url, self.results) - self.assertEqual(request, None) + assert request is None def test_cb_kwargs(self): spider = DemoSpider() @@ -298,35 +298,31 @@ class ContractsManagerTest(unittest.TestCase): # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request_cb_kwargs) - self.assertEqual(len(contracts), 3) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]), + assert len(contracts) == 3 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, CallbackKeywordArgumentsContract, ReturnsContract] ) contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs) - self.assertEqual(len(contracts), 3) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]), + assert len(contracts) == 3 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, CallbackKeywordArgumentsContract, ReturnsContract] ) contracts = self.conman.extract_contracts( spider.returns_item_cb_kwargs_error_unexpected_keyword ) - self.assertEqual(len(contracts), 3) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]), + assert len(contracts) == 3 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, CallbackKeywordArgumentsContract, ReturnsContract] ) contracts = self.conman.extract_contracts( spider.returns_item_cb_kwargs_error_missing_argument ) - self.assertEqual(len(contracts), 2) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, ReturnsContract]), + assert len(contracts) == 2 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, ReturnsContract] ) # returns_request @@ -360,17 +356,15 @@ class ContractsManagerTest(unittest.TestCase): # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request_meta) - self.assertEqual(len(contracts), 3) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, MetadataContract, ReturnsContract]), + assert len(contracts) == 3 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, MetadataContract, ReturnsContract] ) contracts = self.conman.extract_contracts(spider.returns_item_meta) - self.assertEqual(len(contracts), 3) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, MetadataContract, ReturnsContract]), + assert len(contracts) == 3 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, MetadataContract, ReturnsContract] ) response = ResponseMetaMock() @@ -505,8 +499,8 @@ class ContractsManagerTest(unittest.TestCase): request = self.conman.from_method(spider.returns_request, self.results) request.errback(failure_mock) - self.assertFalse(self.results.failures) - self.assertTrue(self.results.errors) + assert not self.results.failures + assert self.results.errors @defer.inlineCallbacks def test_same_url(self): @@ -537,19 +531,19 @@ class ContractsManagerTest(unittest.TestCase): crawler = get_crawler(TestSameUrlSpider) yield crawler.crawl() - self.assertEqual(crawler.spider.visited, 2) + assert crawler.spider.visited == 2 def test_form_contract(self): spider = DemoSpider() request = self.conman.from_method(spider.custom_form, self.results) - self.assertEqual(request.method, "POST") - self.assertIsInstance(request, FormRequest) + assert request.method == "POST" + assert isinstance(request, FormRequest) def test_inherited_contracts(self): spider = InheritsDemoSpider() requests = self.conman.from_spider(spider, self.results) - self.assertTrue(requests) + assert requests class CustomFailContractPreProcess(Contract): @@ -566,8 +560,8 @@ class CustomFailContractPostProcess(Contract): raise KeyboardInterrupt("Post-process exception") -class CustomContractPrePostProcess(unittest.TestCase): - def setUp(self): +class TestCustomContractPrePostProcess: + def setup_method(self): self.results = TextTestResult(stream=None, descriptions=False, verbosity=0) def test_pre_hook_keyboard_interrupt(self): @@ -576,18 +570,13 @@ class CustomContractPrePostProcess(unittest.TestCase): contract = CustomFailContractPreProcess(spider.returns_request) conman = ContractsManager([contract]) - try: - request = conman.from_method(spider.returns_request, self.results) - contract.add_pre_hook(request, self.results) - # Expect this to raise a KeyboardInterrupt + request = conman.from_method(spider.returns_request, self.results) + contract.add_pre_hook(request, self.results) + with pytest.raises(KeyboardInterrupt, match="Pre-process exception"): request.callback(response, **request.cb_kwargs) - except KeyboardInterrupt as e: - self.assertEqual(str(e), "Pre-process exception") - else: - self.fail("KeyboardInterrupt not raised") - self.assertFalse(self.results.failures) - self.assertFalse(self.results.errors) + assert not self.results.failures + assert not self.results.errors def test_post_hook_keyboard_interrupt(self): spider = DemoSpider() @@ -595,15 +584,10 @@ class CustomContractPrePostProcess(unittest.TestCase): contract = CustomFailContractPostProcess(spider.returns_request) conman = ContractsManager([contract]) - try: - request = conman.from_method(spider.returns_request, self.results) - contract.add_post_hook(request, self.results) - # Expect this to raise a KeyboardInterrupt + request = conman.from_method(spider.returns_request, self.results) + contract.add_post_hook(request, self.results) + with pytest.raises(KeyboardInterrupt, match="Post-process exception"): request.callback(response, **request.cb_kwargs) - except KeyboardInterrupt as e: - self.assertEqual(str(e), "Post-process exception") - else: - self.fail("KeyboardInterrupt not raised") - self.assertFalse(self.results.failures) - self.assertFalse(self.results.errors) + assert not self.results.failures + assert not self.results.errors diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index dffba303f..1bffd69ed 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -30,16 +30,13 @@ from scrapy.utils.test import get_crawler from tests.mockserver import PayloadResource, ssl_context_factory -class SlotTest(unittest.TestCase): +class TestSlot: def test_repr(self): slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) - self.assertEqual( - repr(slot), - "Slot(concurrency=8, delay=0.10, randomize_delay=True)", - ) + assert repr(slot) == "Slot(concurrency=8, delay=0.10, randomize_delay=True)" -class ContextFactoryBaseTestCase(unittest.TestCase): +class TestContextFactoryBase(unittest.TestCase): context_factory = None def _listen(self, site): @@ -90,7 +87,7 @@ class ContextFactoryBaseTestCase(unittest.TestCase): return await maybe_deferred_to_future(d) -class ContextFactoryTestCase(ContextFactoryBaseTestCase): +class TestContextFactory(TestContextFactoryBase): @deferred_f_from_coro_f async def testPayload(self): s = "0123456789" * 10 @@ -100,7 +97,7 @@ class ContextFactoryTestCase(ContextFactoryBaseTestCase): body = await self.get_page( self.getURL("payload"), client_context_factory, body=s ) - self.assertEqual(body, to_bytes(s)) + assert body == to_bytes(s) def test_override_getContext(self): class MyFactory(ScrapyClientContextFactory): @@ -112,14 +109,14 @@ class ContextFactoryTestCase(ContextFactoryBaseTestCase): with warnings.catch_warnings(record=True) as w: MyFactory() - self.assertEqual(len(w), 1) - self.assertIn( - "Overriding ScrapyClientContextFactory.getContext() is deprecated", - str(w[0].message), + assert len(w) == 1 + assert ( + "Overriding ScrapyClientContextFactory.getContext() is deprecated" + in str(w[0].message) ) -class ContextFactoryTLSMethodTestCase(ContextFactoryBaseTestCase): +class TestContextFactoryTLSMethod(TestContextFactoryBase): async def _assert_factory_works( self, client_context_factory: ScrapyClientContextFactory ) -> None: @@ -127,7 +124,7 @@ class ContextFactoryTLSMethodTestCase(ContextFactoryBaseTestCase): body = await self.get_page( self.getURL("payload"), client_context_factory, body=s ) - self.assertEqual(body, to_bytes(s)) + assert body == to_bytes(s) @deferred_f_from_coro_f async def test_setting_default(self): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 3aca2bbce..5766f9313 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -55,7 +55,7 @@ from tests.spiders import ( ) -class CrawlTestCase(TestCase): +class TestCrawl(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -69,7 +69,7 @@ class CrawlTestCase(TestCase): def test_follow_all(self): crawler = get_crawler(FollowAllSpider) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url + assert len(crawler.spider.urls_visited) == 11 # 10 + start_url @defer.inlineCallbacks def test_fixed_delay(self): @@ -94,9 +94,7 @@ class CrawlTestCase(TestCase): times = crawler.spider.times total_time = times[-1] - times[0] average = total_time / (len(times) - 1) - self.assertTrue( - average > delay * tolerance, f"download delay too small: {average}" - ) + assert average > delay * tolerance, f"download delay too small: {average}" # Ensure that the same test parameters would cause a failure if no # download delay is set. Otherwise, it means we are using a combination @@ -108,34 +106,32 @@ class CrawlTestCase(TestCase): times = crawler.spider.times total_time = times[-1] - times[0] average = total_time / (len(times) - 1) - self.assertFalse( - average > delay / tolerance, "test total or delay values are too small" - ) + assert average <= delay / tolerance, "test total or delay values are too small" @defer.inlineCallbacks def test_timeout_success(self): crawler = get_crawler(DelaySpider) yield crawler.crawl(n=0.5, mockserver=self.mockserver) - self.assertTrue(crawler.spider.t1 > 0) - self.assertTrue(crawler.spider.t2 > 0) - self.assertTrue(crawler.spider.t2 > crawler.spider.t1) + assert crawler.spider.t1 > 0 + assert crawler.spider.t2 > 0 + assert crawler.spider.t2 > crawler.spider.t1 @defer.inlineCallbacks def test_timeout_failure(self): crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) yield crawler.crawl(n=0.5, mockserver=self.mockserver) - self.assertTrue(crawler.spider.t1 > 0) - self.assertTrue(crawler.spider.t2 == 0) - self.assertTrue(crawler.spider.t2_err > 0) - self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) + assert crawler.spider.t1 > 0 + assert crawler.spider.t2 == 0 + assert crawler.spider.t2_err > 0 + assert crawler.spider.t2_err > crawler.spider.t1 # server hangs after receiving response headers crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver) - self.assertTrue(crawler.spider.t1 > 0) - self.assertTrue(crawler.spider.t2 == 0) - self.assertTrue(crawler.spider.t2_err > 0) - self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) + assert crawler.spider.t1 > 0 + assert crawler.spider.t2 == 0 + assert crawler.spider.t2_err > 0 + assert crawler.spider.t2_err > crawler.spider.t1 @defer.inlineCallbacks def test_retry_503(self): @@ -173,10 +169,10 @@ class CrawlTestCase(TestCase): crawler = get_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) - self.assertEqual(len(log.records), 1) + assert len(log.records) == 1 record = log.records[0] - self.assertIsNotNone(record.exc_info) - self.assertIs(record.exc_info[0], ZeroDivisionError) + assert record.exc_info is not None + assert record.exc_info[0] is ZeroDivisionError @defer.inlineCallbacks def test_start_requests_bug_yielding(self): @@ -184,10 +180,10 @@ class CrawlTestCase(TestCase): crawler = get_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) - self.assertEqual(len(log.records), 1) + assert len(log.records) == 1 record = log.records[0] - self.assertIsNotNone(record.exc_info) - self.assertIs(record.exc_info[0], ZeroDivisionError) + assert record.exc_info is not None + assert record.exc_info[0] is ZeroDivisionError @defer.inlineCallbacks def test_start_requests_items(self): @@ -195,7 +191,7 @@ class CrawlTestCase(TestCase): crawler = get_crawler(StartRequestsItemSpider) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(log.records), 0) + assert len(log.records) == 0 @defer.inlineCallbacks def test_start_requests_unsupported_output(self): @@ -203,23 +199,18 @@ class CrawlTestCase(TestCase): crawler = get_crawler(StartRequestsGoodAndBadOutput) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(log.records), 2) - self.assertEqual( - log.records[0].msg, - ( - "Got 'data:,b' among start requests. Only requests and items " - "are supported. It will be ignored." - ), + assert len(log.records) == 2 + assert log.records[0].msg == ( + "Got 'data:,b' among start requests. Only requests and items " + "are supported. It will be ignored." ) - self.assertTrue( - re.match( - ( - r"^Got among start " - r"requests\. Only requests and items are supported\. It " - r"will be ignored\.$" - ), - log.records[1].msg, - ) + assert re.match( + ( + r"^Got among start " + r"requests\. Only requests and items are supported\. It " + r"will be ignored\.$" + ), + log.records[1].msg, ) @defer.inlineCallbacks @@ -227,10 +218,9 @@ class CrawlTestCase(TestCase): settings = {"CONCURRENT_REQUESTS": 1} crawler = get_crawler(BrokenStartRequestsSpider, settings) yield crawler.crawl(mockserver=self.mockserver) - self.assertTrue( - crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99), - crawler.spider.seedsseen, - ) + assert crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index( + 99 + ), crawler.spider.seedsseen @defer.inlineCallbacks def test_start_requests_dupes(self): @@ -239,7 +229,7 @@ class CrawlTestCase(TestCase): yield crawler.crawl( dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver ) - self.assertEqual(crawler.spider.visited, 6) + assert crawler.spider.visited == 6 crawler = get_crawler(DuplicateStartRequestsSpider, settings) yield crawler.crawl( @@ -248,7 +238,7 @@ class CrawlTestCase(TestCase): dupe_factor=4, mockserver=self.mockserver, ) - self.assertEqual(crawler.spider.visited, 3) + assert crawler.spider.visited == 3 @defer.inlineCallbacks def test_unbounded_response(self): @@ -282,7 +272,7 @@ with multiples lines yield crawler.crawl( self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver ) - self.assertEqual(str(log).count("Got response 200"), 1) + assert str(log).count("Got response 200") == 1 @defer.inlineCallbacks def test_retry_conn_lost(self): @@ -305,8 +295,8 @@ with multiples lines self._assert_retried(log) def _assert_retried(self, log): - self.assertEqual(str(log).count("Retrying"), 2) - self.assertEqual(str(log).count("Gave up retrying"), 1) + assert str(log).count("Retrying") == 2 + assert str(log).count("Gave up retrying") == 1 @defer.inlineCallbacks def test_referer_header(self): @@ -321,20 +311,20 @@ with multiples lines crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=req0, mockserver=self.mockserver) # basic asserts in case of weird communication errors - self.assertIn("responses", crawler.spider.meta) - self.assertNotIn("failures", crawler.spider.meta) + assert "responses" in crawler.spider.meta + assert "failures" not in crawler.spider.meta # start requests doesn't set Referer header echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) - self.assertNotIn("Referer", echo0["headers"]) + assert "Referer" not in echo0["headers"] # following request sets Referer to start request url echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body)) - self.assertEqual(echo1["headers"].get("Referer"), [req0.url]) + assert echo1["headers"].get("Referer") == [req0.url] # next request avoids Referer header echo2 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) - self.assertNotIn("Referer", echo2["headers"]) + assert "Referer" not in echo2["headers"] # last request explicitly sets a Referer header echo3 = json.loads(to_unicode(crawler.spider.meta["responses"][3].body)) - self.assertEqual(echo3["headers"].get("Referer"), ["http://example.com"]) + assert echo3["headers"].get("Referer") == ["http://example.com"] @defer.inlineCallbacks def test_engine_status(self): @@ -349,10 +339,10 @@ with multiples lines yield crawler.crawl( seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver ) - self.assertEqual(len(est), 1, est) + assert len(est) == 1, est s = dict(est[0]) - self.assertEqual(s["engine.spider.name"], crawler.spider.name) - self.assertEqual(s["len(engine.scraper.slot.active)"], 1) + assert s["engine.spider.name"] == crawler.spider.name + assert s["len(engine.scraper.slot.active)"] == 1 @defer.inlineCallbacks def test_format_engine_status(self): @@ -367,7 +357,7 @@ with multiples lines yield crawler.crawl( seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver ) - self.assertEqual(len(est), 1, est) + assert len(est) == 1, est est = est[0].split("\n")[2:-2] # remove header & footer # convert to dict est = [x.split(":") for x in est] @@ -376,8 +366,8 @@ with multiples lines it = iter(est) s = dict(zip(it, it)) - self.assertEqual(s["engine.spider.name"], crawler.spider.name) - self.assertEqual(s["len(engine.scraper.slot.active)"], "1") + assert s["engine.spider.name"] == crawler.spider.name + assert s["len(engine.scraper.slot.active)"] == "1" @defer.inlineCallbacks def test_graceful_crawl_error_handling(self): @@ -398,7 +388,7 @@ with multiples lines crawler = get_crawler(FaultySpider) yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError) - self.assertFalse(crawler.crawling) + assert not crawler.crawling @defer.inlineCallbacks def test_open_spider_error_on_faulty_pipeline(self): @@ -414,7 +404,7 @@ with multiples lines ), ZeroDivisionError, ) - self.assertFalse(crawler.crawling) + assert not crawler.crawling @defer.inlineCallbacks def test_crawlerrunner_accepts_crawler(self): @@ -426,7 +416,7 @@ with multiples lines self.mockserver.url("/status?n=200"), mockserver=self.mockserver, ) - self.assertIn("Got response 200", str(log)) + assert "Got response 200" in str(log) @defer.inlineCallbacks def test_crawl_multiple(self): @@ -446,10 +436,10 @@ with multiples lines yield runner.join() self._assert_retried(log) - self.assertIn("Got response 200", str(log)) + assert "Got response 200" in str(log) -class CrawlSpiderTestCase(TestCase): +class TestCrawlSpider(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -480,9 +470,9 @@ class CrawlSpiderTestCase(TestCase): with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertIn("[parse] status 200 (foo: None)", str(log)) - self.assertIn("[parse] status 201 (foo: None)", str(log)) - self.assertIn("[parse] status 202 (foo: bar)", str(log)) + assert "[parse] status 200 (foo: None)" in str(log) + assert "[parse] status 201 (foo: None)" in str(log) + assert "[parse] status 202 (foo: bar)" in str(log) @defer.inlineCallbacks def test_crawlspider_with_async_callback(self): @@ -490,9 +480,9 @@ class CrawlSpiderTestCase(TestCase): with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertIn("[parse_async] status 200 (foo: None)", str(log)) - self.assertIn("[parse_async] status 201 (foo: None)", str(log)) - self.assertIn("[parse_async] status 202 (foo: bar)", str(log)) + assert "[parse_async] status 200 (foo: None)" in str(log) + assert "[parse_async] status 201 (foo: None)" in str(log) + assert "[parse_async] status 202 (foo: bar)" in str(log) @defer.inlineCallbacks def test_crawlspider_with_async_generator_callback(self): @@ -500,9 +490,9 @@ class CrawlSpiderTestCase(TestCase): with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertIn("[parse_async_gen] status 200 (foo: None)", str(log)) - self.assertIn("[parse_async_gen] status 201 (foo: None)", str(log)) - self.assertIn("[parse_async_gen] status 202 (foo: bar)", str(log)) + assert "[parse_async_gen] status 200 (foo: None)" in str(log) + assert "[parse_async_gen] status 201 (foo: None)" in str(log) + assert "[parse_async_gen] status 202 (foo: bar)" in str(log) @defer.inlineCallbacks def test_crawlspider_with_errback(self): @@ -510,12 +500,12 @@ class CrawlSpiderTestCase(TestCase): with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertIn("[parse] status 200 (foo: None)", str(log)) - self.assertIn("[parse] status 201 (foo: None)", str(log)) - self.assertIn("[parse] status 202 (foo: bar)", str(log)) - self.assertIn("[errback] status 404", str(log)) - self.assertIn("[errback] status 500", str(log)) - self.assertIn("[errback] status 501", str(log)) + assert "[parse] status 200 (foo: None)" in str(log) + assert "[parse] status 201 (foo: None)" in str(log) + assert "[parse] status 202 (foo: bar)" in str(log) + assert "[errback] status 404" in str(log) + assert "[errback] status 500" in str(log) + assert "[errback] status 501" in str(log) @defer.inlineCallbacks def test_crawlspider_process_request_cb_kwargs(self): @@ -523,9 +513,9 @@ class CrawlSpiderTestCase(TestCase): with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertIn("[parse] status 200 (foo: process_request)", str(log)) - self.assertIn("[parse] status 201 (foo: process_request)", str(log)) - self.assertIn("[parse] status 202 (foo: bar)", str(log)) + assert "[parse] status 200 (foo: process_request)" in str(log) + assert "[parse] status 201 (foo: process_request)" in str(log) + assert "[parse] status 202 (foo: bar)" in str(log) @defer.inlineCallbacks def test_async_def_parse(self): @@ -534,7 +524,7 @@ class CrawlSpiderTestCase(TestCase): yield crawler.crawl( self.mockserver.url("/status?n=200"), mockserver=self.mockserver ) - self.assertIn("Got response 200", str(log)) + assert "Got response 200" in str(log) @pytest.mark.only_asyncio @defer.inlineCallbacks @@ -549,15 +539,15 @@ class CrawlSpiderTestCase(TestCase): yield crawler.crawl( self.mockserver.url("/status?n=200"), mockserver=self.mockserver ) - self.assertIn("Got response 200", str(log)) + assert "Got response 200" in str(log) @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncio_parse_items_list(self): log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider) - self.assertIn("Got response 200", str(log)) - self.assertIn({"id": 1}, items) - self.assertIn({"id": 2}, items) + assert "Got response 200" in str(log) + assert {"id": 1} in items + assert {"id": 2} in items @pytest.mark.only_asyncio @defer.inlineCallbacks @@ -573,81 +563,81 @@ class CrawlSpiderTestCase(TestCase): yield crawler.crawl( self.mockserver.url("/status?n=200"), mockserver=self.mockserver ) - self.assertIn("Got response 200", str(log)) - self.assertIn({"foo": 42}, items) + assert "Got response 200" in str(log) + assert {"foo": 42} in items @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse(self): log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider) - self.assertIn("Got response 200", str(log)) + assert "Got response 200" in str(log) itemcount = stats.get_value("item_scraped_count") - self.assertEqual(itemcount, 1) + assert itemcount == 1 @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse_loop(self): log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider) - self.assertIn("Got response 200", str(log)) + assert "Got response 200" in str(log) itemcount = stats.get_value("item_scraped_count") - self.assertEqual(itemcount, 10) + assert itemcount == 10 for i in range(10): - self.assertIn({"foo": i}, items) + assert {"foo": i} in items @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse_exc(self): log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider) log = str(log) - self.assertIn("Spider error processing", log) - self.assertIn("ValueError", log) + assert "Spider error processing" in log + assert "ValueError" in log itemcount = stats.get_value("item_scraped_count") - self.assertEqual(itemcount, 7) + assert itemcount == 7 for i in range(7): - self.assertIn({"foo": i}, items) + assert {"foo": i} in items @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse_complex(self): _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider) itemcount = stats.get_value("item_scraped_count") - self.assertEqual(itemcount, 156) + assert itemcount == 156 # some random items for i in [1, 4, 21, 22, 207, 311]: - self.assertIn({"index": i}, items) + assert {"index": i} in items for i in [10, 30, 122]: - self.assertIn({"index2": i}, items) + assert {"index2": i} in items @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncio_parse_reqs_list(self): log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider) for req_id in range(3): - self.assertIn(f"Got response 200, req_id {req_id}", str(log)) + assert f"Got response 200, req_id {req_id}" in str(log) @pytest.mark.only_not_asyncio @defer.inlineCallbacks def test_async_def_deferred_direct(self): _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider) - self.assertEqual(items, [{"code": 200}]) + assert items == [{"code": 200}] @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_deferred_wrapped(self): log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider) - self.assertEqual(items, [{"code": 200}]) + assert items == [{"code": 200}] @defer.inlineCallbacks def test_async_def_deferred_maybe_wrapped(self): _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider) - self.assertEqual(items, [{"code": 200}]) + assert items == [{"code": 200}] @defer.inlineCallbacks def test_response_ssl_certificate_none(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test", is_secure=False) yield crawler.crawl(seed=url, mockserver=self.mockserver) - self.assertIsNone(crawler.spider.meta["responses"][0].certificate) + assert crawler.spider.meta["responses"][0].certificate is None @defer.inlineCallbacks def test_response_ssl_certificate(self): @@ -655,9 +645,9 @@ class CrawlSpiderTestCase(TestCase): url = self.mockserver.url("/echo?body=test", is_secure=True) yield crawler.crawl(seed=url, mockserver=self.mockserver) cert = crawler.spider.meta["responses"][0].certificate - self.assertIsInstance(cert, Certificate) - self.assertEqual(cert.getSubject().commonName, b"localhost") - self.assertEqual(cert.getIssuer().commonName, b"localhost") + assert isinstance(cert, Certificate) + assert cert.getSubject().commonName == b"localhost" + assert cert.getIssuer().commonName == b"localhost" @pytest.mark.xfail( reason="Responses with no body return early and contain no certificate" @@ -668,9 +658,9 @@ class CrawlSpiderTestCase(TestCase): url = self.mockserver.url("/status?n=200", is_secure=True) yield crawler.crawl(seed=url, mockserver=self.mockserver) cert = crawler.spider.meta["responses"][0].certificate - self.assertIsInstance(cert, Certificate) - self.assertEqual(cert.getSubject().commonName, b"localhost") - self.assertEqual(cert.getIssuer().commonName, b"localhost") + assert isinstance(cert, Certificate) + assert cert.getSubject().commonName == b"localhost" + assert cert.getIssuer().commonName == b"localhost" @defer.inlineCallbacks def test_dns_server_ip_address_none(self): @@ -678,7 +668,7 @@ class CrawlSpiderTestCase(TestCase): url = self.mockserver.url("/status?n=200") yield crawler.crawl(seed=url, mockserver=self.mockserver) ip_address = crawler.spider.meta["responses"][0].ip_address - self.assertIsNone(ip_address) + assert ip_address is None @defer.inlineCallbacks def test_dns_server_ip_address(self): @@ -687,61 +677,57 @@ class CrawlSpiderTestCase(TestCase): expected_netloc, _ = urlparse(url).netloc.split(":") yield crawler.crawl(seed=url, mockserver=self.mockserver) ip_address = crawler.spider.meta["responses"][0].ip_address - self.assertIsInstance(ip_address, IPv4Address) - self.assertEqual(str(ip_address), gethostbyname(expected_netloc)) + assert isinstance(ip_address, IPv4Address) + assert str(ip_address) == gethostbyname(expected_netloc) @defer.inlineCallbacks def test_bytes_received_stop_download_callback(self): crawler = get_crawler(BytesReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) - self.assertIsNone(crawler.spider.meta.get("failure")) - self.assertIsInstance(crawler.spider.meta["response"], Response) - self.assertEqual( - crawler.spider.meta["response"].body, - crawler.spider.meta.get("bytes_received"), + assert crawler.spider.meta.get("failure") is None + assert isinstance(crawler.spider.meta["response"], Response) + assert crawler.spider.meta["response"].body == crawler.spider.meta.get( + "bytes_received" ) - self.assertLess( - len(crawler.spider.meta["response"].body), - crawler.spider.full_response_length, + assert ( + len(crawler.spider.meta["response"].body) + < crawler.spider.full_response_length ) @defer.inlineCallbacks def test_bytes_received_stop_download_errback(self): crawler = get_crawler(BytesReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) - self.assertIsNone(crawler.spider.meta.get("response")) - self.assertIsInstance(crawler.spider.meta["failure"], Failure) - self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload) - self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response) - self.assertEqual( - crawler.spider.meta["failure"].value.response.body, - crawler.spider.meta.get("bytes_received"), - ) - self.assertLess( - len(crawler.spider.meta["failure"].value.response.body), - crawler.spider.full_response_length, + assert crawler.spider.meta.get("response") is None + assert isinstance(crawler.spider.meta["failure"], Failure) + assert isinstance(crawler.spider.meta["failure"].value, StopDownload) + assert isinstance(crawler.spider.meta["failure"].value.response, Response) + assert crawler.spider.meta[ + "failure" + ].value.response.body == crawler.spider.meta.get("bytes_received") + assert ( + len(crawler.spider.meta["failure"].value.response.body) + < crawler.spider.full_response_length ) @defer.inlineCallbacks def test_headers_received_stop_download_callback(self): crawler = get_crawler(HeadersReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) - self.assertIsNone(crawler.spider.meta.get("failure")) - self.assertIsInstance(crawler.spider.meta["response"], Response) - self.assertEqual( - crawler.spider.meta["response"].headers, - crawler.spider.meta.get("headers_received"), + assert crawler.spider.meta.get("failure") is None + assert isinstance(crawler.spider.meta["response"], Response) + assert crawler.spider.meta["response"].headers == crawler.spider.meta.get( + "headers_received" ) @defer.inlineCallbacks def test_headers_received_stop_download_errback(self): crawler = get_crawler(HeadersReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) - self.assertIsNone(crawler.spider.meta.get("response")) - self.assertIsInstance(crawler.spider.meta["failure"], Failure) - self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload) - self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response) - self.assertEqual( - crawler.spider.meta["failure"].value.response.headers, - crawler.spider.meta.get("headers_received"), - ) + assert crawler.spider.meta.get("response") is None + assert isinstance(crawler.spider.meta["failure"], Failure) + assert isinstance(crawler.spider.meta["failure"].value, StopDownload) + assert isinstance(crawler.spider.meta["failure"].value.response, Response) + assert crawler.spider.meta[ + "failure" + ].value.response.headers == crawler.spider.meta.get("headers_received") diff --git a/tests/test_crawler.py b/tests/test_crawler.py index df5ebfa7b..0bbcc0843 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -39,13 +39,13 @@ def get_raw_crawler(spidercls=None, settings_dict=None): return Crawler(spidercls or DefaultSpider, settings) -class BaseCrawlerTest(unittest.TestCase): +class TestBaseCrawler(unittest.TestCase): def assertOptionIsDefault(self, settings, key): - self.assertIsInstance(settings, Settings) - self.assertEqual(settings[key], getattr(default_settings, key)) + assert isinstance(settings, Settings) + assert settings[key] == getattr(default_settings, key) -class CrawlerTestCase(BaseCrawlerTest): +class TestCrawler(TestBaseCrawler): def test_populate_spidercls_settings(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} project_settings = {**BASE_SETTINGS, "TEST1": "project", "TEST3": "project"} @@ -58,16 +58,16 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = Crawler(CustomSettingsSpider, settings) crawler._apply_settings() - self.assertEqual(crawler.settings.get("TEST1"), "spider") - self.assertEqual(crawler.settings.get("TEST2"), "spider") - self.assertEqual(crawler.settings.get("TEST3"), "project") + assert crawler.settings.get("TEST1") == "spider" + assert crawler.settings.get("TEST2") == "spider" + assert crawler.settings.get("TEST3") == "project" - self.assertFalse(settings.frozen) - self.assertTrue(crawler.settings.frozen) + assert not settings.frozen + assert crawler.settings.frozen def test_crawler_accepts_dict(self): crawler = get_crawler(DefaultSpider, {"foo": "bar"}) - self.assertEqual(crawler.settings["foo"], "bar") + assert crawler.settings["foo"] == "bar" self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") def test_crawler_accepts_None(self): @@ -107,23 +107,23 @@ class CrawlerTestCase(BaseCrawlerTest): }, } crawler = get_crawler(settings_dict=settings) - self.assertEqual(len(TrackingAddon.instances), 1) + assert len(TrackingAddon.instances) == 1 expected = TrackingAddon.instances[-1] addon = crawler.get_addon(TrackingAddon) - self.assertEqual(addon, expected) + assert addon == expected addon = crawler.get_addon(DefaultSpider) - self.assertIsNone(addon) + assert addon is None addon = crawler.get_addon(ParentAddon) - self.assertEqual(addon, expected) + assert addon == expected class ChildAddon(TrackingAddon): pass addon = crawler.get_addon(ChildAddon) - self.assertIsNone(addon) + assert addon is None @inlineCallbacks def test_get_downloader_middleware(self): @@ -162,18 +162,18 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingDownloaderMiddleware yield crawler.crawl() - self.assertEqual(len(TrackingDownloaderMiddleware.instances), 1) - self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1]) + assert len(TrackingDownloaderMiddleware.instances) == 1 + assert MySpider.result == TrackingDownloaderMiddleware.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider yield crawler.crawl() - self.assertIsNone(MySpider.result) + assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentDownloaderMiddleware yield crawler.crawl() - self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1]) + assert MySpider.result == TrackingDownloaderMiddleware.instances[-1] class ChildDownloaderMiddleware(TrackingDownloaderMiddleware): pass @@ -181,7 +181,7 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildDownloaderMiddleware yield crawler.crawl() - self.assertIsNone(MySpider.result) + assert MySpider.result is None def test_get_downloader_middleware_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) @@ -242,18 +242,18 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingExtension yield crawler.crawl() - self.assertEqual(len(TrackingExtension.instances), 1) - self.assertEqual(MySpider.result, TrackingExtension.instances[-1]) + assert len(TrackingExtension.instances) == 1 + assert MySpider.result == TrackingExtension.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider yield crawler.crawl() - self.assertIsNone(MySpider.result) + assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentExtension yield crawler.crawl() - self.assertEqual(MySpider.result, TrackingExtension.instances[-1]) + assert MySpider.result == TrackingExtension.instances[-1] class ChildExtension(TrackingExtension): pass @@ -261,7 +261,7 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildExtension yield crawler.crawl() - self.assertIsNone(MySpider.result) + assert MySpider.result is None def test_get_extension_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) @@ -322,18 +322,18 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingItemPipeline yield crawler.crawl() - self.assertEqual(len(TrackingItemPipeline.instances), 1) - self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1]) + assert len(TrackingItemPipeline.instances) == 1 + assert MySpider.result == TrackingItemPipeline.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider yield crawler.crawl() - self.assertIsNone(MySpider.result) + assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentItemPipeline yield crawler.crawl() - self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1]) + assert MySpider.result == TrackingItemPipeline.instances[-1] class ChildItemPipeline(TrackingItemPipeline): pass @@ -341,7 +341,7 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildItemPipeline yield crawler.crawl() - self.assertIsNone(MySpider.result) + assert MySpider.result is None def test_get_item_pipeline_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) @@ -402,18 +402,18 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingSpiderMiddleware yield crawler.crawl() - self.assertEqual(len(TrackingSpiderMiddleware.instances), 1) - self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1]) + assert len(TrackingSpiderMiddleware.instances) == 1 + assert MySpider.result == TrackingSpiderMiddleware.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider yield crawler.crawl() - self.assertIsNone(MySpider.result) + assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentSpiderMiddleware yield crawler.crawl() - self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1]) + assert MySpider.result == TrackingSpiderMiddleware.instances[-1] class ChildSpiderMiddleware(TrackingSpiderMiddleware): pass @@ -421,7 +421,7 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildSpiderMiddleware yield crawler.crawl() - self.assertIsNone(MySpider.result) + assert MySpider.result is None def test_get_spider_middleware_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) @@ -446,7 +446,7 @@ class CrawlerTestCase(BaseCrawlerTest): yield crawler.crawl() -class SpiderSettingsTestCase(unittest.TestCase): +class TestSpiderSettings: def test_spider_custom_settings(self): class MySpider(scrapy.Spider): name = "spider" @@ -454,10 +454,10 @@ class SpiderSettingsTestCase(unittest.TestCase): crawler = get_crawler(MySpider) enabled_exts = [e.__class__ for e in crawler.extensions.middlewares] - self.assertIn(AutoThrottle, enabled_exts) + assert AutoThrottle in enabled_exts -class CrawlerLoggingTestCase(unittest.TestCase): +class TestCrawlerLogging: def test_no_root_handler_installed(self): handler = get_scrapy_root_handler() if handler is not None: @@ -469,8 +469,8 @@ class CrawlerLoggingTestCase(unittest.TestCase): get_crawler(MySpider) assert get_scrapy_root_handler() is None - def test_spider_custom_settings_log_level(self): - log_file = Path(self.mktemp()) + def test_spider_custom_settings_log_level(self, tmp_path): + log_file = Path(tmp_path, "log.txt") log_file.write_text("previous message\n", encoding="utf-8") class MySpider(scrapy.Spider): @@ -481,9 +481,9 @@ class CrawlerLoggingTestCase(unittest.TestCase): } configure_logging() - self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) + assert get_scrapy_root_handler().level == logging.DEBUG crawler = get_crawler(MySpider) - self.assertEqual(get_scrapy_root_handler().level, logging.INFO) + assert get_scrapy_root_handler().level == logging.INFO info_count = crawler.stats.get_value("log_count/INFO") logging.debug("debug message") logging.info("info message") @@ -492,18 +492,18 @@ class CrawlerLoggingTestCase(unittest.TestCase): logged = log_file.read_text(encoding="utf-8") - self.assertIn("previous message", logged) - self.assertNotIn("debug message", logged) - self.assertIn("info message", logged) - self.assertIn("warning message", logged) - self.assertIn("error message", logged) - self.assertEqual(crawler.stats.get_value("log_count/ERROR"), 1) - self.assertEqual(crawler.stats.get_value("log_count/WARNING"), 1) - self.assertEqual(crawler.stats.get_value("log_count/INFO") - info_count, 1) - self.assertEqual(crawler.stats.get_value("log_count/DEBUG", 0), 0) + assert "previous message" in logged + assert "debug message" not in logged + assert "info message" in logged + assert "warning message" in logged + assert "error message" in logged + assert crawler.stats.get_value("log_count/ERROR") == 1 + assert crawler.stats.get_value("log_count/WARNING") == 1 + assert crawler.stats.get_value("log_count/INFO") - info_count == 1 + assert crawler.stats.get_value("log_count/DEBUG", 0) == 0 - def test_spider_custom_settings_log_append(self): - log_file = Path(self.mktemp()) + def test_spider_custom_settings_log_append(self, tmp_path): + log_file = Path(tmp_path, "log.txt") log_file.write_text("previous message\n", encoding="utf-8") class MySpider(scrapy.Spider): @@ -519,8 +519,8 @@ class CrawlerLoggingTestCase(unittest.TestCase): logged = log_file.read_text(encoding="utf-8") - self.assertNotIn("previous message", logged) - self.assertIn("debug message", logged) + assert "previous message" not in logged + assert "debug message" in logged class SpiderLoaderWithWrongInterface: @@ -532,7 +532,7 @@ class CustomSpiderLoader(SpiderLoader): pass -class CrawlerRunnerTestCase(BaseCrawlerTest): +class TestCrawlerRunner(TestBaseCrawler): def test_spider_manager_verify_interface(self): settings = Settings( { @@ -544,7 +544,7 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): def test_crawler_runner_accepts_dict(self): runner = CrawlerRunner({"foo": "bar"}) - self.assertEqual(runner.settings["foo"], "bar") + assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") def test_crawler_runner_accepts_None(self): @@ -552,10 +552,10 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") -class CrawlerProcessTest(BaseCrawlerTest): +class TestCrawlerProcess(TestBaseCrawler): def test_crawler_process_accepts_dict(self): runner = CrawlerProcess({"foo": "bar"}) - self.assertEqual(runner.settings["foo"], "bar") + assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") def test_crawler_process_accepts_None(self): @@ -579,7 +579,7 @@ class NoRequestsSpider(scrapy.Spider): @pytest.mark.usefixtures("reactor_pytest") -class CrawlerRunnerHasSpider(unittest.TestCase): +class TestCrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): return CrawlerRunner() @@ -587,14 +587,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase): def test_crawler_runner_bootstrap_successful(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) - self.assertFalse(runner.bootstrap_failed) + assert not runner.bootstrap_failed @inlineCallbacks def test_crawler_runner_bootstrap_successful_for_several(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) yield runner.crawl(NoRequestsSpider) - self.assertFalse(runner.bootstrap_failed) + assert not runner.bootstrap_failed @inlineCallbacks def test_crawler_runner_bootstrap_failed(self): @@ -605,9 +605,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase): except ValueError: pass else: - self.fail("Exception should be raised from spider") + pytest.fail("Exception should be raised from spider") - self.assertTrue(runner.bootstrap_failed) + assert runner.bootstrap_failed @inlineCallbacks def test_crawler_runner_bootstrap_failed_for_several(self): @@ -618,11 +618,11 @@ class CrawlerRunnerHasSpider(unittest.TestCase): except ValueError: pass else: - self.fail("Exception should be raised from spider") + pytest.fail("Exception should be raised from spider") yield runner.crawl(NoRequestsSpider) - self.assertTrue(runner.bootstrap_failed) + assert runner.bootstrap_failed @inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): @@ -664,31 +664,34 @@ class ScriptRunnerMixin: return stderr.decode("utf-8") -class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): +class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): script_dir = Path(__file__).parent.resolve() / "CrawlerProcess" def test_simple(self): log = self.run_script("simple.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log ) def test_multi(self): log = self.run_script("multi.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log ) - self.assertNotIn("ReactorAlreadyInstalledError", log) + assert "ReactorAlreadyInstalledError" not in log def test_reactor_default(self): log = self.run_script("reactor_default.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log ) - self.assertNotIn("ReactorAlreadyInstalledError", log) + assert "ReactorAlreadyInstalledError" not in log def test_reactor_default_twisted_reactor_select(self): log = self.run_script("reactor_default_twisted_reactor_select.py") @@ -703,50 +706,46 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): # If that ever becomes the case on more platforms (i.e. if Linux # also starts using the select reactor by default in a future # version of Twisted), then we will need to rethink this test. - self.assertIn("Spider closed (finished)", log) + assert "Spider closed (finished)" in log else: - self.assertNotIn("Spider closed (finished)", log) - self.assertIn( - ( - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ), - log, - ) + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log def test_reactor_select(self): log = self.run_script("reactor_select.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn("ReactorAlreadyInstalledError", log) + assert "Spider closed (finished)" in log + assert "ReactorAlreadyInstalledError" not in log def test_reactor_select_twisted_reactor_select(self): log = self.run_script("reactor_select_twisted_reactor_select.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn("ReactorAlreadyInstalledError", log) + assert "Spider closed (finished)" in log + assert "ReactorAlreadyInstalledError" not in log def test_reactor_select_subclass_twisted_reactor_select(self): log = self.run_script("reactor_select_subclass_twisted_reactor_select.py") - self.assertNotIn("Spider closed (finished)", log) - self.assertIn( - ( - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ), - log, - ) + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log def test_asyncio_enabled_no_reactor(self): log = self.run_script("asyncio_enabled_no_reactor.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) def test_asyncio_enabled_reactor(self): log = self.run_script("asyncio_enabled_reactor.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) @pytest.mark.skipif( @@ -755,129 +754,129 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) def test_ipv6_default_name_resolver(self): log = self.run_script("default_name_resolver.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", - log, + assert "Spider closed (finished)" in log + assert ( + "'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1," + in log ) - self.assertIn( - "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", - log, + assert ( + "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1." + in log ) def test_caching_hostname_resolver_ipv6(self): log = self.run_script("caching_hostname_resolver_ipv6.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn("twisted.internet.error.DNSLookupError", log) + assert "Spider closed (finished)" in log + assert "twisted.internet.error.DNSLookupError" not in log def test_caching_hostname_resolver_finite_execution(self): with MockServer() as mock_server: http_address = mock_server.http_address.replace("0.0.0.0", "127.0.0.1") log = self.run_script("caching_hostname_resolver.py", http_address) - self.assertIn("Spider closed (finished)", log) - self.assertNotIn("ERROR: Error downloading", log) - self.assertNotIn("TimeoutError", log) - self.assertNotIn("twisted.internet.error.DNSLookupError", log) + assert "Spider closed (finished)" in log + assert "ERROR: Error downloading" not in log + assert "TimeoutError" not in log + assert "twisted.internet.error.DNSLookupError" not in log def test_twisted_reactor_select(self): log = self.run_script("twisted_reactor_select.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.selectreactor.SelectReactor", log - ) + assert "Spider closed (finished)" in log + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log @pytest.mark.skipif( platform.system() == "Windows", reason="PollReactor is not supported on Windows" ) def test_twisted_reactor_poll(self): log = self.run_script("twisted_reactor_poll.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) + assert "Spider closed (finished)" in log + assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log def test_twisted_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) def test_twisted_reactor_asyncio_custom_settings(self): log = self.run_script("twisted_reactor_custom_settings.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) def test_twisted_reactor_asyncio_custom_settings_same(self): log = self.run_script("twisted_reactor_custom_settings_same.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) def test_twisted_reactor_asyncio_custom_settings_conflict(self): log = self.run_script("twisted_reactor_custom_settings_conflict.py") - self.assertIn( - "Using reactor: twisted.internet.selectreactor.SelectReactor", log - ) - self.assertIn( - "(twisted.internet.selectreactor.SelectReactor) does not match the requested one", - log, + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + assert ( + "(twisted.internet.selectreactor.SelectReactor) does not match the requested one" + in log ) @pytest.mark.requires_uvloop def test_custom_loop_asyncio(self): log = self.run_script("asyncio_custom_loop.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) - self.assertIn("Using asyncio event loop: uvloop.Loop", log) + assert "Using asyncio event loop: uvloop.Loop" in log @pytest.mark.requires_uvloop def test_custom_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) - self.assertIn("Using asyncio event loop: uvloop.Loop", log) - self.assertIn("async pipeline opened!", log) + assert "Using asyncio event loop: uvloop.Loop" in log + assert "async pipeline opened!" in log @pytest.mark.requires_uvloop def test_asyncio_enabled_reactor_same_loop(self): log = self.run_script("asyncio_enabled_reactor_same_loop.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) - self.assertIn("Using asyncio event loop: uvloop.Loop", log) + assert "Using asyncio event loop: uvloop.Loop" in log @pytest.mark.requires_uvloop def test_asyncio_enabled_reactor_different_loop(self): log = self.run_script("asyncio_enabled_reactor_different_loop.py") - self.assertNotIn("Spider closed (finished)", log) - self.assertIn( - ( - "does not match the one specified in the ASYNCIO_EVENT_LOOP " - "setting (uvloop.Loop)" - ), - log, - ) + assert "Spider closed (finished)" not in log + assert ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ) in log def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) - self.assertNotIn("Using asyncio event loop: uvloop.Loop", log) - self.assertIn("async pipeline opened!", log) + assert "Using asyncio event loop: uvloop.Loop" not in log + assert "async pipeline opened!" in log def test_args_change_settings(self): log = self.run_script("args_settings.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn("The value of FOO is 42", log) + assert "Spider closed (finished)" in log + assert "The value of FOO is 42" in log def test_shutdown_graceful(self): sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK @@ -910,23 +909,23 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.wait() -class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): +class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): script_dir = Path(__file__).parent.resolve() / "CrawlerRunner" def test_response_ip_address(self): log = self.run_script("ip_address.py") - self.assertIn("INFO: Spider closed (finished)", log) - self.assertIn("INFO: Host: not.a.real.domain", log) - self.assertIn("INFO: Type: ", log) - self.assertIn("INFO: IP address: 127.0.0.1", log) + assert "INFO: Spider closed (finished)" in log + assert "INFO: Host: not.a.real.domain" in log + assert "INFO: Type: " in log + assert "INFO: IP address: 127.0.0.1" in log def test_change_default_reactor(self): log = self.run_script("change_reactor.py") - self.assertIn( - "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", - log, + assert ( + "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) - self.assertIn("DEBUG: Using asyncio event loop", log) + assert "DEBUG: Using asyncio event loop" in log @pytest.mark.parametrize( diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index a39ed0694..162747581 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -4,11 +4,12 @@ from configparser import ConfigParser from importlib import import_module from pathlib import Path +import pytest from twisted import version as twisted_version from twisted.trial import unittest -class ScrapyUtilsTest(unittest.TestCase): +class TestScrapyUtils: def test_required_openssl_version(self): try: module = import_module("OpenSSL") @@ -27,7 +28,7 @@ class ScrapyUtilsTest(unittest.TestCase): See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011 """ if not os.environ.get("_SCRAPY_PINNED", None): - self.skipTest("Not in a pinned environment") + pytest.skip("Not in a pinned environment") tox_config_file_path = Path(__file__).parent / ".." / "tox.ini" config_parser = ConfigParser() @@ -36,4 +37,4 @@ class ScrapyUtilsTest(unittest.TestCase): match = re.search(pattern, config_parser["pinned"]["deps"]) pinned_twisted_version_string = match[1] - self.assertEqual(twisted_version.short(), pinned_twisted_version_string) + assert twisted_version.short() == pinned_twisted_version_string diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 4f8b005d7..15b3ad5af 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -76,7 +76,7 @@ class CrawlTestCase(TestCase): for k, v in slots.items() } - self.assertTrue(max(list(error_delta.values())) < tolerance) + assert max(list(error_delta.values())) < tolerance def test_params(): diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 703c23529..d5e1b37f7 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -2,7 +2,6 @@ import hashlib import shutil import sys import tempfile -import unittest from pathlib import Path from warnings import catch_warnings @@ -39,7 +38,7 @@ class DirectDupeFilter: method = "n/a" -class RFPDupeFilterTest(unittest.TestCase): +class TestRFPDupeFilter: def test_df_from_crawler_scheduler(self): settings = { "DUPEFILTER_DEBUG": True, @@ -47,8 +46,8 @@ class RFPDupeFilterTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) - self.assertTrue(scheduler.df.debug) - self.assertEqual(scheduler.df.method, "from_crawler") + assert scheduler.df.debug + assert scheduler.df.method == "from_crawler" def test_df_direct_scheduler(self): settings = { @@ -56,7 +55,7 @@ class RFPDupeFilterTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) - self.assertEqual(scheduler.df.method, "n/a") + assert scheduler.df.method == "n/a" def test_filter(self): dupefilter = _get_dupefilter() @@ -256,16 +255,16 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.close("finished") -class BaseDupeFilterTestCase(unittest.TestCase): +class TestBaseDupeFilter: def test_log_deprecation(self): dupefilter = _get_dupefilter( settings={"DUPEFILTER_CLASS": BaseDupeFilter}, ) with catch_warnings(record=True) as warning_list: dupefilter.log(None, None) - self.assertEqual(len(warning_list), 1) - self.assertEqual( - str(warning_list[0].message), - "Calling BaseDupeFilter.log() is deprecated.", + assert len(warning_list) == 1 + assert ( + str(warning_list[0].message) + == "Calling BaseDupeFilter.log() is deprecated." ) - self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + assert warning_list[0].category == ScrapyDeprecationWarning diff --git a/tests/test_engine.py b/tests/test_engine.py index e9470493f..4bac8d273 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -243,8 +243,9 @@ class CrawlerRun: self.signals_caught[sig] = signalargs -class EngineTestBase(unittest.TestCase): - def _assert_visited_urls(self, run: CrawlerRun): +class TestEngineBase(unittest.TestCase): + @staticmethod + def _assert_visited_urls(run: CrawlerRun) -> None: must_be_visited = [ "/", "/redirect", @@ -259,8 +260,9 @@ class EngineTestBase(unittest.TestCase): f"URLs not visited: {list(urls_expected - urls_visited)}" ) - def _assert_scheduled_requests(self, run: CrawlerRun, count=None): - self.assertEqual(count, len(run.reqplug)) + @staticmethod + def _assert_scheduled_requests(run: CrawlerRun, count: int) -> None: + assert len(run.reqplug) == count paths_expected = ["/item999.html", "/item2.html", "/item1.html"] @@ -270,101 +272,104 @@ class EngineTestBase(unittest.TestCase): scheduled_requests_count = len(run.reqplug) dropped_requests_count = len(run.reqdropped) responses_count = len(run.respplug) - self.assertEqual( - scheduled_requests_count, dropped_requests_count + responses_count - ) - self.assertEqual(len(run.reqreached), responses_count) + assert scheduled_requests_count == dropped_requests_count + responses_count + assert len(run.reqreached) == responses_count - def _assert_dropped_requests(self, run: CrawlerRun): - self.assertEqual(len(run.reqdropped), 1) + @staticmethod + def _assert_dropped_requests(run: CrawlerRun) -> None: + assert len(run.reqdropped) == 1 - def _assert_downloaded_responses(self, run: CrawlerRun, count): + @staticmethod + def _assert_downloaded_responses(run: CrawlerRun, count: int) -> None: # response tests - self.assertEqual(count, len(run.respplug)) - self.assertEqual(count, len(run.reqreached)) + assert len(run.respplug) == count + assert len(run.reqreached) == count for response, _ in run.respplug: if run.getpath(response.url) == "/item999.html": - self.assertEqual(404, response.status) + assert response.status == 404 if run.getpath(response.url) == "/redirect": - self.assertEqual(302, response.status) + assert response.status == 302 - def _assert_items_error(self, run: CrawlerRun): - self.assertEqual(2, len(run.itemerror)) + @staticmethod + def _assert_items_error(run: CrawlerRun) -> None: + assert len(run.itemerror) == 2 for item, response, spider, failure in run.itemerror: - self.assertEqual(failure.value.__class__, ZeroDivisionError) - self.assertEqual(spider, run.spider) + assert failure.value.__class__ is ZeroDivisionError + assert spider == run.spider - self.assertEqual(item["url"], response.url) + assert item["url"] == response.url if "item1.html" in item["url"]: - self.assertEqual("Item 1 name", item["name"]) - self.assertEqual("100", item["price"]) + assert item["name"] == "Item 1 name" + assert item["price"] == "100" if "item2.html" in item["url"]: - self.assertEqual("Item 2 name", item["name"]) - self.assertEqual("200", item["price"]) + assert item["name"] == "Item 2 name" + assert item["price"] == "200" - def _assert_scraped_items(self, run: CrawlerRun): - self.assertEqual(2, len(run.itemresp)) + @staticmethod + def _assert_scraped_items(run: CrawlerRun) -> None: + assert len(run.itemresp) == 2 for item, response in run.itemresp: item = ItemAdapter(item) - self.assertEqual(item["url"], response.url) + assert item["url"] == response.url if "item1.html" in item["url"]: - self.assertEqual("Item 1 name", item["name"]) - self.assertEqual("100", item["price"]) + assert item["name"] == "Item 1 name" + assert item["price"] == "100" if "item2.html" in item["url"]: - self.assertEqual("Item 2 name", item["name"]) - self.assertEqual("200", item["price"]) + assert item["name"] == "Item 2 name" + assert item["price"] == "200" - def _assert_headers_received(self, run: CrawlerRun): + @staticmethod + def _assert_headers_received(run: CrawlerRun) -> None: for headers in run.headers.values(): - self.assertIn(b"Server", headers) - self.assertIn(b"TwistedWeb", headers[b"Server"]) - self.assertIn(b"Date", headers) - self.assertIn(b"Content-Type", headers) + assert b"Server" in headers + assert b"TwistedWeb" in headers[b"Server"] + assert b"Date" in headers + assert b"Content-Type" in headers - def _assert_bytes_received(self, run: CrawlerRun): - self.assertEqual(9, len(run.bytes)) + @staticmethod + def _assert_bytes_received(run: CrawlerRun) -> None: + assert len(run.bytes) == 9 for request, data in run.bytes.items(): joined_data = b"".join(data) if run.getpath(request.url) == "/": - self.assertEqual(joined_data, get_testdata("test_site", "index.html")) + assert joined_data == get_testdata("test_site", "index.html") elif run.getpath(request.url) == "/item1.html": - self.assertEqual(joined_data, get_testdata("test_site", "item1.html")) + assert joined_data == get_testdata("test_site", "item1.html") elif run.getpath(request.url) == "/item2.html": - self.assertEqual(joined_data, get_testdata("test_site", "item2.html")) + assert joined_data == get_testdata("test_site", "item2.html") elif run.getpath(request.url) == "/redirected": - self.assertEqual(joined_data, b"Redirected here") + assert joined_data == b"Redirected here" elif run.getpath(request.url) == "/redirect": - self.assertEqual( - joined_data, - b"\n\n" + assert ( + joined_data == b"\n\n" b" \n" b' \n' b" \n" b' \n' b' click here\n' b" \n" - b"\n", + b"\n" ) elif run.getpath(request.url) == "/tem999.html": - self.assertEqual( - joined_data, - b"\n\n" + assert ( + joined_data == b"\n\n" b" 404 - No Such Resource\n" b" \n" b"

No Such Resource

\n" b"

File not found.

\n" b" \n" - b"\n", + b"\n" ) elif run.getpath(request.url) == "/numbers": # signal was fired multiple times - self.assertTrue(len(data) > 1) + assert len(data) > 1 # bytes were received in order numbers = [str(x).encode("utf8") for x in range(2**18)] - self.assertEqual(joined_data, b"".join(numbers)) + assert joined_data == b"".join(numbers) - def _assert_signals_caught(self, run: CrawlerRun): + @staticmethod + def _assert_signals_caught(run: CrawlerRun) -> None: assert signals.engine_started in run.signals_caught assert signals.engine_stopped in run.signals_caught assert signals.spider_opened in run.signals_caught @@ -372,19 +377,14 @@ class EngineTestBase(unittest.TestCase): assert signals.spider_closed in run.signals_caught assert signals.headers_received in run.signals_caught - self.assertEqual( - {"spider": run.spider}, run.signals_caught[signals.spider_opened] - ) - self.assertEqual( - {"spider": run.spider}, run.signals_caught[signals.spider_idle] - ) - self.assertEqual( - {"spider": run.spider, "reason": "finished"}, - run.signals_caught[signals.spider_closed], - ) + assert {"spider": run.spider} == run.signals_caught[signals.spider_opened] + assert {"spider": run.spider} == run.signals_caught[signals.spider_idle] + assert {"spider": run.spider, "reason": "finished"} == run.signals_caught[ + signals.spider_closed + ] -class EngineTest(EngineTestBase): +class TestEngine(TestEngineBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( @@ -419,10 +419,9 @@ class EngineTest(EngineTestBase): def test_crawler_change_close_reason_on_idle(self): run = CrawlerRun(ChangeCloseReasonSpider) yield run.run() - self.assertEqual( - {"spider": run.spider, "reason": "custom_reason"}, - run.signals_caught[signals.spider_closed], - ) + assert {"spider": run.spider, "reason": "custom_reason"} == run.signals_caught[ + signals.spider_closed + ] @defer.inlineCallbacks def test_close_downloader(self): @@ -470,7 +469,7 @@ class EngineTest(EngineTestBase): finally: timer.cancel() - self.assertNotIn(b"Traceback", stderr) + assert b"Traceback" not in stderr def test_request_scheduled_signal(caplog): diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index 5dd04c310..f09b0e091 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -7,8 +7,8 @@ from tests.test_engine import ( CrawlerRun, DataClassItemsSpider, DictItemsSpider, - EngineTestBase, MySpider, + TestEngineBase, ) @@ -18,7 +18,7 @@ class BytesReceivedCrawlerRun(CrawlerRun): raise StopDownload(fail=False) -class BytesReceivedEngineTest(EngineTestBase): +class TestBytesReceivedEngine(TestEngineBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( @@ -61,14 +61,15 @@ class BytesReceivedEngineTest(EngineTestBase): self._assert_headers_received(run) self._assert_bytes_received(run) - def _assert_bytes_received(self, run: CrawlerRun): - self.assertEqual(9, len(run.bytes)) + @staticmethod + def _assert_bytes_received(run: CrawlerRun) -> None: + assert len(run.bytes) == 9 for request, data in run.bytes.items(): joined_data = b"".join(data) - self.assertTrue(len(data) == 1) # signal was fired only once + assert len(data) == 1 # signal was fired only once if run.getpath(request.url) == "/numbers": # Received bytes are not the complete response. The exact amount depends # on the buffer size, which can vary, so we only check that the amount # of received bytes is strictly less than the full response. numbers = [str(x).encode("utf8") for x in range(2**18)] - self.assertTrue(len(joined_data) < len(b"".join(numbers))) + assert len(joined_data) < len(b"".join(numbers)) diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index 06929d1e4..dbb0ea0d2 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -7,8 +7,8 @@ from tests.test_engine import ( CrawlerRun, DataClassItemsSpider, DictItemsSpider, - EngineTestBase, MySpider, + TestEngineBase, ) @@ -18,7 +18,7 @@ class HeadersReceivedCrawlerRun(CrawlerRun): raise StopDownload(fail=False) -class HeadersReceivedEngineTest(EngineTestBase): +class TestHeadersReceivedEngine(TestEngineBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( @@ -60,10 +60,12 @@ class HeadersReceivedEngineTest(EngineTestBase): self._assert_bytes_received(run) self._assert_headers_received(run) - def _assert_bytes_received(self, run: CrawlerRun): - self.assertEqual(0, len(run.bytes)) + @staticmethod + def _assert_bytes_received(run: CrawlerRun) -> None: + assert len(run.bytes) == 0 - def _assert_visited_urls(self, run: CrawlerRun): + @staticmethod + def _assert_visited_urls(run: CrawlerRun) -> None: must_be_visited = ["/", "/redirect", "/redirected"] urls_visited = {rp[0].url for rp in run.respplug} urls_expected = {run.geturl(p) for p in must_be_visited} diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 1b4ad2f2f..c3c03d6c3 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -36,7 +36,7 @@ from tests.mockserver import ( PayloadResource, ssl_context_factory, ) -from tests.test_core_downloader import ContextFactoryBaseTestCase +from tests.test_core_downloader import TestContextFactoryBase def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): @@ -63,7 +63,7 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class ScrapyHTTPPageGetterTests(unittest.TestCase): +class TestScrapyHTTPPageGetter: def test_earlyHeaders(self): # basic test stolen from twisted HTTPageGetter factory = client.ScrapyHTTPClientFactory( @@ -177,9 +177,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): protocol = client.ScrapyHTTPPageGetter() protocol.factory = factory protocol.makeConnection(transport) - self.assertEqual( - set(transport.value().splitlines()), set(testvalue.splitlines()) - ) + assert set(transport.value().splitlines()) == set(testvalue.splitlines()) return testvalue def test_non_standard_line_endings(self): @@ -192,9 +190,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): protocol.dataReceived(b"Hello: World\n") protocol.dataReceived(b"Foo: Bar\n") protocol.dataReceived(b"\n") - self.assertEqual( - protocol.headers, Headers({"Hello": ["World"], "Foo": ["Bar"]}) - ) + assert protocol.headers == Headers({"Hello": ["World"], "Foo": ["Bar"]}) class EncodingResource(resource.Resource): @@ -207,7 +203,7 @@ class EncodingResource(resource.Resource): @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class WebClientTestCase(unittest.TestCase): +class TestWebClient(unittest.TestCase): def _listen(self, site): return reactor.listenTCP(0, site, interface="127.0.0.1") @@ -319,7 +315,7 @@ class WebClientTestCase(unittest.TestCase): return getPage(self.getURL("notsuchfile")).addCallback(self._cbNoSuchFile) def _cbNoSuchFile(self, pageData): - self.assertIn(b"404 - No Such Resource", pageData) + assert b"404 - No Such Resource" in pageData def testFactoryInfo(self): url = self.getURL("file") @@ -329,20 +325,20 @@ class WebClientTestCase(unittest.TestCase): return factory.deferred.addCallback(self._cbFactoryInfo, factory) def _cbFactoryInfo(self, ignoredResult, factory): - self.assertEqual(factory.status, b"200") - self.assertTrue(factory.version.startswith(b"HTTP/")) - self.assertEqual(factory.message, b"OK") - self.assertEqual(factory.response_headers[b"content-length"], b"10") + assert factory.status == b"200" + assert factory.version.startswith(b"HTTP/") + assert factory.message == b"OK" + assert factory.response_headers[b"content-length"] == b"10" def testRedirect(self): return getPage(self.getURL("redirect")).addCallback(self._cbRedirect) def _cbRedirect(self, pageData): - self.assertEqual( - pageData, - b'\n\n \n \n' + assert ( + pageData + == b'\n\n \n \n' b' \n \n ' - b'click here\n \n\n', + b'click here\n \n\n' ) def test_encoding(self): @@ -356,14 +352,12 @@ class WebClientTestCase(unittest.TestCase): def _check_Encoding(self, response, original_body): content_encoding = to_unicode(response.headers[b"Content-Encoding"]) - self.assertEqual(content_encoding, EncodingResource.out_encoding) - self.assertEqual( - response.body.decode(content_encoding), to_unicode(original_body) - ) + assert content_encoding == EncodingResource.out_encoding + assert response.body.decode(content_encoding) == to_unicode(original_body) @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class WebClientSSLTestCase(ContextFactoryBaseTestCase): +class WebClientSSLTestCase(TestContextFactoryBase): def testPayload(self): s = "0123456789" * 10 return getPage(self.getURL("payload"), body=s).addCallback( From d2e5486d5a0ddfa9c202e39f5af98257a230d4f3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Mar 2025 13:20:42 +0500 Subject: [PATCH 064/491] Remove the Splash recommendation. --- docs/topics/dynamic-content.rst | 35 +++++++-------------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 801f6d06d..65270433f 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -14,7 +14,7 @@ from it. If you fail to do that, and you can nonetheless access the desired data through the :ref:`DOM ` from your web browser, see -:ref:`topics-javascript-rendering`. +:ref:`topics-headless-browsing`. .. _topics-finding-data-source: @@ -97,7 +97,7 @@ it `. You can reproduce any request with Scrapy. However, some times reproducing all necessary requests may not seem efficient in developer time. If that is your case, and crawling speed is not a major concern for you, you can alternatively -consider :ref:`JavaScript pre-rendering `. +consider :ref:`using a headless browser `. If you get the expected response `sometimes`, but not always, the issue is probably not your request, but the target server. The target server might be @@ -220,9 +220,9 @@ data from it: >>> selector.css('var[name="data"]').get() 'value' -.. _topics-javascript-rendering: +.. _topics-headless-browsing: -Pre-rendering JavaScript +Using a headless browser ======================== On webpages that fetch data from additional requests, reproducing those @@ -232,29 +232,10 @@ network transfer. However, sometimes it can be really hard to reproduce certain requests. Or you may need something that no request can give you, such as a screenshot of a -webpage as seen in a web browser. +webpage as seen in a web browser. In this case using a `headless browser`_ will +help. -In these cases use the Splash_ JavaScript-rendering service, along with -`scrapy-splash`_ for seamless integration. - -Splash returns as HTML the :ref:`DOM ` of a webpage, so that -you can parse it with :ref:`selectors `. It provides great -flexibility through configuration_ or scripting_. - -If you need something beyond what Splash offers, such as interacting with the -DOM on-the-fly from Python code instead of using a previously-written script, -or handling multiple web browser windows, you might need to -:ref:`use a headless browser ` instead. - -.. _configuration: https://splash.readthedocs.io/en/stable/api.html -.. _scripting: https://splash.readthedocs.io/en/stable/scripting-tutorial.html - -.. _topics-headless-browsing: - -Using a headless browser -======================== - -A `headless browser`_ is a special web browser that provides an API for +A headless browser is a special web browser that provides an API for automation. By installing the :ref:`asyncio reactor `, it is possible to integrate ``asyncio``-based libraries which handle headless browsers. @@ -287,7 +268,6 @@ We recommend using `scrapy-playwright`_ for a better integration. .. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29 .. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets .. _JavaScript: https://en.wikipedia.org/wiki/JavaScript -.. _Splash: https://github.com/scrapinghub/splash .. _chompjs: https://github.com/Nykakin/chompjs .. _curl: https://curl.se/ .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser @@ -297,7 +277,6 @@ We recommend using `scrapy-playwright`_ for a better integration. .. _pyppeteer: https://pyppeteer.github.io/pyppeteer/ .. _pytesseract: https://github.com/madmaze/pytesseract .. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright -.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash .. _tabula-py: https://github.com/chezou/tabula-py .. _wget: https://www.gnu.org/software/wget/ .. _wgrep: https://github.com/stav/wgrep From 3ded1dfe31510f00e14a70811b7c01dae8b5a641 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Mar 2025 20:25:15 +0400 Subject: [PATCH 065/491] Converting tests to plain asserts, part 4. (#6702) --- tests/test_spider.py | 344 +++++++++----------- tests/test_spiderloader/__init__.py | 113 ++++--- tests/test_spidermiddleware.py | 105 +++--- tests/test_spidermiddleware_depth.py | 16 +- tests/test_spidermiddleware_httperror.py | 98 +++--- tests/test_spidermiddleware_offsite.py | 11 +- tests/test_spidermiddleware_output_chain.py | 112 +++---- tests/test_spidermiddleware_referer.py | 31 +- tests/test_spidermiddleware_urllength.py | 12 +- tests/test_spiderstate.py | 7 +- tests/test_squeues.py | 45 +-- tests/test_squeues_request.py | 148 ++++----- tests/test_stats.py | 72 ++-- 13 files changed, 529 insertions(+), 585 deletions(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index af29872a8..05f1c59d0 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -31,7 +31,7 @@ from scrapy.utils.test import get_crawler from tests import get_testdata, tests_datadir -class SpiderTest(unittest.TestCase): +class TestSpider(unittest.TestCase): spider_class = Spider def setUp(self): @@ -42,19 +42,19 @@ class SpiderTest(unittest.TestCase): def test_base_spider(self): spider = self.spider_class("example.com") - self.assertEqual(spider.name, "example.com") - self.assertEqual(spider.start_urls, []) + assert spider.name == "example.com" + assert spider.start_urls == [] # pylint: disable=use-implicit-booleaness-not-comparison def test_start_requests(self): spider = self.spider_class("example.com") start_requests = spider.start_requests() - self.assertTrue(inspect.isgenerator(start_requests)) - self.assertEqual(list(start_requests), []) + assert inspect.isgenerator(start_requests) + assert not list(start_requests) def test_spider_args(self): """``__init__`` method arguments are assigned to spider attributes""" spider = self.spider_class("example.com", foo="bar") - self.assertEqual(spider.foo, "bar") + assert spider.foo == "bar" def test_spider_without_name(self): """``__init__`` method arguments are assigned to spider attributes""" @@ -67,10 +67,10 @@ class SpiderTest(unittest.TestCase): def test_from_crawler_crawler_and_settings_population(self): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertTrue(hasattr(spider, "crawler")) - self.assertIs(spider.crawler, crawler) - self.assertTrue(hasattr(spider, "settings")) - self.assertIs(spider.settings, crawler.settings) + assert hasattr(spider, "crawler") + assert spider.crawler is crawler + assert hasattr(spider, "settings") + assert spider.settings is crawler.settings def test_from_crawler_init_call(self): with mock.patch.object( @@ -92,7 +92,7 @@ class SpiderTest(unittest.TestCase): crawler.signals.send_catch_log( signal=signals.spider_closed, spider=spider, reason=None ) - self.assertTrue(spider.closed_called) + assert spider.closed_called def test_update_settings(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} @@ -101,9 +101,9 @@ class SpiderTest(unittest.TestCase): settings = Settings(project_settings, priority="project") self.spider_class.update_settings(settings) - self.assertEqual(settings.get("TEST1"), "spider") - self.assertEqual(settings.get("TEST2"), "spider") - self.assertEqual(settings.get("TEST3"), "project") + assert settings.get("TEST1") == "spider" + assert settings.get("TEST2") == "spider" + assert settings.get("TEST3") == "project" @inlineCallbacks def test_settings_in_from_crawler(self): @@ -121,11 +121,11 @@ class SpiderTest(unittest.TestCase): return spider crawler = Crawler(TestSpider, project_settings) - self.assertEqual(crawler.settings.get("TEST1"), "spider") - self.assertEqual(crawler.settings.get("TEST2"), "spider") - self.assertEqual(crawler.settings.get("TEST3"), "project") + assert crawler.settings.get("TEST1") == "spider" + assert crawler.settings.get("TEST2") == "spider" + assert crawler.settings.get("TEST3") == "project" yield crawler.crawl() - self.assertEqual(crawler.settings.get("TEST1"), "spider_instance") + assert crawler.settings.get("TEST1") == "spider_instance" def test_logger(self): spider = self.spider_class("example.com") @@ -134,8 +134,8 @@ class SpiderTest(unittest.TestCase): lc.check(("example.com", "INFO", "test log msg")) record = lc.records[0] - self.assertIn("spider", record.__dict__) - self.assertIs(record.spider, spider) + assert "spider" in record.__dict__ + assert record.spider is spider def test_log(self): spider = self.spider_class("example.com") @@ -144,11 +144,11 @@ class SpiderTest(unittest.TestCase): mock_logger.log.assert_called_once_with("INFO", "test log msg") -class InitSpiderTest(SpiderTest): +class TestInitSpider(TestSpider): spider_class = InitSpider -class XMLFeedSpiderTest(SpiderTest): +class TestXMLFeedSpider(TestSpider): spider_class = XMLFeedSpider def test_register_namespace(self): @@ -180,28 +180,24 @@ class XMLFeedSpiderTest(SpiderTest): for iterator in ("iternodes", "xml"): spider = _XMLSpider("example", iterator=iterator) output = list(spider._parse(response)) - self.assertEqual(len(output), 2, iterator) - self.assertEqual( - output, - [ - { - "loc": ["http://www.example.com/Special-Offers.html"], - "updated": ["2009-08-16"], - "custom": ["fuu"], - "other": ["bar"], - }, - { - "loc": [], - "updated": ["2009-08-16"], - "other": ["foo"], - "custom": [], - }, - ], - iterator, - ) + assert len(output) == 2, iterator + assert output == [ + { + "loc": ["http://www.example.com/Special-Offers.html"], + "updated": ["2009-08-16"], + "custom": ["fuu"], + "other": ["bar"], + }, + { + "loc": [], + "updated": ["2009-08-16"], + "other": ["foo"], + "custom": [], + }, + ], iterator -class CSVFeedSpiderTest(SpiderTest): +class TestCSVFeedSpider(TestSpider): spider_class = CSVFeedSpider def test_parse_rows(self): @@ -222,7 +218,7 @@ class CSVFeedSpiderTest(SpiderTest): assert len(rows) == 4 -class CrawlSpiderTest(SpiderTest): +class TestCrawlSpider(TestSpider): test_body = b"""Page title<title> <body> <p><a href="item/12.html">Item 12</a></p> @@ -247,16 +243,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] def test_process_links(self): response = HtmlResponse( @@ -273,16 +266,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] def test_process_links_filter(self): response = HtmlResponse( @@ -302,15 +292,12 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 2) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - ], - ) + assert len(output) == 2 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + ] def test_process_links_generator(self): response = HtmlResponse( @@ -327,16 +314,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] def test_process_request(self): response = HtmlResponse( @@ -355,16 +339,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.com/somepage/item/12.html", - "http://example.com/about.html", - "http://example.com/nofollow.html", - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.com/somepage/item/12.html", + "http://example.com/about.html", + "http://example.com/nofollow.html", + ] def test_process_request_with_response(self): response = HtmlResponse( @@ -386,20 +367,18 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) - self.assertEqual( - [r.meta["response_class"] for r in output], - ["HtmlResponse", "HtmlResponse", "HtmlResponse"], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + assert [r.meta["response_class"] for r in output] == [ + "HtmlResponse", + "HtmlResponse", + "HtmlResponse", + ] def test_process_request_instance_method(self): response = HtmlResponse( @@ -416,16 +395,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"), - safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"), - safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"), - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"), + safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"), + safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"), + ] def test_process_request_instance_method_with_response(self): response = HtmlResponse( @@ -448,32 +424,30 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) - self.assertEqual( - [r.meta["response_class"] for r in output], - ["HtmlResponse", "HtmlResponse", "HtmlResponse"], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + assert [r.meta["response_class"] for r in output] == [ + "HtmlResponse", + "HtmlResponse", + "HtmlResponse", + ] def test_follow_links_attribute_population(self): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertTrue(hasattr(spider, "_follow_links")) - self.assertTrue(spider._follow_links) + assert hasattr(spider, "_follow_links") + assert spider._follow_links settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False} crawler = get_crawler(settings_dict=settings_dict) spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertTrue(hasattr(spider, "_follow_links")) - self.assertFalse(spider._follow_links) + assert hasattr(spider, "_follow_links") + assert not spider._follow_links def test_start_url(self): spider = self.spider_class("example.com") @@ -483,7 +457,7 @@ class CrawlSpiderTest(SpiderTest): list(spider.start_requests()) -class SitemapSpiderTest(SpiderTest): +class TestSitemapSpider(TestSpider): spider_class = SitemapSpider BODY = b"SITEMAP" @@ -496,7 +470,7 @@ class SitemapSpiderTest(SpiderTest): def assertSitemapBody(self, response, body): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertEqual(spider._get_sitemap_body(response), body) + assert spider._get_sitemap_body(response) == body def test_get_sitemap_body(self): r = XmlResponse(url="http://www.example.com/", body=self.BODY) @@ -543,15 +517,12 @@ Sitemap: /sitemap-relative-url.xml r = TextResponse(url="http://www.example.com/robots.txt", body=robots) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://example.com/sitemap.xml", - "http://example.com/sitemap-product-index.xml", - "http://example.com/sitemap-uppercase.xml", - "http://www.example.com/sitemap-relative-url.xml", - ], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://example.com/sitemap.xml", + "http://example.com/sitemap-product-index.xml", + "http://example.com/sitemap-uppercase.xml", + "http://www.example.com/sitemap-relative-url.xml", + ] def test_alternate_url_locs(self): sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -570,21 +541,17 @@ Sitemap: /sitemap-relative-url.xml </urlset>""" r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/english/"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/" + ] spider.sitemap_alternate_links = True - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://www.example.com/english/", - "http://www.example.com/deutsch/", - "http://www.example.com/schweiz-deutsch/", - "http://www.example.com/italiano/", - ], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/", + "http://www.example.com/deutsch/", + "http://www.example.com/schweiz-deutsch/", + "http://www.example.com/italiano/", + ] def test_sitemap_filter(self): sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -611,16 +578,15 @@ Sitemap: /sitemap-relative-url.xml r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/english/", "http://www.example.com/portuguese/"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/", + "http://www.example.com/portuguese/", + ] spider = FilteredSitemapSpider("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/english/"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/" + ] def test_sitemap_filter_with_alternate_links(self): sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -649,19 +615,15 @@ Sitemap: /sitemap-relative-url.xml r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://www.example.com/english/article_1/", - "http://www.example.com/english/article_2/", - ], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/article_1/", + "http://www.example.com/english/article_2/", + ] spider = FilteredSitemapSpider("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/deutsch/article_1/"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/deutsch/article_1/" + ] def test_sitemapindex_filter(self): sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -689,19 +651,15 @@ Sitemap: /sitemap-relative-url.xml r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://www.example.com/sitemap1.xml", - "http://www.example.com/sitemap2.xml", - ], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/sitemap1.xml", + "http://www.example.com/sitemap2.xml", + ] spider = FilteredSitemapSpider("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/sitemap2.xml"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/sitemap2.xml" + ] def test_compression_bomb_setting(self): settings = {"DOWNLOAD_MAXSIZE": 10_000_000} @@ -711,7 +669,7 @@ Sitemap: /sitemap-relative-url.xml body = body_path.read_bytes() request = Request(url="https://example.com") response = Response(url="https://example.com", body=body, request=request) - self.assertIsNone(spider._get_sitemap_body(response)) + assert spider._get_sitemap_body(response) is None def test_compression_bomb_spider_attr(self): class DownloadMaxSizeSpider(self.spider_class): @@ -723,7 +681,7 @@ Sitemap: /sitemap-relative-url.xml body = body_path.read_bytes() request = Request(url="https://example.com") response = Response(url="https://example.com", body=body, request=request) - self.assertIsNone(spider._get_sitemap_body(response)) + assert spider._get_sitemap_body(response) is None def test_compression_bomb_request_meta(self): crawler = get_crawler() @@ -734,7 +692,7 @@ Sitemap: /sitemap-relative-url.xml url="https://example.com", meta={"download_maxsize": 10_000_000} ) response = Response(url="https://example.com", body=body, request=request) - self.assertIsNone(spider._get_sitemap_body(response)) + assert spider._get_sitemap_body(response) is None def test_download_warnsize_setting(self): settings = {"DOWNLOAD_WARNSIZE": 10_000_000} @@ -814,13 +772,13 @@ Sitemap: /sitemap-relative-url.xml ) -class DeprecationTest(unittest.TestCase): +class TestDeprecation: def test_crawl_spider(self): assert issubclass(CrawlSpider, Spider) assert isinstance(CrawlSpider(name="foo"), Spider) -class NoParseMethodSpiderTest(unittest.TestCase): +class TestNoParseMethodSpider: spider_class = Spider def test_undefined_parse_method(self): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index b103e9ed0..476487a04 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -8,7 +8,6 @@ from tempfile import mkdtemp from unittest import mock import pytest -from twisted.trial import unittest from zope.interface.verify import verifyObject # ugly hack to avoid cyclic imports of scrapy.spiders when running this test @@ -28,8 +27,8 @@ def _copytree(source: Path, target: Path): shutil.copytree(source, target) -class SpiderLoaderTest(unittest.TestCase): - def setUp(self): +class TestSpiderLoader: + def setup_method(self): orig_spiders_dir = module_dir / "test_spiders" self.tmpdir = Path(tempfile.mkdtemp()) self.spiders_dir = self.tmpdir / "test_spiders_xxx" @@ -38,7 +37,7 @@ class SpiderLoaderTest(unittest.TestCase): settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) self.spider_loader = SpiderLoader.from_settings(settings) - def tearDown(self): + def teardown_method(self): del self.spider_loader del sys.modules["test_spiders_xxx"] sys.path.remove(str(self.tmpdir)) @@ -47,37 +46,35 @@ class SpiderLoaderTest(unittest.TestCase): verifyObject(ISpiderLoader, self.spider_loader) def test_list(self): - self.assertEqual( - set(self.spider_loader.list()), {"spider1", "spider2", "spider3", "spider4"} - ) + assert set(self.spider_loader.list()) == { + "spider1", + "spider2", + "spider3", + "spider4", + } def test_load(self): spider1 = self.spider_loader.load("spider1") - self.assertEqual(spider1.__name__, "Spider1") + assert spider1.__name__ == "Spider1" def test_find_by_request(self): - self.assertEqual( - self.spider_loader.find_by_request(Request("http://scrapy1.org/test")), - ["spider1"], - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://scrapy2.org/test")), - ["spider2"], - ) - self.assertEqual( - set(self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))), - {"spider1", "spider2"}, - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://scrapy999.org/test")), [] - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://spider3.com")), [] - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://spider3.com/onlythis")), - ["spider3"], + assert self.spider_loader.find_by_request( + Request("http://scrapy1.org/test") + ) == ["spider1"] + assert self.spider_loader.find_by_request( + Request("http://scrapy2.org/test") + ) == ["spider2"] + assert set( + self.spider_loader.find_by_request(Request("http://scrapy3.org/test")) + ) == {"spider1", "spider2"} + assert ( + self.spider_loader.find_by_request(Request("http://scrapy999.org/test")) + == [] ) + assert self.spider_loader.find_by_request(Request("http://spider3.com")) == [] + assert self.spider_loader.find_by_request( + Request("http://spider3.com/onlythis") + ) == ["spider3"] def test_load_spider_module(self): module = "tests.test_spiderloader.test_spiders.spider1" @@ -113,9 +110,9 @@ class SpiderLoaderTest(unittest.TestCase): runner = CrawlerRunner({"ADDONS": {SpiderModuleAddon: 1}}) crawler = runner.create_crawler("spider_from_addon") - self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) - self.assertEqual(crawler.spidercls.name, "spider_from_addon") - self.assertTrue(len(crawler.settings["SPIDER_MODULES"]) == 1) + assert issubclass(crawler.spidercls, scrapy.Spider) + assert crawler.spidercls.name == "spider_from_addon" + assert len(crawler.settings["SPIDER_MODULES"]) == 1 def test_crawler_runner_loading(self): module = "tests.test_spiderloader.test_spiders.spider1" @@ -129,8 +126,8 @@ class SpiderLoaderTest(unittest.TestCase): runner.create_crawler("spider2") crawler = runner.create_crawler("spider1") - self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) - self.assertEqual(crawler.spidercls.name, "spider1") + assert issubclass(crawler.spidercls, scrapy.Spider) + assert crawler.spidercls.name == "spider1" def test_bad_spider_modules_exception(self): module = "tests.test_spiderloader.test_spiders.doesnotexist" @@ -150,10 +147,10 @@ class SpiderLoaderTest(unittest.TestCase): # at least until all six versions we can import (including botocore.vendored.six) # are updated to 1.16.0+ w.pop(0) - self.assertIn("Could not load spiders from module", str(w[0].message)) + assert "Could not load spiders from module" in str(w[0].message) spiders = spider_loader.list() - self.assertEqual(spiders, []) + assert not spiders def test_syntax_error_exception(self): module = "tests.test_spiderloader.test_spiders.spider1" @@ -179,14 +176,14 @@ class SpiderLoaderTest(unittest.TestCase): # at least until all six versions we can import (including botocore.vendored.six) # are updated to 1.16.0+ w.pop(0) - self.assertIn("Could not load spiders from module", str(w[0].message)) + assert "Could not load spiders from module" in str(w[0].message) spiders = spider_loader.list() - self.assertEqual(spiders, []) + assert not spiders -class DuplicateSpiderNameLoaderTest(unittest.TestCase): - def setUp(self): +class TestDuplicateSpiderNameLoader: + def setup_method(self): orig_spiders_dir = module_dir / "test_spiders" self.tmpdir = Path(mkdtemp()) self.spiders_dir = self.tmpdir / "test_spiders_xxx" @@ -194,7 +191,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): sys.path.append(str(self.tmpdir)) self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) - def tearDown(self): + def teardown_method(self): del sys.modules["test_spiders_xxx"] sys.path.remove(str(self.tmpdir)) @@ -208,18 +205,18 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) - self.assertEqual(len(w), 1) + assert len(w) == 1 msg = str(w[0].message) - self.assertIn("several spiders with the same name", msg) - self.assertIn("'spider3'", msg) - self.assertTrue(msg.count("'spider3'") == 2) + assert "several spiders with the same name" in msg + assert "'spider3'" in msg + assert msg.count("'spider3'") == 2 - self.assertNotIn("'spider1'", msg) - self.assertNotIn("'spider2'", msg) - self.assertNotIn("'spider4'", msg) + assert "'spider1'" not in msg + assert "'spider2'" not in msg + assert "'spider4'" not in msg spiders = set(spider_loader.list()) - self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"}) + assert spiders == {"spider1", "spider2", "spider3", "spider4"} def test_multiple_dupename_warning(self): # copy 2 spider modules so as to have duplicate spider name @@ -236,17 +233,17 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) - self.assertEqual(len(w), 1) + assert len(w) == 1 msg = str(w[0].message) - self.assertIn("several spiders with the same name", msg) - self.assertIn("'spider1'", msg) - self.assertTrue(msg.count("'spider1'") == 2) + assert "several spiders with the same name" in msg + assert "'spider1'" in msg + assert msg.count("'spider1'") == 2 - self.assertIn("'spider2'", msg) - self.assertTrue(msg.count("'spider2'") == 2) + assert "'spider2'" in msg + assert msg.count("'spider2'") == 2 - self.assertNotIn("'spider3'", msg) - self.assertNotIn("'spider4'", msg) + assert "'spider3'" not in msg + assert "'spider4'" not in msg spiders = set(spider_loader.list()) - self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"}) + assert spiders == {"spider1", "spider2", "spider3", "spider4"} diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index a9f3876bb..ddc9b5206 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -18,7 +18,7 @@ from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.test import get_crawler -class SpiderMiddlewareTestCase(TestCase): +class TestSpiderMiddleware(TestCase): def setUp(self): self.request = Request("http://example.com/index.html") self.response = Response(self.request.url, request=self.request) @@ -41,7 +41,7 @@ class SpiderMiddlewareTestCase(TestCase): return results[0] -class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase): +class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_input method""" def test_invalid_process_spider_input(self): @@ -51,11 +51,11 @@ class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase): self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware()) result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, _InvalidOutput) + assert isinstance(result, Failure) + assert isinstance(result.value, _InvalidOutput) -class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase): +class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_output method""" def test_invalid_process_spider_output(self): @@ -65,11 +65,11 @@ class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase): self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware()) result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, _InvalidOutput) + assert isinstance(result, Failure) + assert isinstance(result.value, _InvalidOutput) -class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase): +class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_exception method""" def test_invalid_process_spider_exception(self): @@ -84,11 +84,11 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase): self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, _InvalidOutput) + assert isinstance(result, Failure) + assert isinstance(result.value, _InvalidOutput) -class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): +class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): """Re raise the exception by returning None""" def test_process_spider_exception_return_none(self): @@ -103,11 +103,11 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, ZeroDivisionError) + assert isinstance(result, Failure) + assert isinstance(result.value, ZeroDivisionError) -class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): +class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): """Helpers for testing sync, async and mixed middlewares. Should work for process_spider_output and, when it's supported, process_start_requests. @@ -148,14 +148,13 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): result = yield self._get_middleware_result( *mw_classes, start_index=start_index ) - self.assertIsInstance(result, Iterable) + assert isinstance(result, Iterable) result_list = list(result) - self.assertEqual(len(result_list), self.RESULT_COUNT) - self.assertIsInstance(result_list[0], self.ITEM_TYPE) - self.assertEqual("downgraded to a non-async" in str(log), downgrade) - self.assertEqual( - "doesn't support asynchronous spider output" in str(log), - ProcessSpiderOutputSimpleMiddleware in mw_classes, + assert len(result_list) == self.RESULT_COUNT + assert isinstance(result_list[0], self.ITEM_TYPE) + assert ("downgraded to a non-async" in str(log)) == downgrade + assert ("doesn't support asynchronous spider output" in str(log)) == ( + ProcessSpiderOutputSimpleMiddleware in mw_classes ) @defer.inlineCallbacks @@ -166,11 +165,11 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): result = yield self._get_middleware_result( *mw_classes, start_index=start_index ) - self.assertIsInstance(result, AsyncIterator) + assert isinstance(result, AsyncIterator) result_list = yield deferred_from_coro(collect_asyncgen(result)) - self.assertEqual(len(result_list), self.RESULT_COUNT) - self.assertIsInstance(result_list[0], self.ITEM_TYPE) - self.assertEqual("downgraded to a non-async" in str(log), downgrade) + assert len(result_list) == self.RESULT_COUNT + assert isinstance(result_list[0], self.ITEM_TYPE) + assert ("downgraded to a non-async" in str(log)) == downgrade class ProcessSpiderOutputSimpleMiddleware: @@ -212,7 +211,7 @@ class ProcessSpiderExceptionAsyncIterableMiddleware: yield {"foo": 3} -class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware): """process_spider_output tests for simple callbacks""" ITEM_TYPE = dict @@ -257,7 +256,7 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): return self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN) -class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple): +class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple): """process_spider_output tests for async generator callbacks""" async def _scrape_func(self, *args, **kwargs): @@ -297,7 +296,7 @@ class ProcessSpiderOutputCoroutineMiddleware: return result -class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): @defer.inlineCallbacks def test_non_iterable(self): with pytest.raises( @@ -324,7 +323,7 @@ class ProcessStartRequestsSimpleMiddleware: yield from start_requests -class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessStartRequestsSimple(TestBaseAsyncSpiderMiddleware): """process_start_requests tests for simple start_requests""" ITEM_TYPE = (Request, dict) @@ -373,67 +372,65 @@ class UniversalMiddlewareBothAsync: yield -class UniversalMiddlewareManagerTest(TestCase): - def setUp(self): +class TestUniversalMiddlewareManager: + def setup_method(self): self.mwman = SpiderMiddlewareManager() def test_simple_mw(self): mw = ProcessSpiderOutputSimpleMiddleware() self.mwman._add_middleware(mw) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], mw.process_spider_output + assert ( + self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable ) def test_async_mw(self): mw = ProcessSpiderOutputAsyncGenMiddleware() self.mwman._add_middleware(mw) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], mw.process_spider_output + assert ( + self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable ) def test_universal_mw(self): mw = ProcessSpiderOutputUniversalMiddleware() self.mwman._add_middleware(mw) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], - (mw.process_spider_output, mw.process_spider_output_async), + assert self.mwman.methods["process_spider_output"][0] == ( + mw.process_spider_output, + mw.process_spider_output_async, ) def test_universal_mw_no_sync(self): with LogCapture() as log: self.mwman._add_middleware(UniversalMiddlewareNoSync()) - self.assertIn( + assert ( "UniversalMiddlewareNoSync has process_spider_output_async" - " without process_spider_output", - str(log), + " without process_spider_output" in str(log) ) - self.assertEqual(self.mwman.methods["process_spider_output"][0], None) + assert self.mwman.methods["process_spider_output"][0] is None def test_universal_mw_both_sync(self): mw = UniversalMiddlewareBothSync() with LogCapture() as log: self.mwman._add_middleware(mw) - self.assertIn( + assert ( "UniversalMiddlewareBothSync.process_spider_output_async " - "is not an async generator function", - str(log), + "is not an async generator function" in str(log) ) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], mw.process_spider_output + assert ( + self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable ) def test_universal_mw_both_async(self): with LogCapture() as log: self.mwman._add_middleware(UniversalMiddlewareBothAsync()) - self.assertIn( + assert ( "UniversalMiddlewareBothAsync.process_spider_output " - "is an async generator function while process_spider_output_async exists", - str(log), + "is an async generator function while process_spider_output_async exists" + in str(log) ) - self.assertEqual(self.mwman.methods["process_spider_output"][0], None) + assert self.mwman.methods["process_spider_output"][0] is None -class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): +class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware): ITEM_TYPE = dict MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware @@ -474,7 +471,7 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): return self._test_simple_base(self.MW_UNIVERSAL) -class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest): +class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple): async def _scrape_func(self, *args, **kwargs): for item in super()._scrape_func(): yield item @@ -503,7 +500,7 @@ class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest): return self._test_asyncgen_base(self.MW_UNIVERSAL) -class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): ITEM_TYPE = dict MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index e359d9cfc..dfcc141c3 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -1,5 +1,3 @@ -from unittest import TestCase - from scrapy.http import Request, Response from scrapy.spidermiddlewares.depth import DepthMiddleware from scrapy.spiders import Spider @@ -7,8 +5,8 @@ from scrapy.statscollectors import StatsCollector from scrapy.utils.test import get_crawler -class TestDepthMiddleware(TestCase): - def setUp(self): +class TestDepthMiddleware: + def setup_method(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider("scrapytest.org") @@ -24,18 +22,18 @@ class TestDepthMiddleware(TestCase): result = [Request("http://scrapytest.org")] out = list(self.mw.process_spider_output(resp, result, self.spider)) - self.assertEqual(out, result) + assert out == result rdc = self.stats.get_value("request_depth_count/1", spider=self.spider) - self.assertEqual(rdc, 1) + assert rdc == 1 req.meta["depth"] = 1 out2 = list(self.mw.process_spider_output(resp, result, self.spider)) - self.assertEqual(out2, []) + assert not out2 rdm = self.stats.get_value("request_depth_max", spider=self.spider) - self.assertEqual(rdm, 1) + assert rdm == 1 - def tearDown(self): + def teardown_method(self): self.stats.close_spider(self.spider, "") diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index f9eb93d6b..e306579fa 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,10 +1,9 @@ import logging -from unittest import TestCase import pytest from testfixtures import LogCapture from twisted.internet import defer -from twisted.trial.unittest import TestCase as TrialTestCase +from twisted.trial.unittest import TestCase from scrapy.http import Request, Response from scrapy.settings import Settings @@ -59,8 +58,8 @@ def _responses(request, status_codes): return responses -class TestHttpErrorMiddleware(TestCase): - def setUp(self): +class TestHttpErrorMiddleware: + def setup_method(self): crawler = get_crawler(Spider) self.spider = Spider.from_crawler(crawler, name="foo") self.mw = HttpErrorMiddleware(Settings({})) @@ -68,19 +67,20 @@ class TestHttpErrorMiddleware(TestCase): self.res200, self.res404 = _responses(self.req, [200, 404]) def test_process_spider_input(self): - self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + assert self.mw.process_spider_input(self.res200, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(self.res404, self.spider) def test_process_spider_exception(self): - self.assertEqual( - [], + assert ( self.mw.process_spider_exception( self.res404, HttpError(self.res404), self.spider - ), + ) + == [] ) - self.assertIsNone( + assert ( self.mw.process_spider_exception(self.res404, Exception(), self.spider) + is None ) def test_handle_httpstatus_list(self): @@ -88,26 +88,26 @@ class TestHttpErrorMiddleware(TestCase): res.request = Request( "http://scrapytest.org", meta={"handle_httpstatus_list": [404]} ) - self.assertIsNone(self.mw.process_spider_input(res, self.spider)) + assert self.mw.process_spider_input(res, self.spider) is None self.spider.handle_httpstatus_list = [404] - self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + assert self.mw.process_spider_input(self.res404, self.spider) is None -class TestHttpErrorMiddlewareSettings(TestCase): +class TestHttpErrorMiddlewareSettings: """Similar test, but with settings""" - def setUp(self): + def setup_method(self): self.spider = Spider("foo") self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)})) self.req = Request("http://scrapytest.org") self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + assert self.mw.process_spider_input(self.res200, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(self.res404, self.spider) - self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider)) + assert self.mw.process_spider_input(self.res402, self.spider) is None def test_meta_overrides_settings(self): request = Request( @@ -118,27 +118,27 @@ class TestHttpErrorMiddlewareSettings(TestCase): res402 = self.res402.copy() res402.request = request - self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) + assert self.mw.process_spider_input(res404, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(res402, self.spider) def test_spider_override_settings(self): self.spider.handle_httpstatus_list = [404] - self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + assert self.mw.process_spider_input(self.res404, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(self.res402, self.spider) -class TestHttpErrorMiddlewareHandleAll(TestCase): - def setUp(self): +class TestHttpErrorMiddlewareHandleAll: + def setup_method(self): self.spider = Spider("foo") self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True})) self.req = Request("http://scrapytest.org") self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) - self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + assert self.mw.process_spider_input(self.res200, self.spider) is None + assert self.mw.process_spider_input(self.res404, self.spider) is None def test_meta_overrides_settings(self): request = Request( @@ -149,7 +149,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): res402 = self.res402.copy() res402.request = request - self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) + assert self.mw.process_spider_input(res404, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(res402, self.spider) @@ -169,10 +169,10 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): with pytest.raises(HttpError): mw.process_spider_input(res404, self.spider) - self.assertIsNone(mw.process_spider_input(res402, self.spider)) + assert mw.process_spider_input(res402, self.spider) is None -class TestHttpErrorMiddlewareIntegrational(TrialTestCase): +class TestHttpErrorMiddlewareIntegrational(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -187,28 +187,28 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): crawler = get_crawler(_HttpErrorSpider) yield crawler.crawl(mockserver=self.mockserver) assert not crawler.spider.skipped, crawler.spider.skipped - self.assertEqual(crawler.spider.parsed, {"200"}) - self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) + assert crawler.spider.parsed == {"200"} + assert crawler.spider.failed == {"404", "402", "500"} get_value = crawler.stats.get_value - self.assertEqual(get_value("httperror/response_ignored_count"), 3) - self.assertEqual(get_value("httperror/response_ignored_status_count/404"), 1) - self.assertEqual(get_value("httperror/response_ignored_status_count/402"), 1) - self.assertEqual(get_value("httperror/response_ignored_status_count/500"), 1) + assert get_value("httperror/response_ignored_count") == 3 + assert get_value("httperror/response_ignored_status_count/404") == 1 + assert get_value("httperror/response_ignored_status_count/402") == 1 + assert get_value("httperror/response_ignored_status_count/500") == 1 @defer.inlineCallbacks def test_logging(self): crawler = get_crawler(_HttpErrorSpider) with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402}) - self.assertEqual(crawler.spider.parsed, {"200", "402"}) - self.assertEqual(crawler.spider.skipped, {"402"}) - self.assertEqual(crawler.spider.failed, {"404", "500"}) + assert crawler.spider.parsed == {"200", "402"} + assert crawler.spider.skipped == {"402"} + assert crawler.spider.failed == {"404", "500"} - self.assertIn("Ignoring response <404", str(log)) - self.assertIn("Ignoring response <500", str(log)) - self.assertNotIn("Ignoring response <200", str(log)) - self.assertNotIn("Ignoring response <402", str(log)) + assert "Ignoring response <404" in str(log) + assert "Ignoring response <500" in str(log) + assert "Ignoring response <200" not in str(log) + assert "Ignoring response <402" not in str(log) @defer.inlineCallbacks def test_logging_level(self): @@ -216,22 +216,22 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.INFO) as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(crawler.spider.parsed, {"200"}) - self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) + assert crawler.spider.parsed == {"200"} + assert crawler.spider.failed == {"404", "402", "500"} - self.assertIn("Ignoring response <402", str(log)) - self.assertIn("Ignoring response <404", str(log)) - self.assertIn("Ignoring response <500", str(log)) - self.assertNotIn("Ignoring response <200", str(log)) + assert "Ignoring response <402" in str(log) + assert "Ignoring response <404" in str(log) + assert "Ignoring response <500" in str(log) + assert "Ignoring response <200" not in str(log) # with level WARNING, we shouldn't capture anything from HttpError crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.WARNING) as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(crawler.spider.parsed, {"200"}) - self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) + assert crawler.spider.parsed == {"200"} + assert crawler.spider.failed == {"404", "402", "500"} - self.assertNotIn("Ignoring response <402", str(log)) - self.assertNotIn("Ignoring response <404", str(log)) - self.assertNotIn("Ignoring response <500", str(log)) - self.assertNotIn("Ignoring response <200", str(log)) + assert "Ignoring response <402" not in str(log) + assert "Ignoring response <404" not in str(log) + assert "Ignoring response <500" not in str(log) + assert "Ignoring response <200" not in str(log) diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 906928e01..f4563a0a4 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -1,5 +1,4 @@ import warnings -from unittest import TestCase from urllib.parse import urlparse from scrapy.http import Request, Response @@ -8,8 +7,8 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class TestOffsiteMiddleware(TestCase): - def setUp(self): +class TestOffsiteMiddleware: + def setup_method(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider(**self._get_spiderargs()) self.mw = OffsiteMiddleware.from_crawler(crawler) @@ -46,7 +45,7 @@ class TestOffsiteMiddleware(TestCase): reqs = onsite_reqs + offsite_reqs out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEqual(out, onsite_reqs) + assert out == onsite_reqs class TestOffsiteMiddleware2(TestOffsiteMiddleware): @@ -57,7 +56,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware): res = Response("http://scrapytest.org") reqs = [Request("http://a.com/b.html"), Request("http://b.com/1")] out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEqual(out, reqs) + assert out == reqs class TestOffsiteMiddleware3(TestOffsiteMiddleware2): @@ -77,7 +76,7 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3): res = Response("http://scrapytest.org") reqs = [Request("http://scrapytest.org/1")] out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEqual(out, reqs) + assert out == reqs class TestOffsiteMiddleware5(TestOffsiteMiddleware4): diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index e51957497..6e26a85ea 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -324,9 +324,9 @@ class TestSpiderMiddleware(TestCase): was enqueued from the recovery middleware) """ log = yield self.crawl_log(RecoverySpider) - self.assertIn("Middleware: TabError exception caught", str(log)) - self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1) - self.assertIn("'item_scraped_count': 3", str(log)) + assert "Middleware: TabError exception caught" in str(log) + assert str(log).count("Middleware: TabError exception caught") == 1 + assert "'item_scraped_count': 3" in str(log) @defer.inlineCallbacks def test_recovery_asyncgen(self): @@ -334,9 +334,9 @@ class TestSpiderMiddleware(TestCase): Same as test_recovery but with an async callback. """ log = yield self.crawl_log(RecoveryAsyncGenSpider) - self.assertIn("Middleware: TabError exception caught", str(log)) - self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1) - self.assertIn("'item_scraped_count': 3", str(log)) + assert "Middleware: TabError exception caught" in str(log) + assert str(log).count("Middleware: TabError exception caught") == 1 + assert "'item_scraped_count': 3" in str(log) @defer.inlineCallbacks def test_process_spider_input_without_errback(self): @@ -345,8 +345,8 @@ class TestSpiderMiddleware(TestCase): process_spider_exception chain from the start if the Request has no errback """ log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithoutErrback) - self.assertIn("Middleware: will raise IndexError", str(log1)) - self.assertIn("Middleware: IndexError exception caught", str(log1)) + assert "Middleware: will raise IndexError" in str(log1) + assert "Middleware: IndexError exception caught" in str(log1) @defer.inlineCallbacks def test_process_spider_input_with_errback(self): @@ -355,12 +355,12 @@ class TestSpiderMiddleware(TestCase): process_spider_exception chain if the Request has an errback """ log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithErrback) - self.assertNotIn("Middleware: IndexError exception caught", str(log1)) - self.assertIn("Middleware: will raise IndexError", str(log1)) - self.assertIn("Got a Failure on the Request errback", str(log1)) - self.assertIn("{'from': 'errback'}", str(log1)) - self.assertNotIn("{'from': 'callback'}", str(log1)) - self.assertIn("'item_scraped_count': 1", str(log1)) + assert "Middleware: IndexError exception caught" not in str(log1) + assert "Middleware: will raise IndexError" in str(log1) + assert "Got a Failure on the Request errback" in str(log1) + assert "{'from': 'errback'}" in str(log1) + assert "{'from': 'callback'}" not in str(log1) + assert "'item_scraped_count': 1" in str(log1) @defer.inlineCallbacks def test_generator_callback(self): @@ -370,8 +370,8 @@ class TestSpiderMiddleware(TestCase): exception is raised should be processed normally. """ log2 = yield self.crawl_log(GeneratorCallbackSpider) - self.assertIn("Middleware: ImportError exception caught", str(log2)) - self.assertIn("'item_scraped_count': 2", str(log2)) + assert "Middleware: ImportError exception caught" in str(log2) + assert "'item_scraped_count': 2" in str(log2) @defer.inlineCallbacks def test_async_generator_callback(self): @@ -379,8 +379,8 @@ class TestSpiderMiddleware(TestCase): Same as test_generator_callback but with an async callback. """ log2 = yield self.crawl_log(AsyncGeneratorCallbackSpider) - self.assertIn("Middleware: ImportError exception caught", str(log2)) - self.assertIn("'item_scraped_count': 2", str(log2)) + assert "Middleware: ImportError exception caught" in str(log2) + assert "'item_scraped_count': 2" in str(log2) @defer.inlineCallbacks def test_generator_callback_right_after_callback(self): @@ -389,8 +389,8 @@ class TestSpiderMiddleware(TestCase): even if the middleware is placed right after the spider """ log21 = yield self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider) - self.assertIn("Middleware: ImportError exception caught", str(log21)) - self.assertIn("'item_scraped_count': 2", str(log21)) + assert "Middleware: ImportError exception caught" in str(log21) + assert "'item_scraped_count': 2" in str(log21) @defer.inlineCallbacks def test_not_a_generator_callback(self): @@ -399,8 +399,8 @@ class TestSpiderMiddleware(TestCase): be caught by the process_spider_exception chain. No items should be processed. """ log3 = yield self.crawl_log(NotGeneratorCallbackSpider) - self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3)) - self.assertNotIn("item_scraped_count", str(log3)) + assert "Middleware: ZeroDivisionError exception caught" in str(log3) + assert "item_scraped_count" not in str(log3) @defer.inlineCallbacks def test_not_a_generator_callback_right_after_callback(self): @@ -411,8 +411,8 @@ class TestSpiderMiddleware(TestCase): log31 = yield self.crawl_log( NotGeneratorCallbackSpiderMiddlewareRightAfterSpider ) - self.assertIn("Middleware: ZeroDivisionError exception caught", str(log31)) - self.assertNotIn("item_scraped_count", str(log31)) + assert "Middleware: ZeroDivisionError exception caught" in str(log31) + assert "item_scraped_count" not in str(log31) @defer.inlineCallbacks def test_generator_output_chain(self): @@ -425,22 +425,22 @@ class TestSpiderMiddleware(TestCase): process_spider_exception chain) """ log4 = yield self.crawl_log(GeneratorOutputChainSpider) - self.assertIn("'item_scraped_count': 2", str(log4)) - self.assertIn( - "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert "'item_scraped_count': 2" in str(log4) + assert ( + "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught" + in str(log4) ) - self.assertIn( - "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert ( + "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught" + in str(log4) ) - self.assertNotIn( - "GeneratorFailMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert ( + "GeneratorFailMiddleware.process_spider_exception: LookupError caught" + not in str(log4) ) - self.assertNotIn( - "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert ( + "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught" + not in str(log4) ) item_from_callback = { "processed": [ @@ -457,9 +457,9 @@ class TestSpiderMiddleware(TestCase): "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output", ] } - self.assertIn(str(item_from_callback), str(log4)) - self.assertIn(str(item_recovered), str(log4)) - self.assertNotIn("parse-second-item", str(log4)) + assert str(item_from_callback) in str(log4) + assert str(item_recovered) in str(log4) + assert "parse-second-item" not in str(log4) @defer.inlineCallbacks def test_not_a_generator_output_chain(self): @@ -472,22 +472,22 @@ class TestSpiderMiddleware(TestCase): from the spider callback are lost) """ log5 = yield self.crawl_log(NotGeneratorOutputChainSpider) - self.assertIn("'item_scraped_count': 1", str(log5)) - self.assertIn( - "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", - str(log5), + assert "'item_scraped_count': 1" in str(log5) + assert ( + "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught" + in str(log5) ) - self.assertIn( - "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", - str(log5), + assert ( + "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught" + in str(log5) ) - self.assertNotIn( - "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", - str(log5), + assert ( + "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught" + not in str(log5) ) - self.assertNotIn( - "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", - str(log5), + assert ( + "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught" + not in str(log5) ) item_recovered = { "processed": [ @@ -495,6 +495,6 @@ class TestSpiderMiddleware(TestCase): "NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output", ] } - self.assertIn(str(item_recovered), str(log5)) - self.assertNotIn("parse-first-item", str(log5)) - self.assertNotIn("parse-second-item", str(log5)) + assert str(item_recovered) in str(log5) + assert "parse-first-item" not in str(log5) + assert "parse-second-item" not in str(log5) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 01a87c645..300a40c13 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -2,7 +2,6 @@ from __future__ import annotations import warnings from typing import Any -from unittest import TestCase from urllib.parse import urlparse import pytest @@ -35,7 +34,7 @@ from scrapy.spidermiddlewares.referer import ( from scrapy.spiders import Spider -class TestRefererMiddleware(TestCase): +class TestRefererMiddleware: req_meta: dict[str, Any] = {} resp_headers: dict[str, str] = {} settings: dict[str, Any] = {} @@ -43,7 +42,7 @@ class TestRefererMiddleware(TestCase): ("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"), ] - def setUp(self): + def setup_method(self): self.spider = Spider("foo") settings = Settings(self.settings) self.mw = RefererMiddleware(settings) @@ -59,7 +58,7 @@ class TestRefererMiddleware(TestCase): response = self.get_response(origin) request = self.get_request(target) out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEqual(out[0].headers.get("Referer"), referrer) + assert out[0].headers.get("Referer") == referrer class MixinDefault: @@ -773,7 +772,7 @@ class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {"referrer_policy": POLICY_UNSAFE_URL} -class TestRequestMetaSettingFallback(TestCase): +class TestRequestMetaSettingFallback: params = [ ( # When an unknown policy is referenced in Request.meta @@ -844,14 +843,14 @@ class TestRequestMetaSettingFallback(TestCase): with warnings.catch_warnings(record=True) as w: policy = mw.policy(response, request) - self.assertIsInstance(policy, policy_class) + assert isinstance(policy, policy_class) if check_warning: - self.assertEqual(len(w), 1) - self.assertEqual(w[0].category, RuntimeWarning, w[0].message) + assert len(w) == 1 + assert w[0].category is RuntimeWarning, w[0].message -class TestSettingsPolicyByName(TestCase): +class TestSettingsPolicyByName: def test_valid_name(self): for s, p in [ (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), @@ -866,7 +865,7 @@ class TestSettingsPolicyByName(TestCase): ]: settings = Settings({"REFERRER_POLICY": s}) mw = RefererMiddleware(settings) - self.assertEqual(mw.default_policy, p) + assert mw.default_policy == p def test_valid_name_casevariants(self): for s, p in [ @@ -882,7 +881,7 @@ class TestSettingsPolicyByName(TestCase): ]: settings = Settings({"REFERRER_POLICY": s.upper()}) mw = RefererMiddleware(settings) - self.assertEqual(mw.default_policy, p) + assert mw.default_policy == p def test_invalid_name(self): settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"}) @@ -902,7 +901,7 @@ class TestSettingsPolicyByName(TestCase): } ) mw1 = RefererMiddleware(settings1) - self.assertEqual(mw1.default_policy, StrictOriginWhenCrossOriginPolicy) + assert mw1.default_policy == StrictOriginWhenCrossOriginPolicy # test parsing with space(s) after the comma settings2 = Settings( @@ -915,7 +914,7 @@ class TestSettingsPolicyByName(TestCase): } ) mw2 = RefererMiddleware(settings2) - self.assertEqual(mw2.default_policy, UnsafeUrlPolicy) + assert mw2.default_policy == UnsafeUrlPolicy def test_multiple_policy_tokens_all_invalid(self): settings = Settings( @@ -1003,7 +1002,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): ), ] - def setUp(self): + def setup_method(self): self.spider = Spider("foo") settings = Settings(self.settings) self.referrermw = RefererMiddleware(settings) @@ -1023,7 +1022,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): out = list( self.referrermw.process_spider_output(response, [request], self.spider) ) - self.assertEqual(out[0].headers.get("Referer"), init_referrer) + assert out[0].headers.get("Referer") == init_referrer for status, url in redirections: response = Response( @@ -1035,7 +1034,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): self.referrermw.request_scheduled(request, self.spider) assert isinstance(request, Request) - self.assertEqual(request.headers.get("Referer"), final_referrer) + assert request.headers.get("Referer") == final_referrer class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 1a0f2e223..5cc3cdc6c 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -1,5 +1,3 @@ -from unittest import TestCase - from testfixtures import LogCapture from scrapy.http import Request, Response @@ -8,8 +6,8 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class TestUrlLengthMiddleware(TestCase): - def setUp(self): +class TestUrlLengthMiddleware: + def setup_method(self): self.maxlength = 25 crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": self.maxlength}) self.spider = crawler._create_spider("foo") @@ -27,7 +25,7 @@ class TestUrlLengthMiddleware(TestCase): ) def test_middleware_works(self): - self.assertEqual(self.process_spider_output(), [self.short_url_req]) + assert self.process_spider_output() == [self.short_url_req] def test_logging(self): with LogCapture() as log: @@ -36,6 +34,6 @@ class TestUrlLengthMiddleware(TestCase): ric = self.stats.get_value( "urllength/request_ignored_count", spider=self.spider ) - self.assertEqual(ric, 1) + assert ric == 1 - self.assertIn(f"Ignoring link (url length > {self.maxlength})", str(log)) + assert f"Ignoring link (url length > {self.maxlength})" in str(log) diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index 72692afab..cd31891a0 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -3,7 +3,6 @@ from datetime import datetime, timezone from tempfile import mkdtemp import pytest -from twisted.trial import unittest from scrapy.exceptions import NotConfigured from scrapy.extensions.spiderstate import SpiderState @@ -11,7 +10,7 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class SpiderStateTest(unittest.TestCase): +class TestSpiderState: def test_store_load(self): jobdir = mkdtemp() try: @@ -27,7 +26,7 @@ class SpiderStateTest(unittest.TestCase): spider2 = Spider(name="default") ss2 = SpiderState(jobdir) ss2.spider_opened(spider2) - self.assertEqual(spider.state, {"one": 1, "dt": dt}) + assert spider.state == {"one": 1, "dt": dt} ss2.spider_closed(spider2) finally: shutil.rmtree(jobdir) @@ -38,7 +37,7 @@ class SpiderStateTest(unittest.TestCase): spider = Spider(name="default") ss = SpiderState() ss.spider_opened(spider) - self.assertEqual(spider.state, {}) + assert spider.state == {} ss.spider_closed(spider) def test_not_configured(self): diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 8556b75dd..6283b9ad6 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -50,9 +50,9 @@ class FifoDiskQueueTestMixin: q.push("a") q.push(123) q.push({"a": "dict"}) - self.assertEqual(q.pop(), "a") - self.assertEqual(q.pop(), 123) - self.assertEqual(q.pop(), {"a": "dict"}) + assert q.pop() == "a" + assert q.pop() == 123 + assert q.pop() == {"a": "dict"} test_nonserializable_object = nonserializable_object_test @@ -92,7 +92,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): q.push(i) i2 = q.pop() assert isinstance(i2, MyItem) - self.assertEqual(i, i2) + assert i == i2 def test_serialize_loader(self): q = self.queue() @@ -101,7 +101,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): loader2 = q.pop() assert isinstance(loader2, MyLoader) assert loader2.default_item_class is MyItem - self.assertEqual(loader2.name_out("x"), "xx") + assert loader2.name_out("x") == "xx" def test_serialize_request_recursive(self): q = self.queue() @@ -110,23 +110,26 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): q.push(r) r2 = q.pop() assert isinstance(r2, Request) - self.assertEqual(r.url, r2.url) + assert r.url == r2.url assert r2.meta["request"] is r2 def test_non_pickable_object(self): q = self.queue() - try: + with pytest.raises( + ValueError, + match="Can't (get|pickle) local object|Can't pickle .*: it's not found as", + ) as exc_info: q.push(lambda x: x) - except ValueError as exc: - if hasattr(sys, "pypy_version_info"): - self.assertIsInstance(exc.__context__, pickle.PicklingError) - else: - self.assertIsInstance(exc.__context__, AttributeError) + if hasattr(sys, "pypy_version_info"): + assert isinstance(exc_info.value.__context__, pickle.PicklingError) + else: + assert isinstance(exc_info.value.__context__, AttributeError) sel = Selector(text="<html><body><p>some text</p></body></html>") - try: + with pytest.raises( + ValueError, match="can't pickle Selector objects" + ) as exc_info: q.push(sel) - except ValueError as exc: - self.assertIsInstance(exc.__context__, TypeError) + assert isinstance(exc_info.value.__context__, TypeError) class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): @@ -151,9 +154,9 @@ class LifoDiskQueueTestMixin: q.push("a") q.push(123) q.push({"a": "dict"}) - self.assertEqual(q.pop(), {"a": "dict"}) - self.assertEqual(q.pop(), 123) - self.assertEqual(q.pop(), "a") + assert q.pop() == {"a": "dict"} + assert q.pop() == 123 + assert q.pop() == "a" test_nonserializable_object = nonserializable_object_test @@ -173,7 +176,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): q.push(i) i2 = q.pop() assert isinstance(i2, MyItem) - self.assertEqual(i, i2) + assert i == i2 def test_serialize_loader(self): q = self.queue() @@ -182,7 +185,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): loader2 = q.pop() assert isinstance(loader2, MyLoader) assert loader2.default_item_class is MyItem - self.assertEqual(loader2.name_out("x"), "xx") + assert loader2.name_out("x") == "xx" def test_serialize_request_recursive(self): q = self.queue() @@ -191,5 +194,5 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): q.push(r) r2 = q.pop() assert isinstance(r2, Request) - self.assertEqual(r.url, r2.url) + assert r.url == r2.url assert r2.meta["request"] is r2 diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 88f6657d8..6c153f40e 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -22,14 +22,14 @@ from scrapy.squeues import ( from scrapy.utils.test import get_crawler -class BaseQueueTestCase(unittest.TestCase): - def setUp(self): +class TestBaseQueue: + def setup_method(self): self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-") self.qpath = self.tempfilename() self.qdir = tempfile.mkdtemp() self.crawler = get_crawler(Spider) - def tearDown(self): + def teardown_method(self): shutil.rmtree(self.tmpdir) def tempfilename(self): @@ -48,36 +48,36 @@ class RequestQueueTestMixin: if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None req = Request("http://www.example.com") q.push(req) - self.assertEqual(len(q), 1) - self.assertEqual(q.peek().url, req.url) - self.assertEqual(q.pop().url, req.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 1 + assert q.peek().url == req.url + assert q.pop().url == req.url + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None q.close() def test_one_element_without_peek(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.pop() is None req = Request("http://www.example.com") q.push(req) - self.assertEqual(len(q), 1) + assert len(q) == 1 with pytest.raises( NotImplementedError, match="The underlying queue class does not implement 'peek'", ): q.peek() - self.assertEqual(q.pop().url, req.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert q.pop().url == req.url + assert len(q) == 0 + assert q.pop() is None q.close() @@ -86,35 +86,35 @@ class FifoQueueMixin(RequestQueueTestMixin): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") q.push(req1) q.push(req2) q.push(req3) - self.assertEqual(len(q), 3) - self.assertEqual(q.peek().url, req1.url) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.peek().url, req2.url) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.peek().url, req3.url) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 3 + assert q.peek().url == req1.url + assert q.pop().url == req1.url + assert len(q) == 2 + assert q.peek().url == req2.url + assert q.pop().url == req2.url + assert len(q) == 1 + assert q.peek().url == req3.url + assert q.pop().url == req3.url + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None q.close() def test_fifo_without_peek(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") @@ -126,14 +126,14 @@ class FifoQueueMixin(RequestQueueTestMixin): match="The underlying queue class does not implement 'peek'", ): q.peek() - self.assertEqual(len(q), 3) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 3 + assert q.pop().url == req1.url + assert len(q) == 2 + assert q.pop().url == req2.url + assert len(q) == 1 + assert q.pop().url == req3.url + assert len(q) == 0 + assert q.pop() is None q.close() @@ -142,35 +142,35 @@ class LifoQueueMixin(RequestQueueTestMixin): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") q.push(req1) q.push(req2) q.push(req3) - self.assertEqual(len(q), 3) - self.assertEqual(q.peek().url, req3.url) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.peek().url, req2.url) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.peek().url, req1.url) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 3 + assert q.peek().url == req3.url + assert q.pop().url == req3.url + assert len(q) == 2 + assert q.peek().url == req2.url + assert q.pop().url == req2.url + assert len(q) == 1 + assert q.peek().url == req1.url + assert q.pop().url == req1.url + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None q.close() def test_lifo_without_peek(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") @@ -182,46 +182,46 @@ class LifoQueueMixin(RequestQueueTestMixin): match="The underlying queue class does not implement 'peek'", ): q.peek() - self.assertEqual(len(q), 3) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 3 + assert q.pop().url == req3.url + assert len(q) == 2 + assert q.pop().url == req2.url + assert len(q) == 1 + assert q.pop().url == req1.url + assert len(q) == 0 + assert q.pop() is None q.close() -class PickleFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): +class TestPickleFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue): def queue(self): return PickleFifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/fifo") -class PickleLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): +class TestPickleLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue): def queue(self): return PickleLifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/lifo") -class MarshalFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): +class TestMarshalFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue): def queue(self): return MarshalFifoDiskQueue.from_crawler( crawler=self.crawler, key="marshal/fifo" ) -class MarshalLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): +class TestMarshalLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue): def queue(self): return MarshalLifoDiskQueue.from_crawler( crawler=self.crawler, key="marshal/lifo" ) -class FifoMemoryQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): +class TestFifoMemoryQueueRequest(FifoQueueMixin, TestBaseQueue): def queue(self): return FifoMemoryQueue.from_crawler(crawler=self.crawler) -class LifoMemoryQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): +class TestLifoMemoryQueueRequest(LifoQueueMixin, TestBaseQueue): def queue(self): return LifoMemoryQueue.from_crawler(crawler=self.crawler) diff --git a/tests/test_stats.py b/tests/test_stats.py index 3d4c7e88e..537614364 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -1,4 +1,3 @@ -import unittest from datetime import datetime from unittest import mock @@ -8,8 +7,8 @@ from scrapy.statscollectors import DummyStatsCollector, StatsCollector from scrapy.utils.test import get_crawler -class CoreStatsExtensionTest(unittest.TestCase): - def setUp(self): +class TestCoreStatsExtension: + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") @@ -24,19 +23,16 @@ class CoreStatsExtensionTest(unittest.TestCase): ext.response_received(self.spider) ext.item_dropped({}, self.spider, ZeroDivisionError()) ext.spider_closed(self.spider, "finished") - self.assertEqual( - ext.stats._stats, - { - "start_time": fixed_datetime, - "finish_time": fixed_datetime, - "item_scraped_count": 1, - "response_received_count": 1, - "item_dropped_count": 1, - "item_dropped_reasons_count/ZeroDivisionError": 1, - "finish_reason": "finished", - "elapsed_time_seconds": 0.0, - }, - ) + assert ext.stats._stats == { + "start_time": fixed_datetime, + "finish_time": fixed_datetime, + "item_scraped_count": 1, + "response_received_count": 1, + "item_dropped_count": 1, + "item_dropped_reasons_count/ZeroDivisionError": 1, + "finish_reason": "finished", + "elapsed_time_seconds": 0.0, + } def test_core_stats_dummy_stats_collector(self): self.crawler.stats = DummyStatsCollector(self.crawler) @@ -46,51 +42,51 @@ class CoreStatsExtensionTest(unittest.TestCase): ext.response_received(self.spider) ext.item_dropped({}, self.spider, ZeroDivisionError()) ext.spider_closed(self.spider, "finished") - self.assertEqual(ext.stats._stats, {}) + assert ext.stats._stats == {} -class StatsCollectorTest(unittest.TestCase): - def setUp(self): +class TestStatsCollector: + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") def test_collector(self): stats = StatsCollector(self.crawler) - self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_value("anything"), None) - self.assertEqual(stats.get_value("anything", "default"), "default") + assert stats.get_stats() == {} + assert stats.get_value("anything") is None + assert stats.get_value("anything", "default") == "default" stats.set_value("test", "value") - self.assertEqual(stats.get_stats(), {"test": "value"}) + assert stats.get_stats() == {"test": "value"} stats.set_value("test2", 23) - self.assertEqual(stats.get_stats(), {"test": "value", "test2": 23}) - self.assertEqual(stats.get_value("test2"), 23) + assert stats.get_stats() == {"test": "value", "test2": 23} + assert stats.get_value("test2") == 23 stats.inc_value("test2") - self.assertEqual(stats.get_value("test2"), 24) + assert stats.get_value("test2") == 24 stats.inc_value("test2", 6) - self.assertEqual(stats.get_value("test2"), 30) + assert stats.get_value("test2") == 30 stats.max_value("test2", 6) - self.assertEqual(stats.get_value("test2"), 30) + assert stats.get_value("test2") == 30 stats.max_value("test2", 40) - self.assertEqual(stats.get_value("test2"), 40) + assert stats.get_value("test2") == 40 stats.max_value("test3", 1) - self.assertEqual(stats.get_value("test3"), 1) + assert stats.get_value("test3") == 1 stats.min_value("test2", 60) - self.assertEqual(stats.get_value("test2"), 40) + assert stats.get_value("test2") == 40 stats.min_value("test2", 35) - self.assertEqual(stats.get_value("test2"), 35) + assert stats.get_value("test2") == 35 stats.min_value("test4", 7) - self.assertEqual(stats.get_value("test4"), 7) + assert stats.get_value("test4") == 7 def test_dummy_collector(self): stats = DummyStatsCollector(self.crawler) - self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_value("anything"), None) - self.assertEqual(stats.get_value("anything", "default"), "default") + assert stats.get_stats() == {} + assert stats.get_value("anything") is None + assert stats.get_value("anything", "default") == "default" stats.set_value("test", "value") stats.inc_value("v1") stats.max_value("v2", 100) stats.min_value("v3", 100) stats.open_spider("a") stats.set_value("test", "value", spider=self.spider) - self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_stats("a"), {}) + assert stats.get_stats() == {} + assert stats.get_stats("a") == {} From 40833afc86d45543a521b0f147387f7f770a8adc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 7 Mar 2025 22:33:41 +0400 Subject: [PATCH 066/491] Work around a queuelib test file close problem. (#6703) --- tests/test_squeues.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 6283b9ad6..0b6ed8e11 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -130,6 +130,9 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): ) as exc_info: q.push(sel) assert isinstance(exc_info.value.__context__, TypeError) + # This seems to help with https://github.com/scrapy/queuelib/issues/70. + # It will need to remain under a queuelib version check after that bug is fixed. + del exc_info class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): From 1469b2739ea566a57e0b5f8e6bb104fd19460d24 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sat, 8 Mar 2025 20:50:54 +0400 Subject: [PATCH 067/491] Drop tests/test_loader_deprecated.py. (#6704) --- tests/test_loader_deprecated.py | 750 -------------------------------- 1 file changed, 750 deletions(-) delete mode 100644 tests/test_loader_deprecated.py diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py deleted file mode 100644 index 0d7921b1d..000000000 --- a/tests/test_loader_deprecated.py +++ /dev/null @@ -1,750 +0,0 @@ -""" -These tests are kept as references from the ones that were ported to a itemloaders library. -Once we remove the references from scrapy, we can remove these tests. -""" - -import unittest -from functools import partial - -import pytest -from itemloaders.processors import ( - Compose, - Identity, - Join, - MapCompose, - SelectJmes, - TakeFirst, -) - -from scrapy.item import Field, Item -from scrapy.loader import ItemLoader - - -# test items -class NameItem(Item): - name = Field() - - -class SummaryItem(NameItem): - url = Field() - summary = Field() - - -# test item loaders -class NameItemLoader(ItemLoader): - default_item_class = SummaryItem - - -class ProcessorItemLoader(NameItemLoader): - name_in = MapCompose(lambda v: v.title()) - - -class DefaultedItemLoader(NameItemLoader): - default_input_processor = MapCompose(lambda v: v[:-1]) - - -# test processors -def processor_with_args(value, other=None, loader_context=None): - if "key" in loader_context: - return loader_context["key"] - return value - - -class BasicItemLoaderTest(unittest.TestCase): - def test_load_item_using_default_loader(self): - i = SummaryItem() - i["summary"] = "lala" - il = ItemLoader(item=i) - il.add_value("name", "marta") - item = il.load_item() - assert item is i - self.assertEqual(item["summary"], ["lala"]) - self.assertEqual(item["name"], ["marta"]) - - def test_load_item_using_custom_loader(self): - il = ProcessorItemLoader() - il.add_value("name", "marta") - item = il.load_item() - self.assertEqual(item["name"], ["Marta"]) - - def test_load_item_ignore_none_field_values(self): - def validate_sku(value): - # Let's assume a SKU is only digits. - return value if value.isdigit() else None - - class MyLoader(ItemLoader): - name_out = Compose(lambda vs: vs[0]) # take first which allows empty values - price_out = Compose(TakeFirst(), float) - sku_out = Compose(TakeFirst(), validate_sku) - - valid_fragment = "SKU: 1234" - invalid_fragment = "SKU: not available" - sku_re = "SKU: (.+)" - - il = MyLoader(item={}) - # Should not return "sku: None". - il.add_value("sku", [invalid_fragment], re=sku_re) - # Should not ignore empty values. - il.add_value("name", "") - il.add_value("price", ["0"]) - self.assertEqual( - il.load_item(), - { - "name": "", - "price": 0.0, - }, - ) - - il.replace_value("sku", [valid_fragment], re=sku_re) - self.assertEqual(il.load_item()["sku"], "1234") - - def test_self_referencing_loader(self): - class MyLoader(ItemLoader): - url_out = TakeFirst() - - def img_url_out(self, values): - return (self.get_output_value("url") or "") + values[0] - - il = MyLoader(item={}) - il.add_value("url", "http://example.com/") - il.add_value("img_url", "1234.png") - self.assertEqual( - il.load_item(), - { - "url": "http://example.com/", - "img_url": "http://example.com/1234.png", - }, - ) - - il = MyLoader(item={}) - il.add_value("img_url", "1234.png") - self.assertEqual( - il.load_item(), - { - "img_url": "1234.png", - }, - ) - - def test_add_value(self): - il = ProcessorItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_collected_values("name"), ["Marta"]) - self.assertEqual(il.get_output_value("name"), ["Marta"]) - il.add_value("name", "pepe") - self.assertEqual(il.get_collected_values("name"), ["Marta", "Pepe"]) - self.assertEqual(il.get_output_value("name"), ["Marta", "Pepe"]) - - # test add object value - il.add_value("summary", {"key": 1}) - self.assertEqual(il.get_collected_values("summary"), [{"key": 1}]) - - il.add_value(None, "Jim", lambda x: {"name": x}) - self.assertEqual(il.get_collected_values("name"), ["Marta", "Pepe", "Jim"]) - - def test_add_zero(self): - il = NameItemLoader() - il.add_value("name", 0) - self.assertEqual(il.get_collected_values("name"), [0]) - - def test_replace_value(self): - il = ProcessorItemLoader() - il.replace_value("name", "marta") - self.assertEqual(il.get_collected_values("name"), ["Marta"]) - self.assertEqual(il.get_output_value("name"), ["Marta"]) - il.replace_value("name", "pepe") - self.assertEqual(il.get_collected_values("name"), ["Pepe"]) - self.assertEqual(il.get_output_value("name"), ["Pepe"]) - - il.replace_value(None, "Jim", lambda x: {"name": x}) - self.assertEqual(il.get_collected_values("name"), ["Jim"]) - - def test_get_value(self): - il = NameItemLoader() - self.assertEqual("FOO", il.get_value(["foo", "bar"], TakeFirst(), str.upper)) - self.assertEqual( - ["foo", "bar"], il.get_value(["name:foo", "name:bar"], re="name:(.*)$") - ) - self.assertEqual( - "foo", il.get_value(["name:foo", "name:bar"], TakeFirst(), re="name:(.*)$") - ) - - il.add_value("name", ["name:foo", "name:bar"], TakeFirst(), re="name:(.*)$") - self.assertEqual(["foo"], il.get_collected_values("name")) - il.replace_value("name", "name:bar", re="name:(.*)$") - self.assertEqual(["bar"], il.get_collected_values("name")) - - def test_iter_on_input_processor_input(self): - class NameFirstItemLoader(NameItemLoader): - name_in = TakeFirst() - - il = NameFirstItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_collected_values("name"), ["marta"]) - il = NameFirstItemLoader() - il.add_value("name", ["marta", "jose"]) - self.assertEqual(il.get_collected_values("name"), ["marta"]) - - il = NameFirstItemLoader() - il.replace_value("name", "marta") - self.assertEqual(il.get_collected_values("name"), ["marta"]) - il = NameFirstItemLoader() - il.replace_value("name", ["marta", "jose"]) - self.assertEqual(il.get_collected_values("name"), ["marta"]) - - il = NameFirstItemLoader() - il.add_value("name", "marta") - il.add_value("name", ["jose", "pedro"]) - self.assertEqual(il.get_collected_values("name"), ["marta", "jose"]) - - def test_map_compose_filter(self): - def filter_world(x): - return None if x == "world" else x - - proc = MapCompose(filter_world, str.upper) - self.assertEqual( - proc(["hello", "world", "this", "is", "scrapy"]), - ["HELLO", "THIS", "IS", "SCRAPY"], - ) - - def test_map_compose_filter_multil(self): - class TestItemLoader(NameItemLoader): - name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1]) - - il = TestItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["Mart"]) - item = il.load_item() - self.assertEqual(item["name"], ["Mart"]) - - def test_default_input_processor(self): - il = DefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["mart"]) - - def test_inherited_default_input_processor(self): - class InheritDefaultedItemLoader(DefaultedItemLoader): - pass - - il = InheritDefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["mart"]) - - def test_input_processor_inheritance(self): - class ChildItemLoader(ProcessorItemLoader): - url_in = MapCompose(lambda v: v.lower()) - - il = ChildItemLoader() - il.add_value("url", "HTTP://scrapy.ORG") - self.assertEqual(il.get_output_value("url"), ["http://scrapy.org"]) - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["Marta"]) - - class ChildChildItemLoader(ChildItemLoader): - url_in = MapCompose(lambda v: v.upper()) - summary_in = MapCompose(lambda v: v) - - il = ChildChildItemLoader() - il.add_value("url", "http://scrapy.org") - self.assertEqual(il.get_output_value("url"), ["HTTP://SCRAPY.ORG"]) - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["Marta"]) - - def test_empty_map_compose(self): - class IdentityDefaultedItemLoader(DefaultedItemLoader): - name_in = MapCompose() - - il = IdentityDefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["marta"]) - - def test_identity_input_processor(self): - class IdentityDefaultedItemLoader(DefaultedItemLoader): - name_in = Identity() - - il = IdentityDefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["marta"]) - - def test_extend_custom_input_processors(self): - class ChildItemLoader(ProcessorItemLoader): - name_in = MapCompose(ProcessorItemLoader.name_in, str.swapcase) - - il = ChildItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["mARTA"]) - - def test_extend_default_input_processors(self): - class ChildDefaultedItemLoader(DefaultedItemLoader): - name_in = MapCompose( - DefaultedItemLoader.default_input_processor, str.swapcase - ) - - il = ChildDefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["MART"]) - - def test_output_processor_using_function(self): - il = ProcessorItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - - class TakeFirstItemLoader(ProcessorItemLoader): - name_out = " ".join - - il = TakeFirstItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), "Mar Ta") - - def test_output_processor_error(self): - class TestItemLoader(ItemLoader): - default_item_class = SummaryItem - name_out = MapCompose(float) - - il = TestItemLoader() - il.add_value("name", ["$10"]) - try: - float("$10") - except Exception as e: - expected_exc_str = str(e) - - exc = None - try: - il.load_item() - except Exception as e: - exc = e - assert isinstance(exc, ValueError) - s = str(exc) - assert "name" in s, s - assert "$10" in s, s - assert "ValueError" in s, s - assert expected_exc_str in s, s - - def test_output_processor_using_classes(self): - il = ProcessorItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - - class TakeFirstItemLoader(ProcessorItemLoader): - name_out = Join() - - il = TakeFirstItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), "Mar Ta") - - class TakeFirstItemLoader2(ProcessorItemLoader): - name_out = Join("<br>") - - il = TakeFirstItemLoader2() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), "Mar<br>Ta") - - def test_default_output_processor(self): - il = ProcessorItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - - class LalaItemLoader(ProcessorItemLoader): - default_output_processor = Identity() - - il = LalaItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - - def test_loader_context_on_declaration(self): - class ChildItemLoader(ProcessorItemLoader): - url_in = MapCompose(processor_with_args, key="val") - - il = ChildItemLoader() - il.add_value("url", "text") - self.assertEqual(il.get_output_value("url"), ["val"]) - il.replace_value("url", "text2") - self.assertEqual(il.get_output_value("url"), ["val"]) - - def test_loader_context_on_instantiation(self): - class ChildItemLoader(ProcessorItemLoader): - url_in = MapCompose(processor_with_args) - - il = ChildItemLoader(key="val") - il.add_value("url", "text") - self.assertEqual(il.get_output_value("url"), ["val"]) - il.replace_value("url", "text2") - self.assertEqual(il.get_output_value("url"), ["val"]) - - def test_loader_context_on_assign(self): - class ChildItemLoader(ProcessorItemLoader): - url_in = MapCompose(processor_with_args) - - il = ChildItemLoader() - il.context["key"] = "val" - il.add_value("url", "text") - self.assertEqual(il.get_output_value("url"), ["val"]) - il.replace_value("url", "text2") - self.assertEqual(il.get_output_value("url"), ["val"]) - - def test_item_passed_to_input_processor_functions(self): - def processor(value, loader_context): - return loader_context["item"]["name"] - - class ChildItemLoader(ProcessorItemLoader): - url_in = MapCompose(processor) - - it = SummaryItem(name="marta") - il = ChildItemLoader(item=it) - il.add_value("url", "text") - self.assertEqual(il.get_output_value("url"), ["marta"]) - il.replace_value("url", "text2") - self.assertEqual(il.get_output_value("url"), ["marta"]) - - def test_compose_processor(self): - class TestItemLoader(NameItemLoader): - name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1]) - - il = TestItemLoader() - il.add_value("name", ["marta", "other"]) - self.assertEqual(il.get_output_value("name"), "Mart") - item = il.load_item() - self.assertEqual(item["name"], "Mart") - - def test_partial_processor(self): - def join(values, sep=None, loader_context=None, ignored=None): - if sep is not None: - return sep.join(values) - if loader_context and "sep" in loader_context: - return loader_context["sep"].join(values) - return "".join(values) - - class TestItemLoader(NameItemLoader): - name_out = Compose(partial(join, sep="+")) - url_out = Compose(partial(join, loader_context={"sep": "."})) - summary_out = Compose(partial(join, ignored="foo")) - - il = TestItemLoader() - il.add_value("name", ["rabbit", "hole"]) - il.add_value("url", ["rabbit", "hole"]) - il.add_value("summary", ["rabbit", "hole"]) - item = il.load_item() - self.assertEqual(item["name"], "rabbit+hole") - self.assertEqual(item["url"], "rabbit.hole") - self.assertEqual(item["summary"], "rabbithole") - - def test_error_input_processor(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - name_in = MapCompose(float) - - il = TestItemLoader() - with pytest.raises( - ValueError, - match="Error with input processor MapCompose: .* " - "error='ValueError: Error in MapCompose .* " - "error='ValueError: could not convert", - ): - il.add_value("name", ["marta", "other"]) - - def test_error_output_processor(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - name_out = Compose(Join(), float) - - il = TestItemLoader() - il.add_value("name", "marta") - with pytest.raises( - ValueError, - match="Error with output processor: .* " - "error='ValueError: Error in Compose .* " - "error='ValueError: could not convert", - ): - il.load_item() - - def test_error_processor_as_argument(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - - il = TestItemLoader() - with pytest.raises( - ValueError, - match=r"Error with processor Compose .* " - r"error='ValueError: Error in Compose .* " - r"error='TypeError: float\(\) argument", - ): - il.add_value("name", ["marta", "other"], Compose(float)) - - -class InitializationFromDictTest(unittest.TestCase): - item_class = dict - - def test_keep_single_value(self): - """Loaded item should contain values from the initial item""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo"]}) - - def test_keep_list(self): - """Loaded item should contain values from the initial item""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "bar"]}) - - def test_add_value_singlevalue_singlevalue(self): - """Values added after initialization should be appended""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - il.add_value("name", "bar") - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "bar"]}) - - def test_add_value_singlevalue_list(self): - """Values added after initialization should be appended""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - il.add_value("name", ["item", "loader"]) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "item", "loader"]}) - - def test_add_value_list_singlevalue(self): - """Values added after initialization should be appended""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - il.add_value("name", "qwerty") - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "bar", "qwerty"]}) - - def test_add_value_list_list(self): - """Values added after initialization should be appended""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - il.add_value("name", ["item", "loader"]) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "bar", "item", "loader"]}) - - def test_get_output_value_singlevalue(self): - """Getting output value must not remove value from item""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value("name"), ["foo"]) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, {"name": ["foo"]}) - - def test_get_output_value_list(self): - """Getting output value must not remove value from item""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value("name"), ["foo", "bar"]) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, {"name": ["foo", "bar"]}) - - def test_values_single(self): - """Values from initial item must be added to loader._values""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - self.assertEqual(il._values.get("name"), ["foo"]) - - def test_values_list(self): - """Values from initial item must be added to loader._values""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - self.assertEqual(il._values.get("name"), ["foo", "bar"]) - - -class BaseNoInputReprocessingLoader(ItemLoader): - title_in = MapCompose(str.upper) - title_out = TakeFirst() - - -class NoInputReprocessingDictLoader(BaseNoInputReprocessingLoader): - default_item_class = dict - - -class NoInputReprocessingFromDictTest(unittest.TestCase): - """ - Loaders initialized from loaded items must not reprocess fields (dict instances) - """ - - def test_avoid_reprocessing_with_initial_values_single(self): - il = NoInputReprocessingDictLoader(item={"title": "foo"}) - il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "foo"}) - self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"} - ) - - def test_avoid_reprocessing_with_initial_values_list(self): - il = NoInputReprocessingDictLoader(item={"title": ["foo", "bar"]}) - il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "foo"}) - self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"} - ) - - def test_avoid_reprocessing_without_initial_values_single(self): - il = NoInputReprocessingDictLoader() - il.add_value("title", "foo") - il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "FOO"}) - self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"} - ) - - def test_avoid_reprocessing_without_initial_values_list(self): - il = NoInputReprocessingDictLoader() - il.add_value("title", ["foo", "bar"]) - il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "FOO"}) - self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"} - ) - - -class TestOutputProcessorDict(unittest.TestCase): - def test_output_processor(self): - class TempDict(dict): - def __init__(self, *args, **kwargs): - super().__init__(self, *args, **kwargs) - self.setdefault("temp", 0.3) - - class TempLoader(ItemLoader): - default_item_class = TempDict - default_input_processor = Identity() - default_output_processor = Compose(TakeFirst()) - - loader = TempLoader() - item = loader.load_item() - self.assertIsInstance(item, TempDict) - self.assertEqual(dict(item), {"temp": 0.3}) - - -class ProcessorsTest(unittest.TestCase): - def test_take_first(self): - proc = TakeFirst() - self.assertEqual(proc([None, "", "hello", "world"]), "hello") - self.assertEqual(proc([None, "", 0, "hello", "world"]), 0) - - def test_identity(self): - proc = Identity() - self.assertEqual( - proc([None, "", "hello", "world"]), [None, "", "hello", "world"] - ) - - def test_join(self): - proc = Join() - with pytest.raises(TypeError): - proc([None, "", "hello", "world"]) - self.assertEqual(proc(["", "hello", "world"]), " hello world") - self.assertEqual(proc(["hello", "world"]), "hello world") - self.assertIsInstance(proc(["hello", "world"]), str) - - def test_compose(self): - proc = Compose(lambda v: v[0], str.upper) - self.assertEqual(proc(["hello", "world"]), "HELLO") - proc = Compose(str.upper) - self.assertEqual(proc(None), None) - proc = Compose(str.upper, stop_on_none=False) - with pytest.raises( - ValueError, - match="Error in Compose with .* error='TypeError: (descriptor 'upper'|'str' object expected)", - ): - proc(None) - proc = Compose(str.upper, lambda x: x + 1) - with pytest.raises( - ValueError, - match="Error in Compose with .* error='TypeError: (can only|unsupported operand)", - ): - proc("hello") - - def test_mapcompose(self): - def filter_world(x): - return None if x == "world" else x - - proc = MapCompose(filter_world, str.upper) - self.assertEqual( - proc(["hello", "world", "this", "is", "scrapy"]), - ["HELLO", "THIS", "IS", "SCRAPY"], - ) - proc = MapCompose(filter_world, str.upper) - self.assertEqual(proc(None), []) - proc = MapCompose(filter_world, str.upper) - with pytest.raises( - ValueError, - match="Error in MapCompose with .* error='TypeError: (descriptor 'upper'|'str' object expected)", - ): - proc([1]) - proc = MapCompose(filter_world, lambda x: x + 1) - with pytest.raises( - ValueError, - match="Error in MapCompose with .* error='TypeError: (can only|unsupported operand)", - ): - proc("hello") - - -class SelectJmesTestCase(unittest.TestCase): - test_list_equals = { - "simple": ("foo.bar", {"foo": {"bar": "baz"}}, "baz"), - "invalid": ("foo.bar.baz", {"foo": {"bar": "baz"}}, None), - "top_level": ("foo", {"foo": {"bar": "baz"}}, {"bar": "baz"}), - "double_vs_single_quote_string": ("foo.bar", {"foo": {"bar": "baz"}}, "baz"), - "dict": ( - "foo.bar[*].name", - {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, - ["one", "two"], - ), - "list": ("[1]", [1, 2], 2), - } - - def test_output(self): - for k, v in self.test_list_equals.items(): - expr, test_list, expected = v - test = SelectJmes(expr)(test_list) - self.assertEqual( - test, expected, msg=f'test "{k}" got {test} expected {expected}' - ) - - -# Functions as processors - - -def function_processor_strip(iterable): - return [x.strip() for x in iterable] - - -def function_processor_upper(iterable): - return [x.upper() for x in iterable] - - -class FunctionProcessorItem(Item): - foo = Field( - input_processor=function_processor_strip, - output_processor=function_processor_upper, - ) - - -class FunctionProcessorDictLoader(ItemLoader): - default_item_class = dict - foo_in = function_processor_strip - foo_out = function_processor_upper - - -class FunctionProcessorTestCase(unittest.TestCase): - def test_processor_defined_in_item_loader(self): - lo = FunctionProcessorDictLoader() - lo.add_value("foo", " bar ") - lo.add_value("foo", [" asdf ", " qwerty "]) - self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]}) From 044c3f69edd1bf926408649361ada7f2146db04e Mon Sep 17 00:00:00 2001 From: Mehraz Hossain Rumman <59512321+MehrazRumman@users.noreply.github.com> Date: Mon, 10 Mar 2025 01:18:57 +0600 Subject: [PATCH 068/491] Deprecate InitSpider (#6714) --- scrapy/spiders/init.py | 17 ++++++++++++++++- tests/test_spider.py | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index 4ec2919f7..a7dba989e 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,9 +1,11 @@ from __future__ import annotations +import warnings from collections.abc import Iterable from typing import TYPE_CHECKING, Any, cast from scrapy import Request +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.spiders import Spider from scrapy.utils.spider import iterate_spider_output @@ -12,7 +14,20 @@ if TYPE_CHECKING: class InitSpider(Spider): - """Base Spider with initialization facilities""" + """Base Spider with initialization facilities + + .. warning:: This class is deprecated. Copy its code into your project if needed. + It will be removed in a future Scrapy version. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + warnings.warn( + "InitSpider is deprecated. Copy its code from Scrapy's source if needed. " + "Will be removed in a future version.", + ScrapyDeprecationWarning, + stacklevel=2, + ) def start_requests(self) -> Iterable[Request]: self._postinit_reqs: Iterable[Request] = super().start_requests() diff --git a/tests/test_spider.py b/tests/test_spider.py index 05f1c59d0..4e8330c06 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -144,6 +144,7 @@ class TestSpider(unittest.TestCase): mock_logger.log.assert_called_once_with("INFO", "test log msg") +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestInitSpider(TestSpider): spider_class = InitSpider From 02ed71d8877d1f3f270a9085c3cdb7fc7e917b8a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 9 Mar 2025 23:20:24 +0400 Subject: [PATCH 069/491] Converting tests to plain asserts, part 6. (#6709) --- tests/test_item.py | 97 ++-- tests/test_link.py | 12 +- tests/test_linkextractors.py | 937 +++++++++++++++------------------- tests/test_pipeline_crawl.py | 46 +- tests/test_pipeline_files.py | 213 ++++---- tests/test_pipeline_images.py | 204 ++++---- tests/test_pipeline_media.py | 149 +++--- 7 files changed, 730 insertions(+), 928 deletions(-) diff --git a/tests/test_item.py b/tests/test_item.py index 47c5c3db6..bf51eb398 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,4 +1,3 @@ -import unittest from abc import ABCMeta from unittest import mock @@ -7,9 +6,9 @@ import pytest from scrapy.item import Field, Item, ItemMeta -class ItemTest(unittest.TestCase): +class TestItem: def assertSortedEqual(self, first, second, msg=None): - return self.assertEqual(sorted(first), sorted(second), msg) + assert sorted(first) == sorted(second), msg def test_simple(self): class TestItem(Item): @@ -17,7 +16,7 @@ class ItemTest(unittest.TestCase): i = TestItem() i["name"] = "name" - self.assertEqual(i["name"], "name") + assert i["name"] == "name" def test_init(self): class TestItem(Item): @@ -28,13 +27,13 @@ class ItemTest(unittest.TestCase): i["name"] i2 = TestItem(name="john doe") - self.assertEqual(i2["name"], "john doe") + assert i2["name"] == "john doe" i3 = TestItem({"name": "john doe"}) - self.assertEqual(i3["name"], "john doe") + assert i3["name"] == "john doe" i4 = TestItem(i3) - self.assertEqual(i4["name"], "john doe") + assert i4["name"] == "john doe" with pytest.raises(KeyError): TestItem({"name": "john doe", "other": "foo"}) @@ -59,11 +58,11 @@ class ItemTest(unittest.TestCase): i["number"] = 123 itemrepr = repr(i) - self.assertEqual(itemrepr, "{'name': 'John Doe', 'number': 123}") + assert itemrepr == "{'name': 'John Doe', 'number': 123}" i2 = eval(itemrepr) # pylint: disable=eval-used - self.assertEqual(i2["name"], "John Doe") - self.assertEqual(i2["number"], 123) + assert i2["name"] == "John Doe" + assert i2["number"] == 123 def test_private_attr(self): class TestItem(Item): @@ -71,7 +70,7 @@ class ItemTest(unittest.TestCase): i = TestItem() i._private = "test" - self.assertEqual(i._private, "test") + assert i._private == "test" def test_raise_getattr(self): class TestItem(Item): @@ -103,9 +102,9 @@ class ItemTest(unittest.TestCase): with pytest.raises(KeyError): i.get_name() i["name"] = "lala" - self.assertEqual(i.get_name(), "lala") + assert i.get_name() == "lala" i.change_name("other") - self.assertEqual(i.get_name(), "other") + assert i.get_name() == "other" def test_metaclass(self): class TestItem(Item): @@ -115,8 +114,8 @@ class ItemTest(unittest.TestCase): i = TestItem() i["name"] = "John" - self.assertEqual(list(i.keys()), ["name"]) - self.assertEqual(list(i.values()), ["John"]) + assert list(i.keys()) == ["name"] + assert list(i.values()) == ["John"] i["keys"] = "Keys" i["values"] = "Values" @@ -142,8 +141,8 @@ class ItemTest(unittest.TestCase): i = TestItem() i["keys"] = 3 - self.assertEqual(list(i.keys()), ["keys"]) - self.assertEqual(list(i.values()), [3]) + assert list(i.keys()) == ["keys"] + assert list(i.values()) == [3] def test_metaclass_multiple_inheritance_simple(self): class A(Item): @@ -161,17 +160,17 @@ class ItemTest(unittest.TestCase): pass item = D(save="X", load="Y") - self.assertEqual(item["save"], "X") - self.assertEqual(item["load"], "Y") - self.assertEqual(D.fields, {"load": {"default": "A"}, "save": {"default": "A"}}) + assert item["save"] == "X" + assert item["load"] == "Y" + assert D.fields == {"load": {"default": "A"}, "save": {"default": "A"}} # D class inverted class E(C, B): pass - self.assertEqual(E(save="X")["save"], "X") - self.assertEqual(E(load="X")["load"], "X") - self.assertEqual(E.fields, {"load": {"default": "C"}, "save": {"default": "C"}}) + assert E(save="X")["save"] == "X" + assert E(load="X")["load"] == "X" + assert E.fields == {"load": {"default": "C"}, "save": {"default": "C"}} def test_metaclass_multiple_inheritance_diamond(self): class A(Item): @@ -190,31 +189,25 @@ class ItemTest(unittest.TestCase): fields = {"update": Field(default="D")} load = Field(default="D") - self.assertEqual(D(save="X")["save"], "X") - self.assertEqual(D(load="X")["load"], "X") - self.assertEqual( - D.fields, - { - "save": {"default": "C"}, - "load": {"default": "D"}, - "update": {"default": "D"}, - }, - ) + assert D(save="X")["save"] == "X" + assert D(load="X")["load"] == "X" + assert D.fields == { + "save": {"default": "C"}, + "load": {"default": "D"}, + "update": {"default": "D"}, + } # D class inverted class E(C, B): load = Field(default="E") - self.assertEqual(E(save="X")["save"], "X") - self.assertEqual(E(load="X")["load"], "X") - self.assertEqual( - E.fields, - { - "save": {"default": "C"}, - "load": {"default": "E"}, - "update": {"default": "C"}, - }, - ) + assert E(save="X")["save"] == "X" + assert E(load="X")["load"] == "X" + assert E.fields == { + "save": {"default": "C"}, + "load": {"default": "E"}, + "update": {"default": "C"}, + } def test_metaclass_multiple_inheritance_without_metaclass(self): class A(Item): @@ -234,8 +227,8 @@ class ItemTest(unittest.TestCase): with pytest.raises(KeyError): D(not_allowed="value") - self.assertEqual(D(save="X")["save"], "X") - self.assertEqual(D.fields, {"save": {"default": "A"}, "load": {"default": "A"}}) + assert D(save="X")["save"] == "X" + assert D.fields == {"save": {"default": "A"}, "load": {"default": "A"}} # D class inverted class E(C, B): @@ -243,8 +236,8 @@ class ItemTest(unittest.TestCase): with pytest.raises(KeyError): E(not_allowed="value") - self.assertEqual(E(save="X")["save"], "X") - self.assertEqual(E.fields, {"save": {"default": "A"}, "load": {"default": "A"}}) + assert E(save="X")["save"] == "X" + assert E.fields == {"save": {"default": "A"}, "load": {"default": "A"}} def test_to_dict(self): class TestItem(Item): @@ -252,7 +245,7 @@ class ItemTest(unittest.TestCase): i = TestItem() i["name"] = "John" - self.assertEqual(dict(i), {"name": "John"}) + assert dict(i) == {"name": "John"} def test_copy(self): class TestItem(Item): @@ -260,9 +253,9 @@ class ItemTest(unittest.TestCase): item = TestItem({"name": "lower"}) copied_item = item.copy() - self.assertNotEqual(id(item), id(copied_item)) + assert id(item) != id(copied_item) copied_item["name"] = copied_item["name"].upper() - self.assertNotEqual(item["name"], copied_item["name"]) + assert item["name"] != copied_item["name"] def test_deepcopy(self): class TestItem(Item): @@ -274,7 +267,7 @@ class ItemTest(unittest.TestCase): assert item["tags"] != copied_item["tags"] -class ItemMetaTest(unittest.TestCase): +class TestItemMeta: def test_new_method_propagates_classcell(self): new_mock = mock.Mock(side_effect=ABCMeta.__new__) base = ItemMeta.__bases__[0] @@ -297,7 +290,7 @@ class ItemMetaTest(unittest.TestCase): assert "__classcell__" in attrs -class ItemMetaClassCellRegression(unittest.TestCase): +class TestItemMetaClassCellRegression: def test_item_meta_classcell_regression(self): class MyItem(Item, metaclass=ItemMeta): def __init__(self, *args, **kwargs): # pylint: disable=useless-parent-delegation diff --git a/tests/test_link.py b/tests/test_link.py index ed9d27a37..f96961075 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -1,18 +1,16 @@ -import unittest - import pytest from scrapy.link import Link -class LinkTest(unittest.TestCase): +class TestLink: def _assert_same_links(self, link1, link2): - self.assertEqual(link1, link2) - self.assertEqual(hash(link1), hash(link2)) + assert link1 == link2 + assert hash(link1) == hash(link2) def _assert_different_links(self, link1, link2): - self.assertNotEqual(link1, link2) - self.assertNotEqual(hash(link1), hash(link2)) + assert link1 != link2 + assert hash(link1) != hash(link2) def test_eq_and_hash(self): l1 = Link("http://www.example.com") diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index e751e0a63..1bff369af 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -2,7 +2,6 @@ from __future__ import annotations import pickle import re -import unittest import pytest from packaging.version import Version @@ -16,175 +15,139 @@ from tests import get_testdata # a hack to skip base class tests in pytest class Base: - class LinkExtractorTestCase(unittest.TestCase): + class TestLinkExtractorBase: extractor_cls: type | None = None - def setUp(self): + def setup_method(self): body = get_testdata("link_extractor", "linkextractor.html") self.response = HtmlResponse(url="http://example.com/index", body=body) def test_urls_type(self): """Test that the resulting urls are str objects""" lx = self.extractor_cls() - self.assertTrue( - all( - isinstance(link.url, str) - for link in lx.extract_links(self.response) - ) + assert all( + isinstance(link.url, str) for link in lx.extract_links(self.response) ) def test_extract_all_links(self): lx = self.extractor_cls() page4_url = "http://example.com/page%204.html" - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - Link(url="http://www.google.com/something", text=""), - Link(url="http://example.com/innertag.html", text="inner tag"), - Link(url=page4_url, text="href with whitespaces"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + Link(url="http://www.google.com/something", text=""), + Link(url="http://example.com/innertag.html", text="inner tag"), + Link(url=page4_url, text="href with whitespaces"), + ] def test_extract_filter_allow(self): lx = self.extractor_cls(allow=("sample",)) - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + ] def test_extract_filter_allow_with_duplicates(self): lx = self.extractor_cls(allow=("sample",), unique=False) - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html", - text="sample 3 repetition", - ), - Link( - url="http://example.com/sample3.html", - text="sample 3 repetition", - ), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition", + ), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition", + ), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + ] def test_extract_filter_allow_with_duplicates_canonicalize(self): lx = self.extractor_cls(allow=("sample",), unique=False, canonicalize=True) - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html", - text="sample 3 repetition", - ), - Link( - url="http://example.com/sample3.html", - text="sample 3 repetition", - ), - Link( - url="http://example.com/sample3.html", - text="sample 3 repetition with fragment", - ), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition", + ), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition", + ), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition with fragment", + ), + ] def test_extract_filter_allow_no_duplicates_canonicalize(self): lx = self.extractor_cls(allow=("sample",), unique=True, canonicalize=True) - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + ] def test_extract_filter_allow_and_deny(self): lx = self.extractor_cls(allow=("sample",), deny=("3",)) - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + ] def test_extract_filter_allowed_domains(self): lx = self.extractor_cls(allow_domains=("google.com",)) - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://www.google.com/something", text=""), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://www.google.com/something", text=""), + ] def test_extraction_using_single_values(self): """Test the extractor's behaviour among different situations""" lx = self.extractor_cls(allow="sample") - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + ] lx = self.extractor_cls(allow="sample", deny="3") - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + ] lx = self.extractor_cls(allow_domains="google.com") - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://www.google.com/something", text=""), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://www.google.com/something", text=""), + ] lx = self.extractor_cls(deny_domains="example.com") - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://www.google.com/something", text=""), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://www.google.com/something", text=""), + ] def test_nofollow(self): """Test the extractor's behaviour for links with rel='nofollow'""" @@ -210,47 +173,44 @@ class Base: response = HtmlResponse("http://example.org/somepage/index.html", body=html) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.org/about.html", text="About us"), - Link(url="http://example.org/follow.html", text="Follow this link"), - Link( - url="http://example.org/nofollow.html", - text="Dont follow this one", - nofollow=True, - ), - Link( - url="http://example.org/nofollow2.html", - text="Choose to follow or not", - ), - Link( - url="http://google.com/something", - text="External link not to follow", - nofollow=True, - ), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.org/about.html", text="About us"), + Link(url="http://example.org/follow.html", text="Follow this link"), + Link( + url="http://example.org/nofollow.html", + text="Dont follow this one", + nofollow=True, + ), + Link( + url="http://example.org/nofollow2.html", + text="Choose to follow or not", + ), + Link( + url="http://google.com/something", + text="External link not to follow", + nofollow=True, + ), + ] def test_matches(self): url1 = "http://lotsofstuff.com/stuff1/index" url2 = "http://evenmorestuff.com/uglystuff/index" lx = self.extractor_cls(allow=(r"stuff1",)) - self.assertTrue(lx.matches(url1)) - self.assertFalse(lx.matches(url2)) + assert lx.matches(url1) + assert not lx.matches(url2) lx = self.extractor_cls(deny=(r"uglystuff",)) - self.assertTrue(lx.matches(url1)) - self.assertFalse(lx.matches(url2)) + assert lx.matches(url1) + assert not lx.matches(url2) lx = self.extractor_cls(allow_domains=("evenmorestuff.com",)) - self.assertFalse(lx.matches(url1)) - self.assertTrue(lx.matches(url2)) + assert not lx.matches(url1) + assert lx.matches(url2) lx = self.extractor_cls(deny_domains=("lotsofstuff.com",)) - self.assertFalse(lx.matches(url1)) - self.assertTrue(lx.matches(url2)) + assert not lx.matches(url1) + assert lx.matches(url2) lx = self.extractor_cls( allow=["blah1"], @@ -258,20 +218,17 @@ class Base: allow_domains=["blah1.com"], deny_domains=["blah2.com"], ) - self.assertTrue(lx.matches("http://blah1.com/blah1")) - self.assertFalse(lx.matches("http://blah1.com/blah2")) - self.assertFalse(lx.matches("http://blah2.com/blah1")) - self.assertFalse(lx.matches("http://blah2.com/blah2")) + assert lx.matches("http://blah1.com/blah1") + assert not lx.matches("http://blah1.com/blah2") + assert not lx.matches("http://blah2.com/blah1") + assert not lx.matches("http://blah2.com/blah2") def test_restrict_xpaths(self): lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]',)) - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + ] def test_restrict_xpaths_encoding(self): """Test restrict_xpaths with encodings""" @@ -291,10 +248,9 @@ class Base: ) lx = self.extractor_cls(restrict_xpaths="//div[@class='links']") - self.assertEqual( - lx.extract_links(response), - [Link(url="http://example.org/about.html", text="About us\xa3")], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.org/about.html", text="About us\xa3") + ] def test_restrict_xpaths_with_html_entities(self): html = b'<html><body><p><a href="/♥/you?c=€">text</a></p></body></html>' @@ -304,47 +260,40 @@ class Base: encoding="iso8859-15", ) links = self.extractor_cls(restrict_xpaths="//p").extract_links(response) - self.assertEqual( - links, [Link(url="http://example.org/%E2%99%A5/you?c=%A4", text="text")] - ) + assert links == [ + Link(url="http://example.org/%E2%99%A5/you?c=%A4", text="text") + ] def test_restrict_xpaths_concat_in_handle_data(self): """html entities cause SGMLParser to call handle_data hook twice""" body = b"""<html><body><div><a href="/foo">>\xbe\xa9<\xb6\xab</a></body></html>""" response = HtmlResponse("http://example.org", body=body, encoding="gb18030") lx = self.extractor_cls(restrict_xpaths="//div") - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.org/foo", - text=">\u4eac<\u4e1c", - fragment="", - nofollow=False, - ) - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.org/foo", + text=">\u4eac<\u4e1c", + fragment="", + nofollow=False, + ) + ] def test_restrict_css(self): lx = self.extractor_cls(restrict_css=("#subwrapper a",)) - self.assertEqual( - lx.extract_links(self.response), - [Link(url="http://example.com/sample2.html", text="sample 2")], - ) + assert lx.extract_links(self.response) == [ + Link(url="http://example.com/sample2.html", text="sample 2") + ] def test_restrict_css_and_restrict_xpaths_together(self): lx = self.extractor_cls( restrict_xpaths=('//div[@id="subwrapper"]',), restrict_css=("#subwrapper + a",), ) - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + ] def test_area_tag_with_unicode_present(self): body = b"""<html><body>\xbe\xa9<map><area href="http://example.org/foo" /></map></body></html>""" @@ -353,17 +302,14 @@ class Base: lx.extract_links(response) lx.extract_links(response) lx.extract_links(response) - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.org/foo", - text="", - fragment="", - nofollow=False, - ) - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.org/foo", + text="", + fragment="", + nofollow=False, + ) + ] def test_encoded_url(self): body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>""" @@ -371,17 +317,14 @@ class Base: "http://known.fm/AC%2FDC/", body=body, encoding="utf8" ) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://known.fm/AC%2FDC/?page=2", - text="BinB", - fragment="", - nofollow=False, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://known.fm/AC%2FDC/?page=2", + text="BinB", + fragment="", + nofollow=False, + ), + ] def test_encoded_url_in_restricted_xpath(self): body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>""" @@ -389,38 +332,29 @@ class Base: "http://known.fm/AC%2FDC/", body=body, encoding="utf8" ) lx = self.extractor_cls(restrict_xpaths="//div") - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://known.fm/AC%2FDC/?page=2", - text="BinB", - fragment="", - nofollow=False, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://known.fm/AC%2FDC/?page=2", + text="BinB", + fragment="", + nofollow=False, + ), + ] def test_ignored_extensions(self): # jpg is ignored by default html = b"""<a href="page.html">asd</a> and <a href="photo.jpg">""" response = HtmlResponse("http://example.org/", body=html) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.org/page.html", text="asd"), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.org/page.html", text="asd"), + ] # override denied extensions lx = self.extractor_cls(deny_extensions=["html"]) - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.org/photo.jpg"), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.org/photo.jpg"), + ] def test_process_value(self): """Test restrict_xpaths with encodings""" @@ -439,10 +373,9 @@ class Base: return m.group(1) if m else None lx = self.extractor_cls(process_value=process_value) - self.assertEqual( - lx.extract_links(response), - [Link(url="http://example.org/other/page.html", text="Text")], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.org/other/page.html", text="Text") + ] def test_base_url_with_restrict_xpaths(self): html = b"""<html><head><title>Page title<title><base href="http://otherdomain.com/base/" /> @@ -450,53 +383,46 @@ class Base: </body></html>""" response = HtmlResponse("http://example.org/somepage/index.html", body=html) lx = self.extractor_cls(restrict_xpaths="//p") - self.assertEqual( - lx.extract_links(response), - [Link(url="http://otherdomain.com/base/item/12.html", text="Item 12")], - ) + assert lx.extract_links(response) == [ + Link(url="http://otherdomain.com/base/item/12.html", text="Item 12") + ] def test_attrs(self): lx = self.extractor_cls(attrs="href") page4_url = "http://example.com/page%204.html" - self.assertEqual( - lx.extract_links(self.response), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - Link(url="http://www.google.com/something", text=""), - Link(url="http://example.com/innertag.html", text="inner tag"), - Link(url=page4_url, text="href with whitespaces"), - ], - ) + assert lx.extract_links(self.response) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + Link(url="http://www.google.com/something", text=""), + Link(url="http://example.com/innertag.html", text="inner tag"), + Link(url=page4_url, text="href with whitespaces"), + ] lx = self.extractor_cls( attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=() ) - self.assertEqual( - lx.extract_links(self.response), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample2.jpg", text=""), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - Link(url="http://www.google.com/something", text=""), - Link(url="http://example.com/innertag.html", text="inner tag"), - Link(url=page4_url, text="href with whitespaces"), - ], - ) + assert lx.extract_links(self.response) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample2.jpg", text=""), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + Link(url="http://www.google.com/something", text=""), + Link(url="http://example.com/innertag.html", text="inner tag"), + Link(url=page4_url, text="href with whitespaces"), + ] lx = self.extractor_cls(attrs=None) - self.assertEqual(lx.extract_links(self.response), []) + assert lx.extract_links(self.response) == [] def test_tags(self): html = ( @@ -506,43 +432,31 @@ class Base: response = HtmlResponse("http://example.com/index.html", body=html) lx = self.extractor_cls(tags=None) - self.assertEqual(lx.extract_links(response), []) + assert lx.extract_links(response) == [] lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + ] lx = self.extractor_cls(tags="area") - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.com/sample1.html", text=""), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.com/sample1.html", text=""), + ] lx = self.extractor_cls(tags="a") - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.com/sample2.html", text="sample 2"), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.com/sample2.html", text="sample 2"), + ] lx = self.extractor_cls( tags=("a", "img"), attrs=("href", "src"), deny_extensions=() ) - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample2.jpg", text=""), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample2.jpg", text=""), + ] def test_tags_attrs(self): html = b""" @@ -554,42 +468,36 @@ class Base: response = HtmlResponse("http://example.com/index.html", body=html) lx = self.extractor_cls(tags="div", attrs="data-url") - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.com/get?id=1", - text="Item 1", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/get?id=2", - text="Item 2", - fragment="", - nofollow=False, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.com/get?id=1", + text="Item 1", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/get?id=2", + text="Item 2", + fragment="", + nofollow=False, + ), + ] lx = self.extractor_cls(tags=("div",), attrs=("data-url",)) - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.com/get?id=1", - text="Item 1", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/get?id=2", - text="Item 2", - fragment="", - nofollow=False, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.com/get?id=1", + text="Item 1", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/get?id=2", + text="Item 2", + fragment="", + nofollow=False, + ), + ] def test_xhtml(self): xhtml = b""" @@ -623,78 +531,72 @@ class Base: response = HtmlResponse("http://example.com/index.xhtml", body=xhtml) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.com/about.html", - text="About us", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/follow.html", - text="Follow this link", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/nofollow.html", - text="Dont follow this one", - fragment="", - nofollow=True, - ), - Link( - url="http://example.com/nofollow2.html", - text="Choose to follow or not", - fragment="", - nofollow=False, - ), - Link( - url="http://google.com/something", - text="External link not to follow", - nofollow=True, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.com/about.html", + text="About us", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/follow.html", + text="Follow this link", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/nofollow.html", + text="Dont follow this one", + fragment="", + nofollow=True, + ), + Link( + url="http://example.com/nofollow2.html", + text="Choose to follow or not", + fragment="", + nofollow=False, + ), + Link( + url="http://google.com/something", + text="External link not to follow", + nofollow=True, + ), + ] response = XmlResponse("http://example.com/index.xhtml", body=xhtml) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.com/about.html", - text="About us", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/follow.html", - text="Follow this link", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/nofollow.html", - text="Dont follow this one", - fragment="", - nofollow=True, - ), - Link( - url="http://example.com/nofollow2.html", - text="Choose to follow or not", - fragment="", - nofollow=False, - ), - Link( - url="http://google.com/something", - text="External link not to follow", - nofollow=True, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.com/about.html", + text="About us", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/follow.html", + text="Follow this link", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/nofollow.html", + text="Dont follow this one", + fragment="", + nofollow=True, + ), + Link( + url="http://example.com/nofollow2.html", + text="Choose to follow or not", + fragment="", + nofollow=False, + ), + Link( + url="http://google.com/something", + text="External link not to follow", + nofollow=True, + ), + ] def test_link_wrong_href(self): html = b""" @@ -704,21 +606,18 @@ class Base: """ response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() - self.assertEqual( - list(lx.extract_links(response)), - [ - Link( - url="http://example.org/item1.html", - text="Item 1", - nofollow=False, - ), - Link( - url="http://example.org/item3.html", - text="Item 3", - nofollow=False, - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link( + url="http://example.org/item1.html", + text="Item 1", + nofollow=False, + ), + Link( + url="http://example.org/item3.html", + text="Item 3", + nofollow=False, + ), + ] def test_ftp_links(self): body = b""" @@ -729,21 +628,18 @@ class Base: "http://www.example.com/index.html", body=body, encoding="utf8" ) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="ftp://www.external.com/", - text="An Item", - fragment="", - nofollow=False, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="ftp://www.external.com/", + text="An Item", + fragment="", + nofollow=False, + ), + ] def test_pickle_extractor(self): lx = self.extractor_cls() - self.assertIsInstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls) + assert isinstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls) def test_link_extractor_aggregation(self): """When a parameter like restrict_css is used, the underlying @@ -770,14 +666,11 @@ class Base: """, ) actual = lx.extract_links(response) - self.assertEqual( - actual, - [ - Link(url="https://example.com/a", text="a1"), - Link(url="https://example.com/b?a=1&b=2", text="b1"), - Link(url="https://example.com/b?b=2&a=1", text="b2"), - ], - ) + assert actual == [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/b?b=2&a=1", text="b2"), + ] # unique=True (default), canonicalize=True lx = self.extractor_cls(restrict_css=("div",), canonicalize=True) @@ -795,13 +688,10 @@ class Base: """, ) actual = lx.extract_links(response) - self.assertEqual( - actual, - [ - Link(url="https://example.com/a", text="a1"), - Link(url="https://example.com/b?a=1&b=2", text="b1"), - ], - ) + assert actual == [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + ] # unique=False, canonicalize=False (default) lx = self.extractor_cls(restrict_css=("div",), unique=False) @@ -819,15 +709,12 @@ class Base: """, ) actual = lx.extract_links(response) - self.assertEqual( - actual, - [ - Link(url="https://example.com/a", text="a1"), - Link(url="https://example.com/b?a=1&b=2", text="b1"), - Link(url="https://example.com/a", text="a2"), - Link(url="https://example.com/b?b=2&a=1", text="b2"), - ], - ) + assert actual == [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/a", text="a2"), + Link(url="https://example.com/b?b=2&a=1", text="b2"), + ] # unique=False, canonicalize=True lx = self.extractor_cls( @@ -847,18 +734,15 @@ class Base: """, ) actual = lx.extract_links(response) - self.assertEqual( - actual, - [ - Link(url="https://example.com/a", text="a1"), - Link(url="https://example.com/b?a=1&b=2", text="b1"), - Link(url="https://example.com/a", text="a2"), - Link(url="https://example.com/b?a=1&b=2", text="b2"), - ], - ) + assert actual == [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/a", text="a2"), + Link(url="https://example.com/b?a=1&b=2", text="b2"), + ] -class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): +class TestLxmlLinkExtractor(Base.TestLinkExtractorBase): extractor_cls = LxmlLinkExtractor def test_link_wrong_href(self): @@ -869,17 +753,10 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): """ response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() - self.assertEqual( - list(lx.extract_links(response)), - [ - Link( - url="http://example.org/item1.html", text="Item 1", nofollow=False - ), - Link( - url="http://example.org/item3.html", text="Item 3", nofollow=False - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link(url="http://example.org/item1.html", text="Item 1", nofollow=False), + Link(url="http://example.org/item3.html", text="Item 3", nofollow=False), + ] def test_link_restrict_text(self): html = b""" @@ -890,45 +767,36 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): response = HtmlResponse("http://example.org/index.html", body=html) # Simple text inclusion test lx = self.extractor_cls(restrict_text="dog") - self.assertEqual( - list(lx.extract_links(response)), - [ - Link( - url="http://example.org/item2.html", - text="Pic of a dog", - nofollow=False, - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link( + url="http://example.org/item2.html", + text="Pic of a dog", + nofollow=False, + ), + ] # Unique regex test lx = self.extractor_cls(restrict_text=r"of.*dog") - self.assertEqual( - list(lx.extract_links(response)), - [ - Link( - url="http://example.org/item2.html", - text="Pic of a dog", - nofollow=False, - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link( + url="http://example.org/item2.html", + text="Pic of a dog", + nofollow=False, + ), + ] # Multiple regex test lx = self.extractor_cls(restrict_text=[r"of.*dog", r"of.*cat"]) - self.assertEqual( - list(lx.extract_links(response)), - [ - Link( - url="http://example.org/item1.html", - text="Pic of a cat", - nofollow=False, - ), - Link( - url="http://example.org/item2.html", - text="Pic of a dog", - nofollow=False, - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link( + url="http://example.org/item1.html", + text="Pic of a cat", + nofollow=False, + ), + Link( + url="http://example.org/item2.html", + text="Pic of a dog", + nofollow=False, + ), + ] @pytest.mark.skipif( Version(w3lib_version) < Version("2.0.0"), @@ -945,30 +813,27 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): """ response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() - self.assertEqual( - list(lx.extract_links(response)), - [ - Link( - url="http://example.org/item2.html", - text="Good Link", - nofollow=False, - ), - Link( - url="http://example.org/item3.html", - text="Good Link 2", - nofollow=False, - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link( + url="http://example.org/item2.html", + text="Good Link", + nofollow=False, + ), + Link( + url="http://example.org/item3.html", + text="Good Link 2", + nofollow=False, + ), + ] def test_link_allowed_is_false_with_empty_url(self): bad_link = Link("") - self.assertFalse(LxmlLinkExtractor()._link_allowed(bad_link)) + assert not LxmlLinkExtractor()._link_allowed(bad_link) def test_link_allowed_is_false_with_bad_url_prefix(self): bad_link = Link("htp://should_be_http.example") - self.assertFalse(LxmlLinkExtractor()._link_allowed(bad_link)) + assert not LxmlLinkExtractor()._link_allowed(bad_link) def test_link_allowed_is_false_with_missing_url_prefix(self): bad_link = Link("should_have_prefix.example") - self.assertFalse(LxmlLinkExtractor()._link_allowed(bad_link)) + assert not LxmlLinkExtractor()._link_allowed(bad_link) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 84d714e5c..162dfdaf4 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -53,7 +53,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider): ) -class FileDownloadCrawlTestCase(TestCase): +class TestFileDownloadCrawl(TestCase): pipeline_class = "scrapy.pipelines.files.FilesPipeline" store_setting_key = "FILES_STORE" media_key = "files" @@ -98,52 +98,46 @@ class FileDownloadCrawlTestCase(TestCase): return crawler def _assert_files_downloaded(self, items, logs): - self.assertEqual(len(items), 1) - self.assertIn(self.media_key, items[0]) + assert len(items) == 1 + assert self.media_key in items[0] # check that logs show the expected number of successful file downloads file_dl_success = "File (downloaded): Downloaded file from" - self.assertEqual(logs.count(file_dl_success), 3) + assert logs.count(file_dl_success) == 3 # check that the images/files status is `downloaded` for item in items: for i in item[self.media_key]: - self.assertEqual(i["status"], "downloaded") + assert i["status"] == "downloaded" # check that the images/files checksums are what we know they should be if self.expected_checksums is not None: checksums = {i["checksum"] for item in items for i in item[self.media_key]} - self.assertEqual(checksums, self.expected_checksums) + assert checksums == self.expected_checksums # check that the image files where actually written to the media store for item in items: for i in item[self.media_key]: - self.assertTrue((self.tmpmediastore / i["path"]).exists()) + assert (self.tmpmediastore / i["path"]).exists() def _assert_files_download_failure(self, crawler, items, code, logs): # check that the item does NOT have the "images/files" field populated - self.assertEqual(len(items), 1) - self.assertIn(self.media_key, items[0]) - self.assertFalse(items[0][self.media_key]) + assert len(items) == 1 + assert self.media_key in items[0] + assert not items[0][self.media_key] # check that there was 1 successful fetch and 3 other responses with non-200 code - self.assertEqual( - crawler.stats.get_value("downloader/request_method_count/GET"), 4 - ) - self.assertEqual(crawler.stats.get_value("downloader/response_count"), 4) - self.assertEqual( - crawler.stats.get_value("downloader/response_status_count/200"), 1 - ) - self.assertEqual( - crawler.stats.get_value(f"downloader/response_status_count/{code}"), 3 - ) + assert crawler.stats.get_value("downloader/request_method_count/GET") == 4 + assert crawler.stats.get_value("downloader/response_count") == 4 + assert crawler.stats.get_value("downloader/response_status_count/200") == 1 + assert crawler.stats.get_value(f"downloader/response_status_count/{code}") == 3 # check that logs do show the failure on the file downloads file_dl_failure = f"File (code: {code}): Error downloading file from" - self.assertEqual(logs.count(file_dl_failure), 3) + assert logs.count(file_dl_failure) == 3 # check that no files were written to the media store - self.assertEqual(list(self.tmpmediastore.iterdir()), []) + assert not list(self.tmpmediastore.iterdir()) @defer.inlineCallbacks def test_download_media(self): @@ -193,9 +187,7 @@ class FileDownloadCrawlTestCase(TestCase): mockserver=self.mockserver, ) self._assert_files_downloaded(self.items, str(log)) - self.assertEqual( - crawler.stats.get_value("downloader/response_status_count/302"), 3 - ) + assert crawler.stats.get_value("downloader/response_status_count/302") == 3 @defer.inlineCallbacks def test_download_media_file_path_error(self): @@ -218,7 +210,7 @@ class FileDownloadCrawlTestCase(TestCase): media_urls_key=self.media_urls_key, mockserver=self.mockserver, ) - self.assertIn("ZeroDivisionError", str(log)) + assert "ZeroDivisionError" in str(log) skip_pillow: str | None @@ -230,7 +222,7 @@ else: skip_pillow = None -class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase): +class ImageDownloadCrawlTestCase(TestFileDownloadCrawl): skip = skip_pillow pipeline_class = "scrapy.pipelines.images.ImagesPipeline" diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 05fd17207..e515c16a0 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -77,7 +77,7 @@ def get_ftp_content_and_delete( return b"".join(ftp_data) -class FilesPipelineTestCase(unittest.TestCase): +class TestFilesPipeline(unittest.TestCase): def setUp(self): self.tempdir = mkdtemp() settings_dict = {"FILES_STORE": self.tempdir} @@ -91,73 +91,73 @@ class FilesPipelineTestCase(unittest.TestCase): def test_file_path(self): file_path = self.pipeline.file_path - self.assertEqual( - file_path(Request("https://dev.mydeco.com/mydeco.pdf")), - "full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf", + assert ( + file_path(Request("https://dev.mydeco.com/mydeco.pdf")) + == "full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf" ) - self.assertEqual( + assert ( file_path( Request( "http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt" ) - ), - "full/4ce274dd83db0368bafd7e406f382ae088e39219.txt", + ) + == "full/4ce274dd83db0368bafd7e406f382ae088e39219.txt" ) - self.assertEqual( + assert ( file_path( Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc") - ), - "full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc", + ) + == "full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc" ) - self.assertEqual( + assert ( file_path( Request( "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg" ) - ), - "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg", + ) + == "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg" ) - self.assertEqual( - file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), - "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2", + assert ( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")) + == "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2" ) - self.assertEqual( - file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), - "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1", + assert ( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532")) + == "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1" ) - self.assertEqual( + assert ( file_path( Request("http://www.dorma.co.uk/images/product_details/2532"), response=Response("http://www.dorma.co.uk/images/product_details/2532"), info=object(), - ), - "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1", + ) + == "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1" ) - self.assertEqual( + assert ( file_path( Request( "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha" ) - ), - "full/76c00cef2ef669ae65052661f68d451162829507", + ) + == "full/76c00cef2ef669ae65052661f68d451162829507" ) - self.assertEqual( + assert ( file_path( Request( "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\ //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y" ) - ), - "full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png", + ) + == "full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png" ) def test_fs_store(self): assert isinstance(self.pipeline.store, FSFilesStore) - self.assertEqual(self.pipeline.store.basedir, self.tempdir) + assert self.pipeline.store.basedir == self.tempdir path = "some/image/key.jpg" fullpath = Path(self.tempdir, "some", "image", "key.jpg") - self.assertEqual(self.pipeline.store._get_filesystem_path(path), fullpath) + assert self.pipeline.store._get_filesystem_path(path) == fullpath @defer.inlineCallbacks def test_file_not_expired(self): @@ -180,8 +180,8 @@ class FilesPipelineTestCase(unittest.TestCase): p.start() result = yield self.pipeline.process_item(item, None) - self.assertEqual(result["files"][0]["checksum"], "abc") - self.assertEqual(result["files"][0]["status"], "uptodate") + assert result["files"][0]["checksum"] == "abc" + assert result["files"][0]["status"] == "uptodate" for p in patchers: p.stop() @@ -211,8 +211,8 @@ class FilesPipelineTestCase(unittest.TestCase): p.start() result = yield self.pipeline.process_item(item, None) - self.assertNotEqual(result["files"][0]["checksum"], "abc") - self.assertEqual(result["files"][0]["status"], "downloaded") + assert result["files"][0]["checksum"] != "abc" + assert result["files"][0]["status"] == "downloaded" for p in patchers: p.stop() @@ -242,8 +242,8 @@ class FilesPipelineTestCase(unittest.TestCase): p.start() result = yield self.pipeline.process_item(item, None) - self.assertNotEqual(result["files"][0]["checksum"], "abc") - self.assertEqual(result["files"][0]["status"], "cached") + assert result["files"][0]["checksum"] != "abc" + assert result["files"][0]["status"] == "cached" for p in patchers: p.stop() @@ -262,14 +262,14 @@ class FilesPipelineTestCase(unittest.TestCase): ).file_path item = {"path": "path-to-store-file"} request = Request("http://example.com") - self.assertEqual(file_path(request, item=item), "full/path-to-store-file") + assert file_path(request, item=item) == "full/path-to-store-file" class FilesPipelineTestCaseFieldsMixin: - def setUp(self): + def setup_method(self): self.tempdir = mkdtemp() - def tearDown(self): + def teardown_method(self): rmtree(self.tempdir) def test_item_fields_default(self): @@ -279,12 +279,12 @@ class FilesPipelineTestCaseFieldsMixin: get_crawler(None, {"FILES_STORE": self.tempdir}) ) requests = list(pipeline.get_media_requests(item, None)) - self.assertEqual(requests[0].url, url) + assert requests[0].url == url results = [(True, {"url": url})] item = pipeline.item_completed(results, item, None) files = ItemAdapter(item).get("files") - self.assertEqual(files, [results[0][1]]) - self.assertIsInstance(item, self.item_class) + assert files == [results[0][1]] + assert isinstance(item, self.item_class) def test_item_fields_override_settings(self): url = "http://www.example.com/files/1.txt" @@ -300,17 +300,15 @@ class FilesPipelineTestCaseFieldsMixin: ) ) requests = list(pipeline.get_media_requests(item, None)) - self.assertEqual(requests[0].url, url) + assert requests[0].url == url results = [(True, {"url": url})] item = pipeline.item_completed(results, item, None) custom_files = ItemAdapter(item).get("custom_files") - self.assertEqual(custom_files, [results[0][1]]) - self.assertIsInstance(item, self.item_class) + assert custom_files == [results[0][1]] + assert isinstance(item, self.item_class) -class FilesPipelineTestCaseFieldsDict( - FilesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestFilesPipelineFieldsDict(FilesPipelineTestCaseFieldsMixin): item_class = dict @@ -324,9 +322,7 @@ class FilesPipelineTestItem(Item): custom_files = Field() -class FilesPipelineTestCaseFieldsItem( - FilesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestFilesPipelineFieldsItem(FilesPipelineTestCaseFieldsMixin): item_class = FilesPipelineTestItem @@ -341,9 +337,7 @@ class FilesPipelineTestDataClass: custom_files: list = dataclasses.field(default_factory=list) -class FilesPipelineTestCaseFieldsDataClass( - FilesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestFilesPipelineFieldsDataClass(FilesPipelineTestCaseFieldsMixin): item_class = FilesPipelineTestDataClass @@ -358,13 +352,11 @@ class FilesPipelineTestAttrsItem: custom_files: list[dict[str, str]] = attr.ib(default=list) -class FilesPipelineTestCaseFieldsAttrsItem( - FilesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestFilesPipelineFieldsAttrsItem(FilesPipelineTestCaseFieldsMixin): item_class = FilesPipelineTestAttrsItem -class FilesPipelineTestCaseCustomSettings(unittest.TestCase): +class TestFilesPipelineCustomSettings: default_cls_settings = { "EXPIRES": 90, "FILES_URLS_FIELD": "file_urls", @@ -376,10 +368,10 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): ("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field"), } - def setUp(self): + def setup_method(self): self.tempdir = mkdtemp() - def tearDown(self): + def teardown_method(self): rmtree(self.tempdir) def _generate_fake_settings(self, prefix=None): @@ -420,10 +412,10 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): one_pipeline = FilesPipeline(self.tempdir, crawler=get_crawler(None)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: default_value = self.default_cls_settings[pipe_attr] - self.assertEqual(getattr(one_pipeline, pipe_attr), default_value) + assert getattr(one_pipeline, pipe_attr) == default_value custom_value = custom_settings[settings_attr] - self.assertNotEqual(default_value, custom_value) - self.assertEqual(getattr(another_pipeline, pipe_ins_attr), custom_value) + assert default_value != custom_value + assert getattr(another_pipeline, pipe_ins_attr) == custom_value def test_subclass_attributes_preserved_if_no_settings(self): """ @@ -433,8 +425,8 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": self.tempdir})) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: custom_value = getattr(pipe, pipe_ins_attr) - self.assertNotEqual(custom_value, self.default_cls_settings[pipe_attr]) - self.assertEqual(getattr(pipe, pipe_ins_attr), getattr(pipe, pipe_attr)) + assert custom_value != self.default_cls_settings[pipe_attr] + assert getattr(pipe, pipe_ins_attr) == getattr(pipe, pipe_attr) def test_subclass_attrs_preserved_custom_settings(self): """ @@ -447,8 +439,8 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: value = getattr(pipeline, pipe_ins_attr) setting_value = settings.get(settings_attr) - self.assertNotEqual(value, self.default_cls_settings[pipe_attr]) - self.assertEqual(value, setting_value) + assert value != self.default_cls_settings[pipe_attr] + assert value == setting_value def test_no_custom_settings_for_subclasses(self): """ @@ -465,7 +457,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = self.default_cls_settings.get(pipe_attr.upper()) - self.assertEqual(getattr(user_pipeline, pipe_ins_attr), custom_value) + assert getattr(user_pipeline, pipe_ins_attr) == custom_value def test_custom_settings_for_subclasses(self): """ @@ -484,8 +476,8 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = settings.get(prefix + "_" + settings_attr) - self.assertNotEqual(custom_value, self.default_cls_settings[pipe_attr]) - self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value) + assert custom_value != self.default_cls_settings[pipe_attr] + assert getattr(user_pipeline, pipe_inst_attr) == custom_value def test_custom_settings_and_class_attrs_for_subclasses(self): """ @@ -502,8 +494,8 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): pipe_inst_attr, ) in self.file_cls_attr_settings_map: custom_value = settings.get(prefix + "_" + settings_attr) - self.assertNotEqual(custom_value, self.default_cls_settings[pipe_cls_attr]) - self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value) + assert custom_value != self.default_cls_settings[pipe_cls_attr] + assert getattr(user_pipeline, pipe_inst_attr) == custom_value def test_cls_attrs_with_DEFAULT_prefix(self): class UserDefinedFilesPipeline(FilesPipeline): @@ -513,12 +505,13 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): pipeline = UserDefinedFilesPipeline.from_crawler( get_crawler(None, {"FILES_STORE": self.tempdir}) ) - self.assertEqual( - pipeline.files_result_field, - UserDefinedFilesPipeline.DEFAULT_FILES_RESULT_FIELD, + assert ( + pipeline.files_result_field + == UserDefinedFilesPipeline.DEFAULT_FILES_RESULT_FIELD ) - self.assertEqual( - pipeline.files_urls_field, UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD + assert ( + pipeline.files_urls_field + == UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD ) def test_user_defined_subclass_default_key_names(self): @@ -535,7 +528,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: expected_value = settings.get(settings_attr) - self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), expected_value) + assert getattr(pipeline_cls, pipe_inst_attr) == expected_value def test_file_pipeline_using_pathlike_objects(self): class CustomFilesPipelineWithPathLikeDir(FilesPipeline): @@ -546,12 +539,12 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): get_crawler(None, {"FILES_STORE": Path("./Temp")}) ) request = Request("http://example.com/image01.jpg") - self.assertEqual(pipeline.file_path(request), Path("subdir/image01.jpg")) + assert pipeline.file_path(request) == Path("subdir/image01.jpg") def test_files_store_constructor_with_pathlike_object(self): path = Path("./FileDir") fs_store = FSFilesStore(path) - self.assertEqual(fs_store.basedir, str(path)) + assert fs_store.basedir == str(path) @pytest.mark.requires_botocore @@ -593,13 +586,8 @@ class TestS3FilesStore(unittest.TestCase): ) stub.assert_no_pending_responses() - self.assertEqual( - buffer.method_calls, - [ - mock.call.seek(0), - # The call to read does not happen with Stubber - ], - ) + # The call to read does not happen with Stubber + assert buffer.method_calls == [mock.call.seek(0)] @defer.inlineCallbacks def test_stat(self): @@ -626,13 +614,10 @@ class TestS3FilesStore(unittest.TestCase): ) file_stats = yield store.stat_file("", info=None) - self.assertEqual( - file_stats, - { - "checksum": checksum, - "last_modified": last_modified.timestamp(), - }, - ) + assert file_stats == { + "checksum": checksum, + "last_modified": last_modified.timestamp(), + } stub.assert_no_pending_responses() @@ -655,16 +640,16 @@ class TestGCSFilesStore(unittest.TestCase): expected_policy = {"role": "READER", "entity": "allAuthenticatedUsers"} yield store.persist_file(path, buf, info=None, meta=meta, headers=None) s = yield store.stat_file(path, info=None) - self.assertIn("last_modified", s) - self.assertIn("checksum", s) - self.assertEqual(s["checksum"], "cdcda85605e46d0af6110752770dce3c") + assert "last_modified" in s + assert "checksum" in s + assert s["checksum"] == "cdcda85605e46d0af6110752770dce3c" u = urlparse(uri) content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:] + path) - self.assertEqual(content, data) - self.assertEqual(blob.metadata, {"foo": "bar"}) - self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL) - self.assertEqual(blob.content_type, "application/octet-stream") - self.assertIn(expected_policy, acl) + assert content == data + assert blob.metadata == {"foo": "bar"} + assert blob.cache_control == GCSFilesStore.CACHE_CONTROL + assert blob.content_type == "application/octet-stream" + assert expected_policy in acl @defer.inlineCallbacks def test_blob_path_consistency(self): @@ -702,12 +687,12 @@ class TestFTPFileStore(unittest.TestCase): with MockFTPServer() as ftp_server: store = FTPFilesStore(ftp_server.url("/")) empty_dict = yield store.stat_file(path, info=None) - self.assertEqual(empty_dict, {}) + assert empty_dict == {} yield store.persist_file(path, buf, info=None, meta=meta, headers=None) stat = yield store.stat_file(path, info=None) - self.assertIn("last_modified", stat) - self.assertIn("checksum", stat) - self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6") + assert "last_modified" in stat + assert "checksum" in stat + assert stat["checksum"] == "d113d66b2ec7258724a268bd88eef6b6" path = f"{store.basedir}/{path}" content = get_ftp_content_and_delete( path, @@ -717,7 +702,7 @@ class TestFTPFileStore(unittest.TestCase): store.password, store.USE_ACTIVE_MODE, ) - self.assertEqual(data, content) + assert data == content class ItemWithFiles(Item): @@ -739,12 +724,12 @@ def _prepare_request_object(item_url, flags=None): # this is separate from the one in test_pipeline_media.py to specifically test FilesPipeline subclasses -class BuildFromCrawlerTestCase(unittest.TestCase): - def setUp(self): +class TestBuildFromCrawler: + def setup_method(self): self.tempdir = mkdtemp() self.crawler = get_crawler(None, {"FILES_STORE": self.tempdir}) - def tearDown(self): + def teardown_method(self): rmtree(self.tempdir) def test_simple(self): @@ -755,7 +740,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 0) + assert len(w) == 0 assert pipe.store def test_has_old_init(self): @@ -768,7 +753,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 2) + assert len(w) == 2 assert pipe._init_called def test_has_from_settings(self): @@ -785,7 +770,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 3) + assert len(w) == 3 assert pipe.store assert pipe._from_settings_called @@ -805,6 +790,6 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 0) + assert len(w) == 0 assert pipe.store assert pipe._from_crawler_called diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 1d89e44ce..fef6bbbe9 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -9,109 +9,106 @@ from tempfile import mkdtemp import attr import pytest from itemadapter import ItemAdapter -from twisted.trial import unittest from scrapy.http import Request, Response from scrapy.item import Field, Item from scrapy.pipelines.images import ImageException, ImagesPipeline from scrapy.utils.test import get_crawler -skip_pillow: str | None try: from PIL import Image except ImportError: - skip_pillow = "Missing Python Imaging Library, install https://pypi.org/pypi/Pillow" + pytest.skip( + "Missing Python Imaging Library, install https://pypi.org/pypi/Pillow", + allow_module_level=True, + ) else: encoders = {"jpeg_encoder", "jpeg_decoder"} if not encoders.issubset(set(Image.core.__dict__)): # type: ignore[attr-defined] - skip_pillow = "Missing JPEG encoders" - else: - skip_pillow = None + pytest.skip("Missing JPEG encoders", allow_module_level=True) -class ImagesPipelineTestCase(unittest.TestCase): - skip = skip_pillow - - def setUp(self): +class TestImagesPipeline: + def setup_method(self): self.tempdir = mkdtemp() crawler = get_crawler() self.pipeline = ImagesPipeline(self.tempdir, crawler=crawler) - def tearDown(self): + def teardown_method(self): rmtree(self.tempdir) def test_file_path(self): file_path = self.pipeline.file_path - self.assertEqual( - file_path(Request("https://dev.mydeco.com/mydeco.gif")), - "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg", + assert ( + file_path(Request("https://dev.mydeco.com/mydeco.gif")) + == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" ) - self.assertEqual( + assert ( file_path( Request( "http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg" ) - ), - "full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg", + ) + == "full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg" ) - self.assertEqual( + assert ( file_path( Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif") - ), - "full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg", + ) + == "full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg" ) - self.assertEqual( + assert ( file_path( Request( "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg" ) - ), - "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg", + ) + == "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg" ) - self.assertEqual( - file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), - "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg", + assert ( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")) + == "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg" ) - self.assertEqual( - file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), - "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg", + assert ( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532")) + == "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg" ) - self.assertEqual( + assert ( file_path( Request("http://www.dorma.co.uk/images/product_details/2532"), response=Response("http://www.dorma.co.uk/images/product_details/2532"), info=object(), - ), - "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg", + ) + == "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg" ) def test_thumbnail_name(self): thumb_path = self.pipeline.thumb_path name = "50" - self.assertEqual( - thumb_path(Request("file:///tmp/foo.jpg"), name), - "thumbs/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg", + assert ( + thumb_path(Request("file:///tmp/foo.jpg"), name) + == "thumbs/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg" ) - self.assertEqual( - thumb_path(Request("file://foo.png"), name), - "thumbs/50/e55b765eba0ec7348e50a1df496040449071b96a.jpg", + assert ( + thumb_path(Request("file://foo.png"), name) + == "thumbs/50/e55b765eba0ec7348e50a1df496040449071b96a.jpg" ) - self.assertEqual( - thumb_path(Request("file:///tmp/foo"), name), - "thumbs/50/0329ad83ebb8e93ea7c7906d46e9ed55f7349a50.jpg", + assert ( + thumb_path(Request("file:///tmp/foo"), name) + == "thumbs/50/0329ad83ebb8e93ea7c7906d46e9ed55f7349a50.jpg" ) - self.assertEqual( - thumb_path(Request("file:///tmp/some.name/foo"), name), - "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg", + assert ( + thumb_path(Request("file:///tmp/some.name/foo"), name) + == "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg" ) - self.assertEqual( + assert ( thumb_path( Request("file:///tmp/some.name/foo"), name, response=Response("file:///tmp/some.name/foo"), info=object(), - ), - "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg", + ) + == "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg" ) def test_thumbnail_name_from_item(self): @@ -130,8 +127,8 @@ class ImagesPipelineTestCase(unittest.TestCase): ).thumb_path item = {"path": "path-to-store-file"} request = Request("http://example.com") - self.assertEqual( - thumb_path(request, "small", item=item), "thumb/small/path-to-store-file" + assert ( + thumb_path(request, "small", item=item) == "thumb/small/path-to-store-file" ) def test_get_images_exception(self): @@ -169,16 +166,13 @@ class ImagesPipelineTestCase(unittest.TestCase): ) path, new_im, new_buf = next(get_images_gen) - self.assertEqual(path, "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg") - self.assertEqual(orig_im, new_im) - self.assertEqual(buf.getvalue(), new_buf.getvalue()) + assert path == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" + assert orig_im == new_im + assert buf.getvalue() == new_buf.getvalue() thumb_path, thumb_img, thumb_buf = next(get_images_gen) - self.assertEqual( - thumb_path, "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" - ) - self.assertEqual(thumb_img, thumb_img) - self.assertEqual(orig_thumb_buf.getvalue(), thumb_buf.getvalue()) + assert thumb_path == "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" + assert orig_thumb_buf.getvalue() == thumb_buf.getvalue() def test_convert_image(self): SIZE = (100, 100) @@ -186,37 +180,35 @@ class ImagesPipelineTestCase(unittest.TestCase): COLOUR = (0, 127, 255) im, buf = _create_image("JPEG", "RGB", SIZE, COLOUR) converted, converted_buf = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + assert converted.mode == "RGB" + assert converted.getcolors() == [(10000, COLOUR)] # check that we don't convert JPEGs again - self.assertEqual(converted_buf, buf) + assert converted_buf == buf # check that thumbnail keep image ratio thumbnail, _ = self.pipeline.convert_image( converted, size=(10, 25), response_body=converted_buf ) - self.assertEqual(thumbnail.mode, "RGB") - self.assertEqual(thumbnail.size, (10, 10)) + assert thumbnail.mode == "RGB" + assert thumbnail.size == (10, 10) # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR) converted, _ = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + assert converted.mode == "RGB" + assert converted.getcolors() == [(10000, (205, 230, 255))] # transparency case with palette: P and PNG COLOUR = (0, 127, 255, 50) im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR) im = im.convert("P") converted, _ = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + assert converted.mode == "RGB" + assert converted.getcolors() == [(10000, (205, 230, 255))] class ImagesPipelineTestCaseFieldsMixin: - skip = skip_pillow - def test_item_fields_default(self): url = "http://www.example.com/images/1.jpg" item = self.item_class(name="item1", image_urls=[url]) @@ -224,12 +216,12 @@ class ImagesPipelineTestCaseFieldsMixin: get_crawler(None, {"IMAGES_STORE": "s3://example/images/"}) ) requests = list(pipeline.get_media_requests(item, None)) - self.assertEqual(requests[0].url, url) + assert requests[0].url == url results = [(True, {"url": url})] item = pipeline.item_completed(results, item, None) images = ItemAdapter(item).get("images") - self.assertEqual(images, [results[0][1]]) - self.assertIsInstance(item, self.item_class) + assert images == [results[0][1]] + assert isinstance(item, self.item_class) def test_item_fields_override_settings(self): url = "http://www.example.com/images/1.jpg" @@ -245,17 +237,15 @@ class ImagesPipelineTestCaseFieldsMixin: ) ) requests = list(pipeline.get_media_requests(item, None)) - self.assertEqual(requests[0].url, url) + assert requests[0].url == url results = [(True, {"url": url})] item = pipeline.item_completed(results, item, None) custom_images = ItemAdapter(item).get("custom_images") - self.assertEqual(custom_images, [results[0][1]]) - self.assertIsInstance(item, self.item_class) + assert custom_images == [results[0][1]] + assert isinstance(item, self.item_class) -class ImagesPipelineTestCaseFieldsDict( - ImagesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestImagesPipelineFieldsDict(ImagesPipelineTestCaseFieldsMixin): item_class = dict @@ -269,9 +259,7 @@ class ImagesPipelineTestItem(Item): custom_images = Field() -class ImagesPipelineTestCaseFieldsItem( - ImagesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestImagesPipelineFieldsItem(ImagesPipelineTestCaseFieldsMixin): item_class = ImagesPipelineTestItem @@ -286,9 +274,7 @@ class ImagesPipelineTestDataClass: custom_images: list = dataclasses.field(default_factory=list) -class ImagesPipelineTestCaseFieldsDataClass( - ImagesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestImagesPipelineFieldsDataClass(ImagesPipelineTestCaseFieldsMixin): item_class = ImagesPipelineTestDataClass @@ -303,15 +289,11 @@ class ImagesPipelineTestAttrsItem: custom_images: list[dict[str, str]] = attr.ib(default=list) -class ImagesPipelineTestCaseFieldsAttrsItem( - ImagesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestImagesPipelineFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin): item_class = ImagesPipelineTestAttrsItem -class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): - skip = skip_pillow - +class TestImagesPipelineCustomSettings: img_cls_attribute_names = [ # Pipeline attribute names with corresponding setting names. ("EXPIRES", "IMAGES_EXPIRES"), @@ -332,10 +314,10 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): "IMAGES_RESULT_FIELD": "images", } - def setUp(self): + def setup_method(self): self.tempdir = mkdtemp() - def tearDown(self): + def teardown_method(self): rmtree(self.tempdir) def _generate_fake_settings(self, prefix=None): @@ -397,11 +379,11 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr in self.img_cls_attribute_names: expected_default_value = self.default_pipeline_settings.get(pipe_attr) custom_value = custom_settings.get(settings_attr) - self.assertNotEqual(expected_default_value, custom_value) - self.assertEqual( - getattr(default_sts_pipe, pipe_attr.lower()), expected_default_value + assert expected_default_value != custom_value + assert ( + getattr(default_sts_pipe, pipe_attr.lower()) == expected_default_value ) - self.assertEqual(getattr(user_sts_pipe, pipe_attr.lower()), custom_value) + assert getattr(user_sts_pipe, pipe_attr.lower()) == custom_value def test_subclass_attrs_preserved_default_settings(self): """ @@ -415,8 +397,8 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr in self.img_cls_attribute_names: # Instance attribute (lowercase) must be equal to class attribute (uppercase). attr_value = getattr(pipeline, pipe_attr.lower()) - self.assertNotEqual(attr_value, self.default_pipeline_settings[pipe_attr]) - self.assertEqual(attr_value, getattr(pipeline, pipe_attr)) + assert attr_value != self.default_pipeline_settings[pipe_attr] + assert attr_value == getattr(pipeline, pipe_attr) def test_subclass_attrs_preserved_custom_settings(self): """ @@ -430,9 +412,9 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): # Instance attribute (lowercase) must be equal to # value defined in settings. value = getattr(pipeline, pipe_attr.lower()) - self.assertNotEqual(value, self.default_pipeline_settings[pipe_attr]) + assert value != self.default_pipeline_settings[pipe_attr] setings_value = settings.get(settings_attr) - self.assertEqual(value, setings_value) + assert value == setings_value def test_no_custom_settings_for_subclasses(self): """ @@ -449,7 +431,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr in self.img_cls_attribute_names: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = self.default_pipeline_settings.get(pipe_attr.upper()) - self.assertEqual(getattr(user_pipeline, pipe_attr.lower()), custom_value) + assert getattr(user_pipeline, pipe_attr.lower()) == custom_value def test_custom_settings_for_subclasses(self): """ @@ -468,8 +450,8 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr in self.img_cls_attribute_names: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = settings.get(prefix + "_" + settings_attr) - self.assertNotEqual(custom_value, self.default_pipeline_settings[pipe_attr]) - self.assertEqual(getattr(user_pipeline, pipe_attr.lower()), custom_value) + assert custom_value != self.default_pipeline_settings[pipe_attr] + assert getattr(user_pipeline, pipe_attr.lower()) == custom_value def test_custom_settings_and_class_attrs_for_subclasses(self): """ @@ -482,8 +464,8 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: custom_value = settings.get(prefix + "_" + settings_attr) - self.assertNotEqual(custom_value, self.default_pipeline_settings[pipe_attr]) - self.assertEqual(getattr(user_pipeline, pipe_attr.lower()), custom_value) + assert custom_value != self.default_pipeline_settings[pipe_attr] + assert getattr(user_pipeline, pipe_attr.lower()) == custom_value def test_cls_attrs_with_DEFAULT_prefix(self): class UserDefinedImagePipeline(ImagesPipeline): @@ -493,13 +475,13 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): pipeline = UserDefinedImagePipeline.from_crawler( get_crawler(None, {"IMAGES_STORE": self.tempdir}) ) - self.assertEqual( - pipeline.images_result_field, - UserDefinedImagePipeline.DEFAULT_IMAGES_RESULT_FIELD, + assert ( + pipeline.images_result_field + == UserDefinedImagePipeline.DEFAULT_IMAGES_RESULT_FIELD ) - self.assertEqual( - pipeline.images_urls_field, - UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD, + assert ( + pipeline.images_urls_field + == UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD ) def test_user_defined_subclass_default_key_names(self): @@ -516,7 +498,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr in self.img_cls_attribute_names: expected_value = settings.get(settings_attr) - self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), expected_value) + assert getattr(pipeline_cls, pipe_attr.lower()) == expected_value def _create_image(format, *a, **kw): diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index c6fdd3767..d915fc2a3 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -2,6 +2,7 @@ from __future__ import annotations import warnings +import pytest from testfixtures import LogCapture from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks @@ -18,15 +19,6 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all from scrapy.utils.test import get_crawler -try: - from PIL import Image # noqa: F401 -except ImportError: - skip_pillow: str | None = ( - "Missing Python Imaging Library, install https://pypi.org/pypi/Pillow" - ) -else: - skip_pillow = None - def _mocked_download_func(request, info): assert request.callback is NO_CALLBACK @@ -51,7 +43,7 @@ class UserDefinedPipeline(MediaPipeline): return "" -class BaseMediaPipelineTestCase(unittest.TestCase): +class TestBaseMediaPipeline(unittest.TestCase): pipeline_class = UserDefinedPipeline settings = None @@ -123,9 +115,9 @@ class BaseMediaPipelineTestCase(unittest.TestCase): failure = Failure(file_exc) # The Failure should encapsulate a FileException ... - self.assertEqual(failure.value, file_exc) + assert failure.value == file_exc # ... and it should have the StopIteration exception set as its context - self.assertEqual(failure.value.__context__, def_gen_return_exc) + assert failure.value.__context__ == def_gen_return_exc # Let's calculate the request fingerprint and fake some runtime data... fp = self.fingerprint(request) @@ -136,12 +128,12 @@ class BaseMediaPipelineTestCase(unittest.TestCase): # When calling the method that caches the Request's result ... self.pipe._cache_result_and_execute_waiters(failure, fp, info) # ... it should store the Twisted Failure ... - self.assertEqual(info.downloaded[fp], failure) + assert info.downloaded[fp] == failure # ... encapsulating the original FileException ... - self.assertEqual(info.downloaded[fp].value, file_exc) + assert info.downloaded[fp].value == file_exc # ... but it should not store the StopIteration exception on its context context = getattr(info.downloaded[fp].value, "__context__", None) - self.assertIsNone(context) + assert context is None def test_default_item_completed(self): item = {"name": "name"} @@ -158,7 +150,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): assert len(log.records) == 1 record = log.records[0] assert record.levelname == "ERROR" - self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail)) + assert record.exc_info == failure_to_exc_info(fail) # disable failure logging and check again self.pipe.LOG_FAILED_RESULTS = False @@ -208,7 +200,7 @@ class MockedMediaPipeline(UserDefinedPipeline): return item -class MediaPipelineTestCase(BaseMediaPipelineTestCase): +class TestMediaPipeline(TestBaseMediaPipeline): pipeline_class = MockedMediaPipeline def _errback(self, result): @@ -225,16 +217,13 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ) item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, {})]) - self.assertEqual( - self.pipe._mockcalled, - [ - "get_media_requests", - "media_to_download", - "media_downloaded", - "item_completed", - ], - ) + assert new_item["results"] == [(True, {})] + assert self.pipe._mockcalled == [ + "get_media_requests", + "media_to_download", + "media_downloaded", + "item_completed", + ] @inlineCallbacks def test_result_failure(self): @@ -247,17 +236,14 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ) item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(False, fail)]) - self.assertEqual( - self.pipe._mockcalled, - [ - "get_media_requests", - "media_to_download", - "media_failed", - "request_errback", - "item_completed", - ], - ) + assert new_item["results"] == [(False, fail)] + assert self.pipe._mockcalled == [ + "get_media_requests", + "media_to_download", + "media_failed", + "request_errback", + "item_completed", + ] @inlineCallbacks def test_mix_of_success_and_failure(self): @@ -268,18 +254,18 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): req2 = Request("http://url2", meta={"response": fail}) item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, {}), (False, fail)]) + assert new_item["results"] == [(True, {}), (False, fail)] m = self.pipe._mockcalled # only once - self.assertEqual(m[0], "get_media_requests") # first hook called - self.assertEqual(m.count("get_media_requests"), 1) - self.assertEqual(m.count("item_completed"), 1) - self.assertEqual(m[-1], "item_completed") # last hook called + assert m[0] == "get_media_requests" # first hook called + assert m.count("get_media_requests") == 1 + assert m.count("item_completed") == 1 + assert m[-1] == "item_completed" # last hook called # twice, one per request - self.assertEqual(m.count("media_to_download"), 2) + assert m.count("media_to_download") == 2 # one to handle success and other for failure - self.assertEqual(m.count("media_downloaded"), 1) - self.assertEqual(m.count("media_failed"), 1) + assert m.count("media_downloaded") == 1 + assert m.count("media_failed") == 1 @inlineCallbacks def test_get_media_requests(self): @@ -288,7 +274,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = {"requests": req} # pass a single item new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item - self.assertIn(self.fingerprint(req), self.info.downloaded) + assert self.fingerprint(req) in self.info.downloaded # returns iterable of Requests req1 = Request("http://url1") @@ -305,8 +291,8 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): req1 = Request("http://url1", meta={"response": rsp1}) item = {"requests": req1} new_item = yield self.pipe.process_item(item, self.spider) - self.assertTrue(new_item is item) - self.assertEqual(new_item["results"], [(True, {})]) + assert new_item is item + assert new_item["results"] == [(True, {})] # rsp2 is ignored, rsp1 must be in results because request fingerprints are the same req2 = Request( @@ -314,9 +300,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ) item = {"requests": req2} new_item = yield self.pipe.process_item(item, self.spider) - self.assertTrue(new_item is item) - self.assertEqual(self.fingerprint(req1), self.fingerprint(req2)) - self.assertEqual(new_item["results"], [(True, {})]) + assert new_item is item + assert self.fingerprint(req1) == self.fingerprint(req2) + assert new_item["results"] == [(True, {})] @inlineCallbacks def test_results_are_cached_for_requests_of_single_item(self): @@ -327,17 +313,17 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ) item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) - self.assertTrue(new_item is item) - self.assertEqual(new_item["results"], [(True, {}), (True, {})]) + assert new_item is item + assert new_item["results"] == [(True, {}), (True, {})] @inlineCallbacks def test_wait_if_request_is_downloading(self): def _check_downloading(response): fp = self.fingerprint(req1) - self.assertTrue(fp in self.info.downloading) - self.assertTrue(fp in self.info.waiting) - self.assertTrue(fp not in self.info.downloaded) - self.assertEqual(len(self.info.waiting[fp]), 2) + assert fp in self.info.downloading + assert fp in self.info.waiting + assert fp not in self.info.downloaded + assert len(self.info.waiting[fp]) == 2 return response rsp1 = Response("http://url") @@ -348,39 +334,40 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): return dfd def rsp2_func(): - self.fail("it must cache rsp1 result and must not try to redownload") + pytest.fail("it must cache rsp1 result and must not try to redownload") req1 = Request("http://url", meta={"response": rsp1_func}) req2 = Request(req1.url, meta={"response": rsp2_func}) item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, {}), (True, {})]) + assert new_item["results"] == [(True, {}), (True, {})] @inlineCallbacks def test_use_media_to_download_result(self): req = Request("http://url", meta={"result": "ITSME", "response": self.fail}) item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, "ITSME")]) - self.assertEqual( - self.pipe._mockcalled, - ["get_media_requests", "media_to_download", "item_completed"], - ) + assert new_item["results"] == [(True, "ITSME")] + assert self.pipe._mockcalled == [ + "get_media_requests", + "media_to_download", + "item_completed", + ] def test_key_for_pipe(self): - self.assertEqual( - self.pipe._key_for_pipe("IMAGES", base_class_name="MediaPipeline"), - "MOCKEDMEDIAPIPELINE_IMAGES", + assert ( + self.pipe._key_for_pipe("IMAGES", base_class_name="MediaPipeline") + == "MOCKEDMEDIAPIPELINE_IMAGES" ) -class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): +class TestMediaPipelineAllowRedirectSettings: def _assert_request_no3xx(self, pipeline_class, settings): pipe = pipeline_class(crawler=get_crawler(None, settings)) request = Request("http://url") pipe._modify_media_request(request) - self.assertIn("handle_httpstatus_list", request.meta) + assert "handle_httpstatus_list" in request.meta for status, check in [ (200, True), # These are the status codes we want @@ -396,9 +383,9 @@ class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): (500, True), ]: if check: - self.assertIn(status, request.meta["handle_httpstatus_list"]) + assert status in request.meta["handle_httpstatus_list"] else: - self.assertNotIn(status, request.meta["handle_httpstatus_list"]) + assert status not in request.meta["handle_httpstatus_list"] def test_subclass_standard_setting(self): self._assert_request_no3xx(UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True}) @@ -409,8 +396,8 @@ class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): ) -class BuildFromCrawlerTestCase(unittest.TestCase): - def setUp(self): +class TestBuildFromCrawler: + def setup_method(self): self.crawler = get_crawler(None, {"FILES_STORE": "/foo"}) def test_simple(self): @@ -421,7 +408,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 0) + assert len(w) == 0 def test_has_old_init(self): class Pipeline(UserDefinedPipeline): @@ -433,7 +420,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 2) + assert len(w) == 2 assert pipe._init_called def test_has_from_settings(self): @@ -450,7 +437,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 2) + assert len(w) == 2 assert pipe._from_settings_called def test_has_from_settings_and_from_crawler(self): @@ -474,7 +461,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 2) + assert len(w) == 2 assert pipe._from_settings_called assert pipe._from_crawler_called @@ -497,7 +484,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 2) + assert len(w) == 2 assert pipe._from_settings_called assert pipe._init_called @@ -521,7 +508,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 0) + assert len(w) == 0 assert pipe._from_crawler_called assert pipe._init_called @@ -542,5 +529,5 @@ class BuildFromCrawlerTestCase(unittest.TestCase): # this and the next assert will fail as MediaPipeline.from_crawler() wasn't called assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 0) + assert len(w) == 0 assert pipe._from_crawler_called From 380c2279b92f1aa7386e79fc43109499a057e8cf Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 9 Mar 2025 23:23:51 +0400 Subject: [PATCH 070/491] Converting tests to plain asserts, part 7. (#6710) --- tests/test_downloader_handlers.py | 243 ++++++++-------- tests/test_downloader_handlers_http2.py | 41 ++- tests/test_exporters.py | 168 +++++------ tests/test_feedexport.py | 367 +++++++++++------------- tests/test_http2_client_protocol.py | 123 ++++---- tests/test_http_cookies.py | 52 ++-- tests/test_http_headers.py | 87 +++--- 7 files changed, 512 insertions(+), 569 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 323a51002..19bd02498 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -69,46 +69,46 @@ class OffDH: return cls(crawler) -class LoadTestCase(unittest.TestCase): +class TestLoad: def test_enabled_handler(self): handlers = {"scheme": DummyDH} crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) dh = DownloadHandlers(crawler) - self.assertIn("scheme", dh._schemes) - self.assertIn("scheme", dh._handlers) - self.assertNotIn("scheme", dh._notconfigured) + assert "scheme" in dh._schemes + assert "scheme" in dh._handlers + assert "scheme" not in dh._notconfigured def test_not_configured_handler(self): handlers = {"scheme": OffDH} crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) dh = DownloadHandlers(crawler) - self.assertIn("scheme", dh._schemes) - self.assertNotIn("scheme", dh._handlers) - self.assertIn("scheme", dh._notconfigured) + assert "scheme" in dh._schemes + assert "scheme" not in dh._handlers + assert "scheme" in dh._notconfigured def test_disabled_handler(self): handlers = {"scheme": None} crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) dh = DownloadHandlers(crawler) - self.assertNotIn("scheme", dh._schemes) + assert "scheme" not in dh._schemes for scheme in handlers: # force load handlers dh._get_handler(scheme) - self.assertNotIn("scheme", dh._handlers) - self.assertIn("scheme", dh._notconfigured) + assert "scheme" not in dh._handlers + assert "scheme" in dh._notconfigured def test_lazy_handlers(self): handlers = {"scheme": DummyLazyDH} crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) dh = DownloadHandlers(crawler) - self.assertIn("scheme", dh._schemes) - self.assertNotIn("scheme", dh._handlers) + assert "scheme" in dh._schemes + assert "scheme" not in dh._handlers for scheme in handlers: # force load lazy handler dh._get_handler(scheme) - self.assertIn("scheme", dh._handlers) - self.assertNotIn("scheme", dh._notconfigured) + assert "scheme" in dh._handlers + assert "scheme" not in dh._notconfigured -class FileTestCase(unittest.TestCase): +class TestFile(unittest.TestCase): def setUp(self): # add a special char to check that they are handled correctly self.fd, self.tmpname = mkstemp(suffix="^") @@ -122,10 +122,10 @@ class FileTestCase(unittest.TestCase): def test_download(self): def _test(response): - self.assertEqual(response.url, request.url) - self.assertEqual(response.status, 200) - self.assertEqual(response.body, b"0123456789") - self.assertEqual(response.protocol, None) + assert response.url == request.url + assert response.status == 200 + assert response.body == b"0123456789" + assert response.protocol is None request = Request(path_to_file_uri(self.tmpname)) assert request.url.upper().endswith("%5E") @@ -217,7 +217,7 @@ class DuplicateHeaderResource(resource.Resource): return b"" -class HttpTestCase(unittest.TestCase, ABC): +class TestHttp(unittest.TestCase, ABC): scheme = "http" # only used for HTTPS tests @@ -336,8 +336,8 @@ class HttpTestCase(unittest.TestCase, ABC): def test_host_header_not_in_request_headers(self): def _test(response): - self.assertEqual(response.body, to_bytes(f"{self.host}:{self.portno}")) - self.assertEqual(request.headers, {}) + assert response.body == to_bytes(f"{self.host}:{self.portno}") + assert not request.headers request = Request(self.getURL("host")) return self.download_request(request, Spider("foo")).addCallback(_test) @@ -346,8 +346,8 @@ class HttpTestCase(unittest.TestCase, ABC): host = self.host + ":" + str(self.portno) def _test(response): - self.assertEqual(response.body, host.encode()) - self.assertEqual(request.headers.get("Host"), host.encode()) + assert response.body == host.encode() + assert request.headers.get("Host") == host.encode() request = Request(self.getURL("host"), headers={"Host": host}) return self.download_request(request, Spider("foo")).addCallback(_test) @@ -365,7 +365,7 @@ class HttpTestCase(unittest.TestCase, ABC): """ def _test(response): - self.assertEqual(response.body, b"0") + assert response.body == b"0" request = Request(self.getURL("contentlength"), method="POST") return self.download_request(request, Spider("foo")).addCallback(_test) @@ -376,8 +376,8 @@ class HttpTestCase(unittest.TestCase, ABC): headers = Headers(json.loads(response.text)["headers"]) contentlengths = headers.getlist("Content-Length") - self.assertEqual(len(contentlengths), 1) - self.assertEqual(contentlengths, [b"0"]) + assert len(contentlengths) == 1 + assert contentlengths == [b"0"] request = Request(self.getURL("echo"), method="POST") return self.download_request(request, Spider("foo")).addCallback(_test) @@ -399,7 +399,7 @@ class HttpTestCase(unittest.TestCase, ABC): def _test_response_class(self, filename, body, response_class): def _test(response): - self.assertEqual(type(response), response_class) + assert type(response) is response_class # pylint: disable=unidiomatic-typecheck request = Request(self.getURL(filename), body=body) return self.download_request(request, Spider("foo")).addCallback(_test) @@ -416,17 +416,14 @@ class HttpTestCase(unittest.TestCase, ABC): def test_get_duplicate_header(self): def _test(response): - self.assertEqual( - response.headers.getlist(b"Set-Cookie"), - [b"a=b", b"c=d"], - ) + assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] request = Request(self.getURL("duplicate-header")) return self.download_request(request, Spider("foo")).addCallback(_test) @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class Http10TestCase(HttpTestCase): +class TestHttp10(TestHttp): """HTTP 1.0 test case""" @property @@ -441,11 +438,11 @@ class Http10TestCase(HttpTestCase): return d -class Https10TestCase(Http10TestCase): +class TestHttps10(TestHttp10): scheme = "https" -class Http11TestCase(HttpTestCase): +class TestHttp11(TestHttp): """HTTP 1.1 test case""" @property @@ -466,7 +463,7 @@ class Http11TestCase(HttpTestCase): body = b"Some plain text\ndata with tabs\t and null bytes\0" def _test_type(response): - self.assertEqual(type(response), TextResponse) + assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck request = Request(self.getURL("nocontenttype"), body=body) d = self.download_request(request, Spider("foo")) @@ -583,7 +580,7 @@ class Http11TestCase(HttpTestCase): return d -class Https11TestCase(Http11TestCase): +class TestHttps11(TestHttp11): scheme = "https" tls_log_message = ( @@ -611,7 +608,7 @@ class Https11TestCase(Http11TestCase): yield download_handler.close() -class SimpleHttpsTest(unittest.TestCase): +class TestSimpleHttps(unittest.TestCase): """Base class for special cases tested with just one simple request""" keyfile = "keys/localhost.key" @@ -663,7 +660,7 @@ class SimpleHttpsTest(unittest.TestCase): return d -class Https11WrongHostnameTestCase(SimpleHttpsTest): +class TestHttps11WrongHostname(TestSimpleHttps): # above tests use a server certificate for "localhost", # client connection to "localhost" too. # here we test that even if the server certificate is for another domain, @@ -673,7 +670,7 @@ class Https11WrongHostnameTestCase(SimpleHttpsTest): certfile = "keys/example-com.cert.pem" -class Https11InvalidDNSId(SimpleHttpsTest): +class TestHttps11InvalidDNSId(TestSimpleHttps): """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" def setUp(self): @@ -681,18 +678,18 @@ class Https11InvalidDNSId(SimpleHttpsTest): self.host = "127.0.0.1" -class Https11InvalidDNSPattern(SimpleHttpsTest): +class TestHttps11InvalidDNSPattern(TestSimpleHttps): """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" keyfile = "keys/localhost.ip.key" certfile = "keys/localhost.ip.crt" -class Https11CustomCiphers(SimpleHttpsTest): +class TestHttps11CustomCiphers(TestSimpleHttps): cipher_string = "CAMELLIA256-SHA" -class Http11MockServerTestCase(unittest.TestCase): +class TestHttp11MockServer(unittest.TestCase): """HTTP 1.1 test case with MockServer""" settings_dict: dict | None = None @@ -719,7 +716,7 @@ class Http11MockServerTestCase(unittest.TestCase): ) ) failure = crawler.spider.meta["failure"] - self.assertIsInstance(failure.value, defer.CancelledError) + assert isinstance(failure.value, defer.CancelledError) @defer.inlineCallbacks def test_download(self): @@ -728,9 +725,9 @@ class Http11MockServerTestCase(unittest.TestCase): seed=Request(url=self.mockserver.url("", is_secure=self.is_secure)) ) failure = crawler.spider.meta.get("failure") - self.assertTrue(failure is None) + assert failure is None reason = crawler.spider.meta["close_reason"] - self.assertTrue(reason, "finished") + assert reason == "finished" class UriResource(resource.Resource): @@ -748,7 +745,7 @@ class UriResource(resource.Resource): return b"" -class HttpProxyTestCase(unittest.TestCase, ABC): +class TestHttpProxy(unittest.TestCase, ABC): expected_http_proxy_request_body = b"http://example.com" @property @@ -777,9 +774,9 @@ class HttpProxyTestCase(unittest.TestCase, ABC): def test_download_with_proxy(self): def _test(response): - self.assertEqual(response.status, 200) - self.assertEqual(response.url, request.url) - self.assertEqual(response.body, self.expected_http_proxy_request_body) + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body http_proxy = self.getURL("") request = Request("http://example.com", meta={"proxy": http_proxy}) @@ -787,22 +784,22 @@ class HttpProxyTestCase(unittest.TestCase, ABC): def test_download_without_proxy(self): def _test(response): - self.assertEqual(response.status, 200) - self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b"/path/to/resource") + assert response.status == 200 + assert response.url == request.url + assert response.body == b"/path/to/resource" request = Request(self.getURL("path/to/resource")) return self.download_request(request, Spider("foo")).addCallback(_test) @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class Http10ProxyTestCase(HttpProxyTestCase): +class TestHttp10Proxy(TestHttpProxy): @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: return HTTP10DownloadHandler -class Http11ProxyTestCase(HttpProxyTestCase): +class TestHttp11Proxy(TestHttpProxy): @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: return HTTP11DownloadHandler @@ -817,13 +814,13 @@ class Http11ProxyTestCase(HttpProxyTestCase): request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2}) d = self.download_request(request, Spider("foo")) timeout = yield self.assertFailure(d, error.TimeoutError) - self.assertIn(domain, timeout.osError) + assert domain in timeout.osError def test_download_with_proxy_without_http_scheme(self): def _test(response): - self.assertEqual(response.status, 200) - self.assertEqual(response.url, request.url) - self.assertEqual(response.body, self.expected_http_proxy_request_body) + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body http_proxy = self.getURL("").replace("http://", "") request = Request("http://example.com", meta={"proxy": http_proxy}) @@ -839,8 +836,8 @@ class HttpDownloadHandlerMock: @pytest.mark.requires_botocore -class S3AnonTestCase(unittest.TestCase): - def setUp(self): +class TestS3Anon: + def setup_method(self): crawler = get_crawler() self.s3reqh = build_from_crawler( S3DownloadHandler, @@ -854,13 +851,13 @@ class S3AnonTestCase(unittest.TestCase): def test_anon_request(self): req = Request("s3://aws-publicdatasets/") httpreq = self.download_request(req, self.spider) - self.assertEqual(hasattr(self.s3reqh, "anon"), True) - self.assertEqual(self.s3reqh.anon, True) - self.assertEqual(httpreq.url, "http://aws-publicdatasets.s3.amazonaws.com/") + assert hasattr(self.s3reqh, "anon") + assert self.s3reqh.anon + assert httpreq.url == "http://aws-publicdatasets.s3.amazonaws.com/" @pytest.mark.requires_botocore -class S3TestCase(unittest.TestCase): +class TestS3: download_handler_cls: type = S3DownloadHandler # test use same example keys than amazon developer guide @@ -870,7 +867,7 @@ class S3TestCase(unittest.TestCase): AWS_ACCESS_KEY_ID = "0PN5J17HBGZHT7JJ3X82" AWS_SECRET_ACCESS_KEY = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o" - def setUp(self): + def setup_method(self): crawler = get_crawler() s3reqh = build_from_crawler( S3DownloadHandler, @@ -897,17 +894,13 @@ class S3TestCase(unittest.TestCase): yield def test_extra_kw(self): - try: - crawler = get_crawler() + crawler = get_crawler() + with pytest.raises((TypeError, NotConfigured)): build_from_crawler( S3DownloadHandler, crawler, extra_kw=True, ) - except Exception as e: - self.assertIsInstance(e, (TypeError, NotConfigured)) - else: - raise AssertionError def test_request_signing1(self): # gets an object from the johnsmith bucket. @@ -915,9 +908,9 @@ class S3TestCase(unittest.TestCase): req = Request("s3://johnsmith/photos/puppy.jpg", headers={"Date": date}) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual( - httpreq.headers["Authorization"], - b"AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=", + assert ( + httpreq.headers["Authorization"] + == b"AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=" ) def test_request_signing2(self): @@ -934,9 +927,9 @@ class S3TestCase(unittest.TestCase): ) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual( - httpreq.headers["Authorization"], - b"AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=", + assert ( + httpreq.headers["Authorization"] + == b"AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=" ) def test_request_signing3(self): @@ -952,9 +945,9 @@ class S3TestCase(unittest.TestCase): ) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual( - httpreq.headers["Authorization"], - b"AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=", + assert ( + httpreq.headers["Authorization"] + == b"AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=" ) def test_request_signing4(self): @@ -963,9 +956,9 @@ class S3TestCase(unittest.TestCase): req = Request("s3://johnsmith/?acl", method="GET", headers={"Date": date}) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual( - httpreq.headers["Authorization"], - b"AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=", + assert ( + httpreq.headers["Authorization"] + == b"AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=" ) def test_request_signing6(self): @@ -991,9 +984,9 @@ class S3TestCase(unittest.TestCase): ) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual( - httpreq.headers["Authorization"], - b"AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=", + assert ( + httpreq.headers["Authorization"] + == b"AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=" ) def test_request_signing7(self): @@ -1006,13 +999,13 @@ class S3TestCase(unittest.TestCase): ) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual( - httpreq.headers["Authorization"], - b"AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=", + assert ( + httpreq.headers["Authorization"] + == b"AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=" ) -class BaseFTPTestCase(unittest.TestCase): +class TestFTPBase(unittest.TestCase): username = "scrapy" password = "passwd" req_meta = {"ftp_user": username, "ftp_password": password} @@ -1068,10 +1061,10 @@ class BaseFTPTestCase(unittest.TestCase): d = self.download_handler.download_request(request, None) def _test(r): - self.assertEqual(r.status, 200) - self.assertEqual(r.body, b"I have the power!") - self.assertEqual(r.headers, {b"Local Filename": [b""], b"Size": [b"17"]}) - self.assertIsNone(r.protocol) + assert r.status == 200 + assert r.body == b"I have the power!" + assert r.headers == {b"Local Filename": [b""], b"Size": [b"17"]} + assert r.protocol is None return self._add_test_callbacks(d, _test) @@ -1083,9 +1076,9 @@ class BaseFTPTestCase(unittest.TestCase): d = self.download_handler.download_request(request, None) def _test(r): - self.assertEqual(r.status, 200) - self.assertEqual(r.body, b"Moooooooooo power!") - self.assertEqual(r.headers, {b"Local Filename": [b""], b"Size": [b"18"]}) + assert r.status == 200 + assert r.body == b"Moooooooooo power!" + assert r.headers == {b"Local Filename": [b""], b"Size": [b"18"]} return self._add_test_callbacks(d, _test) @@ -1096,7 +1089,7 @@ class BaseFTPTestCase(unittest.TestCase): d = self.download_handler.download_request(request, None) def _test(r): - self.assertEqual(r.status, 404) + assert r.status == 404 return self._add_test_callbacks(d, _test) @@ -1111,12 +1104,10 @@ class BaseFTPTestCase(unittest.TestCase): d = self.download_handler.download_request(request, None) def _test(r): - self.assertEqual(r.body, fname_bytes) - self.assertEqual( - r.headers, {b"Local Filename": [fname_bytes], b"Size": [b"17"]} - ) - self.assertTrue(local_fname.exists()) - self.assertEqual(local_fname.read_bytes(), b"I have the power!") + assert r.body == fname_bytes + assert r.headers == {b"Local Filename": [fname_bytes], b"Size": [b"17"]} + assert local_fname.exists() + assert local_fname.read_bytes() == b"I have the power!" local_fname.unlink() return self._add_test_callbacks(d, _test) @@ -1131,7 +1122,7 @@ class BaseFTPTestCase(unittest.TestCase): d = self.download_handler.download_request(request, None) def _test(r): - self.assertEqual(type(r), response_class) + assert type(r) is response_class # pylint: disable=unidiomatic-typecheck local_fname.unlink() return self._add_test_callbacks(d, _test) @@ -1143,7 +1134,7 @@ class BaseFTPTestCase(unittest.TestCase): return self._test_response_class("html-file-without-extension", HtmlResponse) -class FTPTestCase(BaseFTPTestCase): +class TestFTP(TestFTPBase): def test_invalid_credentials(self): if self.reactor_pytest == "asyncio" and sys.platform == "win32": raise unittest.SkipTest( @@ -1157,12 +1148,12 @@ class FTPTestCase(BaseFTPTestCase): d = self.download_handler.download_request(request, None) def _test(r): - self.assertEqual(r.type, ConnectionLost) + assert r.type == ConnectionLost return self._add_test_callbacks(d, errback=_test) -class AnonymousFTPTestCase(BaseFTPTestCase): +class TestAnonymousFTP(TestFTPBase): username = "anonymous" req_meta = {} @@ -1188,7 +1179,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase): shutil.rmtree(self.directory) -class DataURITestCase(unittest.TestCase): +class TestDataURI(unittest.TestCase): def setUp(self): crawler = get_crawler() self.download_handler = build_from_crawler(DataURIDownloadHandler, crawler) @@ -1199,44 +1190,44 @@ class DataURITestCase(unittest.TestCase): uri = "data:,A%20brief%20note" def _test(response): - self.assertEqual(response.url, uri) - self.assertFalse(response.headers) + assert response.url == uri + assert not response.headers request = Request(uri) return self.download_request(request, self.spider).addCallback(_test) def test_default_mediatype_encoding(self): def _test(response): - self.assertEqual(response.text, "A brief note") - self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) - self.assertEqual(response.encoding, "US-ASCII") + assert response.text == "A brief note" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "US-ASCII" request = Request("data:,A%20brief%20note") return self.download_request(request, self.spider).addCallback(_test) def test_default_mediatype(self): def _test(response): - self.assertEqual(response.text, "\u038e\u03a3\u038e") - self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) - self.assertEqual(response.encoding, "iso-8859-7") + assert response.text == "\u038e\u03a3\u038e" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "iso-8859-7" request = Request("data:;charset=iso-8859-7,%be%d3%be") return self.download_request(request, self.spider).addCallback(_test) def test_text_charset(self): def _test(response): - self.assertEqual(response.text, "\u038e\u03a3\u038e") - self.assertEqual(response.body, b"\xbe\xd3\xbe") - self.assertEqual(response.encoding, "iso-8859-7") + assert response.text == "\u038e\u03a3\u038e" + assert response.body == b"\xbe\xd3\xbe" + assert response.encoding == "iso-8859-7" request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") return self.download_request(request, self.spider).addCallback(_test) def test_mediatype_parameters(self): def _test(response): - self.assertEqual(response.text, "\u038e\u03a3\u038e") - self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) - self.assertEqual(response.encoding, "utf-8") + assert response.text == "\u038e\u03a3\u038e" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "utf-8" request = Request( "data:text/plain;foo=%22foo;bar%5C%22%22;" @@ -1247,14 +1238,14 @@ class DataURITestCase(unittest.TestCase): def test_base64(self): def _test(response): - self.assertEqual(response.text, "Hello, world.") + assert response.text == "Hello, world." request = Request("data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D") return self.download_request(request, self.spider).addCallback(_test) def test_protocol(self): def _test(response): - self.assertIsNone(response.protocol) + assert response.protocol is None request = Request("data:,") return self.download_request(request, self.spider).addCallback(_test) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 17d5c2d0a..c74c09cbb 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -4,7 +4,6 @@ from unittest import mock import pytest from testfixtures import LogCapture from twisted.internet import defer, error, reactor -from twisted.trial import unittest from twisted.web import server from twisted.web.error import SchemeNotSupported from twisted.web.http import H2_ENABLED @@ -28,25 +27,25 @@ class BaseTestClasses: # A hack to prevent tests from the imported classes to run here too. # See https://stackoverflow.com/q/1323455/113586 for other ways. from tests.test_downloader_handlers import ( - Http11MockServerTestCase as Http11MockServerTestCase, + TestHttp11MockServer as TestHttp11MockServer, ) from tests.test_downloader_handlers import ( - Http11ProxyTestCase as Http11ProxyTestCase, + TestHttp11Proxy as TestHttp11Proxy, ) from tests.test_downloader_handlers import ( - Https11CustomCiphers as Https11CustomCiphers, + TestHttps11 as TestHttps11, ) from tests.test_downloader_handlers import ( - Https11InvalidDNSId as Https11InvalidDNSId, + TestHttps11CustomCiphers as TestHttps11CustomCiphers, ) from tests.test_downloader_handlers import ( - Https11InvalidDNSPattern as Https11InvalidDNSPattern, + TestHttps11InvalidDNSId as TestHttps11InvalidDNSId, ) from tests.test_downloader_handlers import ( - Https11TestCase as Https11TestCase, + TestHttps11InvalidDNSPattern as TestHttps11InvalidDNSPattern, ) from tests.test_downloader_handlers import ( - Https11WrongHostnameTestCase as Https11WrongHostnameTestCase, + TestHttps11WrongHostname as TestHttps11WrongHostname, ) @@ -56,7 +55,7 @@ def _get_dh() -> type[DownloadHandlerProtocol]: return H2DownloadHandler -class Https2TestCase(BaseTestClasses.Https11TestCase): +class TestHttps2(BaseTestClasses.TestHttps11): scheme = "https" HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" @@ -97,22 +96,22 @@ class Https2TestCase(BaseTestClasses.Https11TestCase): yield self.assertFailure(d, SchemeNotSupported) def test_download_broken_content_cause_data_loss(self, url="broken"): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) def test_download_broken_chunked_content_cause_data_loss(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) def test_download_broken_content_allow_data_loss(self, url="broken"): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) def test_download_broken_chunked_content_allow_data_loss(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) def test_download_broken_chunked_content_allow_data_loss_via_setting(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) def test_concurrent_requests_same_domain(self): spider = Spider("foo") @@ -180,31 +179,31 @@ class Https2TestCase(BaseTestClasses.Https11TestCase): return d -class Https2WrongHostnameTestCase(BaseTestClasses.Https11WrongHostnameTestCase): +class Https2WrongHostnameTestCase(BaseTestClasses.TestHttps11WrongHostname): @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: return _get_dh() -class Https2InvalidDNSId(BaseTestClasses.Https11InvalidDNSId): +class Https2InvalidDNSId(BaseTestClasses.TestHttps11InvalidDNSId): @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: return _get_dh() -class Https2InvalidDNSPattern(BaseTestClasses.Https11InvalidDNSPattern): +class Https2InvalidDNSPattern(BaseTestClasses.TestHttps11InvalidDNSPattern): @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: return _get_dh() -class Https2CustomCiphers(BaseTestClasses.Https11CustomCiphers): +class Https2CustomCiphers(BaseTestClasses.TestHttps11CustomCiphers): @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: return _get_dh() -class Http2MockServerTestCase(BaseTestClasses.Http11MockServerTestCase): +class Http2MockServerTestCase(BaseTestClasses.TestHttp11MockServer): """HTTP 2.0 test case with MockServer""" settings_dict = { @@ -215,7 +214,7 @@ class Http2MockServerTestCase(BaseTestClasses.Http11MockServerTestCase): is_secure = True -class Https2ProxyTestCase(BaseTestClasses.Http11ProxyTestCase): +class Https2ProxyTestCase(BaseTestClasses.TestHttp11Proxy): # only used for HTTPS tests keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 48728e078..f55cb6c97 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -54,11 +54,11 @@ class CustomFieldDataclass: age: int = dataclasses.field(metadata={"serializer": custom_serializer}) -class BaseItemExporterTest(unittest.TestCase): +class TestBaseItemExporter: item_class: type = MyItem custom_field_item_class: type = CustomFieldItem - def setUp(self): + def setup_method(self): self.i = self.item_class(name="John\xa3", age="22") self.output = BytesIO() self.ie = self._get_exporter() @@ -72,7 +72,7 @@ class BaseItemExporterTest(unittest.TestCase): def _assert_expected_item(self, exported_dict): for k, v in exported_dict.items(): exported_dict[k] = to_unicode(v) - self.assertEqual(self.i, self.item_class(**exported_dict)) + assert self.i == self.item_class(**exported_dict) def _get_nonstring_types_item(self): return { @@ -105,45 +105,40 @@ class BaseItemExporterTest(unittest.TestCase): def test_serialize_field(self): a = ItemAdapter(self.i) res = self.ie.serialize_field(a.get_field_meta("name"), "name", a["name"]) - self.assertEqual(res, "John\xa3") + assert res == "John\xa3" res = self.ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) - self.assertEqual(res, "22") + assert res == "22" def test_fields_to_export(self): ie = self._get_exporter(fields_to_export=["name"]) - self.assertEqual( - list(ie._get_serialized_fields(self.i)), [("name", "John\xa3")] - ) + assert list(ie._get_serialized_fields(self.i)) == [("name", "John\xa3")] ie = self._get_exporter(fields_to_export=["name"], encoding="latin-1") _, name = next(iter(ie._get_serialized_fields(self.i))) assert isinstance(name, str) - self.assertEqual(name, "John\xa3") + assert name == "John\xa3" ie = self._get_exporter(fields_to_export={"name": "名稱"}) - self.assertEqual( - list(ie._get_serialized_fields(self.i)), [("名稱", "John\xa3")] - ) + assert list(ie._get_serialized_fields(self.i)) == [("名稱", "John\xa3")] def test_field_custom_serializer(self): i = self.custom_field_item_class(name="John\xa3", age="22") a = ItemAdapter(i) ie = self._get_exporter() - self.assertEqual( - ie.serialize_field(a.get_field_meta("name"), "name", a["name"]), "John\xa3" - ) - self.assertEqual( - ie.serialize_field(a.get_field_meta("age"), "age", a["age"]), "24" + assert ( + ie.serialize_field(a.get_field_meta("name"), "name", a["name"]) + == "John\xa3" ) + assert ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) == "24" -class BaseItemExporterDataclassTest(BaseItemExporterTest): +class TestBaseItemExporterDataclass(TestBaseItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class PythonItemExporterTest(BaseItemExporterTest): +class TestPythonItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): return PythonItemExporter(**kwargs) @@ -157,16 +152,13 @@ class PythonItemExporterTest(BaseItemExporterTest): i3 = self.item_class(name="Jesus", age=i2) ie = self._get_exporter() exported = ie.export_item(i3) - self.assertEqual(type(exported), dict) - self.assertEqual( - exported, - { - "age": {"age": {"age": "22", "name": "Joseph"}, "name": "Maria"}, - "name": "Jesus", - }, - ) - self.assertEqual(type(exported["age"]), dict) - self.assertEqual(type(exported["age"]["age"]), dict) + assert isinstance(exported, dict) + assert exported == { + "age": {"age": {"age": "22", "name": "Joseph"}, "name": "Maria"}, + "name": "Jesus", + } + assert isinstance(exported["age"], dict) + assert isinstance(exported["age"]["age"], dict) def test_export_list(self): i1 = self.item_class(name="Joseph", age="22") @@ -174,15 +166,12 @@ class PythonItemExporterTest(BaseItemExporterTest): i3 = self.item_class(name="Jesus", age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) - self.assertEqual( - exported, - { - "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}], - "name": "Jesus", - }, - ) - self.assertEqual(type(exported["age"][0]), dict) - self.assertEqual(type(exported["age"][0]["age"][0]), dict) + assert exported == { + "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}], + "name": "Jesus", + } + assert isinstance(exported["age"][0], dict) + assert isinstance(exported["age"][0]["age"][0], dict) def test_export_item_dict_list(self): i1 = self.item_class(name="Joseph", age="22") @@ -190,29 +179,26 @@ class PythonItemExporterTest(BaseItemExporterTest): i3 = self.item_class(name="Jesus", age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) - self.assertEqual( - exported, - { - "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}], - "name": "Jesus", - }, - ) - self.assertEqual(type(exported["age"][0]), dict) - self.assertEqual(type(exported["age"][0]["age"][0]), dict) + assert exported == { + "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}], + "name": "Jesus", + } + assert isinstance(exported["age"][0], dict) + assert isinstance(exported["age"][0]["age"][0], dict) def test_nonstring_types_item(self): item = self._get_nonstring_types_item() ie = self._get_exporter() exported = ie.export_item(item) - self.assertEqual(exported, item) + assert exported == item -class PythonItemExporterDataclassTest(PythonItemExporterTest): +class TestPythonItemExporterDataclass(TestPythonItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class PprintItemExporterTest(BaseItemExporterTest): +class TestPprintItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): return PprintItemExporter(self.output, **kwargs) @@ -222,12 +208,12 @@ class PprintItemExporterTest(BaseItemExporterTest): ) -class PprintItemExporterDataclassTest(PprintItemExporterTest): +class TestPprintItemExporterDataclass(TestPprintItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class PickleItemExporterTest(BaseItemExporterTest): +class TestPickleItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): return PickleItemExporter(self.output, **kwargs) @@ -245,8 +231,8 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.finish_exporting() del ie # See the first “del self.ie” in this file for context. f.seek(0) - self.assertEqual(self.item_class(**pickle.load(f)), i1) - self.assertEqual(self.item_class(**pickle.load(f)), i2) + assert self.item_class(**pickle.load(f)) == i1 + assert self.item_class(**pickle.load(f)) == i2 def test_nonstring_types_item(self): item = self._get_nonstring_types_item() @@ -256,15 +242,15 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.export_item(item) ie.finish_exporting() del ie # See the first “del self.ie” in this file for context. - self.assertEqual(pickle.loads(fp.getvalue()), item) + assert pickle.loads(fp.getvalue()) == item -class PickleItemExporterDataclassTest(PickleItemExporterTest): +class TestPickleItemExporterDataclass(TestPickleItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class MarshalItemExporterTest(BaseItemExporterTest): +class TestMarshalItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): self.output = tempfile.TemporaryFile() return MarshalItemExporter(self.output, **kwargs) @@ -283,15 +269,15 @@ class MarshalItemExporterTest(BaseItemExporterTest): ie.finish_exporting() del ie # See the first “del self.ie” in this file for context. fp.seek(0) - self.assertEqual(marshal.load(fp), item) + assert marshal.load(fp) == item -class MarshalItemExporterDataclassTest(MarshalItemExporterTest): +class TestMarshalItemExporterDataclass(TestMarshalItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class CsvItemExporterTest(BaseItemExporterTest): +class TestCsvItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): self.output = tempfile.TemporaryFile() return CsvItemExporter(self.output, **kwargs) @@ -303,7 +289,7 @@ class CsvItemExporterTest(BaseItemExporterTest): for line in to_unicode(csv).splitlines(True) ] - return self.assertEqual(split_csv(first), split_csv(second), msg=msg) + assert split_csv(first) == split_csv(second), msg def _check_output(self): self.output.seek(0) @@ -406,12 +392,12 @@ class CsvItemExporterTest(BaseItemExporterTest): ) -class CsvItemExporterDataclassTest(CsvItemExporterTest): +class TestCsvItemExporterDataclass(TestCsvItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class XmlItemExporterTest(BaseItemExporterTest): +class TestXmlItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): return XmlItemExporter(self.output, **kwargs) @@ -426,7 +412,7 @@ class XmlItemExporterTest(BaseItemExporterTest): doc = lxml.etree.fromstring(xmlcontent) return xmltuple(doc) - return self.assertEqual(xmlsplit(first), xmlsplit(second), msg) + assert xmlsplit(first) == xmlsplit(second), msg def assertExportResult(self, item, expected_value): fp = BytesIO() @@ -517,12 +503,12 @@ class XmlItemExporterTest(BaseItemExporterTest): ) -class XmlItemExporterDataclassTest(XmlItemExporterTest): +class TestXmlItemExporterDataclass(TestXmlItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class JsonLinesItemExporterTest(BaseItemExporterTest): +class TestJsonLinesItemExporter(TestBaseItemExporter): _expected_nested: Any = { "name": "Jesus", "age": {"name": "Maria", "age": {"name": "Joseph", "age": "22"}}, @@ -533,7 +519,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): def _check_output(self): exported = json.loads(to_unicode(self.output.getvalue().strip())) - self.assertEqual(exported, ItemAdapter(self.i).asdict()) + assert exported == ItemAdapter(self.i).asdict() def test_nested_item(self): i1 = self.item_class(name="Joseph", age="22") @@ -544,7 +530,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie.finish_exporting() del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) - self.assertEqual(exported, self._expected_nested) + assert exported == self._expected_nested def test_extra_keywords(self): self.ie = self._get_exporter(sort_keys=True) @@ -561,23 +547,23 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) item["time"] = str(item["time"]) - self.assertEqual(exported, item) + assert exported == item -class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest): +class TestJsonLinesItemExporterDataclass(TestJsonLinesItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class JsonItemExporterTest(JsonLinesItemExporterTest): - _expected_nested = [JsonLinesItemExporterTest._expected_nested] +class TestJsonItemExporter(TestJsonLinesItemExporter): + _expected_nested = [TestJsonLinesItemExporter._expected_nested] def _get_exporter(self, **kwargs): return JsonItemExporter(self.output, **kwargs) def _check_output(self): exported = json.loads(to_unicode(self.output.getvalue().strip())) - self.assertEqual(exported, [ItemAdapter(self.i).asdict()]) + assert exported == [ItemAdapter(self.i).asdict()] def assertTwoItemsExported(self, item): self.ie.start_exporting() @@ -586,9 +572,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.finish_exporting() del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) - self.assertEqual( - exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()] - ) + assert exported == [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()] def test_two_items(self): self.assertTwoItemsExported(self.i) @@ -609,7 +593,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - self.assertEqual(exported, [dict(i1), dict(i3)]) + assert exported == [dict(i1), dict(i3)] def test_nested_item(self): i1 = self.item_class(name="Joseph\xa3", age="22") @@ -624,7 +608,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): "name": "Jesus", "age": {"name": "Maria", "age": ItemAdapter(i1).asdict()}, } - self.assertEqual(exported, [expected]) + assert exported == [expected] def test_nested_dict_item(self): i1 = {"name": "Joseph\xa3", "age": "22"} @@ -636,7 +620,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) expected = {"name": "Jesus", "age": {"name": "Maria", "age": i1}} - self.assertEqual(exported, [expected]) + assert exported == [expected] def test_nonstring_types_item(self): item = self._get_nonstring_types_item() @@ -646,10 +630,10 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) item["time"] = str(item["time"]) - self.assertEqual(exported, [item]) + assert exported == [item] -class JsonItemExporterToBytesTest(BaseItemExporterTest): +class TestJsonItemExporterToBytes(TestBaseItemExporter): def _get_exporter(self, **kwargs): kwargs["encoding"] = "latin" return JsonItemExporter(self.output, **kwargs) @@ -665,18 +649,18 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest): self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue(), encoding="latin")) - self.assertEqual(exported, [dict(i1), dict(i3)]) + assert exported == [dict(i1), dict(i3)] -class JsonItemExporterDataclassTest(JsonItemExporterTest): +class TestJsonItemExporterDataclass(TestJsonItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class CustomExporterItemTest(unittest.TestCase): +class TestCustomExporterItem: item_class: type = MyItem - def setUp(self): + def setup_method(self): if self.item_class is None: raise unittest.SkipTest("item class is None") @@ -691,17 +675,13 @@ class CustomExporterItemTest(unittest.TestCase): a = ItemAdapter(i) ie = CustomItemExporter() - self.assertEqual( - ie.serialize_field(a.get_field_meta("name"), "name", a["name"]), "John" - ) - self.assertEqual( - ie.serialize_field(a.get_field_meta("age"), "age", a["age"]), "23" - ) + assert ie.serialize_field(a.get_field_meta("name"), "name", a["name"]) == "John" + assert ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) == "23" i2 = {"name": "John", "age": "22"} - self.assertEqual(ie.serialize_field({}, "name", i2["name"]), "John") - self.assertEqual(ie.serialize_field({}, "age", i2["age"]), "23") + assert ie.serialize_field({}, "name", i2["name"]) == "John" + assert ie.serialize_field({}, "age", i2["age"]) == "23" -class CustomExporterDataclassTest(CustomExporterItemTest): +class TestCustomExporterDataclass(TestCustomExporterItem): item_class = MyDataClass diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8e008ab98..44cd10ec3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -88,7 +88,7 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]: return (client_mock, bucket_mock, blob_mock) -class FileFeedStorageTest(unittest.TestCase): +class TestFileFeedStorage(unittest.TestCase): def test_store_file_uri(self): path = Path(self.mktemp()).resolve() uri = path_to_file_uri(str(path)) @@ -137,14 +137,14 @@ class FileFeedStorageTest(unittest.TestCase): file = storage.open(spider) file.write(b"content") yield storage.store(file) - self.assertTrue(path.exists()) + assert path.exists() try: - self.assertEqual(path.read_bytes(), expected_content) + assert path.read_bytes() == expected_content finally: path.unlink() -class FTPFeedStorageTest(unittest.TestCase): +class TestFTPFeedStorage(unittest.TestCase): def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): name = "test_spider" @@ -166,9 +166,9 @@ class FTPFeedStorageTest(unittest.TestCase): return storage.store(file) def _assert_stored(self, path: Path, content): - self.assertTrue(path.exists()) + assert path.exists() try: - self.assertEqual(path.read_bytes(), content) + assert path.read_bytes() == content finally: path.unlink() @@ -216,10 +216,10 @@ class FTPFeedStorageTest(unittest.TestCase): # RFC3986: 3.2.1. User Information pw_quoted = quote(string.punctuation, safe="") st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {}) - self.assertEqual(st.password, string.punctuation) + assert st.password == string.punctuation -class BlockingFeedStorageTest(unittest.TestCase): +class TestBlockingFeedStorage: def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): name = "test_spider" @@ -232,7 +232,7 @@ class BlockingFeedStorageTest(unittest.TestCase): tmp = b.open(self.get_test_spider()) tmp_path = Path(tmp.name).parent - self.assertEqual(str(tmp_path), tempfile.gettempdir()) + assert str(tmp_path) == tempfile.gettempdir() def test_temp_file(self): b = BlockingFeedStorage() @@ -241,7 +241,7 @@ class BlockingFeedStorageTest(unittest.TestCase): spider = self.get_test_spider({"FEED_TEMPDIR": str(tests_path)}) tmp = b.open(spider) tmp_path = Path(tmp.name).parent - self.assertEqual(tmp_path, tests_path) + assert tmp_path == tests_path def test_invalid_folder(self): b = BlockingFeedStorage() @@ -255,7 +255,7 @@ class BlockingFeedStorageTest(unittest.TestCase): @pytest.mark.requires_boto3 -class S3FeedStorageTest(unittest.TestCase): +class TestS3FeedStorage(unittest.TestCase): def test_parse_credentials(self): aws_credentials = { "AWS_ACCESS_KEY_ID": "settings_key", @@ -268,9 +268,9 @@ class S3FeedStorageTest(unittest.TestCase): crawler, "s3://mybucket/export.csv", ) - self.assertEqual(storage.access_key, "settings_key") - self.assertEqual(storage.secret_key, "settings_secret") - self.assertEqual(storage.session_token, "settings_token") + assert storage.access_key == "settings_key" + assert storage.secret_key == "settings_secret" + assert storage.session_token == "settings_token" # Instantiate directly storage = S3FeedStorage( "s3://mybucket/export.csv", @@ -278,17 +278,17 @@ class S3FeedStorageTest(unittest.TestCase): aws_credentials["AWS_SECRET_ACCESS_KEY"], session_token=aws_credentials["AWS_SESSION_TOKEN"], ) - self.assertEqual(storage.access_key, "settings_key") - self.assertEqual(storage.secret_key, "settings_secret") - self.assertEqual(storage.session_token, "settings_token") + assert storage.access_key == "settings_key" + assert storage.secret_key == "settings_secret" + assert storage.session_token == "settings_token" # URI priority > settings priority storage = S3FeedStorage( "s3://uri_key:uri_secret@mybucket/export.csv", aws_credentials["AWS_ACCESS_KEY_ID"], aws_credentials["AWS_SECRET_ACCESS_KEY"], ) - self.assertEqual(storage.access_key, "uri_key") - self.assertEqual(storage.secret_key, "uri_secret") + assert storage.access_key == "uri_key" + assert storage.secret_key == "uri_secret" @defer.inlineCallbacks def test_store(self): @@ -306,24 +306,23 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(file) - self.assertEqual( - storage.s3_client.upload_fileobj.call_args, - mock.call(Bucket=bucket, Key=key, Fileobj=file), + assert storage.s3_client.upload_fileobj.call_args == mock.call( + Bucket=bucket, Key=key, Fileobj=file ) def test_init_without_acl(self): storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, None) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl is None def test_init_with_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, "custom-acl") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl == "custom-acl" def test_init_with_endpoint_url(self): storage = S3FeedStorage( @@ -332,9 +331,9 @@ class S3FeedStorageTest(unittest.TestCase): "secret_key", endpoint_url="https://example.com", ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.endpoint_url, "https://example.com") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.endpoint_url == "https://example.com" def test_init_with_region_name(self): region_name = "ap-east-1" @@ -344,10 +343,10 @@ class S3FeedStorageTest(unittest.TestCase): "secret_key", region_name=region_name, ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.region_name, region_name) - self.assertEqual(storage.s3_client._client_config.region_name, region_name) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.region_name == region_name + assert storage.s3_client._client_config.region_name == region_name def test_from_crawler_without_acl(self): settings = { @@ -359,9 +358,9 @@ class S3FeedStorageTest(unittest.TestCase): crawler, "s3://mybucket/export.csv", ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, None) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl is None def test_without_endpoint_url(self): settings = { @@ -373,9 +372,9 @@ class S3FeedStorageTest(unittest.TestCase): crawler, "s3://mybucket/export.csv", ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.endpoint_url, None) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.endpoint_url is None def test_without_region_name(self): settings = { @@ -387,9 +386,9 @@ class S3FeedStorageTest(unittest.TestCase): crawler, "s3://mybucket/export.csv", ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.s3_client._client_config.region_name, "us-east-1") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.s3_client._client_config.region_name == "us-east-1" def test_from_crawler_with_acl(self): settings = { @@ -402,9 +401,9 @@ class S3FeedStorageTest(unittest.TestCase): crawler, "s3://mybucket/export.csv", ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, "custom-acl") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl == "custom-acl" def test_from_crawler_with_endpoint_url(self): settings = { @@ -414,9 +413,9 @@ class S3FeedStorageTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings) storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.endpoint_url, "https://example.com") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.endpoint_url == "https://example.com" def test_from_crawler_with_region_name(self): region_name = "ap-east-1" @@ -427,10 +426,10 @@ class S3FeedStorageTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings) storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.region_name, region_name) - self.assertEqual(storage.s3_client._client_config.region_name, region_name) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.region_name == region_name + assert storage.s3_client._client_config.region_name == region_name @defer.inlineCallbacks def test_store_without_acl(self): @@ -439,9 +438,9 @@ class S3FeedStorageTest(unittest.TestCase): "access_key", "secret_key", ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, None) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl is None storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) @@ -450,28 +449,28 @@ class S3FeedStorageTest(unittest.TestCase): .get("ExtraArgs", {}) .get("ACL") ) - self.assertIsNone(acl) + assert acl is None @defer.inlineCallbacks def test_store_with_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, "custom-acl") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl == "custom-acl" storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] - self.assertEqual(acl, "custom-acl") + assert acl == "custom-acl" def test_overwrite_default(self): with LogCapture() as log: S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" ) - self.assertNotIn("S3 does not support appending to files", str(log)) + assert "S3 does not support appending to files" not in str(log) def test_overwrite_false(self): with LogCapture() as log: @@ -482,10 +481,10 @@ class S3FeedStorageTest(unittest.TestCase): "custom-acl", feed_options={"overwrite": False}, ) - self.assertIn("S3 does not support appending to files", str(log)) + assert "S3 does not support appending to files" in str(log) -class GCSFeedStorageTest(unittest.TestCase): +class TestGCSFeedStorage(unittest.TestCase): def test_parse_settings(self): try: from google.cloud.storage import Client # noqa: F401 @@ -543,7 +542,7 @@ class GCSFeedStorageTest(unittest.TestCase): def test_overwrite_default(self): with LogCapture() as log: GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl") - self.assertNotIn("GCS does not support appending to files", str(log)) + assert "GCS does not support appending to files" not in str(log) def test_overwrite_false(self): with LogCapture() as log: @@ -553,10 +552,10 @@ class GCSFeedStorageTest(unittest.TestCase): "custom-acl", feed_options={"overwrite": False}, ) - self.assertIn("GCS does not support appending to files", str(log)) + assert "GCS does not support appending to files" in str(log) -class StdoutFeedStorageTest(unittest.TestCase): +class TestStdoutFeedStorage(unittest.TestCase): @defer.inlineCallbacks def test_store(self): out = BytesIO() @@ -564,20 +563,21 @@ class StdoutFeedStorageTest(unittest.TestCase): file = storage.open(scrapy.Spider("default")) file.write(b"content") yield storage.store(file) - self.assertEqual(out.getvalue(), b"content") + assert out.getvalue() == b"content" def test_overwrite_default(self): with LogCapture() as log: StdoutFeedStorage("stdout:") - self.assertNotIn( - "Standard output (stdout) storage does not support overwriting", str(log) + assert ( + "Standard output (stdout) storage does not support overwriting" + not in str(log) ) def test_overwrite_true(self): with LogCapture() as log: StdoutFeedStorage("stdout:", feed_options={"overwrite": True}) - self.assertIn( - "Standard output (stdout) storage does not support overwriting", str(log) + assert "Standard output (stdout) storage does not support overwriting" in str( + log ) @@ -639,7 +639,7 @@ class LogOnStoreFileStorage: file.close() -class FeedExportTestBase(ABC, unittest.TestCase): +class TestFeedExportBase(ABC, unittest.TestCase): class MyItem(scrapy.Item): foo = scrapy.Field() egg = scrapy.Field() @@ -769,7 +769,7 @@ class ExceptionJsonItemExporter(JsonItemExporter): raise RuntimeError("foo") -class FeedExportTest(FeedExportTestBase): +class TestFeedExport(TestFeedExportBase): @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): """Run spider with specified settings; return exported data.""" @@ -812,8 +812,8 @@ class FeedExportTest(FeedExportTestBase): ) data = yield self.exported_data(items, settings) reader = csv.DictReader(to_unicode(data["csv"]).splitlines()) - self.assertEqual(reader.fieldnames, list(header)) - self.assertEqual(rows, list(reader)) + assert reader.fieldnames == list(header) + assert rows == list(reader) @defer.inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): @@ -828,7 +828,7 @@ class FeedExportTest(FeedExportTestBase): data = yield self.exported_data(items, settings) parsed = [json.loads(to_unicode(line)) for line in data["jl"].splitlines()] rows = [{k: v for k, v in row.items() if v} for row in rows] - self.assertEqual(rows, parsed) + assert rows == parsed @defer.inlineCallbacks def assertExportedXml(self, items, rows, settings=None): @@ -844,7 +844,7 @@ class FeedExportTest(FeedExportTestBase): rows = [{k: v for k, v in row.items() if v} for row in rows] root = lxml.etree.fromstring(data["xml"]) got_rows = [{e.tag: e.text for e in it} for it in root.findall("item")] - self.assertEqual(rows, got_rows) + assert rows == got_rows @defer.inlineCallbacks def assertExportedMultiple(self, items, rows, settings=None): @@ -862,10 +862,10 @@ class FeedExportTest(FeedExportTestBase): # XML root = lxml.etree.fromstring(data["xml"]) xml_rows = [{e.tag: e.text for e in it} for it in root.findall("item")] - self.assertEqual(rows, xml_rows) + assert rows == xml_rows # JSON json_rows = json.loads(to_unicode(data["json"])) - self.assertEqual(rows, json_rows) + assert rows == json_rows @defer.inlineCallbacks def assertExportedPickle(self, items, rows, settings=None): @@ -882,7 +882,7 @@ class FeedExportTest(FeedExportTestBase): import pickle result = self._load_until_eof(data["pickle"], load_func=pickle.load) - self.assertEqual(expected, result) + assert result == expected @defer.inlineCallbacks def assertExportedMarshal(self, items, rows, settings=None): @@ -899,7 +899,7 @@ class FeedExportTest(FeedExportTestBase): import marshal result = self._load_until_eof(data["marshal"], load_func=marshal.load) - self.assertEqual(expected, result) + assert result == expected @defer.inlineCallbacks def test_stats_file_success(self): @@ -912,12 +912,8 @@ class FeedExportTest(FeedExportTestBase): } crawler = get_crawler(ItemSpider, settings) yield crawler.crawl(mockserver=self.mockserver) - self.assertIn( - "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() - ) - self.assertEqual( - crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1 - ) + assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() + assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 @defer.inlineCallbacks def test_stats_file_failed(self): @@ -934,12 +930,8 @@ class FeedExportTest(FeedExportTestBase): side_effect=KeyError("foo"), ): yield crawler.crawl(mockserver=self.mockserver) - self.assertIn( - "feedexport/failed_count/FileFeedStorage", crawler.stats.get_stats() - ) - self.assertEqual( - crawler.stats.get_value("feedexport/failed_count/FileFeedStorage"), 1 - ) + assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats() + assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1 @defer.inlineCallbacks def test_stats_multiple_file(self): @@ -956,17 +948,11 @@ class FeedExportTest(FeedExportTestBase): crawler = get_crawler(ItemSpider, settings) with mock.patch.object(S3FeedStorage, "store"): yield crawler.crawl(mockserver=self.mockserver) - self.assertIn( - "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() - ) - self.assertIn( - "feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats() - ) - self.assertEqual( - crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1 - ) - self.assertEqual( - crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1 + assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() + assert "feedexport/success_count/StdoutFeedStorage" in crawler.stats.get_stats() + assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 + assert ( + crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage") == 1 ) @defer.inlineCallbacks @@ -993,7 +979,7 @@ class FeedExportTest(FeedExportTestBase): "FEED_STORE_EMPTY": False, } data = yield self.exported_no_data(settings) - self.assertEqual(None, data[fmt]) + assert data[fmt] is None @defer.inlineCallbacks def test_start_finish_exporting_items(self): @@ -1012,8 +998,8 @@ class FeedExportTest(FeedExportTestBase): with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): _ = yield self.exported_data(items, settings) - self.assertFalse(listener.start_without_finish) - self.assertFalse(listener.finish_without_start) + assert not listener.start_without_finish + assert not listener.finish_without_start @defer.inlineCallbacks def test_start_finish_exporting_no_items(self): @@ -1030,8 +1016,8 @@ class FeedExportTest(FeedExportTestBase): with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): _ = yield self.exported_data(items, settings) - self.assertFalse(listener.start_without_finish) - self.assertFalse(listener.finish_without_start) + assert not listener.start_without_finish + assert not listener.finish_without_start @defer.inlineCallbacks def test_start_finish_exporting_items_exception(self): @@ -1051,8 +1037,8 @@ class FeedExportTest(FeedExportTestBase): with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): _ = yield self.exported_data(items, settings) - self.assertFalse(listener.start_without_finish) - self.assertFalse(listener.finish_without_start) + assert not listener.start_without_finish + assert not listener.finish_without_start @defer.inlineCallbacks def test_start_finish_exporting_no_items_exception(self): @@ -1070,8 +1056,8 @@ class FeedExportTest(FeedExportTestBase): with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): _ = yield self.exported_data(items, settings) - self.assertFalse(listener.start_without_finish) - self.assertFalse(listener.finish_without_start) + assert not listener.start_without_finish + assert not listener.finish_without_start @defer.inlineCallbacks def test_export_no_items_store_empty(self): @@ -1091,7 +1077,7 @@ class FeedExportTest(FeedExportTestBase): "FEED_EXPORT_INDENT": None, } data = yield self.exported_no_data(settings) - self.assertEqual(expctd, data[fmt]) + assert expctd == data[fmt] @defer.inlineCallbacks def test_export_no_items_multiple_feeds(self): @@ -1109,7 +1095,7 @@ class FeedExportTest(FeedExportTestBase): with LogCapture() as log: yield self.exported_no_data(settings) - self.assertEqual(str(log).count("Storage.store is called"), 0) + assert str(log).count("Storage.store is called") == 0 @defer.inlineCallbacks def test_export_multiple_item_classes(self): @@ -1238,7 +1224,7 @@ class FeedExportTest(FeedExportTestBase): data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): - self.assertEqual(expected, data[fmt]) + assert data[fmt] == expected @defer.inlineCallbacks def test_export_based_on_custom_filters(self): @@ -1297,7 +1283,7 @@ class FeedExportTest(FeedExportTestBase): data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): - self.assertEqual(expected, data[fmt]) + assert data[fmt] == expected @defer.inlineCallbacks def test_export_dicts(self): @@ -1371,7 +1357,7 @@ class FeedExportTest(FeedExportTestBase): "FEED_EXPORT_INDENT": None, } data = yield self.exported_data(items, settings) - self.assertEqual(expected, data[fmt]) + assert data[fmt] == expected formats = { "json": b'[{"foo": "Test\xd6"}]', @@ -1392,7 +1378,7 @@ class FeedExportTest(FeedExportTestBase): "FEED_EXPORT_ENCODING": "latin-1", } data = yield self.exported_data(items, settings) - self.assertEqual(expected, data[fmt]) + assert data[fmt] == expected @defer.inlineCallbacks def test_export_multiple_configs(self): @@ -1432,7 +1418,7 @@ class FeedExportTest(FeedExportTestBase): data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): - self.assertEqual(expected, data[fmt]) + assert data[fmt] == expected @defer.inlineCallbacks def test_export_indentation(self): @@ -1588,7 +1574,7 @@ class FeedExportTest(FeedExportTestBase): }, } data = yield self.exported_data(items, settings) - self.assertEqual(row["expected"], data[row["format"]]) + assert data[row["format"]] == row["expected"] @defer.inlineCallbacks def test_init_exporters_storages_with_crawler(self): @@ -1600,8 +1586,8 @@ class FeedExportTest(FeedExportTestBase): }, } yield self.exported_data(items=[], settings=settings) - self.assertTrue(FromCrawlerCsvItemExporter.init_with_crawler) - self.assertTrue(FromCrawlerFileFeedStorage.init_with_crawler) + assert FromCrawlerCsvItemExporter.init_with_crawler + assert FromCrawlerFileFeedStorage.init_with_crawler @defer.inlineCallbacks def test_str_uri(self): @@ -1610,7 +1596,7 @@ class FeedExportTest(FeedExportTestBase): "FEEDS": {str(self._random_temp_filename()): {"format": "csv"}}, } data = yield self.exported_no_data(settings) - self.assertEqual(data["csv"], b"") + assert data["csv"] == b"" @defer.inlineCallbacks def test_multiple_feeds_success_logs_blocking_feed_storage(self): @@ -1631,7 +1617,7 @@ class FeedExportTest(FeedExportTestBase): print(log) for fmt in ["json", "xml", "csv"]: - self.assertIn(f"Stored {fmt} feed (2 items)", str(log)) + assert f"Stored {fmt} feed (2 items)" in str(log) @defer.inlineCallbacks def test_multiple_feeds_failing_logs_blocking_feed_storage(self): @@ -1652,7 +1638,7 @@ class FeedExportTest(FeedExportTestBase): print(log) for fmt in ["json", "xml", "csv"]: - self.assertIn(f"Error storing {fmt} feed (2 items)", str(log)) + assert f"Error storing {fmt} feed (2 items)" in str(log) @defer.inlineCallbacks def test_extend_kwargs(self): @@ -1689,7 +1675,7 @@ class FeedExportTest(FeedExportTestBase): } data = yield self.exported_data(items, settings) - self.assertEqual(row["expected"], data[feed_options["format"]]) + assert data[feed_options["format"]] == row["expected"] @defer.inlineCallbacks def test_storage_file_no_postprocessing(self): @@ -1711,7 +1697,7 @@ class FeedExportTest(FeedExportTestBase): "FEED_STORAGES": {"file": Storage}, } yield self.exported_no_data(settings) - self.assertIs(Storage.open_file, Storage.store_file) + assert Storage.open_file is Storage.store_file @defer.inlineCallbacks def test_storage_file_postprocessing(self): @@ -1741,11 +1727,11 @@ class FeedExportTest(FeedExportTestBase): "FEED_STORAGES": {"file": Storage}, } yield self.exported_no_data(settings) - self.assertIs(Storage.open_file, Storage.store_file) - self.assertFalse(Storage.file_was_closed) + assert Storage.open_file is Storage.store_file + assert not Storage.file_was_closed -class FeedPostProcessedExportsTest(FeedExportTestBase): +class TestFeedPostProcessedExports(TestFeedExportBase): items = [{"foo": "bar"}] expected = b"foo\r\nbar\r\n" @@ -1827,7 +1813,7 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): try: gzip.decompress(data[filename]) except OSError: - self.fail("Received invalid gzip data.") + pytest.fail("Received invalid gzip data.") @defer.inlineCallbacks def test_gzip_plugin_compresslevel(self): @@ -1863,8 +1849,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): for filename, compressed in filename_to_compressed.items(): result = gzip.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) + assert compressed == data[filename] + assert result == self.expected @defer.inlineCallbacks def test_gzip_plugin_mtime(self): @@ -1898,8 +1884,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): for filename, compressed in filename_to_compressed.items(): result = gzip.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) + assert compressed == data[filename] + assert result == self.expected @defer.inlineCallbacks def test_gzip_plugin_filename(self): @@ -1933,8 +1919,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): for filename, compressed in filename_to_compressed.items(): result = gzip.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) + assert compressed == data[filename] + assert result == self.expected @defer.inlineCallbacks def test_lzma_plugin(self): @@ -1953,7 +1939,7 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): try: lzma.decompress(data[filename]) except lzma.LZMAError: - self.fail("Received invalid lzma data.") + pytest.fail("Received invalid lzma data.") @defer.inlineCallbacks def test_lzma_plugin_format(self): @@ -1985,8 +1971,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): for filename, compressed in filename_to_compressed.items(): result = lzma.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) + assert compressed == data[filename] + assert result == self.expected @defer.inlineCallbacks def test_lzma_plugin_check(self): @@ -2018,8 +2004,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): for filename, compressed in filename_to_compressed.items(): result = lzma.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) + assert compressed == data[filename] + assert result == self.expected @defer.inlineCallbacks def test_lzma_plugin_preset(self): @@ -2051,8 +2037,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): for filename, compressed in filename_to_compressed.items(): result = lzma.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) + assert compressed == data[filename] + assert result == self.expected @defer.inlineCallbacks def test_lzma_plugin_filters(self): @@ -2075,9 +2061,9 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): } data = yield self.exported_data(self.items, settings) - self.assertEqual(compressed, data[filename]) + assert compressed == data[filename] result = lzma.decompress(data[filename]) - self.assertEqual(self.expected, result) + assert result == self.expected @defer.inlineCallbacks def test_bz2_plugin(self): @@ -2096,7 +2082,7 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): try: bz2.decompress(data[filename]) except OSError: - self.fail("Received invalid bz2 data.") + pytest.fail("Received invalid bz2 data.") @defer.inlineCallbacks def test_bz2_plugin_compresslevel(self): @@ -2128,8 +2114,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): for filename, compressed in filename_to_compressed.items(): result = bz2.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) + assert compressed == data[filename] + assert result == self.expected @defer.inlineCallbacks def test_custom_plugin(self): @@ -2145,7 +2131,7 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): } data = yield self.exported_data(self.items, settings) - self.assertEqual(self.expected, data[filename]) + assert data[filename] == self.expected @defer.inlineCallbacks def test_custom_plugin_with_parameter(self): @@ -2163,7 +2149,7 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): } data = yield self.exported_data(self.items, settings) - self.assertEqual(expected, data[filename]) + assert data[filename] == expected @defer.inlineCallbacks def test_custom_plugin_with_compression(self): @@ -2208,7 +2194,7 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): for filename, decompressor in filename_to_decompressor.items(): result = decompressor(data[filename]) - self.assertEqual(expected, result) + assert result == expected @defer.inlineCallbacks def test_exports_compatibility_with_postproc(self): @@ -2262,10 +2248,10 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): expected, result = self.items[0], marshal.loads(result) else: expected = filename_to_expected[filename] - self.assertEqual(expected, result) + assert result == expected -class BatchDeliveriesTest(FeedExportTestBase): +class TestBatchDeliveries(TestFeedExportBase): _file_mark = "_%(batch_time)s_#%(batch_id)02d_" @defer.inlineCallbacks @@ -2310,7 +2296,7 @@ class BatchDeliveriesTest(FeedExportTestBase): json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines() ] expected_batch, rows = rows[:batch_size], rows[batch_size:] - self.assertEqual(expected_batch, got_batch) + assert got_batch == expected_batch @defer.inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None): @@ -2328,9 +2314,9 @@ class BatchDeliveriesTest(FeedExportTestBase): data = yield self.exported_data(items, settings) for batch in data["csv"]: got_batch = csv.DictReader(to_unicode(batch).splitlines()) - self.assertEqual(list(header), got_batch.fieldnames) + assert list(header) == got_batch.fieldnames expected_batch, rows = rows[:batch_size], rows[batch_size:] - self.assertEqual(expected_batch, list(got_batch)) + assert list(got_batch) == expected_batch @defer.inlineCallbacks def assertExportedXml(self, items, rows, settings=None): @@ -2351,7 +2337,7 @@ class BatchDeliveriesTest(FeedExportTestBase): root = lxml.etree.fromstring(batch) got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] expected_batch, rows = rows[:batch_size], rows[batch_size:] - self.assertEqual(expected_batch, got_batch) + assert got_batch == expected_batch @defer.inlineCallbacks def assertExportedMultiple(self, items, rows, settings=None): @@ -2377,13 +2363,13 @@ class BatchDeliveriesTest(FeedExportTestBase): root = lxml.etree.fromstring(batch) got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:] - self.assertEqual(expected_batch, got_batch) + assert got_batch == expected_batch # JSON json_rows = rows.copy() for batch in data["json"]: got_batch = json.loads(batch.decode("utf-8")) expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] - self.assertEqual(expected_batch, got_batch) + assert got_batch == expected_batch @defer.inlineCallbacks def assertExportedPickle(self, items, rows, settings=None): @@ -2405,7 +2391,7 @@ class BatchDeliveriesTest(FeedExportTestBase): for batch in data["pickle"]: got_batch = self._load_until_eof(batch, load_func=pickle.load) expected_batch, rows = rows[:batch_size], rows[batch_size:] - self.assertEqual(expected_batch, got_batch) + assert got_batch == expected_batch @defer.inlineCallbacks def assertExportedMarshal(self, items, rows, settings=None): @@ -2427,7 +2413,7 @@ class BatchDeliveriesTest(FeedExportTestBase): for batch in data["marshal"]: got_batch = self._load_until_eof(batch, load_func=marshal.load) expected_batch, rows = rows[:batch_size], rows[batch_size:] - self.assertEqual(expected_batch, got_batch) + assert got_batch == expected_batch @defer.inlineCallbacks def test_export_items(self): @@ -2472,7 +2458,7 @@ class BatchDeliveriesTest(FeedExportTestBase): } data = yield self.exported_no_data(settings) data = dict(data) - self.assertEqual(0, len(data[fmt])) + assert len(data[fmt]) == 0 @defer.inlineCallbacks def test_export_no_items_store_empty(self): @@ -2496,7 +2482,7 @@ class BatchDeliveriesTest(FeedExportTestBase): } data = yield self.exported_no_data(settings) data = dict(data) - self.assertEqual(expctd, data[fmt][0]) + assert data[fmt][0] == expctd @defer.inlineCallbacks def test_export_multiple_configs(self): @@ -2552,7 +2538,7 @@ class BatchDeliveriesTest(FeedExportTestBase): data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): for expected_batch, got_batch in zip(expected, data[fmt]): - self.assertEqual(expected_batch, got_batch) + assert got_batch == expected_batch @defer.inlineCallbacks def test_batch_item_count_feeds_setting(self): @@ -2576,7 +2562,7 @@ class BatchDeliveriesTest(FeedExportTestBase): data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): for expected_batch, got_batch in zip(expected, data[fmt]): - self.assertEqual(expected_batch, got_batch) + assert got_batch == expected_batch @defer.inlineCallbacks def test_batch_path_differ(self): @@ -2598,7 +2584,7 @@ class BatchDeliveriesTest(FeedExportTestBase): "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } data = yield self.exported_data(items, settings) - self.assertEqual(len(items), len(data["json"])) + assert len(items) == len(data["json"]) @defer.inlineCallbacks def test_stats_batch_file_success(self): @@ -2614,12 +2600,8 @@ class BatchDeliveriesTest(FeedExportTestBase): } crawler = get_crawler(ItemSpider, settings) yield crawler.crawl(total=2, mockserver=self.mockserver) - self.assertIn( - "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() - ) - self.assertEqual( - crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12 - ) + assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() + assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12 @pytest.mark.requires_boto3 @defer.inlineCallbacks @@ -2687,13 +2669,13 @@ class BatchDeliveriesTest(FeedExportTestBase): crawler = get_crawler(TestSpider, settings) yield crawler.crawl() - self.assertEqual(len(CustomS3FeedStorage.stubs), len(items)) + assert len(CustomS3FeedStorage.stubs) == len(items) for stub in CustomS3FeedStorage.stubs[:-1]: stub.assert_no_pending_responses() # Test that the FeedExporer sends the feed_exporter_closed and feed_slot_closed signals -class FeedExporterSignalsTest(unittest.TestCase): +class TestFeedExporterSignals: items = [ {"foo": "bar1", "egg": "spam1"}, {"foo": "bar2", "egg": "spam2", "baz": "quux2"}, @@ -2754,8 +2736,8 @@ class FeedExporterSignalsTest(unittest.TestCase): self.feed_exporter_closed_signal_handler, self.feed_slot_closed_signal_handler, ) - self.assertTrue(self.feed_slot_closed_received) - self.assertTrue(self.feed_exporter_closed_received) + assert self.feed_slot_closed_received + assert self.feed_exporter_closed_received def test_feed_exporter_signals_sent_deferred(self): self.feed_exporter_closed_received = False @@ -2765,11 +2747,11 @@ class FeedExporterSignalsTest(unittest.TestCase): self.feed_exporter_closed_signal_handler_deferred, self.feed_slot_closed_signal_handler_deferred, ) - self.assertTrue(self.feed_slot_closed_received) - self.assertTrue(self.feed_exporter_closed_received) + assert self.feed_slot_closed_received + assert self.feed_exporter_closed_received -class FeedExportInitTest(unittest.TestCase): +class TestFeedExportInit: def test_unsupported_storage(self): settings = { "FEEDS": { @@ -2803,7 +2785,7 @@ class FeedExportInitTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings) exporter = FeedExporter.from_crawler(crawler) - self.assertIsInstance(exporter, FeedExporter) + assert isinstance(exporter, FeedExporter) def test_relative_pathlib_as_uri(self): settings = { @@ -2815,13 +2797,14 @@ class FeedExportInitTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings) exporter = FeedExporter.from_crawler(crawler) - self.assertIsInstance(exporter, FeedExporter) + assert isinstance(exporter, FeedExporter) -class URIParamsTest: +class TestURIParams(ABC): spider_name = "uri_params_spider" deprecated_options = False + @abstractmethod def build_settings(self, uri="file:///tmp/foobar", uri_params=None): raise NotImplementedError @@ -2850,7 +2833,7 @@ class URIParamsTest: warnings.simplefilter("error", ScrapyDeprecationWarning) feed_exporter.open_spider(spider) - self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" def test_none(self): def uri_params(params, spider): @@ -2866,7 +2849,7 @@ class URIParamsTest: feed_exporter.open_spider(spider) - self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" def test_empty_dict(self): def uri_params(params, spider): @@ -2900,7 +2883,7 @@ class URIParamsTest: warnings.simplefilter("error", ScrapyDeprecationWarning) feed_exporter.open_spider(spider) - self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" def test_custom_param(self): def uri_params(params, spider): @@ -2917,10 +2900,10 @@ class URIParamsTest: warnings.simplefilter("error", ScrapyDeprecationWarning) feed_exporter.open_spider(spider) - self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" -class URIParamsSettingTest(URIParamsTest, unittest.TestCase): +class TestURIParamsSetting(TestURIParams): deprecated_options = True def build_settings(self, uri="file:///tmp/foobar", uri_params=None): @@ -2933,7 +2916,7 @@ class URIParamsSettingTest(URIParamsTest, unittest.TestCase): } -class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase): +class TestURIParamsFeedOption(TestURIParams): deprecated_options = False def build_settings(self, uri="file:///tmp/foobar", uri_params=None): diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 0881bbeca..7c1b38877 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -185,7 +185,7 @@ def get_client_certificate( @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") -class Https2ClientProtocolTestCase(TestCase): +class TestHttps2ClientProtocol(TestCase): scheme = "https" key_file = Path(__file__).parent / "keys" / "localhost.key" certificate_file = Path(__file__).parent / "keys" / "localhost.crt" @@ -277,14 +277,14 @@ class Https2ClientProtocolTestCase(TestCase): def _check_GET(self, request: Request, expected_body, expected_status): def check_response(response: Response): - self.assertEqual(response.status, expected_status) - self.assertEqual(response.body, expected_body) - self.assertEqual(response.request, request) + assert response.status == expected_status + assert response.body == expected_body + assert response.request == request content_length_header = response.headers.get("Content-Length") assert content_length_header is not None content_length = int(content_length_header) - self.assertEqual(len(response.body), content_length) + assert len(response.body) == content_length d = self.make_request(request) d.addCallback(check_response) @@ -325,35 +325,35 @@ class Https2ClientProtocolTestCase(TestCase): d = self.make_request(request) def assert_response(response: Response): - self.assertEqual(response.status, expected_status) - self.assertEqual(response.request, request) + assert response.status == expected_status + assert response.request == request content_length_header = response.headers.get("Content-Length") assert content_length_header is not None content_length = int(content_length_header) - self.assertEqual(len(response.body), content_length) + assert len(response.body) == content_length # Parse the body content_encoding_header = response.headers[b"Content-Encoding"] assert content_encoding_header is not None content_encoding = str(content_encoding_header, "utf-8") body = json.loads(str(response.body, content_encoding)) - self.assertIn("request-body", body) - self.assertIn("extra-data", body) - self.assertIn("request-headers", body) + assert "request-body" in body + assert "extra-data" in body + assert "request-headers" in body request_body = body["request-body"] - self.assertEqual(request_body, expected_request_body) + assert request_body == expected_request_body extra_data = body["extra-data"] - self.assertEqual(extra_data, expected_extra_data) + assert extra_data == expected_extra_data # Check if headers were sent successfully request_headers = body["request-headers"] for k, v in request.headers.items(): k_str = str(k, "utf-8") - self.assertIn(k_str, request_headers) - self.assertEqual(request_headers[k_str], str(v[0], "utf-8")) + assert k_str in request_headers + assert request_headers[k_str] == str(v[0], "utf-8") d.addCallback(assert_response) d.addErrback(self.fail) @@ -414,8 +414,8 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(url=self.get_url("/get-data-html-large")) def assert_response(response: Response): - self.assertEqual(response.status, 499) - self.assertEqual(response.request, request) + assert response.status == 499 + assert response.request == request d = self.make_request(request) d.addCallback(assert_response) @@ -430,12 +430,12 @@ class Https2ClientProtocolTestCase(TestCase): ) def assert_cancelled_error(failure): - self.assertIsInstance(failure.value, CancelledError) + assert isinstance(failure.value, CancelledError) error_pattern = re.compile( rf"Cancelling download of {request.url}: received response " rf"size \(\d*\) larger than download max size \(1000\)" ) - self.assertEqual(len(re.findall(error_pattern, str(failure.value))), 1) + assert len(re.findall(error_pattern, str(failure.value))) == 1 d = self.make_request(request) d.addCallback(self.fail) @@ -448,14 +448,12 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(url=self.get_url("/dataloss")) def assert_failure(failure: Failure): - self.assertTrue(len(failure.value.reasons) > 0) + assert len(failure.value.reasons) > 0 from h2.exceptions import InvalidBodyLengthError - self.assertTrue( - any( - isinstance(error, InvalidBodyLengthError) - for error in failure.value.reasons - ) + assert any( + isinstance(error, InvalidBodyLengthError) + for error in failure.value.reasons ) d = self.make_request(request) @@ -467,10 +465,10 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(url=self.get_url("/no-content-length-header")) def assert_content_length(response: Response): - self.assertEqual(response.status, 200) - self.assertEqual(response.body, Data.NO_CONTENT_LENGTH) - self.assertEqual(response.request, request) - self.assertNotIn("Content-Length", response.headers) + assert response.status == 200 + assert response.body == Data.NO_CONTENT_LENGTH + assert response.request == request + assert "Content-Length" not in response.headers d = self.make_request(request) d.addCallback(assert_content_length) @@ -481,14 +479,12 @@ class Https2ClientProtocolTestCase(TestCase): def _check_log_warnsize(self, request, warn_pattern, expected_body): with self.assertLogs("scrapy.core.http2.stream", level="WARNING") as cm: response = yield self.make_request(request) - self.assertEqual(response.status, 200) - self.assertEqual(response.request, request) - self.assertEqual(response.body, expected_body) + assert response.status == 200 + assert response.request == request + assert response.body == expected_body # Check the warning is raised only once for this request - self.assertEqual( - sum(len(re.findall(warn_pattern, log)) for log in cm.output), 1 - ) + assert sum(len(re.findall(warn_pattern, log)) for log in cm.output) == 1 @inlineCallbacks def test_log_expected_warnsize(self): @@ -534,11 +530,11 @@ class Https2ClientProtocolTestCase(TestCase): d_list = [] def assert_inactive_stream(failure): - self.assertIsNotNone(failure.check(ResponseFailed)) + assert failure.check(ResponseFailed) is not None from scrapy.core.http2.stream import InactiveStreamClosed - self.assertTrue( - any(isinstance(e, InactiveStreamClosed) for e in failure.value.reasons) + assert any( + isinstance(e, InactiveStreamClosed) for e in failure.value.reasons ) # Send 100 request (we do not check the result) @@ -578,7 +574,7 @@ class Https2ClientProtocolTestCase(TestCase): assert content_encoding_header is not None content_encoding = str(content_encoding_header, "utf-8") data = json.loads(str(response.body, content_encoding)) - self.assertEqual(data, params) + assert data == params d = self.make_request(request) d.addCallback(assert_query_params) @@ -588,7 +584,7 @@ class Https2ClientProtocolTestCase(TestCase): def test_status_codes(self): def assert_response_status(response: Response, expected_status: int): - self.assertEqual(response.status, expected_status) + assert response.status == expected_status d_list = [] for status in [200, 404]: @@ -604,21 +600,18 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(self.get_url("/status?n=200")) def assert_metadata(response: Response): - self.assertEqual(response.request, request) - self.assertIsInstance(response.certificate, Certificate) - assert response.certificate # typing - self.assertIsNotNone(response.certificate.original) - self.assertEqual( - response.certificate.getIssuer(), self.client_certificate.getIssuer() + assert response.request == request + assert isinstance(response.certificate, Certificate) + assert response.certificate.original is not None + assert ( + response.certificate.getIssuer() == self.client_certificate.getIssuer() ) - self.assertTrue( - response.certificate.getPublicKey().matches( - self.client_certificate.getPublicKey() - ) + assert response.certificate.getPublicKey().matches( + self.client_certificate.getPublicKey() ) - self.assertIsInstance(response.ip_address, IPv4Address) - self.assertEqual(str(response.ip_address), "127.0.0.1") + assert isinstance(response.ip_address, IPv4Address) + assert str(response.ip_address) == "127.0.0.1" d = self.make_request(request) d.addCallback(assert_metadata) @@ -632,11 +625,11 @@ class Https2ClientProtocolTestCase(TestCase): def assert_invalid_hostname(failure: Failure): from scrapy.core.http2.stream import InvalidHostname - self.assertIsNotNone(failure.check(InvalidHostname)) + assert failure.check(InvalidHostname) is not None error_msg = str(failure.value) - self.assertIn("localhost", error_msg) - self.assertIn("127.0.0.1", error_msg) - self.assertIn(str(request), error_msg) + assert "localhost" in error_msg + assert "127.0.0.1" in error_msg + assert str(request) in error_msg d = self.make_request(request) d.addCallback(self.fail) @@ -672,13 +665,13 @@ class Https2ClientProtocolTestCase(TestCase): from scrapy.core.http2.protocol import H2ClientProtocol if isinstance(err, TimeoutError): - self.assertIn( - f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", - str(err), + assert ( + f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s" + in str(err) ) break else: - self.fail() + pytest.fail("No TimeoutError raised.") d.addCallback(self.fail) d.addErrback(assert_timeout_error) @@ -692,15 +685,15 @@ class Https2ClientProtocolTestCase(TestCase): d = self.make_request(request) def assert_request_headers(response: Response): - self.assertEqual(response.status, 200) - self.assertEqual(response.request, request) + assert response.status == 200 + assert response.request == request response_headers = json.loads(str(response.body, "utf-8")) - self.assertIsInstance(response_headers, dict) + assert isinstance(response_headers, dict) for k, v in request.headers.items(): k, v = str(k, "utf-8"), str(v[0], "utf-8") - self.assertIn(k, response_headers) - self.assertEqual(v, response_headers[k]) + assert k in response_headers + assert v == response_headers[k] d.addErrback(self.fail) d.addCallback(assert_request_headers) diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 932644320..660b76d08 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -1,74 +1,72 @@ -from unittest import TestCase - from scrapy.http import Request, Response from scrapy.http.cookies import WrappedRequest, WrappedResponse from scrapy.utils.httpobj import urlparse_cached -class WrappedRequestTest(TestCase): - def setUp(self): +class TestWrappedRequest: + def setup_method(self): self.request = Request( "http://www.example.com/page.html", headers={"Content-Type": "text/html"} ) self.wrapped = WrappedRequest(self.request) def test_get_full_url(self): - self.assertEqual(self.wrapped.get_full_url(), self.request.url) - self.assertEqual(self.wrapped.full_url, self.request.url) + assert self.wrapped.get_full_url() == self.request.url + assert self.wrapped.full_url == self.request.url def test_get_host(self): - self.assertEqual(self.wrapped.get_host(), urlparse_cached(self.request).netloc) - self.assertEqual(self.wrapped.host, urlparse_cached(self.request).netloc) + assert self.wrapped.get_host() == urlparse_cached(self.request).netloc + assert self.wrapped.host == urlparse_cached(self.request).netloc def test_get_type(self): - self.assertEqual(self.wrapped.get_type(), urlparse_cached(self.request).scheme) - self.assertEqual(self.wrapped.type, urlparse_cached(self.request).scheme) + assert self.wrapped.get_type() == urlparse_cached(self.request).scheme + assert self.wrapped.type == urlparse_cached(self.request).scheme def test_is_unverifiable(self): - self.assertFalse(self.wrapped.is_unverifiable()) - self.assertFalse(self.wrapped.unverifiable) + assert not self.wrapped.is_unverifiable() + assert not self.wrapped.unverifiable def test_is_unverifiable2(self): self.request.meta["is_unverifiable"] = True - self.assertTrue(self.wrapped.is_unverifiable()) - self.assertTrue(self.wrapped.unverifiable) + assert self.wrapped.is_unverifiable() + assert self.wrapped.unverifiable def test_get_origin_req_host(self): - self.assertEqual(self.wrapped.origin_req_host, "www.example.com") + assert self.wrapped.origin_req_host == "www.example.com" def test_has_header(self): - self.assertTrue(self.wrapped.has_header("content-type")) - self.assertFalse(self.wrapped.has_header("xxxxx")) + assert self.wrapped.has_header("content-type") + assert not self.wrapped.has_header("xxxxx") def test_get_header(self): - self.assertEqual(self.wrapped.get_header("content-type"), "text/html") - self.assertEqual(self.wrapped.get_header("xxxxx", "def"), "def") - self.assertEqual(self.wrapped.get_header("xxxxx"), None) + assert self.wrapped.get_header("content-type") == "text/html" + assert self.wrapped.get_header("xxxxx", "def") == "def" + assert self.wrapped.get_header("xxxxx") is None wrapped = WrappedRequest( Request( "http://www.example.com/page.html", headers={"empty-binary-header": b""} ) ) - self.assertEqual(wrapped.get_header("empty-binary-header"), "") + assert wrapped.get_header("empty-binary-header") == "" def test_header_items(self): - self.assertEqual(self.wrapped.header_items(), [("Content-Type", ["text/html"])]) + assert self.wrapped.header_items() == [("Content-Type", ["text/html"])] def test_add_unredirected_header(self): self.wrapped.add_unredirected_header("hello", "world") - self.assertEqual(self.request.headers["hello"], b"world") + assert self.request.headers["hello"] == b"world" -class WrappedResponseTest(TestCase): - def setUp(self): +class TestWrappedResponse: + def setup_method(self): self.response = Response( "http://www.example.com/page.html", headers={"Content-TYpe": "text/html"} ) self.wrapped = WrappedResponse(self.response) def test_info(self): - self.assertIs(self.wrapped.info(), self.wrapped) + assert self.wrapped.info() is self.wrapped def test_get_all(self): # get_all result must be native string - self.assertEqual(self.wrapped.get_all("content-type"), ["text/html"]) + assert self.wrapped.get_all("content-type") == ["text/html"] diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 0bbbcda46..2fcf9e83c 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -1,14 +1,13 @@ import copy -import unittest import pytest from scrapy.http import Headers -class HeadersTest(unittest.TestCase): +class TestHeaders: def assertSortedEqual(self, first, second, msg=None): - return self.assertEqual(sorted(first), sorted(second), msg) + assert sorted(first) == sorted(second), msg def test_basics(self): h = Headers({"Content-Type": "text/html", "Content-Length": 1234}) @@ -17,53 +16,53 @@ class HeadersTest(unittest.TestCase): with pytest.raises(KeyError): h["Accept"] - self.assertEqual(h.get("Accept"), None) - self.assertEqual(h.getlist("Accept"), []) + assert h.get("Accept") is None + assert h.getlist("Accept") == [] - self.assertEqual(h.get("Accept", "*/*"), b"*/*") - self.assertEqual(h.getlist("Accept", "*/*"), [b"*/*"]) - self.assertEqual( - h.getlist("Accept", ["text/html", "images/jpeg"]), - [b"text/html", b"images/jpeg"], - ) + assert h.get("Accept", "*/*") == b"*/*" + assert h.getlist("Accept", "*/*") == [b"*/*"] + assert h.getlist("Accept", ["text/html", "images/jpeg"]) == [ + b"text/html", + b"images/jpeg", + ] def test_single_value(self): h = Headers() h["Content-Type"] = "text/html" - self.assertEqual(h["Content-Type"], b"text/html") - self.assertEqual(h.get("Content-Type"), b"text/html") - self.assertEqual(h.getlist("Content-Type"), [b"text/html"]) + assert h["Content-Type"] == b"text/html" + assert h.get("Content-Type") == b"text/html" + assert h.getlist("Content-Type") == [b"text/html"] def test_multivalue(self): h = Headers() h["X-Forwarded-For"] = hlist = ["ip1", "ip2"] - self.assertEqual(h["X-Forwarded-For"], b"ip2") - self.assertEqual(h.get("X-Forwarded-For"), b"ip2") - self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1", b"ip2"]) + assert h["X-Forwarded-For"] == b"ip2" + assert h.get("X-Forwarded-For") == b"ip2" + assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"] assert h.getlist("X-Forwarded-For") is not hlist def test_multivalue_for_one_header(self): h = Headers((("a", "b"), ("a", "c"))) - self.assertEqual(h["a"], b"c") - self.assertEqual(h.get("a"), b"c") - self.assertEqual(h.getlist("a"), [b"b", b"c"]) + assert h["a"] == b"c" + assert h.get("a") == b"c" + assert h.getlist("a") == [b"b", b"c"] def test_encode_utf8(self): h = Headers({"key": "\xa3"}, encoding="utf-8") key, val = dict(h).popitem() assert isinstance(key, bytes), key assert isinstance(val[0], bytes), val[0] - self.assertEqual(val[0], b"\xc2\xa3") + assert val[0] == b"\xc2\xa3" def test_encode_latin1(self): h = Headers({"key": "\xa3"}, encoding="latin1") key, val = dict(h).popitem() - self.assertEqual(val[0], b"\xa3") + assert val[0] == b"\xa3" def test_encode_multiple(self): h = Headers({"key": ["\xa3"]}, encoding="utf-8") key, val = dict(h).popitem() - self.assertEqual(val[0], b"\xc2\xa3") + assert val[0] == b"\xc2\xa3" def test_delete_and_contains(self): h = Headers() @@ -81,17 +80,17 @@ class HeadersTest(unittest.TestCase): h = Headers() olist = h.setdefault("X-Forwarded-For", "ip1") - self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1"]) + assert h.getlist("X-Forwarded-For") == [b"ip1"] assert h.getlist("X-Forwarded-For") is olist def test_iterables(self): idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]} h = Headers(idict) - self.assertDictEqual( - dict(h), - {b"Content-Type": [b"text/html"], b"X-Forwarded-For": [b"ip1", b"ip2"]}, - ) + assert dict(h) == { + b"Content-Type": [b"text/html"], + b"X-Forwarded-For": [b"ip1", b"ip2"], + } self.assertSortedEqual(h.keys(), [b"X-Forwarded-For", b"Content-Type"]) self.assertSortedEqual( h.items(), @@ -102,57 +101,57 @@ class HeadersTest(unittest.TestCase): def test_update(self): h = Headers() h.update({"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]}) - self.assertEqual(h.getlist("Content-Type"), [b"text/html"]) - self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1", b"ip2"]) + assert h.getlist("Content-Type") == [b"text/html"] + assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"] def test_copy(self): h1 = Headers({"header1": ["value1", "value2"]}) h2 = copy.copy(h1) - self.assertEqual(h1, h2) - self.assertEqual(h1.getlist("header1"), h2.getlist("header1")) + assert h1 == h2 + assert h1.getlist("header1") == h2.getlist("header1") assert h1.getlist("header1") is not h2.getlist("header1") assert isinstance(h2, Headers) def test_appendlist(self): h1 = Headers({"header1": "value1"}) h1.appendlist("header1", "value3") - self.assertEqual(h1.getlist("header1"), [b"value1", b"value3"]) + assert h1.getlist("header1") == [b"value1", b"value3"] h1 = Headers() h1.appendlist("header1", "value1") h1.appendlist("header1", "value3") - self.assertEqual(h1.getlist("header1"), [b"value1", b"value3"]) + assert h1.getlist("header1") == [b"value1", b"value3"] def test_setlist(self): h1 = Headers({"header1": "value1"}) - self.assertEqual(h1.getlist("header1"), [b"value1"]) + assert h1.getlist("header1") == [b"value1"] h1.setlist("header1", [b"value2", b"value3"]) - self.assertEqual(h1.getlist("header1"), [b"value2", b"value3"]) + assert h1.getlist("header1") == [b"value2", b"value3"] def test_setlistdefault(self): h1 = Headers({"header1": "value1"}) h1.setlistdefault("header1", ["value2", "value3"]) h1.setlistdefault("header2", ["value2", "value3"]) - self.assertEqual(h1.getlist("header1"), [b"value1"]) - self.assertEqual(h1.getlist("header2"), [b"value2", b"value3"]) + assert h1.getlist("header1") == [b"value1"] + assert h1.getlist("header2") == [b"value2", b"value3"] def test_none_value(self): h1 = Headers() h1["foo"] = "bar" h1["foo"] = None h1.setdefault("foo", "bar") - self.assertEqual(h1.get("foo"), None) - self.assertEqual(h1.getlist("foo"), []) + assert h1.get("foo") is None + assert h1.getlist("foo") == [] def test_int_value(self): h1 = Headers({"hey": 5}) h1["foo"] = 1 h1.setdefault("bar", 2) h1.setlist("buz", [1, "dos", 3]) - self.assertEqual(h1.getlist("foo"), [b"1"]) - self.assertEqual(h1.getlist("bar"), [b"2"]) - self.assertEqual(h1.getlist("buz"), [b"1", b"dos", b"3"]) - self.assertEqual(h1.getlist("hey"), [b"5"]) + assert h1.getlist("foo") == [b"1"] + assert h1.getlist("bar") == [b"2"] + assert h1.getlist("buz") == [b"1", b"dos", b"3"] + assert h1.getlist("hey") == [b"5"] def test_invalid_value(self): with pytest.raises(TypeError, match="Unsupported value type"): From d442227fa74e414f4c7ac6baea6c3c4a1d938219 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 9 Mar 2025 23:24:12 +0400 Subject: [PATCH 071/491] Converting tests to plain asserts, part 8. (#6711) --- tests/test_http_request.py | 608 +++++++++++++++----------------- tests/test_http_response.py | 316 ++++++++--------- tests/test_loader.py | 280 +++++++-------- tests/test_settings/__init__.py | 309 ++++++++-------- 4 files changed, 718 insertions(+), 795 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index e5291157d..6bf0b8e3f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,6 +1,5 @@ import json import re -import unittest import warnings import xmlrpc.client from typing import Any @@ -22,7 +21,7 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode -class RequestTest(unittest.TestCase): +class TestRequest: request_class = Request default_method = "GET" default_headers: dict[bytes, list[bytes]] = {} @@ -40,12 +39,12 @@ class RequestTest(unittest.TestCase): r = self.request_class("http://www.example.com") assert isinstance(r.url, str) - self.assertEqual(r.url, "http://www.example.com") - self.assertEqual(r.method, self.default_method) + assert r.url == "http://www.example.com" + assert r.method == self.default_method assert isinstance(r.headers, Headers) - self.assertEqual(r.headers, self.default_headers) - self.assertEqual(r.meta, self.default_meta) + assert r.headers == self.default_headers + assert r.meta == self.default_meta meta = {"lala": "lolo"} headers = {b"caca": b"coco"} @@ -54,9 +53,9 @@ class RequestTest(unittest.TestCase): ) assert r.meta is not meta - self.assertEqual(r.meta, meta) + assert r.meta == meta assert r.headers is not headers - self.assertEqual(r.headers[b"caca"], b"coco") + assert r.headers[b"caca"] == b"coco" def test_url_scheme(self): # This test passes by not raising any (ValueError) exception @@ -83,61 +82,61 @@ class RequestTest(unittest.TestCase): r = self.request_class(url=url, headers=headers) p = self.request_class(url=url, headers=r.headers) - self.assertEqual(r.headers, p.headers) - self.assertFalse(r.headers is headers) - self.assertFalse(p.headers is r.headers) + assert r.headers == p.headers + assert r.headers is not headers + assert p.headers is not r.headers # headers must not be unicode h = Headers({"key1": "val1", "key2": "val2"}) h["newkey"] = "newval" for k, v in h.items(): - self.assertIsInstance(k, bytes) + assert isinstance(k, bytes) for s in v: - self.assertIsInstance(s, bytes) + assert isinstance(s, bytes) def test_eq(self): url = "http://www.scrapy.org" r1 = self.request_class(url=url) r2 = self.request_class(url=url) - self.assertNotEqual(r1, r2) + assert r1 != r2 set_ = set() set_.add(r1) set_.add(r2) - self.assertEqual(len(set_), 2) + assert len(set_) == 2 def test_url(self): r = self.request_class(url="http://www.scrapy.org/path") - self.assertEqual(r.url, "http://www.scrapy.org/path") + assert r.url == "http://www.scrapy.org/path" def test_url_quoting(self): r = self.request_class(url="http://www.scrapy.org/blank%20space") - self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") + assert r.url == "http://www.scrapy.org/blank%20space" r = self.request_class(url="http://www.scrapy.org/blank space") - self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") + assert r.url == "http://www.scrapy.org/blank%20space" def test_url_encoding(self): r = self.request_class(url="http://www.scrapy.org/price/£") - self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + assert r.url == "http://www.scrapy.org/price/%C2%A3" def test_url_encoding_other(self): # encoding affects only query part of URI, not path # path part should always be UTF-8 encoded before percent-escaping r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8") - self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + assert r.url == "http://www.scrapy.org/price/%C2%A3" r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1") - self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + assert r.url == "http://www.scrapy.org/price/%C2%A3" def test_url_encoding_query(self): r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ") - self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") + assert r1.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" # should be same as above r2 = self.request_class( url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8" ) - self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") + assert r2.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" def test_url_encoding_query_latin1(self): # encoding is used for encoding query-string before percent-escaping; @@ -145,7 +144,7 @@ class RequestTest(unittest.TestCase): r3 = self.request_class( url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1" ) - self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3") + assert r3.url == "http://www.scrapy.org/price/%C2%B5?currency=%A3" def test_url_encoding_nonutf8_untouched(self): # percent-escaping sequences that do not match valid UTF-8 sequences @@ -164,16 +163,16 @@ class RequestTest(unittest.TestCase): # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different # URI from "http://www.example.org/r%E9sum%E9.html". r1 = self.request_class(url="http://www.scrapy.org/price/%a3") - self.assertEqual(r1.url, "http://www.scrapy.org/price/%a3") + assert r1.url == "http://www.scrapy.org/price/%a3" r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") - self.assertEqual(r2.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + assert r2.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3") - self.assertEqual(r3.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + assert r3.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") - self.assertEqual(r4.url, "http://www.example.org/r%E9sum%E9.html") + assert r4.url == "http://www.example.org/r%E9sum%E9.html" def test_body(self): r1 = self.request_class(url="http://www.example.com/") @@ -181,19 +180,19 @@ class RequestTest(unittest.TestCase): r2 = self.request_class(url="http://www.example.com/", body=b"") assert isinstance(r2.body, bytes) - self.assertEqual(r2.encoding, "utf-8") # default encoding + assert r2.encoding == "utf-8" # default encoding r3 = self.request_class( url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8" ) assert isinstance(r3.body, bytes) - self.assertEqual(r3.body, b"Price: \xc2\xa3100") + assert r3.body == b"Price: \xc2\xa3100" r4 = self.request_class( url="http://www.example.com/", body="Price: \xa3100", encoding="latin1" ) assert isinstance(r4.body, bytes) - self.assertEqual(r4.body, b"Price: \xa3100") + assert r4.body == b"Price: \xa3100" def test_copy(self): """Test Request copy""" @@ -219,25 +218,25 @@ class RequestTest(unittest.TestCase): # make sure flags list is shallow copied assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" - self.assertEqual(r1.flags, r2.flags) + assert r1.flags == r2.flags # make sure cb_kwargs dict is shallow copied assert r1.cb_kwargs is not r2.cb_kwargs, ( "cb_kwargs must be a shallow copy, not identical" ) - self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) + assert r1.cb_kwargs == r2.cb_kwargs # make sure meta dict is shallow copied assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" - self.assertEqual(r1.meta, r2.meta) + assert r1.meta == r2.meta # make sure headers attribute is shallow copied assert r1.headers is not r2.headers, ( "headers must be a shallow copy, not identical" ) - self.assertEqual(r1.headers, r2.headers) - self.assertEqual(r1.encoding, r2.encoding) - self.assertEqual(r1.dont_filter, r2.dont_filter) + assert r1.headers == r2.headers + assert r1.encoding == r2.encoding + assert r1.dont_filter == r2.dont_filter # Request.body can be identical since it's an immutable object (str) @@ -258,10 +257,10 @@ class RequestTest(unittest.TestCase): hdrs = Headers(r1.headers) hdrs[b"key"] = b"value" r2 = r1.replace(method="POST", body="New body", headers=hdrs) - self.assertEqual(r1.url, r2.url) - self.assertEqual((r1.method, r2.method), ("GET", "POST")) - self.assertEqual((r1.body, r2.body), (b"", b"New body")) - self.assertEqual((r1.headers, r2.headers), (self.default_headers, hdrs)) + assert r1.url == r2.url + assert (r1.method, r2.method) == ("GET", "POST") + assert (r1.body, r2.body) == (b"", b"New body") + assert (r1.headers, r2.headers) == (self.default_headers, hdrs) # Empty attributes (which may fail if not compared properly) r3 = self.request_class( @@ -270,9 +269,9 @@ class RequestTest(unittest.TestCase): r4 = r3.replace( url="http://www.example.com/2", body=b"", meta={}, dont_filter=False ) - self.assertEqual(r4.url, "http://www.example.com/2") - self.assertEqual(r4.body, b"") - self.assertEqual(r4.meta, {}) + assert r4.url == "http://www.example.com/2" + assert r4.body == b"" + assert r4.meta == {} assert r4.dont_filter is False def test_method_always_str(self): @@ -291,32 +290,32 @@ class RequestTest(unittest.TestCase): pass r1 = self.request_class("http://example.com") - self.assertIsNone(r1.callback) - self.assertIsNone(r1.errback) + assert r1.callback is None + assert r1.errback is None r2 = self.request_class("http://example.com", callback=a_function) - self.assertIs(r2.callback, a_function) - self.assertIsNone(r2.errback) + assert r2.callback is a_function + assert r2.errback is None r3 = self.request_class("http://example.com", errback=a_function) - self.assertIsNone(r3.callback) - self.assertIs(r3.errback, a_function) + assert r3.callback is None + assert r3.errback is a_function r4 = self.request_class( url="http://example.com", callback=a_function, errback=a_function, ) - self.assertIs(r4.callback, a_function) - self.assertIs(r4.errback, a_function) + assert r4.callback is a_function + assert r4.errback is a_function r5 = self.request_class( url="http://example.com", callback=NO_CALLBACK, errback=NO_CALLBACK, ) - self.assertIs(r5.callback, NO_CALLBACK) - self.assertIs(r5.errback, NO_CALLBACK) + assert r5.callback is NO_CALLBACK + assert r5.errback is NO_CALLBACK def test_callback_and_errback_type(self): with pytest.raises(TypeError): @@ -354,53 +353,46 @@ class RequestTest(unittest.TestCase): "2%3A15&comments=' --compressed" ) r = self.request_class.from_curl(curl_command) - self.assertEqual(r.method, "POST") - self.assertEqual(r.url, "http://httpbin.org/post") - self.assertEqual( - r.body, - b"custname=John+Smith&custtel=500&custemail=jsmith%40" + assert r.method == "POST" + assert r.url == "http://httpbin.org/post" + assert ( + r.body == b"custname=John+Smith&custtel=500&custemail=jsmith%40" b"example.org&size=small&topping=cheese&topping=onion" - b"&delivery=12%3A15&comments=", - ) - self.assertEqual( - r.cookies, - { - "_gauges_unique_year": "1", - "_gauges_unique": "1", - "_gauges_unique_month": "1", - "_gauges_unique_hour": "1", - "_gauges_unique_day": "1", - }, - ) - self.assertEqual( - r.headers, - { - b"Origin": [b"http://httpbin.org"], - b"Accept-Encoding": [b"gzip, deflate"], - b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"], - b"Upgrade-Insecure-Requests": [b"1"], - b"User-Agent": [ - b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537." - b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202" - b".75 Chrome/62.0.3202.75 Safari/537.36" - ], - b"Content-Type": [b"application /x-www-form-urlencoded"], - b"Accept": [ - b"text/html,application/xhtml+xml,application/xml;q=0." - b"9,image/webp,image/apng,*/*;q=0.8" - ], - b"Cache-Control": [b"max-age=0"], - b"Referer": [b"http://httpbin.org/forms/post"], - b"Connection": [b"keep-alive"], - }, + b"&delivery=12%3A15&comments=" ) + assert r.cookies == { + "_gauges_unique_year": "1", + "_gauges_unique": "1", + "_gauges_unique_month": "1", + "_gauges_unique_hour": "1", + "_gauges_unique_day": "1", + } + assert r.headers == { + b"Origin": [b"http://httpbin.org"], + b"Accept-Encoding": [b"gzip, deflate"], + b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"], + b"Upgrade-Insecure-Requests": [b"1"], + b"User-Agent": [ + b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537." + b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202" + b".75 Chrome/62.0.3202.75 Safari/537.36" + ], + b"Content-Type": [b"application /x-www-form-urlencoded"], + b"Accept": [ + b"text/html,application/xhtml+xml,application/xml;q=0." + b"9,image/webp,image/apng,*/*;q=0.8" + ], + b"Cache-Control": [b"max-age=0"], + b"Referer": [b"http://httpbin.org/forms/post"], + b"Connection": [b"keep-alive"], + } def test_from_curl_with_kwargs(self): r = self.request_class.from_curl( 'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"} ) - self.assertEqual(r.method, "POST") - self.assertEqual(r.meta, {"key": "value"}) + assert r.method == "POST" + assert r.meta == {"key": "value"} def test_from_curl_ignore_unknown_options(self): # By default: it works and ignores the unknown options: --foo and -z @@ -409,7 +401,7 @@ class RequestTest(unittest.TestCase): r = self.request_class.from_curl( 'curl -X DELETE "http://example.org" --foo -z', ) - self.assertEqual(r.method, "DELETE") + assert r.method == "DELETE" # If `ignore_unknown_options` is set to `False` it raises an error with # the unknown options: --foo and -z @@ -420,17 +412,17 @@ class RequestTest(unittest.TestCase): ) -class FormRequestTest(RequestTest): +class TestFormRequest(TestRequest): request_class = FormRequest def assertQueryEqual(self, first, second, msg=None): first = to_unicode(first).split("&") second = to_unicode(second).split("&") - return self.assertEqual(sorted(first), sorted(second), msg) + assert sorted(first) == sorted(second), msg def test_empty_formdata(self): r1 = self.request_class("http://www.example.com", formdata={}) - self.assertEqual(r1.body, b"") + assert r1.body == b"" def test_formdata_overrides_querystring(self): data = (("a", "one"), ("a", "two"), ("b", "2")) @@ -438,69 +430,61 @@ class FormRequestTest(RequestTest): "http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data ).url.split("#", maxsplit=1)[0] fs = _qs(self.request_class(url, method="GET", formdata=data)) - self.assertEqual(set(fs[b"a"]), {b"one", b"two"}) - self.assertEqual(fs[b"b"], [b"2"]) - self.assertIsNone(fs.get(b"c")) + assert set(fs[b"a"]) == {b"one", b"two"} + assert fs[b"b"] == [b"2"] + assert fs.get(b"c") is None data = {"a": "1", "b": "2"} fs = _qs( self.request_class("http://www.example.com/", method="GET", formdata=data) ) - self.assertEqual(fs[b"a"], [b"1"]) - self.assertEqual(fs[b"b"], [b"2"]) + assert fs[b"a"] == [b"1"] + assert fs[b"b"] == [b"2"] def test_default_encoding_bytes(self): # using default encoding (utf-8) data = {b"one": b"two", b"price": b"\xc2\xa3 100"} r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, "POST") - self.assertEqual(r2.encoding, "utf-8") + assert r2.method == "POST" + assert r2.encoding == "utf-8" self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two") - self.assertEqual( - r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" - ) + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_default_encoding_textual_data(self): # using default encoding (utf-8) data = {"µ one": "two", "price": "£ 100"} r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, "POST") - self.assertEqual(r2.encoding, "utf-8") + assert r2.method == "POST" + assert r2.encoding == "utf-8" self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") - self.assertEqual( - r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" - ) + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_default_encoding_mixed_data(self): # using default encoding (utf-8) data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, "POST") - self.assertEqual(r2.encoding, "utf-8") + assert r2.method == "POST" + assert r2.encoding == "utf-8" self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") - self.assertEqual( - r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" - ) + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_custom_encoding_bytes(self): data = {b"\xb5 one": b"two", b"price": b"\xa3 100"} r2 = self.request_class( "http://www.example.com", formdata=data, encoding="latin1" ) - self.assertEqual(r2.method, "POST") - self.assertEqual(r2.encoding, "latin1") + assert r2.method == "POST" + assert r2.encoding == "latin1" self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two") - self.assertEqual( - r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" - ) + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_custom_encoding_textual_data(self): data = {"price": "£ 100"} r3 = self.request_class( "http://www.example.com", formdata=data, encoding="latin1" ) - self.assertEqual(r3.encoding, "latin1") - self.assertEqual(r3.body, b"price=%A3+100") + assert r3.encoding == "latin1" + assert r3.body == b"price=%A3+100" def test_multi_key_values(self): # using multiples values for a single key @@ -523,16 +507,14 @@ class FormRequestTest(RequestTest): response, formdata={"one": ["two", "three"], "six": "seven"} ) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers[b"Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req) - self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"}) - self.assertEqual(set(fs[b"one"]), {b"two", b"three"}) - self.assertEqual(fs[b"test2"], [b"xxx"]) - self.assertEqual(fs[b"six"], [b"seven"]) + assert set(fs[b"test"]) == {b"val1", b"val2"} + assert set(fs[b"one"]) == {b"two", b"three"} + assert fs[b"test2"] == [b"xxx"] + assert fs[b"six"] == [b"seven"] def test_from_response_post_nonascii_bytes_utf8(self): response = _buildresponse( @@ -547,16 +529,14 @@ class FormRequestTest(RequestTest): response, formdata={"one": ["two", "three"], "six": "seven"} ) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers[b"Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs["test £"]), {"val1", "val2"}) - self.assertEqual(set(fs["one"]), {"two", "three"}) - self.assertEqual(fs["test2"], ["xxx µ"]) - self.assertEqual(fs["six"], ["seven"]) + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] def test_from_response_post_nonascii_bytes_latin1(self): response = _buildresponse( @@ -572,16 +552,14 @@ class FormRequestTest(RequestTest): response, formdata={"one": ["two", "three"], "six": "seven"} ) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers[b"Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req, to_unicode=True, encoding="latin1") - self.assertEqual(set(fs["test £"]), {"val1", "val2"}) - self.assertEqual(set(fs["one"]), {"two", "three"}) - self.assertEqual(fs["test2"], ["xxx µ"]) - self.assertEqual(fs["six"], ["seven"]) + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] def test_from_response_post_nonascii_unicode(self): response = _buildresponse( @@ -596,16 +574,14 @@ class FormRequestTest(RequestTest): response, formdata={"one": ["two", "three"], "six": "seven"} ) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers[b"Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs["test £"]), {"val1", "val2"}) - self.assertEqual(set(fs["one"]), {"two", "three"}) - self.assertEqual(fs["test2"], ["xxx µ"]) - self.assertEqual(fs["six"], ["seven"]) + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] def test_from_response_duplicate_form_key(self): response = _buildresponse("<form></form>", url="http://www.example.com") @@ -614,8 +590,8 @@ class FormRequestTest(RequestTest): method="GET", formdata=(("foo", "bar"), ("foo", "baz")), ) - self.assertEqual(urlparse_cached(req).hostname, "www.example.com") - self.assertEqual(urlparse_cached(req).query, "foo=bar&foo=baz") + assert urlparse_cached(req).hostname == "www.example.com" + assert urlparse_cached(req).query == "foo=bar&foo=baz" def test_from_response_override_duplicate_form_key(self): response = _buildresponse( @@ -628,8 +604,8 @@ class FormRequestTest(RequestTest): response, formdata=(("two", "2"), ("two", "4")) ) fs = _qs(req) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertEqual(fs[b"two"], [b"2", b"4"]) + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2", b"4"] def test_from_response_extra_headers(self): response = _buildresponse( @@ -644,11 +620,9 @@ class FormRequestTest(RequestTest): formdata={"one": ["two", "three"], "six": "seven"}, headers={"Accept-Encoding": "gzip,deflate"}, ) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.headers["Accept-Encoding"], b"gzip,deflate") + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.headers["Accept-Encoding"] == b"gzip,deflate" def test_from_response_get(self): response = _buildresponse( @@ -662,14 +636,14 @@ class FormRequestTest(RequestTest): r1 = self.request_class.from_response( response, formdata={"one": ["two", "three"], "six": "seven"} ) - self.assertEqual(r1.method, "GET") - self.assertEqual(urlparse_cached(r1).hostname, "www.example.com") - self.assertEqual(urlparse_cached(r1).path, "/this/get.php") + assert r1.method == "GET" + assert urlparse_cached(r1).hostname == "www.example.com" + assert urlparse_cached(r1).path == "/this/get.php" fs = _qs(r1) - self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"}) - self.assertEqual(set(fs[b"one"]), {b"two", b"three"}) - self.assertEqual(fs[b"test2"], [b"xxx"]) - self.assertEqual(fs[b"six"], [b"seven"]) + assert set(fs[b"test"]) == {b"val1", b"val2"} + assert set(fs[b"one"]) == {b"two", b"three"} + assert fs[b"test2"] == [b"xxx"] + assert fs[b"six"] == [b"seven"] def test_from_response_override_params(self): response = _buildresponse( @@ -680,8 +654,8 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, formdata={"two": "2"}) fs = _qs(req) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertEqual(fs[b"two"], [b"2"]) + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] def test_from_response_drop_params(self): response = _buildresponse( @@ -692,8 +666,8 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, formdata={"two": None}) fs = _qs(req) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertNotIn(b"two", fs) + assert fs[b"one"] == [b"1"] + assert b"two" not in fs def test_from_response_override_method(self): response = _buildresponse( @@ -702,9 +676,9 @@ class FormRequestTest(RequestTest): </body></html>""" ) request = FormRequest.from_response(response) - self.assertEqual(request.method, "GET") + assert request.method == "GET" request = FormRequest.from_response(response, method="POST") - self.assertEqual(request.method, "POST") + assert request.method == "POST" def test_from_response_override_url(self): response = _buildresponse( @@ -713,11 +687,11 @@ class FormRequestTest(RequestTest): </body></html>""" ) request = FormRequest.from_response(response) - self.assertEqual(request.url, "http://example.com/app") + assert request.url == "http://example.com/app" request = FormRequest.from_response(response, url="http://foo.bar/absolute") - self.assertEqual(request.url, "http://foo.bar/absolute") + assert request.url == "http://foo.bar/absolute" request = FormRequest.from_response(response, url="/relative") - self.assertEqual(request.url, "http://example.com/relative") + assert request.url == "http://example.com/relative" def test_from_response_case_insensitive(self): response = _buildresponse( @@ -729,9 +703,9 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response) fs = _qs(req) - self.assertEqual(fs[b"clickable1"], [b"clicked1"]) - self.assertFalse(b"i1" in fs, fs) # xpath in _get_inputs() - self.assertFalse(b"clickable2" in fs, fs) # xpath in _get_clickable() + assert fs[b"clickable1"] == [b"clicked1"] + assert b"i1" not in fs, fs # xpath in _get_inputs() + assert b"clickable2" not in fs, fs # xpath in _get_clickable() def test_from_response_submit_first_clickable(self): response = _buildresponse( @@ -744,10 +718,10 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, formdata={"two": "2"}) fs = _qs(req) - self.assertEqual(fs[b"clickable1"], [b"clicked1"]) - self.assertFalse(b"clickable2" in fs, fs) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertEqual(fs[b"two"], [b"2"]) + assert fs[b"clickable1"] == [b"clicked1"] + assert b"clickable2" not in fs, fs + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] def test_from_response_submit_not_first_clickable(self): response = _buildresponse( @@ -762,10 +736,10 @@ class FormRequestTest(RequestTest): response, formdata={"two": "2"}, clickdata={"name": "clickable2"} ) fs = _qs(req) - self.assertEqual(fs[b"clickable2"], [b"clicked2"]) - self.assertFalse(b"clickable1" in fs, fs) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertEqual(fs[b"two"], [b"2"]) + assert fs[b"clickable2"] == [b"clicked2"] + assert b"clickable1" not in fs, fs + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] def test_from_response_dont_submit_image_as_input(self): response = _buildresponse( @@ -777,7 +751,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v"]}) + assert fs == {b"i1": [b"i1v"]} def test_from_response_dont_submit_reset_as_input(self): response = _buildresponse( @@ -790,7 +764,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b"i2v"]}) + assert fs == {b"i1": [b"i1v"], b"i2": [b"i2v"]} def test_from_response_clickdata_does_not_ignore_image(self): response = _buildresponse( @@ -801,7 +775,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b"i2v"]}) + assert fs == {b"i1": [b"i1v"], b"i2": [b"i2v"]} def test_from_response_multiple_clickdata(self): response = _buildresponse( @@ -816,9 +790,9 @@ class FormRequestTest(RequestTest): response, clickdata={"name": "clickable", "value": "clicked2"} ) fs = _qs(req) - self.assertEqual(fs[b"clickable"], [b"clicked2"]) - self.assertEqual(fs[b"one"], [b"clicked1"]) - self.assertEqual(fs[b"two"], [b"clicked2"]) + assert fs[b"clickable"] == [b"clicked2"] + assert fs[b"one"] == [b"clicked1"] + assert fs[b"two"] == [b"clicked2"] def test_from_response_unicode_clickdata(self): response = _buildresponse( @@ -833,7 +807,7 @@ class FormRequestTest(RequestTest): response, clickdata={"name": "price in \u00a3"} ) fs = _qs(req, to_unicode=True) - self.assertTrue(fs["price in \u00a3"]) + assert fs["price in \u00a3"] def test_from_response_unicode_clickdata_latin1(self): response = _buildresponse( @@ -849,7 +823,7 @@ class FormRequestTest(RequestTest): response, clickdata={"name": "price in \u00a5"} ) fs = _qs(req, to_unicode=True, encoding="latin1") - self.assertTrue(fs["price in \u00a5"]) + assert fs["price in \u00a5"] def test_from_response_multiple_forms_clickdata(self): response = _buildresponse( @@ -867,9 +841,9 @@ class FormRequestTest(RequestTest): response, formname="form2", clickdata={"name": "clickable"} ) fs = _qs(req) - self.assertEqual(fs[b"clickable"], [b"clicked2"]) - self.assertEqual(fs[b"field2"], [b"value2"]) - self.assertFalse(b"field1" in fs, fs) + assert fs[b"clickable"] == [b"clicked2"] + assert fs[b"field2"] == [b"value2"] + assert b"field1" not in fs, fs def test_from_response_override_clickable(self): response = _buildresponse( @@ -879,7 +853,7 @@ class FormRequestTest(RequestTest): response, formdata={"clickme": "two"}, clickdata={"name": "clickme"} ) fs = _qs(req) - self.assertEqual(fs[b"clickme"], [b"two"]) + assert fs[b"clickme"] == [b"two"] def test_from_response_dont_click(self): response = _buildresponse( @@ -892,8 +866,8 @@ class FormRequestTest(RequestTest): ) r1 = self.request_class.from_response(response, dont_click=True) fs = _qs(r1) - self.assertFalse(b"clickable1" in fs, fs) - self.assertFalse(b"clickable2" in fs, fs) + assert b"clickable1" not in fs, fs + assert b"clickable2" not in fs, fs def test_from_response_ambiguous_clickdata(self): response = _buildresponse( @@ -934,8 +908,8 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, clickdata={"nr": 1}) fs = _qs(req) - self.assertIn(b"clickable2", fs) - self.assertNotIn(b"clickable1", fs) + assert b"clickable2" in fs + assert b"clickable1" not in fs def test_from_response_invalid_nr_index_clickdata(self): response = _buildresponse( @@ -962,7 +936,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, formdata={"bar": "buz"}) fs = _qs(req) - self.assertEqual(fs, {b"foo": [b"xxx"], b"bar": [b"buz"]}) + assert fs == {b"foo": [b"xxx"], b"bar": [b"buz"]} def test_from_response_errors_formnumber(self): response = _buildresponse( @@ -983,12 +957,10 @@ class FormRequestTest(RequestTest): </form>""" ) r1 = self.request_class.from_response(response, formdata={"two": "3"}) - self.assertEqual(r1.method, "POST") - self.assertEqual( - r1.headers["Content-type"], b"application/x-www-form-urlencoded" - ) + assert r1.method == "POST" + assert r1.headers["Content-type"] == b"application/x-www-form-urlencoded" fs = _qs(r1) - self.assertEqual(fs, {b"one": [b"1"], b"two": [b"3"]}) + assert fs == {b"one": [b"1"], b"two": [b"3"]} def test_from_response_formname_exists(self): response = _buildresponse( @@ -1002,9 +974,9 @@ class FormRequestTest(RequestTest): </form>""" ) r1 = self.request_class.from_response(response, formname="form2") - self.assertEqual(r1.method, "POST") + assert r1.method == "POST" fs = _qs(r1) - self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) + assert fs == {b"four": [b"4"], b"three": [b"3"]} def test_from_response_formname_nonexistent(self): response = _buildresponse( @@ -1016,9 +988,9 @@ class FormRequestTest(RequestTest): </form>""" ) r1 = self.request_class.from_response(response, formname="form3") - self.assertEqual(r1.method, "POST") + assert r1.method == "POST" fs = _qs(r1) - self.assertEqual(fs, {b"one": [b"1"]}) + assert fs == {b"one": [b"1"]} def test_from_response_formname_errors_formnumber(self): response = _buildresponse( @@ -1044,9 +1016,9 @@ class FormRequestTest(RequestTest): </form>""" ) r1 = self.request_class.from_response(response, formid="form2") - self.assertEqual(r1.method, "POST") + assert r1.method == "POST" fs = _qs(r1) - self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) + assert fs == {b"four": [b"4"], b"three": [b"3"]} def test_from_response_formname_nonexistent_fallback_formid(self): response = _buildresponse( @@ -1062,9 +1034,9 @@ class FormRequestTest(RequestTest): r1 = self.request_class.from_response( response, formname="form3", formid="form2" ) - self.assertEqual(r1.method, "POST") + assert r1.method == "POST" fs = _qs(r1) - self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) + assert fs == {b"four": [b"4"], b"three": [b"3"]} def test_from_response_formid_nonexistent(self): response = _buildresponse( @@ -1076,9 +1048,9 @@ class FormRequestTest(RequestTest): </form>""" ) r1 = self.request_class.from_response(response, formid="form3") - self.assertEqual(r1.method, "POST") + assert r1.method == "POST" fs = _qs(r1) - self.assertEqual(fs, {b"one": [b"1"]}) + assert fs == {b"one": [b"1"]} def test_from_response_formid_errors_formnumber(self): response = _buildresponse( @@ -1122,7 +1094,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req, to_unicode=True) - self.assertEqual(fs, {"i1": ["i1v2"], "i2": ["i2v1"], "i4": ["i4v2", "i4v3"]}) + assert fs == {"i1": ["i1v2"], "i2": ["i2v1"], "i4": ["i4v2", "i4v3"]} def test_from_response_radio(self): res = _buildresponse( @@ -1139,7 +1111,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"iv2"], b"i2": [b"on"]}) + assert fs == {b"i1": [b"iv2"], b"i2": [b"on"]} def test_from_response_checkbox(self): res = _buildresponse( @@ -1156,7 +1128,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"iv2"], b"i2": [b"on"]}) + assert fs == {b"i1": [b"iv2"], b"i2": [b"on"]} def test_from_response_input_text(self): res = _buildresponse( @@ -1170,7 +1142,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v1"], b"i2": [b""], b"i4": [b"i4v1"]}) + assert fs == {b"i1": [b"i1v1"], b"i2": [b""], b"i4": [b"i4v1"]} def test_from_response_input_hidden(self): res = _buildresponse( @@ -1183,7 +1155,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v1"], b"i2": [b""]}) + assert fs == {b"i1": [b"i1v1"], b"i2": [b""]} def test_from_response_input_textarea(self): res = _buildresponse( @@ -1196,7 +1168,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b""], b"i3": [b""]}) + assert fs == {b"i1": [b"i1v"], b"i2": [b""], b"i3": [b""]} def test_from_response_descendants(self): res = _buildresponse( @@ -1218,7 +1190,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(set(fs), {b"h2", b"i2", b"i1", b"i3", b"h1", b"i5", b"i4"}) + assert set(fs) == {b"h2", b"i2", b"i1", b"i3", b"h1", b"i5", b"i4"} def test_from_response_xpath(self): response = _buildresponse( @@ -1235,13 +1207,13 @@ class FormRequestTest(RequestTest): response, formxpath="//form[@action='post.php']" ) fs = _qs(r1) - self.assertEqual(fs[b"one"], [b"1"]) + assert fs[b"one"] == [b"1"] r1 = self.request_class.from_response( response, formxpath="//form/input[@name='four']" ) fs = _qs(r1) - self.assertEqual(fs[b"three"], [b"3"]) + assert fs[b"three"] == [b"3"] with pytest.raises(ValueError, match="No <form> element found with"): self.request_class.from_response( @@ -1254,7 +1226,7 @@ class FormRequestTest(RequestTest): response, formxpath="//form[@name='\u044a']" ) fs = _qs(r) - self.assertEqual(fs, {}) + assert not fs xpath = "//form[@name='\u03b1']" with pytest.raises(ValueError, match=re.escape(xpath)): @@ -1270,15 +1242,13 @@ class FormRequestTest(RequestTest): url="http://www.example.com/this/list.html", ) req = self.request_class.from_response(response) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req) - self.assertEqual(fs[b"test1"], [b"val1"]) - self.assertEqual(fs[b"test2"], [b"val2"]) - self.assertEqual(fs[b"button1"], [b"submit1"]) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b"submit1"] def test_from_response_button_notype(self): response = _buildresponse( @@ -1290,15 +1260,13 @@ class FormRequestTest(RequestTest): url="http://www.example.com/this/list.html", ) req = self.request_class.from_response(response) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req) - self.assertEqual(fs[b"test1"], [b"val1"]) - self.assertEqual(fs[b"test2"], [b"val2"]) - self.assertEqual(fs[b"button1"], [b"submit1"]) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b"submit1"] def test_from_response_submit_novalue(self): response = _buildresponse( @@ -1310,15 +1278,13 @@ class FormRequestTest(RequestTest): url="http://www.example.com/this/list.html", ) req = self.request_class.from_response(response) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req) - self.assertEqual(fs[b"test1"], [b"val1"]) - self.assertEqual(fs[b"test2"], [b"val2"]) - self.assertEqual(fs[b"button1"], [b""]) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b""] def test_from_response_button_novalue(self): response = _buildresponse( @@ -1330,15 +1296,13 @@ class FormRequestTest(RequestTest): url="http://www.example.com/this/list.html", ) req = self.request_class.from_response(response) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req) - self.assertEqual(fs[b"test1"], [b"val1"]) - self.assertEqual(fs[b"test2"], [b"val2"]) - self.assertEqual(fs[b"button1"], [b""]) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b""] def test_html_base_form_action(self): response = _buildresponse( @@ -1356,12 +1320,12 @@ class FormRequestTest(RequestTest): url="http://a.com/", ) req = self.request_class.from_response(response) - self.assertEqual(req.url, "http://b.com/test_form") + assert req.url == "http://b.com/test_form" def test_spaces_in_action(self): resp = _buildresponse('<body><form action=" path\n"></form></body>') req = self.request_class.from_response(resp) - self.assertEqual(req.url, "http://example.com/path") + assert req.url == "http://example.com/path" def test_from_response_css(self): response = _buildresponse( @@ -1378,11 +1342,11 @@ class FormRequestTest(RequestTest): response, formcss="form[action='post.php']" ) fs = _qs(r1) - self.assertEqual(fs[b"one"], [b"1"]) + assert fs[b"one"] == [b"1"] r1 = self.request_class.from_response(response, formcss="input[name='four']") fs = _qs(r1) - self.assertEqual(fs[b"three"], [b"3"]) + assert fs[b"three"] == [b"3"] with pytest.raises(ValueError, match="No <form> element found with"): self.request_class.from_response(response, formcss="input[name='abc']") @@ -1400,7 +1364,7 @@ class FormRequestTest(RequestTest): "</form>" ) r = self.request_class.from_response(response) - self.assertEqual(r.method, expected) + assert r.method == expected def test_form_response_with_invalid_formdata_type_error(self): """Test that a ValueError is raised for non-iterable and non-dict formdata input""" @@ -1464,23 +1428,20 @@ def _qs(req, encoding="utf-8", to_unicode=False): return parse_qs(uqs, True) -class XmlRpcRequestTest(RequestTest): +class TestXmlRpcRequest(TestRequest): request_class = XmlRpcRequest default_method = "POST" default_headers = {b"Content-Type": [b"text/xml"]} def _test_request(self, **kwargs): r = self.request_class("http://scrapytest.org/rpc2", **kwargs) - self.assertEqual(r.headers[b"Content-Type"], b"text/xml") - self.assertEqual( - r.body, - to_bytes( - xmlrpc.client.dumps(**kwargs), encoding=kwargs.get("encoding", "utf-8") - ), + assert r.headers[b"Content-Type"] == b"text/xml" + assert r.body == to_bytes( + xmlrpc.client.dumps(**kwargs), encoding=kwargs.get("encoding", "utf-8") ) - self.assertEqual(r.method, "POST") - self.assertEqual(r.encoding, kwargs.get("encoding", "utf-8")) - self.assertTrue(r.dont_filter, True) + assert r.method == "POST" + assert r.encoding == kwargs.get("encoding", "utf-8") + assert r.dont_filter, True def test_xmlrpc_dumps(self): self._test_request(params=("value",)) @@ -1497,7 +1458,7 @@ class XmlRpcRequestTest(RequestTest): self._test_request(params=("pas£",), encoding="latin1") -class JsonRequestTest(RequestTest): +class TestJsonRequest(TestRequest): request_class = JsonRequest default_method = "GET" default_headers = { @@ -1505,49 +1466,51 @@ class JsonRequestTest(RequestTest): b"Accept": [b"application/json, text/javascript, */*; q=0.01"], } - def setUp(self): + def setup_method(self): warnings.simplefilter("always") - super().setUp() + + def teardown_method(self): + warnings.resetwarnings() def test_data(self): r1 = self.request_class(url="http://www.example.com/") - self.assertEqual(r1.body, b"") + assert r1.body == b"" body = b"body" r2 = self.request_class(url="http://www.example.com/", body=body) - self.assertEqual(r2.body, body) + assert r2.body == body data = { "name": "value", } r3 = self.request_class(url="http://www.example.com/", data=data) - self.assertEqual(r3.body, to_bytes(json.dumps(data))) + assert r3.body == to_bytes(json.dumps(data)) # empty data r4 = self.request_class(url="http://www.example.com/", data=[]) - self.assertEqual(r4.body, to_bytes(json.dumps([]))) + assert r4.body == to_bytes(json.dumps([])) def test_data_method(self): # data is not passed r1 = self.request_class(url="http://www.example.com/") - self.assertEqual(r1.method, "GET") + assert r1.method == "GET" body = b"body" r2 = self.request_class(url="http://www.example.com/", body=body) - self.assertEqual(r2.method, "GET") + assert r2.method == "GET" data = { "name": "value", } r3 = self.request_class(url="http://www.example.com/", data=data) - self.assertEqual(r3.method, "POST") + assert r3.method == "POST" # method passed explicitly r4 = self.request_class(url="http://www.example.com/", data=data, method="GET") - self.assertEqual(r4.method, "GET") + assert r4.method == "GET" r5 = self.request_class(url="http://www.example.com/", data=[]) - self.assertEqual(r5.method, "POST") + assert r5.method == "POST" def test_body_data(self): """passing both body and data should result a warning""" @@ -1557,10 +1520,10 @@ class JsonRequestTest(RequestTest): } with warnings.catch_warnings(record=True) as _warnings: r5 = self.request_class(url="http://www.example.com/", body=body, data=data) - self.assertEqual(r5.body, body) - self.assertEqual(r5.method, "GET") - self.assertEqual(len(_warnings), 1) - self.assertIn("data will be ignored", str(_warnings[0].message)) + assert r5.body == body + assert r5.method == "GET" + assert len(_warnings) == 1 + assert "data will be ignored" in str(_warnings[0].message) def test_empty_body_data(self): """passing any body value and data should result a warning""" @@ -1569,10 +1532,10 @@ class JsonRequestTest(RequestTest): } with warnings.catch_warnings(record=True) as _warnings: r6 = self.request_class(url="http://www.example.com/", body=b"", data=data) - self.assertEqual(r6.body, b"") - self.assertEqual(r6.method, "GET") - self.assertEqual(len(_warnings), 1) - self.assertIn("data will be ignored", str(_warnings[0].message)) + assert r6.body == b"" + assert r6.method == "GET" + assert len(_warnings) == 1 + assert "data will be ignored" in str(_warnings[0].message) def test_body_none_data(self): data = { @@ -1580,15 +1543,15 @@ class JsonRequestTest(RequestTest): } with warnings.catch_warnings(record=True) as _warnings: r7 = self.request_class(url="http://www.example.com/", body=None, data=data) - self.assertEqual(r7.body, to_bytes(json.dumps(data))) - self.assertEqual(r7.method, "POST") - self.assertEqual(len(_warnings), 0) + assert r7.body == to_bytes(json.dumps(data)) + assert r7.method == "POST" + assert len(_warnings) == 0 def test_body_data_none(self): with warnings.catch_warnings(record=True) as _warnings: r8 = self.request_class(url="http://www.example.com/", body=None, data=None) - self.assertEqual(r8.method, "GET") - self.assertEqual(len(_warnings), 0) + assert r8.method == "GET" + assert len(_warnings) == 0 def test_dumps_sort_keys(self): """Test that sort_keys=True is passed to json.dumps by default""" @@ -1598,7 +1561,7 @@ class JsonRequestTest(RequestTest): with mock.patch("json.dumps", return_value=b"") as mock_dumps: self.request_class(url="http://www.example.com/", data=data) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs["sort_keys"], True) + assert kwargs["sort_keys"] is True def test_dumps_kwargs(self): """Test that dumps_kwargs are passed to json.dumps""" @@ -1614,8 +1577,8 @@ class JsonRequestTest(RequestTest): url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs ) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs["ensure_ascii"], True) - self.assertEqual(kwargs["allow_nan"], True) + assert kwargs["ensure_ascii"] is True + assert kwargs["allow_nan"] is True def test_replace_data(self): data1 = { @@ -1626,7 +1589,7 @@ class JsonRequestTest(RequestTest): } r1 = self.request_class(url="http://www.example.com/", data=data1) r2 = r1.replace(data=data2) - self.assertEqual(r2.body, to_bytes(json.dumps(data2))) + assert r2.body == to_bytes(json.dumps(data2)) def test_replace_sort_keys(self): """Test that replace provides sort_keys=True to json.dumps""" @@ -1640,7 +1603,7 @@ class JsonRequestTest(RequestTest): with mock.patch("json.dumps", return_value=b"") as mock_dumps: r1.replace(data=data2) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs["sort_keys"], True) + assert kwargs["sort_keys"] is True def test_replace_dumps_kwargs(self): """Test that dumps_kwargs are provided to json.dumps when replace is called""" @@ -1660,8 +1623,8 @@ class JsonRequestTest(RequestTest): with mock.patch("json.dumps", return_value=b"") as mock_dumps: r1.replace(data=data2) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs["ensure_ascii"], True) - self.assertEqual(kwargs["allow_nan"], True) + assert kwargs["ensure_ascii"] is True + assert kwargs["allow_nan"] is True def test_replacement_both_body_and_data_warns(self): """Test that we get a warning if both body and data are passed""" @@ -1677,11 +1640,6 @@ class JsonRequestTest(RequestTest): with warnings.catch_warnings(record=True) as _warnings: r1.replace(data=data2, body=body2) - self.assertIn( - "Both body and data passed. data will be ignored", - str(_warnings[0].message), + assert "Both body and data passed. data will be ignored" in str( + _warnings[0].message ) - - def tearDown(self): - warnings.resetwarnings() - super().tearDown() diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 5a943f084..fdef5adea 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,5 +1,4 @@ import codecs -import unittest from unittest import mock import pytest @@ -22,62 +21,56 @@ from scrapy.utils.python import to_unicode from tests import get_testdata -class BaseResponseTest(unittest.TestCase): +class TestResponseBase: response_class = Response def test_init(self): # Response requires url in the constructor with pytest.raises(TypeError): self.response_class() - self.assertTrue( - isinstance(self.response_class("http://example.com/"), self.response_class) + assert isinstance( + self.response_class("http://example.com/"), self.response_class ) with pytest.raises(TypeError): self.response_class(b"http://example.com") with pytest.raises(TypeError): self.response_class(url="http://example.com", body={}) # body can be str or None - self.assertTrue( - isinstance( - self.response_class("http://example.com/", body=b""), - self.response_class, - ) + assert isinstance( + self.response_class("http://example.com/", body=b""), + self.response_class, ) - self.assertTrue( - isinstance( - self.response_class("http://example.com/", body=b"body"), - self.response_class, - ) + assert isinstance( + self.response_class("http://example.com/", body=b"body"), + self.response_class, ) # test presence of all optional parameters - self.assertTrue( - isinstance( - self.response_class( - "http://example.com/", body=b"", headers={}, status=200 - ), - self.response_class, - ) + assert isinstance( + self.response_class( + "http://example.com/", body=b"", headers={}, status=200 + ), + self.response_class, ) r = self.response_class("http://www.example.com") assert isinstance(r.url, str) - self.assertEqual(r.url, "http://www.example.com") - self.assertEqual(r.status, 200) + assert r.url == "http://www.example.com" + assert r.status == 200 assert isinstance(r.headers, Headers) - self.assertEqual(r.headers, {}) + assert not r.headers headers = {"foo": "bar"} body = b"a body" r = self.response_class("http://www.example.com", headers=headers, body=body) assert r.headers is not headers - self.assertEqual(r.headers[b"foo"], b"bar") + assert r.headers[b"foo"] == b"bar" r = self.response_class("http://www.example.com", status=301) - self.assertEqual(r.status, 301) + assert r.status == 301 r = self.response_class("http://www.example.com", status="301") - self.assertEqual(r.status, 301) + assert r.status == 301 with pytest.raises(ValueError, match=r"invalid literal for int\(\)"): self.response_class("http://example.com", status="lala200") @@ -88,18 +81,18 @@ class BaseResponseTest(unittest.TestCase): r1.flags.append("cached") r2 = r1.copy() - self.assertEqual(r1.status, r2.status) - self.assertEqual(r1.body, r2.body) + assert r1.status == r2.status + assert r1.body == r2.body # make sure flags list is shallow copied assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" - self.assertEqual(r1.flags, r2.flags) + assert r1.flags == r2.flags # make sure headers attribute is shallow copied assert r1.headers is not r2.headers, ( "headers must be a shallow copy, not identical" ) - self.assertEqual(r1.headers, r2.headers) + assert r1.headers == r2.headers def test_copy_meta(self): req = Request("http://www.example.com") @@ -144,16 +137,16 @@ class BaseResponseTest(unittest.TestCase): r1 = self.response_class("http://www.example.com") r2 = r1.replace(status=301, body=b"New body", headers=hdrs) assert r1.body == b"" - self.assertEqual(r1.url, r2.url) - self.assertEqual((r1.status, r2.status), (200, 301)) - self.assertEqual((r1.body, r2.body), (b"", b"New body")) - self.assertEqual((r1.headers, r2.headers), ({}, hdrs)) + assert r1.url == r2.url + assert (r1.status, r2.status) == (200, 301) + assert (r1.body, r2.body) == (b"", b"New body") + assert (r1.headers, r2.headers) == ({}, hdrs) # Empty attributes (which may fail if not compared properly) r3 = self.response_class("http://www.example.com", flags=["cached"]) r4 = r3.replace(body=b"", flags=[]) - self.assertEqual(r4.body, b"") - self.assertEqual(r4.flags, []) + assert r4.body == b"" + assert not r4.flags def _assert_response_values(self, response, encoding, body): if isinstance(body, str): @@ -166,11 +159,11 @@ class BaseResponseTest(unittest.TestCase): assert isinstance(response.body, bytes) assert isinstance(response.text, str) self._assert_response_encoding(response, encoding) - self.assertEqual(response.body, body_bytes) - self.assertEqual(response.text, body_unicode) + assert response.body == body_bytes + assert response.text == body_unicode def _assert_response_encoding(self, response, encoding): - self.assertEqual(response.encoding, resolve_encoding(encoding)) + assert response.encoding == resolve_encoding(encoding) def test_immutable_attributes(self): r = self.response_class("http://example.com") @@ -183,7 +176,7 @@ class BaseResponseTest(unittest.TestCase): """Test urljoin shortcut (only for existence, since behavior equals urljoin)""" joined = self.response_class("http://www.example.com").urljoin("/test") absolute = "http://www.example.com/test" - self.assertEqual(joined, absolute) + assert joined == absolute def test_shortcut_attributes(self): r = self.response_class("http://example.com", body=b"hello") @@ -241,7 +234,7 @@ class BaseResponseTest(unittest.TestCase): def test_follow_flags(self): res = self.response_class("http://example.com/") fol = res.follow("http://example.com/", flags=["cached", "allowed"]) - self.assertEqual(fol.flags, ["cached", "allowed"]) + assert fol.flags == ["cached", "allowed"] # Response.follow_all @@ -276,7 +269,7 @@ class BaseResponseTest(unittest.TestCase): def test_follow_all_empty(self): r = self.response_class("http://example.com") - self.assertEqual([], list(r.follow_all([]))) + assert not list(r.follow_all([])) def test_follow_all_invalid(self): r = self.response_class("http://example.com") @@ -327,13 +320,13 @@ class BaseResponseTest(unittest.TestCase): ] fol = re.follow_all(urls, flags=["cached", "allowed"]) for req in fol: - self.assertEqual(req.flags, ["cached", "allowed"]) + assert req.flags == ["cached", "allowed"] def _assert_followed_url(self, follow_obj, target_url, response=None): if response is None: response = self._links_response() req = response.follow(follow_obj) - self.assertEqual(req.url, target_url) + assert req.url == target_url return req def _assert_followed_all_urls(self, follow_obj, target_urls, response=None): @@ -341,7 +334,7 @@ class BaseResponseTest(unittest.TestCase): response = self._links_response() followed = response.follow_all(follow_obj) for req, target in zip(followed, target_urls): - self.assertEqual(req.url, target) + assert req.url == target yield req def _links_response(self): @@ -353,7 +346,7 @@ class BaseResponseTest(unittest.TestCase): return self.response_class("http://example.com/index", body=body) -class TextResponseTest(BaseResponseTest): +class TestTextResponse(TestResponseBase): response_class = TextResponse def test_replace(self): @@ -365,10 +358,10 @@ class TextResponseTest(BaseResponseTest): r3 = r1.replace(url="http://www.example.com/other", encoding="latin1") assert isinstance(r2, self.response_class) - self.assertEqual(r2.url, "http://www.example.com/other") + assert r2.url == "http://www.example.com/other" self._assert_response_encoding(r2, "cp852") - self.assertEqual(r3.url, "http://www.example.com/other") - self.assertEqual(r3._declared_encoding(), "latin1") + assert r3.url == "http://www.example.com/other" + assert r3._declared_encoding() == "latin1" def test_unicode_url(self): # instantiate with unicode url without encoding (should set default encoding) @@ -382,21 +375,21 @@ class TextResponseTest(BaseResponseTest): resp = self.response_class( url="http://www.example.com/price/\xa3", encoding="utf-8" ) - self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3")) + assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3") resp = self.response_class( url="http://www.example.com/price/\xa3", encoding="latin-1" ) - self.assertEqual(resp.url, "http://www.example.com/price/\xa3") + assert resp.url == "http://www.example.com/price/\xa3" resp = self.response_class( "http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]}, ) - self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3")) + assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3") resp = self.response_class( "http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, ) - self.assertEqual(resp.url, "http://www.example.com/price/\xa3") + assert resp.url == "http://www.example.com/price/\xa3" def test_unicode_body(self): unicode_string = ( @@ -412,8 +405,8 @@ class TextResponseTest(BaseResponseTest): ) # check response.text - self.assertTrue(isinstance(r1.text, str)) - self.assertEqual(r1.text, unicode_string) + assert isinstance(r1.text, str) + assert r1.text == unicode_string def test_encoding(self): r1 = self.response_class( @@ -458,18 +451,18 @@ class TextResponseTest(BaseResponseTest): }, ) - self.assertEqual(r1._headers_encoding(), "utf-8") - self.assertEqual(r2._headers_encoding(), None) - self.assertEqual(r2._declared_encoding(), "utf-8") + assert r1._headers_encoding() == "utf-8" + assert r2._headers_encoding() is None + assert r2._declared_encoding() == "utf-8" self._assert_response_encoding(r2, "utf-8") - self.assertEqual(r3._headers_encoding(), "cp1252") - self.assertEqual(r3._declared_encoding(), "cp1252") - self.assertEqual(r4._headers_encoding(), None) - self.assertEqual(r5._headers_encoding(), None) - self.assertEqual(r8._headers_encoding(), "cp1251") - self.assertEqual(r9._headers_encoding(), None) - self.assertEqual(r8._declared_encoding(), "utf-8") - self.assertEqual(r9._declared_encoding(), None) + assert r3._headers_encoding() == "cp1252" + assert r3._declared_encoding() == "cp1252" + assert r4._headers_encoding() is None + assert r5._headers_encoding() is None + assert r8._headers_encoding() == "cp1251" + assert r9._headers_encoding() is None + assert r8._declared_encoding() == "utf-8" + assert r9._declared_encoding() is None self._assert_response_encoding(r5, "utf-8") self._assert_response_encoding(r8, "utf-8") self._assert_response_encoding(r9, "cp1252") @@ -493,7 +486,7 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=UNKNOWN"]}, body=b"\xc2\xa3", ) - self.assertEqual(r._declared_encoding(), None) + assert r._declared_encoding() is None self._assert_response_values(r, "utf-8", "\xa3") def test_utf16(self): @@ -511,14 +504,11 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xef\xbb\xbfWORD\xe3\xab", ) - self.assertEqual(r6.encoding, "utf-8") - self.assertIn( - r6.text, - { - "WORD\ufffd\ufffd", # w3lib < 1.19.0 - "WORD\ufffd", # w3lib >= 1.19.0 - }, - ) + assert r6.encoding == "utf-8" + assert r6.text in { + "WORD\ufffd\ufffd", # w3lib < 1.19.0 + "WORD\ufffd", # w3lib >= 1.19.0 + } def test_bom_is_removed_from_body(self): # Inferring encoding from body also cache decoded body as sideeffect, @@ -532,21 +522,21 @@ class TextResponseTest(BaseResponseTest): # Test response without content-type and BOM encoding response = self.response_class(url, body=body) - self.assertEqual(response.encoding, "utf-8") - self.assertEqual(response.text, "WORD") + assert response.encoding == "utf-8" + assert response.text == "WORD" response = self.response_class(url, body=body) - self.assertEqual(response.text, "WORD") - self.assertEqual(response.encoding, "utf-8") + assert response.text == "WORD" + assert response.encoding == "utf-8" # Body caching sideeffect isn't triggered when encoding is declared in # content-type header but BOM still need to be removed from decoded # body response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.encoding, "utf-8") - self.assertEqual(response.text, "WORD") + assert response.encoding == "utf-8" + assert response.text == "WORD" response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.text, "WORD") - self.assertEqual(response.encoding, "utf-8") + assert response.text == "WORD" + assert response.encoding == "utf-8" def test_replace_wrong_encoding(self): """Test invalid chars are replaced properly""" @@ -577,49 +567,47 @@ class TextResponseTest(BaseResponseTest): body = b"<html><head><title>Some page" response = self.response_class("http://www.example.com", body=body) - self.assertIsInstance(response.selector, Selector) - self.assertEqual(response.selector.type, "html") - self.assertIs(response.selector, response.selector) # property is cached - self.assertIs(response.selector.response, response) + assert isinstance(response.selector, Selector) + assert response.selector.type == "html" + assert response.selector is response.selector # property is cached + assert response.selector.response is response - self.assertEqual( - response.selector.xpath("//title/text()").getall(), ["Some page"] - ) - self.assertEqual(response.selector.css("title::text").getall(), ["Some page"]) - self.assertEqual(response.selector.re("Some (.*)"), ["page"]) + assert response.selector.xpath("//title/text()").getall() == ["Some page"] + assert response.selector.css("title::text").getall() == ["Some page"] + assert response.selector.re("Some (.*)") == ["page"] def test_selector_shortcuts(self): body = b"Some page" response = self.response_class("http://www.example.com", body=body) - self.assertEqual( - response.xpath("//title/text()").getall(), - response.selector.xpath("//title/text()").getall(), + assert ( + response.xpath("//title/text()").getall() + == response.selector.xpath("//title/text()").getall() ) - self.assertEqual( - response.css("title::text").getall(), - response.selector.css("title::text").getall(), + assert ( + response.css("title::text").getall() + == response.selector.css("title::text").getall() ) def test_selector_shortcuts_kwargs(self): body = b'Some page

A nice paragraph.

' response = self.response_class("http://www.example.com", body=body) - self.assertEqual( + assert ( response.xpath( "normalize-space(//p[@class=$pclass])", pclass="content" - ).getall(), - response.xpath('normalize-space(//p[@class="content"])').getall(), + ).getall() + == response.xpath('normalize-space(//p[@class="content"])').getall() ) - self.assertEqual( + assert ( response.xpath( "//title[count(following::p[@class=$pclass])=$pcount]/text()", pclass="content", pcount=1, - ).getall(), - response.xpath( + ).getall() + == response.xpath( '//title[count(following::p[@class="content"])=1]/text()' - ).getall(), + ).getall() ) def test_urljoin_with_base_url(self): @@ -629,21 +617,21 @@ class TextResponseTest(BaseResponseTest): "/test" ) absolute = "https://example.net/test" - self.assertEqual(joined, absolute) + assert joined == absolute body = b'' joined = self.response_class("http://www.example.com", body=body).urljoin( "test" ) absolute = "http://www.example.com/test" - self.assertEqual(joined, absolute) + assert joined == absolute body = b'' joined = self.response_class("http://www.example.com", body=body).urljoin( "test" ) absolute = "http://www.example.com/elsewhere/test" - self.assertEqual(joined, absolute) + assert joined == absolute def test_follow_selector(self): resp = self._links_response() @@ -728,7 +716,7 @@ class TextResponseTest(BaseResponseTest): "http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82", response=resp1, ) - self.assertEqual(req.encoding, "utf8") + assert req.encoding == "utf8" resp2 = self.response_class( "http://example.com", @@ -742,12 +730,12 @@ class TextResponseTest(BaseResponseTest): "http://example.com/foo?%EF%F0%E8%E2%E5%F2", response=resp2, ) - self.assertEqual(req.encoding, "cp1251") + assert req.encoding == "cp1251" def test_follow_flags(self): res = self.response_class("http://example.com/") fol = res.follow("http://example.com/", flags=["cached", "allowed"]) - self.assertEqual(fol.flags, ["cached", "allowed"]) + assert fol.flags == ["cached", "allowed"] def test_follow_all_flags(self): re = self.response_class("http://www.example.com/") @@ -758,7 +746,7 @@ class TextResponseTest(BaseResponseTest): ] fol = re.follow_all(urls, flags=["cached", "allowed"]) for req in fol: - self.assertEqual(req.flags, ["cached", "allowed"]) + assert req.flags == ["cached", "allowed"] def test_follow_all_css(self): expected = [ @@ -767,7 +755,7 @@ class TextResponseTest(BaseResponseTest): ] response = self._links_response() extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')] - self.assertEqual(expected, extracted) + assert expected == extracted def test_follow_all_css_skip_invalid(self): expected = [ @@ -777,9 +765,9 @@ class TextResponseTest(BaseResponseTest): ] response = self._links_response_no_href() extracted1 = [r.url for r in response.follow_all(css=".pagination a")] - self.assertEqual(expected, extracted1) + assert expected == extracted1 extracted2 = [r.url for r in response.follow_all(response.css(".pagination a"))] - self.assertEqual(expected, extracted2) + assert expected == extracted2 def test_follow_all_xpath(self): expected = [ @@ -788,7 +776,7 @@ class TextResponseTest(BaseResponseTest): ] response = self._links_response() extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]') - self.assertEqual(expected, [r.url for r in extracted]) + assert expected == [r.url for r in extracted] def test_follow_all_xpath_skip_invalid(self): expected = [ @@ -800,12 +788,12 @@ class TextResponseTest(BaseResponseTest): extracted1 = [ r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a') ] - self.assertEqual(expected, extracted1) + assert expected == extracted1 extracted2 = [ r.url for r in response.follow_all(response.xpath('//div[@id="pagination"]/a')) ] - self.assertEqual(expected, extracted2) + assert expected == extracted2 def test_follow_all_too_many_arguments(self): response = self._links_response() @@ -820,7 +808,7 @@ class TextResponseTest(BaseResponseTest): def test_json_response(self): json_body = b"""{"ip": "109.187.217.200"}""" json_response = self.response_class("http://www.example.com", body=json_body) - self.assertEqual(json_response.json(), {"ip": "109.187.217.200"}) + assert json_response.json() == {"ip": "109.187.217.200"} text_body = b"""text""" text_response = self.response_class("http://www.example.com", body=text_body) @@ -842,7 +830,7 @@ class TextResponseTest(BaseResponseTest): mock_json.assert_called_once_with(json_body) -class HtmlResponseTest(TextResponseTest): +class TestHtmlResponse(TestTextResponse): response_class = HtmlResponse def test_html_encoding(self): @@ -883,7 +871,7 @@ class HtmlResponseTest(TextResponseTest): self._assert_response_values(r1, "gb2312", body) -class XmlResponseTest(TextResponseTest): +class TestXmlResponse(TestTextResponse): response_class = XmlResponse def test_xml_encoding(self): @@ -917,20 +905,20 @@ class XmlResponseTest(TextResponseTest): body = b'value' response = self.response_class("http://www.example.com", body=body) - self.assertIsInstance(response.selector, Selector) - self.assertEqual(response.selector.type, "xml") - self.assertIs(response.selector, response.selector) # property is cached - self.assertIs(response.selector.response, response) + assert isinstance(response.selector, Selector) + assert response.selector.type == "xml" + assert response.selector is response.selector # property is cached + assert response.selector.response is response - self.assertEqual(response.selector.xpath("//elem/text()").getall(), ["value"]) + assert response.selector.xpath("//elem/text()").getall() == ["value"] def test_selector_shortcuts(self): body = b'value' response = self.response_class("http://www.example.com", body=body) - self.assertEqual( - response.xpath("//elem/text()").getall(), - response.selector.xpath("//elem/text()").getall(), + assert ( + response.xpath("//elem/text()").getall() + == response.selector.xpath("//elem/text()").getall() ) def test_selector_shortcuts_kwargs(self): @@ -940,21 +928,21 @@ class XmlResponseTest(TextResponseTest): """ response = self.response_class("http://www.example.com", body=body) - self.assertEqual( + assert ( response.xpath( "//s:elem/text()", namespaces={"s": "http://scrapy.org"} - ).getall(), - response.selector.xpath( + ).getall() + == response.selector.xpath( "//s:elem/text()", namespaces={"s": "http://scrapy.org"} - ).getall(), + ).getall() ) response.selector.register_namespace("s2", "http://scrapy.org") - self.assertEqual( + assert ( response.xpath( "//s1:elem/text()", namespaces={"s1": "http://scrapy.org"} - ).getall(), - response.selector.xpath("//s2:elem/text()").getall(), + ).getall() + == response.selector.xpath("//s2:elem/text()").getall() ) @@ -968,7 +956,7 @@ class CustomResponse(TextResponse): super().__init__(*args, **kwargs) -class CustomResponseTest(TextResponseTest): +class TestCustomResponse(TestTextResponse): response_class = CustomResponse def test_copy(self): @@ -981,11 +969,11 @@ class CustomResponseTest(TextResponseTest): lost="lost", ) r2 = r1.copy() - self.assertIsInstance(r2, self.response_class) - self.assertEqual(r1.foo, r2.foo) - self.assertEqual(r1.bar, r2.bar) - self.assertEqual(r1.lost, "lost") - self.assertIsNone(r2.lost) + assert isinstance(r2, self.response_class) + assert r1.foo == r2.foo + assert r1.bar == r2.bar + assert r1.lost == "lost" + assert r2.lost is None def test_replace(self): super().test_replace() @@ -998,31 +986,31 @@ class CustomResponseTest(TextResponseTest): ) r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost") - self.assertIsInstance(r2, self.response_class) - self.assertEqual(r1.foo, "foo") - self.assertEqual(r1.bar, "bar") - self.assertEqual(r1.lost, "lost") - self.assertEqual(r2.foo, "new-foo") - self.assertEqual(r2.bar, "new-bar") - self.assertEqual(r2.lost, "new-lost") + assert isinstance(r2, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r2.foo == "new-foo" + assert r2.bar == "new-bar" + assert r2.lost == "new-lost" r3 = r1.replace(foo="new-foo", bar="new-bar") - self.assertIsInstance(r3, self.response_class) - self.assertEqual(r1.foo, "foo") - self.assertEqual(r1.bar, "bar") - self.assertEqual(r1.lost, "lost") - self.assertEqual(r3.foo, "new-foo") - self.assertEqual(r3.bar, "new-bar") - self.assertIsNone(r3.lost) + assert isinstance(r3, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r3.foo == "new-foo" + assert r3.bar == "new-bar" + assert r3.lost is None r4 = r1.replace(foo="new-foo") - self.assertIsInstance(r4, self.response_class) - self.assertEqual(r1.foo, "foo") - self.assertEqual(r1.bar, "bar") - self.assertEqual(r1.lost, "lost") - self.assertEqual(r4.foo, "new-foo") - self.assertEqual(r4.bar, "bar") - self.assertIsNone(r4.lost) + assert isinstance(r4, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r4.foo == "new-foo" + assert r4.bar == "bar" + assert r4.lost is None with pytest.raises( TypeError, diff --git a/tests/test_loader.py b/tests/test_loader.py index 1a933bb8d..224158e7f 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,7 +1,6 @@ from __future__ import annotations import dataclasses -import unittest import attr import pytest @@ -67,7 +66,7 @@ def processor_with_args(value, other=None, loader_context=None): return value -class BasicItemLoaderTest(unittest.TestCase): +class TestBasicItemLoader: def test_add_value_on_unknown_field(self): il = ProcessorItemLoader() with pytest.raises(KeyError): @@ -80,14 +79,14 @@ class BasicItemLoaderTest(unittest.TestCase): il.add_value("name", "marta") item = il.load_item() assert item is i - self.assertEqual(item["summary"], ["lala"]) - self.assertEqual(item["name"], ["marta"]) + assert item["summary"] == ["lala"] + assert item["name"] == ["marta"] def test_load_item_using_custom_loader(self): il = ProcessorItemLoader() il.add_value("name", "marta") item = il.load_item() - self.assertEqual(item["name"], ["Marta"]) + assert item["name"] == ["Marta"] class InitializationTestMixin: @@ -98,16 +97,16 @@ class InitializationTestMixin: input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo"]} def test_keep_list(self): """Loaded item should contain values from the initial item""" input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar"]} def test_add_value_singlevalue_singlevalue(self): """Values added after initialization should be appended""" @@ -115,8 +114,8 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) il.add_value("name", "bar") loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar"]} def test_add_value_singlevalue_list(self): """Values added after initialization should be appended""" @@ -124,10 +123,8 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) il.add_value("name", ["item", "loader"]) loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual( - ItemAdapter(loaded_item).asdict(), {"name": ["foo", "item", "loader"]} - ) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "item", "loader"]} def test_add_value_list_singlevalue(self): """Values added after initialization should be appended""" @@ -135,10 +132,8 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) il.add_value("name", "qwerty") loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual( - ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar", "qwerty"]} - ) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar", "qwerty"]} def test_add_value_list_list(self): """Values added after initialization should be appended""" @@ -146,56 +141,55 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) il.add_value("name", ["item", "loader"]) loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual( - ItemAdapter(loaded_item).asdict(), - {"name": ["foo", "bar", "item", "loader"]}, - ) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == { + "name": ["foo", "bar", "item", "loader"] + } def test_get_output_value_singlevalue(self): """Getting output value must not remove value from item""" input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value("name"), ["foo"]) + assert il.get_output_value("name") == ["foo"] loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo"]} def test_get_output_value_list(self): """Getting output value must not remove value from item""" input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value("name"), ["foo", "bar"]) + assert il.get_output_value("name") == ["foo", "bar"] loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar"]} def test_values_single(self): """Values from initial item must be added to loader._values""" input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) - self.assertEqual(il._values.get("name"), ["foo"]) + assert il._values.get("name") == ["foo"] def test_values_list(self): """Values from initial item must be added to loader._values""" input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) - self.assertEqual(il._values.get("name"), ["foo", "bar"]) + assert il._values.get("name") == ["foo", "bar"] -class InitializationFromDictTest(InitializationTestMixin, unittest.TestCase): +class TestInitializationFromDict(InitializationTestMixin): item_class = dict -class InitializationFromItemTest(InitializationTestMixin, unittest.TestCase): +class TestInitializationFromItem(InitializationTestMixin): item_class = NameItem -class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase): +class TestInitializationFromAttrsItem(InitializationTestMixin): item_class = AttrsNameItem -class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase): +class TestInitializationFromDataClass(InitializationTestMixin): item_class = NameDataClass @@ -212,7 +206,7 @@ class NoInputReprocessingItemLoader(BaseNoInputReprocessingLoader): default_item_class = NoInputReprocessingItem -class NoInputReprocessingFromItemTest(unittest.TestCase): +class TestNoInputReprocessingFromItem: """ Loaders initialized from loaded items must not reprocess fields (Item instances) """ @@ -220,41 +214,41 @@ class NoInputReprocessingFromItemTest(unittest.TestCase): def test_avoid_reprocessing_with_initial_values_single(self): il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title="foo")) il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "foo"}) - self.assertEqual( - NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "foo"} - ) + assert il_loaded == {"title": "foo"} + assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == { + "title": "foo" + } def test_avoid_reprocessing_with_initial_values_list(self): il = NoInputReprocessingItemLoader( item=NoInputReprocessingItem(title=["foo", "bar"]) ) il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "foo"}) - self.assertEqual( - NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "foo"} - ) + assert il_loaded == {"title": "foo"} + assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == { + "title": "foo" + } def test_avoid_reprocessing_without_initial_values_single(self): il = NoInputReprocessingItemLoader() il.add_value("title", "FOO") il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "FOO"}) - self.assertEqual( - NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "FOO"} - ) + assert il_loaded == {"title": "FOO"} + assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == { + "title": "FOO" + } def test_avoid_reprocessing_without_initial_values_list(self): il = NoInputReprocessingItemLoader() il.add_value("title", ["foo", "bar"]) il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "FOO"}) - self.assertEqual( - NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "FOO"} - ) + assert il_loaded == {"title": "FOO"} + assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == { + "title": "FOO" + } -class TestOutputProcessorItem(unittest.TestCase): +class TestOutputProcessorItem: def test_output_processor(self): class TempItem(Item): temp = Field() @@ -270,11 +264,11 @@ class TestOutputProcessorItem(unittest.TestCase): loader = TempLoader() item = loader.load_item() - self.assertIsInstance(item, TempItem) - self.assertEqual(dict(item), {"temp": 0.3}) + assert isinstance(item, TempItem) + assert dict(item) == {"temp": 0.3} -class SelectortemLoaderTest(unittest.TestCase): +class TestSelectortemLoader: response = HtmlResponse( url="", encoding="utf-8", @@ -292,7 +286,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method(self): l = ProcessorItemLoader() - self.assertEqual(l.selector, None) + assert l.selector is None def test_init_method_errors(self): l = ProcessorItemLoader() @@ -312,150 +306,149 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method_with_selector(self): sel = Selector(text="
marta
") l = ProcessorItemLoader(selector=sel) - self.assertIs(l.selector, sel) + assert l.selector is sel l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] def test_init_method_with_selector_css(self): sel = Selector(text="
marta
") l = ProcessorItemLoader(selector=sel) - self.assertIs(l.selector, sel) + assert l.selector is sel l.add_css("name", "div::text") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] def test_init_method_with_base_response(self): """Selector should be None after initialization""" response = Response("https://scrapy.org") l = ProcessorItemLoader(response=response) - self.assertIs(l.selector, None) + assert l.selector is None def test_init_method_with_response(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] def test_init_method_with_response_css(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_css("name", "div::text") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.add_css("url", "a::attr(href)") - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] # combining/accumulating CSS selectors and XPath expressions l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta", "Marta"]) + assert l.get_output_value("name") == ["Marta", "Marta"] l.add_xpath("url", "//img/@src") - self.assertEqual( - l.get_output_value("url"), ["http://www.scrapy.org", "/images/logo.png"] - ) + assert l.get_output_value("url") == [ + "http://www.scrapy.org", + "/images/logo.png", + ] def test_add_xpath_re(self): l = ProcessorItemLoader(response=self.response) l.add_xpath("name", "//div/text()", re="ma") - self.assertEqual(l.get_output_value("name"), ["Ma"]) + assert l.get_output_value("name") == ["Ma"] def test_replace_xpath(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_xpath("name", "//p/text()") - self.assertEqual(l.get_output_value("name"), ["Paragraph"]) + assert l.get_output_value("name") == ["Paragraph"] l.replace_xpath("name", ["//p/text()", "//div/text()"]) - self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"]) + assert l.get_output_value("name") == ["Paragraph", "Marta"] def test_get_xpath(self): l = ProcessorItemLoader(response=self.response) - self.assertEqual(l.get_xpath("//p/text()"), ["paragraph"]) - self.assertEqual(l.get_xpath("//p/text()", TakeFirst()), "paragraph") - self.assertEqual(l.get_xpath("//p/text()", TakeFirst(), re="pa"), "pa") + assert l.get_xpath("//p/text()") == ["paragraph"] + assert l.get_xpath("//p/text()", TakeFirst()) == "paragraph" + assert l.get_xpath("//p/text()", TakeFirst(), re="pa") == "pa" - self.assertEqual( - l.get_xpath(["//p/text()", "//div/text()"]), ["paragraph", "marta"] - ) + assert l.get_xpath(["//p/text()", "//div/text()"]) == ["paragraph", "marta"] def test_replace_xpath_multi_fields(self): l = ProcessorItemLoader(response=self.response) l.add_xpath(None, "//div/text()", TakeFirst(), lambda x: {"name": x}) - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_xpath(None, "//p/text()", TakeFirst(), lambda x: {"name": x}) - self.assertEqual(l.get_output_value("name"), ["Paragraph"]) + assert l.get_output_value("name") == ["Paragraph"] def test_replace_xpath_re(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_xpath("name", "//div/text()", re="ma") - self.assertEqual(l.get_output_value("name"), ["Ma"]) + assert l.get_output_value("name") == ["Ma"] def test_add_css_re(self): l = ProcessorItemLoader(response=self.response) l.add_css("name", "div::text", re="ma") - self.assertEqual(l.get_output_value("name"), ["Ma"]) + assert l.get_output_value("name") == ["Ma"] l.add_css("url", "a::attr(href)", re="http://(.+)") - self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"]) + assert l.get_output_value("url") == ["www.scrapy.org"] def test_replace_css(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_css("name", "div::text") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_css("name", "p::text") - self.assertEqual(l.get_output_value("name"), ["Paragraph"]) + assert l.get_output_value("name") == ["Paragraph"] l.replace_css("name", ["p::text", "div::text"]) - self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"]) + assert l.get_output_value("name") == ["Paragraph", "Marta"] l.add_css("url", "a::attr(href)", re="http://(.+)") - self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"]) + assert l.get_output_value("url") == ["www.scrapy.org"] l.replace_css("url", "img::attr(src)") - self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) + assert l.get_output_value("url") == ["/images/logo.png"] def test_get_css(self): l = ProcessorItemLoader(response=self.response) - self.assertEqual(l.get_css("p::text"), ["paragraph"]) - self.assertEqual(l.get_css("p::text", TakeFirst()), "paragraph") - self.assertEqual(l.get_css("p::text", TakeFirst(), re="pa"), "pa") + assert l.get_css("p::text") == ["paragraph"] + assert l.get_css("p::text", TakeFirst()) == "paragraph" + assert l.get_css("p::text", TakeFirst(), re="pa") == "pa" - self.assertEqual(l.get_css(["p::text", "div::text"]), ["paragraph", "marta"]) - self.assertEqual( - l.get_css(["a::attr(href)", "img::attr(src)"]), - ["http://www.scrapy.org", "/images/logo.png"], - ) + assert l.get_css(["p::text", "div::text"]) == ["paragraph", "marta"] + assert l.get_css(["a::attr(href)", "img::attr(src)"]) == [ + "http://www.scrapy.org", + "/images/logo.png", + ] def test_replace_css_multi_fields(self): l = ProcessorItemLoader(response=self.response) l.add_css(None, "div::text", TakeFirst(), lambda x: {"name": x}) - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_css(None, "p::text", TakeFirst(), lambda x: {"name": x}) - self.assertEqual(l.get_output_value("name"), ["Paragraph"]) + assert l.get_output_value("name") == ["Paragraph"] l.add_css(None, "a::attr(href)", TakeFirst(), lambda x: {"url": x}) - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] l.replace_css(None, "img::attr(src)", TakeFirst(), lambda x: {"url": x}) - self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) + assert l.get_output_value("url") == ["/images/logo.png"] def test_replace_css_re(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_css("url", "a::attr(href)") - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] l.replace_css("url", "a::attr(href)", re=r"http://www\.(.+)") - self.assertEqual(l.get_output_value("url"), ["scrapy.org"]) + assert l.get_output_value("url") == ["scrapy.org"] -class SubselectorLoaderTest(unittest.TestCase): +class TestSubselectorLoader: response = HtmlResponse( url="", encoding="utf-8", @@ -483,17 +476,13 @@ class SubselectorLoaderTest(unittest.TestCase): nl.add_css("name_div", "#id") nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value("name"), ["marta"]) - self.assertEqual(l.get_output_value("name_div"), ['
marta
']) - self.assertEqual(l.get_output_value("name_value"), ["marta"]) + assert l.get_output_value("name") == ["marta"] + assert l.get_output_value("name_div") == ['
marta
'] + assert l.get_output_value("name_value") == ["marta"] - self.assertEqual(l.get_output_value("name"), nl.get_output_value("name")) - self.assertEqual( - l.get_output_value("name_div"), nl.get_output_value("name_div") - ) - self.assertEqual( - l.get_output_value("name_value"), nl.get_output_value("name_value") - ) + assert l.get_output_value("name") == nl.get_output_value("name") + assert l.get_output_value("name_div") == nl.get_output_value("name_div") + assert l.get_output_value("name_value") == nl.get_output_value("name_value") def test_nested_css(self): l = NestedItemLoader(response=self.response) @@ -502,17 +491,13 @@ class SubselectorLoaderTest(unittest.TestCase): nl.add_css("name_div", "#id") nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value("name"), ["marta"]) - self.assertEqual(l.get_output_value("name_div"), ['
marta
']) - self.assertEqual(l.get_output_value("name_value"), ["marta"]) + assert l.get_output_value("name") == ["marta"] + assert l.get_output_value("name_div") == ['
marta
'] + assert l.get_output_value("name_value") == ["marta"] - self.assertEqual(l.get_output_value("name"), nl.get_output_value("name")) - self.assertEqual( - l.get_output_value("name_div"), nl.get_output_value("name_div") - ) - self.assertEqual( - l.get_output_value("name_value"), nl.get_output_value("name_value") - ) + assert l.get_output_value("name") == nl.get_output_value("name") + assert l.get_output_value("name_div") == nl.get_output_value("name_div") + assert l.get_output_value("name_value") == nl.get_output_value("name_value") def test_nested_replace(self): l = NestedItemLoader(response=self.response) @@ -520,11 +505,11 @@ class SubselectorLoaderTest(unittest.TestCase): nl2 = nl1.nested_xpath("a") l.add_xpath("url", "//footer/a/@href") - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] nl1.replace_xpath("url", "img/@src") - self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) + assert l.get_output_value("url") == ["/images/logo.png"] nl2.replace_xpath("url", "@href") - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] def test_nested_ordering(self): l = NestedItemLoader(response=self.response) @@ -536,15 +521,12 @@ class SubselectorLoaderTest(unittest.TestCase): nl2.add_xpath("url", "text()") l.add_xpath("url", "//footer/a/@href") - self.assertEqual( - l.get_output_value("url"), - [ - "/images/logo.png", - "http://www.scrapy.org", - "homepage", - "http://www.scrapy.org", - ], - ) + assert l.get_output_value("url") == [ + "/images/logo.png", + "http://www.scrapy.org", + "homepage", + "http://www.scrapy.org", + ] def test_nested_load_item(self): l = NestedItemLoader(response=self.response) @@ -561,9 +543,9 @@ class SubselectorLoaderTest(unittest.TestCase): assert item is nl1.item assert item is nl2.item - self.assertEqual(item["name"], ["marta"]) - self.assertEqual(item["url"], ["http://www.scrapy.org"]) - self.assertEqual(item["image"], ["/images/logo.png"]) + assert item["name"] == ["marta"] + assert item["url"] == ["http://www.scrapy.org"] + assert item["image"] == ["/images/logo.png"] # Functions as processors @@ -588,9 +570,9 @@ class FunctionProcessorItemLoader(ItemLoader): default_item_class = FunctionProcessorItem -class FunctionProcessorTestCase(unittest.TestCase): +class TestFunctionProcessor: def test_processor_defined_in_item(self): lo = FunctionProcessorItemLoader() lo.add_value("foo", " bar ") lo.add_value("foo", [" asdf ", " qwerty "]) - self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]}) + assert dict(lo.load_item()) == {"foo": ["BAR", "ASDF", "QWERTY"]} diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 909b365a9..d7d900546 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,4 +1,6 @@ -import unittest +# pylint: disable=unsubscriptable-object,unsupported-membership-test,use-implicit-booleaness-not-comparison +# (too many false positives) + from unittest import mock import pytest @@ -14,31 +16,31 @@ from scrapy.settings import ( from . import default_settings -class SettingsGlobalFuncsTest(unittest.TestCase): +class TestSettingsGlobalFuncs: def test_get_settings_priority(self): for prio_str, prio_num in SETTINGS_PRIORITIES.items(): - self.assertEqual(get_settings_priority(prio_str), prio_num) - self.assertEqual(get_settings_priority(99), 99) + assert get_settings_priority(prio_str) == prio_num + assert get_settings_priority(99) == 99 -class SettingsAttributeTest(unittest.TestCase): - def setUp(self): +class TestSettingsAttribute: + def setup_method(self): self.attribute = SettingsAttribute("value", 10) def test_set_greater_priority(self): self.attribute.set("value2", 20) - self.assertEqual(self.attribute.value, "value2") - self.assertEqual(self.attribute.priority, 20) + assert self.attribute.value == "value2" + assert self.attribute.priority == 20 def test_set_equal_priority(self): self.attribute.set("value2", 10) - self.assertEqual(self.attribute.value, "value2") - self.assertEqual(self.attribute.priority, 10) + assert self.attribute.value == "value2" + assert self.attribute.priority == 10 def test_set_less_priority(self): self.attribute.set("value2", 0) - self.assertEqual(self.attribute.value, "value") - self.assertEqual(self.attribute.priority, 10) + assert self.attribute.value == "value" + assert self.attribute.priority == 10 def test_overwrite_basesettings(self): original_dict = {"one": 10, "two": 20} @@ -47,61 +49,59 @@ class SettingsAttributeTest(unittest.TestCase): new_dict = {"three": 11, "four": 21} attribute.set(new_dict, 10) - self.assertIsInstance(attribute.value, BaseSettings) - self.assertCountEqual(attribute.value, new_dict) - self.assertCountEqual(original_settings, original_dict) + assert isinstance(attribute.value, BaseSettings) + assert set(attribute.value) == set(new_dict) + assert set(original_settings) == set(original_dict) new_settings = BaseSettings({"five": 12}, 0) attribute.set(new_settings, 0) # Insufficient priority - self.assertCountEqual(attribute.value, new_dict) + assert set(attribute.value) == set(new_dict) attribute.set(new_settings, 10) - self.assertCountEqual(attribute.value, new_settings) + assert set(attribute.value) == set(new_settings) def test_repr(self): - self.assertEqual( - repr(self.attribute), "" - ) + assert repr(self.attribute) == "" -class BaseSettingsTest(unittest.TestCase): - def setUp(self): +class TestBaseSettings: + def setup_method(self): self.settings = BaseSettings() def test_setdefault_not_existing_value(self): settings = BaseSettings() value = settings.setdefault("TEST_OPTION", "value") - self.assertEqual(settings["TEST_OPTION"], "value") - self.assertEqual(value, "value") - self.assertIsNotNone(value) + assert settings["TEST_OPTION"] == "value" + assert value == "value" + assert value is not None def test_setdefault_existing_value(self): settings = BaseSettings({"TEST_OPTION": "value"}) value = settings.setdefault("TEST_OPTION", None) - self.assertEqual(settings["TEST_OPTION"], "value") - self.assertEqual(value, "value") + assert settings["TEST_OPTION"] == "value" + assert value == "value" def test_set_new_attribute(self): self.settings.set("TEST_OPTION", "value", 0) - self.assertIn("TEST_OPTION", self.settings.attributes) + assert "TEST_OPTION" in self.settings.attributes attr = self.settings.attributes["TEST_OPTION"] - self.assertIsInstance(attr, SettingsAttribute) - self.assertEqual(attr.value, "value") - self.assertEqual(attr.priority, 0) + assert isinstance(attr, SettingsAttribute) + assert attr.value == "value" + assert attr.priority == 0 def test_set_settingsattribute(self): myattr = SettingsAttribute(0, 30) # Note priority 30 self.settings.set("TEST_ATTR", myattr, 10) - self.assertEqual(self.settings.get("TEST_ATTR"), 0) - self.assertEqual(self.settings.getpriority("TEST_ATTR"), 30) + assert self.settings.get("TEST_ATTR") == 0 + assert self.settings.getpriority("TEST_ATTR") == 30 def test_set_instance_identity_on_update(self): attr = SettingsAttribute("value", 0) self.settings.attributes = {"TEST_OPTION": attr} self.settings.set("TEST_OPTION", "othervalue", 10) - self.assertIn("TEST_OPTION", self.settings.attributes) - self.assertIs(attr, self.settings.attributes["TEST_OPTION"]) + assert "TEST_OPTION" in self.settings.attributes + assert attr is self.settings.attributes["TEST_OPTION"] def test_set_calls_settings_attributes_methods_on_update(self): attr = SettingsAttribute("value", 10) @@ -114,7 +114,7 @@ class BaseSettingsTest(unittest.TestCase): for priority in (0, 10, 20): self.settings.set("TEST_OPTION", "othervalue", priority) mock_set.assert_called_once_with("othervalue", priority) - self.assertFalse(mock_setattr.called) + assert not mock_setattr.called mock_set.reset_mock() mock_setattr.reset_mock() @@ -122,19 +122,19 @@ class BaseSettingsTest(unittest.TestCase): settings = BaseSettings() settings.set("key", "a", "default") settings["key"] = "b" - self.assertEqual(settings["key"], "b") - self.assertEqual(settings.getpriority("key"), 20) + assert settings["key"] == "b" + assert settings.getpriority("key") == 20 settings["key"] = "c" - self.assertEqual(settings["key"], "c") + assert settings["key"] == "c" settings["key2"] = "x" - self.assertIn("key2", settings) - self.assertEqual(settings["key2"], "x") - self.assertEqual(settings.getpriority("key2"), 20) + assert "key2" in settings + assert settings["key2"] == "x" + assert settings.getpriority("key2") == 20 def test_setdict_alias(self): with mock.patch.object(self.settings, "set") as mock_set: self.settings.setdict({"TEST_1": "value1", "TEST_2": "value2"}, 10) - self.assertEqual(mock_set.call_count, 2) + assert mock_set.call_count == 2 calls = [ mock.call("TEST_1", "value1", 10), mock.call("TEST_2", "value2", 10), @@ -149,10 +149,10 @@ class BaseSettingsTest(unittest.TestCase): self.settings.attributes = {} self.settings.setmodule(ModuleMock(), 10) - self.assertIn("UPPERCASE_VAR", self.settings.attributes) - self.assertNotIn("MIXEDcase_VAR", self.settings.attributes) - self.assertNotIn("lowercase_var", self.settings.attributes) - self.assertEqual(len(self.settings.attributes), 1) + assert "UPPERCASE_VAR" in self.settings.attributes + assert "MIXEDcase_VAR" not in self.settings.attributes + assert "lowercase_var" not in self.settings.attributes + assert len(self.settings.attributes) == 1 def test_setmodule_alias(self): with mock.patch.object(self.settings, "set") as mock_set: @@ -168,13 +168,13 @@ class BaseSettingsTest(unittest.TestCase): self.settings.attributes = {} self.settings.setmodule("tests.test_settings.default_settings", 10) - self.assertCountEqual(self.settings.attributes.keys(), ctrl_attributes.keys()) + assert set(self.settings.attributes) == set(ctrl_attributes) for key in ctrl_attributes: attr = self.settings.attributes[key] ctrl_attr = ctrl_attributes[key] - self.assertEqual(attr.value, ctrl_attr.value) - self.assertEqual(attr.priority, ctrl_attr.priority) + assert attr.value == ctrl_attr.value + assert attr.priority == ctrl_attr.priority def test_update(self): settings = BaseSettings({"key_lowprio": 0}, priority=0) @@ -186,21 +186,21 @@ class BaseSettingsTest(unittest.TestCase): custom_dict = {"key_lowprio": 2, "key_highprio": 12, "newkey_two": None} settings.update(custom_dict, priority=20) - self.assertEqual(settings["key_lowprio"], 2) - self.assertEqual(settings.getpriority("key_lowprio"), 20) - self.assertEqual(settings["key_highprio"], 10) - self.assertIn("newkey_two", settings) - self.assertEqual(settings.getpriority("newkey_two"), 20) + assert settings["key_lowprio"] == 2 + assert settings.getpriority("key_lowprio") == 20 + assert settings["key_highprio"] == 10 + assert "newkey_two" in settings + assert settings.getpriority("newkey_two") == 20 settings.update(custom_settings) - self.assertEqual(settings["key_lowprio"], 1) - self.assertEqual(settings.getpriority("key_lowprio"), 30) - self.assertEqual(settings["key_highprio"], 10) - self.assertIn("newkey_one", settings) - self.assertEqual(settings.getpriority("newkey_one"), 50) + assert settings["key_lowprio"] == 1 + assert settings.getpriority("key_lowprio") == 30 + assert settings["key_highprio"] == 10 + assert "newkey_one" in settings + assert settings.getpriority("newkey_one") == 50 settings.update({"key_lowprio": 3}, priority=20) - self.assertEqual(settings["key_lowprio"], 1) + assert settings["key_lowprio"] == 1 @pytest.mark.xfail( raises=TypeError, reason="BaseSettings.update doesn't support kwargs input" @@ -220,21 +220,21 @@ class BaseSettingsTest(unittest.TestCase): def test_update_jsonstring(self): settings = BaseSettings({"number": 0, "dict": BaseSettings({"key": "val"})}) settings.update('{"number": 1, "newnumber": 2}') - self.assertEqual(settings["number"], 1) - self.assertEqual(settings["newnumber"], 2) + assert settings["number"] == 1 + assert settings["newnumber"] == 2 settings.set("dict", '{"key": "newval", "newkey": "newval2"}') - self.assertEqual(settings["dict"]["key"], "newval") - self.assertEqual(settings["dict"]["newkey"], "newval2") + assert settings["dict"]["key"] == "newval" + assert settings["dict"]["newkey"] == "newval2" def test_delete(self): settings = BaseSettings({"key": None}) settings.set("key_highprio", None, priority=50) settings.delete("key") settings.delete("key_highprio") - self.assertNotIn("key", settings) - self.assertIn("key_highprio", settings) + assert "key" not in settings + assert "key_highprio" in settings del settings["key_highprio"] - self.assertNotIn("key_highprio", settings) + assert "key_highprio" not in settings with pytest.raises(KeyError): settings.delete("notkey") with pytest.raises(KeyError): @@ -271,40 +271,40 @@ class BaseSettingsTest(unittest.TestCase): for key, value in test_configuration.items() } - self.assertTrue(settings.getbool("TEST_ENABLED1")) - self.assertTrue(settings.getbool("TEST_ENABLED2")) - self.assertTrue(settings.getbool("TEST_ENABLED3")) - self.assertTrue(settings.getbool("TEST_ENABLED4")) - self.assertTrue(settings.getbool("TEST_ENABLED5")) - self.assertFalse(settings.getbool("TEST_ENABLEDx")) - self.assertTrue(settings.getbool("TEST_ENABLEDx", True)) - self.assertFalse(settings.getbool("TEST_DISABLED1")) - self.assertFalse(settings.getbool("TEST_DISABLED2")) - self.assertFalse(settings.getbool("TEST_DISABLED3")) - self.assertFalse(settings.getbool("TEST_DISABLED4")) - self.assertFalse(settings.getbool("TEST_DISABLED5")) - self.assertEqual(settings.getint("TEST_INT1"), 123) - self.assertEqual(settings.getint("TEST_INT2"), 123) - self.assertEqual(settings.getint("TEST_INTx"), 0) - self.assertEqual(settings.getint("TEST_INTx", 45), 45) - self.assertEqual(settings.getfloat("TEST_FLOAT1"), 123.45) - self.assertEqual(settings.getfloat("TEST_FLOAT2"), 123.45) - self.assertEqual(settings.getfloat("TEST_FLOATx"), 0.0) - self.assertEqual(settings.getfloat("TEST_FLOATx", 55.0), 55.0) - self.assertEqual(settings.getlist("TEST_LIST1"), ["one", "two"]) - self.assertEqual(settings.getlist("TEST_LIST2"), ["one", "two"]) - self.assertEqual(settings.getlist("TEST_LIST3"), []) - self.assertEqual(settings.getlist("TEST_LISTx"), []) - self.assertEqual(settings.getlist("TEST_LISTx", ["default"]), ["default"]) - self.assertEqual(settings["TEST_STR"], "value") - self.assertEqual(settings.get("TEST_STR"), "value") - self.assertEqual(settings["TEST_STRx"], None) - self.assertEqual(settings.get("TEST_STRx"), None) - self.assertEqual(settings.get("TEST_STRx", "default"), "default") - self.assertEqual(settings.getdict("TEST_DICT1"), {"key1": "val1", "ke2": 3}) - self.assertEqual(settings.getdict("TEST_DICT2"), {"key1": "val1", "ke2": 3}) - self.assertEqual(settings.getdict("TEST_DICT3"), {}) - self.assertEqual(settings.getdict("TEST_DICT3", {"key1": 5}), {"key1": 5}) + assert settings.getbool("TEST_ENABLED1") + assert settings.getbool("TEST_ENABLED2") + assert settings.getbool("TEST_ENABLED3") + assert settings.getbool("TEST_ENABLED4") + assert settings.getbool("TEST_ENABLED5") + assert not settings.getbool("TEST_ENABLEDx") + assert settings.getbool("TEST_ENABLEDx", True) + assert not settings.getbool("TEST_DISABLED1") + assert not settings.getbool("TEST_DISABLED2") + assert not settings.getbool("TEST_DISABLED3") + assert not settings.getbool("TEST_DISABLED4") + assert not settings.getbool("TEST_DISABLED5") + assert settings.getint("TEST_INT1") == 123 + assert settings.getint("TEST_INT2") == 123 + assert settings.getint("TEST_INTx") == 0 + assert settings.getint("TEST_INTx", 45) == 45 + assert settings.getfloat("TEST_FLOAT1") == 123.45 + assert settings.getfloat("TEST_FLOAT2") == 123.45 + assert settings.getfloat("TEST_FLOATx") == 0.0 + assert settings.getfloat("TEST_FLOATx", 55.0) == 55.0 + assert settings.getlist("TEST_LIST1") == ["one", "two"] + assert settings.getlist("TEST_LIST2") == ["one", "two"] + assert settings.getlist("TEST_LIST3") == [] + assert settings.getlist("TEST_LISTx") == [] + assert settings.getlist("TEST_LISTx", ["default"]) == ["default"] + assert settings["TEST_STR"] == "value" + assert settings.get("TEST_STR") == "value" + assert settings["TEST_STRx"] is None + assert settings.get("TEST_STRx") is None + assert settings.get("TEST_STRx", "default") == "default" + assert settings.getdict("TEST_DICT1") == {"key1": "val1", "ke2": 3} + assert settings.getdict("TEST_DICT2") == {"key1": "val1", "ke2": 3} + assert settings.getdict("TEST_DICT3") == {} + assert settings.getdict("TEST_DICT3", {"key1": 5}) == {"key1": 5} with pytest.raises( ValueError, match="dictionary update sequence element #0 has length 3; 2 is required|sequence of pairs expected", @@ -321,8 +321,8 @@ class BaseSettingsTest(unittest.TestCase): def test_getpriority(self): settings = BaseSettings({"key": "value"}, priority=99) - self.assertEqual(settings.getpriority("key"), 99) - self.assertEqual(settings.getpriority("nonexistentkey"), None) + assert settings.getpriority("key") == 99 + assert settings.getpriority("nonexistentkey") is None def test_getwithbase(self): s = BaseSettings( @@ -333,16 +333,16 @@ class BaseSettingsTest(unittest.TestCase): } ) s["TEST"].set(2, 200, "cmdline") - self.assertCountEqual(s.getwithbase("TEST"), {1: 1, 2: 200, 3: 30}) - self.assertCountEqual(s.getwithbase("HASNOBASE"), s["HASNOBASE"]) - self.assertEqual(s.getwithbase("NONEXISTENT"), {}) + assert set(s.getwithbase("TEST")) == {1, 2, 3} + assert set(s.getwithbase("HASNOBASE")) == set(s["HASNOBASE"]) + assert s.getwithbase("NONEXISTENT") == {} def test_maxpriority(self): # Empty settings should return 'default' - self.assertEqual(self.settings.maxpriority(), 0) + assert self.settings.maxpriority() == 0 self.settings.set("A", 0, 10) self.settings.set("B", 0, 30) - self.assertEqual(self.settings.maxpriority(), 30) + assert self.settings.maxpriority() == 30 def test_copy(self): values = { @@ -356,17 +356,15 @@ class BaseSettingsTest(unittest.TestCase): self.settings.setdict(values) copy = self.settings.copy() self.settings.set("TEST_BOOL", False) - self.assertTrue(copy.get("TEST_BOOL")) + assert copy.get("TEST_BOOL") test_list = self.settings.get("TEST_LIST") test_list.append("three") - self.assertListEqual(copy.get("TEST_LIST"), ["one", "two"]) + assert copy.get("TEST_LIST") == ["one", "two"] test_list_of_lists = self.settings.get("TEST_LIST_OF_LISTS") test_list_of_lists[0].append("first_three") - self.assertListEqual( - copy.get("TEST_LIST_OF_LISTS")[0], ["first_one", "first_two"] - ) + assert copy.get("TEST_LIST_OF_LISTS")[0] == ["first_one", "first_two"] def test_copy_to_dict(self): s = BaseSettings( @@ -379,17 +377,14 @@ class BaseSettingsTest(unittest.TestCase): "HASNOBASE": BaseSettings({3: 3000}, "default"), } ) - self.assertDictEqual( - s.copy_to_dict(), - { - "HASNOBASE": {3: 3000}, - "TEST": {1: 10, 3: 30}, - "TEST_BASE": {1: 1, 2: 2}, - "TEST_LIST": [1, 2], - "TEST_BOOLEAN": False, - "TEST_STRING": "a string", - }, - ) + assert s.copy_to_dict() == { + "HASNOBASE": {3: 3000}, + "TEST": {1: 10, 3: 30}, + "TEST_BASE": {1: 1, 2: 2}, + "TEST_LIST": [1, 2], + "TEST_BOOLEAN": False, + "TEST_STRING": "a string", + } def test_freeze(self): self.settings.freeze() @@ -400,55 +395,55 @@ class BaseSettingsTest(unittest.TestCase): def test_frozencopy(self): frozencopy = self.settings.frozencopy() - self.assertTrue(frozencopy.frozen) - self.assertIsNot(frozencopy, self.settings) + assert frozencopy.frozen + assert frozencopy is not self.settings -class SettingsTest(unittest.TestCase): - def setUp(self): +class TestSettings: + def setup_method(self): self.settings = Settings() @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {"default": 10}) @mock.patch("scrapy.settings.default_settings", default_settings) def test_initial_defaults(self): settings = Settings() - self.assertEqual(len(settings.attributes), 2) - self.assertIn("TEST_DEFAULT", settings.attributes) + assert len(settings.attributes) == 2 + assert "TEST_DEFAULT" in settings.attributes attr = settings.attributes["TEST_DEFAULT"] - self.assertIsInstance(attr, SettingsAttribute) - self.assertEqual(attr.value, "defvalue") - self.assertEqual(attr.priority, 10) + assert isinstance(attr, SettingsAttribute) + assert attr.value == "defvalue" + assert attr.priority == 10 @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {}) @mock.patch("scrapy.settings.default_settings", {}) def test_initial_values(self): settings = Settings({"TEST_OPTION": "value"}, 10) - self.assertEqual(len(settings.attributes), 1) - self.assertIn("TEST_OPTION", settings.attributes) + assert len(settings.attributes) == 1 + assert "TEST_OPTION" in settings.attributes attr = settings.attributes["TEST_OPTION"] - self.assertIsInstance(attr, SettingsAttribute) - self.assertEqual(attr.value, "value") - self.assertEqual(attr.priority, 10) + assert isinstance(attr, SettingsAttribute) + assert attr.value == "value" + assert attr.priority == 10 @mock.patch("scrapy.settings.default_settings", default_settings) def test_autopromote_dicts(self): settings = Settings() mydict = settings.get("TEST_DICT") - self.assertIsInstance(mydict, BaseSettings) - self.assertIn("key", mydict) - self.assertEqual(mydict["key"], "val") # pylint: disable=unsubscriptable-object - self.assertEqual(mydict.getpriority("key"), 0) + assert isinstance(mydict, BaseSettings) + assert "key" in mydict + assert mydict["key"] == "val" + assert mydict.getpriority("key") == 0 @mock.patch("scrapy.settings.default_settings", default_settings) def test_getdict_autodegrade_basesettings(self): settings = Settings() mydict = settings.getdict("TEST_DICT") - self.assertIsInstance(mydict, dict) - self.assertEqual(len(mydict), 1) - self.assertIn("key", mydict) - self.assertEqual(mydict["key"], "val") + assert isinstance(mydict, dict) + assert len(mydict) == 1 + assert "key" in mydict + assert mydict["key"] == "val" def test_passing_objects_as_values(self): from scrapy.core.downloader.handlers.file import FileDownloadHandler @@ -470,19 +465,19 @@ class SettingsTest(unittest.TestCase): } ) - self.assertIn("ITEM_PIPELINES", settings.attributes) + assert "ITEM_PIPELINES" in settings.attributes mypipeline, priority = settings.getdict("ITEM_PIPELINES").popitem() - self.assertEqual(priority, 800) - self.assertEqual(mypipeline, TestPipeline) - self.assertIsInstance(mypipeline(), TestPipeline) - self.assertEqual(mypipeline().process_item("item", None), "item") + assert priority == 800 + assert mypipeline == TestPipeline + assert isinstance(mypipeline(), TestPipeline) + assert mypipeline().process_item("item", None) == "item" myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp") - self.assertEqual(myhandler, FileDownloadHandler) + assert myhandler == FileDownloadHandler myhandler_instance = build_from_crawler(myhandler, get_crawler()) - self.assertIsInstance(myhandler_instance, FileDownloadHandler) - self.assertTrue(hasattr(myhandler_instance, "download_request")) + assert isinstance(myhandler_instance, FileDownloadHandler) + assert hasattr(myhandler_instance, "download_request") def test_pop_item_with_default_value(self): settings = Settings() @@ -491,14 +486,14 @@ class SettingsTest(unittest.TestCase): settings.pop("DUMMY_CONFIG") dummy_config_value = settings.pop("DUMMY_CONFIG", "dummy_value") - self.assertEqual(dummy_config_value, "dummy_value") + assert dummy_config_value == "dummy_value" def test_pop_item_with_immutable_settings(self): settings = Settings( {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"} ) - self.assertEqual(settings.pop("DUMMY_CONFIG"), "dummy_value") + assert settings.pop("DUMMY_CONFIG") == "dummy_value" settings.freeze() From 7bbe775040d5d695bc3b48a73de5d6fa99312b4c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 9 Mar 2025 23:24:45 +0400 Subject: [PATCH 072/491] Converting tests to plain asserts, part 5. (#6712) --- tests/test_logformatter.py | 93 +++++++-------- tests/test_logstats.py | 45 ++++---- tests/test_mail.py | 71 ++++++------ tests/test_middleware.py | 30 ++--- tests/test_pipelines.py | 16 +-- tests/test_pqueues.py | 113 +++++++++--------- tests/test_proxy_connect.py | 11 +- tests/test_request_attribute_binding.py | 26 ++--- tests/test_request_cb_kwargs.py | 30 ++--- tests/test_request_dict.py | 50 ++++---- tests/test_request_left.py | 8 +- tests/test_responsetypes.py | 8 +- tests/test_robotstxt_interface.py | 106 +++++++---------- tests/test_scheduler.py | 119 +++++++++---------- tests/test_scheduler_base.py | 59 +++++----- tests/test_selector.py | 145 +++++++++++------------- tests/test_signals.py | 6 +- tests/test_toplevel.py | 20 ++-- tests/test_urlparse_monkeypatches.py | 11 +- 19 files changed, 446 insertions(+), 521 deletions(-) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 962692a31..3c9f97631 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -1,11 +1,10 @@ import logging -import unittest import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.python.failure import Failure -from twisted.trial.unittest import TestCase as TwistedTestCase +from twisted.trial.unittest import TestCase from scrapy.exceptions import DropItem from scrapy.http import Request, Response @@ -24,8 +23,8 @@ class CustomItem(Item): return f"name: {self['name']}" -class LogFormatterTestCase(unittest.TestCase): - def setUp(self): +class TestLogFormatter: + def setup_method(self): self.formatter = LogFormatter() self.spider = Spider("default") self.spider.crawler = get_crawler() @@ -35,9 +34,7 @@ class LogFormatterTestCase(unittest.TestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, "Crawled (200) (referer: None)" - ) + assert logline == "Crawled (200) (referer: None)" def test_crawled_without_referer(self): req = Request( @@ -46,9 +43,9 @@ class LogFormatterTestCase(unittest.TestCase): res = Response("http://www.example.com", flags=["cached"]) logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, - "Crawled (200) (referer: http://example.com) ['cached']", + assert ( + logline + == "Crawled (200) (referer: http://example.com) ['cached']" ) def test_flags_in_request(self): @@ -56,9 +53,9 @@ class LogFormatterTestCase(unittest.TestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, - "Crawled (200) ['test', 'flag'] (referer: None)", + assert ( + logline + == "Crawled (200) ['test', 'flag'] (referer: None)" ) def test_dropped(self): @@ -69,7 +66,7 @@ class LogFormatterTestCase(unittest.TestCase): logline = logkws["msg"] % logkws["args"] lines = logline.splitlines() assert all(isinstance(x, str) for x in lines) - self.assertEqual(lines, ["Dropped: \u2018", "{}"]) + assert lines == ["Dropped: \u2018", "{}"] def test_dropitem_default_log_level(self): item = {} @@ -79,38 +76,38 @@ class LogFormatterTestCase(unittest.TestCase): spider.crawler = get_crawler(Spider) logkws = self.formatter.dropped(item, exception, response, spider) - self.assertEqual(logkws["level"], logging.WARNING) + assert logkws["level"] == logging.WARNING spider.crawler.settings.frozen = False spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = logging.INFO spider.crawler.settings.frozen = True logkws = self.formatter.dropped(item, exception, response, spider) - self.assertEqual(logkws["level"], logging.INFO) + assert logkws["level"] == logging.INFO spider.crawler.settings.frozen = False spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = "INFO" spider.crawler.settings.frozen = True logkws = self.formatter.dropped(item, exception, response, spider) - self.assertEqual(logkws["level"], logging.INFO) + assert logkws["level"] == logging.INFO spider.crawler.settings.frozen = False spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = 10 spider.crawler.settings.frozen = True logkws = self.formatter.dropped(item, exception, response, spider) - self.assertEqual(logkws["level"], logging.DEBUG) + assert logkws["level"] == logging.DEBUG spider.crawler.settings.frozen = False spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = 0 spider.crawler.settings.frozen = True logkws = self.formatter.dropped(item, exception, response, spider) - self.assertEqual(logkws["level"], logging.NOTSET) + assert logkws["level"] == logging.NOTSET unsupported_value = object() spider.crawler.settings.frozen = False spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = unsupported_value spider.crawler.settings.frozen = True logkws = self.formatter.dropped(item, exception, response, spider) - self.assertEqual(logkws["level"], unsupported_value) + assert logkws["level"] == unsupported_value with pytest.raises(TypeError): logging.log(logkws["level"], "message") @@ -121,11 +118,11 @@ class LogFormatterTestCase(unittest.TestCase): exception = DropItem("Test drop", log_level="INFO") logkws = self.formatter.dropped(item, exception, response, self.spider) - self.assertEqual(logkws["level"], logging.INFO) + assert logkws["level"] == logging.INFO exception = DropItem("Test drop", log_level="ERROR") logkws = self.formatter.dropped(item, exception, response, self.spider) - self.assertEqual(logkws["level"], logging.ERROR) + assert logkws["level"] == logging.ERROR def test_item_error(self): # In practice, the complete traceback is shown by passing the @@ -135,7 +132,7 @@ class LogFormatterTestCase(unittest.TestCase): response = Response("http://www.example.com") logkws = self.formatter.item_error(item, exception, response, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual(logline, "Error processing {'key': 'value'}") + assert logline == "Error processing {'key': 'value'}" def test_spider_error(self): # In practice, the complete traceback is shown by passing the @@ -147,9 +144,9 @@ class LogFormatterTestCase(unittest.TestCase): response = Response("http://www.example.com", request=request) logkws = self.formatter.spider_error(failure, request, response, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, - "Spider error processing (referer: http://example.org)", + assert ( + logline + == "Spider error processing (referer: http://example.org)" ) def test_download_error_short(self): @@ -159,7 +156,7 @@ class LogFormatterTestCase(unittest.TestCase): request = Request("http://www.example.com") logkws = self.formatter.download_error(failure, request, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual(logline, "Error downloading ") + assert logline == "Error downloading " def test_download_error_long(self): # In practice, the complete traceback is shown by passing the @@ -170,9 +167,7 @@ class LogFormatterTestCase(unittest.TestCase): failure, request, self.spider, "Some message" ) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, "Error downloading : Some message" - ) + assert logline == "Error downloading : Some message" def test_scraped(self): item = CustomItem() @@ -182,9 +177,7 @@ class LogFormatterTestCase(unittest.TestCase): logline = logkws["msg"] % logkws["args"] lines = logline.splitlines() assert all(isinstance(x, str) for x in lines) - self.assertEqual( - lines, ["Scraped from <200 http://www.example.com>", "name: \xa3"] - ) + assert lines == ["Scraped from <200 http://www.example.com>", "name: \xa3"] class LogFormatterSubclass(LogFormatter): @@ -200,8 +193,8 @@ class LogFormatterSubclass(LogFormatter): } -class LogformatterSubclassTest(LogFormatterTestCase): - def setUp(self): +class TestLogformatterSubclass(TestLogFormatter): + def setup_method(self): self.formatter = LogFormatterSubclass() self.spider = Spider("default") self.spider.crawler = get_crawler(Spider) @@ -211,8 +204,8 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, "Crawled (200) (referer: None) []" + assert ( + logline == "Crawled (200) (referer: None) []" ) def test_crawled_without_referer(self): @@ -224,9 +217,9 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, - "Crawled (200) (referer: http://example.com) ['cached']", + assert ( + logline + == "Crawled (200) (referer: http://example.com) ['cached']" ) def test_flags_in_request(self): @@ -234,9 +227,9 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, - "Crawled (200) (referer: None) ['test', 'flag']", + assert ( + logline + == "Crawled (200) (referer: None) ['test', 'flag']" ) @@ -261,7 +254,7 @@ class DropSomeItemsPipeline: self.drop = True -class ShowOrSkipMessagesTestCase(TwistedTestCase): +class TestShowOrSkipMessages(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -284,9 +277,9 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): crawler = get_crawler(ItemSpider, self.base_settings) with LogCapture() as lc: yield crawler.crawl(mockserver=self.mockserver) - self.assertIn("Scraped from <200 http://127.0.0.1:", str(lc)) - self.assertIn("Crawled (200) body

") - self.assertEqual(msg.get("Content-Type"), "text/html") + assert msg.get_payload() == "

body

" + assert msg.get("Content-Type") == "text/html" def test_send_attach(self): attach = BytesIO() @@ -70,22 +69,22 @@ class MailSenderTest(unittest.TestCase): ) assert self.catched_msg - self.assertEqual(self.catched_msg["to"], ["test@scrapy.org"]) - self.assertEqual(self.catched_msg["subject"], "subject") - self.assertEqual(self.catched_msg["body"], "body") + assert self.catched_msg["to"] == ["test@scrapy.org"] + assert self.catched_msg["subject"] == "subject" + assert self.catched_msg["body"] == "body" msg = self.catched_msg["msg"] - self.assertEqual(msg["to"], "test@scrapy.org") - self.assertEqual(msg["subject"], "subject") + assert msg["to"] == "test@scrapy.org" + assert msg["subject"] == "subject" payload = msg.get_payload() assert isinstance(payload, list) - self.assertEqual(len(payload), 2) + assert len(payload) == 2 text, attach = payload - self.assertEqual(text.get_payload(decode=True), b"body") - self.assertEqual(text.get_charset(), Charset("us-ascii")) - self.assertEqual(attach.get_payload(decode=True), b"content") + assert text.get_payload(decode=True) == b"body" + assert text.get_charset() == Charset("us-ascii") + assert attach.get_payload(decode=True) == b"content" def _catch_mail_sent(self, **kwargs): self.catched_msg = {**kwargs} @@ -103,14 +102,14 @@ class MailSenderTest(unittest.TestCase): ) assert self.catched_msg - self.assertEqual(self.catched_msg["subject"], subject) - self.assertEqual(self.catched_msg["body"], body) + assert self.catched_msg["subject"] == subject + assert self.catched_msg["body"] == body msg = self.catched_msg["msg"] - self.assertEqual(msg["subject"], subject) - self.assertEqual(msg.get_payload(decode=True).decode("utf-8"), body) - self.assertEqual(msg.get_charset(), Charset("utf-8")) - self.assertEqual(msg.get("Content-Type"), 'text/plain; charset="utf-8"') + assert msg["subject"] == subject + assert msg.get_payload(decode=True).decode("utf-8") == body + assert msg.get_charset() == Charset("utf-8") + assert msg.get("Content-Type") == 'text/plain; charset="utf-8"' def test_send_attach_utf8(self): subject = "sübjèçt" @@ -131,22 +130,22 @@ class MailSenderTest(unittest.TestCase): ) assert self.catched_msg - self.assertEqual(self.catched_msg["subject"], subject) - self.assertEqual(self.catched_msg["body"], body) + assert self.catched_msg["subject"] == subject + assert self.catched_msg["body"] == body msg = self.catched_msg["msg"] - self.assertEqual(msg["subject"], subject) - self.assertEqual(msg.get_charset(), Charset("utf-8")) - self.assertEqual(msg.get("Content-Type"), 'multipart/mixed; charset="utf-8"') + assert msg["subject"] == subject + assert msg.get_charset() == Charset("utf-8") + assert msg.get("Content-Type") == 'multipart/mixed; charset="utf-8"' payload = msg.get_payload() assert isinstance(payload, list) - self.assertEqual(len(payload), 2) + assert len(payload) == 2 text, attach = payload - self.assertEqual(text.get_payload(decode=True).decode("utf-8"), body) - self.assertEqual(text.get_charset(), Charset("utf-8")) - self.assertEqual(attach.get_payload(decode=True).decode("utf-8"), body) + assert text.get_payload(decode=True).decode("utf-8") == body + assert text.get_charset() == Charset("utf-8") + assert attach.get_payload(decode=True).decode("utf-8") == body def test_create_sender_factory_with_host(self): mailsender = MailSender(debug=False, smtphost="smtp.testhost.com") @@ -156,4 +155,4 @@ class MailSenderTest(unittest.TestCase): ) context = factory.buildProtocol("test@scrapy.org").context - self.assertIsInstance(context, ClientTLSOptions) + assert isinstance(context, ClientTLSOptions) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 0cc532570..d004d4d93 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -1,5 +1,3 @@ -from twisted.trial import unittest - from scrapy.exceptions import NotConfigured from scrapy.middleware import MiddlewareManager from scrapy.utils.test import get_crawler @@ -51,37 +49,27 @@ class MyMiddlewareManager(MiddlewareManager): self.methods["process"].append(mw.process) -class MiddlewareManagerTest(unittest.TestCase): +class TestMiddlewareManager: def test_init(self): m1, m2, m3 = M1(), M2(), M3() mwman = MyMiddlewareManager(m1, m2, m3) - self.assertEqual( - list(mwman.methods["open_spider"]), [m1.open_spider, m2.open_spider] - ) - self.assertEqual( - list(mwman.methods["close_spider"]), [m2.close_spider, m1.close_spider] - ) - self.assertEqual(list(mwman.methods["process"]), [m1.process, m3.process]) + assert list(mwman.methods["open_spider"]) == [m1.open_spider, m2.open_spider] + assert list(mwman.methods["close_spider"]) == [m2.close_spider, m1.close_spider] + assert list(mwman.methods["process"]) == [m1.process, m3.process] def test_methods(self): mwman = MyMiddlewareManager(M1(), M2(), M3()) - self.assertEqual( - [x.__self__.__class__ for x in mwman.methods["open_spider"]], [M1, M2] - ) - self.assertEqual( - [x.__self__.__class__ for x in mwman.methods["close_spider"]], [M2, M1] - ) - self.assertEqual( - [x.__self__.__class__ for x in mwman.methods["process"]], [M1, M3] - ) + assert [x.__self__.__class__ for x in mwman.methods["open_spider"]] == [M1, M2] + assert [x.__self__.__class__ for x in mwman.methods["close_spider"]] == [M2, M1] + assert [x.__self__.__class__ for x in mwman.methods["process"]] == [M1, M3] def test_enabled(self): m1, m2, m3 = M1(), M2(), M3() mwman = MiddlewareManager(m1, m2, m3) - self.assertEqual(mwman.middlewares, (m1, m2, m3)) + assert mwman.middlewares == (m1, m2, m3) def test_enabled_from_settings(self): crawler = get_crawler() mwman = MyMiddlewareManager.from_crawler(crawler) classes = [x.__class__ for x in mwman.middlewares] - self.assertEqual(classes, [M1, M3]) + assert classes == [M1, M3] diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 0ae86235c..743d9774b 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -76,7 +76,7 @@ class ItemSpider(Spider): return {"field": 42} -class PipelineTestCase(unittest.TestCase): +class TestPipeline(unittest.TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -87,8 +87,8 @@ class PipelineTestCase(unittest.TestCase): cls.mockserver.__exit__(None, None, None) def _on_item_scraped(self, item): - self.assertIsInstance(item, dict) - self.assertTrue(item.get("pipeline_passed")) + assert isinstance(item, dict) + assert item.get("pipeline_passed") self.items.append(item) def _create_crawler(self, pipeline_class): @@ -104,30 +104,30 @@ class PipelineTestCase(unittest.TestCase): def test_simple_pipeline(self): crawler = self._create_crawler(SimplePipeline) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 1) + assert len(self.items) == 1 @defer.inlineCallbacks def test_deferred_pipeline(self): crawler = self._create_crawler(DeferredPipeline) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 1) + assert len(self.items) == 1 @defer.inlineCallbacks def test_asyncdef_pipeline(self): crawler = self._create_crawler(AsyncDefPipeline) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 1) + assert len(self.items) == 1 @pytest.mark.only_asyncio @defer.inlineCallbacks def test_asyncdef_asyncio_pipeline(self): crawler = self._create_crawler(AsyncDefAsyncioPipeline) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 1) + assert len(self.items) == 1 @pytest.mark.only_not_asyncio @defer.inlineCallbacks def test_asyncdef_not_asyncio_pipeline(self): crawler = self._create_crawler(AsyncDefNotAsyncioPipeline) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 1) + assert len(self.items) == 1 diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index c223c4562..d5c710ed2 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -1,5 +1,4 @@ import tempfile -import unittest import pytest import queuelib @@ -12,8 +11,8 @@ from scrapy.utils.test import get_crawler from tests.test_scheduler import MockDownloader, MockEngine -class PriorityQueueTest(unittest.TestCase): - def setUp(self): +class TestPriorityQueue: + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") @@ -22,20 +21,20 @@ class PriorityQueueTest(unittest.TestCase): queue = ScrapyPriorityQueue.from_crawler( self.crawler, FifoMemoryQueue, temp_dir ) - self.assertIsNone(queue.pop()) - self.assertEqual(len(queue), 0) + assert queue.pop() is None + assert len(queue) == 0 req1 = Request("https://example.org/1", priority=1) queue.push(req1) - self.assertEqual(len(queue), 1) + assert len(queue) == 1 dequeued = queue.pop() - self.assertEqual(len(queue), 0) - self.assertEqual(dequeued.url, req1.url) - self.assertEqual(dequeued.priority, req1.priority) - self.assertEqual(queue.close(), []) + assert len(queue) == 0 + assert dequeued.url == req1.url + assert dequeued.priority == req1.priority + assert not queue.close() def test_no_peek_raises(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") + pytest.skip("queuelib.queue.FifoMemoryQueue.peek is defined") temp_dir = tempfile.mkdtemp() queue = ScrapyPriorityQueue.from_crawler( self.crawler, FifoMemoryQueue, temp_dir @@ -50,53 +49,53 @@ class PriorityQueueTest(unittest.TestCase): def test_peek(self): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined") + pytest.skip("queuelib.queue.FifoMemoryQueue.peek is undefined") temp_dir = tempfile.mkdtemp() queue = ScrapyPriorityQueue.from_crawler( self.crawler, FifoMemoryQueue, temp_dir ) - self.assertEqual(len(queue), 0) - self.assertIsNone(queue.peek()) + assert len(queue) == 0 + assert queue.peek() is None req1 = Request("https://example.org/1") req2 = Request("https://example.org/2") req3 = Request("https://example.org/3") queue.push(req1) queue.push(req2) queue.push(req3) - self.assertEqual(len(queue), 3) - self.assertEqual(queue.peek().url, req1.url) - self.assertEqual(queue.pop().url, req1.url) - self.assertEqual(len(queue), 2) - self.assertEqual(queue.peek().url, req2.url) - self.assertEqual(queue.pop().url, req2.url) - self.assertEqual(len(queue), 1) - self.assertEqual(queue.peek().url, req3.url) - self.assertEqual(queue.pop().url, req3.url) - self.assertEqual(queue.close(), []) + assert len(queue) == 3 + assert queue.peek().url == req1.url + assert queue.pop().url == req1.url + assert len(queue) == 2 + assert queue.peek().url == req2.url + assert queue.pop().url == req2.url + assert len(queue) == 1 + assert queue.peek().url == req3.url + assert queue.pop().url == req3.url + assert not queue.close() def test_queue_push_pop_priorities(self): temp_dir = tempfile.mkdtemp() queue = ScrapyPriorityQueue.from_crawler( self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3] ) - self.assertIsNone(queue.pop()) - self.assertEqual(len(queue), 0) + assert queue.pop() is None + assert len(queue) == 0 req1 = Request("https://example.org/1", priority=1) req2 = Request("https://example.org/2", priority=2) req3 = Request("https://example.org/3", priority=3) queue.push(req1) queue.push(req2) queue.push(req3) - self.assertEqual(len(queue), 3) + assert len(queue) == 3 dequeued = queue.pop() - self.assertEqual(len(queue), 2) - self.assertEqual(dequeued.url, req3.url) - self.assertEqual(dequeued.priority, req3.priority) - self.assertEqual(queue.close(), [-1, -2]) + assert len(queue) == 2 + assert dequeued.url == req3.url + assert dequeued.priority == req3.priority + assert queue.close() == [-1, -2] -class DownloaderAwarePriorityQueueTest(unittest.TestCase): - def setUp(self): +class TestDownloaderAwarePriorityQueue: + def setup_method(self): crawler = get_crawler(Spider) crawler.engine = MockEngine(downloader=MockDownloader()) self.queue = DownloaderAwarePriorityQueue.from_crawler( @@ -105,30 +104,30 @@ class DownloaderAwarePriorityQueueTest(unittest.TestCase): key="foo/bar", ) - def tearDown(self): + def teardown_method(self): self.queue.close() def test_push_pop(self): - self.assertEqual(len(self.queue), 0) - self.assertIsNone(self.queue.pop()) + assert len(self.queue) == 0 + assert self.queue.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") self.queue.push(req1) self.queue.push(req2) self.queue.push(req3) - self.assertEqual(len(self.queue), 3) - self.assertEqual(self.queue.pop().url, req1.url) - self.assertEqual(len(self.queue), 2) - self.assertEqual(self.queue.pop().url, req2.url) - self.assertEqual(len(self.queue), 1) - self.assertEqual(self.queue.pop().url, req3.url) - self.assertEqual(len(self.queue), 0) - self.assertIsNone(self.queue.pop()) + assert len(self.queue) == 3 + assert self.queue.pop().url == req1.url + assert len(self.queue) == 2 + assert self.queue.pop().url == req2.url + assert len(self.queue) == 1 + assert self.queue.pop().url == req3.url + assert len(self.queue) == 0 + assert self.queue.pop() is None def test_no_peek_raises(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") + pytest.skip("queuelib.queue.FifoMemoryQueue.peek is defined") self.queue.push(Request("https://example.org")) with pytest.raises( NotImplementedError, @@ -138,21 +137,21 @@ class DownloaderAwarePriorityQueueTest(unittest.TestCase): def test_peek(self): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined") - self.assertEqual(len(self.queue), 0) + pytest.skip("queuelib.queue.FifoMemoryQueue.peek is undefined") + assert len(self.queue) == 0 req1 = Request("https://example.org/1") req2 = Request("https://example.org/2") req3 = Request("https://example.org/3") self.queue.push(req1) self.queue.push(req2) self.queue.push(req3) - self.assertEqual(len(self.queue), 3) - self.assertEqual(self.queue.peek().url, req1.url) - self.assertEqual(self.queue.pop().url, req1.url) - self.assertEqual(len(self.queue), 2) - self.assertEqual(self.queue.peek().url, req2.url) - self.assertEqual(self.queue.pop().url, req2.url) - self.assertEqual(len(self.queue), 1) - self.assertEqual(self.queue.peek().url, req3.url) - self.assertEqual(self.queue.pop().url, req3.url) - self.assertIsNone(self.queue.peek()) + assert len(self.queue) == 3 + assert self.queue.peek().url == req1.url + assert self.queue.pop().url == req1.url + assert len(self.queue) == 2 + assert self.queue.peek().url == req2.url + assert self.queue.pop().url == req2.url + assert len(self.queue) == 1 + assert self.queue.peek().url == req3.url + assert self.queue.pop().url == req3.url + assert self.queue.peek() is None diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 6ed7e93a6..885b7b7ae 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -6,6 +6,7 @@ from pathlib import Path from subprocess import PIPE, Popen from urllib.parse import urlsplit, urlunsplit +import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -61,7 +62,7 @@ def _wrong_credentials(proxy_url): return urlunsplit(bad_auth_proxy) -class ProxyConnectTestCase(TestCase): +class TestProxyConnect(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -75,7 +76,7 @@ class ProxyConnectTestCase(TestCase): try: import mitmproxy # noqa: F401 except ImportError: - self.skipTest("mitmproxy is not installed") + pytest.skip("mitmproxy is not installed") self._oldenv = os.environ.copy() @@ -113,12 +114,12 @@ class ProxyConnectTestCase(TestCase): yield crawler.crawl(seed=request) self._assert_got_response_code(200, log) echo = json.loads(crawler.spider.meta["responses"][0].text) - self.assertTrue("Proxy-Authorization" not in echo["headers"]) + assert "Proxy-Authorization" not in echo["headers"] def _assert_got_response_code(self, code, log): print(log) - self.assertEqual(str(log).count(f"Crawled ({code})"), 1) + assert str(log).count(f"Crawled ({code})") == 1 def _assert_got_tunnel_error(self, log): print(log) - self.assertIn("TunnelError", str(log)) + assert "TunnelError" in str(log) diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 0072660a7..9b42fd6c7 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -56,7 +56,7 @@ class AlternativeCallbacksMiddleware: return response.replace(request=new_request) -class CrawlTestCase(TestCase): +class TestCrawl(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -72,7 +72,7 @@ class CrawlTestCase(TestCase): crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] - self.assertEqual(response.request.url, url) + assert response.request.url == url @defer.inlineCallbacks def test_response_error(self): @@ -82,8 +82,8 @@ class CrawlTestCase(TestCase): yield crawler.crawl(seed=url, mockserver=self.mockserver) failure = crawler.spider.meta["failure"] response = failure.value.response - self.assertEqual(failure.request.url, url) - self.assertEqual(response.request.url, url) + assert failure.request.url == url + assert response.request.url == url @defer.inlineCallbacks def test_downloader_middleware_raise_exception(self): @@ -98,8 +98,8 @@ class CrawlTestCase(TestCase): ) yield crawler.crawl(seed=url, mockserver=self.mockserver) failure = crawler.spider.meta["failure"] - self.assertEqual(failure.request.url, url) - self.assertIsInstance(failure.value, ZeroDivisionError) + assert failure.request.url == url + assert isinstance(failure.value, ZeroDivisionError) @defer.inlineCallbacks def test_downloader_middleware_override_request_in_process_response(self): @@ -131,10 +131,10 @@ class CrawlTestCase(TestCase): yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] - self.assertEqual(response.request.url, OVERRIDDEN_URL) + assert response.request.url == OVERRIDDEN_URL - self.assertEqual(signal_params["response"].url, url) - self.assertEqual(signal_params["request"].url, OVERRIDDEN_URL) + assert signal_params["response"].url == url + assert signal_params["request"].url == OVERRIDDEN_URL log.check_present( ( @@ -164,8 +164,8 @@ class CrawlTestCase(TestCase): ) yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] - self.assertEqual(response.body, b"Caught ZeroDivisionError") - self.assertEqual(response.request.url, OVERRIDDEN_URL) + assert response.body == b"Caught ZeroDivisionError" + assert response.request.url == OVERRIDDEN_URL @defer.inlineCallbacks def test_downloader_middleware_do_not_override_in_process_exception(self): @@ -187,8 +187,8 @@ class CrawlTestCase(TestCase): ) yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] - self.assertEqual(response.body, b"Caught ZeroDivisionError") - self.assertEqual(response.request.url, url) + assert response.body == b"Caught ZeroDivisionError" + assert response.request.url == url @defer.inlineCallbacks def test_downloader_middleware_alternative_callback(self): diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index a21cb43ff..ab6baa5f0 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -151,7 +151,7 @@ class KeywordArgumentsSpider(MockServerSpider): self.crawler.stats.inc_value("boolean_checks", 1) -class CallbackKeywordArgumentsTestCase(TestCase): +class TestCallbackKeywordArguments(TestCase): maxDiff = None @classmethod @@ -168,27 +168,19 @@ class CallbackKeywordArgumentsTestCase(TestCase): crawler = get_crawler(KeywordArgumentsSpider) with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertTrue(all(crawler.spider.checks)) - self.assertEqual( - len(crawler.spider.checks), crawler.stats.get_value("boolean_checks") - ) + assert all(crawler.spider.checks) + assert len(crawler.spider.checks) == crawler.stats.get_value("boolean_checks") # check exceptions for argument mismatch exceptions = {} for line in log.records: for key in ("takes_less", "takes_more"): if key in line.getMessage(): exceptions[key] = line - self.assertEqual(exceptions["takes_less"].exc_info[0], TypeError) - self.assertTrue( - str(exceptions["takes_less"].exc_info[1]).endswith( - "parse_takes_less() got an unexpected keyword argument 'number'" - ), - msg="Exception message: " + str(exceptions["takes_less"].exc_info[1]), - ) - self.assertEqual(exceptions["takes_more"].exc_info[0], TypeError) - self.assertTrue( - str(exceptions["takes_more"].exc_info[1]).endswith( - "parse_takes_more() missing 1 required positional argument: 'other'" - ), - msg="Exception message: " + str(exceptions["takes_more"].exc_info[1]), - ) + assert exceptions["takes_less"].exc_info[0] is TypeError + assert str(exceptions["takes_less"].exc_info[1]).endswith( + "parse_takes_less() got an unexpected keyword argument 'number'" + ), "Exception message: " + str(exceptions["takes_less"].exc_info[1]) + assert exceptions["takes_more"].exc_info[0] is TypeError + assert str(exceptions["takes_more"].exc_info[1]).endswith( + "parse_takes_more() missing 1 required positional argument: 'other'" + ), "Exception message: " + str(exceptions["takes_more"].exc_info[1]) diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 2c605a015..ea7018541 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -1,5 +1,3 @@ -import unittest - import pytest from scrapy import Request, Spider @@ -11,8 +9,8 @@ class CustomRequest(Request): pass -class RequestSerializationTest(unittest.TestCase): - def setUp(self): +class TestRequestSerialization: + def setup_method(self): self.spider = MethodsSpider() def test_basic(self): @@ -50,23 +48,23 @@ class RequestSerializationTest(unittest.TestCase): self._assert_same_request(request, request2) def _assert_same_request(self, r1, r2): - self.assertEqual(r1.__class__, r2.__class__) - self.assertEqual(r1.url, r2.url) - self.assertEqual(r1.callback, r2.callback) - self.assertEqual(r1.errback, r2.errback) - self.assertEqual(r1.method, r2.method) - self.assertEqual(r1.body, r2.body) - self.assertEqual(r1.headers, r2.headers) - self.assertEqual(r1.cookies, r2.cookies) - self.assertEqual(r1.meta, r2.meta) - self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) - self.assertEqual(r1.encoding, r2.encoding) - self.assertEqual(r1._encoding, r2._encoding) - self.assertEqual(r1.priority, r2.priority) - self.assertEqual(r1.dont_filter, r2.dont_filter) - self.assertEqual(r1.flags, r2.flags) + assert r1.__class__ == r2.__class__ + assert r1.url == r2.url + assert r1.callback == r2.callback + assert r1.errback == r2.errback + assert r1.method == r2.method + assert r1.body == r2.body + assert r1.headers == r2.headers + assert r1.cookies == r2.cookies + assert r1.meta == r2.meta + assert r1.cb_kwargs == r2.cb_kwargs + assert r1.encoding == r2.encoding + assert r1._encoding == r2._encoding + assert r1.priority == r2.priority + assert r1.dont_filter == r2.dont_filter + assert r1.flags == r2.flags if isinstance(r1, JsonRequest): - self.assertEqual(r1.dumps_kwargs, r2.dumps_kwargs) + assert r1.dumps_kwargs == r2.dumps_kwargs def test_request_class(self): r1 = FormRequest("http://www.example.com") @@ -92,8 +90,8 @@ class RequestSerializationTest(unittest.TestCase): ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) - self.assertEqual(request_dict["callback"], "parse_item_reference") - self.assertEqual(request_dict["errback"], "handle_error_reference") + assert request_dict["callback"] == "parse_item_reference" + assert request_dict["errback"] == "handle_error_reference" def test_private_reference_callback_serialization(self): r = Request( @@ -103,12 +101,8 @@ class RequestSerializationTest(unittest.TestCase): ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) - self.assertEqual( - request_dict["callback"], "_MethodsSpider__parse_item_reference" - ) - self.assertEqual( - request_dict["errback"], "_MethodsSpider__handle_error_reference" - ) + assert request_dict["callback"] == "_MethodsSpider__parse_item_reference" + assert request_dict["errback"] == "_MethodsSpider__handle_error_reference" def test_private_callback_serialization(self): r = Request( diff --git a/tests/test_request_left.py b/tests/test_request_left.py index cf4c8a2d5..d55905f9c 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -38,22 +38,22 @@ class TestCatching(TestCase): def test_success(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl(self.mockserver.url("/status?n=200")) - self.assertEqual(crawler.spider.caught_times, 1) + assert crawler.spider.caught_times == 1 @defer.inlineCallbacks def test_timeout(self): crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1}) yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) - self.assertEqual(crawler.spider.caught_times, 1) + assert crawler.spider.caught_times == 1 @defer.inlineCallbacks def test_disconnect(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl(self.mockserver.url("/drop")) - self.assertEqual(crawler.spider.caught_times, 1) + assert crawler.spider.caught_times == 1 @defer.inlineCallbacks def test_noconnect(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl("http://thereisdefinetelynosuchdomain.com") - self.assertEqual(crawler.spider.caught_times, 1) + assert crawler.spider.caught_times == 1 diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index f9f56ff97..5b04c7436 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -1,5 +1,3 @@ -import unittest - from scrapy.http import ( Headers, HtmlResponse, @@ -11,7 +9,7 @@ from scrapy.http import ( from scrapy.responsetypes import responsetypes -class ResponseTypesTest(unittest.TestCase): +class TestResponseTypes: def test_from_filename(self): mappings = [ ("data.bin", Response), @@ -123,6 +121,4 @@ class ResponseTypesTest(unittest.TestCase): def test_custom_mime_types_loaded(self): # check that mime.types files shipped with scrapy are loaded - self.assertEqual( - responsetypes.mimetypes.guess_type("x.scrapytest")[0], "x-scrapy/test" - ) + assert responsetypes.mimetypes.guess_type("x.scrapytest")[0] == "x-scrapy/test" diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 0d00ff660..221ccabe6 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,4 +1,4 @@ -from twisted.trial import unittest +import pytest from scrapy.robotstxt import decode_robotstxt @@ -32,8 +32,8 @@ class BaseRobotParserTest: rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertTrue(rp.allowed("https://www.site.local/allowed", "*")) - self.assertFalse(rp.allowed("https://www.site.local/disallowed", "*")) + assert rp.allowed("https://www.site.local/allowed", "*") + assert not rp.allowed("https://www.site.local/disallowed", "*") def test_allowed_wildcards(self): robotstxt_robotstxt_body = b"""User-agent: first @@ -47,42 +47,36 @@ class BaseRobotParserTest: crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertTrue(rp.allowed("https://www.site.local/disallowed", "first")) - self.assertFalse( - rp.allowed("https://www.site.local/disallowed/xyz/end", "first") - ) - self.assertFalse( - rp.allowed("https://www.site.local/disallowed/abc/end", "first") - ) - self.assertTrue( - rp.allowed("https://www.site.local/disallowed/xyz/endinglater", "first") - ) + assert rp.allowed("https://www.site.local/disallowed", "first") + assert not rp.allowed("https://www.site.local/disallowed/xyz/end", "first") + assert not rp.allowed("https://www.site.local/disallowed/abc/end", "first") + assert rp.allowed("https://www.site.local/disallowed/xyz/endinglater", "first") - self.assertTrue(rp.allowed("https://www.site.local/allowed", "second")) - self.assertTrue(rp.allowed("https://www.site.local/is_still_allowed", "second")) - self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second")) + assert rp.allowed("https://www.site.local/allowed", "second") + assert rp.allowed("https://www.site.local/is_still_allowed", "second") + assert rp.allowed("https://www.site.local/is_allowed_too", "second") def test_length_based_precedence(self): robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertTrue(rp.allowed("https://www.site.local/page", "*")) + assert rp.allowed("https://www.site.local/page", "*") def test_order_based_precedence(self): robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertFalse(rp.allowed("https://www.site.local/page", "*")) + assert not rp.allowed("https://www.site.local/page", "*") def test_empty_response(self): """empty response should equal 'allow all'""" rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b"") - self.assertTrue(rp.allowed("https://site.local/", "*")) - self.assertTrue(rp.allowed("https://site.local/", "chrome")) - self.assertTrue(rp.allowed("https://site.local/index.html", "*")) - self.assertTrue(rp.allowed("https://site.local/disallowed", "*")) + assert rp.allowed("https://site.local/", "*") + assert rp.allowed("https://site.local/", "chrome") + assert rp.allowed("https://site.local/index.html", "*") + assert rp.allowed("https://site.local/disallowed", "*") def test_garbage_response(self): """garbage response should be discarded, equal 'allow all'""" @@ -90,10 +84,10 @@ class BaseRobotParserTest: rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertTrue(rp.allowed("https://site.local/", "*")) - self.assertTrue(rp.allowed("https://site.local/", "chrome")) - self.assertTrue(rp.allowed("https://site.local/index.html", "*")) - self.assertTrue(rp.allowed("https://site.local/disallowed", "*")) + assert rp.allowed("https://site.local/", "*") + assert rp.allowed("https://site.local/", "chrome") + assert rp.allowed("https://site.local/index.html", "*") + assert rp.allowed("https://site.local/disallowed", "*") def test_unicode_url_and_useragent(self): robotstxt_robotstxt_body = """ @@ -109,79 +103,67 @@ class BaseRobotParserTest: rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertTrue(rp.allowed("https://site.local/", "*")) - self.assertFalse(rp.allowed("https://site.local/admin/", "*")) - self.assertFalse(rp.allowed("https://site.local/static/", "*")) - self.assertTrue(rp.allowed("https://site.local/admin/", "UnicödeBöt")) - self.assertFalse( - rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*") - ) - self.assertFalse(rp.allowed("https://site.local/wiki/Käyttäjä:", "*")) - self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*")) - self.assertFalse( - rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt") - ) + assert rp.allowed("https://site.local/", "*") + assert not rp.allowed("https://site.local/admin/", "*") + assert not rp.allowed("https://site.local/static/", "*") + assert rp.allowed("https://site.local/admin/", "UnicödeBöt") + assert not rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*") + assert not rp.allowed("https://site.local/wiki/Käyttäjä:", "*") + assert rp.allowed("https://site.local/some/randome/page.html", "*") + assert not rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt") -class DecodeRobotsTxtTest(unittest.TestCase): +class TestDecodeRobotsTxt: def test_native_string_conversion(self): robotstxt_body = b"User-agent: *\nDisallow: /\n" decoded_content = decode_robotstxt( robotstxt_body, spider=None, to_native_str_type=True ) - self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") + assert decoded_content == "User-agent: *\nDisallow: /\n" def test_decode_utf8(self): robotstxt_body = b"User-agent: *\nDisallow: /\n" decoded_content = decode_robotstxt(robotstxt_body, spider=None) - self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") + assert decoded_content == "User-agent: *\nDisallow: /\n" def test_decode_non_utf8(self): robotstxt_body = b"User-agent: *\n\xffDisallow: /\n" decoded_content = decode_robotstxt(robotstxt_body, spider=None) - self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") + assert decoded_content == "User-agent: *\nDisallow: /\n" -class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): - def setUp(self): +class TestPythonRobotParser(BaseRobotParserTest): + def setup_method(self): from scrapy.robotstxt import PythonRobotParser super()._setUp(PythonRobotParser) def test_length_based_precedence(self): - raise unittest.SkipTest( + pytest.skip( "RobotFileParser does not support length based directives precedence." ) def test_allowed_wildcards(self): - raise unittest.SkipTest("RobotFileParser does not support wildcards.") + pytest.skip("RobotFileParser does not support wildcards.") -class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase): - if not rerp_available(): - skip = "Rerp parser is not installed" - - def setUp(self): +@pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed") +class TestRerpRobotParser(BaseRobotParserTest): + def setup_method(self): from scrapy.robotstxt import RerpRobotParser super()._setUp(RerpRobotParser) def test_length_based_precedence(self): - raise unittest.SkipTest( - "Rerp does not support length based directives precedence." - ) + pytest.skip("Rerp does not support length based directives precedence.") -class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase): - if not protego_available(): - skip = "Protego parser is not installed" - - def setUp(self): +@pytest.mark.skipif(not protego_available(), reason="Protego parser is not installed") +class TestProtegoRobotParser(BaseRobotParserTest): + def setup_method(self): from scrapy.robotstxt import ProtegoRobotParser super()._setUp(ProtegoRobotParser) def test_order_based_precedence(self): - raise unittest.SkipTest( - "Protego does not support order based directives precedence." - ) + pytest.skip("Protego does not support order based directives precedence.") diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index f2f8b96cd..1d6992a32 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -2,7 +2,7 @@ from __future__ import annotations import shutil import tempfile -import unittest +from abc import ABC, abstractmethod from typing import Any, NamedTuple import pytest @@ -65,10 +65,14 @@ class MockCrawler(Crawler): self.stats = load_object(self.settings["STATS_CLASS"])(self) -class SchedulerHandler: - priority_queue_cls: str | None = None +class SchedulerHandler(ABC): jobdir = None + @property + @abstractmethod + def priority_queue_cls(self) -> str: + raise NotImplementedError + def create_scheduler(self): self.mock_crawler = MockCrawler(self.priority_queue_cls, self.jobdir) self.scheduler = Scheduler.from_crawler(self.mock_crawler) @@ -80,10 +84,10 @@ class SchedulerHandler: self.mock_crawler.stop() self.mock_crawler.engine.downloader.close() - def setUp(self): + def setup_method(self): self.create_scheduler() - def tearDown(self): + def teardown_method(self): self.close_scheduler() @@ -99,16 +103,16 @@ _PRIORITIES = [ _URLS = {"http://foo.com/a", "http://foo.com/b", "http://foo.com/c"} -class BaseSchedulerInMemoryTester(SchedulerHandler): +class TestSchedulerInMemoryBase(SchedulerHandler): def test_length(self): - self.assertFalse(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), 0) + assert not self.scheduler.has_pending_requests() + assert len(self.scheduler) == 0 for url in _URLS: self.scheduler.enqueue_request(Request(url)) - self.assertTrue(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), len(_URLS)) + assert self.scheduler.has_pending_requests() + assert len(self.scheduler) == len(_URLS) def test_dequeue(self): for url in _URLS: @@ -118,7 +122,7 @@ class BaseSchedulerInMemoryTester(SchedulerHandler): while self.scheduler.has_pending_requests(): urls.add(self.scheduler.next_request().url) - self.assertEqual(urls, _URLS) + assert urls == _URLS def test_dequeue_priorities(self): for url, priority in _PRIORITIES: @@ -128,25 +132,23 @@ class BaseSchedulerInMemoryTester(SchedulerHandler): while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) - self.assertEqual( - priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x) - ) + assert priorities == sorted([x[1] for x in _PRIORITIES], key=lambda x: -x) -class BaseSchedulerOnDiskTester(SchedulerHandler): - def setUp(self): +class TestSchedulerOnDiskBase(SchedulerHandler): + def setup_method(self): self.jobdir = tempfile.mkdtemp() self.create_scheduler() - def tearDown(self): + def teardown_method(self): self.close_scheduler() shutil.rmtree(self.jobdir) self.jobdir = None def test_length(self): - self.assertFalse(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), 0) + assert not self.scheduler.has_pending_requests() + assert len(self.scheduler) == 0 for url in _URLS: self.scheduler.enqueue_request(Request(url)) @@ -154,8 +156,8 @@ class BaseSchedulerOnDiskTester(SchedulerHandler): self.close_scheduler() self.create_scheduler() - self.assertTrue(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), len(_URLS)) + assert self.scheduler.has_pending_requests() + assert len(self.scheduler) == len(_URLS) def test_dequeue(self): for url in _URLS: @@ -168,7 +170,7 @@ class BaseSchedulerOnDiskTester(SchedulerHandler): while self.scheduler.has_pending_requests(): urls.add(self.scheduler.next_request().url) - self.assertEqual(urls, _URLS) + assert urls == _URLS def test_dequeue_priorities(self): for url, priority in _PRIORITIES: @@ -181,17 +183,19 @@ class BaseSchedulerOnDiskTester(SchedulerHandler): while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) - self.assertEqual( - priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x) - ) + assert priorities == sorted([x[1] for x in _PRIORITIES], key=lambda x: -x) -class TestSchedulerInMemory(BaseSchedulerInMemoryTester, unittest.TestCase): - priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue" +class TestSchedulerInMemory(TestSchedulerInMemoryBase): + @property + def priority_queue_cls(self) -> str: + return "scrapy.pqueues.ScrapyPriorityQueue" -class TestSchedulerOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): - priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue" +class TestSchedulerOnDisk(TestSchedulerOnDiskBase): + @property + def priority_queue_cls(self) -> str: + return "scrapy.pqueues.ScrapyPriorityQueue" _URLS_WITH_SLOTS = [ @@ -204,37 +208,34 @@ _URLS_WITH_SLOTS = [ ] -class TestMigration(unittest.TestCase): - def setUp(self): - self.tmpdir = tempfile.mkdtemp() +class TestMigration: + def test_migration(self, tmpdir): + class PrevSchedulerHandler(SchedulerHandler): + jobdir = tmpdir - def tearDown(self): - shutil.rmtree(self.tmpdir) + @property + def priority_queue_cls(self) -> str: + return "scrapy.pqueues.ScrapyPriorityQueue" - def _migration(self, tmp_dir): - prev_scheduler_handler = SchedulerHandler() - prev_scheduler_handler.priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue" - prev_scheduler_handler.jobdir = tmp_dir + class NextSchedulerHandler(SchedulerHandler): + jobdir = tmpdir + @property + def priority_queue_cls(self) -> str: + return "scrapy.pqueues.DownloaderAwarePriorityQueue" + + prev_scheduler_handler = PrevSchedulerHandler() prev_scheduler_handler.create_scheduler() for url in _URLS: prev_scheduler_handler.scheduler.enqueue_request(Request(url)) prev_scheduler_handler.close_scheduler() - next_scheduler_handler = SchedulerHandler() - next_scheduler_handler.priority_queue_cls = ( - "scrapy.pqueues.DownloaderAwarePriorityQueue" - ) - next_scheduler_handler.jobdir = tmp_dir - - next_scheduler_handler.create_scheduler() - - def test_migration(self): + next_scheduler_handler = NextSchedulerHandler() with pytest.raises( ValueError, match="DownloaderAwarePriorityQueue accepts ``slot_startprios`` as a dict", ): - self._migration(self.tmpdir) + next_scheduler_handler.create_scheduler() def _is_scheduling_fair(enqueued_slots, dequeued_slots): @@ -263,9 +264,12 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots): class DownloaderAwareSchedulerTestMixin: - priority_queue_cls: str | None = "scrapy.pqueues.DownloaderAwarePriorityQueue" reopen = False + @property + def priority_queue_cls(self) -> str: + return "scrapy.pqueues.DownloaderAwarePriorityQueue" + def test_logic(self): for url, slot in _URLS_WITH_SLOTS: request = Request(url) @@ -290,20 +294,18 @@ class DownloaderAwareSchedulerTestMixin: slot = downloader.get_slot_key(request) downloader.decrement(slot) - self.assertTrue( - _is_scheduling_fair([s for u, s in _URLS_WITH_SLOTS], dequeued_slots) - ) - self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0) + assert _is_scheduling_fair([s for u, s in _URLS_WITH_SLOTS], dequeued_slots) + assert sum(len(s.active) for s in downloader.slots.values()) == 0 class TestSchedulerWithDownloaderAwareInMemory( - DownloaderAwareSchedulerTestMixin, BaseSchedulerInMemoryTester, unittest.TestCase + DownloaderAwareSchedulerTestMixin, TestSchedulerInMemoryBase ): pass class TestSchedulerWithDownloaderAwareOnDisk( - DownloaderAwareSchedulerTestMixin, BaseSchedulerOnDiskTester, unittest.TestCase + DownloaderAwareSchedulerTestMixin, TestSchedulerOnDiskBase ): reopen = True @@ -337,13 +339,12 @@ class TestIntegrationWithDownloaderAwareInMemory(TestCase): url = mockserver.url("/status?n=200", is_secure=False) start_urls = [url] * 6 yield self.crawler.crawl(start_urls) - self.assertEqual( - self.crawler.stats.get_value("downloader/response_count"), - len(start_urls), + assert self.crawler.stats.get_value("downloader/response_count") == len( + start_urls ) -class TestIncompatibility(unittest.TestCase): +class TestIncompatibility: def _incompatible(self): settings = { "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.DownloaderAwarePriorityQueue", diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index c2bb8cec5..4a36d3cdb 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -1,12 +1,11 @@ from __future__ import annotations -from unittest import TestCase from urllib.parse import urljoin import pytest from testfixtures import LogCapture from twisted.internet import defer -from twisted.trial.unittest import TestCase as TwistedTestCase +from twisted.trial.unittest import TestCase from scrapy.core.scheduler import BaseScheduler from scrapy.http import Request @@ -65,17 +64,17 @@ class PathsSpider(Spider): class InterfaceCheckMixin: def test_scheduler_class(self): - self.assertTrue(isinstance(self.scheduler, BaseScheduler)) - self.assertTrue(issubclass(self.scheduler.__class__, BaseScheduler)) + assert isinstance(self.scheduler, BaseScheduler) + assert issubclass(self.scheduler.__class__, BaseScheduler) -class BaseSchedulerTest(TestCase, InterfaceCheckMixin): - def setUp(self): +class TestBaseScheduler(InterfaceCheckMixin): + def setup_method(self): self.scheduler = BaseScheduler() def test_methods(self): - self.assertIsNone(self.scheduler.open(Spider("foo"))) - self.assertIsNone(self.scheduler.close("finished")) + assert self.scheduler.open(Spider("foo")) is None + assert self.scheduler.close("finished") is None with pytest.raises(NotImplementedError): self.scheduler.has_pending_requests() with pytest.raises(NotImplementedError): @@ -84,8 +83,8 @@ class BaseSchedulerTest(TestCase, InterfaceCheckMixin): self.scheduler.next_request() -class MinimalSchedulerTest(TestCase, InterfaceCheckMixin): - def setUp(self): +class TestMinimalScheduler(InterfaceCheckMixin): + def setup_method(self): self.scheduler = MinimalScheduler() def test_open_close(self): @@ -101,51 +100,51 @@ class MinimalSchedulerTest(TestCase, InterfaceCheckMixin): len(self.scheduler) def test_enqueue_dequeue(self): - self.assertFalse(self.scheduler.has_pending_requests()) + assert not self.scheduler.has_pending_requests() for url in URLS: - self.assertTrue(self.scheduler.enqueue_request(Request(url))) - self.assertFalse(self.scheduler.enqueue_request(Request(url))) - self.assertTrue(self.scheduler.has_pending_requests) + assert self.scheduler.enqueue_request(Request(url)) + assert not self.scheduler.enqueue_request(Request(url)) + assert self.scheduler.has_pending_requests dequeued = [] while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() dequeued.append(request.url) - self.assertEqual(set(dequeued), set(URLS)) - self.assertFalse(self.scheduler.has_pending_requests()) + assert set(dequeued) == set(URLS) + assert not self.scheduler.has_pending_requests() -class SimpleSchedulerTest(TwistedTestCase, InterfaceCheckMixin): +class SimpleSchedulerTest(TestCase, InterfaceCheckMixin): def setUp(self): self.scheduler = SimpleScheduler() @defer.inlineCallbacks def test_enqueue_dequeue(self): open_result = yield self.scheduler.open(Spider("foo")) - self.assertEqual(open_result, "open") - self.assertFalse(self.scheduler.has_pending_requests()) + assert open_result == "open" + assert not self.scheduler.has_pending_requests() for url in URLS: - self.assertTrue(self.scheduler.enqueue_request(Request(url))) - self.assertFalse(self.scheduler.enqueue_request(Request(url))) + assert self.scheduler.enqueue_request(Request(url)) + assert not self.scheduler.enqueue_request(Request(url)) - self.assertTrue(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), len(URLS)) + assert self.scheduler.has_pending_requests() + assert len(self.scheduler) == len(URLS) dequeued = [] while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() dequeued.append(request.url) - self.assertEqual(set(dequeued), set(URLS)) + assert set(dequeued) == set(URLS) - self.assertFalse(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), 0) + assert not self.scheduler.has_pending_requests() + assert len(self.scheduler) == 0 close_result = yield self.scheduler.close("") - self.assertEqual(close_result, "close") + assert close_result == "close" -class MinimalSchedulerCrawlTest(TwistedTestCase): +class MinimalSchedulerCrawlTest(TestCase): scheduler_cls = MinimalScheduler @defer.inlineCallbacks @@ -158,8 +157,8 @@ class MinimalSchedulerCrawlTest(TwistedTestCase): crawler = get_crawler(PathsSpider, settings) yield crawler.crawl(mockserver) for path in PATHS: - self.assertIn(f"{{'path': '{path}'}}", str(log)) - self.assertIn(f"'item_scraped_count': {len(PATHS)}", str(log)) + assert f"{{'path': '{path}'}}" in str(log) + assert f"'item_scraped_count': {len(PATHS)}" in str(log) class SimpleSchedulerCrawlTest(MinimalSchedulerCrawlTest): diff --git a/tests/test_selector.py b/tests/test_selector.py index 2d7a1442e..5c8eadf0b 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -3,7 +3,6 @@ import weakref import parsel import pytest from packaging import version -from twisted.trial import unittest from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.selector import Selector @@ -12,7 +11,7 @@ PARSEL_VERSION = version.parse(getattr(parsel, "__version__", "0.0")) PARSEL_18_PLUS = PARSEL_VERSION >= version.parse("1.8.0") -class SelectorTestCase(unittest.TestCase): +class TestSelector: def test_simple_selection(self): """Simple selector tests""" body = b"

" @@ -20,57 +19,46 @@ class SelectorTestCase(unittest.TestCase): sel = Selector(response) xl = sel.xpath("//input") - self.assertEqual(2, len(xl)) + assert len(xl) == 2 for x in xl: assert isinstance(x, Selector) - self.assertEqual( - sel.xpath("//input").getall(), [x.get() for x in sel.xpath("//input")] - ) - self.assertEqual( - [x.get() for x in sel.xpath("//input[@name='a']/@name")], ["a"] - ) - self.assertEqual( - [ - x.get() - for x in sel.xpath( - "number(concat(//input[@name='a']/@value, //input[@name='b']/@value))" - ) - ], - ["12.0"], - ) - self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(), ["xpathrules"]) - self.assertEqual( - [ - x.get() - for x in sel.xpath( - "concat(//input[@name='a']/@value, //input[@name='b']/@value)" - ) - ], - ["12"], - ) + assert sel.xpath("//input").getall() == [x.get() for x in sel.xpath("//input")] + assert [x.get() for x in sel.xpath("//input[@name='a']/@name")] == ["a"] + assert [ + x.get() + for x in sel.xpath( + "number(concat(//input[@name='a']/@value, //input[@name='b']/@value))" + ) + ] == ["12.0"] + assert sel.xpath("concat('xpath', 'rules')").getall() == ["xpathrules"] + assert [ + x.get() + for x in sel.xpath( + "concat(//input[@name='a']/@value, //input[@name='b']/@value)" + ) + ] == ["12"] def test_root_base_url(self): body = b'' url = "http://example.com" response = TextResponse(url=url, body=body, encoding="utf-8") sel = Selector(response) - self.assertEqual(url, sel.root.base) + assert url == sel.root.base def test_flavor_detection(self): text = b'

Hello

' sel = Selector(XmlResponse("http://example.com", body=text, encoding="utf-8")) - self.assertEqual(sel.type, "xml") - self.assertEqual( - sel.xpath("//div").getall(), - ['

Hello

'], - ) + assert sel.type == "xml" + assert sel.xpath("//div").getall() == [ + '

Hello

' + ] sel = Selector(HtmlResponse("http://example.com", body=text, encoding="utf-8")) - self.assertEqual(sel.type, "html") - self.assertEqual( - sel.xpath("//div").getall(), ['

Hello

'] - ) + assert sel.type == "html" + assert sel.xpath("//div").getall() == [ + '

Hello

' + ] def test_http_header_encoding_precedence(self): # '\xa3' = pound symbol in unicode @@ -92,7 +80,7 @@ class SelectorTestCase(unittest.TestCase): url="http://example.com", headers=headers, body=html_utf8 ) x = Selector(response) - self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ["\xa3"]) + assert x.xpath("//span[@id='blank']/text()").getall() == ["\xa3"] def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence @@ -116,7 +104,7 @@ class SelectorTestCase(unittest.TestCase): Selector(TextResponse(url="http://example.com", body=b""), text="") -class JMESPathTestCase(unittest.TestCase): +class TestJMESPath: @pytest.mark.skipif( not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" ) @@ -149,16 +137,13 @@ class JMESPathTestCase(unittest.TestCase): } """ resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") - self.assertEqual( - resp.jmespath("html").get(), - "
def
", + assert ( + resp.jmespath("html").get() + == "
def
" ) - self.assertEqual( - resp.jmespath("html").xpath("//div/a/text()").getall(), - ["a", "b", "d"], - ) - self.assertEqual(resp.jmespath("html").css("div > b").getall(), ["f"]) - self.assertEqual(resp.jmespath("content").jmespath("name.age").get(), "18") + assert resp.jmespath("html").xpath("//div/a/text()").getall() == ["a", "b", "d"] + assert resp.jmespath("html").css("div > b").getall() == ["f"] + assert resp.jmespath("content").jmespath("name.age").get() == "18" @pytest.mark.skipif( not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" @@ -194,15 +179,19 @@ class JMESPathTestCase(unittest.TestCase): """ resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") - self.assertEqual( - resp.xpath("//div/content/text()").jmespath("user[*].name").getall(), - ["A", "B", "C", "D"], - ) - self.assertEqual( - resp.xpath("//div/content").jmespath("user[*].name").getall(), - ["A", "B", "C", "D"], - ) - self.assertEqual(resp.xpath("//div/content").jmespath("total").get(), "4") + assert resp.xpath("//div/content/text()").jmespath("user[*].name").getall() == [ + "A", + "B", + "C", + "D", + ] + assert resp.xpath("//div/content").jmespath("user[*].name").getall() == [ + "A", + "B", + "C", + "D", + ] + assert resp.xpath("//div/content").jmespath("total").get() == "4" @pytest.mark.skipif( not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" @@ -238,30 +227,26 @@ class JMESPathTestCase(unittest.TestCase): """ resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") - self.assertEqual( - resp.xpath("//div/content/text()").jmespath("user[*].name").re(r"(\w+)"), - ["A", "B", "C", "D"], - ) - self.assertEqual( - resp.xpath("//div/content").jmespath("user[*].name").re(r"(\w+)"), - ["A", "B", "C", "D"], + assert resp.xpath("//div/content/text()").jmespath("user[*].name").re( + r"(\w+)" + ) == ["A", "B", "C", "D"] + assert resp.xpath("//div/content").jmespath("user[*].name").re(r"(\w+)") == [ + "A", + "B", + "C", + "D", + ] + + assert resp.xpath("//div/content").jmespath("unavailable").re(r"(\d+)") == [] + + assert ( + resp.xpath("//div/content").jmespath("unavailable").re_first(r"(\d+)") + is None ) - self.assertEqual( - resp.xpath("//div/content").jmespath("unavailable").re(r"(\d+)"), [] - ) - - self.assertEqual( - resp.xpath("//div/content").jmespath("unavailable").re_first(r"(\d+)"), - None, - ) - - self.assertEqual( - resp.xpath("//div/content") - .jmespath("user[*].age.to_string(@)") - .re(r"(\d+)"), - ["18", "32", "22", "25"], - ) + assert resp.xpath("//div/content").jmespath("user[*].age.to_string(@)").re( + r"(\d+)" + ) == ["18", "32", "22", "25"] @pytest.mark.skipif(PARSEL_18_PLUS, reason="parsel >= 1.8 supports jmespath") def test_jmespath_not_available(self) -> None: diff --git a/tests/test_signals.py b/tests/test_signals.py index a508eb41a..f5075fb60 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -20,7 +20,7 @@ class ItemSpider(Spider): return {"index": response.meta["index"]} -class AsyncSignalTestCase(unittest.TestCase): +class TestAsyncSignal(unittest.TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -43,6 +43,6 @@ class AsyncSignalTestCase(unittest.TestCase): crawler = get_crawler(ItemSpider) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 10) + assert len(self.items) == 10 for index in range(10): - self.assertIn({"index": index}, self.items) + assert {"index": index} in self.items diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py index d272101b8..a4f31096e 100644 --- a/tests/test_toplevel.py +++ b/tests/test_toplevel.py @@ -1,33 +1,31 @@ -from unittest import TestCase - import scrapy -class ToplevelTestCase(TestCase): +class TestToplevel: def test_version(self): - self.assertIs(type(scrapy.__version__), str) + assert isinstance(scrapy.__version__, str) def test_version_info(self): - self.assertIs(type(scrapy.version_info), tuple) + assert isinstance(scrapy.version_info, tuple) def test_request_shortcut(self): from scrapy.http import FormRequest, Request - self.assertIs(scrapy.Request, Request) - self.assertIs(scrapy.FormRequest, FormRequest) + assert scrapy.Request is Request + assert scrapy.FormRequest is FormRequest def test_spider_shortcut(self): from scrapy.spiders import Spider - self.assertIs(scrapy.Spider, Spider) + assert scrapy.Spider is Spider def test_selector_shortcut(self): from scrapy.selector import Selector - self.assertIs(scrapy.Selector, Selector) + assert scrapy.Selector is Selector def test_item_shortcut(self): from scrapy.item import Field, Item - self.assertIs(scrapy.Item, Item) - self.assertIs(scrapy.Field, Field) + assert scrapy.Item is Item + assert scrapy.Field is Field diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py index c695968d7..0e1e89e81 100644 --- a/tests/test_urlparse_monkeypatches.py +++ b/tests/test_urlparse_monkeypatches.py @@ -1,11 +1,10 @@ -import unittest from urllib.parse import urlparse -class UrlparseTestCase(unittest.TestCase): +class TestUrlparse: def test_s3_url(self): p = urlparse("s3://bucket/key/name?param=value") - self.assertEqual(p.scheme, "s3") - self.assertEqual(p.hostname, "bucket") - self.assertEqual(p.path, "/key/name") - self.assertEqual(p.query, "param=value") + assert p.scheme == "s3" + assert p.hostname == "bucket" + assert p.path == "/key/name" + assert p.query == "param=value" From 0bbfca6c1d1327a9919f2c0efc0c75aadb5b0033 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 10 Mar 2025 13:15:28 +0500 Subject: [PATCH 073/491] Better fix for test_non_pickable_object on Windows. --- tests/test_squeues.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 0b6ed8e11..21bbeece2 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -130,9 +130,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): ) as exc_info: q.push(sel) assert isinstance(exc_info.value.__context__, TypeError) - # This seems to help with https://github.com/scrapy/queuelib/issues/70. - # It will need to remain under a queuelib version check after that bug is fixed. - del exc_info + q.close() class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): From bee74fb753afa373ecdb1dbf272af9a108b14b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 11:56:44 +0100 Subject: [PATCH 074/491] Remove trailing whitespace --- .github/workflows/publish.yml | 2 +- .pre-commit-config.yaml | 4 + README.rst | 2 +- docs/_tests/quotes.html | 138 +++++++++++++++---------------- docs/_tests/quotes1.html | 138 +++++++++++++++---------------- docs/faq.rst | 2 +- docs/intro/install.rst | 4 +- docs/intro/tutorial.rst | 6 +- docs/news.rst | 8 +- docs/topics/api.rst | 2 +- docs/topics/architecture.rst | 4 +- docs/topics/exporters.rst | 2 +- docs/topics/extensions.rst | 16 ++-- docs/topics/feed-exports.rst | 10 +-- docs/topics/logging.rst | 10 +-- docs/topics/media-pipeline.rst | 14 ++-- docs/topics/practices.rst | 2 +- docs/topics/request-response.rst | 6 +- docs/topics/selectors.rst | 6 +- docs/topics/spiders.rst | 4 +- docs/topics/stats.rst | 2 +- sep/sep-004.rst | 2 +- sep/sep-007.rst | 2 +- sep/sep-008.rst | 4 +- sep/sep-014.rst | 4 +- sep/sep-018.rst | 4 +- 26 files changed, 201 insertions(+), 197 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8e01ffd88..d1589f4f7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.13" - - run: | + - run: | python -m pip install --upgrade build python -m build - name: Publish to PyPI diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 18402b908..0d1a76247 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,3 +11,7 @@ repos: - id: blacken-docs additional_dependencies: - black==24.10.0 +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace diff --git a/README.rst b/README.rst index 3f468953e..cf7c6043c 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ .. image:: https://scrapy.org/img/scrapylogo.png :target: https://scrapy.org/ - + ====== Scrapy ====== diff --git a/docs/_tests/quotes.html b/docs/_tests/quotes.html index f4002ecd1..d1cfd9020 100644 --- a/docs/_tests/quotes.html +++ b/docs/_tests/quotes.html @@ -16,13 +16,13 @@

- + Login - +

- +
@@ -34,16 +34,16 @@
Tags: - - + + change - + deep-thoughts - + thinking - + world - +
@@ -54,12 +54,12 @@
Tags: - - + + abilities - + choices - +
@@ -70,18 +70,18 @@
Tags: - - + + inspirational - + life - + live - + miracle - + miracles - +
@@ -92,16 +92,16 @@
Tags: - - + + aliteracy - + books - + classic - + humor - +
@@ -112,12 +112,12 @@
Tags: - - + + be-yourself - + inspirational - +
@@ -128,14 +128,14 @@
Tags: - - + + adulthood - + success - + value - +
@@ -146,12 +146,12 @@
Tags: - - + + life - + love - +
@@ -162,16 +162,16 @@
Tags: - - + + edison - + failure - + inspirational - + paraphrased - +
@@ -182,10 +182,10 @@
Tags: - - + + misattributed-eleanor-roosevelt - +
@@ -196,73 +196,73 @@
Tags: - - + + humor - + obvious - + simile - +
- +

Top Ten tags

- + love - + inspirational - + life - + humor - + books - + reading - + friendship - + friends - + truth - + simile - - + +
diff --git a/docs/_tests/quotes1.html b/docs/_tests/quotes1.html index f4002ecd1..d1cfd9020 100644 --- a/docs/_tests/quotes1.html +++ b/docs/_tests/quotes1.html @@ -16,13 +16,13 @@

- + Login - +

- +
@@ -34,16 +34,16 @@
Tags: - - + + change - + deep-thoughts - + thinking - + world - +
@@ -54,12 +54,12 @@
Tags: - - + + abilities - + choices - +
@@ -70,18 +70,18 @@
Tags: - - + + inspirational - + life - + live - + miracle - + miracles - +
@@ -92,16 +92,16 @@
Tags: - - + + aliteracy - + books - + classic - + humor - +
@@ -112,12 +112,12 @@
Tags: - - + + be-yourself - + inspirational - +
@@ -128,14 +128,14 @@
Tags: - - + + adulthood - + success - + value - +
@@ -146,12 +146,12 @@
Tags: - - + + life - + love - +
@@ -162,16 +162,16 @@
Tags: - - + + edison - + failure - + inspirational - + paraphrased - +
@@ -182,10 +182,10 @@
Tags: - - + + misattributed-eleanor-roosevelt - +
@@ -196,73 +196,73 @@
Tags: - - + + humor - + obvious - + simile - +
- +

Top Ten tags

- + love - + inspirational - + life - + humor - + books - + reading - + friendship - + friends - + truth - + simile - - + +
diff --git a/docs/faq.rst b/docs/faq.rst index f81ec3601..cef3e69f3 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -410,7 +410,7 @@ How can I make a blank request? ------------------------------- .. code-block:: python - + from scrapy import Request diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 82a0e18c5..488a66f36 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -111,7 +111,7 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: To install Scrapy on Windows using ``pip``: .. warning:: - This installation method requires “Microsoft Visual C++” for installing some + This installation method requires “Microsoft Visual C++” for installing some Scrapy dependencies, which demands significantly more disk space than Anaconda. #. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer. @@ -123,7 +123,7 @@ To install Scrapy on Windows using ``pip``: #. Check the installation details and make sure following packages are selected as optional components: * **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) ) - + * **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0)) #. Install the Visual Studio Build Tools. diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 6e6caebf1..5041b49ea 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -292,7 +292,7 @@ As an alternative, you could've written: >>> response.css("title::text")[0].get() 'Quotes to Scrape' -Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will +Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will raise an :exc:`IndexError` exception if there are no results: .. code-block:: pycon @@ -302,8 +302,8 @@ raise an :exc:`IndexError` exception if there are no results: ... IndexError: list index out of range -You might want to use ``.get()`` directly on the -:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` +You might want to use ``.get()`` directly on the +:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` if there are no results: .. code-block:: pycon diff --git a/docs/news.rst b/docs/news.rst index 8230c3aef..9a68f8852 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -934,10 +934,10 @@ Modified requirements Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` - instead of ``False``. In earlier Scrapy versions empty files were created - even when this setting was ``False`` (which was a bug that is now fixed), - so the new default should keep the old behavior. (:issue:`872`, +- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` + instead of ``False``. In earlier Scrapy versions empty files were created + even when this setting was ``False`` (which was a bug that is now fixed), + so the new default should keep the old behavior. (:issue:`872`, :issue:`5847`) Deprecation removals diff --git a/docs/topics/api.rst b/docs/topics/api.rst index f7cffb61b..edc625be8 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -88,7 +88,7 @@ how you :ref:`configure the downloader middlewares The execution engine, which coordinates the core crawling logic between the scheduler, downloader and spiders. - Some extension may want to access the Scrapy engine, to inspect or + Some extension may want to access the Scrapy engine, to inspect or modify the downloader and scheduler behaviour, although this is an advanced use and this API is not yet stable. diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 0370dc538..4e53b6e3d 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -87,8 +87,8 @@ of the system, and triggering events when certain actions occur. See the Scheduler --------- -The :ref:`scheduler ` receives requests from the engine and -enqueues them for feeding them later (also to the engine) when the engine +The :ref:`scheduler ` receives requests from the engine and +enqueues them for feeding them later (also to the engine) when the engine requests them. .. _component-downloader: diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 7a85c099b..5c078568b 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -224,7 +224,7 @@ BaseItemExporter .. [1] Not all exporters respect the specified field order. .. [2] When using :ref:`item objects ` that do not expose all their possible fields, exporters that do not support exporting - a different subset of fields per item will only export the fields + a different subset of fields per item will only export the fields found in the first item exported. .. attribute:: export_empty_fields diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index c47a3226a..23bbcfcb5 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -256,14 +256,14 @@ Spider state extension Manages spider state data by loading it before a crawl and saving it after. Give a value to the :setting:`JOBDIR` setting to enable this extension. -When enabled, this extension manages the :attr:`~scrapy.Spider.state` +When enabled, this extension manages the :attr:`~scrapy.Spider.state` attribute of your :class:`~scrapy.Spider` instance: - -- When your spider closes (:signal:`spider_closed`), the contents of its - :attr:`~scrapy.Spider.state` attribute are serialized into a file named + +- When your spider closes (:signal:`spider_closed`), the contents of its + :attr:`~scrapy.Spider.state` attribute are serialized into a file named ``spider.state`` in the :setting:`JOBDIR` folder. -- When your spider opens (:signal:`spider_opened`), if a previously-generated - ``spider.state`` file exists in the :setting:`JOBDIR` folder, it is loaded +- When your spider opens (:signal:`spider_opened`), if a previously-generated + ``spider.state`` file exists in the :setting:`JOBDIR` folder, it is loaded into the :attr:`~scrapy.Spider.state` attribute. @@ -291,8 +291,8 @@ settings: .. note:: - When a certain closing condition is met, requests which are - currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS` + When a certain closing condition is met, requests which are + currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS` requests) are still processed. .. setting:: CLOSESPIDER_TIMEOUT diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 07a3f3678..7f401f0c7 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -180,7 +180,7 @@ FTP supports two different connection modes: `active or passive mode by default. To use the active connection mode instead, set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. -The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. .. caution:: The value ``True`` in ``overwrite`` will cause you to lose the @@ -222,7 +222,7 @@ feeds using these settings: - :setting:`AWS_ENDPOINT_URL` - :setting:`AWS_REGION_NAME` -The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. .. caution:: The value ``True`` in ``overwrite`` will cause you to lose the @@ -255,7 +255,7 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following - :setting:`FEED_STORAGE_GCS_ACL` - :setting:`GCS_PROJECT_ID` -The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. .. caution:: The value ``True`` in ``overwrite`` will cause you to lose the @@ -587,8 +587,8 @@ FEED_STORE_EMPTY Default: ``True`` Whether to export empty feeds (i.e. feeds with no items). -If ``False``, and there are no items to export, no new files are created and -existing files are not modified, even if the :ref:`overwrite feed option +If ``False``, and there are no items to export, no new files are created and +existing files are not modified, even if the :ref:`overwrite feed option ` is enabled. .. setting:: FEED_STORAGES diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index fe1c4d162..a398d6c83 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -266,9 +266,9 @@ e.g. in the spider's ``__init__`` method: If you run this spider again then INFO messages from ``scrapy.spidermiddlewares.httperror`` logger will be gone. -You can also filter log records by :class:`~logging.LogRecord` data. For +You can also filter log records by :class:`~logging.LogRecord` data. For example, you can filter log records by message content using a substring or -a regular expression. Create a :class:`logging.Filter` subclass +a regular expression. Create a :class:`logging.Filter` subclass and equip it with a regular expression pattern to filter out unwanted messages: @@ -284,8 +284,8 @@ filter out unwanted messages: if match: return False -A project-level filter may be attached to the root -handler created by Scrapy, this is a wieldy way to +A project-level filter may be attached to the root +handler created by Scrapy, this is a wieldy way to filter all loggers in different parts of the project (middlewares, spider, etc.): @@ -301,7 +301,7 @@ filter all loggers in different parts of the project for handler in logging.root.handlers: handler.addFilter(ContentFilter()) -Alternatively, you may choose a specific logger +Alternatively, you may choose a specific logger and hide it without affecting other loggers: .. code-block:: python diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index f086a943e..cc1fe8703 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -414,7 +414,7 @@ class name. E.g. given pipeline class called MyPipeline you can set setting key: and pipeline class MyPipeline will have expiration time set to 180. -The last modified time from the file is used to determine the age of the file in days, +The last modified time from the file is used to determine the age of the file in days, which is then compared to the set expiration time to determine if the file is expired. .. _topics-images-thumbnails: @@ -519,7 +519,7 @@ See here the methods that you can override in your custom Files Pipeline: In addition to ``response``, this method receives the original :class:`request `, - :class:`info ` and + :class:`info ` and :class:`item ` You can override this method to customize the download path of each file. @@ -541,9 +541,9 @@ See here the methods that you can override in your custom Files Pipeline: def file_path(self, request, response=None, info=None, *, item=None): return "files/" + PurePosixPath(urlparse_cached(request).path).name - Similarly, you can use the ``item`` to determine the file path based on some item + Similarly, you can use the ``item`` to determine the file path based on some item property. - + By default the :meth:`file_path` method returns ``full/.``. @@ -677,7 +677,7 @@ See here the methods that you can override in your custom Images Pipeline: In addition to ``response``, this method receives the original :class:`request `, - :class:`info ` and + :class:`info ` and :class:`item ` You can override this method to customize the download path of each file. @@ -699,9 +699,9 @@ See here the methods that you can override in your custom Images Pipeline: def file_path(self, request, response=None, info=None, *, item=None): return "files/" + PurePosixPath(urlparse_cached(request).path).name - Similarly, you can use the ``item`` to determine the file path based on some item + Similarly, you can use the ``item`` to determine the file path based on some item property. - + By default the :meth:`file_path` method returns ``full/.``. diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 5f6798601..db91cd073 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -309,7 +309,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites: services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a super proxy that you can attach your own proxies to. * use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy - plugin `__ and additional + plugin `__ and additional features, like `AI web scraping `__ If you are still unable to prevent your bot getting banned, consider contacting diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index b187f3aaf..55e8518a5 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -1309,7 +1309,7 @@ JsonResponse objects .. class:: JsonResponse(url[, ...]) - The :class:`JsonResponse` class is a subclass of :class:`TextResponse` - that is used when the response has a `JSON MIME type - `_ in its `Content-Type` + The :class:`JsonResponse` class is a subclass of :class:`TextResponse` + that is used when the response has a `JSON MIME type + `_ in its `Content-Type` header. diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index b95e6eab3..dbef07b73 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -559,7 +559,7 @@ For example, suppose you want to extract all ``

`` elements inside ``

`` elements. First, you would get all ``
`` elements: .. code-block:: pycon - + >>> divs = response.xpath("//div") At first, you may be tempted to use the following approach, which is wrong, as @@ -610,7 +610,7 @@ As it turns out, Scrapy selectors allow you to chain selectors, so most of the t you can just select by class using CSS and then switch to XPath when needed: .. code-block:: pycon - + >>> from scrapy import Selector >>> sel = Selector( ... text='
' @@ -1032,7 +1032,7 @@ whereas the CSS lookup is translated into XPath and thus runs more efficiently, so performance-wise its uses are limited to situations that are not easily described with CSS selectors. -Parsel also simplifies adding your own XPath extensions with +Parsel also simplifies adding your own XPath extensions with :func:`~parsel.xpathfuncs.set_xpathfunc`. .. _topics-selectors-ref: diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index e1b1c5ad6..0a67240d6 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -379,8 +379,8 @@ The above example can also be written as follows: def start_requests(self): yield scrapy.Request(f"http://www.example.com/categories/{self.category}") -If you are :ref:`running Scrapy from a script `, you can -specify spider arguments when calling +If you are :ref:`running Scrapy from a script `, you can +specify spider arguments when calling :class:`CrawlerProcess.crawl ` or :class:`CrawlerRunner.crawl `: diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index be8ecb7a5..9572a3785 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -86,7 +86,7 @@ Available Stats Collectors Besides the basic :class:`StatsCollector` there are other Stats Collectors available in Scrapy which extend the basic Stats Collector. You can select which Stats Collector to use through the :setting:`STATS_CLASS` setting. The -default Stats Collector used is the :class:`MemoryStatsCollector`. +default Stats Collector used is the :class:`MemoryStatsCollector`. .. currentmodule:: scrapy.statscollectors diff --git a/sep/sep-004.rst b/sep/sep-004.rst index b1cef2600..7a4ebe886 100644 --- a/sep/sep-004.rst +++ b/sep/sep-004.rst @@ -11,7 +11,7 @@ SEP-004: Library API ==================== .. note:: the library API has been implemented, but slightly different from proposed in this SEP. You can run a Scrapy crawler inside a Twisted - reactor, but not outside it. + reactor, but not outside it. Introduction ============ diff --git a/sep/sep-007.rst b/sep/sep-007.rst index 0ca2036ce..73ce0d338 100644 --- a/sep/sep-007.rst +++ b/sep/sep-007.rst @@ -96,7 +96,7 @@ specified, else utf-8 is used) and returns a new unicode object. E.g: ``clean_spaces`` ---------------- - + Converts multispaces into single spaces for the given string. E.g: :: diff --git a/sep/sep-008.rst b/sep/sep-008.rst index be5987e39..1c38b1c40 100644 --- a/sep/sep-008.rst +++ b/sep/sep-008.rst @@ -73,8 +73,8 @@ Alternative Public API Proposal - ``ItemLoader.get_stored_values()`` or ``ItemLoader.get_values()`` *(returns the ``ItemLoader values)* - ``ItemLoader.get_output_value()`` -- ``ItemLoader.get_input_processor()`` or ``ItemLoader.get_in_processor()`` *(short version)* -- ``ItemLoader.get_output_processor()`` or ``ItemLoader.get_out_processor()`` *(short version)* +- ``ItemLoader.get_input_processor()`` or ``ItemLoader.get_in_processor()`` *(short version)* +- ``ItemLoader.get_output_processor()`` or ``ItemLoader.get_out_processor()`` *(short version)* - ``ItemLoader.context`` diff --git a/sep/sep-014.rst b/sep/sep-014.rst index e03a2b0f6..0a2e6b51e 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -21,7 +21,7 @@ Current flaws and inconsistencies 2. Link extractors are inflexible and hard to maintain, link processing/filtering is tightly coupled. (e.g. canonicalize) 3. Isn't possible to crawl an url directly from command line because the Spider - does not know which callback use. + does not know which callback use. These flaws will be corrected by the changes proposed in this SEP. @@ -55,7 +55,7 @@ Request Extractors Request Extractors takes response object and determines which requests follow. This is an enhancement to ``LinkExtractors`` which returns urls (links), -Request Extractors return Request objects. +Request Extractors return Request objects. Request Processors ------------------ diff --git a/sep/sep-018.rst b/sep/sep-018.rst index 13ab501ed..e6d601fe1 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -200,7 +200,7 @@ the same spider: # extract item from response return item -The Spider Middleware that implements spider code +The Spider Middleware that implements spider code ================================================= There's gonna be one middleware that will take care of calling the proper @@ -625,7 +625,7 @@ Resolved: not the original one (think of redirections), but it does carry the ``meta`` of the original one. The original one may not be available anymore (in memory) if we're using a persistent scheduler., but in that case it would be - the deserialized request from the persistent scheduler queue. + the deserialized request from the persistent scheduler queue. - No - this would make implementation more complex and we're not sure it's really needed From 9b7db1a068895254aae1618fb684baf9cb0c2784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 12:43:50 +0100 Subject: [PATCH 075/491] Move some reference docs of Request to the code (#6721) --- docs/topics/request-response.rst | 75 +++++++++++++------------------- scrapy/http/request/__init__.py | 49 +++++++++++++++++++++ 2 files changed, 80 insertions(+), 44 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 55e8518a5..0375e0ff1 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -31,23 +31,12 @@ Request objects If the URL is invalid, a :exc:`ValueError` exception is raised. :type url: str - :param callback: the function that will be called with the response of this - request (once it's downloaded) as its first parameter. + :param callback: sets :attr:`callback`, defaults to ``None``. - In addition to a function, the following values are supported: - - - ``None`` (default), which indicates that the spider's - :meth:`~scrapy.Spider.parse` method must be used. - - - :func:`~scrapy.http.request.NO_CALLBACK` - - For more information, see - :ref:`topics-request-response-ref-request-callback-arguments`. - - .. note:: If exceptions are raised during processing, ``errback`` is - called instead. - - :type callback: collections.abc.Callable + .. versionchanged:: 2.0 + The *callback* parameter is no longer required when the *errback* + parameter is specified. + :type callback: Callable[Concatenate[Response, ...], Any] | None :param method: the HTTP method of this request. Defaults to ``'GET'``. :type method: str @@ -144,23 +133,15 @@ Request objects Negative values are allowed in order to indicate relatively low-priority. :type priority: int - :param dont_filter: indicates that this request should not be filtered by - the scheduler or some middlewares. This is used when you want to perform - an identical request multiple times, to ignore the duplicates filter. - Use it with care, or you will get into crawling loops. Default to ``False``. + :param dont_filter: sets :attr:`dont_filter`, defaults to ``False``. :type dont_filter: bool - :param errback: a function that will be called if any exception was - raised while processing the request. This includes pages that failed - with 404 HTTP errors and such. It receives a - :exc:`~twisted.python.failure.Failure` as first parameter. - For more information, - see :ref:`topics-request-response-ref-errbacks` below. + :param errback: sets :attr:`errback`, defaults to ``None``. - .. versionchanged:: 2.0 - The *callback* parameter is no longer required when the *errback* - parameter is specified. - :type errback: collections.abc.Callable + .. versionchanged:: 2.0 + The *callback* parameter is no longer required when the *errback* + parameter is specified. + :type errback: Callable[[Failure], Any] | None :param flags: Flags sent to the request, can be used for logging or similar purposes. :type flags: list @@ -194,6 +175,25 @@ Request objects This attribute is read-only. To change the body of a Request use :meth:`replace`. + .. autoattribute:: callback + + .. autoattribute:: errback + + .. attribute:: Request.cb_kwargs + + A dictionary that contains arbitrary metadata for this request. Its contents + will be passed to the Request's callback as keyword arguments. It is empty + for new Requests, which means by default callbacks only get a + :class:`~scrapy.http.Response` object as argument. + + This dict is :doc:`shallow copied ` when the request is + cloned using the ``copy()`` or ``replace()`` methods, and can also be + accessed, in your spider, from the ``response.cb_kwargs`` attribute. + + In case of a failure to process the request, this dict can be accessed as + ``failure.request.cb_kwargs`` in the request's errback. For more information, + see :ref:`errback-cb_kwargs`. + .. attribute:: Request.meta :value: {} @@ -237,20 +237,7 @@ Request objects Also mind that the :meth:`copy` and :meth:`replace` request methods :doc:`shallow-copy ` request metadata. - .. attribute:: Request.cb_kwargs - - A dictionary that contains arbitrary metadata for this request. Its contents - will be passed to the Request's callback as keyword arguments. It is empty - for new Requests, which means by default callbacks only get a - :class:`~scrapy.http.Response` object as argument. - - This dict is :doc:`shallow copied ` when the request is - cloned using the ``copy()`` or ``replace()`` methods, and can also be - accessed, in your spider, from the ``response.cb_kwargs`` attribute. - - In case of a failure to process the request, this dict can be accessed as - ``failure.request.cb_kwargs`` in the request's errback. For more information, - see :ref:`errback-cb_kwargs`. + .. autoattribute:: dont_filter .. autoattribute:: Request.attributes diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index e24f6874d..6d3b7a926 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -138,11 +138,60 @@ class Request(object_ref): ) if not (callable(errback) or errback is None): raise TypeError(f"errback must be a callable, got {type(errback).__name__}") + + #: :class:`~collections.abc.Callable` to parse the + #: :class:`~scrapy.http.Response` to this request once received. + #: + #: The callable must expect the response as its first parameter, and + #: support any additional keyword arguments set through + #: :attr:`cb_kwargs`. + #: + #: In addition to an arbitrary callable, the following values are also + #: supported: + #: + #: - ``None`` (default), which indicates that the + #: :meth:`~scrapy.Spider.parse` method of the spider must be used. + #: + #: - :func:`~scrapy.http.request.NO_CALLBACK`. + #: + #: If an unhandled exception is raised during request or response + #: processing, i.e. by a :ref:`spider middleware + #: `, :ref:`downloader middleware + #: ` or download handler + #: (:setting:`DOWNLOAD_HANDLERS`), :attr:`errback` is called instead. + #: + #: .. tip:: + #: :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware` + #: raises exceptions for non-2xx responses by default, sending them + #: to the :attr:`errback` instead. + #: + #: .. seealso:: + #: :ref:`topics-request-response-ref-request-callback-arguments` self.callback: CallbackT | None = callback + + #: :class:`~collections.abc.Callable` to handle exceptions raised + #: during request or response processing. + #: + #: The callable must expect a :exc:`~twisted.python.failure.Failure` as + #: its first parameter. + #: + #: .. seealso:: :ref:`topics-request-response-ref-errbacks` self.errback: Callable[[Failure], Any] | None = errback self.cookies: CookiesT = cookies or {} self.headers: Headers = Headers(headers or {}, encoding=encoding) + + #: Whether this request may be filtered out by :ref:`components + #: ` that support filtering out requests (``False``, + #: default), or those components should not filter out this request + #: (``True``). + #: + #: This attribute is commonly set to ``True`` to prevent duplicate + #: requests from being filtered out. + #: + #: When defining the start URLs of a spider through + #: :attr:`~scrapy.Spider.start_urls`, this attribute is enabled by + #: default. See :meth:`~scrapy.Spider.start_requests`. self.dont_filter: bool = dont_filter self._meta: dict[str, Any] | None = dict(meta) if meta else None From 26ecc93228bc3dbb9d62daa419cb344ba6f88caa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 14:12:11 +0100 Subject: [PATCH 076/491] Run CI only on the main branch, on release branches and on PRs (#6720) --- .github/workflows/checks.yml | 7 ++++++- .github/workflows/tests-macos.yml | 7 ++++++- .github/workflows/tests-ubuntu.yml | 7 ++++++- .github/workflows/tests-windows.yml | 7 ++++++- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a064bf5b2..312af3b2e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,5 +1,10 @@ name: Checks -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index c28a99982..ce0e1a6c2 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,5 +1,10 @@ name: macOS -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 6c7842217..f74575ee1 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,5 +1,10 @@ name: Ubuntu -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 45e4ca157..21d621240 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,5 +1,10 @@ name: Windows -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} From 5a0690c89d718b33bd63c1cd724c50c9ceb809e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 14:52:48 +0100 Subject: [PATCH 077/491] Remove or post-pone the use of itemadapter.is_item, as a potentially expensive call (#6719) --- docs/faq.rst | 14 +++++++------ docs/topics/items.rst | 5 ++--- scrapy/commands/parse.py | 8 ++++---- scrapy/core/engine.py | 9 +-------- scrapy/core/scraper.py | 10 +--------- scrapy/exporters.py | 4 ++-- .../project/module/middlewares.py.tmpl | 2 +- scrapy/utils/serialize.py | 4 ++-- tests/test_crawl.py | 20 ++++++------------- 9 files changed, 27 insertions(+), 49 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index cef3e69f3..da255f29e 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -361,16 +361,18 @@ method for this purpose. For example: from copy import deepcopy - from itemadapter import is_item, ItemAdapter + from itemadapter import ItemAdapter + from scrapy import Request class MultiplyItemsMiddleware: def process_spider_output(self, response, result, spider): - for item in result: - if is_item(item): - adapter = ItemAdapter(item) - for _ in range(adapter["multiply_by"]): - yield deepcopy(item) + for item_or_request in result: + if isinstance(item_or_request, Request): + continue + adapter = ItemAdapter(item) + for _ in range(adapter["multiply_by"]): + yield deepcopy(item) Does Scrapy support IPv6 addresses? ----------------------------------- diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 7cc476863..0365c95b3 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -384,9 +384,8 @@ Supporting All Item Types In code that receives an item, such as methods of :ref:`item pipelines ` or :ref:`spider middlewares `, it is a good practice to use the -:class:`~itemadapter.ItemAdapter` class and the -:func:`~itemadapter.is_item` function to write code that works for -any supported item type. +:class:`~itemadapter.ItemAdapter` class to write code that works for any +supported item type. Other classes related to items ============================== diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 61aea3ee4..c6ed20b3b 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -6,7 +6,7 @@ import json import logging from typing import TYPE_CHECKING, Any, TypeVar, overload -from itemadapter import ItemAdapter, is_item +from itemadapter import ItemAdapter from twisted.internet.defer import Deferred, maybeDeferred from w3lib.url import is_url @@ -211,10 +211,10 @@ class Command(BaseRunSpiderCommand): ) -> tuple[list[Any], list[Request], argparse.Namespace, int, Spider, CallbackT]: items, requests = [], [] for x in spider_output: - if is_item(x): - items.append(x) - elif isinstance(x, Request): + if isinstance(x, Request): requests.append(x) + else: + items.append(x) return items, requests, opts, depth, spider, callback def run_callback( diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 61f444e31..b7a73700b 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -11,7 +11,6 @@ import logging from time import time from typing import TYPE_CHECKING, Any, TypeVar, cast -from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall from twisted.python.failure import Failure @@ -194,14 +193,8 @@ class ExecutionEngine: else: if isinstance(request_or_item, Request): self.crawl(request_or_item) - elif is_item(request_or_item): - self.scraper.start_itemproc(request_or_item, response=None) else: - logger.error( - f"Got {request_or_item!r} among start requests. Only " - f"requests and items are supported. It will be " - f"ignored." - ) + self.scraper.start_itemproc(request_or_item, response=None) if self.spider_is_idle() and self.slot.close_if_idle: self._spider_idle() diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 03301717d..b664b61f6 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -8,7 +8,6 @@ from collections import deque from collections.abc import AsyncIterable, Iterator from typing import TYPE_CHECKING, Any, TypeVar, Union, cast -from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure @@ -298,17 +297,10 @@ class Scraper: if isinstance(output, Request): assert self.crawler.engine is not None # typing self.crawler.engine.crawl(request=output) - elif is_item(output): - return self.start_itemproc(output, response=response) elif output is None: pass else: - typename = type(output).__name__ - logger.error( - "Spider must return request, item, or None, got %(typename)r in %(request)s", - {"request": request, "typename": typename}, - extra={"spider": spider}, - ) + return self.start_itemproc(output, response=response) return None def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[Any]: diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 46c6aa3fa..0a641752e 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -356,12 +356,12 @@ class PythonItemExporter(BaseItemExporter): def _serialize_value(self, value: Any) -> Any: if isinstance(value, Item): return self.export_item(value) + if isinstance(value, (str, bytes)): + return to_unicode(value, encoding=self.encoding) if is_item(value): return dict(self._serialize_item(value)) if is_listlike(value): return [self._serialize_value(v) for v in value] - if isinstance(value, (str, bytes)): - return to_unicode(value, encoding=self.encoding) return value def _serialize_item(self, item: Any) -> Iterable[tuple[str | bytes, Any]]: diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 8c9a86dce..dcb2d63de 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -6,7 +6,7 @@ from scrapy import signals # useful for handling different item types with a single interface -from itemadapter import is_item, ItemAdapter +from itemadapter import ItemAdapter class ${ProjectName}SpiderMiddleware: diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 308e351c6..bcfae0c00 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -28,12 +28,12 @@ class ScrapyJSONEncoder(json.JSONEncoder): return str(o) if isinstance(o, defer.Deferred): return str(o) - if is_item(o): - return ItemAdapter(o).asdict() if isinstance(o, Request): return f"<{type(o).__name__} {o.method} {o.url}>" if isinstance(o, Response): return f"<{type(o).__name__} {o.status} {o.url}>" + if is_item(o): + return ItemAdapter(o).asdict() return super().default(o) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 5766f9313..6f4045fc8 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,6 +1,5 @@ import json import logging -import re import unittest from ipaddress import IPv4Address from socket import gethostbyname @@ -195,23 +194,16 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_requests_unsupported_output(self): + """Anything that is not a request is assumed to be an item, avoiding a + potentially expensive call to itemadapter.is_item, and letting instead + things fail when ItemAdapter is actually used on the corresponding + non-item object.""" + with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(StartRequestsGoodAndBadOutput) yield crawler.crawl(mockserver=self.mockserver) - assert len(log.records) == 2 - assert log.records[0].msg == ( - "Got 'data:,b' among start requests. Only requests and items " - "are supported. It will be ignored." - ) - assert re.match( - ( - r"^Got among start " - r"requests\. Only requests and items are supported\. It " - r"will be ignored\.$" - ), - log.records[1].msg, - ) + assert len(log.records) == 0 @defer.inlineCallbacks def test_start_requests_laziness(self): From ba28d96d3ef2488e9c5c86d4a5c28a1aac269a2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 14:53:04 +0100 Subject: [PATCH 078/491] Centralize from_crawler docs (and somewhat related changes) (#6723) --- docs/topics/addons.rst | 17 +----- docs/topics/api.rst | 9 +-- docs/topics/components.rst | 80 +++++++++++++++++++++++++-- docs/topics/downloader-middleware.rst | 19 +------ docs/topics/email.rst | 14 +---- docs/topics/extensions.rst | 80 ++++++--------------------- docs/topics/item-pipeline.rst | 18 ++---- docs/topics/request-response.rst | 24 +------- docs/topics/settings.rst | 39 ++++--------- docs/topics/spider-middleware.rst | 69 +++++++++-------------- 10 files changed, 150 insertions(+), 219 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 8ec7b0295..17e3c177a 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -32,7 +32,8 @@ This is an example where two add-ons are enabled in a project's Writing your own add-ons ======================== -Add-ons are Python classes that include one or both of the following methods: +Add-ons are :ref:`components ` that include one or both of +the following methods: .. method:: update_settings(settings) @@ -54,20 +55,6 @@ Add-ons are Python classes that include one or both of the following methods: :param settings: The settings object storing Scrapy/component configuration :type settings: :class:`~scrapy.settings.BaseSettings` -They can also have the following method: - -.. classmethod:: from_crawler(cls, crawler) - :noindex: - - If present, this class method is called to create an add-on instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the add-on. The crawler object provides access to all Scrapy core - components like settings and signals; it is a way for the add-on to access - them and hook its functionality into Scrapy. - - :param crawler: The crawler that uses this add-on - :type crawler: :class:`~scrapy.crawler.Crawler` - The settings set by the add-on should use the ``addon`` priority (see :ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):: diff --git a/docs/topics/api.rst b/docs/topics/api.rst index edc625be8..5a00fd570 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -12,10 +12,11 @@ extensions and middlewares. Crawler API =========== -The main entry point to Scrapy API is the :class:`~scrapy.crawler.Crawler` -object, passed to extensions through the ``from_crawler`` class method. This -object provides access to all Scrapy core components, and it's the only way for -extensions to access them and hook their functionality into Scrapy. +The main entry point to the Scrapy API is the :class:`~scrapy.crawler.Crawler` +object, which :ref:`components ` can :ref:`get for +initialization `. It provides access to all Scrapy core +components, and it is the only way for components to access them and hook their +functionality into Scrapy. .. module:: scrapy.crawler :synopsis: The Scrapy crawler diff --git a/docs/topics/components.rst b/docs/topics/components.rst index d34b3884b..3a7644379 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -9,6 +9,8 @@ A Scrapy component is any class whose objects are built using That includes the classes that you may assign to the following settings: +- :setting:`ADDONS` + - :setting:`DNS_RESOLVER` - :setting:`DOWNLOAD_HANDLERS` @@ -41,10 +43,80 @@ Third-party Scrapy components may also let you define additional Scrapy components, usually configurable through :ref:`settings `, to modify their behavior. +.. _from-crawler: + +Initializing from the crawler +============================= + +Any Scrapy component may optionally define the following class method: + +.. classmethod:: from_crawler(cls, crawler: scrapy.crawler.Crawler, *args, **kwargs) + + Return an instance of the component based on *crawler*. + + *args* and *kwargs* are component-specific arguments that some components + receive. However, most components do not get any arguments, and instead + :ref:`use settings `. + + If a component class defines this method, this class method is called to + create any instance of the component. + + The *crawler* object provides access to all Scrapy core components like + :ref:`settings ` and :ref:`signals `, + allowing the component to access them and hook its functionality into + Scrapy. + +.. _component-settings: + +Settings +======== + +Components can be configured through :ref:`settings `. + +Components can read any setting from the +:attr:`~scrapy.crawler.Crawler.settings` attribute of the +:class:`~scrapy.crawler.Crawler` object they can :ref:`get for initialization +`. That includes both built-in and custom settings. + +For example: + +.. code-block:: python + + class MyExtension: + @classmethod + def from_crawler(cls, crawler): + settings = crawler.settings + return cls(settings.getbool("LOG_ENABLED")) + + def __init__(self, log_is_enabled=False): + if log_is_enabled: + print("log is enabled!") + +Components do not need to declare their custom settings programmatically. +However, they should document them, so that users know they exist and how to +use them. + +It is a good practice to prefix custom settings with the name of the component, +to avoid collisions with custom settings of other existing (or future) +components. For example, an extension called ``WarcCaching`` could prefix its +custom settings with ``WARC_CACHING_``. + +Another good practice, mainly for components meant for :ref:`component priority +dictionaries `, is to provide a boolean setting +called ``_ENABLED`` (e.g. ``WARC_CACHING_ENABLED``) to allow toggling +that component on and off without changing the component priority dictionary +setting. You can usually check the value of such a setting during +initialization, and if ``False``, raise +:exc:`~scrapy.exceptions.NotConfigured`. + +When choosing a name for a custom setting, it is also a good idea to have a +look at the names of :ref:`built-in settings `, to try to +maintain consistency with them. + .. _enforce-component-requirements: -Enforcing component requirements -================================ +Enforcing requirements +====================== Sometimes, your components may only be intended to work under certain conditions. For example, they may require a minimum version of Scrapy to work as @@ -58,8 +130,8 @@ In the case of :ref:`downloader middlewares `, :ref:`extensions `, :ref:`item pipelines `, and :ref:`spider middlewares `, you should raise -:exc:`scrapy.exceptions.NotConfigured`, passing a description of the issue as a -parameter to the exception so that it is printed in the logs, for the user to +:exc:`~scrapy.exceptions.NotConfigured`, passing a description of the issue as +a parameter to the exception so that it is printed in the logs, for the user to see. For other components, feel free to raise whatever other exception feels right to you; for example, :exc:`RuntimeError` would make sense for a Scrapy version mismatch, while :exc:`ValueError` may be better if the issue is the diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index ab7e6a0ec..60b6aab78 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -61,12 +61,8 @@ particular setting. See each middleware documentation for more info. Writing your own downloader middleware ====================================== -Each downloader middleware is a Python class that defines one or more of the -methods defined below. - -The main entry point is the ``from_crawler`` class method, which receives a -:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` -object gives you access, for example, to the :ref:`settings `. +Each downloader middleware is a :ref:`component ` that +defines one or more of these methods: .. module:: scrapy.downloadermiddlewares @@ -167,17 +163,6 @@ object gives you access, for example, to the :ref:`settings `. :param spider: the spider for which this request is intended :type spider: :class:`~scrapy.Spider` object - .. method:: from_crawler(cls, crawler) - - If present, this classmethod is called to create a middleware instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the middleware. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for middleware to - access them and hook its functionality into Scrapy. - - :param crawler: crawler that uses this middleware - :type crawler: :class:`~scrapy.crawler.Crawler` object - .. _topics-downloader-middleware-ref: Built-in downloader middleware reference diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 8f7a2357a..1d7bad787 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -50,9 +50,9 @@ And here is how to use it to send an e-mail (without attachments): MailSender class reference ========================== -MailSender is the preferred class to use for sending emails from Scrapy, as it -uses :doc:`Twisted non-blocking IO `, like the -rest of the framework. +The MailSender :ref:`components ` is the preferred class to +use for sending emails from Scrapy, as it uses :doc:`Twisted non-blocking IO +`, like the rest of the framework. .. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None) @@ -81,14 +81,6 @@ rest of the framework. :param smtpssl: enforce using a secure SSL connection :type smtpssl: bool - .. classmethod:: from_crawler(crawler) - - Instantiate using a :class:`scrapy.Crawler` instance, which will - respect :ref:`these Scrapy settings `. - - :param crawler: the crawler - :type settings: :class:`scrapy.Crawler` object - .. method:: send(to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None) Send email to the given recipients. diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 23bbcfcb5..e1e3dd6b4 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -4,34 +4,21 @@ Extensions ========== -The extensions framework provides a mechanism for inserting your own -custom functionality into Scrapy. +Extensions are :ref:`components ` that allow inserting your +own custom functionality into Scrapy. -Extensions are just regular classes. +Unlike other components, extensions do not have a specific role in Scrapy. They +are “wildcard” components that can be used for anything that does not fit the +role of any other type of component. -Extension settings -================== +Loading and activating extensions +================================= -Extensions use the :ref:`Scrapy settings ` to manage their -settings, just like any other Scrapy code. +Extensions are loaded at startup by creating a single instance of the extension +class per spider being run. -It is customary for extensions to prefix their settings with their own name, to -avoid collision with existing (and future) extensions. For example, a -hypothetical extension to handle `Google Sitemaps`_ would use settings like -``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on. - -.. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps - -Loading & activating extensions -=============================== - -Extensions are loaded and activated at startup by instantiating a single -instance of the extension class per spider being run. All the extension -initialization code must be performed in the class ``__init__`` method. - -To make an extension available, add it to the :setting:`EXTENSIONS` setting in -your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented -by a string: the full Python path to the extension's class name. For example: +To enable an extension, add it to the :setting:`EXTENSIONS` setting. For +example: .. code-block:: python @@ -40,55 +27,24 @@ by a string: the full Python path to the extension's class name. For example: "scrapy.extensions.telnet.TelnetConsole": 500, } - -As you can see, the :setting:`EXTENSIONS` setting is a dict where the keys are -the extension paths, and their values are the orders, which define the -extension *loading* order. The :setting:`EXTENSIONS` setting is merged with the -:setting:`EXTENSIONS_BASE` setting defined in Scrapy (and not meant to be -overridden) and then sorted by order to get the final sorted list of enabled -extensions. +:setting:`EXTENSIONS` is merged with :setting:`EXTENSIONS_BASE` (not meant to +be overridden), and the priorities in the resulting value determine the +*loading* order. As extensions typically do not depend on each other, their loading order is irrelevant in most cases. This is why the :setting:`EXTENSIONS_BASE` setting -defines all extensions with the same order (``0``). However, this feature can -be exploited if you need to add an extension which depends on other extensions -already loaded. - -Available, enabled and disabled extensions -========================================== - -Not all available extensions will be enabled. Some of them usually depend on a -particular setting. For example, the HTTP Cache extension is available by default -but disabled unless the :setting:`HTTPCACHE_ENABLED` setting is set. - -Disabling an extension -====================== - -In order to disable an extension that comes enabled by default (i.e. those -included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to -``None``. For example: - -.. code-block:: python - - EXTENSIONS = { - "scrapy.extensions.corestats.CoreStats": None, - } +defines all extensions with the same order (``0``). However, you may need to +carefully use priorities if you add an extension that depends on other +extensions being already loaded. Writing your own extension ========================== -Each extension is a Python class. The main entry point for a Scrapy extension -(this also includes middlewares and pipelines) is the ``from_crawler`` -class method which receives a ``Crawler`` instance. Through the Crawler object -you can access settings, signals, stats, and also control the crawling behaviour. +Each extension is a :ref:`component `. Typically, extensions connect to :ref:`signals ` and perform tasks triggered by them. -Finally, if the ``from_crawler`` method raises the -:exc:`~scrapy.exceptions.NotConfigured` exception, the extension will be -disabled. Otherwise, the extension will be enabled. - Sample extension ---------------- diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 310f153e8..dc27ce6ca 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -23,7 +23,8 @@ Typical uses of item pipelines are: Writing your own item pipeline ============================== -Each item pipeline component is a Python class that must implement the following method: +Each item pipeline is a :ref:`component ` that must +implement the following method: .. method:: process_item(self, item, spider) @@ -60,17 +61,6 @@ Additionally, they may also implement the following methods: :param spider: the spider which was closed :type spider: :class:`~scrapy.Spider` object -.. classmethod:: from_crawler(cls, crawler) - - If present, this class method is called to create a pipeline instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the pipeline. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for pipeline to - access them and hook its functionality into Scrapy. - - :param crawler: crawler that uses this pipeline - :type crawler: :class:`~scrapy.crawler.Crawler` object - Item pipeline example ===================== @@ -139,8 +129,8 @@ In this example we'll write items to MongoDB_ using pymongo_. MongoDB address and database name are specified in Scrapy settings; MongoDB collection is named after item class. -The main point of this example is to show how to use :meth:`from_crawler` -method and how to clean up the resources properly. +The main point of this example is to show how to :ref:`get the crawler +` and how to clean up the resources properly. .. skip: next .. code-block:: python diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 0375e0ff1..77837378e 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -463,35 +463,17 @@ import path. Writing your own request fingerprinter ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A request fingerprinter is a class that must implement the following method: +A request fingerprinter is a :ref:`component ` that must +implement the following method: .. currentmodule:: None -.. method:: fingerprint(self, request) +.. method:: fingerprint(self, request: scrapy.Request) Return a :class:`bytes` object that uniquely identifies *request*. See also :ref:`request-fingerprint-restrictions`. - :param request: request to fingerprint - :type request: scrapy.Request - -Additionally, it may also implement the following method: - -.. classmethod:: from_crawler(cls, crawler) - :noindex: - - If present, this class method is called to create a request fingerprinter - instance from a :class:`~scrapy.crawler.Crawler` object. It must return a - new instance of the request fingerprinter. - - *crawler* provides access to all Scrapy core components like settings and - signals; it is a way for the request fingerprinter to access them and hook - its functionality into Scrapy. - - :param crawler: crawler that uses this request fingerprinter - :type crawler: :class:`~scrapy.crawler.Crawler` object - .. currentmodule:: scrapy.http The :meth:`fingerprint` method of the default request fingerprinter, diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 7646aca4f..b2bb7148f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -204,7 +204,7 @@ How to access settings .. highlight:: python -In a spider, the settings are available through ``self.settings``: +In a spider, settings are available through ``self.settings``: .. code-block:: python @@ -217,37 +217,17 @@ In a spider, the settings are available through ``self.settings``: .. note:: The ``settings`` attribute is set in the base Spider class after the spider - is initialized. If you want to use the settings before the initialization + is initialized. If you want to use settings before the initialization (e.g., in your spider's ``__init__()`` method), you'll need to override the :meth:`~scrapy.Spider.from_crawler` method. -Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings` -attribute of the Crawler that is passed to ``from_crawler`` method in -extensions, middlewares and item pipelines: +:ref:`Components ` can also :ref:`access settings +`. -.. code-block:: python - - class MyExtension: - def __init__(self, log_is_enabled=False): - if log_is_enabled: - print("log is enabled!") - - @classmethod - def from_crawler(cls, crawler): - settings = crawler.settings - return cls(settings.getbool("LOG_ENABLED")) - -The settings object can be used like a dict (e.g., -``settings['LOG_ENABLED']``), but it's usually preferred to extract the setting -in the format you need it to avoid type errors, using one of the methods -provided by the :class:`~scrapy.settings.Settings` API. - -Rationale for setting names -=========================== - -Setting names are usually prefixed with the component that they configure. For -example, proper setting names for a fictional robots.txt extension would be -``ROBOTSTXT_ENABLED``, ``ROBOTSTXT_OBEY``, ``ROBOTSTXT_CACHEDIR``, etc. +The ``settings`` object can be used like a :class:`dict` (e.g. +``settings["LOG_ENABLED"]``). However, to support non-string setting values, +which may be passed from the command line as strings, it is recommended to use +one of the methods provided by the :class:`~scrapy.settings.Settings` API. .. _component-priority-dictionaries: @@ -1211,7 +1191,8 @@ EXTENSIONS Default:: ``{}`` -A dict containing the extensions enabled in your project, and their orders. +:ref:`Component priority dictionary ` of +enabled extensions. See :ref:`topics-extensions`. .. setting:: EXTENSIONS_BASE diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2b59cabe1..567a875b6 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -63,17 +63,38 @@ particular setting. See each middleware documentation for more info. Writing your own spider middleware ================================== -Each spider middleware is a Python class that defines one or more of the -methods defined below. - -The main entry point is the ``from_crawler`` class method, which receives a -:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` -object gives you access, for example, to the :ref:`settings `. +Each spider middleware is a :ref:`component ` that defines +one or more of these methods: .. module:: scrapy.spidermiddlewares .. class:: SpiderMiddleware + .. method:: process_start_requests(start_requests, spider) + + This method is called with the start requests of the spider, and works + similarly to the :meth:`process_spider_output` method, except that it + doesn't have a response associated and must return only requests (not + items). + + It receives an iterable (in the ``start_requests`` parameter) and must + return another iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects `. + + .. note:: When implementing this method in your spider middleware, you + should always return an iterable (that follows the input one) and + not consume all ``start_requests`` iterator because it can be very + large (or even unbounded) and cause a memory overflow. The Scrapy + engine is designed to pull start requests while it has capacity to + process them, so the start requests iterator can be effectively + endless where there is some other condition for stopping the spider + (like a time limit or item/page count). + + :param start_requests: the start requests + :type start_requests: an iterable of :class:`~scrapy.Request` + + :param spider: the spider to whom the start requests belong + :type spider: :class:`~scrapy.Spider` object + .. method:: process_spider_input(response, spider) This method is called for each response that goes through the spider @@ -168,42 +189,6 @@ object gives you access, for example, to the :ref:`settings `. :param spider: the spider which raised the exception :type spider: :class:`~scrapy.Spider` object - .. method:: process_start_requests(start_requests, spider) - - This method is called with the start requests of the spider, and works - similarly to the :meth:`process_spider_output` method, except that it - doesn't have a response associated and must return only requests (not - items). - - It receives an iterable (in the ``start_requests`` parameter) and must - return another iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects `. - - .. note:: When implementing this method in your spider middleware, you - should always return an iterable (that follows the input one) and - not consume all ``start_requests`` iterator because it can be very - large (or even unbounded) and cause a memory overflow. The Scrapy - engine is designed to pull start requests while it has capacity to - process them, so the start requests iterator can be effectively - endless where there is some other condition for stopping the spider - (like a time limit or item/page count). - - :param start_requests: the start requests - :type start_requests: an iterable of :class:`~scrapy.Request` - - :param spider: the spider to whom the start requests belong - :type spider: :class:`~scrapy.Spider` object - - .. method:: from_crawler(cls, crawler) - - If present, this classmethod is called to create a middleware instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the middleware. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for middleware to - access them and hook its functionality into Scrapy. - - :param crawler: crawler that uses this middleware - :type crawler: :class:`~scrapy.crawler.Crawler` object - .. _topics-spider-middleware-ref: Built-in spider middleware reference From 803b4f258d85ab4f85c7c230e6c025f54e836269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 14:53:42 +0100 Subject: [PATCH 079/491] tox: move to posargs pytest parameters that can be too noisy when running specific tests (#6724) --- tox.ini | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 041fcffca..70c841603 100644 --- a/tox.ini +++ b/tox.ini @@ -39,7 +39,7 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - pytest --cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml {posargs:--durations=10 docs scrapy tests} --doctest-modules + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --durations=10 docs scrapy tests --doctest-modules} install_command = python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} @@ -58,7 +58,7 @@ deps = pytest >= 8.2.0 w3lib >= 2.2.0 commands = - mypy {posargs: scrapy tests} + mypy {posargs:scrapy tests} [testenv:typing-tests] basepython = python3.9 @@ -67,7 +67,7 @@ deps = {[testenv:typing]deps} pytest-mypy-testing==0.1.3 commands = - pytest {posargs: tests_typing} + pytest {posargs:tests_typing} [testenv:pre-commit] basepython = python3 @@ -119,7 +119,7 @@ install_command = python -I -m pip install {opts} {packages} commands = ; tests for docs fail with parsel < 1.8.0 - pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --durations=10 scrapy tests} [testenv:pinned] basepython = {[pinned]basepython} @@ -266,7 +266,7 @@ deps = {[testenv]deps} botocore>=1.4.87 commands = - pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests -m requires_botocore} [testenv:botocore-pinned] basepython = {[pinned]basepython} @@ -277,4 +277,4 @@ install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands = - pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests -m requires_botocore} From eb654aa1a8d2ef6433957fcc1361420b6141094e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 11 Mar 2025 21:00:36 +0400 Subject: [PATCH 080/491] Convert remaining unittest assert* calls, use the tmp_path fixture. (#6725) --- pyproject.toml | 4 -- scrapy/utils/test.py | 2 +- tests/test_downloadermiddleware_robotstxt.py | 13 ++++- tests/test_engine.py | 8 ++- tests/test_pipeline_files.py | 58 ++++++++------------ tests/test_pipeline_images.py | 44 +++++++-------- tests/test_utils_template.py | 16 +----- 7 files changed, 62 insertions(+), 83 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 82d8056f6..84bf41a94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -374,10 +374,6 @@ ignore = [ "B904", # Use capitalized environment variable "SIM112", - - # Temporarily silenced PT rules - # Use a regular `assert` instead of unittest-style `assertEqual` - "PT009", ] [tool.ruff.lint.per-file-ignores] diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index db1f5c419..b69f43438 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -156,7 +156,7 @@ def assert_samelines( category=ScrapyDeprecationWarning, stacklevel=2, ) - testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) + testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) # noqa: PT009 def get_from_asyncio_queue(value: _T) -> Awaitable[_T]: diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 38f0333bb..ad335f852 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,3 +1,4 @@ +from typing import Any from unittest import mock import pytest @@ -171,7 +172,11 @@ Disallow: /some/randome/page.html middleware = RobotsTxtMiddleware(self.crawler) middleware._logerror = mock.MagicMock(side_effect=middleware._logerror) deferred = middleware.process_request(Request("http://site.local"), None) - deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called)) + + def check_called(_: Any) -> None: + assert middleware._logerror.called + + deferred.addCallback(check_called) return deferred def test_robotstxt_immediate_error(self): @@ -202,7 +207,11 @@ Disallow: /some/randome/page.html mw_module_logger.error = mock.MagicMock() d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware) - d.addCallback(lambda _: self.assertFalse(mw_module_logger.error.called)) + + def check_not_called(_: Any) -> None: + assert not mw_module_logger.error.called # type: ignore[attr-defined] + + d.addCallback(check_not_called) return d def test_robotstxt_user_agent_setting(self): diff --git a/tests/test_engine.py b/tests/test_engine.py index 4bac8d273..ba4c6dc40 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -433,10 +433,12 @@ class TestEngine(TestEngineBase): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) yield e.open_spider(MySpider(), []) e.start() + + def cb(exc: BaseException) -> None: + assert str(exc), "Engine already running" + try: - yield self.assertFailure(e.start(), RuntimeError).addBoth( - lambda exc: self.assertEqual(str(exc), "Engine already running") - ) + yield self.assertFailure(e.start(), RuntimeError).addBoth(cb) finally: yield e.stop() diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e515c16a0..9a582e4b7 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -266,17 +266,11 @@ class TestFilesPipeline(unittest.TestCase): class FilesPipelineTestCaseFieldsMixin: - def setup_method(self): - self.tempdir = mkdtemp() - - def teardown_method(self): - rmtree(self.tempdir) - - def test_item_fields_default(self): + def test_item_fields_default(self, tmp_path): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", file_urls=[url]) pipeline = FilesPipeline.from_crawler( - get_crawler(None, {"FILES_STORE": self.tempdir}) + get_crawler(None, {"FILES_STORE": tmp_path}) ) requests = list(pipeline.get_media_requests(item, None)) assert requests[0].url == url @@ -286,14 +280,14 @@ class FilesPipelineTestCaseFieldsMixin: assert files == [results[0][1]] assert isinstance(item, self.item_class) - def test_item_fields_override_settings(self): + def test_item_fields_override_settings(self, tmp_path): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", custom_file_urls=[url]) pipeline = FilesPipeline.from_crawler( get_crawler( None, { - "FILES_STORE": self.tempdir, + "FILES_STORE": tmp_path, "FILES_URLS_FIELD": "custom_file_urls", "FILES_RESULT_FIELD": "custom_files", }, @@ -368,13 +362,7 @@ class TestFilesPipelineCustomSettings: ("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field"), } - def setup_method(self): - self.tempdir = mkdtemp() - - def teardown_method(self): - rmtree(self.tempdir) - - def _generate_fake_settings(self, prefix=None): + def _generate_fake_settings(self, tmp_path, prefix=None): def random_string(): return "".join([chr(random.randint(97, 123)) for _ in range(10)]) @@ -382,7 +370,7 @@ class TestFilesPipelineCustomSettings: "FILES_EXPIRES": random.randint(100, 1000), "FILES_URLS_FIELD": random_string(), "FILES_RESULT_FIELD": random_string(), - "FILES_STORE": self.tempdir, + "FILES_STORE": tmp_path, } if not prefix: return settings @@ -400,16 +388,16 @@ class TestFilesPipelineCustomSettings: return UserDefinedFilePipeline - def test_different_settings_for_different_instances(self): + def test_different_settings_for_different_instances(self, tmp_path): """ If there are different instances with different settings they should keep different settings. """ - custom_settings = self._generate_fake_settings() + custom_settings = self._generate_fake_settings(tmp_path) another_pipeline = FilesPipeline.from_crawler( get_crawler(None, custom_settings) ) - one_pipeline = FilesPipeline(self.tempdir, crawler=get_crawler(None)) + one_pipeline = FilesPipeline(tmp_path, crawler=get_crawler(None)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: default_value = self.default_cls_settings[pipe_attr] assert getattr(one_pipeline, pipe_attr) == default_value @@ -417,24 +405,24 @@ class TestFilesPipelineCustomSettings: assert default_value != custom_value assert getattr(another_pipeline, pipe_ins_attr) == custom_value - def test_subclass_attributes_preserved_if_no_settings(self): + def test_subclass_attributes_preserved_if_no_settings(self, tmp_path): """ If subclasses override class attributes and there are no special settings those values should be kept. """ pipe_cls = self._generate_fake_pipeline() - pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": self.tempdir})) + pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": tmp_path})) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: custom_value = getattr(pipe, pipe_ins_attr) assert custom_value != self.default_cls_settings[pipe_attr] assert getattr(pipe, pipe_ins_attr) == getattr(pipe, pipe_attr) - def test_subclass_attrs_preserved_custom_settings(self): + def test_subclass_attrs_preserved_custom_settings(self, tmp_path): """ If file settings are defined but they are not defined for subclass settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline() - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: value = getattr(pipeline, pipe_ins_attr) @@ -442,7 +430,7 @@ class TestFilesPipelineCustomSettings: assert value != self.default_cls_settings[pipe_attr] assert value == setting_value - def test_no_custom_settings_for_subclasses(self): + def test_no_custom_settings_for_subclasses(self, tmp_path): """ If there are no settings for subclass and no subclass attributes, pipeline should use attributes of base class. @@ -452,14 +440,14 @@ class TestFilesPipelineCustomSettings: pass user_pipeline = UserDefinedFilesPipeline.from_crawler( - get_crawler(None, {"FILES_STORE": self.tempdir}) + get_crawler(None, {"FILES_STORE": tmp_path}) ) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = self.default_cls_settings.get(pipe_attr.upper()) assert getattr(user_pipeline, pipe_ins_attr) == custom_value - def test_custom_settings_for_subclasses(self): + def test_custom_settings_for_subclasses(self, tmp_path): """ If there are custom settings for subclass and NO class attributes, pipeline should use custom settings. @@ -469,7 +457,7 @@ class TestFilesPipelineCustomSettings: pass prefix = UserDefinedFilesPipeline.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) user_pipeline = UserDefinedFilesPipeline.from_crawler( get_crawler(None, settings) ) @@ -479,14 +467,14 @@ class TestFilesPipelineCustomSettings: assert custom_value != self.default_cls_settings[pipe_attr] assert getattr(user_pipeline, pipe_inst_attr) == custom_value - def test_custom_settings_and_class_attrs_for_subclasses(self): + def test_custom_settings_and_class_attrs_for_subclasses(self, tmp_path): """ If there are custom settings for subclass AND class attributes setting keys are preferred and override attributes. """ pipeline_cls = self._generate_fake_pipeline() prefix = pipeline_cls.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for ( pipe_cls_attr, @@ -497,13 +485,13 @@ class TestFilesPipelineCustomSettings: assert custom_value != self.default_cls_settings[pipe_cls_attr] assert getattr(user_pipeline, pipe_inst_attr) == custom_value - def test_cls_attrs_with_DEFAULT_prefix(self): + def test_cls_attrs_with_DEFAULT_prefix(self, tmp_path): class UserDefinedFilesPipeline(FilesPipeline): DEFAULT_FILES_RESULT_FIELD = "this" DEFAULT_FILES_URLS_FIELD = "that" pipeline = UserDefinedFilesPipeline.from_crawler( - get_crawler(None, {"FILES_STORE": self.tempdir}) + get_crawler(None, {"FILES_STORE": tmp_path}) ) assert ( pipeline.files_result_field @@ -514,12 +502,12 @@ class TestFilesPipelineCustomSettings: == UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD ) - def test_user_defined_subclass_default_key_names(self): + def test_user_defined_subclass_default_key_names(self, tmp_path): """Test situation when user defines subclass of FilesPipeline, but uses attribute names for default pipeline (without prefixing them with pipeline class name). """ - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) class UserPipe(FilesPipeline): pass diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index fef6bbbe9..f2ee18bd9 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -314,13 +314,7 @@ class TestImagesPipelineCustomSettings: "IMAGES_RESULT_FIELD": "images", } - def setup_method(self): - self.tempdir = mkdtemp() - - def teardown_method(self): - rmtree(self.tempdir) - - def _generate_fake_settings(self, prefix=None): + def _generate_fake_settings(self, tmp_path, prefix=None): """ :param prefix: string for setting keys :return: dictionary of image pipeline settings @@ -331,7 +325,7 @@ class TestImagesPipelineCustomSettings: settings = { "IMAGES_EXPIRES": random.randint(100, 1000), - "IMAGES_STORE": self.tempdir, + "IMAGES_STORE": tmp_path, "IMAGES_RESULT_FIELD": random_string(), "IMAGES_URLS_FIELD": random_string(), "IMAGES_MIN_WIDTH": random.randint(1, 1000), @@ -368,13 +362,13 @@ class TestImagesPipelineCustomSettings: return UserDefinedImagePipeline - def test_different_settings_for_different_instances(self): + def test_different_settings_for_different_instances(self, tmp_path): """ If there are two instances of ImagesPipeline class with different settings, they should have different settings. """ - custom_settings = self._generate_fake_settings() - default_sts_pipe = ImagesPipeline(self.tempdir, crawler=get_crawler(None)) + custom_settings = self._generate_fake_settings(tmp_path) + default_sts_pipe = ImagesPipeline(tmp_path, crawler=get_crawler(None)) user_sts_pipe = ImagesPipeline.from_crawler(get_crawler(None, custom_settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: expected_default_value = self.default_pipeline_settings.get(pipe_attr) @@ -385,14 +379,14 @@ class TestImagesPipelineCustomSettings: ) assert getattr(user_sts_pipe, pipe_attr.lower()) == custom_value - def test_subclass_attrs_preserved_default_settings(self): + def test_subclass_attrs_preserved_default_settings(self, tmp_path): """ If image settings are not defined at all subclass of ImagePipeline takes values from class attributes. """ pipeline_cls = self._generate_fake_pipeline_subclass() pipeline = pipeline_cls.from_crawler( - get_crawler(None, {"IMAGES_STORE": self.tempdir}) + get_crawler(None, {"IMAGES_STORE": tmp_path}) ) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Instance attribute (lowercase) must be equal to class attribute (uppercase). @@ -400,13 +394,13 @@ class TestImagesPipelineCustomSettings: assert attr_value != self.default_pipeline_settings[pipe_attr] assert attr_value == getattr(pipeline, pipe_attr) - def test_subclass_attrs_preserved_custom_settings(self): + def test_subclass_attrs_preserved_custom_settings(self, tmp_path): """ If image settings are defined but they are not defined for subclass default values taken from settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline_subclass() - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Instance attribute (lowercase) must be equal to @@ -416,7 +410,7 @@ class TestImagesPipelineCustomSettings: setings_value = settings.get(settings_attr) assert value == setings_value - def test_no_custom_settings_for_subclasses(self): + def test_no_custom_settings_for_subclasses(self, tmp_path): """ If there are no settings for subclass and no subclass attributes, pipeline should use attributes of base class. @@ -426,14 +420,14 @@ class TestImagesPipelineCustomSettings: pass user_pipeline = UserDefinedImagePipeline.from_crawler( - get_crawler(None, {"IMAGES_STORE": self.tempdir}) + get_crawler(None, {"IMAGES_STORE": tmp_path}) ) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = self.default_pipeline_settings.get(pipe_attr.upper()) assert getattr(user_pipeline, pipe_attr.lower()) == custom_value - def test_custom_settings_for_subclasses(self): + def test_custom_settings_for_subclasses(self, tmp_path): """ If there are custom settings for subclass and NO class attributes, pipeline should use custom settings. @@ -443,7 +437,7 @@ class TestImagesPipelineCustomSettings: pass prefix = UserDefinedImagePipeline.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) user_pipeline = UserDefinedImagePipeline.from_crawler( get_crawler(None, settings) ) @@ -453,27 +447,27 @@ class TestImagesPipelineCustomSettings: assert custom_value != self.default_pipeline_settings[pipe_attr] assert getattr(user_pipeline, pipe_attr.lower()) == custom_value - def test_custom_settings_and_class_attrs_for_subclasses(self): + def test_custom_settings_and_class_attrs_for_subclasses(self, tmp_path): """ If there are custom settings for subclass AND class attributes setting keys are preferred and override attributes. """ pipeline_cls = self._generate_fake_pipeline_subclass() prefix = pipeline_cls.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: custom_value = settings.get(prefix + "_" + settings_attr) assert custom_value != self.default_pipeline_settings[pipe_attr] assert getattr(user_pipeline, pipe_attr.lower()) == custom_value - def test_cls_attrs_with_DEFAULT_prefix(self): + def test_cls_attrs_with_DEFAULT_prefix(self, tmp_path): class UserDefinedImagePipeline(ImagesPipeline): DEFAULT_IMAGES_URLS_FIELD = "something" DEFAULT_IMAGES_RESULT_FIELD = "something_else" pipeline = UserDefinedImagePipeline.from_crawler( - get_crawler(None, {"IMAGES_STORE": self.tempdir}) + get_crawler(None, {"IMAGES_STORE": tmp_path}) ) assert ( pipeline.images_result_field @@ -484,12 +478,12 @@ class TestImagesPipelineCustomSettings: == UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD ) - def test_user_defined_subclass_default_key_names(self): + def test_user_defined_subclass_default_key_names(self, tmp_path): """Test situation when user defines subclass of ImagePipeline, but uses attribute names for default pipeline (without prefixing them with pipeline class name). """ - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) class UserPipe(ImagesPipeline): pass diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 0b845fdb0..41d9b8933 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -1,24 +1,14 @@ -from pathlib import Path -from shutil import rmtree -from tempfile import mkdtemp - from scrapy.utils.template import render_templatefile class TestUtilsRenderTemplateFile: - def setup_method(self): - self.tmp_path = mkdtemp() - - def teardown_method(self): - rmtree(self.tmp_path) - - def test_simple_render(self): + def test_simple_render(self, tmp_path): context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"} template = "from ${project_name}.spiders.${name} import ${classname}" rendered = "from proj.spiders.spi import TheSpider" - template_path = Path(self.tmp_path, "templ.py.tmpl") - render_path = Path(self.tmp_path, "templ.py") + template_path = tmp_path / "templ.py.tmpl" + render_path = tmp_path / "templ.py" template_path.write_text(template, encoding="utf8") assert template_path.is_file() # Failure of test itself From d0dabbc09706b082e2250790cd7a00c033ad8021 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 12 Mar 2025 00:18:30 +0400 Subject: [PATCH 081/491] Enable AsyncioSelectorReactor by default. (#6713) * Enable AsyncioSelectorReactor by default. * Improve get_crawler(), switch more tests to it. * Fix the remaining default-reactor test failures. * Address documentation feedback. * Make pinned envs more consistent. --- .github/workflows/tests-ubuntu.yml | 14 ++-- .github/workflows/tests-windows.yml | 16 ++++- conftest.py | 12 ++-- docs/topics/asyncio.rst | 23 +++++-- docs/topics/media-pipeline.rst | 2 +- docs/topics/settings.rst | 14 ++-- scrapy/pipelines/images.py | 2 +- scrapy/settings/default_settings.py | 2 +- .../templates/project/module/settings.py.tmpl | 1 - scrapy/utils/log.py | 4 +- scrapy/utils/test.py | 23 ++++++- .../CrawlerProcess/asyncio_enabled_reactor.py | 14 ++-- tests/CrawlerProcess/reactor_default.py | 4 +- tests/CrawlerProcess/reactor_select.py | 4 +- tests/CrawlerRunner/ip_address.py | 6 ++ tests/test_addons.py | 5 +- tests/test_crawl.py | 4 +- tests/test_crawler.py | 46 +++++++------ tests/test_dependencies.py | 2 +- tests/test_downloader_handlers.py | 6 +- tests/test_downloaderslotssettings.py | 2 +- tests/test_extension_periodic_log.py | 19 +++--- tests/test_pipeline_crawl.py | 31 +++++---- tests/test_spider.py | 8 ++- tests/test_utils_asyncio.py | 5 +- tox.ini | 68 ++++++++----------- 26 files changed, 197 insertions(+), 140 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index f74575ee1..444aa3557 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -34,25 +34,25 @@ jobs: TOXENV: py - python-version: "3.13" env: - TOXENV: asyncio + TOXENV: default-reactor - python-version: pypy3.10 env: TOXENV: pypy3 # pinned deps - - python-version: 3.9.19 + - python-version: "3.9.21" env: TOXENV: pinned - - python-version: 3.9.19 + - python-version: "3.9.21" env: - TOXENV: asyncio-pinned + TOXENV: default-reactor-pinned - python-version: pypy3.10 env: TOXENV: pypy3-pinned - - python-version: 3.9.19 + - python-version: "3.9.21" env: TOXENV: extra-deps-pinned - - python-version: 3.9.19 + - python-version: "3.9.21" env: TOXENV: botocore-pinned @@ -78,7 +78,7 @@ jobs: if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update - sudo apt-get install libxml2-dev libxslt-dev libjpeg-dev + sudo apt-get install libxml2-dev libxslt-dev - name: Run tests env: ${{ matrix.env }} diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 21d621240..537a01e29 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -19,7 +19,7 @@ jobs: include: - python-version: "3.9" env: - TOXENV: windows-pinned + TOXENV: py - python-version: "3.10" env: TOXENV: py @@ -34,7 +34,19 @@ jobs: TOXENV: py - python-version: "3.13" env: - TOXENV: asyncio + TOXENV: default-reactor + + # pinned deps + - python-version: "3.9.13" + env: + TOXENV: pinned + - python-version: "3.9.13" + env: + TOXENV: extra-deps-pinned + + - python-version: "3.13" + env: + TOXENV: extra-deps steps: - uses: actions/checkout@v4 diff --git a/conftest.py b/conftest.py index f33ffb1a4..9999e41d2 100644 --- a/conftest.py +++ b/conftest.py @@ -51,7 +51,7 @@ def chdir(tmpdir): def pytest_addoption(parser): parser.addoption( "--reactor", - default="default", + default="asyncio", choices=["default", "asyncio"], ) @@ -67,17 +67,17 @@ def reactor_pytest(request): @pytest.fixture(autouse=True) def only_asyncio(request, reactor_pytest): - if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio": - pytest.skip("This test is only run with --reactor=asyncio") + if request.node.get_closest_marker("only_asyncio") and reactor_pytest == "default": + pytest.skip("This test is only run without --reactor=default") @pytest.fixture(autouse=True) def only_not_asyncio(request, reactor_pytest): if ( request.node.get_closest_marker("only_not_asyncio") - and reactor_pytest == "asyncio" + and reactor_pytest != "default" ): - pytest.skip("This test is only run without --reactor=asyncio") + pytest.skip("This test is only run with --reactor=default") @pytest.fixture(autouse=True) @@ -117,7 +117,7 @@ def requires_boto3(request): def pytest_configure(config): - if config.getoption("--reactor") == "asyncio": + if config.getoption("--reactor") != "default": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 07baea071..35afdc11b 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -16,15 +16,19 @@ asyncio reactor `, you may use :mod:`asyncio` and Installing the asyncio reactor ============================== -To enable :mod:`asyncio` support, set the :setting:`TWISTED_REACTOR` setting to -``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``. +To enable :mod:`asyncio` support, your :setting:`TWISTED_REACTOR` setting needs +to be set to ``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, +which is the default value. If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` reactor manually. You can do that using -:func:`~scrapy.utils.reactor.install_reactor`:: +:func:`~scrapy.utils.reactor.install_reactor`: - install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') +.. skip: next +.. code-block:: python + + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") .. _asyncio-preinstalled-reactor: @@ -144,3 +148,14 @@ Using custom asyncio loops You can also use custom asyncio event loops with the asyncio reactor. Set the :setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to use it instead of the default asyncio event loop. + + +.. _disable-asyncio: + +Switching to a non-asyncio reactor +================================== + +If for some reason your code doesn't work with the asyncio reactor, you can use +a different reactor by setting the :setting:`TWISTED_REACTOR` setting to its +import path (e.g. ``'twisted.internet.epollreactor.EPollReactor'``) or to +``None``, which will use the default reactor for your platform. diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index cc1fe8703..01da53342 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you can configure some extra functions like generating thumbnails and filtering the images based on their size. -The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for +The Images Pipeline requires Pillow_ 8.0.0 or greater. It is used for thumbnailing and normalizing images to JPEG/RGB format. .. _Pillow: https://github.com/python-pillow/Pillow diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index b2bb7148f..ca0af569f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1911,7 +1911,7 @@ TWISTED_REACTOR .. versionadded:: 2.0 -Default: ``None`` +Default: ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"`` Import path of a given :mod:`~twisted.internet.reactor`. @@ -1996,17 +1996,19 @@ which raises :exc:`Exception`, becomes: self.crawler.engine.close_spider(self, "timeout") -The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which -means that Scrapy will use the existing reactor if one is already installed, or -install the default reactor defined by Twisted for the current platform. This -is to maintain backward compatibility and avoid possible problems caused by -using a non-default reactor. +If this setting is set ``None``, Scrapy will use the existing reactor if one is +already installed, or install the default reactor defined by Twisted for the +current platform. .. versionchanged:: 2.7 The :command:`startproject` command now sets this setting to ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated ``settings.py`` file. +.. versionchanged:: VERSION + The default value was changed from ``None`` to + ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``. + For additional information, see :doc:`core/howto/choosing-reactor`. diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 29dc13f0a..63c6908dc 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -68,7 +68,7 @@ class ImagesPipeline(FilesPipeline): self._Image = Image except ImportError: raise NotConfigured( - "ImagesPipeline requires installing Pillow 4.0.0 or later" + "ImagesPipeline requires installing Pillow 8.0.0 or later" ) super().__init__( diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index c473b369c..645e50301 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -341,7 +341,7 @@ TELNETCONSOLE_HOST = "127.0.0.1" TELNETCONSOLE_USERNAME = "scrapy" TELNETCONSOLE_PASSWORD = None -TWISTED_REACTOR = None +TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 0bb31ffaa..db7400af8 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -90,5 +90,4 @@ ROBOTSTXT_OBEY = True #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" # Set settings whose default value is deprecated to a future-proof value -TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index b865cf48d..24e17ecb6 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -182,11 +182,9 @@ def log_scrapy_info(settings: Settings) -> None: def log_reactor_info() -> None: - from twisted.internet import reactor + from twisted.internet import asyncioreactor, reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) - from twisted.internet import asyncioreactor - if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor): logger.debug( "Using asyncio event loop: %s.%s", diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index b69f43438..2da526cd8 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -18,6 +18,7 @@ from twisted.trial.unittest import SkipTest from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.boto import is_botocore_available from scrapy.utils.deprecate import create_deprecated_class +from scrapy.utils.reactor import is_asyncio_reactor_installed from scrapy.utils.spider import DefaultSpider if TYPE_CHECKING: @@ -109,6 +110,19 @@ def get_ftp_content_and_delete( TestSpider = create_deprecated_class("TestSpider", DefaultSpider) +def get_reactor_settings() -> dict[str, Any]: + """Return a settings dict that works with the installed reactor. + + ``Crawler._apply_settings()`` checks that the installed reactor matches the + settings, so tests that run the crawler in the current process may need to + pass a correct ``"TWISTED_REACTOR"`` setting value when creating it. + """ + settings: dict[str, Any] = {} + if not is_asyncio_reactor_installed(): + settings["TWISTED_REACTOR"] = None + return settings + + def get_crawler( spidercls: type[Spider] | None = None, settings_dict: dict[str, Any] | None = None, @@ -120,9 +134,12 @@ def get_crawler( """ from scrapy.crawler import CrawlerRunner - # Set by default settings that prevent deprecation warnings. - settings: dict[str, Any] = {} - settings.update(settings_dict or {}) + # When needed, useful settings can be added here, e.g. ones that prevent + # deprecation warnings. + settings: dict[str, Any] = { + **get_reactor_settings(), + **(settings_dict or {}), + } runner = CrawlerRunner(settings) crawler = runner.create_crawler(spidercls or DefaultSpider) crawler._apply_settings() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index f013eed27..0c380610d 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,14 +1,8 @@ -import asyncio -import sys +import scrapy +from scrapy.crawler import CrawlerProcess +from scrapy.utils.reactor import install_reactor -from twisted.internet import asyncioreactor - -if sys.platform == "win32": - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -asyncioreactor.install(asyncio.get_event_loop()) - -import scrapy # noqa: E402 -from scrapy.crawler import CrawlerProcess # noqa: E402 +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index 078cb72cb..e2933338b 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -1,4 +1,5 @@ from twisted.internet import reactor # noqa: F401 +from twisted.python import log import scrapy from scrapy.crawler import CrawlerProcess @@ -13,5 +14,6 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess(settings={}) -process.crawl(NoRequestsSpider) +d = process.crawl(NoRequestsSpider) +d.addErrback(log.err) process.start() diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index 814a2a46d..b61e52625 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -1,4 +1,5 @@ from twisted.internet import selectreactor +from twisted.python import log import scrapy from scrapy.crawler import CrawlerProcess @@ -15,5 +16,6 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess(settings={}) -process.crawl(NoRequestsSpider) +d = process.crawl(NoRequestsSpider) +d.addErrback(log.err) process.start() diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 2f1bb7713..b1b297777 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -1,3 +1,9 @@ +# ruff: noqa: E402 + +from scrapy.utils.reactor import install_reactor + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + from urllib.parse import urlparse from twisted.internet import reactor diff --git a/tests/test_addons.py b/tests/test_addons.py index 686bf9952..b4294c815 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -9,7 +9,7 @@ from scrapy import Spider from scrapy.crawler import Crawler, CrawlerRunner from scrapy.exceptions import NotConfigured from scrapy.settings import BaseSettings, Settings -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings class SimpleAddon: @@ -105,6 +105,7 @@ class TestAddonManager(unittest.TestCase): } settings_dict = { "ADDONS": {get_addon_cls(config): 1}, + **get_reactor_settings(), } crawler = get_crawler(settings_dict=settings_dict) assert crawler.settings.getint("KEY") == 15 @@ -119,6 +120,7 @@ class TestAddonManager(unittest.TestCase): settings_dict = { "KEY": 20, # priority=project "ADDONS": {get_addon_cls(config): 1}, + **get_reactor_settings(), } settings = Settings(settings_dict) settings.set("KEY", 0, priority="default") @@ -196,6 +198,7 @@ class TestAddonManager(unittest.TestCase): return spider settings = Settings() + settings.setdict(get_reactor_settings()) settings.set("KEY", "default", priority="default") runner = CrawlerRunner(settings) crawler = runner.create_crawler(MySpider) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 6f4045fc8..f49deac1f 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -18,7 +18,7 @@ from scrapy.exceptions import StopDownload from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import to_unicode -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings from tests import NON_EXISTING_RESOLVABLE from tests.mockserver import MockServer from tests.spiders import ( @@ -412,7 +412,7 @@ with multiples lines @defer.inlineCallbacks def test_crawl_multiple(self): - runner = CrawlerRunner() + runner = CrawlerRunner(get_reactor_settings()) runner.crawl( SimpleSpider, self.mockserver.url("/status?n=200"), diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0bbcc0843..98352b66e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -25,7 +25,7 @@ from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings from tests.mockserver import MockServer, get_mockserver_env BASE_SETTINGS: dict[str, Any] = {} @@ -35,6 +35,7 @@ def get_raw_crawler(spidercls=None, settings_dict=None): """get_crawler alternative that only calls the __init__ method of the crawler.""" settings = Settings() + settings.setdict(get_reactor_settings()) settings.setdict(settings_dict or {}) return Crawler(spidercls or DefaultSpider, settings) @@ -48,7 +49,12 @@ class TestBaseCrawler(unittest.TestCase): class TestCrawler(TestBaseCrawler): def test_populate_spidercls_settings(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = {**BASE_SETTINGS, "TEST1": "project", "TEST3": "project"} + project_settings = { + **BASE_SETTINGS, + "TEST1": "project", + "TEST3": "project", + **get_reactor_settings(), + } class CustomSettingsSpider(DefaultSpider): custom_settings = spider_settings @@ -581,7 +587,7 @@ class NoRequestsSpider(scrapy.Spider): @pytest.mark.usefixtures("reactor_pytest") class TestCrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): - return CrawlerRunner() + return CrawlerRunner(get_reactor_settings()) @inlineCallbacks def test_crawler_runner_bootstrap_successful(self): @@ -626,13 +632,7 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): @inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): - if self.reactor_pytest == "asyncio": - CrawlerRunner( - settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - } - ) - else: + if self.reactor_pytest == "default": runner = CrawlerRunner( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", @@ -643,6 +643,12 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)", ): yield runner.crawl(NoRequestsSpider) + else: + CrawlerRunner( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + ) class ScriptRunnerMixin: @@ -672,7 +678,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): assert "Spider closed (finished)" in log assert ( "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log + in log ) def test_multi(self): @@ -680,18 +686,17 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): assert "Spider closed (finished)" in log assert ( "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log + in log ) assert "ReactorAlreadyInstalledError" not in log def test_reactor_default(self): log = self.run_script("reactor_default.py") - assert "Spider closed (finished)" in log + assert "Spider closed (finished)" not in log assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log - ) - assert "ReactorAlreadyInstalledError" not in log + "does not match the requested one " + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + ) in log def test_reactor_default_twisted_reactor_select(self): log = self.run_script("reactor_default_twisted_reactor_select.py") @@ -716,8 +721,11 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_reactor_select(self): log = self.run_script("reactor_select.py") - assert "Spider closed (finished)" in log - assert "ReactorAlreadyInstalledError" not in log + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + ) in log def test_reactor_select_twisted_reactor_select(self): log = self.run_script("reactor_select_twisted_reactor_select.py") diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 162747581..c2df67c66 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -33,7 +33,7 @@ class TestScrapyUtils: tox_config_file_path = Path(__file__).parent / ".." / "tox.ini" config_parser = ConfigParser() config_parser.read(tox_config_file_path) - pattern = r"Twisted\[http2\]==([\d.]+)" + pattern = r"Twisted==([\d.]+)" match = re.search(pattern, config_parser["pinned"]["deps"]) pinned_twisted_version_string = match[1] diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 19bd02498..bc18e76e1 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -307,7 +307,7 @@ class TestHttp(unittest.TestCase, ABC): @defer.inlineCallbacks def test_timeout_download_from_spider_nodata_rcvd(self): - if self.reactor_pytest == "asyncio" and sys.platform == "win32": + if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 raise unittest.SkipTest( "This test produces DirtyReactorAggregateError on Windows with asyncio" @@ -322,7 +322,7 @@ class TestHttp(unittest.TestCase, ABC): @defer.inlineCallbacks def test_timeout_download_from_spider_server_hangs(self): - if self.reactor_pytest == "asyncio" and sys.platform == "win32": + if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 raise unittest.SkipTest( "This test produces DirtyReactorAggregateError on Windows with asyncio" @@ -1136,7 +1136,7 @@ class TestFTPBase(unittest.TestCase): class TestFTP(TestFTPBase): def test_invalid_credentials(self): - if self.reactor_pytest == "asyncio" and sys.platform == "win32": + if self.reactor_pytest != "default" and sys.platform == "win32": raise unittest.SkipTest( "This test produces DirtyReactorAggregateError on Windows with asyncio" ) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 15b3ad5af..4fca9eefb 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -64,7 +64,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_delay(self): - crawler = CrawlerRunner().create_crawler(DownloaderSlotsSettingsTestSpider) + crawler = get_crawler(DownloaderSlotsSettingsTestSpider) yield crawler.crawl(mockserver=self.mockserver) slots = crawler.engine.downloader.slots times = crawler.spider.times diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index ca5ffdc26..85bd42857 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -1,9 +1,11 @@ -import datetime -import typing -import unittest +from __future__ import annotations + +import datetime +import unittest +from typing import Any, Callable -from scrapy.crawler import Crawler from scrapy.extensions.periodic_log import PeriodicLog +from scrapy.utils.test import get_crawler from .spiders import MetaSpider @@ -59,9 +61,8 @@ class CustomPeriodicLog(PeriodicLog): self.stats._stats = stats_dump_2 -def extension(settings=None): - crawler = Crawler(MetaSpider, settings=settings) - crawler._apply_settings() +def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog: + crawler = get_crawler(MetaSpider, settings) return CustomPeriodicLog.from_crawler(crawler) @@ -94,7 +95,7 @@ class TestPeriodicLog(unittest.TestCase): ext.spider_closed(spider, reason="finished") return ext, a, b - def check(settings: dict, condition: typing.Callable): + def check(settings: dict[str, Any], condition: Callable) -> None: ext, a, b = emulate(settings) assert list(a["delta"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) @@ -151,7 +152,7 @@ class TestPeriodicLog(unittest.TestCase): ext.spider_closed(spider, reason="finished") return ext, a, b - def check(settings: dict, condition: typing.Callable): + def check(settings: dict[str, Any], condition: Callable) -> None: ext, a, b = emulate(settings) assert list(a["stats"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 162dfdaf4..c5f1b6321 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -3,18 +3,22 @@ from __future__ import annotations import shutil from pathlib import Path from tempfile import mkdtemp +from typing import TYPE_CHECKING, Any from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase from w3lib.url import add_or_replace_parameter -from scrapy import signals -from scrapy.crawler import CrawlerRunner +from scrapy import Spider, signals from scrapy.utils.misc import load_object +from scrapy.utils.test import get_crawler from tests.mockserver import MockServer from tests.spiders import SimpleSpider +if TYPE_CHECKING: + from scrapy.crawler import Crawler + class MediaDownloadSpider(SimpleSpider): name = "mediadownload" @@ -80,7 +84,6 @@ class TestFileDownloadCrawl(TestCase): "ITEM_PIPELINES": {self.pipeline_class: 1}, self.store_setting_key: str(self.tmpmediastore), } - self.runner = CrawlerRunner(self.settings) self.items = [] def tearDown(self): @@ -90,10 +93,12 @@ class TestFileDownloadCrawl(TestCase): def _on_item_scraped(self, item): self.items.append(item) - def _create_crawler(self, spider_class, runner=None, **kwargs): - if runner is None: - runner = self.runner - crawler = runner.create_crawler(spider_class, **kwargs) + def _create_crawler( + self, spider_class: type[Spider], settings: dict[str, Any] | None = None + ) -> Crawler: + if settings is None: + settings = self.settings + crawler = get_crawler(spider_class, settings) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) return crawler @@ -175,10 +180,11 @@ class TestFileDownloadCrawl(TestCase): @defer.inlineCallbacks def test_download_media_redirected_allowed(self): - settings = dict(self.settings) - settings.update({"MEDIA_ALLOW_REDIRECTS": True}) - runner = CrawlerRunner(settings) - crawler = self._create_crawler(RedirectedMediaDownloadSpider, runner=runner) + settings = { + **self.settings, + "MEDIA_ALLOW_REDIRECTS": True, + } + crawler = self._create_crawler(RedirectedMediaDownloadSpider, settings) with LogCapture() as log: yield crawler.crawl( self.mockserver.url("/files/images/"), @@ -201,8 +207,7 @@ class TestFileDownloadCrawl(TestCase): **self.settings, "ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1}, } - runner = CrawlerRunner(settings) - crawler = self._create_crawler(MediaDownloadSpider, runner=runner) + crawler = self._create_crawler(MediaDownloadSpider, settings) with LogCapture() as log: yield crawler.crawl( self.mockserver.url("/files/images/"), diff --git a/tests/test_spider.py b/tests/test_spider.py index 4e8330c06..aaf72390d 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -27,7 +27,7 @@ from scrapy.spiders import ( XMLFeedSpider, ) from scrapy.spiders.init import InitSpider -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings from tests import get_testdata, tests_datadir @@ -108,7 +108,11 @@ class TestSpider(unittest.TestCase): @inlineCallbacks def test_settings_in_from_crawler(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = {"TEST1": "project", "TEST3": "project"} + project_settings = { + "TEST1": "project", + "TEST3": "project", + **get_reactor_settings(), + } class TestSpider(self.spider_class): name = "test" diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index a65a36219..901e03d59 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -2,6 +2,7 @@ import asyncio import warnings import pytest +from twisted.trial.unittest import TestCase from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.reactor import ( @@ -12,10 +13,10 @@ from scrapy.utils.reactor import ( @pytest.mark.usefixtures("reactor_pytest") -class TestAsyncio: +class TestAsyncio(TestCase): def test_is_asyncio_reactor_installed(self): # the result should depend only on the pytest --reactor argument - assert is_asyncio_reactor_installed() == (self.reactor_pytest == "asyncio") + assert is_asyncio_reactor_installed() == (self.reactor_pytest != "default") def test_install_asyncio_reactor(self): from twisted.internet import reactor as original_reactor diff --git a/tox.ini b/tox.ini index 70c841603..eb084f0f5 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,7 @@ deps = {[test-requirements]deps} # mitmproxy does not support PyPy - mitmproxy; implementation_name != 'pypy' + mitmproxy; implementation_name != "pypy" setenv = COVERAGE_CORE=sysmon passenv = @@ -96,19 +96,18 @@ commands = [pinned] basepython = python3.9 deps = + Protego==0.1.15 + Twisted==21.7.0 cryptography==37.0.0 cssselect==0.9.1 - h2==3.0 itemadapter==0.1.0 + lxml==4.6.0 parsel==1.5.0 - Protego==0.1.15 pyOpenSSL==22.0.0 queuelib==1.4.2 service_identity==18.1.0 - Twisted[http2]==21.7.0 w3lib==1.17.0 zope.interface==5.1.0 - lxml==4.6.0 {[test-requirements]deps} # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies @@ -131,60 +130,50 @@ setenv = {[pinned]setenv} commands = {[pinned]commands} -[testenv:windows-pinned] -basepython = {[pinned]basepython} -deps = - {[pinned]deps} - PyDispatcher==2.0.5 -install_command = {[pinned]install_command} -setenv = - {[pinned]setenv} -commands = {[pinned]commands} - [testenv:extra-deps] basepython = python3 deps = {[testenv]deps} - boto3 - google-cloud-storage - robotexclusionrulesparser Pillow Twisted[http2] - uvloop; platform_system != "Windows" + boto3 bpython # optional for shell wrapper tests - brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests - brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests - zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests + brotli; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests + brotlicffi; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests + google-cloud-storage ipython + robotexclusionrulesparser + uvloop; platform_system != "Windows" + zstandard; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests [testenv:extra-deps-pinned] basepython = {[pinned]basepython} deps = {[pinned]deps} + Pillow==8.0.0 boto3==1.20.0 - google-cloud-storage==1.29.0 - Pillow==7.1.0 - robotexclusionrulesparser==1.6.2 - brotlipy - uvloop==0.14.0; platform_system != "Windows" bpython==0.7.1 - zstandard==0.1; implementation_name != 'pypy' + brotli==0.5.2; implementation_name != "pypy" + brotlicffi==0.8.0; implementation_name == "pypy" + brotlipy + google-cloud-storage==1.29.0 ipython==2.0.0 - brotli==0.5.2; implementation_name != 'pypy' - brotlicffi==0.8.0; implementation_name == 'pypy' + robotexclusionrulesparser==1.6.2 + uvloop==0.14.0; platform_system != "Windows" + zstandard==0.1; implementation_name != "pypy" install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands = {[pinned]commands} -[testenv:asyncio] +[testenv:default-reactor] commands = - {[testenv]commands} --reactor=asyncio + {[testenv]commands} --reactor=default -[testenv:asyncio-pinned] +[testenv:default-reactor-pinned] basepython = {[pinned]basepython} deps = {[testenv:pinned]deps} -commands = {[pinned]commands} --reactor=asyncio +commands = {[pinned]commands} --reactor=default install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -204,21 +193,20 @@ commands = {[testenv:pypy3]commands} [testenv:pypy3-pinned] basepython = pypy3.10 deps = + PyPyDispatcher==2.1.0 + {[test-requirements]deps} + Protego==0.1.15 + Twisted==21.7.0 cryptography==41.0.5 cssselect==0.9.1 - h2==3.1 itemadapter==0.1.0 + lxml==4.6.0 parsel==1.5.0 - Protego==0.1.15 pyOpenSSL==23.3.0 queuelib==1.4.2 service_identity==18.1.0 - Twisted[http2]==21.7.0 w3lib==1.17.0 zope.interface==5.1.0 - lxml==4.6.0 - {[test-requirements]deps} - PyPyDispatcher==2.1.0 commands = ; disabling both coverage and docs tests pytest {posargs:--durations=10 scrapy tests} From fc566a7ff9913be5fce2b961a6599f861e1ef59a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Mar 2025 23:46:17 +0400 Subject: [PATCH 082/491] Don't install the reactor in is_asyncio_reactor_installed(). (#6732) --- docs/topics/asyncio.rst | 2 ++ scrapy/utils/reactor.py | 13 ++++++++ .../asyncio_enabled_no_reactor.py | 8 +++++ .../CrawlerProcess/asyncio_enabled_reactor.py | 33 ++++++++++++++++++- tests/test_crawler.py | 2 ++ 5 files changed, 57 insertions(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 35afdc11b..b61a6e4a8 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -116,6 +116,8 @@ example: f"of Scrapy for more information." ) +.. autofunction:: scrapy.utils.reactor.is_asyncio_reactor_installed + .. _asyncio-windows: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 679e38206..099c81f0e 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -175,7 +175,20 @@ def verify_installed_asyncio_event_loop(loop_path: str) -> None: ) +def is_reactor_installed() -> bool: + return "twisted.internet.reactor" in sys.modules + + def is_asyncio_reactor_installed() -> bool: + """Check whether the installed reactor is :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. + + Raise a :exc:`RuntimeError` if no reactor is installed. + """ + if not is_reactor_installed(): + raise RuntimeError( + "is_asyncio_reactor_installed() called without an installed reactor." + ) + from twisted.internet import reactor return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py index 6df6d76fa..6f82cf589 100644 --- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py @@ -1,5 +1,12 @@ import scrapy from scrapy.crawler import CrawlerProcess +from scrapy.utils.reactor import is_asyncio_reactor_installed + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") class NoRequestsSpider(scrapy.Spider): @@ -12,6 +19,7 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, } ) process.crawl(NoRequestsSpider) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index 0c380610d..a8bf1bc3c 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,9 +1,39 @@ import scrapy from scrapy.crawler import CrawlerProcess -from scrapy.utils.reactor import install_reactor +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + is_reactor_installed, +) + +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed before is_asyncio_reactor_installed()." + ) + +try: + is_asyncio_reactor_installed() +except RuntimeError: + pass +else: + raise RuntimeError("is_asyncio_reactor_installed() did not raise RuntimeError.") + +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed after is_asyncio_reactor_installed()." + ) install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +if not is_asyncio_reactor_installed(): + raise RuntimeError("Wrong reactor installed after install_reactor().") + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") + class NoRequestsSpider(scrapy.Spider): name = "no_request" @@ -15,6 +45,7 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, } ) process.crawl(NoRequestsSpider) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 98352b66e..6c465f000 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -747,6 +747,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" in log ) + assert "RuntimeError" not in log def test_asyncio_enabled_reactor(self): log = self.run_script("asyncio_enabled_reactor.py") @@ -755,6 +756,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" in log ) + assert "RuntimeError" not in log @pytest.mark.skipif( parse_version(w3lib_version) >= parse_version("2.0.0"), From 9057bf4e1e08dccac4fa6d9f0f191d1f4708a43a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 15 Mar 2025 14:47:16 +0400 Subject: [PATCH 083/491] More docs about Deferred<->Future interoperability. (#6734) --- docs/topics/asyncio.rst | 24 ++++++++++++++++-------- docs/topics/coroutines.rst | 24 ++++++++++++++++++++++++ scrapy/utils/defer.py | 9 +++++---- 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index b61a6e4a8..0490129b3 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -72,24 +72,32 @@ those imports happen. .. _asyncio-await-dfd: -Awaiting on Deferreds -===================== +Integrating Deferred code and asyncio code +========================================== -When the asyncio reactor isn't installed, you can await on Deferreds in the -coroutines directly. When it is installed, this is not possible anymore, due to -specifics of the Scrapy coroutine integration (the coroutines are wrapped into -:class:`asyncio.Future` objects, not into -:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into -Futures. Scrapy provides two helpers for this: +Coroutine functions can await on Deferreds by wrapping them into +:class:`asyncio.Future` objects. Scrapy provides two helpers for this: .. autofunction:: scrapy.utils.defer.deferred_to_future .. autofunction:: scrapy.utils.defer.maybe_deferred_to_future + +.. tip:: If you don't need to support reactors other than the default + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, you + can use :func:`~scrapy.utils.defer.deferred_to_future`, otherwise you + should use :func:`~scrapy.utils.defer.maybe_deferred_to_future`. + .. tip:: If you need to use these functions in code that aims to be compatible with lower versions of Scrapy that do not provide these functions, down to Scrapy 2.0 (earlier versions do not support :mod:`asyncio`), you can copy the implementation of these functions into your own code. +Coroutines and futures can be wrapped into Deferreds (for example, when a +Scrapy API requires passing a Deferred to it) using the following helpers: + +.. autofunction:: scrapy.utils.defer.deferred_from_coro +.. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f + .. _enforce-asyncio-requirement: diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 57aa3a62d..1c80857f6 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -9,6 +9,7 @@ Coroutines Scrapy has :ref:`partial support ` for the :ref:`coroutine syntax `. + .. _coroutine-support: Supported callables @@ -51,6 +52,29 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): .. versionadded:: 2.7 + +.. _coroutine-deferred-apis: + +Using Deferred-based APIs +========================= + +In addition to native coroutine APIs Scrapy has some APIs that return a +:class:`~twisted.internet.defer.Deferred` object or take a user-supplied +function that returns a :class:`~twisted.internet.defer.Deferred` object. These +APIs are also asynchronous but don't yet support native ``async def`` syntax. +For example: + +- The :meth:`ExecutionEngine.download` method returns a + :class:`~twisted.internet.defer.Deferred` object. +- A custom download handler needs to define a ``download_request()`` method that + returns a :class:`~twisted.internet.defer.Deferred` object. + +In most cases you can use these APIs in code that otherwise uses coroutines, by +wrapping a :class:`~twisted.internet.defer.Deferred` object into a +:class:`~asyncio.Future` object or vice versa. See :ref:`asyncio-await-dfd` for +more information about this. + + General usage ============= diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 8f52836c4..42ad28d8d 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -362,7 +362,8 @@ def deferred_from_coro(o: _T) -> _T: ... def deferred_from_coro(o: _T) -> Deferred | _T: - """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" + """Converts a coroutine or other awaitable object into a Deferred, + or returns the object as is if it isn't a coroutine.""" if isinstance(o, Deferred): return o if asyncio.isfuture(o) or inspect.isawaitable(o): @@ -442,12 +443,12 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: What you can await in Scrapy callables defined as coroutines depends on the value of :setting:`TWISTED_REACTOR`: - - When not using the asyncio reactor, you can only await on - :class:`~twisted.internet.defer.Deferred` objects. - - When :ref:`using the asyncio reactor `, you can only await on :class:`asyncio.Future` objects. + - When not using the asyncio reactor, you can only await on + :class:`~twisted.internet.defer.Deferred` objects. + If you want to write code that uses ``Deferred`` objects but works with any reactor, use this function on all ``Deferred`` objects:: From 872924721344502cca92a72ce68d5f78a9b9e5bc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 15 Mar 2025 23:33:10 +0400 Subject: [PATCH 084/491] Fix running simple tests with --reactor=default. (#6735) --- conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conftest.py b/conftest.py index 9999e41d2..8e0c429a0 100644 --- a/conftest.py +++ b/conftest.py @@ -119,6 +119,9 @@ def requires_boto3(request): def pytest_configure(config): if config.getoption("--reactor") != "default": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + else: + # install the reactor explicitly + from twisted.internet import reactor # noqa: F401 # Generate localhost certificate files, needed by some tests From 2ee01efe496db9ff8506ca37fe2571f5f4ac2849 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 20 Mar 2025 07:49:44 -0300 Subject: [PATCH 085/491] feat: Add count to spider_exceptions stats (#6740) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add overall exception_count to spider_exceptions stats * Remove variable * Update test_closespider.py * Update test_closespider.py * Rename exception_count → count --- scrapy/core/scraper.py | 1 + tests/test_closespider.py | 1 + 2 files changed, 2 insertions(+) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index b664b61f6..496adb500 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -246,6 +246,7 @@ class Scraper: spider=spider, ) assert self.crawler.stats + self.crawler.stats.inc_value("spider_exceptions/count", spider=spider) self.crawler.stats.inc_value( f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider ) diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 476662789..4a17b254b 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -88,6 +88,7 @@ class TestCloseSpider(TestCase): assert reason == "closespider_errorcount" key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}" errorcount = crawler.stats.get_value(key) + assert crawler.stats.get_value("spider_exceptions/count") >= close_on assert errorcount >= close_on @defer.inlineCallbacks From 3ca882fba86750199c2f41ef24b5495a4afa7988 Mon Sep 17 00:00:00 2001 From: Mehraz Hossain Rumman <59512321+MehrazRumman@users.noreply.github.com> Date: Thu, 20 Mar 2025 18:02:10 +0600 Subject: [PATCH 086/491] Syntax Error Fixed (#6738) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Syntax error fix issue #6731 * test case added * extra logic removed * mock spider fixture * Update scrapy/utils/misc.py Co-authored-by: Adrián Chaves * settings.rst updated * settings.rst updated * settings.rst updated --------- Co-authored-by: Adrián Chaves --- docs/topics/settings.rst | 15 +++ scrapy/settings/default_settings.py | 2 + scrapy/utils/misc.py | 2 + ...t_return_with_argument_inside_generator.py | 110 +++++++++++++----- 4 files changed, 98 insertions(+), 31 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index ca0af569f..a59a61050 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2047,6 +2047,21 @@ also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and there is no overriding User-Agent header specified for the request. +.. setting:: WARN_ON_GENERATOR_RETURN_VALUE + +WARN_ON_GENERATOR_RETURN_VALUE +------------------------------ + +Default: ``True`` + +When enabled, Scrapy will warn if generator-based callback methods (like +``parse``) contain return statements with non-``None`` values. This helps detect +potential mistakes in spider development. + +Disable this setting to prevent syntax errors that may occur when dynamically +modifying generator function source code during runtime, skip AST parsing of +callback functions, or improve performance in auto-reloading development +environments. Settings documented elsewhere: ------------------------------ diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 645e50301..680fded7a 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -351,3 +351,5 @@ SPIDER_CONTRACTS_BASE = { "scrapy.contracts.default.ReturnsContract": 2, "scrapy.contracts.default.ScrapesContract": 3, } + +WARN_ON_GENERATOR_RETURN_VALUE = True diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index d319e7950..b7b436260 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -286,6 +286,8 @@ def warn_on_generator_with_return_value( Logs a warning if a callable is a generator function and includes a 'return' statement with a value different than None """ + if not spider.settings.getbool("WARN_ON_GENERATOR_RETURN_VALUE"): + return try: if is_generator_with_return_value(callable): warnings.warn( diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 81a83c3d7..ad31e5185 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -2,6 +2,8 @@ import warnings from functools import partial from unittest import mock +import pytest + from scrapy.utils.misc import ( is_generator_with_return_value, warn_on_generator_with_return_value, @@ -40,7 +42,24 @@ def generator_that_returns_stuff(): class TestUtilsMisc: - def test_generators_return_something(self): + @pytest.fixture + def mock_spider(self): + class MockSettings: + def __init__(self, settings_dict=None): + self.settings_dict = settings_dict or { + "WARN_ON_GENERATOR_RETURN_VALUE": True + } + + def getbool(self, name, default=False): + return self.settings_dict.get(name, default) + + class MockSpider: + def __init__(self): + self.settings = MockSettings() + + return MockSpider() + + def test_generators_return_something(self, mock_spider): def f1(): yield 1 return 2 @@ -75,30 +94,30 @@ https://example.org assert is_generator_with_return_value(i1) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, top_level_return_something) + warn_on_generator_with_return_value(mock_spider, top_level_return_something) assert len(w) == 1 assert ( - 'The "NoneType.top_level_return_something" method is a generator' + 'The "MockSpider.top_level_return_something" method is a generator' in str(w[0].message) ) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, f1) + warn_on_generator_with_return_value(mock_spider, f1) assert len(w) == 1 - assert 'The "NoneType.f1" method is a generator' in str(w[0].message) + assert 'The "MockSpider.f1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, g1) + warn_on_generator_with_return_value(mock_spider, g1) assert len(w) == 1 - assert 'The "NoneType.g1" method is a generator' in str(w[0].message) + assert 'The "MockSpider.g1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, h1) + warn_on_generator_with_return_value(mock_spider, h1) assert len(w) == 1 - assert 'The "NoneType.h1" method is a generator' in str(w[0].message) + assert 'The "MockSpider.h1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, i1) + warn_on_generator_with_return_value(mock_spider, i1) assert len(w) == 1 - assert 'The "NoneType.i1" method is a generator' in str(w[0].message) + assert 'The "MockSpider.i1" method is a generator' in str(w[0].message) - def test_generators_return_none(self): + def test_generators_return_none(self, mock_spider): def f2(): yield 1 @@ -142,31 +161,31 @@ https://example.org assert not is_generator_with_return_value(l2) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, top_level_return_none) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, f2) + warn_on_generator_with_return_value(mock_spider, f2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, g2) + warn_on_generator_with_return_value(mock_spider, g2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, h2) + warn_on_generator_with_return_value(mock_spider, h2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, i2) + warn_on_generator_with_return_value(mock_spider, i2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, j2) + warn_on_generator_with_return_value(mock_spider, j2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, k2) + warn_on_generator_with_return_value(mock_spider, k2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, l2) + warn_on_generator_with_return_value(mock_spider, l2) assert len(w) == 0 - def test_generators_return_none_with_decorator(self): + def test_generators_return_none_with_decorator(self, mock_spider): def decorator(func): def inner_func(): func() @@ -223,36 +242,36 @@ https://example.org assert not is_generator_with_return_value(l3) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, top_level_return_none) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, f3) + warn_on_generator_with_return_value(mock_spider, f3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, g3) + warn_on_generator_with_return_value(mock_spider, g3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, h3) + warn_on_generator_with_return_value(mock_spider, h3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, i3) + warn_on_generator_with_return_value(mock_spider, i3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, j3) + warn_on_generator_with_return_value(mock_spider, j3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, k3) + warn_on_generator_with_return_value(mock_spider, k3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, l3) + warn_on_generator_with_return_value(mock_spider, l3) assert len(w) == 0 @mock.patch( "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error ) - def test_indentation_error(self): + def test_indentation_error(self, mock_spider): with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, top_level_return_none) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) assert len(w) == 1 assert "Unable to determine" in str(w[0].message) @@ -262,3 +281,32 @@ https://example.org partial_cb = partial(cb, arg1=42) assert not is_generator_with_return_value(partial_cb) + + def test_warn_on_generator_with_return_value_settings_disabled(self): + class MockSettings: + def __init__(self, settings_dict=None): + self.settings_dict = settings_dict or {} + + def getbool(self, name, default=False): + return self.settings_dict.get(name, default) + + class MockSpider: + def __init__(self): + self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False}) + + spider = MockSpider() + + def gen_with_return(): + yield 1 + return "value" + + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(spider, gen_with_return) + assert len(w) == 0 + + spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True + + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(spider, gen_with_return) + assert len(w) == 1 + assert "is a generator" in str(w[0].message) From e50914e0f5b98ee4c9cb1f182ec2cd684fdf9900 Mon Sep 17 00:00:00 2001 From: Suejung Shin Date: Fri, 21 Mar 2025 04:28:47 -0700 Subject: [PATCH 087/491] Codecov: Add test analytics (#6741) --- .github/workflows/tests-macos.yml | 4 ++++ .github/workflows/tests-ubuntu.yml | 4 ++++ .github/workflows/tests-windows.yml | 4 ++++ .gitignore | 1 + tox.ini | 8 ++++---- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index ce0e1a6c2..d740808cc 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -33,3 +33,7 @@ jobs: - name: Upload coverage report uses: codecov/codecov-action@v5 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 444aa3557..34819f227 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -88,3 +88,7 @@ jobs: - name: Upload coverage report uses: codecov/codecov-action@v5 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 537a01e29..bbbb704e5 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -64,3 +64,7 @@ jobs: - name: Upload coverage report uses: codecov/codecov-action@v5 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 diff --git a/.gitignore b/.gitignore index 6c5c50e08..0a3f0ac1c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ htmlcov/ .pytest_cache/ .coverage.* coverage.* +*.junit.xml test-output.* .cache/ .mypy_cache/ diff --git a/tox.ini b/tox.ini index eb084f0f5..59572442d 100644 --- a/tox.ini +++ b/tox.ini @@ -39,7 +39,7 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --durations=10 docs scrapy tests --doctest-modules} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=testenv.junit.xml -o junit_family=legacy --durations=10 docs scrapy tests --doctest-modules} install_command = python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} @@ -118,7 +118,7 @@ install_command = python -I -m pip install {opts} {packages} commands = ; tests for docs fail with parsel < 1.8.0 - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --durations=10 scrapy tests} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=pinned.junit.xml -o junit_family=legacy --durations=10 scrapy tests} [testenv:pinned] basepython = {[pinned]basepython} @@ -254,7 +254,7 @@ deps = {[testenv]deps} botocore>=1.4.87 commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests -m requires_botocore} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy -m requires_botocore} [testenv:botocore-pinned] basepython = {[pinned]basepython} @@ -265,4 +265,4 @@ install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests -m requires_botocore} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore-pinned.junit.xml -o junit_family=legacy -m requires_botocore} From 9f99da8f865efff11f6c8736567b8fbd4413091c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 24 Mar 2025 13:26:25 +0500 Subject: [PATCH 088/491] Convert test_downloadermiddleware_robotstxt.py from callbacks to awaits. (#6743) --- tests/test_downloadermiddleware_robotstxt.py | 168 +++++++++---------- 1 file changed, 76 insertions(+), 92 deletions(-) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index ad335f852..9518f1835 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,9 +1,11 @@ -from typing import Any +from __future__ import annotations + +from typing import TYPE_CHECKING from unittest import mock import pytest from twisted.internet import error, reactor -from twisted.internet.defer import Deferred, DeferredList, maybeDeferred +from twisted.internet.defer import Deferred, maybeDeferred from twisted.python import failure from twisted.trial import unittest @@ -13,8 +15,12 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from tests.test_robotstxt_interface import rerp_available +if TYPE_CHECKING: + from scrapy.crawler import Crawler + class TestRobotsTxtMiddleware(unittest.TestCase): def setUp(self): @@ -31,7 +37,7 @@ class TestRobotsTxtMiddleware(unittest.TestCase): with pytest.raises(NotConfigured): RobotsTxtMiddleware(self.crawler) - def _get_successful_crawler(self): + def _get_successful_crawler(self) -> Crawler: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) ROBOTS = """ @@ -54,54 +60,41 @@ Disallow: /some/randome/page.html crawler.engine.download.side_effect = return_response return crawler - def test_robotstxt(self): + @deferred_f_from_coro_f + async def test_robotstxt(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) - return DeferredList( - [ - self.assertNotIgnored(Request("http://site.local/allowed"), middleware), - maybeDeferred(self.assertRobotsTxtRequested, "http://site.local"), - self.assertIgnored(Request("http://site.local/admin/main"), middleware), - self.assertIgnored(Request("http://site.local/static/"), middleware), - self.assertIgnored( - Request("http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:"), - middleware, - ), - self.assertIgnored( - Request("http://site.local/wiki/Käyttäjä:"), middleware - ), - ], - fireOnOneErrback=True, + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + self.assertRobotsTxtRequested("http://site.local") + await self.assertIgnored(Request("http://site.local/admin/main"), middleware) + await self.assertIgnored(Request("http://site.local/static/"), middleware) + await self.assertIgnored( + Request("http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:"), middleware + ) + await self.assertIgnored( + Request("http://site.local/wiki/Käyttäjä:"), middleware ) - def test_robotstxt_ready_parser(self): + @deferred_f_from_coro_f + async def test_robotstxt_ready_parser(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) - d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware) - d.addCallback( - lambda _: self.assertNotIgnored( - Request("http://site.local/allowed"), middleware - ) - ) - return d + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) - def test_robotstxt_meta(self): + @deferred_f_from_coro_f + async def test_robotstxt_meta(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) meta = {"dont_obey_robotstxt": True} - return DeferredList( - [ - self.assertNotIgnored( - Request("http://site.local/allowed", meta=meta), middleware - ), - self.assertNotIgnored( - Request("http://site.local/admin/main", meta=meta), middleware - ), - self.assertNotIgnored( - Request("http://site.local/static/", meta=meta), middleware - ), - ], - fireOnOneErrback=True, + await self.assertNotIgnored( + Request("http://site.local/allowed", meta=meta), middleware + ) + await self.assertNotIgnored( + Request("http://site.local/admin/main", meta=meta), middleware + ) + await self.assertNotIgnored( + Request("http://site.local/static/", meta=meta), middleware ) - def _get_garbage_crawler(self): + def _get_garbage_crawler(self) -> Crawler: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) response = Response( @@ -116,22 +109,16 @@ Disallow: /some/randome/page.html crawler.engine.download.side_effect = return_response return crawler - def test_robotstxt_garbage(self): + @deferred_f_from_coro_f + async def test_robotstxt_garbage(self): # garbage response should be discarded, equal 'allow all' middleware = RobotsTxtMiddleware(self._get_garbage_crawler()) - return DeferredList( - [ - self.assertNotIgnored(Request("http://site.local"), middleware), - self.assertNotIgnored(Request("http://site.local/allowed"), middleware), - self.assertNotIgnored( - Request("http://site.local/admin/main"), middleware - ), - self.assertNotIgnored(Request("http://site.local/static/"), middleware), - ], - fireOnOneErrback=True, - ) + await self.assertNotIgnored(Request("http://site.local"), middleware) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware) + await self.assertNotIgnored(Request("http://site.local/static/"), middleware) - def _get_emptybody_crawler(self): + def _get_emptybody_crawler(self) -> Crawler: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) response = Response("http://site.local/robots.txt") @@ -144,21 +131,16 @@ Disallow: /some/randome/page.html crawler.engine.download.side_effect = return_response return crawler - def test_robotstxt_empty_response(self): + @deferred_f_from_coro_f + async def test_robotstxt_empty_response(self): # empty response should equal 'allow all' middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) - return DeferredList( - [ - self.assertNotIgnored(Request("http://site.local/allowed"), middleware), - self.assertNotIgnored( - Request("http://site.local/admin/main"), middleware - ), - self.assertNotIgnored(Request("http://site.local/static/"), middleware), - ], - fireOnOneErrback=True, - ) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware) + await self.assertNotIgnored(Request("http://site.local/static/"), middleware) - def test_robotstxt_error(self): + @deferred_f_from_coro_f + async def test_robotstxt_error(self): self.crawler.settings.set("ROBOTSTXT_OBEY", True) err = error.DNSLookupError("Robotstxt address not found") @@ -171,15 +153,13 @@ Disallow: /some/randome/page.html middleware = RobotsTxtMiddleware(self.crawler) middleware._logerror = mock.MagicMock(side_effect=middleware._logerror) - deferred = middleware.process_request(Request("http://site.local"), None) + await maybe_deferred_to_future( + middleware.process_request(Request("http://site.local"), None) + ) + assert middleware._logerror.called - def check_called(_: Any) -> None: - assert middleware._logerror.called - - deferred.addCallback(check_called) - return deferred - - def test_robotstxt_immediate_error(self): + @deferred_f_from_coro_f + async def test_robotstxt_immediate_error(self): self.crawler.settings.set("ROBOTSTXT_OBEY", True) err = error.DNSLookupError("Robotstxt address not found") @@ -191,9 +171,10 @@ Disallow: /some/randome/page.html self.crawler.engine.download.side_effect = immediate_failure middleware = RobotsTxtMiddleware(self.crawler) - return self.assertNotIgnored(Request("http://site.local"), middleware) + await self.assertNotIgnored(Request("http://site.local"), middleware) - def test_ignore_robotstxt_request(self): + @deferred_f_from_coro_f + async def test_ignore_robotstxt_request(self): self.crawler.settings.set("ROBOTSTXT_OBEY", True) def ignore_request(request): @@ -206,13 +187,8 @@ Disallow: /some/randome/page.html middleware = RobotsTxtMiddleware(self.crawler) mw_module_logger.error = mock.MagicMock() - d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware) - - def check_not_called(_: Any) -> None: - assert not mw_module_logger.error.called # type: ignore[attr-defined] - - d.addCallback(check_not_called) - return d + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + assert not mw_module_logger.error.called # type: ignore[attr-defined] def test_robotstxt_user_agent_setting(self): crawler = self._get_successful_crawler() @@ -236,19 +212,27 @@ Disallow: /some/randome/page.html Deferred, ) - def assertNotIgnored(self, request, middleware): + async def assertNotIgnored( + self, request: Request, middleware: RobotsTxtMiddleware + ) -> None: spider = None # not actually used - dfd = maybeDeferred(middleware.process_request, request, spider) - dfd.addCallback(self.assertIsNone) - return dfd + result = await maybe_deferred_to_future( + maybeDeferred(middleware.process_request, request, spider) # type: ignore[call-overload] + ) + assert result is None - def assertIgnored(self, request, middleware): + async def assertIgnored( + self, request: Request, middleware: RobotsTxtMiddleware + ) -> None: spider = None # not actually used - return self.assertFailure( - maybeDeferred(middleware.process_request, request, spider), IgnoreRequest + await maybe_deferred_to_future( + self.assertFailure( + middleware.process_request(request, spider), # type: ignore[arg-type] + IgnoreRequest, + ) ) - def assertRobotsTxtRequested(self, base_url): + def assertRobotsTxtRequested(self, base_url: str) -> None: calls = self.crawler.engine.download.call_args_list request = calls[0][0][0] assert request.url == f"{base_url}/robots.txt" From daf9db72b2dc4ff8a6344fb9a0a4817fb953fdd4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 23 Apr 2025 18:29:04 +0400 Subject: [PATCH 089/491] Base class for universal spider middlewares (#6693) * Initial BaseSpiderMiddleware. * Rename the new methods. * Remove the spider argument from new BaseSpiderMiddleware methods. * Add docs for BaseSpiderMiddleware. * Silence pylint. * Add BaseSpiderMiddleware tests. * Add a release note. --- docs/news.rst | 17 ++++ docs/topics/spider-middleware.rst | 13 +++ scrapy/spidermiddlewares/base.py | 97 ++++++++++++++++++++ scrapy/spidermiddlewares/depth.py | 38 ++++---- scrapy/spidermiddlewares/offsite.py | 45 ++++------ scrapy/spidermiddlewares/referer.py | 33 +++---- scrapy/spidermiddlewares/urllength.py | 77 ++++++---------- tests/test_spidermiddleware_base.py | 120 +++++++++++++++++++++++++ tests/test_spidermiddleware_depth.py | 7 +- tests/test_spidermiddleware_offsite.py | 2 +- 10 files changed, 329 insertions(+), 120 deletions(-) create mode 100644 scrapy/spidermiddlewares/base.py create mode 100644 tests/test_spidermiddleware_base.py diff --git a/docs/news.rst b/docs/news.rst index 9a68f8852..9f476ee21 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,23 @@ Release notes ============= +.. _release-VERSION: + +Scrapy VERSION (unreleased) +--------------------------- + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The ``from_settings()`` method of + :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` is removed + without a deprecation period (this was needed because after the + introduction of the + :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware` base class and + switching built-in spider middlewares to it those middlewares need the + :class:`~scrapy.crawler.Crawler` instance at run time). Please use + ``from_crawler()`` instead. + .. _release-2.12.0: Scrapy 2.12.0 (2024-11-18) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 567a875b6..2211a822f 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -189,6 +189,19 @@ one or more of these methods: :param spider: the spider which raised the exception :type spider: :class:`~scrapy.Spider` object +Base class for custom spider middlewares +---------------------------------------- + +Scrapy provides a base class for custom spider middlewares. It's not required +to use it but it can help with simplifying middleware implementations and +reducing the amount of boilerplate code in :ref:`universal middlewares +`. + +.. module:: scrapy.spidermiddlewares.base + +.. autoclass:: BaseSpiderMiddleware + :members: + .. _topics-spider-middleware-ref: Built-in spider middleware reference diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py new file mode 100644 index 000000000..650192095 --- /dev/null +++ b/scrapy/spidermiddlewares/base.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from scrapy import Request, Spider + +if TYPE_CHECKING: + from collections.abc import AsyncIterable, Iterable + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.http import Response + + +class BaseSpiderMiddleware: + """Optional base class for spider middlewares. + + This class provides helper methods for asynchronous ``process_spider_output`` + methods. Middlewares that don't have a ``process_spider_output`` method don't need + to use it. + + You can override the + :meth:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware.get_processed_request` + method to add processing code for requests and the + :meth:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware.get_processed_item` + method to add processing code for items. These methods take a single + request or item from the spider output iterable and return a request or + item (the same or a new one), or ``None`` to remove this request or item + from the processing. + """ + + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls(crawler) + + def process_spider_output( + self, response: Response, result: Iterable[Any], spider: Spider + ) -> Iterable[Any]: + for o in result: + if isinstance(o, Request): + o = self.get_processed_request(o, response) + else: + o = self.get_processed_item(o, response) + if o is not None: + yield o + + async def process_spider_output_async( + self, response: Response, result: AsyncIterable[Any], spider: Spider + ) -> AsyncIterable[Any]: + async for o in result: + if isinstance(o, Request): + o = self.get_processed_request(o, response) + else: + o = self.get_processed_item(o, response) + if o is not None: + yield o + + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + """Return a processed request from the spider output. + + This method is called with a single request from the spider output. + It should return the same or a different request, or ``None`` to + ignore it. + + :param request: the input request + :type request: :class:`~scrapy.Request` object + + :param response: the response being processed + :type response: :class:`~scrapy.http.Response` object + + :return: the processed request or ``None`` + """ + return request + + def get_processed_item(self, item: Any, response: Response) -> Any: + """Return a processed item from the spider output. + + This method is called with a single item from the spider output. + It should return the same or a different item, or ``None`` to + ignore it. + + :param item: the input item + :type item: item object + + :param response: the response being processed + :type response: :class:`~scrapy.http.Response` object + + :return: the processed item or ``None`` + """ + return item diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 3164c1c03..65905f483 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -9,7 +9,7 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any -from scrapy.http import Request, Response +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware if TYPE_CHECKING: from collections.abc import AsyncIterable, Iterable @@ -19,14 +19,17 @@ if TYPE_CHECKING: from scrapy import Spider from scrapy.crawler import Crawler + from scrapy.http import Request, Response from scrapy.statscollectors import StatsCollector logger = logging.getLogger(__name__) -class DepthMiddleware: - def __init__( +class DepthMiddleware(BaseSpiderMiddleware): + crawler: Crawler + + def __init__( # pylint: disable=super-init-not-called self, maxdepth: int, stats: StatsCollector, @@ -45,21 +48,22 @@ class DepthMiddleware: verbose = settings.getbool("DEPTH_STATS_VERBOSE") prio = settings.getint("DEPTH_PRIORITY") assert crawler.stats - return cls(maxdepth, crawler.stats, verbose, prio) + o = cls(maxdepth, crawler.stats, verbose, prio) + o.crawler = crawler + return o def process_spider_output( self, response: Response, result: Iterable[Any], spider: Spider ) -> Iterable[Any]: self._init_depth(response, spider) - return (r for r in result if self._filter(r, response, spider)) + yield from super().process_spider_output(response, result, spider) async def process_spider_output_async( self, response: Response, result: AsyncIterable[Any], spider: Spider ) -> AsyncIterable[Any]: self._init_depth(response, spider) - async for r in result: - if self._filter(r, response, spider): - yield r + async for o in super().process_spider_output_async(response, result, spider): + yield o def _init_depth(self, response: Response, spider: Spider) -> None: # base case (depth=0) @@ -68,9 +72,9 @@ class DepthMiddleware: if self.verbose_stats: self.stats.inc_value("request_depth_count/0", spider=spider) - def _filter(self, request: Any, response: Response, spider: Spider) -> bool: - if not isinstance(request, Request): - return True + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: depth = response.meta["depth"] + 1 request.meta["depth"] = depth if self.prio: @@ -79,10 +83,12 @@ class DepthMiddleware: logger.debug( "Ignoring link (depth > %(maxdepth)d): %(requrl)s ", {"maxdepth": self.maxdepth, "requrl": request.url}, - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) - return False + return None if self.verbose_stats: - self.stats.inc_value(f"request_depth_count/{depth}", spider=spider) - self.stats.max_value("request_depth_max", depth, spider=spider) - return True + self.stats.inc_value( + f"request_depth_count/{depth}", spider=self.crawler.spider + ) + self.stats.max_value("request_depth_max", depth, spider=self.crawler.spider) + return request diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 646beb911..0918c9fac 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -9,11 +9,11 @@ from __future__ import annotations import logging import re import warnings -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING from scrapy import Spider, signals from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.http import Request, Response +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware from scrapy.utils.httpobj import urlparse_cached warnings.warn( @@ -23,61 +23,52 @@ warnings.warn( ) if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable - # typing.Self requires Python 3.11 from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http import Request, Response from scrapy.statscollectors import StatsCollector logger = logging.getLogger(__name__) -class OffsiteMiddleware: - def __init__(self, stats: StatsCollector): +class OffsiteMiddleware(BaseSpiderMiddleware): + crawler: Crawler + + def __init__(self, stats: StatsCollector): # pylint: disable=super-init-not-called self.stats: StatsCollector = stats @classmethod def from_crawler(cls, crawler: Crawler) -> Self: assert crawler.stats o = cls(crawler.stats) + o.crawler = crawler crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o - def process_spider_output( - self, response: Response, result: Iterable[Any], spider: Spider - ) -> Iterable[Any]: - return (r for r in result if self._filter(r, spider)) - - async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: - async for r in result: - if self._filter(r, spider): - yield r - - def _filter(self, request: Any, spider: Spider) -> bool: - if not isinstance(request, Request): - return True + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + assert self.crawler.spider if ( request.dont_filter or request.meta.get("allow_offsite") - or self.should_follow(request, spider) + or self.should_follow(request, self.crawler.spider) ): - return True + return request domain = urlparse_cached(request).hostname if domain and domain not in self.domains_seen: self.domains_seen.add(domain) logger.debug( "Filtered offsite request to %(domain)r: %(request)s", {"domain": domain, "request": request}, - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) - self.stats.inc_value("offsite/domains", spider=spider) - self.stats.inc_value("offsite/filtered", spider=spider) - return False + self.stats.inc_value("offsite/domains", spider=self.crawler.spider) + self.stats.inc_value("offsite/filtered", spider=self.crawler.spider) + return None def should_follow(self, request: Request, spider: Spider) -> bool: regex = self.host_regex diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a3a1e5b92..b2ba8ba8c 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -6,7 +6,7 @@ originated it. from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, cast from urllib.parse import urlparse from w3lib.url import safe_url_string @@ -14,13 +14,12 @@ from w3lib.url import safe_url_string from scrapy import Spider, signals from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware from scrapy.utils.misc import load_object from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable - # typing.Self requires Python 3.11 from typing_extensions import Self @@ -327,8 +326,8 @@ def _load_policy_class( return None -class RefererMiddleware: - def __init__(self, settings: BaseSettings | None = None): +class RefererMiddleware(BaseSpiderMiddleware): + def __init__(self, settings: BaseSettings | None = None): # pylint: disable=super-init-not-called self.default_policy: type[ReferrerPolicy] = DefaultReferrerPolicy if settings is not None: settings_policy = _load_policy_class(settings.get("REFERRER_POLICY")) @@ -370,23 +369,13 @@ class RefererMiddleware: cls = _load_policy_class(policy_name, warning_only=True) return cls() if cls else self.default_policy() - def process_spider_output( - self, response: Response, result: Iterable[Any], spider: Spider - ) -> Iterable[Any]: - return (self._set_referer(r, response) for r in result) - - async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: - async for r in result: - yield self._set_referer(r, response) - - def _set_referer(self, r: Any, response: Response) -> Any: - if isinstance(r, Request): - referrer = self.policy(response, r).referrer(response.url, r.url) - if referrer is not None: - r.headers.setdefault("Referer", referrer) - return r + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + referrer = self.policy(response, request).referrer(response.url, request.url) + if referrer is not None: + request.headers.setdefault("Referer", referrer) + return request def request_scheduled(self, request: Request, spider: Spider) -> None: # check redirected request to patch "Referer" header if necessary diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index a1cd1bb7c..177c19e1b 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -7,72 +7,49 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -import warnings -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.http import Request, Response +from scrapy.exceptions import NotConfigured +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable - # typing.Self requires Python 3.11 from typing_extensions import Self - from scrapy import Spider from scrapy.crawler import Crawler - from scrapy.settings import BaseSettings + from scrapy.http import Request, Response logger = logging.getLogger(__name__) -class UrlLengthMiddleware: - def __init__(self, maxlength: int): +class UrlLengthMiddleware(BaseSpiderMiddleware): + crawler: Crawler + + def __init__(self, maxlength: int): # pylint: disable=super-init-not-called self.maxlength: int = maxlength - @classmethod - def from_settings(cls, settings: BaseSettings) -> Self: - warnings.warn( - f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return cls._from_settings(settings) - @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - return cls._from_settings(crawler.settings) - - @classmethod - def _from_settings(cls, settings: BaseSettings) -> Self: - maxlength = settings.getint("URLLENGTH_LIMIT") + maxlength = crawler.settings.getint("URLLENGTH_LIMIT") if not maxlength: raise NotConfigured - return cls(maxlength) + o = cls(maxlength) + o.crawler = crawler + return o - def process_spider_output( - self, response: Response, result: Iterable[Any], spider: Spider - ) -> Iterable[Any]: - return (r for r in result if self._filter(r, spider)) - - async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: - async for r in result: - if self._filter(r, spider): - yield r - - def _filter(self, request: Any, spider: Spider) -> bool: - if isinstance(request, Request) and len(request.url) > self.maxlength: - logger.info( - "Ignoring link (url length > %(maxlength)d): %(url)s ", - {"maxlength": self.maxlength, "url": request.url}, - extra={"spider": spider}, - ) - assert spider.crawler.stats - spider.crawler.stats.inc_value( - "urllength/request_ignored_count", spider=spider - ) - return False - return True + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + if len(request.url) <= self.maxlength: + return request + logger.info( + "Ignoring link (url length > %(maxlength)d): %(url)s ", + {"maxlength": self.maxlength, "url": request.url}, + extra={"spider": self.crawler.spider}, + ) + assert self.crawler.stats + self.crawler.stats.inc_value( + "urllength/request_ignored_count", spider=self.crawler.spider + ) + return None diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py new file mode 100644 index 000000000..46be879f3 --- /dev/null +++ b/tests/test_spidermiddleware_base.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import pytest + +from scrapy import Request, Spider +from scrapy.http import Response +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware +from scrapy.utils.test import get_crawler + +if TYPE_CHECKING: + from scrapy.crawler import Crawler + + +@pytest.fixture +def crawler() -> Crawler: + return get_crawler(Spider) + + +def test_trivial(crawler): + class TrivialSpiderMiddleware(BaseSpiderMiddleware): + pass + + mw = TrivialSpiderMiddleware.from_crawler(crawler) + assert hasattr(mw, "crawler") + assert mw.crawler is crawler + test_req = Request("data:,") + spider_output = [test_req, {"foo": "bar"}] + processed = list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ) + assert processed == [test_req, {"foo": "bar"}] + + +def test_processed_request(crawler): + class ProcessReqSpiderMiddleware(BaseSpiderMiddleware): + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + if request.url == "data:2,": + return None + if request.url == "data:3,": + return Request("data:30,") + return request + + mw = ProcessReqSpiderMiddleware.from_crawler(crawler) + test_req1 = Request("data:1,") + test_req2 = Request("data:2,") + test_req3 = Request("data:3,") + spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3] + processed = list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ) + assert len(processed) == 3 + assert isinstance(processed[0], Request) + assert processed[0].url == "data:1," + assert processed[1] == {"foo": "bar"} + assert isinstance(processed[2], Request) + assert processed[2].url == "data:30," + + +def test_processed_item(crawler): + class ProcessItemSpiderMiddleware(BaseSpiderMiddleware): + def get_processed_item(self, item: Any, response: Response) -> Any: + if item["foo"] == 2: + return None + if item["foo"] == 3: + item["foo"] = 30 + return item + + mw = ProcessItemSpiderMiddleware.from_crawler(crawler) + test_req = Request("data:,") + spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}] + processed = list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ) + assert processed == [{"foo": 1}, test_req, {"foo": 30}] + + +def test_processed_both(crawler): + class ProcessBothSpiderMiddleware(BaseSpiderMiddleware): + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + if request.url == "data:2,": + return None + if request.url == "data:3,": + return Request("data:30,") + return request + + def get_processed_item(self, item: Any, response: Response) -> Any: + if item["foo"] == 2: + return None + if item["foo"] == 3: + item["foo"] = 30 + return item + + mw = ProcessBothSpiderMiddleware.from_crawler(crawler) + test_req1 = Request("data:1,") + test_req2 = Request("data:2,") + test_req3 = Request("data:3,") + spider_output = [ + test_req1, + {"foo": 1}, + {"foo": 2}, + test_req2, + {"foo": 3}, + test_req3, + ] + processed = list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ) + assert len(processed) == 4 + assert isinstance(processed[0], Request) + assert processed[0].url == "data:1," + assert processed[1] == {"foo": 1} + assert processed[2] == {"foo": 30} + assert isinstance(processed[3], Request) + assert processed[3].url == "data:30," diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index dfcc141c3..9b4aa624c 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -1,19 +1,18 @@ from scrapy.http import Request, Response from scrapy.spidermiddlewares.depth import DepthMiddleware from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector from scrapy.utils.test import get_crawler class TestDepthMiddleware: def setup_method(self): - crawler = get_crawler(Spider) + crawler = get_crawler(Spider, {"DEPTH_LIMIT": 1, "DEPTH_STATS_VERBOSE": True}) self.spider = crawler._create_spider("scrapytest.org") - self.stats = StatsCollector(crawler) + self.stats = crawler.stats self.stats.open_spider(self.spider) - self.mw = DepthMiddleware(1, self.stats, True) + self.mw = DepthMiddleware.from_crawler(crawler) def test_process_spider_output(self): req = Request("http://scrapytest.org") diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index f4563a0a4..e4f4b8f9b 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -10,7 +10,7 @@ from scrapy.utils.test import get_crawler class TestOffsiteMiddleware: def setup_method(self): crawler = get_crawler(Spider) - self.spider = crawler._create_spider(**self._get_spiderargs()) + self.spider = crawler.spider = crawler._create_spider(**self._get_spiderargs()) self.mw = OffsiteMiddleware.from_crawler(crawler) self.mw.spider_opened(self.spider) From b1f85b5a173f48bad2881465efbab5c22aa0327d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 24 Apr 2025 20:03:36 +0500 Subject: [PATCH 090/491] Release notes for 2.13.0, up to b4c253102139e842859a9abf1455e62504cc9511. --- docs/news.rst | 198 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 193 insertions(+), 5 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 9f476ee21..b9b5ce320 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,21 +5,209 @@ Release notes .. _release-VERSION: -Scrapy VERSION (unreleased) ---------------------------- +Scrapy 2.13.0 (unreleased) +-------------------------- + +Highlights: + +- Added the :reqmeta:`allow_offsite` request meta key + +- HTTP/1.0 support is deprecated + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for PyPy 3.9. + (:issue:`6613`) Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - The ``from_settings()`` method of - :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` is removed - without a deprecation period (this was needed because after the - introduction of the + :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`, + deprecated in 2.12.0, is removed earlier than the usual deprecation period + (this was needed because after the introduction of the :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware` base class and switching built-in spider middlewares to it those middlewares need the :class:`~scrapy.crawler.Crawler` instance at run time). Please use ``from_crawler()`` instead. +Deprecations +~~~~~~~~~~~~ + +- Functions that were imported from :mod:`w3lib.url` and re-exported in + :mod:`scrapy.utils.url` are now deprecated, you should import them from + ``w3lib.url`` directly. They are: + + - ``scrapy.utils.url.add_or_replace_parameter()`` + + - ``scrapy.utils.url.add_or_replace_parameters()`` + + - ``scrapy.utils.url.any_to_uri()`` + + - ``scrapy.utils.url.canonicalize_url()`` + + - ``scrapy.utils.url.file_uri_to_path()`` + + - ``scrapy.utils.url.is_url()`` + + - ``scrapy.utils.url.parse_data_uri()`` + + - ``scrapy.utils.url.parse_url()`` + + - ``scrapy.utils.url.path_to_file_uri()`` + + - ``scrapy.utils.url.safe_download_url()`` + + - ``scrapy.utils.url.safe_url_string()`` + + - ``scrapy.utils.url.url_query_cleaner()`` + + - ``scrapy.utils.url.url_query_parameter()`` + + - ``scrapy.utils.url._unquotepath()`` + + - ``scrapy.utils.url._safe_chars`` attribute + + (:issue:`4577`, :issue:`6583`, :issue:`6586`) + +- HTTP/1.0 support code is deprecated. It was disabled by default and + couldn't be used together with HTTP/1.1. If you still need it, you should + write your own download handler or copy the code from Scrapy. The + deprecations include: + + - ``scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler`` + + - ``scrapy.core.downloader.webclient.ScrapyHTTPClientFactory`` + + - ``scrapy.core.downloader.webclient.ScrapyHTTPPageGetter`` + + - Overriding + ``scrapy.core.downloader.contextfactory.ScrapyClientContextFactory.getContext()`` + + (:issue:`6634`) + +- ``scrapy.utils.versions.scrapy_components_versions()`` is deprecated, use + :func:`scrapy.utils.versions.get_versions()` instead. + (:issue:`6582`) + +- ``BaseDupeFilter.log()`` is deprecated. It does nothing and shouldn't be + called. + (:issue:`4151`) + +New features +~~~~~~~~~~~~ + +- Added the :reqmeta:`allow_offsite` request meta key that can be used + instead of the more general :attr:`~scrapy.Request.dont_filter` request + attribute to skip processing of the request by + :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` (but not + by other code that checks :attr:`~scrapy.Request.dont_filter`). + (:issue:`3690`, :issue:`6151`, :issue:`6366`) + +- :ref:`Scrapy add-ons ` can now define a class method called + ``update_pre_crawler_settings()`` to update :ref:`pre-crawler settings + `. + (:issue:`6544`, :issue:`6568`) + +- Added the :setting:`DEFAULT_DROPITEM_LOG_LEVEL` setting and the + :attr:`scrapy.exceptions.DropItem.log_level` attribute that allow + customizing the log level of the message that is logged when an item is + dropped. + (:issue:`6603`, :issue:`6608`) + +- Added the :setting:`LOG_VERSIONS` setting that allows customizing the + list of software which versions are logged when the spider starts. + (:issue:`6582`) + +Improvements +~~~~~~~~~~~~ + +- Improved the error message when running a ``scrapy`` command that requires + a project (such as ``scrapy crawl``) outside of a project directory. + (:issue:`2349`, :issue:`3426`) + +- An empty :setting:`ADDONS` setting added to the ``settings.py`` template + for new projects. + (:issue:`6587`) + +Bug fixes +~~~~~~~~~ + +- Fixed calculation of ``items_per_minute`` and ``responses_per_minute`` + stats. + (:issue:`6599`) + +- Fixed an error initializing + :class:`scrapy.extensions.feedexport.GCSFeedStorage`. + (:issue:`6617`, :issue:`6628`) + +- Fixed an error running ``scrapy bench``. + (:issue:`6632`, :issue:`6633`) + +Documentation +~~~~~~~~~~~~~ + +- Improved the contribution docs. + (:issue:`6561`, :issue:`6575`) + +- Other documentation improvements and fixes. + (:issue:`4151`, + :issue:`6526`, + :issue:`6620`, + :issue:`6621`, + :issue:`6622`, + :issue:`6623`, + :issue:`6624`) + +Packaging +~~~~~~~~~ + +- Switched from ``setup.py`` to ``pyproject.toml``. + (:issue:`6514`, :issue:`6547`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Replaced most linters with ``ruff``. + (:issue:`6565`, :issue:`6576`, :issue:`6577`, :issue:`6581`, :issue:`6584`, + :issue:`6595`, :issue:`6601`, :issue:`6631`) + +- Improved accuracy and performance of collecting test coverage. + (:issue:`6567`) + +- Fixed an error that prevented running tests from directories other than the + top level source directory. + (:issue:`6567`) + +- Reduced the amount of ``mockserver`` calls in tests to improve the overall + test run time. + (:issue:`6637`, :issue:`6648`) + +- Fixed tests that were running the same test code more than once. + (:issue:`6646`) + +- Type hints improvements and fixes. + (:issue:`6578`, :issue:`6579`, :issue:`6593`, :issue:`6605`) + +- CI and test improvements and fixes. + (:issue:`5360`, + :issue:`6271`, + :issue:`6547`, + :issue:`6560`, + :issue:`6602`, + :issue:`6607`, + :issue:`6609`, + :issue:`6613`, + :issue:`6619`, + :issue:`6626`) + +- Code cleanups. + (:issue:`6600`, + :issue:`6606`, + :issue:`6635`) + + .. _release-2.12.0: Scrapy 2.12.0 (2024-11-18) From 095140f134745960751c8c4b34da7de6b91e4a82 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 25 Apr 2025 19:43:29 +0500 Subject: [PATCH 091/491] Cover the current master in the release notes (up to daf9db7). --- docs/news.rst | 223 +++++++++++++++++++++++++++---- docs/topics/addons.rst | 2 + docs/topics/coroutines.rst | 6 + docs/topics/settings.rst | 2 +- scrapy/spidermiddlewares/base.py | 2 + 5 files changed, 210 insertions(+), 25 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index b9b5ce320..74a26c383 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,16 +3,22 @@ Release notes ============= -.. _release-VERSION: +.. _release-2.13.0: Scrapy 2.13.0 (unreleased) -------------------------- Highlights: +- The asyncio reactor is now enabled by default + - Added the :reqmeta:`allow_offsite` request meta key -- HTTP/1.0 support is deprecated +- :ref:`Spider middlewares that don't support asynchronous spider output + ` are deprecated + +- Added a base class for :ref:`universal spider middlewares + ` Modified requirements ~~~~~~~~~~~~~~~~~~~~~ @@ -23,21 +29,62 @@ Modified requirements Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +- The default value of the :setting:`TWISTED_REACTOR` setting was changed + from ``None`` to + ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``. This value + was used in newly generated projects since Scrapy 2.7.0 but now existing + projects that don't explicitly set this setting will also use the asyncio + reactor. You can :ref:`change this setting in your project + ` to use a different reactor. + (:issue:`6659`, :issue:`6713`) + - The ``from_settings()`` method of :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`, - deprecated in 2.12.0, is removed earlier than the usual deprecation period - (this was needed because after the introduction of the + deprecated in Scrapy 2.12.0, is removed earlier than the usual deprecation + period (this was needed because after the introduction of the :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware` base class and switching built-in spider middlewares to it those middlewares need the :class:`~scrapy.crawler.Crawler` instance at run time). Please use ``from_crawler()`` instead. + (:issue:`6693`) + +- ``scrapy.utils.url.escape_ajax()`` is no longer called when a + :class:`~scrapy.Request` instance is created. It was only useful for + websites supporting the ``_escaped_fragment_`` feature which most modern + websites don't support. If you still need this you can modify the URLs + before passing them to :class:`~scrapy.Request`. + (:issue:`6523`, :issue:`6651`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed old deprecated name aliases for some signals: + + - ``stats_spider_opened`` (use ``spider_opened`` instead) + + - ``stats_spider_closing`` and ``stats_spider_closed`` (use + ``spider_closed`` instead) + + - ``item_passed`` (use ``item_scraped`` instead) + + - ``request_received`` (use ``request_scheduled`` instead) + + (:issue:`6654`, :issue:`6655`) Deprecations ~~~~~~~~~~~~ +- :ref:`Spider middlewares that don't support asynchronous spider output + ` are deprecated. The async iterable + downgrading feature, needed for using such middlewares with asynchronous + callbacks and with other spider middlewares that produce asynchronous + iterables, is also deprecated. Please update all such middlewares to + support asynchronous spider output. + (:issue:`6664`) + - Functions that were imported from :mod:`w3lib.url` and re-exported in :mod:`scrapy.utils.url` are now deprecated, you should import them from - ``w3lib.url`` directly. They are: + :mod:`w3lib.url` directly. They are: - ``scrapy.utils.url.add_or_replace_parameter()`` @@ -65,10 +112,6 @@ Deprecations - ``scrapy.utils.url.url_query_parameter()`` - - ``scrapy.utils.url._unquotepath()`` - - - ``scrapy.utils.url._safe_chars`` attribute - (:issue:`4577`, :issue:`6583`, :issue:`6586`) - HTTP/1.0 support code is deprecated. It was disabled by default and @@ -87,6 +130,37 @@ Deprecations (:issue:`6634`) +- The following modules and functions used only in tests are deprecated: + + - the ``scrapy/utils/testproc`` module + + - the ``scrapy/utils/testsite`` module + + - ``scrapy.utils.test.assert_gcs_environ()`` + + - ``scrapy.utils.test.get_ftp_content_and_delete()`` + + - ``scrapy.utils.test.get_gcs_content_and_delete()`` + + - ``scrapy.utils.test.mock_google_cloud_storage()`` + + - ``scrapy.utils.test.skip_if_no_boto()`` + + If you need to use them in your tests or code, you can copy the code from Scrapy. + (:issue:`6696`) + +- ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` is + deprecated. It was disabled by default and isn't useful for most of the + existing websites. + (:issue:`6523`, :issue:`6651`, :issue:`6656`) + +- ``scrapy.utils.url.escape_ajax()`` is deprecated. + (:issue:`6523`, :issue:`6651`) + +- ``scrapy.spiders.init.InitSpider`` is deprecated. If you find it useful, + you can copy its code from Scrapy. + (:issue:`6708`, :issue:`6714`) + - ``scrapy.utils.versions.scrapy_components_versions()`` is deprecated, use :func:`scrapy.utils.versions.get_versions()` instead. (:issue:`6582`) @@ -105,29 +179,63 @@ New features by other code that checks :attr:`~scrapy.Request.dont_filter`). (:issue:`3690`, :issue:`6151`, :issue:`6366`) +- Added an optional base class for spider middlewares, + :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware`, which can be + helpful for writing :ref:`universal spider middlewares + ` without boilerplate and code duplication. + The built-in spider middlewares now inherit from this class. + (:issue:`6693`) + - :ref:`Scrapy add-ons ` can now define a class method called ``update_pre_crawler_settings()`` to update :ref:`pre-crawler settings `. (:issue:`6544`, :issue:`6568`) +- Added :ref:`helpers ` for modifying :ref:`component + priority dictionary ` settings. + (:issue:`6614`) + +- Responses that use an unknown/unsupported encoding now produce a warning. + If Scrapy knows that installing an additional package (such as brotli_) + will allow decoding the response, that will be mentioned in the warning. + (:issue:`4697`, :issue:`6618`) + +- Added the ``spider_exceptions/count`` stat which tracks the total count of + exceptions (tracked also by per-type ``spider_exceptions/*`` stats). + (:issue:`6739`, :issue:`6740`) + - Added the :setting:`DEFAULT_DROPITEM_LOG_LEVEL` setting and the :attr:`scrapy.exceptions.DropItem.log_level` attribute that allow customizing the log level of the message that is logged when an item is dropped. (:issue:`6603`, :issue:`6608`) +- Added support for the ``-b, --cookie`` curl argument to + :meth:`scrapy.Request.from_curl`. + (:issue:`6684`) + - Added the :setting:`LOG_VERSIONS` setting that allows customizing the - list of software which versions are logged when the spider starts. + list of software whose versions are logged when the spider starts. (:issue:`6582`) +- Added the :setting:`WARN_ON_GENERATOR_RETURN_VALUE` setting that allows + disabling run time analysis of callback code used to warn about incorrect + ``return`` statements in generator-based callbacks. You may need to disable + this setting if this analysis breaks on your callback code. + (:issue:`6731`, :issue:`6738`) + Improvements ~~~~~~~~~~~~ +- Removed or postponed some calls of :func:`itemadapter.is_item` to increase + performance. + (:issue:`6719`) + - Improved the error message when running a ``scrapy`` command that requires a project (such as ``scrapy crawl``) outside of a project directory. (:issue:`2349`, :issue:`3426`) -- An empty :setting:`ADDONS` setting added to the ``settings.py`` template +- Added an empty :setting:`ADDONS` setting to the ``settings.py`` template for new projects. (:issue:`6587`) @@ -145,12 +253,46 @@ Bug fixes - Fixed an error running ``scrapy bench``. (:issue:`6632`, :issue:`6633`) +- Fixed duplicated log messages about the reactor and the event loop. + (:issue:`6636`, :issue:`6657`) + +- Fixed resolving type annotations of ``SitemapSpider._parse_sitemap()`` at + run time, required by tools such as scrapy-poet_. + (:issue:`6665`, :issue:`6671`) + + .. _scrapy-poet: https://github.com/scrapinghub/scrapy-poet + +- Calling :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` without + an installed reactor now raises an exception instead of installing a + reactor. + (:issue:`6732`, :issue:`6735`) + +- Restored support for the ``x-gzip`` content encoding. + (:issue:`6618`) + Documentation ~~~~~~~~~~~~~ -- Improved the contribution docs. +- Improved the :ref:`docs ` about asynchronous + iterable support in spider middlewares. + (:issue:`6688`) + +- Improved the :ref:`docs ` about using + :class:`~twisted.internet.defer.Deferred`-based APIs in coroutine-based + code. + (:issue:`6734`) + +- Improved the :ref:`contribution docs `. (:issue:`6561`, :issue:`6575`) +- Removed the ``Splash`` recommendation from the :ref:`headless browser + ` suggestion. We no longer recommend using + ``Splash`` and recommend using other headless browser solutions instead. + (:issue:`6642`, :issue:`6701`) + +- Added the dark mode to the HTML documentation. + (:issue:`6653`) + - Other documentation improvements and fixes. (:issue:`4151`, :issue:`6526`, @@ -158,7 +300,9 @@ Documentation :issue:`6621`, :issue:`6622`, :issue:`6623`, - :issue:`6624`) + :issue:`6624`, + :issue:`6721`, + :issue:`6723`) Packaging ~~~~~~~~~ @@ -169,12 +313,20 @@ Packaging Quality assurance ~~~~~~~~~~~~~~~~~ -- Replaced most linters with ``ruff``. - (:issue:`6565`, :issue:`6576`, :issue:`6577`, :issue:`6581`, :issue:`6584`, - :issue:`6595`, :issue:`6601`, :issue:`6631`) +- Replaced most linters with ruff_. + (:issue:`6565`, + :issue:`6576`, + :issue:`6577`, + :issue:`6581`, + :issue:`6584`, + :issue:`6595`, + :issue:`6601`, + :issue:`6631`) + + .. _ruff: https://docs.astral.sh/ruff/ - Improved accuracy and performance of collecting test coverage. - (:issue:`6567`) + (:issue:`6255`, :issue:`6610`) - Fixed an error that prevented running tests from directories other than the top level source directory. @@ -185,10 +337,28 @@ Quality assurance (:issue:`6637`, :issue:`6648`) - Fixed tests that were running the same test code more than once. - (:issue:`6646`) + (:issue:`6646`, :issue:`6647`, :issue:`6650`) + +- Refactored tests to use more ``pytest`` features instead of ``unittest`` + ones where possible. + (:issue:`6678`, + :issue:`6680`, + :issue:`6695`, + :issue:`6699`, + :issue:`6700`, + :issue:`6702`, + :issue:`6709`, + :issue:`6710`, + :issue:`6711`, + :issue:`6712`, + :issue:`6725`) - Type hints improvements and fixes. - (:issue:`6578`, :issue:`6579`, :issue:`6593`, :issue:`6605`) + (:issue:`6578`, + :issue:`6579`, + :issue:`6593`, + :issue:`6605`, + :issue:`6694`) - CI and test improvements and fixes. (:issue:`5360`, @@ -200,7 +370,16 @@ Quality assurance :issue:`6609`, :issue:`6613`, :issue:`6619`, - :issue:`6626`) + :issue:`6626`, + :issue:`6679`, + :issue:`6703`, + :issue:`6704`, + :issue:`6716`, + :issue:`6720`, + :issue:`6722`, + :issue:`6724`, + :issue:`6741`, + :issue:`6743`) - Code cleanups. (:issue:`6600`, @@ -806,8 +985,6 @@ Bug fixes - Restored support for brotlipy_, which had been dropped in Scrapy 2.11.1 in favor of brotli_. (:issue:`6261`) - .. _brotli: https://github.com/google/brotli - .. note:: brotlipy is deprecated, both in Scrapy and upstream. Use brotli instead if you can. @@ -2391,8 +2568,6 @@ Scrapy 2.5.1 (2021-10-05) need to upgrade scrapy-splash to a greater version for it to continue to work. -.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash - .. _release-2.5.0: diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 17e3c177a..815501e66 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -124,6 +124,8 @@ Set some basic configuration: "ITEM_PIPELINES", MyPipeline, 200 ) +.. _priority-dict-helpers: + .. tip:: When editing a :ref:`component priority dictionary ` setting, like :setting:`ITEM_PIPELINES`, consider using setting methods like diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 1c80857f6..1a84f893c 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -344,3 +344,9 @@ For example: feature will be removed, and all spider middlewares will be expected to define their ``process_spider_output`` method as an asynchronous generator. + +Since 2.13.0, Scrapy provides a base class, +:class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware`, which implements +the ``process_spider_output()`` and ``process_spider_output_async()`` methods, +so instead of duplicating the processing code you can override the +``get_processed_request()`` and/or the ``get_processed_item()`` method. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a59a61050..73ac36646 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2005,7 +2005,7 @@ current platform. ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated ``settings.py`` file. -.. versionchanged:: VERSION +.. versionchanged:: 2.13 The default value was changed from ``None`` to ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``. diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py index 650192095..5e4370d45 100644 --- a/scrapy/spidermiddlewares/base.py +++ b/scrapy/spidermiddlewares/base.py @@ -17,6 +17,8 @@ if TYPE_CHECKING: class BaseSpiderMiddleware: """Optional base class for spider middlewares. + .. versionadded:: 2.13 + This class provides helper methods for asynchronous ``process_spider_output`` methods. Middlewares that don't have a ``process_spider_output`` method don't need to use it. From 4aba7e5f6675703159220ee22bba6953c5685ef6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 25 Apr 2025 20:16:26 +0500 Subject: [PATCH 092/491] Mention the deprecation of TestSpider. --- docs/news.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 74a26c383..7bb25e6b6 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -149,6 +149,11 @@ Deprecations If you need to use them in your tests or code, you can copy the code from Scrapy. (:issue:`6696`) +- ``scrapy.utils.test.TestSpider`` is deprecated. If you need an empty spider + class you can use :class:`scrapy.utils.spider.DefaultSpider` or create your + own subclass of :class:`scrapy.Spider`. + (:issue:`6678`) + - ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` is deprecated. It was disabled by default and isn't useful for most of the existing websites. From eced5ca2d3c85c36b18f0da3d5d888e7e66a7014 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 30 Apr 2025 00:51:23 +0500 Subject: [PATCH 093/491] Remove the unnecessary spider argument from Scraper and related code. --- scrapy/core/engine.py | 12 +-- scrapy/core/scraper.py | 178 +++++++++++++++++++++++++++------------- scrapy/core/spidermw.py | 6 +- tests/test_engine.py | 7 +- 4 files changed, 131 insertions(+), 72 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b7a73700b..653e5e05c 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -265,7 +265,7 @@ class ExecutionEngine: self.crawl(result) return None - d = self.scraper.enqueue_scrape(result, request, self.spider) + d = self.scraper.enqueue_scrape(result, request) d.addErrback( lambda f: logger.error( "Error while enqueuing downloader output", @@ -290,14 +290,14 @@ class ExecutionEngine: """Inject the request into the spider <-> downloader pipeline""" if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - self._schedule_request(request, self.spider) + self._schedule_request(request) self.slot.nextcall.schedule() # type: ignore[union-attr] - def _schedule_request(self, request: Request, spider: Spider) -> None: + def _schedule_request(self, request: Request) -> None: request_scheduled_result = self.signals.send_catch_log( signals.request_scheduled, request=request, - spider=spider, + spider=self.spider, dont_log=IgnoreRequest, ) for handler, result in request_scheduled_result: @@ -305,7 +305,7 @@ class ExecutionEngine: return if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log( - signals.request_dropped, request=request, spider=spider + signals.request_dropped, request=request, spider=self.spider ) def download(self, request: Request) -> Deferred[Response]: @@ -438,7 +438,7 @@ class ExecutionEngine: dfd.addBoth(lambda _: self.downloader.close()) dfd.addErrback(log_failure("Downloader close failure")) - dfd.addBoth(lambda _: self.scraper.close_spider(spider)) + dfd.addBoth(lambda _: self.scraper.close_spider()) dfd.addErrback(log_failure("Scraper close failure")) if hasattr(self.slot.scheduler, "close"): diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 496adb500..2942dfa58 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -4,6 +4,7 @@ extracts information from them""" from __future__ import annotations import logging +import warnings from collections import deque from collections.abc import AsyncIterable, Iterator from typing import TYPE_CHECKING, Any, TypeVar, Union, cast @@ -13,7 +14,12 @@ from twisted.python.failure import Failure from scrapy import Spider, signals from scrapy.core.spidermw import SpiderMiddlewareManager -from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest +from scrapy.exceptions import ( + CloseSpider, + DropItem, + IgnoreRequest, + ScrapyDeprecationWarning, +) from scrapy.http import Request, Response from scrapy.utils.defer import ( aiter_errback, @@ -110,27 +116,43 @@ class Scraper: self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE")) yield self.itemproc.open_spider(spider) - def close_spider(self, spider: Spider) -> Deferred[Spider]: + def close_spider(self, spider: Spider | None = None) -> Deferred[Spider]: """Close a spider being scraped and release its resources""" + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to Scraper.close_spider() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if self.slot is None: raise RuntimeError("Scraper slot not assigned") self.slot.closing = Deferred() self.slot.closing.addCallback(self.itemproc.close_spider) - self._check_if_closing(spider) + self._check_if_closing() return self.slot.closing def is_idle(self) -> bool: """Return True if there isn't any more spiders to process""" return not self.slot - def _check_if_closing(self, spider: Spider) -> None: + def _check_if_closing(self) -> None: assert self.slot is not None # typing + assert self.crawler.spider if self.slot.closing and self.slot.is_idle(): - self.slot.closing.callback(spider) + assert self.crawler.spider + self.slot.closing.callback(self.crawler.spider) def enqueue_scrape( - self, result: Response | Failure, request: Request, spider: Spider + self, result: Response | Failure, request: Request, spider: Spider | None = None ) -> _HandleOutputDeferred: + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to Scraper.enqueue_scrape() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if self.slot is None: raise RuntimeError("Scraper slot not assigned") dfd = self.slot.add_response_request(result, request) @@ -138,8 +160,8 @@ class Scraper: def finish_scraping(_: _T) -> _T: assert self.slot is not None self.slot.finish_response(result, request) - self._check_if_closing(spider) - self._scrape_next(spider) + self._check_if_closing() + self._scrape_next() return _ dfd.addBoth(finish_scraping) @@ -148,20 +170,20 @@ class Scraper: "Scraper bug processing %(request)s", {"request": request}, exc_info=failure_to_exc_info(f), - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) ) - self._scrape_next(spider) + self._scrape_next() return dfd - def _scrape_next(self, spider: Spider) -> None: + def _scrape_next(self) -> None: assert self.slot is not None # typing while self.slot.queue: response, request, deferred = self.slot.next_response_request_deferred() - self._scrape(response, request, spider).chainDeferred(deferred) + self._scrape(response, request).chainDeferred(deferred) def _scrape( - self, result: Response | Failure, request: Request, spider: Spider + self, result: Response | Failure, request: Request ) -> _HandleOutputDeferred: """ Handle the downloaded response or failure through the spider callback/errback @@ -171,40 +193,49 @@ class Scraper: f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}" ) dfd: Deferred[Iterable[Any] | AsyncIterable[Any]] = self._scrape2( - result, request, spider + result, request ) # returns spider's processed output - dfd.addErrback(self.handle_spider_error, request, result, spider) + dfd.addErrback(self.handle_spider_error, request, result) dfd2: _HandleOutputDeferred = dfd.addCallback( - self.handle_spider_output, request, cast(Response, result), spider + self.handle_spider_output, request, cast(Response, result) ) return dfd2 def _scrape2( - self, result: Response | Failure, request: Request, spider: Spider + self, result: Response | Failure, request: Request ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: """ Handle the different cases of request's result been a Response or a Failure """ if isinstance(result, Response): # Deferreds are invariant so Mutable*Chain isn't matched to *Iterable + assert self.crawler.spider return self.spidermw.scrape_response( # type: ignore[return-value] - self.call_spider, result, request, spider + self.call_spider, result, request, self.crawler.spider ) # else result is a Failure - dfd = self.call_spider(result, request, spider) - dfd.addErrback(self._log_download_errors, result, request, spider) + dfd = self.call_spider(result, request) + dfd.addErrback(self._log_download_errors, result, request) return dfd def call_spider( - self, result: Response | Failure, request: Request, spider: Spider + self, result: Response | Failure, request: Request, spider: Spider | None = None ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to Scraper.call_spider() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + + assert self.crawler.spider dfd: Deferred[Any] if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request assert result.request - callback = result.request.callback or spider._parse - warn_on_generator_with_return_value(spider, callback) + callback = result.request.callback or self.crawler.spider._parse + warn_on_generator_with_return_value(self.crawler.spider, callback) dfd = defer_succeed(result) dfd.addCallbacks( callback=callback, callbackKeywords=result.request.cb_kwargs @@ -214,7 +245,9 @@ class Scraper: result.request = request # type: ignore[attr-defined] dfd = defer_fail(result) if request.errback: - warn_on_generator_with_return_value(spider, request.errback) + warn_on_generator_with_return_value( + self.crawler.spider, request.errback + ) dfd.addErrback(request.errback) dfd2: Deferred[Iterable[Any] | AsyncIterable[Any]] = dfd.addCallback( iterate_spider_output @@ -226,29 +259,44 @@ class Scraper: _failure: Failure, request: Request, response: Response | Failure, - spider: Spider, + spider: Spider | None = None, ) -> None: + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to Scraper.handle_spider_error() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + + assert self.crawler.spider exc = _failure.value if isinstance(exc, CloseSpider): assert self.crawler.engine is not None # typing - self.crawler.engine.close_spider(spider, exc.reason or "cancelled") + self.crawler.engine.close_spider( + self.crawler.spider, exc.reason or "cancelled" + ) return - logkws = self.logformatter.spider_error(_failure, request, response, spider) + logkws = self.logformatter.spider_error( + _failure, request, response, self.crawler.spider + ) logger.log( *logformatter_adapter(logkws), exc_info=failure_to_exc_info(_failure), - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) self.signals.send_catch_log( signal=signals.spider_error, failure=_failure, response=response, - spider=spider, + spider=self.crawler.spider, ) assert self.crawler.stats - self.crawler.stats.inc_value("spider_exceptions/count", spider=spider) self.crawler.stats.inc_value( - f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider + "spider_exceptions/count", spider=self.crawler.spider + ) + self.crawler.stats.inc_value( + f"spider_exceptions/{_failure.value.__class__.__name__}", + spider=self.crawler.spider, ) def handle_spider_output( @@ -256,41 +304,40 @@ class Scraper: result: Iterable[_T] | AsyncIterable[_T], request: Request, response: Response, - spider: Spider, + spider: Spider | None = None, ) -> _HandleOutputDeferred: + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to Scraper.handle_spider_output() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if not result: return defer_succeed(None) it: Iterable[_T] | AsyncIterable[_T] dfd: Deferred[_ParallelResult] if isinstance(result, AsyncIterable): - it = aiter_errback( - result, self.handle_spider_error, request, response, spider - ) + it = aiter_errback(result, self.handle_spider_error, request, response) dfd = parallel_async( it, self.concurrent_items, self._process_spidermw_output, - request, response, - spider, ) else: - it = iter_errback( - result, self.handle_spider_error, request, response, spider - ) + it = iter_errback(result, self.handle_spider_error, request, response) dfd = parallel( it, self.concurrent_items, self._process_spidermw_output, - request, response, - spider, ) # returning Deferred[_ParallelResult] instead of Deferred[Union[_ParallelResult, None]] return dfd # type: ignore[return-value] def _process_spidermw_output( - self, output: Any, request: Request, response: Response, spider: Spider + self, output: Any, response: Response ) -> Deferred[Any] | None: """Process each Request/Item (given in the output parameter) returned from the given spider @@ -314,7 +361,7 @@ class Scraper: assert self.crawler.spider is not None # typing self.slot.itemproc_size += 1 dfd = self.itemproc.process_item(item, self.crawler.spider) - dfd.addBoth(self._itemproc_finished, item, response, self.crawler.spider) + dfd.addBoth(self._itemproc_finished, item, response) return dfd def _log_download_errors( @@ -322,7 +369,6 @@ class Scraper: spider_failure: Failure, download_failure: Failure, request: Request, - spider: Spider, ) -> Failure | None: """Log and silence errors that come from the engine (typically download errors that got propagated thru here). @@ -332,24 +378,25 @@ class Scraper: ExecutionEngine._handle_downloader_output() as "result" """ if not download_failure.check(IgnoreRequest): + assert self.crawler.spider if download_failure.frames: logkws = self.logformatter.download_error( - download_failure, request, spider + download_failure, request, self.crawler.spider ) logger.log( *logformatter_adapter(logkws), - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, exc_info=failure_to_exc_info(download_failure), ) else: errmsg = download_failure.getErrorMessage() if errmsg: logkws = self.logformatter.download_error( - download_failure, request, spider, errmsg + download_failure, request, self.crawler.spider, errmsg ) logger.log( *logformatter_adapter(logkws), - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) if spider_failure is not download_failure: @@ -357,41 +404,54 @@ class Scraper: return None def _itemproc_finished( - self, output: Any, item: Any, response: Response | None, spider: Spider + self, output: Any, item: Any, response: Response | None ) -> Deferred[Any]: """ItemProcessor finished for the given ``item`` and returned ``output``""" assert self.slot is not None # typing + assert self.crawler.spider self.slot.itemproc_size -= 1 if isinstance(output, Failure): ex = output.value if isinstance(ex, DropItem): - logkws = self.logformatter.dropped(item, ex, response, spider) + logkws = self.logformatter.dropped( + item, ex, response, self.crawler.spider + ) if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) + logger.log( + *logformatter_adapter(logkws), + extra={"spider": self.crawler.spider}, + ) return self.signals.send_catch_log_deferred( signal=signals.item_dropped, item=item, response=response, - spider=spider, + spider=self.crawler.spider, exception=output.value, ) assert ex - logkws = self.logformatter.item_error(item, ex, response, spider) + logkws = self.logformatter.item_error( + item, ex, response, self.crawler.spider + ) logger.log( *logformatter_adapter(logkws), - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, exc_info=failure_to_exc_info(output), ) return self.signals.send_catch_log_deferred( signal=signals.item_error, item=item, response=response, - spider=spider, + spider=self.crawler.spider, failure=output, ) - logkws = self.logformatter.scraped(output, response, spider) + logkws = self.logformatter.scraped(output, response, self.crawler.spider) if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) + logger.log( + *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} + ) return self.signals.send_catch_log_deferred( - signal=signals.item_scraped, item=output, response=response, spider=spider + signal=signals.item_scraped, + item=output, + response=response, + spider=self.crawler.spider, ) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 85a3b5895..b8b0aec44 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -40,7 +40,7 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") ScrapeFunc = Callable[ - [Union[Response, Failure], Request, Spider], Union[Iterable[_T], AsyncIterable[_T]] + [Union[Response, Failure], Request], Union[Iterable[_T], AsyncIterable[_T]] ] @@ -86,8 +86,8 @@ class SpiderMiddlewareManager(MiddlewareManager): except _InvalidOutput: raise except Exception: - return scrape_func(Failure(), request, spider) - return scrape_func(response, request, spider) + return scrape_func(Failure(), request) + return scrape_func(response, request) def _evaluate_iterable( self, diff --git a/tests/test_engine.py b/tests/test_engine.py index ba4c6dc40..8928e4daf 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -487,18 +487,17 @@ def test_request_scheduled_signal(caplog): if "drop" in request.url: raise IgnoreRequest - spider = MySpider() - crawler = get_crawler(spider.__class__) + crawler = get_crawler(MySpider) engine = ExecutionEngine(crawler, lambda _: None) engine.downloader._slot_gc_loop.stop() scheduler = TestScheduler() engine.slot = Slot((), None, Mock(), scheduler) crawler.signals.connect(signal_handler, request_scheduled) keep_request = Request("https://keep.example") - engine._schedule_request(keep_request, spider) + engine._schedule_request(keep_request) drop_request = Request("https://drop.example") caplog.set_level(DEBUG) - engine._schedule_request(drop_request, spider) + engine._schedule_request(drop_request) assert scheduler.enqueued == [keep_request], ( f"{scheduler.enqueued!r} != [{keep_request!r}]" ) From 23c206af35a8a7772d63cd63fdccee085dc38e40 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 1 May 2025 22:59:18 +0500 Subject: [PATCH 094/491] Improve test coverage of Scraper. --- scrapy/core/scraper.py | 32 +++++++----------- tests/test_crawl.py | 77 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 20 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 2942dfa58..6f69d668e 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -374,32 +374,24 @@ class Scraper: errors that got propagated thru here). spider_failure: the value passed into the errback of self.call_spider() + (likely raised in the request errback) + download_failure: the value passed into _scrape2() from ExecutionEngine._handle_downloader_output() as "result" + (likely raised in the download handler or a downloader middleware) """ if not download_failure.check(IgnoreRequest): assert self.crawler.spider - if download_failure.frames: - logkws = self.logformatter.download_error( - download_failure, request, self.crawler.spider - ) - logger.log( - *logformatter_adapter(logkws), - extra={"spider": self.crawler.spider}, - exc_info=failure_to_exc_info(download_failure), - ) - else: - errmsg = download_failure.getErrorMessage() - if errmsg: - logkws = self.logformatter.download_error( - download_failure, request, self.crawler.spider, errmsg - ) - logger.log( - *logformatter_adapter(logkws), - extra={"spider": self.crawler.spider}, - ) - + logkws = self.logformatter.download_error( + download_failure, request, self.crawler.spider + ) + logger.log( + *logformatter_adapter(logkws), + extra={"spider": self.crawler.spider}, + exc_info=failure_to_exc_info(download_failure), + ) if spider_failure is not download_failure: + # a request errback raised a different exception, it needs to be handled later return spider_failure return None diff --git a/tests/test_crawl.py b/tests/test_crawl.py index f49deac1f..a8174d537 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import logging import unittest @@ -723,3 +725,78 @@ class TestCrawlSpider(TestCase): assert crawler.spider.meta[ "failure" ].value.response.headers == crawler.spider.meta.get("headers_received") + + @defer.inlineCallbacks + def test_spider_errback(self): + failures = [] + + def eb(failure: Failure) -> Failure: + failures.append(failure) + return failure + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert len(failures) == 1 + assert "HTTP status code is not handled or not allowed" in str(log) + assert "Spider error processing" not in str(log) + + @defer.inlineCallbacks + def test_spider_errback_silence(self): + failures = [] + + def eb(failure: Failure) -> None: + failures.append(failure) + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert len(failures) == 1 + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + + @defer.inlineCallbacks + def test_spider_errback_exception(self): + def eb(failure: Failure) -> None: + raise ValueError("foo") + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert "Spider error processing" in str(log) + + @defer.inlineCallbacks + def test_spider_errback_downloader_error(self): + failures = [] + + def eb(failure: Failure) -> Failure: + failures.append(failure) + return failure + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert len(failures) == 1 + assert "Error downloading" in str(log) + assert "Spider error processing" not in str(log) + + @defer.inlineCallbacks + def test_spider_errback_exception_downloader_error(self): + def eb(failure: Failure) -> None: + raise ValueError("foo") + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "Error downloading" in str(log) + assert "Spider error processing" in str(log) From da9078c4bb942be8f55495d9a44ea522f3cdcbc4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 1 May 2025 23:12:39 +0500 Subject: [PATCH 095/491] Add tests for raising CloseSpider in callbacks. --- tests/test_crawl.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index a8174d537..b85f56909 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -16,7 +16,7 @@ from twisted.trial.unittest import TestCase from scrapy import signals from scrapy.crawler import CrawlerRunner -from scrapy.exceptions import StopDownload +from scrapy.exceptions import CloseSpider, StopDownload from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import to_unicode @@ -800,3 +800,25 @@ class TestCrawlSpider(TestCase): ) assert "Error downloading" in str(log) assert "Spider error processing" in str(log) + + @defer.inlineCallbacks + def test_raise_closespider(self): + def cb(response): + raise CloseSpider + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) + assert "Closing spider (cancelled)" in str(log) + assert "Spider error processing" not in str(log) + + @defer.inlineCallbacks + def test_raise_closespider_reason(self): + def cb(response): + raise CloseSpider("my_reason") + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) + assert "Closing spider (my_reason)" in str(log) + assert "Spider error processing" not in str(log) From 5dfe7cd7b87ffc8bb287934fa3d6ffbcd63da332 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 5 May 2025 11:36:52 +0400 Subject: [PATCH 096/491] Improve tests for start items. (#6770) --- tests/test_crawl.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index b85f56909..b7a8a9628 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -188,11 +188,18 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_requests_items(self): + items = [] + + def _on_item_scraped(item): + items.append(item) + with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(StartRequestsItemSpider) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) assert len(log.records) == 0 + assert items == [{"name": "test item"}] @defer.inlineCallbacks def test_start_requests_unsupported_output(self): @@ -201,11 +208,19 @@ class TestCrawl(TestCase): things fail when ItemAdapter is actually used on the corresponding non-item object.""" + items = [] + + def _on_item_scraped(item): + items.append(item) + with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(StartRequestsGoodAndBadOutput) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) assert len(log.records) == 0 + assert len(items) == 3 + assert not any(isinstance(item, Request) for item in items) @defer.inlineCallbacks def test_start_requests_laziness(self): From ff1ac75c9ef538b49212dbb1d4112b3653efab12 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 5 May 2025 11:37:38 +0400 Subject: [PATCH 097/491] Fix shutdown tests. (#6772) --- tests/CrawlerProcess/sleeping.py | 4 +++- tests/test_crawler.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py index 45479ea4f..cb8f869e1 100644 --- a/tests/CrawlerProcess/sleeping.py +++ b/tests/CrawlerProcess/sleeping.py @@ -1,3 +1,5 @@ +import sys + from twisted.internet.defer import Deferred import scrapy @@ -14,7 +16,7 @@ class SleepingSpider(scrapy.Spider): from twisted.internet import reactor d = Deferred() - reactor.callLater(int(self.sleep), d.callback, None) + reactor.callLater(int(sys.argv[1]), d.callback, None) await maybe_deferred_to_future(d) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 6c465f000..efb346dde 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -890,7 +890,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_shutdown_graceful(self): sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py", "-a", "sleep=3") + args = self.get_script_args("sleeping.py", "3") p = PopenSpawn(args, timeout=5) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") @@ -904,7 +904,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): from twisted.internet import reactor sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py", "-a", "sleep=10") + args = self.get_script_args("sleeping.py", "10") p = PopenSpawn(args, timeout=5) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") From 2a1edbd473e47b15183e57d975d71db0cb3a2197 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 5 May 2025 11:44:17 +0400 Subject: [PATCH 098/491] Remove usages of TestCase._wait(). (#6773) --- tests/test_downloadermiddleware.py | 146 +++++++++++++---------------- tests/test_spidermiddleware.py | 53 ++++++----- 2 files changed, 90 insertions(+), 109 deletions(-) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 8e718ad5b..408160ccb 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,17 +1,18 @@ +from __future__ import annotations + import asyncio from gzip import BadGzipFile from unittest import mock import pytest -from twisted.internet import defer -from twisted.internet.defer import Deferred -from twisted.python.failure import Failure +from twisted.internet.defer import Deferred, succeed from twisted.trial.unittest import TestCase from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, get_from_asyncio_queue @@ -29,38 +30,36 @@ class TestManagerBase(TestCase): def tearDown(self): return self.crawler.engine.close_spider(self.spider) - def _download(self, request, response=None): + async def _download( + self, request: Request, response: Response | None = None + ) -> Response | Request: """Executes downloader mw manager's download method and returns - the result (Request or Response) or raise exception in case of + the result (Request or Response) or raises exception in case of failure. """ if not response: response = Response(request.url) - def download_func(request, spider): - return response + def download_func(request: Request, spider: Spider) -> Deferred[Response]: + return succeed(response) - dfd = self.mwman.download(download_func, request, self.spider) - # catch deferred result and return the value - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - ret = results[0] - if isinstance(ret, Failure): - ret.raiseException() - return ret + return await maybe_deferred_to_future( + self.mwman.download(download_func, request, self.spider) + ) class TestDefaults(TestManagerBase): """Tests default behavior with default settings""" - def test_request_response(self): + @deferred_f_from_coro_f + async def test_request_response(self): req = Request("http://example.com/index.html") resp = Response(req.url, status=200) - ret = self._download(req, resp) + ret = await self._download(req, resp) assert isinstance(ret, Response), "Non-response returned" - def test_3xx_and_invalid_gzipped_body_must_redirect(self): + @deferred_f_from_coro_f + async def test_3xx_and_invalid_gzipped_body_must_redirect(self): """Regression test for a failure when redirecting a compressed request. @@ -85,13 +84,14 @@ class TestDefaults(TestManagerBase): "Location": "http://example.com/login", }, ) - ret = self._download(request=req, response=resp) + ret = await self._download(req, resp) assert isinstance(ret, Request), f"Not redirected: {ret!r}" assert to_bytes(ret.url) == resp.headers["Location"], ( "Not redirected to location header" ) - def test_200_and_invalid_gzipped_body_must_fail(self): + @deferred_f_from_coro_f + async def test_200_and_invalid_gzipped_body_must_fail(self): req = Request("http://example.com") body = b"

You are being redirected

" resp = Response( @@ -106,13 +106,14 @@ class TestDefaults(TestManagerBase): }, ) with pytest.raises(BadGzipFile): - self._download(request=req, response=resp) + await self._download(req, resp) class TestResponseFromProcessRequest(TestManagerBase): """Tests middleware returning a response from process_request.""" - def test_download_func_not_called(self): + @deferred_f_from_coro_f + async def test_download_func_not_called(self): resp = Response("http://example.com/index.html") class ResponseMiddleware: @@ -123,19 +124,17 @@ class TestResponseFromProcessRequest(TestManagerBase): req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - assert results[0] is resp + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp assert not download_func.called -class TestProcessRequestInvalidOutput(TestManagerBase): - """Invalid return value for process_request method should raise an exception""" - - def test_invalid_process_request(self): +class TestInvalidOutput(TestManagerBase): + @deferred_f_from_coro_f + async def test_invalid_process_request(self): + """Invalid return value for process_request method should raise an exception""" req = Request("http://example.com/index.html") class InvalidProcessRequestMiddleware: @@ -143,18 +142,12 @@ class TestProcessRequestInvalidOutput(TestManagerBase): return 1 self.mwman._add_middleware(InvalidProcessRequestMiddleware()) - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - assert isinstance(results[0], Failure) - assert isinstance(results[0].value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._download(req) - -class TestProcessResponseInvalidOutput(TestManagerBase): - """Invalid return value for process_response method should raise an exception""" - - def test_invalid_process_response(self): + @deferred_f_from_coro_f + async def test_invalid_process_response(self): + """Invalid return value for process_response method should raise an exception""" req = Request("http://example.com/index.html") class InvalidProcessResponseMiddleware: @@ -162,18 +155,12 @@ class TestProcessResponseInvalidOutput(TestManagerBase): return 1 self.mwman._add_middleware(InvalidProcessResponseMiddleware()) - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - assert isinstance(results[0], Failure) - assert isinstance(results[0].value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._download(req) - -class TestProcessExceptionInvalidOutput(TestManagerBase): - """Invalid return value for process_exception method should raise an exception""" - - def test_invalid_process_exception(self): + @deferred_f_from_coro_f + async def test_invalid_process_exception(self): + """Invalid return value for process_exception method should raise an exception""" req = Request("http://example.com/index.html") class InvalidProcessExceptionMiddleware: @@ -184,18 +171,15 @@ class TestProcessExceptionInvalidOutput(TestManagerBase): return 1 self.mwman._add_middleware(InvalidProcessExceptionMiddleware()) - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - assert isinstance(results[0], Failure) - assert isinstance(results[0].value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._download(req) class TestMiddlewareUsingDeferreds(TestManagerBase): """Middlewares using Deferreds should work""" - def test_deferred(self): + @deferred_f_from_coro_f + async def test_deferred(self): resp = Response("http://example.com/index.html") class DeferredMiddleware: @@ -211,12 +195,10 @@ class TestMiddlewareUsingDeferreds(TestManagerBase): self.mwman._add_middleware(DeferredMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - assert results[0] is resp + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp assert not download_func.called @@ -224,27 +206,27 @@ class TestMiddlewareUsingDeferreds(TestManagerBase): class TestMiddlewareUsingCoro(TestManagerBase): """Middlewares using asyncio coroutines should work""" - def test_asyncdef(self): + @deferred_f_from_coro_f + async def test_asyncdef(self): resp = Response("http://example.com/index.html") class CoroMiddleware: async def process_request(self, request, spider): - await defer.succeed(42) + await succeed(42) return resp self.mwman._add_middleware(CoroMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - assert results[0] is resp + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp assert not download_func.called @pytest.mark.only_asyncio - def test_asyncdef_asyncio(self): + @deferred_f_from_coro_f + async def test_asyncdef_asyncio(self): resp = Response("http://example.com/index.html") class CoroMiddleware: @@ -255,10 +237,8 @@ class TestMiddlewareUsingCoro(TestManagerBase): self.mwman._add_middleware(CoroMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - assert results[0] is resp + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp assert not download_func.called diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index ddc9b5206..1d671134e 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,12 +1,12 @@ from __future__ import annotations from collections.abc import AsyncIterator, Iterable +from typing import Any from unittest import mock import pytest from testfixtures import LogCapture from twisted.internet import defer -from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from scrapy.core.spidermw import SpiderMiddlewareManager @@ -14,7 +14,11 @@ from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen -from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + deferred_from_coro, + maybe_deferred_to_future, +) from scrapy.utils.test import get_crawler @@ -26,53 +30,51 @@ class TestSpiderMiddleware(TestCase): self.spider = self.crawler._create_spider("foo") self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - def _scrape_response(self): + async def _scrape_response(self) -> Any: """Execute spider mw manager's scrape_response method and return the result. Raise exception in case of failure. """ scrape_func = mock.MagicMock() - dfd = self.mwman.scrape_response( - scrape_func, self.response, self.request, self.spider + return await maybe_deferred_to_future( + self.mwman.scrape_response( + scrape_func, self.response, self.request, self.spider + ) ) - # catch deferred result and return the value - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - return results[0] class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_input method""" - def test_invalid_process_spider_input(self): + @deferred_f_from_coro_f + async def test_invalid_process_spider_input(self): class InvalidProcessSpiderInputMiddleware: def process_spider_input(self, response, spider): return 1 self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware()) - result = self._scrape_response() - assert isinstance(result, Failure) - assert isinstance(result.value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._scrape_response() class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_output method""" - def test_invalid_process_spider_output(self): + @deferred_f_from_coro_f + async def test_invalid_process_spider_output(self): class InvalidProcessSpiderOutputMiddleware: def process_spider_output(self, response, result, spider): return 1 self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware()) - result = self._scrape_response() - assert isinstance(result, Failure) - assert isinstance(result.value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._scrape_response() class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_exception method""" - def test_invalid_process_spider_exception(self): + @deferred_f_from_coro_f + async def test_invalid_process_spider_exception(self): class InvalidProcessSpiderOutputExceptionMiddleware: def process_spider_exception(self, response, exception, spider): return 1 @@ -83,15 +85,15 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) - result = self._scrape_response() - assert isinstance(result, Failure) - assert isinstance(result.value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._scrape_response() class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): """Re raise the exception by returning None""" - def test_process_spider_exception_return_none(self): + @deferred_f_from_coro_f + async def test_process_spider_exception_return_none(self): class ProcessSpiderExceptionReturnNoneMiddleware: def process_spider_exception(self, response, exception, spider): return None @@ -102,9 +104,8 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) - result = self._scrape_response() - assert isinstance(result, Failure) - assert isinstance(result.value, ZeroDivisionError) + with pytest.raises(ZeroDivisionError): + await self._scrape_response() class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): From 509b572efc85ac8ef96224d560a8851322a3f606 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 5 May 2025 11:51:53 +0400 Subject: [PATCH 099/491] Migrate the build system to hatchling. (#6771) --- MANIFEST.in | 22 ---------------------- pyproject.toml | 45 ++++++++++++++++++++++++++++++--------------- 2 files changed, 30 insertions(+), 37 deletions(-) delete mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 7700ae7bd..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,22 +0,0 @@ -include CODE_OF_CONDUCT.md -include CONTRIBUTING.md -include INSTALL.md -include NEWS -include SECURITY.md - -include scrapy/VERSION -include scrapy/mime.types -include scrapy/py.typed - -include codecov.yml -include conftest.py -include tox.ini - -recursive-include scrapy/templates * -recursive-include docs * -prune docs/build - -recursive-include extras * -recursive-include tests * - -global-exclude __pycache__ *.py[cod] diff --git a/pyproject.toml b/pyproject.toml index 84bf41a94..e14efdd17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] -requires = ["setuptools >= 61.0"] -build-backend = "setuptools.build_meta" +requires = ["hatchling>=1.27.0"] +build-backend = "hatchling.build" [project] name = "Scrapy" @@ -10,29 +10,28 @@ dependencies = [ "Twisted>=21.7.0", "cryptography>=37.0.0", "cssselect>=0.9.1", + "defusedxml>=0.7.1", + "itemadapter>=0.1.0", "itemloaders>=1.0.1", + "lxml>=4.6.0", + "packaging", "parsel>=1.5.0", + "protego>=0.1.15", "pyOpenSSL>=22.0.0", "queuelib>=1.4.2", "service_identity>=18.1.0", + "tldextract", "w3lib>=1.17.0", "zope.interface>=5.1.0", - "protego>=0.1.15", - "itemadapter>=0.1.0", - "packaging", - "tldextract", - "lxml>=4.6.0", - "defusedxml>=0.7.1", # Platform-specific dependencies 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"', 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"', ] classifiers = [ - "Framework :: Scrapy", "Development Status :: 5 - Production/Stable", "Environment :: Console", + "Framework :: Scrapy", "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", @@ -47,6 +46,8 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules", ] +license = "BSD-3-Clause" +license-files = ["LICENSE", "AUTHORS"] readme = "README.rst" requires-python = ">=3.9" authors = [{ name = "Scrapy developers", email = "pablo@pablohoffman.com" }] @@ -63,12 +64,26 @@ releasenotes = "https://docs.scrapy.org/en/latest/news.html" [project.scripts] scrapy = "scrapy.cmdline:execute" -[tool.setuptools.packages.find] -where = ["."] -include = ["scrapy", "scrapy.*",] +[tool.hatch.build.targets.sdist] +include = [ + "/docs", + "/extras", + "/scrapy", + "/tests", + "/tests_typing", + "/CODE_OF_CONDUCT.md", + "/CONTRIBUTING.md", + "/INSTALL.md", + "/NEWS", + "/SECURITY.md", + "/codecov.yml", + "/conftest.py", + "/tox.ini", +] -[tool.setuptools.dynamic] -version = {file = "./scrapy/VERSION"} +[tool.hatch.version] +path = "scrapy/VERSION" +pattern = "^(?P.+)$" [tool.mypy] ignore_missing_imports = true From b93290f28affdc0bdd780672ec1adb9d7def4940 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 5 May 2025 19:38:04 +0500 Subject: [PATCH 100/491] Add a list of Deferred-only APIs. --- docs/topics/coroutines.rst | 95 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 5 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 1c80857f6..439474310 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -62,18 +62,103 @@ In addition to native coroutine APIs Scrapy has some APIs that return a :class:`~twisted.internet.defer.Deferred` object or take a user-supplied function that returns a :class:`~twisted.internet.defer.Deferred` object. These APIs are also asynchronous but don't yet support native ``async def`` syntax. -For example: +In the future we plan to add support for the ``async def`` syntax to these APIs +or replace them with other APIs where changing the existing ones is +possible. -- The :meth:`ExecutionEngine.download` method returns a - :class:`~twisted.internet.defer.Deferred` object. -- A custom download handler needs to define a ``download_request()`` method that - returns a :class:`~twisted.internet.defer.Deferred` object. +The following Scrapy methods return :class:`~twisted.internet.defer.Deferred` +objects (this list is not complete as it only includes methods that we think +may be useful for user code): + +- :class:`scrapy.crawler.Crawler`: + + - :meth:`~scrapy.crawler.Crawler.crawl` + + - :meth:`~scrapy.crawler.Crawler.stop` + +- :class:`scrapy.crawler.CrawlerRunner` (also inherited by + :class:`scrapy.crawler.CrawlerProcess`): + + - :meth:`~scrapy.crawler.CrawlerRunner.crawl` + + - :meth:`~scrapy.crawler.CrawlerRunner.stop` + + - :meth:`~scrapy.crawler.CrawlerRunner.join` + +- :class:`scrapy.core.engine.ExecutionEngine`: + + - :meth:`~scrapy.core.engine.ExecutionEngine.download` + +- :class:`scrapy.signalmanager.SignalManager`: + + - :meth:`~scrapy.signalmanager.SignalManager.send_catch_log_deferred` + +- :class:`~scrapy.mail.MailSender` + + - :meth:`~scrapy.mail.MailSender.send` + +The following user-supplied methods can return +:class:`~twisted.internet.defer.Deferred` objects (the methods that can also +return coroutines are listed in :ref:`coroutine-support`): + +- Custom download handlers (see :setting:`DOWNLOAD_HANDLERS`): + + - ``download_request()`` + + - ``close()`` + +- Custom downloader implementations (see :setting:`DOWNLOADER`): + + - ``fetch()`` + +- Custom scheduler implementations (see :setting:`SCHEDULER`): + + - :meth:`~scrapy.core.scheduler.BaseScheduler.open` + + - :meth:`~scrapy.core.scheduler.BaseScheduler.close` + +- Custom dupefilters (see :setting:`DUPEFILTER_CLASS`): + + - ``open()`` + + - ``close()`` + +- Custom feed storages (see :setting:`FEED_STORAGES`): + + - ``store()`` + +- Subclasses of :class:`scrapy.pipelines.media.MediaPipeline`: + + - ``media_to_download()`` + + - ``item_completed()`` + +- Custom storages used by subclasses of + :class:`scrapy.pipelines.files.FilesPipeline`: + + - ``persist_file()`` + + - ``stat_file()`` In most cases you can use these APIs in code that otherwise uses coroutines, by wrapping a :class:`~twisted.internet.defer.Deferred` object into a :class:`~asyncio.Future` object or vice versa. See :ref:`asyncio-await-dfd` for more information about this. +For example: + +- The :meth:`ExecutionEngine.download() + ` method returns a + :class:`~twisted.internet.defer.Deferred` object that fires with the + downloaded response. You can use this object directly in Deferred-based + code or convert it into a :class:`~asyncio.Future` object with + :func:`~scrapy.utils.defer.maybe_deferred_to_future`. +- A custom download handler needs to define a ``download_request()`` method + that returns a :class:`~twisted.internet.defer.Deferred` object. You can + write a method that works with Deferreds and returns one directly, or you + can write a coroutine and convert it into a functions that returns a + Deferred with :func:`~scrapy.utils.defer.deferred_f_from_coro_f`. + General usage ============= From 523fc25c4d7550d721e8160f4f09cb61f94d53d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 5 May 2025 18:51:15 +0200 Subject: [PATCH 101/491] Document default values set by startproject (#6775) --- docs/topics/feed-exports.rst | 10 +++++----- docs/topics/settings.rst | 23 +++++++++++++++-------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 7f401f0c7..2184f2d0e 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -539,18 +539,18 @@ as a fallback value if that key is not provided for a specific feed definition: FEED_EXPORT_ENCODING -------------------- -Default: ``None`` +Default: ``"utf-8"`` (:ref:`fallback `: ``None``) The encoding to be used for the feed. -If unset or set to ``None`` (default) it uses UTF-8 for everything except JSON output, -which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons. +If set to ``None``, it uses UTF-8 for everything except JSON output, which uses +safe numeric encoding (``\uXXXX`` sequences) for historic reasons. -Use ``utf-8`` if you want UTF-8 for JSON too. +Use ``"utf-8"`` if you want UTF-8 for JSON too. .. versionchanged:: 2.8 The :command:`startproject` command now sets this setting to - ``utf-8`` in the generated ``settings.py`` file. + ``"utf-8"`` in the generated ``settings.py`` file. .. setting:: FEED_EXPORT_FIELDS diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a59a61050..3a61306d6 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -162,8 +162,17 @@ Those command-specific default settings are specified in the 6. Default global settings -------------------------- -The global defaults are located in the ``scrapy.settings.default_settings`` -module and documented in the :ref:`topics-settings-ref` section. +The ``scrapy.settings.default_settings`` module defines global default values +for some :ref:`built-in settings `. + +.. note:: :command:`startproject` generates a ``settings.py`` file that sets + some settings to different values. + + The reference documentation of settings indicates the default value if one + exists. If :command:`startproject` sets a value, that value is documented + as default, and the value from ``scrapy.settings.default_settings`` is + documented as “fallback”. + Compatibility with pickle ========================= @@ -461,7 +470,7 @@ Note that the event loop class must inherit from :class:`asyncio.AbstractEventLo BOT_NAME -------- -Default: ``'scrapybot'`` +Default: ```` (:ref:`fallback `: ``'scrapybot'``) The name of the bot implemented by this Scrapy project (also known as the project name). This name will be used for the logging too. @@ -1563,7 +1572,7 @@ email notifying about it. If zero, no warning will be produced. NEWSPIDER_MODULE ---------------- -Default: ``''`` +Default: ``".spiders"`` (:ref:`fallback `: ``""``) Module where to create new spiders using the :command:`genspider` command. @@ -1622,9 +1631,7 @@ Adjust redirect request priority relative to original request: ROBOTSTXT_OBEY -------------- -Default: ``False`` - -Scope: ``scrapy.downloadermiddlewares.robotstxt`` +Default: ``True`` (:ref:`fallback `: ``False``) If enabled, Scrapy will respect robots.txt policies. For more information see :ref:`topics-dlmw-robots`. @@ -1838,7 +1845,7 @@ the spider. For more info see :ref:`topics-spider-middleware-setting`. SPIDER_MODULES -------------- -Default: ``[]`` +Default: ``[".spiders"]`` (:ref:`fallback `: ``[]``) A list of modules where Scrapy will look for spiders. From acb5f895cd0b0f63f3dafdd3025b314830ab4a67 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 5 May 2025 22:28:36 +0500 Subject: [PATCH 102/491] Update docs/topics/coroutines.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/coroutines.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 439474310..8af4ce71d 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -156,7 +156,7 @@ For example: - A custom download handler needs to define a ``download_request()`` method that returns a :class:`~twisted.internet.defer.Deferred` object. You can write a method that works with Deferreds and returns one directly, or you - can write a coroutine and convert it into a functions that returns a + can write a coroutine and convert it into a function that returns a Deferred with :func:`~scrapy.utils.defer.deferred_f_from_coro_f`. From 4899d416e701c4d405fdc77fa2ba31327541f292 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 6 May 2025 14:31:28 +0400 Subject: [PATCH 103/491] Add PyPy 3.11 to CI. (#6697) --- .github/workflows/tests-ubuntu.yml | 5 ++++- tox.ini | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 34819f227..06da46ca1 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -38,6 +38,9 @@ jobs: - python-version: pypy3.10 env: TOXENV: pypy3 + - python-version: pypy3.11 + env: + TOXENV: pypy3 # pinned deps - python-version: "3.9.21" @@ -59,7 +62,7 @@ jobs: - python-version: "3.13" env: TOXENV: extra-deps - - python-version: pypy3.10 + - python-version: pypy3.11 env: TOXENV: pypy3-extra-deps - python-version: "3.13" diff --git a/tox.ini b/tox.ini index 59572442d..e63e44189 100644 --- a/tox.ini +++ b/tox.ini @@ -143,7 +143,7 @@ deps = google-cloud-storage ipython robotexclusionrulesparser - uvloop; platform_system != "Windows" + uvloop; platform_system != "Windows" and implementation_name != "pypy" zstandard; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests [testenv:extra-deps-pinned] @@ -159,7 +159,7 @@ deps = google-cloud-storage==1.29.0 ipython==2.0.0 robotexclusionrulesparser==1.6.2 - uvloop==0.14.0; platform_system != "Windows" + uvloop==0.14.0; platform_system != "Windows" and implementation_name != "pypy" zstandard==0.1; implementation_name != "pypy" install_command = {[pinned]install_command} setenv = From 373e501f78703e9fae2b9e970071052a13b1a18e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 7 May 2025 16:11:22 +0200 Subject: [PATCH 104/491] Link to scrapy.org from the docs (#6780) --- docs/_templates/layout.html | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 docs/_templates/layout.html diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 000000000..6ec565e24 --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,23 @@ +{% extends "!layout.html" %} + +{# Overriden to include a link to scrapy.org, not just to the docs root #} +{%- block sidebartitle %} + +{# the logo helper function was removed in Sphinx 6 and deprecated since Sphinx 4 #} +{# the master_doc variable was renamed to root_doc in Sphinx 4 (master_doc still exists in later Sphinx versions) #} +{%- set _logo_url = logo_url|default(pathto('_static/' + (logo or ""), 1)) %} +{%- set _root_doc = root_doc|default(master_doc) %} +scrapy.org / docs + +{%- if READTHEDOCS or DEBUG %} + {%- if theme_version_selector or theme_language_selector %} +
+
+
+
+ {%- endif %} +{%- endif %} + +{%- include "searchbox.html" %} + +{%- endblock %} From 036f3e562716aaf67a4d0ff1c8011281394ef240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 7 May 2025 19:04:03 +0200 Subject: [PATCH 105/491] Support asynchronous start requests (#6729) --- docs/faq.rst | 24 +- docs/intro/tutorial.rst | 39 +- docs/news.rst | 133 ++++++- docs/topics/api.rst | 6 + docs/topics/architecture.rst | 2 +- docs/topics/components.rst | 4 + docs/topics/coroutines.rst | 28 +- docs/topics/jobs.rst | 6 +- docs/topics/request-response.rst | 10 +- docs/topics/scheduler.rst | 8 +- docs/topics/settings.rst | 62 ++- docs/topics/signals.rst | 19 +- docs/topics/spider-middleware.rst | 46 ++- docs/topics/spiders.rst | 119 +++--- docs/topics/telnetconsole.rst | 12 +- extras/qpsclient.py | 4 + pyproject.toml | 4 +- scrapy/commands/bench.py | 8 +- scrapy/commands/check.py | 10 +- scrapy/commands/fetch.py | 8 +- scrapy/commands/parse.py | 6 +- scrapy/commands/shell.py | 4 +- scrapy/core/engine.py | 248 +++++++----- scrapy/core/scheduler.py | 287 ++++++++++---- scrapy/core/scraper.py | 333 ++++++++-------- scrapy/core/spidermw.py | 203 ++++++++-- scrapy/crawler.py | 8 +- scrapy/extensions/telnet.py | 1 - scrapy/http/request/__init__.py | 12 +- scrapy/logformatter.py | 2 +- scrapy/pqueues.py | 126 ++++-- scrapy/settings/default_settings.py | 3 + scrapy/shell.py | 23 +- scrapy/signalmanager.py | 21 +- scrapy/signals.py | 1 + scrapy/spidermiddlewares/base.py | 63 +-- scrapy/spidermiddlewares/depth.py | 11 +- scrapy/spidermiddlewares/offsite.py | 5 +- scrapy/spidermiddlewares/referer.py | 5 +- scrapy/spidermiddlewares/start.py | 31 ++ scrapy/spidermiddlewares/urllength.py | 2 +- scrapy/spiders/__init__.py | 79 +++- scrapy/spiders/crawl.py | 6 +- scrapy/spiders/init.py | 10 +- scrapy/spiders/sitemap.py | 6 +- .../project/module/middlewares.py.tmpl | 13 +- scrapy/utils/asyncgen.py | 8 +- scrapy/utils/defer.py | 34 +- scrapy/utils/engine.py | 8 +- scrapy/utils/python.py | 17 +- scrapy/utils/reactor.py | 25 +- sep/sep-018.rst | 2 +- tests/CrawlerProcess/args_settings.py | 5 +- tests/CrawlerProcess/asyncio_custom_loop.py | 5 +- .../asyncio_enabled_no_reactor.py | 5 +- .../CrawlerProcess/asyncio_enabled_reactor.py | 5 +- .../asyncio_enabled_reactor_different_loop.py | 5 +- .../asyncio_enabled_reactor_same_loop.py | 5 +- .../caching_hostname_resolver.py | 2 +- tests/CrawlerProcess/multi.py | 5 +- tests/CrawlerProcess/reactor_default.py | 5 +- .../reactor_default_twisted_reactor_select.py | 5 +- tests/CrawlerProcess/reactor_select.py | 5 +- ..._select_subclass_twisted_reactor_select.py | 5 +- .../reactor_select_twisted_reactor_select.py | 5 +- tests/CrawlerProcess/simple.py | 5 +- tests/CrawlerRunner/change_reactor.py | 5 +- tests/CrawlerRunner/ip_address.py | 2 +- tests/__init__.py | 6 + tests/spiders.py | 36 +- .../__init__.py | 17 +- tests/test_commands.py | 63 +-- tests/test_contracts.py | 5 +- tests/test_crawl.py | 70 +--- tests/test_crawler.py | 13 +- tests/test_downloadermiddleware.py | 2 +- tests/test_downloaderslotssettings.py | 2 +- tests/test_engine.py | 39 +- tests/test_engine_loop.py | 364 ++++++++++++++++++ tests/test_pipelines.py | 2 +- tests/test_request_cb_kwargs.py | 14 +- tests/test_scheduler.py | 35 +- tests/test_signals.py | 21 +- tests/test_spider.py | 57 ++- tests/test_spider_start.py | 186 +++++++++ tests/test_spidermiddleware.py | 53 +-- tests/test_spidermiddleware_base.py | 74 ++-- tests/test_spidermiddleware_httperror.py | 2 +- tests/test_spidermiddleware_output_chain.py | 16 +- tests/test_spidermiddleware_process_start.py | 352 +++++++++++++++++ tests/test_spidermiddleware_start.py | 44 +++ tests/utils/__init__.py | 9 + tox.ini | 2 +- 93 files changed, 2777 insertions(+), 936 deletions(-) create mode 100644 scrapy/spidermiddlewares/start.py create mode 100644 tests/test_engine_loop.py create mode 100644 tests/test_spider_start.py create mode 100644 tests/test_spidermiddleware_process_start.py create mode 100644 tests/test_spidermiddleware_start.py diff --git a/docs/faq.rst b/docs/faq.rst index da255f29e..1d09a0e63 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -96,30 +96,13 @@ How can I simulate a user login in my spider? See :ref:`topics-request-response-ref-request-userlogin`. + .. _faq-bfo-dfo: Does Scrapy crawl in breadth-first or depth-first order? -------------------------------------------------------- -By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which -basically means that it crawls in `DFO order`_. This order is more convenient -in most cases. - -If you do want to crawl in true `BFO order`_, you can do it by -setting the following settings: - -.. code-block:: python - - DEPTH_PRIORITY = 1 - SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue" - SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" - -While pending requests are below the configured values of -:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or -:setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent -concurrently. As a result, the first few requests of a crawl rarely follow the -desired order. Lowering those settings to ``1`` enforces the desired order, but -it significantly slows down the crawl as a whole. +:ref:`DFO by default, but other orders are possible `. My Scrapy crawler has memory leaks. What can I do? @@ -436,6 +419,3 @@ See :issue:`2680`. .. _Python standard library modules: https://docs.python.org/3/py-modindex.html .. _Python package: https://pypi.org/ .. _user agents: https://en.wikipedia.org/wiki/User_agent -.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) -.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search -.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 5041b49ea..c4e04364b 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -94,7 +94,7 @@ This is the code for our first Spider. Save it in a file named class QuotesSpider(scrapy.Spider): name = "quotes" - def start_requests(self): + async def start(self): urls = [ "https://quotes.toscrape.com/page/1/", "https://quotes.toscrape.com/page/2/", @@ -116,10 +116,10 @@ and defines some attributes and methods: unique within a project, that is, you can't set the same name for different Spiders. -* :meth:`~scrapy.Spider.start_requests`: must return an iterable of - Requests (you can return a list of requests or write a generator function) - which the Spider will begin to crawl from. Subsequent requests will be - generated successively from these initial requests. +* :meth:`~scrapy.Spider.start`: must be an asynchronous generator that + yields requests (and, optionally, items) for the spider to start crawling. + Subsequent requests will be generated successively from these initial + requests. * :meth:`~scrapy.Spider.parse`: a method that will be called to handle the response downloaded for each of the requests made. The response parameter @@ -164,21 +164,22 @@ for the respective URLs, as our ``parse`` method instructs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Scrapy schedules the :class:`scrapy.Request ` objects -returned by the ``start_requests`` method of the Spider. Upon receiving a -response for each one, it instantiates :class:`~scrapy.http.Response` objects -and calls the callback method associated with the request (in this case, the -``parse`` method) passing the response as an argument. +Scrapy sends the first :class:`scrapy.Request ` objects yielded +by the :meth:`~scrapy.Spider.start` spider method. Upon receiving a +response for each one, Scrapy calls the callback method associated with the +request (in this case, the ``parse`` method) with a +:class:`~scrapy.http.Response` object. -A shortcut to the start_requests method ---------------------------------------- -Instead of implementing a :meth:`~scrapy.Spider.start_requests` method -that generates :class:`scrapy.Request ` objects from URLs, -you can just define a :attr:`~scrapy.Spider.start_urls` class attribute -with a list of URLs. This list will then be used by the default implementation -of :meth:`~scrapy.Spider.start_requests` to create the initial requests -for your spider. +A shortcut to the ``start`` method +---------------------------------- + +Instead of implementing a :meth:`~scrapy.Spider.start` method that yields +:class:`~scrapy.Request` objects from URLs, you can define a +:attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This +list will then be used by the default implementation of +:meth:`~scrapy.Spider.start` to create the initial requests for your +spider. .. code-block:: python @@ -794,7 +795,7 @@ with a specific tag, building the URL based on the argument: class QuotesSpider(scrapy.Spider): name = "quotes" - def start_requests(self): + async def start(self): url = "https://quotes.toscrape.com/" tag = getattr(self, "tag", None) if tag is not None: diff --git a/docs/news.rst b/docs/news.rst index 9f476ee21..64a3ad2b1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -8,6 +8,11 @@ Release notes Scrapy VERSION (unreleased) --------------------------- +Highlights: + +- Replaced ``start_requests()`` (sync) with :meth:`~scrapy.Spider.start` + (async) and changed how it is iterated. + Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -20,6 +25,116 @@ Backward-incompatible changes :class:`~scrapy.crawler.Crawler` instance at run time). Please use ``from_crawler()`` instead. +- The iteration of start requests and items no longer stops once there are + requests in the scheduler, and instead runs continuously until all start + requests have been scheduled. + + To reproduce the previous behavior, see :ref:`start-requests-lazy`. + +- An unhandled exception from the + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.open_spider` method of a + :ref:`spider middleware ` no longer stops the + crawl. + +- In ``scrapy.core.engine.ExecutionEngine``: + + - The second parameter of ``open_spider()``, ``start_requests``, has been + removed. The start requests are determined by the ``spider`` parameter + instead (see :meth:`~scrapy.Spider.start`). + + - The ``slot`` attribute has been renamed to ``_slot`` and should not be + used. + +- In ``scrapy.core.engine``, the ``Slot`` class has been renamed to ``_Slot`` + and should not be used. + +- The ``slot`` :ref:`telnet variable ` has been removed. + +- In ``scrapy.core.spidermw.SpiderMiddlewareManager``, + ``process_start_requests()`` has been replaced by ``process_start()``. + +- The now-deprecated ``start_requests()`` method, when it returns an iterable + instead of being defined as a generator, is now executed *after* the + :ref:`scheduler ` instance has been created. + +- When using :setting:`JOBDIR`, :ref:`start requests ` are + now serialized into their own, ``s``-suffixed priority folders. You can set + :setting:`SCHEDULER_START_DISK_QUEUE` to ``None`` or ``""`` to change that, + but the side effects may be undesirable. See + :setting:`SCHEDULER_START_DISK_QUEUE` for details. + +Deprecations +~~~~~~~~~~~~ + +- The ``start_requests()`` method of :class:`~scrapy.Spider` is deprecated, + use :meth:`~scrapy.Spider.start` instead, or both to maintain support for + lower Scrapy versions. + + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- The ``process_start_requests()`` method of :ref:`spider middlewares + ` is deprecated, use + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead, + or both to maintain support for lower Scrapy versions. + + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- The ``__init__`` method of priority queue classes (see + :setting:`SCHEDULER_PRIORITY_QUEUE`) should now support a keyword-only + ``start_queue_cls`` parameter. + + (:issue:`6752`) + +New features +~~~~~~~~~~~~ + +- You can now yield the start requests and items of a spider from the + :meth:`~scrapy.Spider.start` spider method and from the + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` spider + middleware method, both :term:`asynchronous generators `. + + This makes it possible to use asynchronous code to generate those start + requests and items, e.g. reading them from a queue service or database + using an asynchronous client, without workarounds. + + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- Start requests are now :ref:`scheduled ` as soon as + possible. + + As a result, their :attr:`~scrapy.Request.priority` is now taken into + account as soon as :setting:`CONCURRENT_REQUESTS` is reached. + + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- :class:`Crawler.signals ` has a new + :meth:`~scrapy.signalmanager.SignalManager.wait_for` method. + +- Added a new :signal:`scheduler_empty` signal. + +- Added new settings: :setting:`SCHEDULER_START_DISK_QUEUE` and + :setting:`SCHEDULER_START_MEMORY_QUEUE`. + +- Added :class:`~scrapy.spidermiddlewares.start.StartSpiderMiddleware`, which + sets :reqmeta:`is_start_request` to ``True`` on :ref:`start requests + `. + +- Exposed a new method of :class:`Crawler.engine + `: + :meth:`~scrapy.core.engine.ExecutionEngine.needs_backout`. + +Bug fixes +~~~~~~~~~ + +- Yielding an item from :meth:`Spider.start ` or from + :meth:`SpiderMiddleware.process_start + ` no longer delays + the next iteration of starting requests and items by up to 5 seconds. + + (:issue:`6729`) + + .. _release-2.12.0: Scrapy 2.12.0 (2024-11-18) @@ -29,7 +144,7 @@ Highlights: - Dropped support for Python 3.8, added support for Python 3.13 -- :meth:`~scrapy.Spider.start_requests` can now yield items +- ``scrapy.Spider.start_requests()`` can now yield items - Added :class:`~scrapy.http.JsonResponse` @@ -320,9 +435,13 @@ Deprecations New features ~~~~~~~~~~~~ -- :meth:`~scrapy.Spider.start_requests` can now yield items. +- ``scrapy.Spider.start_requests()`` can now yield items. (:issue:`5289`, :issue:`6417`) + .. note:: Some spider middlewares may need to be updated for Scrapy 2.12 + support before you can use them in combination with the ability to + yield items from ``start_requests()``. + - Added a new :class:`~scrapy.http.Response` subclass, :class:`~scrapy.http.JsonResponse`, for responses with a `JSON MIME type `_. @@ -812,7 +931,7 @@ Backward-incompatible changes in :meth:`scrapy.Spider.from_crawler`. If you want to access the final setting values and the initialized :class:`~scrapy.crawler.Crawler` attributes in the spider code as early as possible you can do this in - :meth:`~scrapy.Spider.start_requests` or in a handler of the + ``scrapy.Spider.start_requests()`` or in a handler of the :signal:`engine_started` signal. (:issue:`6038`) - The :meth:`TextResponse.json ` method now @@ -3388,7 +3507,7 @@ New features * :class:`~scrapy.spiders.Spider` objects now raise an :exc:`AttributeError` exception if they do not have a :class:`~scrapy.spiders.Spider.start_urls` - attribute nor reimplement :class:`~scrapy.spiders.Spider.start_requests`, + attribute nor reimplement ``scrapy.spiders.Spider.start_requests()``, but have a ``start_url`` attribute (:issue:`4133`, :issue:`4170`) * :class:`~scrapy.exporters.BaseItemExporter` subclasses may now use @@ -6309,7 +6428,7 @@ Scrapy 0.18.4 (released 2013-10-10) - IPython refuses to update the namespace. fix #396 (:commit:`3d32c4f`) - Fix AlreadyCalledError replacing a request in shell command. closes #407 (:commit:`b1d8919`) -- Fix start_requests laziness and early hangs (:commit:`89faf52`) +- Fix ``start_requests()`` laziness and early hangs (:commit:`89faf52`) Scrapy 0.18.3 (released 2013-10-03) ----------------------------------- @@ -6502,7 +6621,7 @@ Scrapy changes: - added options ``-o`` and ``-t`` to the :command:`runspider` command - documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED` - major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals. -- added :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests` method to spider middlewares +- added a ``process_start_requests()`` method to spider middlewares - dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info. - dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info. - documented :ref:`topics-api` @@ -6565,7 +6684,7 @@ Scrapy 0.14.2 - fixed bug in MemoryUsage extension: get_engine_status() takes exactly 1 argument (0 given) (:commit:`11133e9`) - fixed struct.error on http compression middleware. closes #87 (:commit:`1423140`) - ajax crawling wasn't expanding for unicode urls (:commit:`0de3fb4`) -- Catch start_requests iterator errors. refs #83 (:commit:`454a21d`) +- Catch ``start_requests()`` iterator errors. refs #83 (:commit:`454a21d`) - Speed-up libxml2 XPathSelector (:commit:`2fbd662`) - updated versioning doc according to recent changes (:commit:`0a070f5`) - scrapyd: fixed documentation link (:commit:`2b4e4c3`) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 5a00fd570..8e8f3a0c9 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -280,3 +280,9 @@ class (which they all inherit from). Close the given spider. After this is called, no more specific stats can be accessed or collected. + +Engine API +========== + +.. autoclass:: scrapy.core.engine.ExecutionEngine() + :members: needs_backout diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 4e53b6e3d..e8c510ea5 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -150,7 +150,7 @@ requests). Use a Spider middleware if you need to * post-process output of spider callbacks - change/add/remove requests or items; -* post-process start_requests; +* post-process start requests or items; * handle spider exceptions; * call errback instead of callback for some of the requests based on response content. diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 3a7644379..56f8c6498 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -37,6 +37,10 @@ That includes the classes that you may assign to the following settings: - :setting:`SCHEDULER_PRIORITY_QUEUE` +- :setting:`SCHEDULER_START_DISK_QUEUE` + +- :setting:`SCHEDULER_START_MEMORY_QUEUE` + - :setting:`SPIDER_MIDDLEWARES` Third-party Scrapy components may also let you define additional Scrapy diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 8af4ce71d..448bf07e7 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -6,8 +6,8 @@ Coroutines .. versionadded:: 2.0 -Scrapy has :ref:`partial support ` for the -:ref:`coroutine syntax `. +Scrapy :ref:`supports ` the :ref:`coroutine syntax ` +(i.e. ``async def``). .. _coroutine-support: @@ -18,6 +18,11 @@ Supported callables The following callables may be defined as coroutines using ``async def``, and hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): +- The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be + defined as an :term:`asynchronous generator`. + + .. versionadded: VERSION + - :class:`~scrapy.Request` callbacks. If you are using any custom or third-party :ref:`spider middleware @@ -38,20 +43,26 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): methods of :ref:`downloader middlewares `. -- :ref:`Signal handlers that support deferreds `. - - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method of :ref:`spider middlewares `. - It must be defined as an :term:`asynchronous generator`. The input - ``result`` parameter is an :term:`asynchronous iterable`. + If defined as a coroutine, it must be an :term:`asynchronous generator`. + The input ``result`` parameter is an :term:`asynchronous iterable`. See also :ref:`sync-async-spider-middleware` and :ref:`universal-spider-middleware`. .. versionadded:: 2.7 +- The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method + of :ref:`spider middlewares `, which *must* be + defined as an :term:`asynchronous generator`. + + .. versionadded:: VERSION + +- :ref:`Signal handlers that support deferreds `. + .. _coroutine-deferred-apis: @@ -232,8 +243,9 @@ This means you can use many useful Python libraries providing such code: Common use cases for asynchronous code include: -* requesting data from websites, databases and other services (in callbacks, - pipelines and middlewares); +* requesting data from websites, databases and other services (in + :meth:`~scrapy.spiders.Spider.start`, callbacks, pipelines and + middlewares); * storing data in databases (in pipelines and middlewares); * delaying the spider initialization until some external event (in the :signal:`spider_opened` handler); diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 0e705dc64..50bcaa6d6 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -46,9 +46,9 @@ Keeping persistent state between batches Sometimes you'll want to keep some persistent spider state between pause/resume batches. You can use the ``spider.state`` attribute for that, which should be a -dict. There's :ref:`a built-in extension ` that takes care of serializing, storing and -loading that attribute from the job directory, when the spider starts and -stops. +dict. There's :ref:`a built-in extension ` +that takes care of serializing, storing and loading that attribute from the job +directory, when the spider starts and stops. Here's an example of a callback that uses the spider state (other spider code is omitted for brevity): diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 77837378e..6ca0973d8 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -127,10 +127,7 @@ Request objects body to bytes (if given as a string). :type encoding: str - :param priority: the priority of this request (defaults to ``0``). - The priority is used by the scheduler to define the order used to process - requests. Requests with a higher priority value will execute earlier. - Negative values are allowed in order to indicate relatively low-priority. + :param priority: sets :attr:`priority`, defaults to ``0``. :type priority: int :param dont_filter: sets :attr:`dont_filter`, defaults to ``False``. @@ -179,6 +176,8 @@ Request objects .. autoattribute:: errback + .. autoattribute:: priority + .. attribute:: Request.cb_kwargs A dictionary that contains arbitrary metadata for this request. Its contents @@ -353,7 +352,7 @@ errors if needed: "https://example.invalid/", # DNS error expected ] - def start_requests(self): + async def start(self): for u in self.start_urls: yield scrapy.Request( u, @@ -647,6 +646,7 @@ Those are: * ``ftp_user`` (See :setting:`FTP_USER` for more info) * :reqmeta:`handle_httpstatus_all` * :reqmeta:`handle_httpstatus_list` +* :reqmeta:`is_start_request` * :reqmeta:`max_retry_times` * :reqmeta:`proxy` * :reqmeta:`redirect_reasons` diff --git a/docs/topics/scheduler.rst b/docs/topics/scheduler.rst index 57c24b76a..b6e54ebd7 100644 --- a/docs/topics/scheduler.rst +++ b/docs/topics/scheduler.rst @@ -26,9 +26,9 @@ Minimal scheduler interface :members: -Default Scrapy scheduler -======================== +Default scheduler +================= -.. autoclass:: Scheduler +.. autoclass:: Scheduler() :members: - :special-members: __len__ + :special-members: __init__, __len__ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 3a61306d6..537e51e40 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1326,6 +1326,7 @@ Default: ``{}`` A dict containing the pipelines enabled by default in Scrapy. You should never modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead. + .. setting:: JOBDIR JOBDIR @@ -1336,6 +1337,7 @@ Default: ``None`` A string indicating the directory for storing the state of a crawl when :ref:`pausing and resuming crawls `. + .. setting:: LOG_ENABLED LOG_ENABLED @@ -1700,23 +1702,28 @@ SCHEDULER_DISK_QUEUE Default: ``'scrapy.squeues.PickleLifoDiskQueue'`` -Type of disk queue that will be used by scheduler. Other available types are -``scrapy.squeues.PickleFifoDiskQueue``, ``scrapy.squeues.MarshalFifoDiskQueue``, +Type of disk queue that will be used by the scheduler. Other available types +are ``scrapy.squeues.PickleFifoDiskQueue``, +``scrapy.squeues.MarshalFifoDiskQueue``, ``scrapy.squeues.MarshalLifoDiskQueue``. + .. setting:: SCHEDULER_MEMORY_QUEUE SCHEDULER_MEMORY_QUEUE ---------------------- + Default: ``'scrapy.squeues.LifoMemoryQueue'`` -Type of in-memory queue used by scheduler. Other available type is: +Type of in-memory queue used by the scheduler. Other available type is: ``scrapy.squeues.FifoMemoryQueue``. + .. setting:: SCHEDULER_PRIORITY_QUEUE SCHEDULER_PRIORITY_QUEUE ------------------------ + Default: ``'scrapy.pqueues.ScrapyPriorityQueue'`` Type of priority queue used by the scheduler. Another available type is @@ -1726,6 +1733,51 @@ Type of priority queue used by the scheduler. Another available type is domains in parallel. But currently ``scrapy.pqueues.DownloaderAwarePriorityQueue`` does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`. + +.. setting:: SCHEDULER_START_DISK_QUEUE + +SCHEDULER_START_DISK_QUEUE +-------------------------- + +Default: ``'scrapy.squeues.PickleFifoDiskQueue'`` + +Type of disk queue (see :setting:`JOBDIR`) that the :ref:`scheduler +` uses for :ref:`start requests `. + +For available choices, see :setting:`SCHEDULER_DISK_QUEUE`. + +.. queue-common-starts + +Use ``None`` or ``""`` to disable these separate queues entirely, and instead +have start requests share the same queues as other requests. + +.. note:: + + Disabling separate start request queues makes :ref:`start request order + ` unintuitive: start requests will be sent in order + only until :setting:`CONCURRENT_REQUESTS` is reached, then remaining start + requests will be sent in reverse order. + +.. queue-common-ends + + +.. setting:: SCHEDULER_START_MEMORY_QUEUE + +SCHEDULER_START_MEMORY_QUEUE +---------------------------- + +Default: ``'scrapy.squeues.FifoMemoryQueue'`` + +Type of in-memory queue that the :ref:`scheduler ` uses for +:ref:`start requests `. + +For available choices, see :setting:`SCHEDULER_MEMORY_QUEUE`. + +.. include:: settings.rst + :start-after: queue-common-starts + :end-before: queue-common-ends + + .. setting:: SCRAPER_SLOT_MAX_ACTIVE_SIZE SCRAPER_SLOT_MAX_ACTIVE_SIZE @@ -1957,7 +2009,7 @@ In order to use the reactor installed by Scrapy: self.timeout = int(kwargs.pop("timeout", "60")) super(QuotesSpider, self).__init__(*args, **kwargs) - def start_requests(self): + async def start(self): reactor.callLater(self.timeout, self.stop) urls = ["https://quotes.toscrape.com/page/1"] @@ -1986,7 +2038,7 @@ which raises :exc:`Exception`, becomes: self.timeout = int(kwargs.pop("timeout", "60")) super(QuotesSpider, self).__init__(*args, **kwargs) - def start_requests(self): + async def start(self): from twisted.internet import reactor reactor.callLater(self.timeout, self.stop) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index b45b12540..66cb87fc5 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -131,6 +131,19 @@ engine_stopped This signal supports returning deferreds from its handlers. +scheduler_empty +~~~~~~~~~~~~~~~ + +.. signal:: scheduler_empty +.. function:: scheduler_empty() + + Sent whenever the engine asks for a pending request from the + :ref:`scheduler ` (i.e. calls its + :meth:`~scrapy.core.scheduler.BaseScheduler.next_request` method) and the + scheduler returns none. + + See :ref:`start-requests-lazy` for an example. + Item signals ------------ @@ -160,7 +173,7 @@ item_scraped :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was scraped, or ``None`` - if it was yielded from :meth:`~scrapy.Spider.start_requests`. + if it was yielded from :meth:`~scrapy.Spider.start`. :type response: :class:`~scrapy.http.Response` | ``None`` item_dropped @@ -181,7 +194,7 @@ item_dropped :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was dropped, or ``None`` - if it was yielded from :meth:`~scrapy.Spider.start_requests`. + if it was yielded from :meth:`~scrapy.Spider.start`. :type response: :class:`~scrapy.http.Response` | ``None`` :param exception: the exception (which must be a @@ -205,7 +218,7 @@ item_error :param response: the response being processed when the exception was raised, or ``None`` if it was yielded from - :meth:`~scrapy.Spider.start_requests`. + :meth:`~scrapy.Spider.start`. :type response: :class:`~scrapy.http.Response` | ``None`` :param spider: the spider which raised the exception diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2211a822f..638035e64 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -70,30 +70,29 @@ one or more of these methods: .. class:: SpiderMiddleware - .. method:: process_start_requests(start_requests, spider) + .. method:: process_start(start: AsyncIterator[Any], /) -> AsyncIterator[Any] + :async: - This method is called with the start requests of the spider, and works - similarly to the :meth:`process_spider_output` method, except that it - doesn't have a response associated and must return only requests (not - items). + Iterate over the output of :meth:`~scrapy.Spider.start` or that + of the :meth:`process_start` method of an earlier spider middleware, + overriding it. For example: - It receives an iterable (in the ``start_requests`` parameter) and must - return another iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects `. + .. code-block:: python - .. note:: When implementing this method in your spider middleware, you - should always return an iterable (that follows the input one) and - not consume all ``start_requests`` iterator because it can be very - large (or even unbounded) and cause a memory overflow. The Scrapy - engine is designed to pull start requests while it has capacity to - process them, so the start requests iterator can be effectively - endless where there is some other condition for stopping the spider - (like a time limit or item/page count). + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request - :param start_requests: the start requests - :type start_requests: an iterable of :class:`~scrapy.Request` + You may yield the same type of objects as :meth:`~scrapy.Spider.start`. - :param spider: the spider to whom the start requests belong - :type spider: :class:`~scrapy.Spider` object + To write spider middlewares that work on Scrapy versions lower than + VERSION, define also a synchronous ``process_start_requests()`` method + that returns an iterable. For example: + + .. code-block:: python + + def process_start_requests(self, start, spider): + yield from start .. method:: process_spider_input(response, spider) @@ -154,6 +153,7 @@ one or more of these methods: :type spider: :class:`~scrapy.Spider` object .. method:: process_spider_output_async(response, result, spider) + :async: .. versionadded:: 2.7 @@ -417,6 +417,14 @@ String value Class name (as a string) .. _"unsafe-url": https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url +StartSpiderMiddleware +--------------------- + +.. module:: scrapy.spidermiddlewares.start + +.. autoclass:: StartSpiderMiddleware + + UrlLengthMiddleware ------------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 0a67240d6..891c4da05 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -12,16 +12,16 @@ parsing pages for a particular site (or, in some cases, a group of sites). For spiders, the scraping cycle goes through something like this: -1. You start by generating the initial Requests to crawl the first URLs, and +1. You start by generating the initial requests to crawl the first URLs, and specify a callback function to be called with the response downloaded from those requests. - The first requests to perform are obtained by calling the - :meth:`~scrapy.Spider.start_requests` method which (by default) - generates :class:`~scrapy.Request` for the URLs specified in the - :attr:`~scrapy.Spider.start_urls` and the - :attr:`~scrapy.Spider.parse` method as callback function for the - Requests. + The first requests to perform are obtained by iterating the + :meth:`~scrapy.Spider.start` method, which by default yields a + :class:`~scrapy.Request` object for each URL in the + :attr:`~scrapy.Spider.start_urls` spider attribute, with the + :attr:`~scrapy.Spider.parse` method set as :attr:`~scrapy.Request.callback` + function to handle each :class:`~scrapy.http.Response`. 2. In the callback function, you parse the response (web page) and return :ref:`item objects `, @@ -48,14 +48,7 @@ scrapy.Spider ============= .. class:: scrapy.spiders.Spider -.. class:: scrapy.Spider() - - This is the simplest spider, and the one from which every other spider - must inherit (including spiders that come bundled with Scrapy, as well as spiders - that you write yourself). It doesn't provide any special functionality. It just - provides a default :meth:`start_requests` implementation which sends requests from - the :attr:`start_urls` spider attribute and calls the spider's method ``parse`` - for each of the resulting responses. +.. autoclass:: scrapy.Spider .. attribute:: name @@ -81,12 +74,7 @@ scrapy.Spider Let's say your target url is ``https://www.example.com/1.html``, then add ``'example.com'`` to the list. - .. attribute:: start_urls - - A list of URLs where the spider will begin to crawl from, when no - particular URLs are specified. So, the first pages downloaded will be those - listed here. The subsequent :class:`~scrapy.Request` will be generated successively from data - contained in the start URLs. + .. autoattribute:: start_urls .. attribute:: custom_settings @@ -149,7 +137,7 @@ scrapy.Spider The final settings and the initialized :class:`~scrapy.crawler.Crawler` attributes are available in the - :meth:`start_requests` method, handlers of the + :meth:`start` method, handlers of the :signal:`engine_started` signal and later. :param crawler: crawler to which the spider will be bound @@ -201,42 +189,7 @@ scrapy.Spider super().update_settings(settings) settings.setdefault("FEEDS", {}).update(cls.custom_feed) - .. method:: start_requests() - - This method must return an iterable with the first Requests to crawl and/or with :ref:`item objects - ` for - this spider. It is called by Scrapy when the spider is opened for - scraping. Scrapy calls it only once, so it is safe to implement - :meth:`start_requests` as a generator. - - The default implementation generates ``Request(url, dont_filter=True)`` - for each url in :attr:`start_urls`. - - If you want to change the Requests used to start scraping a domain, this is - the method to override. For example, if you need to start by logging in using - a POST request, you could do: - - .. code-block:: python - - import scrapy - - - class MySpider(scrapy.Spider): - name = "myspider" - - def start_requests(self): - return [ - scrapy.FormRequest( - "http://www.example.com/login", - formdata={"user": "john", "pass": "secret"}, - callback=self.logged_in, - ) - ] - - def logged_in(self, response): - # here you would extract links to follow and return Requests for - # each of them, with another callback - pass + .. automethod:: start .. method:: parse(response) @@ -308,8 +261,9 @@ Return multiple Requests and items from a single callback: for href in response.xpath("//a/@href").getall(): yield scrapy.Request(response.urljoin(href), self.parse) -Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; -to give data more structure you can use :class:`~scrapy.Item` objects: +Instead of :attr:`~.start_urls` you can use :meth:`~scrapy.Spider.start` +directly; to give data more structure you can use :class:`~scrapy.Item` +objects: .. skip: next .. code-block:: python @@ -322,7 +276,7 @@ to give data more structure you can use :class:`~scrapy.Item` objects: name = "example.com" allowed_domains = ["example.com"] - def start_requests(self): + async def start(self): yield scrapy.Request("http://www.example.com/1.html", self.parse) yield scrapy.Request("http://www.example.com/2.html", self.parse) yield scrapy.Request("http://www.example.com/3.html", self.parse) @@ -376,7 +330,7 @@ The above example can also be written as follows: class MySpider(scrapy.Spider): name = "myspider" - def start_requests(self): + async def start(self): yield scrapy.Request(f"http://www.example.com/categories/{self.category}") If you are :ref:`running Scrapy from a script `, you can @@ -410,6 +364,38 @@ used by :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`:: Spider arguments can also be passed through the Scrapyd ``schedule.json`` API. See `Scrapyd documentation`_. +.. _start-requests: + +Start requests +============== + +**Start requests** are :class:`~scrapy.Request` objects yielded from the +:meth:`~scrapy.Spider.start` method of a spider or from the +:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method of a +:ref:`spider middleware `. + +.. seealso:: :ref:`start-request-order` + +.. _start-requests-lazy: + +Delaying start request iteration +-------------------------------- + +You can override the :meth:`~scrapy.Spider.start` method as follows to pause +its iteration whenever there are scheduled requests: + +.. code-block:: python + + async def start(self): + async for item_or_request in super().start(): + if self.crawler.engine.needs_backoff(): + await self.crawler.signals.wait_for(signals.scheduler_empty) + yield item_or_request + +This can help minimize the number of requests in the scheduler at any given +time, to minimize resource usage (memory or disk, depending on +:setting:`JOBDIR`). + .. _builtin-spiders: Generic Spiders @@ -940,10 +926,11 @@ Combine SitemapSpider with other sources of urls: other_urls = ["http://www.example.com/about"] - def start_requests(self): - requests = list(super(MySpider, self).start_requests()) - requests += [scrapy.Request(x, self.parse_other) for x in self.other_urls] - return requests + async def start(self): + async for item_or_request in super().start(): + yield item_or_request + for url in self.other_urls: + yield Request(url, self.parse_other) def parse_shop(self, response): pass # ... scrape shop here ... diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 0e4a8fa6c..3e9bbe56e 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -59,6 +59,8 @@ Default Username and Password can be overridden by the settings You need the telnet program which comes installed by default in Windows, and most Linux distros. +.. _telnet-vars: + Available variables in the telnet console ========================================= @@ -77,8 +79,6 @@ convenience: +----------------+-------------------------------------------------------------------+ | ``spider`` | the active spider | +----------------+-------------------------------------------------------------------+ -| ``slot`` | the engine slot | -+----------------+-------------------------------------------------------------------+ | ``extensions`` | the Extension Manager (Crawler.extensions attribute) | +----------------+-------------------------------------------------------------------+ | ``stats`` | the Stats Collector (Crawler.stats attribute) | @@ -114,10 +114,10 @@ using the telnet console:: engine.scraper.is_idle() : False engine.spider.name : followall engine.spider_is_idle() : False - engine.slot.closing : False - len(engine.slot.inprogress) : 16 - len(engine.slot.scheduler.dqs or []) : 0 - len(engine.slot.scheduler.mqs) : 92 + engine._slot.closing : False + len(engine._slot.inprogress) : 16 + len(engine._slot.scheduler.dqs or []) : 0 + len(engine._slot.scheduler.mqs) : 92 len(engine.scraper.slot.queue) : 0 len(engine.scraper.slot.active) : 0 engine.scraper.slot.active_size : 0 diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 119dfdabb..269b27336 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -34,6 +34,10 @@ class QPSSpider(Spider): elif self.download_delay is not None: self.download_delay = float(self.download_delay) + async def start(self): + for item_or_request in self.start_requests(): + yield item_or_request + def start_requests(self): url = self.benchurl if self.latency is not None: diff --git a/pyproject.toml b/pyproject.toml index e14efdd17..187587eb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -238,7 +238,9 @@ markers = [ "requires_botocore: marks tests that need botocore (but not boto3)", "requires_boto3: marks tests that need botocore and boto3", ] -filterwarnings = [] +filterwarnings = [ + "ignore::DeprecationWarning:twisted.web.static" +] [tool.ruff.lint] extend-select = [ diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 16dae6ac4..96bb1ae84 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -13,9 +13,7 @@ from scrapy.linkextractors import LinkExtractor if TYPE_CHECKING: import argparse - from collections.abc import Iterable - - from scrapy import Request + from collections.abc import AsyncIterator class Command(ScrapyCommand): @@ -61,10 +59,10 @@ class _BenchSpider(scrapy.Spider): baseurl = "http://localhost:8998" link_extractor = LinkExtractor() - def start_requests(self) -> Iterable[Request]: + async def start(self) -> AsyncIterator[Any]: qargs = {"total": self.total, "show": self.show} url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}" - return [scrapy.Request(url, dont_filter=True)] + yield scrapy.Request(url, dont_filter=True) def parse(self, response: Response) -> Any: assert isinstance(response, TextResponse) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 1ce155da7..56dc1ea55 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -80,10 +80,14 @@ class Command(ScrapyCommand): assert self.crawler_process spider_loader = self.crawler_process.spider_loader + async def start(self): + for request in conman.from_spider(self, result): + yield request + with set_environ(SCRAPY_CHECK="true"): for spidername in args or spider_loader.list(): spidercls = spider_loader.load(spidername) - spidercls.start_requests = lambda s: conman.from_spider(s, result) # type: ignore[assignment,method-assign,return-value] + spidercls.start = start # type: ignore[assignment,method-assign,return-value] tested_methods = conman.tested_methods_from_spidercls(spidercls) if opts.list: @@ -101,10 +105,10 @@ class Command(ScrapyCommand): for method in sorted(methods): print(f" * {method}") else: - start = time.time() + start_time = time.time() self.crawler_process.start() stop = time.time() result.printErrors() - result.printSummary(start, stop) + result.printSummary(start_time, stop) self.exitcode = int(not result.wasSuccessful()) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 8a8d04ff6..ef6e13de2 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -89,5 +89,11 @@ class Command(ScrapyCommand): spidercls = spider_loader.load(opts.spider) else: spidercls = spidercls_for_request(spider_loader, request, spidercls) - self.crawler_process.crawl(spidercls, start_requests=lambda: [request]) + + async def start(self): + yield request + + spidercls.start = start # type: ignore[method-assign,attr-defined] + + self.crawler_process.crawl(spidercls) self.crawler_process.start() diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index c6ed20b3b..0dd9954cb 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -22,7 +22,7 @@ from scrapy.utils.spider import spidercls_for_request if TYPE_CHECKING: import argparse - from collections.abc import AsyncGenerator, Coroutine, Iterable + from collections.abc import AsyncGenerator, AsyncIterator, Coroutine, Iterable from twisted.python.failure import Failure @@ -258,11 +258,11 @@ class Command(BaseRunSpiderCommand): if not self.spidercls: logger.error("Unable to find spider for: %(url)s", {"url": url}) - def _start_requests(spider: Spider) -> Iterable[Request]: + async def start(spider: Spider) -> AsyncIterator[Any]: yield self.prepare_request(spider, Request(url), opts) if self.spidercls: - self.spidercls.start_requests = _start_requests # type: ignore[assignment,method-assign] + self.spidercls.start = start # type: ignore[assignment,method-assign] def start_parsing(self, url: str, opts: argparse.Namespace) -> None: assert self.crawler_process diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 3047ae396..9dabfcd9c 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -24,9 +24,9 @@ if TYPE_CHECKING: class Command(ScrapyCommand): requires_project = False default_settings = { + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", "KEEP_ALIVE": True, "LOGSTATS_INTERVAL": 0, - "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", } def syntax(self) -> str: @@ -85,7 +85,7 @@ class Command(ScrapyCommand): crawler._apply_settings() # The Shell class needs a persistent engine in the crawler crawler.engine = crawler._create_engine() - crawler.engine.start() + crawler.engine.start(_start_request_processing=False) self._start_crawler_thread() diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 653e5e05c..7f5dd0405 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -9,6 +9,7 @@ from __future__ import annotations import logging from time import time +from traceback import format_exc from typing import TYPE_CHECKING, Any, TypeVar, cast from twisted.internet.defer import Deferred, inlineCallbacks, succeed @@ -16,15 +17,19 @@ from twisted.internet.task import LoopingCall from twisted.python.failure import Failure from scrapy import signals -from scrapy.core.scraper import Scraper, _HandleOutputDeferred +from scrapy.core.scraper import Scraper from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + maybe_deferred_to_future, +) from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: - from collections.abc import Callable, Generator, Iterable, Iterator + from collections.abc import AsyncIterator, Callable, Generator from scrapy.core.downloader import Downloader from scrapy.core.scheduler import BaseScheduler @@ -40,17 +45,15 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -class Slot: +class _Slot: def __init__( self, - start_requests: Iterable[Request], close_if_idle: bool, nextcall: CallLaterOnce[None], scheduler: BaseScheduler, ) -> None: self.closing: Deferred[None] | None = None self.inprogress: set[Request] = set() - self.start_requests: Iterator[Request] | None = iter(start_requests) self.close_if_idle: bool = close_if_idle self.nextcall: CallLaterOnce[None] = nextcall self.scheduler: BaseScheduler = scheduler @@ -78,6 +81,8 @@ class Slot: class ExecutionEngine: + _SLOT_HEARTBEAT_INTERVAL: float = 5.0 + def __init__( self, crawler: Crawler, @@ -88,20 +93,25 @@ class ExecutionEngine: self.signals: SignalManager = crawler.signals assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter - self.slot: Slot | None = None + self._slot: _Slot | None = None self.spider: Spider | None = None self.running: bool = False self.paused: bool = False - self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( - crawler.settings - ) - downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) - self.downloader: Downloader = downloader_cls(crawler) - self.scraper: Scraper = Scraper(crawler) self._spider_closed_callback: Callable[[Spider], Deferred[None] | None] = ( spider_closed_callback ) self.start_time: float | None = None + self._start: AsyncIterator[Any] | None = None + downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) + try: + self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( + crawler.settings + ) + self.downloader: Downloader = downloader_cls(crawler) + self.scraper: Scraper = Scraper(crawler) + except Exception: + self.close() + raise def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: from scrapy.core.scheduler import BaseScheduler @@ -114,22 +124,28 @@ class ExecutionEngine: ) return scheduler_cls - @inlineCallbacks - def start(self) -> Generator[Deferred[Any], Any, None]: + @deferred_f_from_coro_f + async def start(self, _start_request_processing=True) -> None: if self.running: raise RuntimeError("Engine already running") self.start_time = time() - yield self.signals.send_catch_log_deferred(signal=signals.engine_started) + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred(signal=signals.engine_started) + ) self.running = True self._closewait: Deferred[None] = Deferred() - yield self._closewait + if _start_request_processing: + self._start_request_processing() + await maybe_deferred_to_future(self._closewait) def stop(self) -> Deferred[None]: """Gracefully stop the execution engine""" - @inlineCallbacks - def _finish_stopping_engine(_: Any) -> Generator[Deferred[Any], Any, None]: - yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) + @deferred_f_from_coro_f + async def _finish_stopping_engine(_: Any) -> None: + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred(signal=signals.engine_stopped) + ) self._closewait.callback(None) if not self.running: @@ -163,59 +179,85 @@ class ExecutionEngine: def unpause(self) -> None: self.paused = False - def _next_request(self) -> None: - if self.slot is None: - return + async def _process_start_next(self): + """Processes the next item or request from Spider.start(). - assert self.spider is not None # typing - - if self.paused: - return - - while ( - not self._needs_backout() - and self._next_request_from_scheduler() is not None - ): - pass - - if self.slot.start_requests is not None and not self._needs_backout(): - try: - request_or_item = next(self.slot.start_requests) - except StopIteration: - self.slot.start_requests = None - except Exception: - self.slot.start_requests = None - logger.error( - "Error while obtaining start requests", - exc_info=True, - extra={"spider": self.spider}, - ) + If a request, it is scheduled. If an item, it is sent to item + pipelines. + """ + try: + item_or_request = await self._start.__anext__() + except StopAsyncIteration: + self._start = None + except Exception as exception: + self._start = None + exception_traceback = format_exc() + logger.error( + f"Error while reading start items and requests: {exception}.\n{exception_traceback}", + exc_info=True, + ) + else: + if not self.spider: + return # spider already closed + if isinstance(item_or_request, Request): + self.crawl(item_or_request) else: - if isinstance(request_or_item, Request): - self.crawl(request_or_item) - else: - self.scraper.start_itemproc(request_or_item, response=None) + self.scraper.start_itemproc(item_or_request, response=None) + self._slot.nextcall.schedule() - if self.spider_is_idle() and self.slot.close_if_idle: + @deferred_f_from_coro_f + async def _start_request_processing(self) -> None: + """Starts consuming Spider.start() output and sending scheduled + requests.""" + # Starts the processing of scheduled requests, as well as a periodic + # call to that processing method for scenarios where the scheduler + # reports having pending requests but returns none. + assert self._slot is not None # typing + self._slot.nextcall.schedule() + self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) + + while self._start and self.spider: + await self._process_start_next() + if not self.needs_backout(): + # Give room for the outcome of self._process_start_next() to be + # processed before continuing with the next iteration. + self._slot.nextcall.schedule() + await self._slot.nextcall.wait() + + def _start_scheduled_requests(self) -> None: + if self._slot is None or self._slot.closing is not None or self.paused: + return + + while not self.needs_backout(): + if not self._start_scheduled_request(): + break + + if self.spider_is_idle() and self._slot.close_if_idle: self._spider_idle() - def _needs_backout(self) -> bool: - assert self.slot is not None # typing + def needs_backout(self) -> bool: + """Returns ``True`` if no more requests can be sent at the moment, or + ``False`` otherwise. + + See :ref:`start-requests-lazy` for an example. + """ + assert self._slot is not None # typing assert self.scraper.slot is not None # typing return ( not self.running - or bool(self.slot.closing) + or bool(self._slot.closing) or self.downloader.needs_backout() or self.scraper.slot.needs_backout() ) - def _next_request_from_scheduler(self) -> Deferred[None] | None: - assert self.slot is not None # typing + def _start_scheduled_request(self) -> bool: + assert self._slot is not None # typing assert self.spider is not None # typing - request = self.slot.scheduler.next_request() + request = self._slot.scheduler.next_request() if request is None: - return None + self.signals.send_catch_log(signals.scheduler_empty) + return False d: Deferred[Response | Request] = self._download(request) d.addBoth(self._handle_downloader_output, request) @@ -228,8 +270,8 @@ class ExecutionEngine: ) def _remove_request(_: Any) -> None: - assert self.slot - self.slot.remove_request(request) + assert self._slot + self._slot.remove_request(request) d2: Deferred[None] = d.addBoth(_remove_request) d2.addErrback( @@ -239,7 +281,7 @@ class ExecutionEngine: extra={"spider": self.spider}, ) ) - slot = self.slot + slot = self._slot d2.addBoth(lambda _: slot.nextcall.schedule()) d2.addErrback( lambda f: logger.info( @@ -248,13 +290,12 @@ class ExecutionEngine: extra={"spider": self.spider}, ) ) - return d2 + return True + @inlineCallbacks def _handle_downloader_output( self, result: Request | Response | Failure, request: Request - ) -> _HandleOutputDeferred | None: - assert self.spider is not None # typing - + ) -> Generator[Deferred[Any], Any, None]: if not isinstance(result, (Request, Response, Failure)): raise TypeError( f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}" @@ -263,35 +304,35 @@ class ExecutionEngine: # downloader middleware can return requests (for example, redirects) if isinstance(result, Request): self.crawl(result) - return None + return - d = self.scraper.enqueue_scrape(result, request) - d.addErrback( - lambda f: logger.error( - "Error while enqueuing downloader output", - exc_info=failure_to_exc_info(f), + try: + yield self.scraper.enqueue_scrape(result, request) + except Exception: + assert self.spider is not None + logger.error( + "Error while enqueuing scrape", + exc_info=True, extra={"spider": self.spider}, ) - ) - return d def spider_is_idle(self) -> bool: - if self.slot is None: + if self._slot is None: raise RuntimeError("Engine slot not assigned") if not self.scraper.slot.is_idle(): # type: ignore[union-attr] return False if self.downloader.active: # downloader has pending requests return False - if self.slot.start_requests is not None: # not all start requests are handled + if self._start is not None: # not all start requests are handled return False - return not self.slot.scheduler.has_pending_requests() + return not self._slot.scheduler.has_pending_requests() def crawl(self, request: Request) -> None: """Inject the request into the spider <-> downloader pipeline""" if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") self._schedule_request(request) - self.slot.nextcall.schedule() # type: ignore[union-attr] + self._slot.nextcall.schedule() # type: ignore[union-attr] def _schedule_request(self, request: Request) -> None: request_scheduled_result = self.signals.send_catch_log( @@ -303,7 +344,7 @@ class ExecutionEngine: for handler, result in request_scheduled_result: if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest): return - if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] + if not self._slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log( signals.request_dropped, request=request, spider=self.spider ) @@ -320,14 +361,14 @@ class ExecutionEngine: def _downloaded( self, result: Response | Request | Failure, request: Request ) -> Deferred[Response] | Response | Failure: - assert self.slot is not None # typing - self.slot.remove_request(request) + assert self._slot is not None # typing + self._slot.remove_request(request) return self.download(result) if isinstance(result, Request) else result def _download(self, request: Request) -> Deferred[Response | Request]: - assert self.slot is not None # typing + assert self._slot is not None # typing - self.slot.add_request(request) + self._slot.add_request(request) def _on_success(result: Response | Request) -> Response | Request: if not isinstance(result, (Response, Request)): @@ -352,8 +393,8 @@ class ExecutionEngine: return result def _on_complete(_: _T) -> _T: - assert self.slot is not None - self.slot.nextcall.schedule() + assert self._slot is not None + self._slot.nextcall.schedule() return _ assert self.spider is not None @@ -362,31 +403,28 @@ class ExecutionEngine: dwld.addBoth(_on_complete) return dwld - @inlineCallbacks - def open_spider( + @deferred_f_from_coro_f + async def open_spider( self, spider: Spider, - start_requests: Iterable[Request] = (), close_if_idle: bool = True, - ) -> Generator[Deferred[Any], Any, None]: - if self.slot is not None: + ) -> None: + if self._slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={"spider": spider}) - nextcall = CallLaterOnce(self._next_request) - scheduler = build_from_crawler(self.scheduler_cls, self.crawler) - start_requests = yield self.scraper.spidermw.process_start_requests( - start_requests, spider - ) - self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.spider = spider + nextcall = CallLaterOnce(self._start_scheduled_requests) + scheduler = build_from_crawler(self.scheduler_cls, self.crawler) + self._slot = _Slot(close_if_idle, nextcall, scheduler) + self._start = await self.scraper.spidermw.process_start(spider) if hasattr(scheduler, "open") and (d := scheduler.open(spider)): - yield d - yield self.scraper.open_spider(spider) + await maybe_deferred_to_future(d) + await maybe_deferred_to_future(self.scraper.open_spider(spider)) assert self.crawler.stats self.crawler.stats.open_spider(spider) - yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) - self.slot.nextcall.schedule() - self.slot.heartbeat.start(5) + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) + ) def _spider_idle(self) -> None: """ @@ -415,17 +453,17 @@ class ExecutionEngine: def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred[None]: """Close (cancel) spider and clear all its outstanding requests""" - if self.slot is None: + if self._slot is None: raise RuntimeError("Engine slot not assigned") - if self.slot.closing is not None: - return self.slot.closing + if self._slot.closing is not None: + return self._slot.closing logger.info( "Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider} ) - dfd = self.slot.close() + dfd = self._slot.close() def log_failure(msg: str) -> Callable[[Failure], None]: def errback(failure: Failure) -> None: @@ -441,8 +479,8 @@ class ExecutionEngine: dfd.addBoth(lambda _: self.scraper.close_spider()) dfd.addErrback(log_failure("Scraper close failure")) - if hasattr(self.slot.scheduler, "close"): - dfd.addBoth(lambda _: cast(Slot, self.slot).scheduler.close(reason)) + if hasattr(self._slot.scheduler, "close"): + dfd.addBoth(lambda _: cast(_Slot, self._slot).scheduler.close(reason)) dfd.addErrback(log_failure("Scheduler close failure")) dfd.addBoth( diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 4bb143dfd..57d27b7cf 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -5,13 +5,16 @@ import logging from abc import abstractmethod from pathlib import Path from typing import TYPE_CHECKING, Any, cast +from warnings import warn # working around https://github.com/sphinx-doc/sphinx/issues/10400 from twisted.internet.defer import Deferred # noqa: TC002 +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.spiders import Spider # noqa: TC001 from scrapy.utils.job import job_dir from scrapy.utils.misc import build_from_crawler, load_object +from scrapy.utils.python import global_object_name if TYPE_CHECKING: # requires queuelib >= 1.6.2 @@ -50,18 +53,17 @@ class BaseSchedulerMeta(type): class BaseScheduler(metaclass=BaseSchedulerMeta): - """ - The scheduler component is responsible for storing requests received from - the engine, and feeding them back upon request (also to the engine). + """The scheduler component is responsible for storing requests received + from the engine, and feeding them back upon request (also to the engine). The original sources of said requests are: - * Spider: ``start_requests`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks + * Spider: ``start`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks * Spider middleware: ``process_spider_output`` and ``process_spider_exception`` methods * Downloader middleware: ``process_request``, ``process_response`` and ``process_exception`` methods The order in which the scheduler returns its stored requests (via the ``next_request`` method) - plays a great part in determining the order in which those requests are downloaded. + plays a great part in determining the order in which those requests are downloaded. See :ref:`request-order`. The methods defined in this class constitute the minimal interface that the Scrapy engine will interact with. """ @@ -126,55 +128,112 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): class Scheduler(BaseScheduler): + """Default scheduler. + + Requests are stored into priority queues + (:setting:`SCHEDULER_PRIORITY_QUEUE`) that sort requests by + :attr:`~scrapy.http.Request.priority`. + + By default, a single, memory-based priority queue is used for all requests. + When using :setting:`JOBDIR`, a disk-based priority queue is also created, + and only unserializable requests are stored in the memory-based priority + queue. For a given priority value, requests in memory take precedence over + requests in disk. + + Each priority queue stores requests in separate internal queues, one per + priority value. The memory priority queue uses + :setting:`SCHEDULER_MEMORY_QUEUE` queues, while the disk priority queue + uses :setting:`SCHEDULER_DISK_QUEUE` queues. The internal queues determine + :ref:`request order ` when requests have the same priority. + :ref:`Start requests ` are stored into separate internal + queues by default, and :ref:`ordered differently `. + + Duplicate requests are filtered out with an instance of + :setting:`DUPEFILTER_CLASS`. + + .. _request-order: + + Request order + ============= + + With default settings, pending requests are stored in a LIFO_ queue + (:ref:`except for start requests `). As a result, + crawling happens in `DFO order`_, which is usually the most convenient + crawl order. However, you can enforce :ref:`BFO ` or :ref:`a custom + order ` (:ref:`except for the first few requests + `). + + .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) + .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search + + .. _start-request-order: + + Start request order + ------------------- + + :ref:`Start requests ` are sent in the order they are + yielded from :meth:`~scrapy.Spider.start`, and given the same + :attr:`~scrapy.http.Request.priority`, start requests take precedence over + other requests. + + You can set :setting:`SCHEDULER_START_MEMORY_QUEUE` and + :setting:`SCHEDULER_START_DISK_QUEUE` to ``None`` to handle start requests + the same as other requests when it comes to order and priority. + + + .. _bfo: + + Crawling in BFO order + --------------------- + + If you do want to crawl in `BFO order`_, you can do it by setting the + following :ref:`settings `: + + | :setting:`DEPTH_PRIORITY` = ``1`` + | :setting:`SCHEDULER_DISK_QUEUE` = ``"scrapy.squeues.PickleFifoDiskQueue"`` + | :setting:`SCHEDULER_MEMORY_QUEUE` = ``"scrapy.squeues.FifoMemoryQueue"`` + + .. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search + + + .. _custom-request-order: + + Crawling in a custom order + -------------------------- + + You can manually set :attr:`~scrapy.http.Request.priority` on requests to + force a specific request order. + + + .. _concurrency-v-order: + + Concurrency affects order + ------------------------- + + While pending requests are below the configured values of + :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` + or :setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent + concurrently. + + As a result, the first few requests of a crawl may not follow the desired + order. Lowering those settings to ``1`` enforces the desired order except + for the very first request, but it significantly slows down the crawl as a + whole. """ - Default Scrapy scheduler. This implementation also handles duplication - filtering via the :setting:`dupefilter `. - This scheduler stores requests into several priority queues (defined by the - :setting:`SCHEDULER_PRIORITY_QUEUE` setting). In turn, said priority queues - are backed by either memory or disk based queues (respectively defined by the - :setting:`SCHEDULER_MEMORY_QUEUE` and :setting:`SCHEDULER_DISK_QUEUE` settings). - - Request prioritization is almost entirely delegated to the priority queue. The only - prioritization performed by this scheduler is using the disk-based queue if present - (i.e. if the :setting:`JOBDIR` setting is defined) and falling back to the memory-based - queue if a serialization error occurs. If the disk queue is not present, the memory one - is used directly. - - :param dupefilter: An object responsible for checking and filtering duplicate requests. - The value for the :setting:`DUPEFILTER_CLASS` setting is used by default. - :type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar: - any class that implements the `BaseDupeFilter` interface - - :param jobdir: The path of a directory to be used for persisting the crawl's state. - The value for the :setting:`JOBDIR` setting is used by default. - See :ref:`topics-jobs`. - :type jobdir: :class:`str` or ``None`` - - :param dqclass: A class to be used as persistent request queue. - The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default. - :type dqclass: class - - :param mqclass: A class to be used as non-persistent request queue. - The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default. - :type mqclass: class - - :param logunser: A boolean that indicates whether or not unserializable requests should be logged. - The value for the :setting:`SCHEDULER_DEBUG` setting is used by default. - :type logunser: bool - - :param stats: A stats collector object to record stats about the request scheduling process. - The value for the :setting:`STATS_CLASS` setting is used by default. - :type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar: - any class that implements the `StatsCollector` interface - - :param pqclass: A class to be used as priority queue for requests. - The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default. - :type pqclass: class - - :param crawler: The crawler object corresponding to the current crawl. - :type crawler: :class:`scrapy.crawler.Crawler` - """ + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + dupefilter_cls = load_object(crawler.settings["DUPEFILTER_CLASS"]) + return cls( + dupefilter=build_from_crawler(dupefilter_cls, crawler), + jobdir=job_dir(crawler.settings), + dqclass=load_object(crawler.settings["SCHEDULER_DISK_QUEUE"]), + mqclass=load_object(crawler.settings["SCHEDULER_MEMORY_QUEUE"]), + logunser=crawler.settings.getbool("SCHEDULER_DEBUG"), + stats=crawler.stats, + pqclass=load_object(crawler.settings["SCHEDULER_PRIORITY_QUEUE"]), + crawler=crawler, + ) def __init__( self, @@ -187,6 +246,42 @@ class Scheduler(BaseScheduler): pqclass: type[ScrapyPriorityQueue] | None = None, crawler: Crawler | None = None, ): + """Initialize the scheduler. + + :param dupefilter: An object responsible for checking and filtering duplicate requests. + The value for the :setting:`DUPEFILTER_CLASS` setting is used by default. + :type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar: + any class that implements the `BaseDupeFilter` interface + + :param jobdir: The path of a directory to be used for persisting the crawl's state. + The value for the :setting:`JOBDIR` setting is used by default. + See :ref:`topics-jobs`. + :type jobdir: :class:`str` or ``None`` + + :param dqclass: A class to be used as persistent request queue. + The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default. + :type dqclass: class + + :param mqclass: A class to be used as non-persistent request queue. + The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default. + :type mqclass: class + + :param logunser: A boolean that indicates whether or not unserializable requests should be logged. + The value for the :setting:`SCHEDULER_DEBUG` setting is used by default. + :type logunser: bool + + :param stats: A stats collector object to record stats about the request scheduling process. + The value for the :setting:`STATS_CLASS` setting is used by default. + :type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar: + any class that implements the `StatsCollector` interface + + :param pqclass: A class to be used as priority queue for requests. + The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default. + :type pqclass: class + + :param crawler: The crawler object corresponding to the current crawl. + :type crawler: :class:`scrapy.crawler.Crawler` + """ self.df: BaseDupeFilter = dupefilter self.dqdir: str | None = self._dqdir(jobdir) self.pqclass: type[ScrapyPriorityQueue] | None = pqclass @@ -195,23 +290,22 @@ class Scheduler(BaseScheduler): self.logunser: bool = logunser self.stats: StatsCollector | None = stats self.crawler: Crawler | None = crawler - - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - """ - Factory method, initializes the scheduler with arguments taken from the crawl settings - """ - dupefilter_cls = load_object(crawler.settings["DUPEFILTER_CLASS"]) - return cls( - dupefilter=build_from_crawler(dupefilter_cls, crawler), - jobdir=job_dir(crawler.settings), - dqclass=load_object(crawler.settings["SCHEDULER_DISK_QUEUE"]), - mqclass=load_object(crawler.settings["SCHEDULER_MEMORY_QUEUE"]), - logunser=crawler.settings.getbool("SCHEDULER_DEBUG"), - stats=crawler.stats, - pqclass=load_object(crawler.settings["SCHEDULER_PRIORITY_QUEUE"]), - crawler=crawler, + self._sdqclass: type[BaseQueue] | None = self._get_start_queue_cls( + crawler, "DISK" ) + self._smqclass: type[BaseQueue] | None = self._get_start_queue_cls( + crawler, "MEMORY" + ) + + def _get_start_queue_cls( + self, crawler: Crawler | None, queue: str + ) -> type[BaseQueue] | None: + if crawler is None: + return None + cls = crawler.settings[f"SCHEDULER_START_{queue}_QUEUE"] + if not cls: + return None + return load_object(cls) def has_pending_requests(self) -> bool: return len(self) > 0 @@ -324,12 +418,27 @@ class Scheduler(BaseScheduler): """Create a new priority queue instance, with in-memory storage""" assert self.crawler assert self.pqclass - return build_from_crawler( - self.pqclass, - self.crawler, - downstream_queue_cls=self.mqclass, - key="", - ) + try: + return build_from_crawler( + self.pqclass, + self.crawler, + downstream_queue_cls=self.mqclass, + key="", + start_queue_cls=self._smqclass, + ) + except TypeError: + warn( + f"The __init__ method of {global_object_name(self.pqclass)} " + f"does not support a `start_queue_cls` keyword-only " + f"parameter.", + ScrapyDeprecationWarning, + ) + return build_from_crawler( + self.pqclass, + self.crawler, + downstream_queue_cls=self.mqclass, + key="", + ) def _dq(self) -> ScrapyPriorityQueue: """Create a new priority queue instance, with disk storage""" @@ -337,13 +446,29 @@ class Scheduler(BaseScheduler): assert self.dqdir assert self.pqclass state = self._read_dqs_state(self.dqdir) - q = build_from_crawler( - self.pqclass, - self.crawler, - downstream_queue_cls=self.dqclass, - key=self.dqdir, - startprios=state, - ) + try: + q = build_from_crawler( + self.pqclass, + self.crawler, + downstream_queue_cls=self.dqclass, + key=self.dqdir, + startprios=state, + start_queue_cls=self._sdqclass, + ) + except TypeError: + warn( + f"The __init__ method of {global_object_name(self.pqclass)} " + f"does not support a `start_queue_cls` keyword-only " + f"parameter.", + ScrapyDeprecationWarning, + ) + q = build_from_crawler( + self.pqclass, + self.crawler, + downstream_queue_cls=self.dqclass, + key=self.dqdir, + startprios=state, + ) if q: logger.info( "Resuming crawl (%(queuesize)d requests scheduled)", diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 6f69d668e..9378f2651 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -6,10 +6,10 @@ from __future__ import annotations import logging import warnings from collections import deque -from collections.abc import AsyncIterable, Iterator -from typing import TYPE_CHECKING, Any, TypeVar, Union, cast +from collections.abc import AsyncIterator +from typing import TYPE_CHECKING, Any, TypeVar, Union -from twisted.internet.defer import Deferred, inlineCallbacks +from twisted.internet.defer import Deferred, inlineCallbacks, maybeDeferred from twisted.python.failure import Failure from scrapy import Spider, signals @@ -22,10 +22,12 @@ from scrapy.exceptions import ( ) from scrapy.http import Request, Response from scrapy.utils.defer import ( + _defer_sleep, aiter_errback, - defer_fail, - defer_succeed, + deferred_f_from_coro_f, + deferred_from_coro, iter_errback, + maybe_deferred_to_future, parallel, parallel_async, ) @@ -46,9 +48,7 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -_ParallelResult = list[tuple[bool, Iterator[Any]]] -_HandleOutputDeferred = Deferred[Union[_ParallelResult, None]] -QueueTuple = tuple[Union[Response, Failure], Request, _HandleOutputDeferred] +QueueTuple = tuple[Union[Response, Failure], Request, Deferred[None]] class Slot: @@ -66,8 +66,9 @@ class Slot: def add_response_request( self, result: Response | Failure, request: Request - ) -> _HandleOutputDeferred: - deferred: _HandleOutputDeferred = Deferred() + ) -> Deferred[None]: + # this Deferred will be awaited in enqueue_scrape() + deferred: Deferred[None] = Deferred() self.queue.append((result, request, deferred)) if isinstance(result, Response): self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE) @@ -76,9 +77,9 @@ class Slot: return deferred def next_response_request_deferred(self) -> QueueTuple: - response, request, deferred = self.queue.popleft() + result, request, deferred = self.queue.popleft() self.active.add(request) - return response, request, deferred + return result, request, deferred def finish_response(self, result: Response | Failure, request: Request) -> None: self.active.remove(request) @@ -143,9 +144,10 @@ class Scraper: assert self.crawler.spider self.slot.closing.callback(self.crawler.spider) + @inlineCallbacks def enqueue_scrape( self, result: Response | Failure, request: Request, spider: Spider | None = None - ) -> _HandleOutputDeferred: + ) -> Generator[Deferred[Any], Any, None]: if spider is not None: warnings.warn( "Passing a 'spider' argument to Scraper.enqueue_scrape() is deprecated.", @@ -156,103 +158,106 @@ class Scraper: if self.slot is None: raise RuntimeError("Scraper slot not assigned") dfd = self.slot.add_response_request(result, request) - - def finish_scraping(_: _T) -> _T: - assert self.slot is not None + self._scrape_next() + try: + yield dfd + except Exception: + logger.error( + "Scraper bug processing %(request)s", + {"request": request}, + exc_info=True, + extra={"spider": self.crawler.spider}, + ) + finally: self.slot.finish_response(result, request) self._check_if_closing() self._scrape_next() - return _ - - dfd.addBoth(finish_scraping) - dfd.addErrback( - lambda f: logger.error( - "Scraper bug processing %(request)s", - {"request": request}, - exc_info=failure_to_exc_info(f), - extra={"spider": self.crawler.spider}, - ) - ) - self._scrape_next() - return dfd def _scrape_next(self) -> None: assert self.slot is not None # typing while self.slot.queue: - response, request, deferred = self.slot.next_response_request_deferred() - self._scrape(response, request).chainDeferred(deferred) + result, request, deferred = self.slot.next_response_request_deferred() + self._scrape(result, request).chainDeferred(deferred) - def _scrape( - self, result: Response | Failure, request: Request - ) -> _HandleOutputDeferred: - """ - Handle the downloaded response or failure through the spider callback/errback - """ + @deferred_f_from_coro_f + async def _scrape(self, result: Response | Failure, request: Request) -> None: + """Handle the downloaded response or failure through the spider callback/errback.""" if not isinstance(result, (Response, Failure)): raise TypeError( f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}" ) - dfd: Deferred[Iterable[Any] | AsyncIterable[Any]] = self._scrape2( - result, request - ) # returns spider's processed output - dfd.addErrback(self.handle_spider_error, request, result) - dfd2: _HandleOutputDeferred = dfd.addCallback( - self.handle_spider_output, request, cast(Response, result) - ) - return dfd2 - def _scrape2( - self, result: Response | Failure, request: Request - ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: - """ - Handle the different cases of request's result been a Response or a Failure - """ + assert self.crawler.spider if isinstance(result, Response): - # Deferreds are invariant so Mutable*Chain isn't matched to *Iterable - assert self.crawler.spider - return self.spidermw.scrape_response( # type: ignore[return-value] - self.call_spider, result, request, self.crawler.spider - ) - # else result is a Failure - dfd = self.call_spider(result, request) - dfd.addErrback(self._log_download_errors, result, request) - return dfd + try: + # call the spider middlewares and the request callback with the response + output = await maybe_deferred_to_future( + self.spidermw.scrape_response( + self.call_spider, result, request, self.crawler.spider + ) + ) + except Exception: + self.handle_spider_error(Failure(), request, result) + else: + await self.handle_spider_output_async(output, request, result) + return + + try: + # call the request errback with the downloader error + await self.call_spider_async(result, request) + except Exception as spider_exc: + # the errback didn't silence the exception + if not result.check(IgnoreRequest): + logkws = self.logformatter.download_error( + result, request, self.crawler.spider + ) + logger.log( + *logformatter_adapter(logkws), + extra={"spider": self.crawler.spider}, + exc_info=failure_to_exc_info(result), + ) + if spider_exc is not result.value: + # the errback raised a different exception, handle it + self.handle_spider_error(Failure(), request, result) def call_spider( self, result: Response | Failure, request: Request, spider: Spider | None = None - ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: + ) -> Deferred[Iterable[Any] | AsyncIterator[Any]]: if spider is not None: warnings.warn( "Passing a 'spider' argument to Scraper.call_spider() is deprecated.", category=ScrapyDeprecationWarning, stacklevel=2, ) + return deferred_from_coro(self.call_spider_async(result, request)) + async def call_spider_async( + self, result: Response | Failure, request: Request + ) -> Iterable[Any] | AsyncIterator[Any]: + """Call the request callback or errback with the response or failure.""" + await maybe_deferred_to_future(_defer_sleep()) assert self.crawler.spider - dfd: Deferred[Any] if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request assert result.request callback = result.request.callback or self.crawler.spider._parse warn_on_generator_with_return_value(self.crawler.spider, callback) - dfd = defer_succeed(result) - dfd.addCallbacks( - callback=callback, callbackKeywords=result.request.cb_kwargs - ) + output = callback(result, **result.request.cb_kwargs) else: # result is a Failure # TODO: properly type adding this attribute to a Failure result.request = request # type: ignore[attr-defined] - dfd = defer_fail(result) - if request.errback: - warn_on_generator_with_return_value( - self.crawler.spider, request.errback - ) - dfd.addErrback(request.errback) - dfd2: Deferred[Iterable[Any] | AsyncIterable[Any]] = dfd.addCallback( - iterate_spider_output + if not request.errback: + result.raiseException() + warn_on_generator_with_return_value(self.crawler.spider, request.errback) + output = request.errback(result) + if isinstance(output, Failure): + output.raiseException() + # else the errback returned actual output (like a callback), + # which needs to be passed to iterate_spider_output() + return await maybe_deferred_to_future( + maybeDeferred(iterate_spider_output, output) ) - return dfd2 def handle_spider_error( self, @@ -261,6 +266,7 @@ class Scraper: response: Response | Failure, spider: Spider | None = None, ) -> None: + """Handle an exception raised by a spider callback or errback.""" if spider is not None: warnings.warn( "Passing a 'spider' argument to Scraper.handle_spider_error() is deprecated.", @@ -301,57 +307,68 @@ class Scraper: def handle_spider_output( self, - result: Iterable[_T] | AsyncIterable[_T], + result: Iterable[_T] | AsyncIterator[_T], request: Request, response: Response, spider: Spider | None = None, - ) -> _HandleOutputDeferred: + ) -> Deferred[None]: + """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" if spider is not None: warnings.warn( "Passing a 'spider' argument to Scraper.handle_spider_output() is deprecated.", category=ScrapyDeprecationWarning, stacklevel=2, ) + return deferred_from_coro( + self.handle_spider_output_async(result, request, response) + ) - if not result: - return defer_succeed(None) - it: Iterable[_T] | AsyncIterable[_T] - dfd: Deferred[_ParallelResult] - if isinstance(result, AsyncIterable): - it = aiter_errback(result, self.handle_spider_error, request, response) - dfd = parallel_async( + async def handle_spider_output_async( + self, + result: Iterable[_T] | AsyncIterator[_T], + request: Request, + response: Response, + ) -> None: + """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" + if isinstance(result, AsyncIterator): + ait = aiter_errback(result, self.handle_spider_error, request, response) + await maybe_deferred_to_future( + parallel_async( + ait, + self.concurrent_items, + self._process_spidermw_output, + response, + ) + ) + return + it = iter_errback(result, self.handle_spider_error, request, response) + await maybe_deferred_to_future( + parallel( it, self.concurrent_items, self._process_spidermw_output, response, ) - else: - it = iter_errback(result, self.handle_spider_error, request, response) - dfd = parallel( - it, - self.concurrent_items, - self._process_spidermw_output, - response, - ) - # returning Deferred[_ParallelResult] instead of Deferred[Union[_ParallelResult, None]] - return dfd # type: ignore[return-value] + ) - def _process_spidermw_output( - self, output: Any, response: Response - ) -> Deferred[Any] | None: + @deferred_f_from_coro_f + async def _process_spidermw_output(self, output: Any, response: Response) -> None: """Process each Request/Item (given in the output parameter) returned - from the given spider + from the given spider. + + Items are sent to the item pipelines, requests are scheduled. """ if isinstance(output, Request): assert self.crawler.engine is not None # typing self.crawler.engine.crawl(request=output) - elif output is None: - pass - else: - return self.start_itemproc(output, response=response) - return None + return + if output is not None: + await maybe_deferred_to_future( + self.start_itemproc(output, response=response) + ) - def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[Any]: + @deferred_f_from_coro_f + async def start_itemproc(self, item: Any, *, response: Response | None) -> None: """Send *item* to the item pipelines for processing. *response* is the source of the item data. If the item does not come @@ -360,90 +377,56 @@ class Scraper: assert self.slot is not None # typing assert self.crawler.spider is not None # typing self.slot.itemproc_size += 1 - dfd = self.itemproc.process_item(item, self.crawler.spider) - dfd.addBoth(self._itemproc_finished, item, response) - return dfd - - def _log_download_errors( - self, - spider_failure: Failure, - download_failure: Failure, - request: Request, - ) -> Failure | None: - """Log and silence errors that come from the engine (typically download - errors that got propagated thru here). - - spider_failure: the value passed into the errback of self.call_spider() - (likely raised in the request errback) - - download_failure: the value passed into _scrape2() from - ExecutionEngine._handle_downloader_output() as "result" - (likely raised in the download handler or a downloader middleware) - """ - if not download_failure.check(IgnoreRequest): - assert self.crawler.spider - logkws = self.logformatter.download_error( - download_failure, request, self.crawler.spider + try: + output = await maybe_deferred_to_future( + self.itemproc.process_item(item, self.crawler.spider) ) - logger.log( - *logformatter_adapter(logkws), - extra={"spider": self.crawler.spider}, - exc_info=failure_to_exc_info(download_failure), - ) - if spider_failure is not download_failure: - # a request errback raised a different exception, it needs to be handled later - return spider_failure - return None - - def _itemproc_finished( - self, output: Any, item: Any, response: Response | None - ) -> Deferred[Any]: - """ItemProcessor finished for the given ``item`` and returned ``output``""" - assert self.slot is not None # typing - assert self.crawler.spider - self.slot.itemproc_size -= 1 - if isinstance(output, Failure): - ex = output.value - if isinstance(ex, DropItem): - logkws = self.logformatter.dropped( - item, ex, response, self.crawler.spider + except DropItem as ex: + logkws = self.logformatter.dropped(item, ex, response, self.crawler.spider) + if logkws is not None: + logger.log( + *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} ) - if logkws is not None: - logger.log( - *logformatter_adapter(logkws), - extra={"spider": self.crawler.spider}, - ) - return self.signals.send_catch_log_deferred( + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred( signal=signals.item_dropped, item=item, response=response, spider=self.crawler.spider, - exception=output.value, + exception=ex, ) - assert ex + ) + except Exception as ex: logkws = self.logformatter.item_error( item, ex, response, self.crawler.spider ) logger.log( *logformatter_adapter(logkws), extra={"spider": self.crawler.spider}, - exc_info=failure_to_exc_info(output), + exc_info=True, ) - return self.signals.send_catch_log_deferred( - signal=signals.item_error, - item=item, - response=response, - spider=self.crawler.spider, - failure=output, + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred( + signal=signals.item_error, + item=item, + response=response, + spider=self.crawler.spider, + failure=Failure(), + ) ) - logkws = self.logformatter.scraped(output, response, self.crawler.spider) - if logkws is not None: - logger.log( - *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} + else: + logkws = self.logformatter.scraped(output, response, self.crawler.spider) + if logkws is not None: + logger.log( + *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} + ) + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred( + signal=signals.item_scraped, + item=output, + response=response, + spider=self.crawler.spider, + ) ) - return self.signals.send_catch_log_deferred( - signal=signals.item_scraped, - item=output, - response=response, - spider=self.crawler.spider, - ) + finally: + self.slot.itemproc_size -= 1 diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index b8b0aec44..4a0cd9464 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -7,16 +7,17 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from collections.abc import AsyncIterable, Callable, Iterable +from collections.abc import AsyncIterator, Callable, Iterable from inspect import isasyncgenfunction, iscoroutine from itertools import islice from typing import TYPE_CHECKING, Any, TypeVar, Union, cast +from warnings import warn from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from scrapy import Request, Spider -from scrapy.exceptions import _InvalidOutput +from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput from scrapy.http import Response from scrapy.middleware import MiddlewareManager from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen @@ -40,12 +41,13 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") ScrapeFunc = Callable[ - [Union[Response, Failure], Request], Union[Iterable[_T], AsyncIterable[_T]] + [Union[Response, Failure], Request], + Deferred[Union[Iterable[_T], AsyncIterator[_T]]], ] def _isiterable(o: Any) -> bool: - return isinstance(o, (Iterable, AsyncIterable)) + return isinstance(o, (Iterable, AsyncIterator)) class SpiderMiddlewareManager(MiddlewareManager): @@ -55,12 +57,75 @@ class SpiderMiddlewareManager(MiddlewareManager): def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES")) + def __init__(self, *middlewares: Any) -> None: + self._check_deprecated_process_start_requests_use(middlewares) + super().__init__(*middlewares) + + def _check_deprecated_process_start_requests_use( + self, middlewares: tuple[Any] + ) -> None: + deprecated_middlewares = [ + middleware + for middleware in middlewares + if hasattr(middleware, "process_start_requests") + and not hasattr(middleware, "process_start") + ] + modern_middlewares = [ + middleware + for middleware in middlewares + if not hasattr(middleware, "process_start_requests") + and hasattr(middleware, "process_start") + ] + if deprecated_middlewares and modern_middlewares: + raise ValueError( + "You are trying to combine spider middlewares that only " + "define the deprecated process_start_requests() method () " + "with spider middlewares that only define the " + "process_start() method (). This is not possible. You must " + "either disable or make universal 1 of those 2 sets of " + "spider middlewares. Making a spider middleware universal " + "means having it define both methods. See the release notes " + "of Scrapy VERSION for details: " + "https://docs.scrapy.org/en/VERSION/news.html" + ) + + self._use_start_requests = bool(deprecated_middlewares) + if self._use_start_requests: + deprecated_middleware_list = ", ".join( + global_object_name(middleware.__class__) + for middleware in deprecated_middlewares + ) + warn( + f"The following enabled spider middlewares, directly or " + f"through their parent classes, define the deprecated " + f"process_start_requests() method: " + f"{deprecated_middleware_list}. process_start_requests() has " + f"been deprecated in favor of a new method, process_start(), " + f"to support asynchronous code execution. " + f"process_start_requests() will stop being called in a future " + f"version of Scrapy. If you use Scrapy VERSION or higher " + f"only, replace process_start_requests() with " + f"process_start(); note that process_start() is a coroutine " + f"(async def). If you need to maintain compatibility with " + f"lower Scrapy versions, when defining " + f"process_start_requests() in a spider middleware class, " + f"define process_start() as well. See the release notes of " + f"Scrapy VERSION for details: " + f"https://docs.scrapy.org/en/VERSION/news.html", + ScrapyDeprecationWarning, + ) + def _add_middleware(self, mw: Any) -> None: super()._add_middleware(mw) if hasattr(mw, "process_spider_input"): self.methods["process_spider_input"].append(mw.process_spider_input) - if hasattr(mw, "process_start_requests"): - self.methods["process_start_requests"].appendleft(mw.process_start_requests) + if self._use_start_requests: + if hasattr(mw, "process_start_requests"): + self.methods["process_start_requests"].appendleft( + mw.process_start_requests + ) + elif hasattr(mw, "process_start"): + self.methods["process_start"].appendleft(mw.process_start) process_spider_output = self._get_async_method_pair(mw, "process_spider_output") self.methods["process_spider_output"].appendleft(process_spider_output) process_spider_exception = getattr(mw, "process_spider_exception", None) @@ -72,7 +137,7 @@ class SpiderMiddlewareManager(MiddlewareManager): response: Response, request: Request, spider: Spider, - ) -> Iterable[_T] | AsyncIterable[_T]: + ) -> Deferred[Iterable[_T] | AsyncIterator[_T]]: for method in self.methods["process_spider_input"]: method = cast(Callable, method) try: @@ -93,10 +158,10 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - iterable: Iterable[_T] | AsyncIterable[_T], + iterable: Iterable[_T] | AsyncIterator[_T], exception_processor_index: int, recover_to: MutableChain[_T] | MutableAsyncChain[_T], - ) -> Iterable[_T] | AsyncIterable[_T]: + ) -> Iterable[_T] | AsyncIterator[_T]: def process_sync(iterable: Iterable[_T]) -> Iterable[_T]: try: yield from iterable @@ -112,7 +177,7 @@ class SpiderMiddlewareManager(MiddlewareManager): assert isinstance(recover_to, MutableChain) recover_to.extend(exception_result) - async def process_async(iterable: AsyncIterable[_T]) -> AsyncIterable[_T]: + async def process_async(iterable: AsyncIterator[_T]) -> AsyncIterator[_T]: try: async for r in iterable: yield r @@ -128,7 +193,7 @@ class SpiderMiddlewareManager(MiddlewareManager): assert isinstance(recover_to, MutableAsyncChain) recover_to.extend(exception_result) - if isinstance(iterable, AsyncIterable): + if isinstance(iterable, AsyncIterator): return process_async(iterable) return process_sync(iterable) @@ -187,13 +252,13 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - result: Iterable[_T] | AsyncIterable[_T], + result: Iterable[_T] | AsyncIterator[_T], start_index: int = 0, ) -> Generator[Deferred[Any], Any, MutableChain[_T] | MutableAsyncChain[_T]]: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered: MutableChain[_T] | MutableAsyncChain[_T] - last_result_is_async = isinstance(result, AsyncIterable) + last_result_is_async = isinstance(result, AsyncIterator) recovered = MutableAsyncChain() if last_result_is_async else MutableChain() # There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async. @@ -220,7 +285,7 @@ class SpiderMiddlewareManager(MiddlewareManager): need_downgrade = True try: if need_upgrade: - # Iterable -> AsyncIterable + # Iterable -> AsyncIterator result = as_async_generator(result) elif need_downgrade: logger.warning( @@ -230,10 +295,10 @@ class SpiderMiddlewareManager(MiddlewareManager): f" https://docs.scrapy.org/en/latest/topics/coroutines.html#for-middleware-users" f" for more information." ) - assert isinstance(result, AsyncIterable) - # AsyncIterable -> Iterable + assert isinstance(result, AsyncIterator) + # AsyncIterator -> Iterable result = yield deferred_from_coro(collect_asyncgen(result)) - if isinstance(recovered, AsyncIterable): + if isinstance(recovered, AsyncIterator): recovered_collected = yield deferred_from_coro( collect_asyncgen(recovered) ) @@ -266,7 +331,7 @@ class SpiderMiddlewareManager(MiddlewareManager): f"{type(result)}" ) raise _InvalidOutput(msg) - last_result_is_async = isinstance(result, AsyncIterable) + last_result_is_async = isinstance(result, AsyncIterator) if last_result_is_async: return MutableAsyncChain(result, recovered) @@ -276,23 +341,23 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - result: Iterable[_T] | AsyncIterable[_T], + result: Iterable[_T] | AsyncIterator[_T], ) -> MutableChain[_T] | MutableAsyncChain[_T]: recovered: MutableChain[_T] | MutableAsyncChain[_T] - if isinstance(result, AsyncIterable): + if isinstance(result, AsyncIterator): recovered = MutableAsyncChain() else: recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) result = await maybe_deferred_to_future( cast( - "Deferred[Iterable[_T] | AsyncIterable[_T]]", + "Deferred[Iterable[_T] | AsyncIterator[_T]]", self._process_spider_output(response, spider, result), ) ) - if isinstance(result, AsyncIterable): + if isinstance(result, AsyncIterator): return MutableAsyncChain(result, recovered) - if isinstance(recovered, AsyncIterable): + if isinstance(recovered, AsyncIterator): recovered_collected = await collect_asyncgen(recovered) recovered = MutableChain(recovered_collected) return MutableChain(result, recovered) @@ -305,7 +370,7 @@ class SpiderMiddlewareManager(MiddlewareManager): spider: Spider, ) -> Deferred[MutableChain[_T] | MutableAsyncChain[_T]]: async def process_callback_output( - result: Iterable[_T] | AsyncIterable[_T], + result: Iterable[_T] | AsyncIterator[_T], ) -> MutableChain[_T] | MutableAsyncChain[_T]: return await self._process_callback_output(response, spider, result) @@ -314,7 +379,7 @@ class SpiderMiddlewareManager(MiddlewareManager): ) -> Failure | MutableChain[_T] | MutableAsyncChain[_T]: return self._process_spider_exception(response, spider, _failure) - dfd: Deferred[Iterable[_T] | AsyncIterable[_T]] = mustbe_deferred( + dfd: Deferred[Iterable[_T] | AsyncIterator[_T]] = mustbe_deferred( self._process_spider_input, scrape_func, response, request, spider ) dfd2: Deferred[MutableChain[_T] | MutableAsyncChain[_T]] = dfd.addCallback( @@ -323,10 +388,90 @@ class SpiderMiddlewareManager(MiddlewareManager): dfd2.addErrback(process_spider_exception) return dfd2 - def process_start_requests( - self, start_requests: Iterable[Request], spider: Spider - ) -> Deferred[Iterable[Request]]: - return self._process_chain("process_start_requests", start_requests, spider) + async def process_start(self, spider: Spider) -> AsyncIterator[Any] | None: + self._check_deprecated_start_requests_use(spider) + if self._use_start_requests: + sync_start = iter(spider.start_requests()) + sync_start = await maybe_deferred_to_future( + self._process_chain("process_start_requests", sync_start, spider) + ) + start: AsyncIterator[Any] = as_async_generator(sync_start) + else: + start = spider.start() + start = await maybe_deferred_to_future( + self._process_chain("process_start", start) + ) + return start + + def _check_deprecated_start_requests_use(self, spider: Spider): + start_requests_cls = None + start_cls = None + spidercls = spider.__class__ + mro = spidercls.__mro__ + + for cls in mro: + cls_dict = cls.__dict__ + if start_requests_cls is None and "start_requests" in cls_dict: + start_requests_cls = cls + if start_cls is None and "start" in cls_dict: + start_cls = cls + if start_requests_cls is not None and start_cls is not None: + break + + # Spider defines both, start_requests and start. + assert start_requests_cls is not None + assert start_cls is not None + + if ( + start_requests_cls is not Spider + and start_cls is not start_requests_cls + and mro.index(start_requests_cls) < mro.index(start_cls) + ): + src = global_object_name(start_requests_cls) + if start_requests_cls is not spidercls: + src += f" (inherited by {global_object_name(spidercls)})" + warn( + f"{src} defines the deprecated start_requests() method. " + f"start_requests() has been deprecated in favor of a new " + f"method, start(), to support asynchronous code " + f"execution. start_requests() will stop being called in a " + f"future version of Scrapy. If you use Scrapy VERSION or " + f"higher only, replace start_requests() with start(); " + f"note that start() is a coroutine (async def). If you " + f"need to maintain compatibility with lower Scrapy versions, " + f"when overriding start_requests() in a spider class, " + f"override start() as well; you can use super() to " + f"reuse the inherited start() implementation without " + f"copy-pasting. See the release notes of Scrapy VERSION for " + f"details: https://docs.scrapy.org/en/VERSION/news.html", + ScrapyDeprecationWarning, + ) + + if ( + self._use_start_requests + and start_cls is not Spider + and start_requests_cls is not start_cls + and mro.index(start_cls) < mro.index(start_requests_cls) + ): + src = global_object_name(start_cls) + if start_cls is not spidercls: + src += f" (inherited by {global_object_name(spidercls)})" + raise ValueError( + f"{src} does not define the deprecated start_requests() " + f"method. However, one or more of your enabled spider " + f"middlewares (reported in an earlier deprecation warning) " + f"define the process_start_requests() method, and not the " + f"process_start() method, making them only compatible with " + f"(deprecated) spiders that define the start_requests() " + f"method. To solve this issue, disable the offending spider " + f"middlewares, upgrade them as described in that earlier " + f"deprecation warning, or make your spider compatible with " + f"deprecated spider middlewares (and earlier Scrapy versions) " + f"by defining a sync start_requests() method that works " + f"similarly to its existing start() method. See the " + f"release notes of Scrapy VERSION for details: " + f"https://docs.scrapy.org/en/VERSION/news.html" + ) # This method is only needed until _async compatibility methods are removed. @staticmethod diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1ec1e31dc..749096db5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -136,6 +136,9 @@ class Crawler: "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} ) + # Cannot use @deferred_f_from_coro_f because that relies on the reactor + # being installed already, which is done within _apply_settings(), inside + # this method. @inlineCallbacks def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred[Any], Any, None]: if self.crawling: @@ -151,9 +154,8 @@ class Crawler: self._apply_settings() self._update_root_log_handler() self.engine = self._create_engine() - start_requests = iter(self.spider.start_requests()) - yield self.engine.open_spider(self.spider, start_requests) - yield maybeDeferred(self.engine.start) + yield self.engine.open_spider(self.spider) + yield self.engine.start() except Exception: self.crawling = False if self.engine is not None: diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index ac832e025..bacee8f0a 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -104,7 +104,6 @@ class TelnetConsole(protocol.ServerFactory): telnet_vars: dict[str, Any] = { "engine": self.crawler.engine, "spider": self.crawler.engine.spider, - "slot": self.crawler.engine.slot, "crawler": self.crawler, "extensions": self.crawler.extensions, "stats": self.crawler.stats, diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 6d3b7a926..2b8d0ab84 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -130,6 +130,16 @@ class Request(object_ref): self._set_body(body) if not isinstance(priority, int): raise TypeError(f"Request priority not an integer: {priority!r}") + + #: Default: ``0`` + #: + #: Value that the :ref:`scheduler ` may use for + #: request prioritization. + #: + #: Built-in schedulers prioritize requests with a higher priority + #: value. + #: + #: Negative values are allowed. self.priority: int = priority if not (callable(callback) or callback is None): @@ -191,7 +201,7 @@ class Request(object_ref): #: #: When defining the start URLs of a spider through #: :attr:`~scrapy.Spider.start_urls`, this attribute is enabled by - #: default. See :meth:`~scrapy.Spider.start_requests`. + #: default. See :meth:`~scrapy.Spider.start`. self.dont_filter: bool = dont_filter self._meta: dict[str, Any] | None = dict(meta) if meta else None diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index f10e91beb..4f08918ae 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -98,7 +98,7 @@ class LogFormatter: """Logs a message when an item is scraped by a spider.""" src: Any if response is None: - src = f"{global_object_name(spider.__class__)}.start_requests" + src = f"{global_object_name(spider.__class__)}.start" elif isinstance(response, Failure): src = response.getErrorMessage() else: diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 324a9b955..e6c6b8bf1 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -72,7 +72,6 @@ class ScrapyPriorityQueue: startprios is a sequence of priorities to start with. If the queue was previously closed leaving some priority buckets non-empty, those priorities should be passed in startprios. - """ @classmethod @@ -82,8 +81,16 @@ class ScrapyPriorityQueue: downstream_queue_cls: type[QueueProtocol], key: str, startprios: Iterable[int] = (), + *, + start_queue_cls: type[QueueProtocol] | None = None, ) -> Self: - return cls(crawler, downstream_queue_cls, key, startprios) + return cls( + crawler, + downstream_queue_cls, + key, + startprios, + start_queue_cls=start_queue_cls, + ) def __init__( self, @@ -91,11 +98,15 @@ class ScrapyPriorityQueue: downstream_queue_cls: type[QueueProtocol], key: str, startprios: Iterable[int] = (), + *, + start_queue_cls: type[QueueProtocol] | None = None, ): self.crawler: Crawler = crawler self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls + self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls self.key: str = key self.queues: dict[int, QueueProtocol] = {} + self._start_queues: dict[int, QueueProtocol] = {} self.curprio: int | None = None self.init_prios(startprios) @@ -104,7 +115,13 @@ class ScrapyPriorityQueue: return for priority in startprios: - self.queues[priority] = self.qfactory(priority) + q = self.qfactory(priority) + if q: + self.queues[priority] = q + if self._start_queue_cls: + q = self._sqfactory(priority) + if q: + self._start_queues[priority] = q self.curprio = min(startprios) @@ -115,29 +132,66 @@ class ScrapyPriorityQueue: self.key + "/" + str(key), ) + def _sqfactory(self, key: int) -> QueueProtocol: + assert self._start_queue_cls is not None + return build_from_crawler( + self._start_queue_cls, + self.crawler, + f"{self.key}/{key}s", + ) + def priority(self, request: Request) -> int: return -request.priority def push(self, request: Request) -> None: priority = self.priority(request) - if priority not in self.queues: - self.queues[priority] = self.qfactory(priority) - q = self.queues[priority] + is_start_request = request.meta.get("is_start_request", False) + if is_start_request and self._start_queue_cls: + if priority not in self._start_queues: + self._start_queues[priority] = self._sqfactory(priority) + q = self._start_queues[priority] + else: + if priority not in self.queues: + self.queues[priority] = self.qfactory(priority) + q = self.queues[priority] q.push(request) # this may fail (eg. serialization error) if self.curprio is None or priority < self.curprio: self.curprio = priority def pop(self) -> Request | None: - if self.curprio is None: - return None - q = self.queues[self.curprio] - m = q.pop() - if not q: - del self.queues[self.curprio] - q.close() - prios = [p for p, q in self.queues.items() if q] - self.curprio = min(prios) if prios else None - return m + while self.curprio is not None: + if self._start_queues: + try: + q = self._start_queues[self.curprio] + except KeyError: + pass + else: + m = q.pop() + if not q: + del self._start_queues[self.curprio] + q.close() + return m + try: + q = self.queues[self.curprio] + except KeyError: + self._update_curprio() + else: + m = q.pop() + if not q: + del self.queues[self.curprio] + q.close() + self._update_curprio() + return m + return None + + def _update_curprio(self) -> None: + prios = { + p + for queues in (self.queues, self._start_queues) + for p, q in queues.items() + if q + } + self.curprio = min(prios) if prios else None def peek(self) -> Request | None: """Returns the next object to be returned by :meth:`pop`, @@ -148,19 +202,31 @@ class ScrapyPriorityQueue: """ if self.curprio is None: return None - queue = self.queues[self.curprio] + try: + queue = self._start_queues[self.curprio] + except KeyError: + queue = self.queues[self.curprio] # Protocols can't declare optional members return cast(Request, queue.peek()) # type: ignore[attr-defined] def close(self) -> list[int]: - active: list[int] = [] - for p, q in self.queues.items(): - active.append(p) - q.close() - return active + active: set[int] = set() + for queues in (self.queues, self._start_queues): + for p, q in queues.items(): + active.add(p) + q.close() + return list(active) def __len__(self) -> int: - return sum(len(x) for x in self.queues.values()) if self.queues else 0 + return ( + sum( + len(x) + for queues in (self.queues, self._start_queues) + for x in queues.values() + ) + if self.queues or self._start_queues + else 0 + ) class DownloaderInterface: @@ -194,8 +260,16 @@ class DownloaderAwarePriorityQueue: downstream_queue_cls: type[QueueProtocol], key: str, startprios: dict[str, Iterable[int]] | None = None, + *, + start_queue_cls: type[QueueProtocol] | None = None, ) -> Self: - return cls(crawler, downstream_queue_cls, key, startprios) + return cls( + crawler, + downstream_queue_cls, + key, + startprios, + start_queue_cls=start_queue_cls, + ) def __init__( self, @@ -203,6 +277,8 @@ class DownloaderAwarePriorityQueue: downstream_queue_cls: type[QueueProtocol], key: str, slot_startprios: dict[str, Iterable[int]] | None = None, + *, + start_queue_cls: type[QueueProtocol] | None = None, ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( @@ -222,6 +298,7 @@ class DownloaderAwarePriorityQueue: self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler) self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls + self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls self.key: str = key self.crawler: Crawler = crawler @@ -237,6 +314,7 @@ class DownloaderAwarePriorityQueue: self.downstream_queue_cls, self.key + "/" + _path_safe(slot), startprios, + start_queue_cls=self._start_queue_cls, ) def pop(self) -> Request | None: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 680fded7a..01443fa17 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -305,6 +305,8 @@ SCHEDULER = "scrapy.core.scheduler.Scheduler" SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleLifoDiskQueue" SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.LifoMemoryQueue" SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue" +SCHEDULER_START_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue" +SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000 @@ -315,6 +317,7 @@ SPIDER_MIDDLEWARES = {} SPIDER_MIDDLEWARES_BASE = { # Engine side + "scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25, "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50, "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, diff --git a/scrapy/shell.py b/scrapy/shell.py index 5e5e57a9a..bb39eccc3 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -24,6 +24,7 @@ from scrapy.spiders import Spider from scrapy.utils.conf import get_config from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console from scrapy.utils.datatypes import SequenceExclude +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.misc import load_object from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop from scrapy.utils.response import open_in_browser @@ -102,25 +103,33 @@ class Shell: # set the asyncio event loop for the current thread event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"] set_asyncio_event_loop(event_loop_path) - spider = self._open_spider(request, spider) + + def crawl_request(_): + assert self.crawler.engine is not None + self.crawler.engine.crawl(request) + + d2 = self._open_spider(request, spider) + d2.addCallback(crawl_request) + d = _request_deferred(request) d.addCallback(lambda x: (x, spider)) - assert self.crawler.engine - self.crawler.engine.crawl(request) return d - def _open_spider(self, request: Request, spider: Spider | None) -> Spider: + @deferred_f_from_coro_f + async def _open_spider(self, request: Request, spider: Spider | None) -> None: if self.spider: - return self.spider + return if spider is None: spider = self.crawler.spider or self.crawler._create_spider() self.crawler.spider = spider assert self.crawler.engine - self.crawler.engine.open_spider(spider, close_if_idle=False) + await maybe_deferred_to_future( + self.crawler.engine.open_spider(spider, close_if_idle=False) + ) + self.crawler.engine._start_request_processing() self.spider = spider - return spider def fetch( self, diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index e106418d6..f8c50b5e3 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,13 +1,12 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import Any from pydispatch import dispatcher +from twisted.internet.defer import Deferred from scrapy.utils import signal as _signal - -if TYPE_CHECKING: - from twisted.internet.defer import Deferred +from scrapy.utils.defer import maybe_deferred_to_future class SignalManager: @@ -75,3 +74,17 @@ class SignalManager: """ kwargs.setdefault("sender", self.sender) _signal.disconnect_all(signal, **kwargs) + + async def wait_for(self, signal): + """Await the next *signal*. + + See :ref:`start-requests-lazy` for an example. + """ + d = Deferred() + + def handle(): + self.disconnect(handle, signal) + d.callback(None) + + self.connect(handle, signal) + await maybe_deferred_to_future(d) diff --git a/scrapy/signals.py b/scrapy/signals.py index 8ef0f34f0..bdeec1ba0 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -7,6 +7,7 @@ signals here without documenting them there. engine_started = object() engine_stopped = object() +scheduler_empty = object() spider_opened = object() spider_idle = object() spider_closed = object() diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py index 650192095..cfb50c599 100644 --- a/scrapy/spidermiddlewares/base.py +++ b/scrapy/spidermiddlewares/base.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any from scrapy import Request, Spider if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable + from collections.abc import AsyncIterator, Iterable # typing.Self requires Python 3.11 from typing_extensions import Self @@ -17,9 +17,9 @@ if TYPE_CHECKING: class BaseSpiderMiddleware: """Optional base class for spider middlewares. - This class provides helper methods for asynchronous ``process_spider_output`` - methods. Middlewares that don't have a ``process_spider_output`` method don't need - to use it. + This class provides helper methods for asynchronous + ``process_spider_output()`` and ``process_start()`` methods. Middlewares + that don't have either of these methods don't need to use this class. You can override the :meth:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware.get_processed_request` @@ -38,59 +38,70 @@ class BaseSpiderMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) + def process_start_requests( + self, start: Iterable[Any], spider: Spider + ) -> Iterable[Any]: + for o in start: + if (o := self._get_processed(o, None)) is not None: + yield o + + async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]: + async for o in start: + if (o := self._get_processed(o, None)) is not None: + yield o + def process_spider_output( self, response: Response, result: Iterable[Any], spider: Spider ) -> Iterable[Any]: for o in result: - if isinstance(o, Request): - o = self.get_processed_request(o, response) - else: - o = self.get_processed_item(o, response) - if o is not None: + if (o := self._get_processed(o, response)) is not None: yield o async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: + self, response: Response, result: AsyncIterator[Any], spider: Spider + ) -> AsyncIterator[Any]: async for o in result: - if isinstance(o, Request): - o = self.get_processed_request(o, response) - else: - o = self.get_processed_item(o, response) - if o is not None: + if (o := self._get_processed(o, response)) is not None: yield o + def _get_processed(self, o: Any, response: Response | None) -> Any: + if isinstance(o, Request): + return self.get_processed_request(o, response) + return self.get_processed_item(o, response) + def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: """Return a processed request from the spider output. - This method is called with a single request from the spider output. - It should return the same or a different request, or ``None`` to - ignore it. + This method is called with a single request from the start seeds or the + spider output. It should return the same or a different request, or + ``None`` to ignore it. :param request: the input request :type request: :class:`~scrapy.Request` object :param response: the response being processed - :type response: :class:`~scrapy.http.Response` object + :type response: :class:`~scrapy.http.Response` object or ``None`` for + start seeds :return: the processed request or ``None`` """ return request - def get_processed_item(self, item: Any, response: Response) -> Any: + def get_processed_item(self, item: Any, response: Response | None) -> Any: """Return a processed item from the spider output. - This method is called with a single item from the spider output. - It should return the same or a different item, or ``None`` to - ignore it. + This method is called with a single item from the start seeds or the + spider output. It should return the same or a different item, or + ``None`` to ignore it. :param item: the input item :type item: item object :param response: the response being processed - :type response: :class:`~scrapy.http.Response` object + :type response: :class:`~scrapy.http.Response` object or ``None`` for + start seeds :return: the processed item or ``None`` """ diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 65905f483..6b115ebe6 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Any from scrapy.spidermiddlewares.base import BaseSpiderMiddleware if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable + from collections.abc import AsyncIterator, Iterable # typing.Self requires Python 3.11 from typing_extensions import Self @@ -59,8 +59,8 @@ class DepthMiddleware(BaseSpiderMiddleware): yield from super().process_spider_output(response, result, spider) async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: + self, response: Response, result: AsyncIterator[Any], spider: Spider + ) -> AsyncIterator[Any]: self._init_depth(response, spider) async for o in super().process_spider_output_async(response, result, spider): yield o @@ -73,8 +73,11 @@ class DepthMiddleware(BaseSpiderMiddleware): self.stats.inc_value("request_depth_count/0", spider=spider) def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: + if response is None: + # start requests + return request depth = response.meta["depth"] + 1 request.meta["depth"] = depth if self.prio: diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 0918c9fac..2463275d5 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -49,8 +49,11 @@ class OffsiteMiddleware(BaseSpiderMiddleware): return o def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: + if response is None: + # skip start requests for backward compatibility + return request assert self.crawler.spider if ( request.dont_filter diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index b2ba8ba8c..f5d406c13 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -370,8 +370,11 @@ class RefererMiddleware(BaseSpiderMiddleware): return cls() if cls else self.default_policy() def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: + if response is None: + # start requests + return request referrer = self.policy(response, request).referrer(response.url, request.url) if referrer is not None: request.headers.setdefault("Referer", referrer) diff --git a/scrapy/spidermiddlewares/start.py b/scrapy/spidermiddlewares/start.py new file mode 100644 index 000000000..5d76b60d2 --- /dev/null +++ b/scrapy/spidermiddlewares/start.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from .base import BaseSpiderMiddleware + +if TYPE_CHECKING: + from scrapy.http import Request + from scrapy.http.response import Response + + +class StartSpiderMiddleware(BaseSpiderMiddleware): + """Set :reqmeta:`is_start_request`. + + .. reqmeta:: is_start_request + + is_start_request + ---------------- + + :attr:`~scrapy.Request.meta` key that is set to ``True`` in :ref:`start + requests `, allowing you to tell start requests apart from + other requests, e.g. in :ref:`downloader middlewares + `. + """ + + def get_processed_request( + self, request: Request, response: Response | None + ) -> Request | None: + if response is None: + request.meta.setdefault("is_start_request", True) + return request diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 177c19e1b..5590165a5 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -39,7 +39,7 @@ class UrlLengthMiddleware(BaseSpiderMiddleware): return o def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: if len(request.url) <= self.maxlength: return request diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index e255e91cc..0a1d85ae6 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,15 +7,17 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging +import warnings from typing import TYPE_CHECKING, Any, cast from scrapy import signals +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import AsyncIterator, Iterable from twisted.internet.defer import Deferred @@ -29,13 +31,19 @@ if TYPE_CHECKING: class Spider(object_ref): - """Base class for scrapy spiders. All spiders must inherit from this - class. + """Base class that any spider must subclass. + + It provides a default :meth:`start` implementation that sends + requests based on the :attr:`start_urls` class attribute and calls the + :meth:`parse` method for each response. """ name: str custom_settings: dict[_SettingsKeyT, Any] | None = None + #: Start URLs. See :meth:`start`. + start_urls: list[str] + def __init__(self, name: str | None = None, **kwargs: Any): if name is not None: self.name: str = name @@ -72,7 +80,70 @@ class Spider(object_ref): self.settings: BaseSettings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) - def start_requests(self) -> Iterable[Request]: + async def start(self) -> AsyncIterator[Any]: + """Yield the initial :class:`~scrapy.Request` objects to send. + + .. versionadded:: VERSION + + For example: + + .. code-block:: python + + from scrapy import Request, Spider + + + class MySpider(Spider): + name = "myspider" + + async def start(self): + yield Request("https://toscrape.com/") + + The default implementation reads URLs from :attr:`start_urls` and + yields a request for each with :attr:`~scrapy.Request.dont_filter` + enabled. It is functionally equivalent to: + + .. code-block:: python + + async def start(self): + for url in self.start_urls: + yield Request(url, dont_filter=True) + + You can also yield :ref:`items `. For example: + + .. code-block:: python + + async def start(self): + yield {"foo": "bar"} + + To write spiders that work on Scrapy versions lower than VERSION, + define also a synchronous ``start_requests()`` method that returns an + iterable. For example: + + .. code-block:: python + + def start_requests(self): + yield Request("https://toscrape.com/") + + .. seealso:: :ref:`start-requests` + """ + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$" + ) + for item_or_request in self.start_requests(): + yield item_or_request + + def start_requests(self) -> Iterable[Any]: + warnings.warn( + ( + "The Spider.start_requests() method is deprecated, use " + "Spider.start() instead. If you are calling " + "super().start_requests() from a Spider.start() override, " + "iterate super().start() instead." + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) if not self.start_urls and hasattr(self, "start_url"): raise AttributeError( "Crawling could not start: 'start_urls' not found " diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 087049425..171d8479c 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -8,7 +8,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import copy -from collections.abc import AsyncIterable, Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast from twisted.python.failure import Failure @@ -156,10 +156,10 @@ class CrawlSpider(Spider): callback: CallbackT | None, cb_kwargs: dict[str, Any], follow: bool = True, - ) -> AsyncIterable[Any]: + ) -> AsyncIterator[Any]: if callback: cb_res = callback(response, **cb_kwargs) or () - if isinstance(cb_res, AsyncIterable): + if isinstance(cb_res, AsyncIterator): cb_res = await collect_asyncgen(cb_res) elif isinstance(cb_res, Awaitable): cb_res = await cb_res diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index a7dba989e..e5548b9fa 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from collections.abc import Iterable +from collections.abc import AsyncIterator, Iterable from typing import TYPE_CHECKING, Any, cast from scrapy import Request @@ -29,6 +29,14 @@ class InitSpider(Spider): stacklevel=2, ) + async def start(self) -> AsyncIterator[Any]: + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$" + ) + for item_or_request in self.start_requests(): + yield item_or_request + def start_requests(self) -> Iterable[Request]: self._postinit_reqs: Iterable[Request] = super().start_requests() return cast(Iterable[Request], iterate_spider_output(self.init_request())) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 39033ac3c..2813a32a0 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -4,7 +4,7 @@ import logging import re # Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation -from collections.abc import Iterable, Sequence # noqa: TC003 +from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003 from typing import TYPE_CHECKING, Any, cast from scrapy.http import Request, Response, XmlResponse @@ -53,6 +53,10 @@ class SitemapSpider(Spider): self._cbs.append((regex(r), c)) self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] + async def start(self) -> AsyncIterator[Any]: + for item_or_request in self.start_requests(): + yield item_or_request + def start_requests(self) -> Iterable[Request]: for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index dcb2d63de..3f0239832 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -43,14 +43,11 @@ class ${ProjectName}SpiderMiddleware: # Should return either None or an iterable of Request or item objects. pass - def process_start_requests(self, start_requests, spider): - # Called with the start requests of the spider, and works - # similarly to the process_spider_output() method, except - # that it doesn’t have a response associated. - - # Must return only requests (not items). - for r in start_requests: - yield r + async def process_start(self, start): + # Called with an async iterator over the spider start() method or the + # maching method of an earlier spider middleware. + async for item_or_request in start: + yield item_or_request def spider_opened(self, spider): spider.logger.info("Spider opened: %s" % spider.name) diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 237bd8331..6d96a41f5 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,20 +1,20 @@ from __future__ import annotations -from collections.abc import AsyncGenerator, AsyncIterable, Iterable +from collections.abc import AsyncGenerator, AsyncIterator, Iterable from typing import TypeVar _T = TypeVar("_T") -async def collect_asyncgen(result: AsyncIterable[_T]) -> list[_T]: +async def collect_asyncgen(result: AsyncIterator[_T]) -> list[_T]: return [x async for x in result] async def as_async_generator( - it: Iterable[_T] | AsyncIterable[_T], + it: Iterable[_T] | AsyncIterator[_T], ) -> AsyncGenerator[_T]: """Wraps an iterable (sync or async) into an async generator.""" - if isinstance(it, AsyncIterable): + if isinstance(it, AsyncIterator): async for r in it: yield r else: diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 42ad28d8d..6e1687f3e 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -14,7 +14,11 @@ from types import CoroutineType from typing import TYPE_CHECKING, Any, Generic, TypeVar, Union, cast, overload from twisted.internet import defer -from twisted.internet.defer import Deferred, DeferredList, ensureDeferred +from twisted.internet.defer import ( + Deferred, + DeferredList, + ensureDeferred, +) from twisted.internet.task import Cooperator from twisted.python import failure @@ -22,7 +26,7 @@ from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed if TYPE_CHECKING: - from collections.abc import AsyncIterable, AsyncIterator, Callable + from collections.abc import AsyncIterator, Callable from twisted.python.failure import Failure @@ -36,6 +40,9 @@ _T = TypeVar("_T") _T2 = TypeVar("_T2") +_DEFER_DELAY = 0.1 + + def defer_fail(_failure: Failure) -> Deferred[Any]: """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -46,7 +53,7 @@ def defer_fail(_failure: Failure) -> Deferred[Any]: from twisted.internet import reactor d: Deferred[Any] = Deferred() - reactor.callLater(0.1, d.errback, _failure) + reactor.callLater(_DEFER_DELAY, d.errback, _failure) return d @@ -60,7 +67,16 @@ def defer_succeed(result: _T) -> Deferred[_T]: from twisted.internet import reactor d: Deferred[_T] = Deferred() - reactor.callLater(0.1, d.callback, result) + reactor.callLater(_DEFER_DELAY, d.callback, result) + return d + + +def _defer_sleep() -> Deferred[None]: + """Like ``defer_succeed`` and ``defer_fail`` but doesn't call any real callbacks.""" + from twisted.internet import reactor + + d: Deferred[None] = Deferred() + reactor.callLater(_DEFER_DELAY, d.callback, None) return d @@ -177,7 +193,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): def __init__( self, - aiterable: AsyncIterable[_T], + aiterable: AsyncIterator[_T], callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], *callable_args: _P.args, **callable_kwargs: _P.kwargs, @@ -234,7 +250,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): def parallel_async( - async_iterable: AsyncIterable[_T], + async_iterable: AsyncIterator[_T], count: int, callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], *args: _P.args, @@ -332,13 +348,13 @@ def iter_errback( async def aiter_errback( - aiterable: AsyncIterable[_T], + aiterable: AsyncIterator[_T], errback: Callable[Concatenate[Failure, _P], Any], *a: _P.args, **kw: _P.kwargs, -) -> AsyncIterable[_T]: +) -> AsyncIterator[_T]: """Wraps an async iterable calling an errback if an error is caught while - iterating it. Similar to scrapy.utils.defer.iter_errback() + iterating it. Similar to :func:`scrapy.utils.defer.iter_errback`. """ it = aiterable.__aiter__() while True: diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 52f29e22c..1e0c53212 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -18,10 +18,10 @@ def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]: "engine.scraper.is_idle()", "engine.spider.name", "engine.spider_is_idle()", - "engine.slot.closing", - "len(engine.slot.inprogress)", - "len(engine.slot.scheduler.dqs or [])", - "len(engine.slot.scheduler.mqs)", + "engine._slot.closing", + "len(engine._slot.inprogress)", + "len(engine._slot.scheduler.dqs or [])", + "len(engine._slot.scheduler.mqs)", "len(engine.scraper.slot.queue)", "len(engine.scraper.slot.active)", "engine.scraper.slot.active_size", diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 2e6869779..c859fbc2a 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -10,7 +10,7 @@ import re import sys import warnings import weakref -from collections.abc import AsyncIterable, Iterable, Mapping +from collections.abc import AsyncIterator, Iterable, Mapping from functools import partial, wraps from itertools import chain from typing import TYPE_CHECKING, Any, TypeVar, overload @@ -19,11 +19,12 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator if TYPE_CHECKING: - from collections.abc import AsyncIterator, Callable, Iterator + from collections.abc import Callable, Iterator from re import Pattern # typing.Concatenate and typing.ParamSpec require Python 3.10 - from typing_extensions import Concatenate, ParamSpec + # typing.Self requires Python 3.11 + from typing_extensions import Concatenate, ParamSpec, Self _P = ParamSpec("_P") @@ -369,25 +370,25 @@ class MutableChain(Iterable[_T]): async def _async_chain( - *iterables: Iterable[_T] | AsyncIterable[_T], + *iterables: Iterable[_T] | AsyncIterator[_T], ) -> AsyncIterator[_T]: for it in iterables: async for o in as_async_generator(it): yield o -class MutableAsyncChain(AsyncIterable[_T]): +class MutableAsyncChain(AsyncIterator[_T]): """ Similar to MutableChain but for async iterables """ - def __init__(self, *args: Iterable[_T] | AsyncIterable[_T]): + def __init__(self, *args: Iterable[_T] | AsyncIterator[_T]): self.data: AsyncIterator[_T] = _async_chain(*args) - def extend(self, *iterables: Iterable[_T] | AsyncIterable[_T]) -> None: + def extend(self, *iterables: Iterable[_T] | AsyncIterator[_T]) -> None: self.data = _async_chain(self.data, _async_chain(*iterables)) - def __aiter__(self) -> AsyncIterator[_T]: + def __aiter__(self) -> Self: return self async def __anext__(self) -> _T: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 099c81f0e..9c2754394 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Generic, TypeVar from warnings import catch_warnings, filterwarnings from twisted.internet import asyncioreactor, error +from twisted.internet.defer import Deferred from scrapy.utils.misc import load_object @@ -54,6 +55,7 @@ class CallLaterOnce(Generic[_T]): self._a: tuple[Any, ...] = a self._kw: dict[str, Any] = kw self._call: DelayedCall | None = None + self._deferreds: list[Deferred] = [] def schedule(self, delay: float = 0) -> None: from twisted.internet import reactor @@ -66,8 +68,23 @@ class CallLaterOnce(Generic[_T]): self._call.cancel() def __call__(self) -> _T: + from twisted.internet import reactor + self._call = None - return self._func(*self._a, **self._kw) + result = self._func(*self._a, **self._kw) + + for d in self._deferreds: + reactor.callLater(0, d.callback, None) + self._deferreds = [] + + return result + + async def wait(self): + from scrapy.utils.defer import maybe_deferred_to_future + + d = Deferred() + self._deferreds.append(d) + await maybe_deferred_to_future(d) def set_asyncio_event_loop_policy() -> None: @@ -114,8 +131,10 @@ def set_asyncio_event_loop(event_loop_path: str | None) -> AbstractEventLoop: """Sets and returns the event loop with specified import path.""" if event_loop_path is not None: event_loop_class: type[AbstractEventLoop] = load_object(event_loop_path) - event_loop = event_loop_class() - asyncio.set_event_loop(event_loop) + event_loop = _get_asyncio_event_loop() + if not isinstance(event_loop, event_loop_class): + event_loop = event_loop_class() + asyncio.set_event_loop(event_loop) else: try: with catch_warnings(): diff --git a/sep/sep-018.rst b/sep/sep-018.rst index e6d601fe1..29b1f860e 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -619,7 +619,7 @@ Resolved: ``manager.scraper.process_request()`` instead of ``manager.engine.crawl()`` - should we support adding additional start requests from a spider middleware? - - Yes - there is a spider middleware method (``start_requests``) for that + - Yes - there is a spider middleware method (``start_requests()``) for that - should ``process_response()`` receive a ``request`` argument with the ``request`` that originated it?. ``response.request`` is the latest request, not the original one (think of redirections), but it does carry the ``meta`` diff --git a/tests/CrawlerProcess/args_settings.py b/tests/CrawlerProcess/args_settings.py index a46a8806b..c8a3d0a5b 100644 --- a/tests/CrawlerProcess/args_settings.py +++ b/tests/CrawlerProcess/args_settings.py @@ -13,9 +13,10 @@ class NoRequestsSpider(scrapy.Spider): spider.settings.set("FOO", kwargs.get("foo")) return spider - def start_requests(self): + async def start(self): self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") - return [] + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/asyncio_custom_loop.py b/tests/CrawlerProcess/asyncio_custom_loop.py index 5e72aa6d4..bd78a0de7 100644 --- a/tests/CrawlerProcess/asyncio_custom_loop.py +++ b/tests/CrawlerProcess/asyncio_custom_loop.py @@ -5,8 +5,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py index 6f82cf589..6bb6fb3c6 100644 --- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py @@ -12,8 +12,9 @@ class ReactorCheckExtension: class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index a8bf1bc3c..f3dab12fe 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -38,8 +38,9 @@ class ReactorCheckExtension: class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index e9d6d8875..d8c467f40 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -15,8 +15,9 @@ from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index c72a0a17c..e7d3ca9cc 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -16,8 +16,9 @@ from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py index 7b0497bde..53d427061 100644 --- a/tests/CrawlerProcess/caching_hostname_resolver.py +++ b/tests/CrawlerProcess/caching_hostname_resolver.py @@ -11,7 +11,7 @@ class CachingHostnameResolverSpider(scrapy.Spider): name = "caching_hostname_resolver_spider" - def start_requests(self): + async def start(self): yield scrapy.Request(self.url) def parse(self, response): diff --git a/tests/CrawlerProcess/multi.py b/tests/CrawlerProcess/multi.py index 9f7eaf2ae..0058896b5 100644 --- a/tests/CrawlerProcess/multi.py +++ b/tests/CrawlerProcess/multi.py @@ -5,8 +5,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index e2933338b..8f59c035c 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -8,8 +8,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index eee808c32..9901dd634 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -8,8 +8,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index b61e52625..53941568a 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -10,8 +10,9 @@ selectreactor.install() class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index 38ca4c4f1..5739d77ae 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -17,8 +17,9 @@ installReactor(reactor) class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py index b397608ec..c488f7526 100644 --- a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py @@ -9,8 +9,9 @@ selectreactor.install() class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py index 2d8769501..9e4ad70d9 100644 --- a/tests/CrawlerProcess/simple.py +++ b/tests/CrawlerProcess/simple.py @@ -5,8 +5,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py index de76e13e8..6c0102241 100644 --- a/tests/CrawlerRunner/change_reactor.py +++ b/tests/CrawlerRunner/change_reactor.py @@ -10,8 +10,9 @@ class NoRequestsSpider(Spider): "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", } - def start_requests(self): - return [] + async def start(self): + return + yield configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"}) diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index b1b297777..5e2184afb 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -32,7 +32,7 @@ def createResolver(servers=None, resolvconf=None, hosts=None): class LocalhostSpider(Spider): name = "localhost_spider" - def start_requests(self): + async def start(self): yield Request(self.url) def parse(self, response): diff --git a/tests/__init__.py b/tests/__init__.py index cd52ade58..ccfabb0da 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -8,6 +8,9 @@ import os import socket from pathlib import Path +from twisted import version as TWISTED_VERSION +from twisted.python.versions import Version + # ignore system-wide proxies for tests # which would send requests to a totally unsuspecting server # (e.g. because urllib does not fully understand the proxy spec) @@ -30,3 +33,6 @@ except socket.gaierror: def get_testdata(*paths: str) -> bytes: """Return test data""" return Path(tests_datadir, *paths).read_bytes() + + +TWISTED_KEEPS_TRACEBACKS = TWISTED_VERSION >= Version("twisted", 24, 10, 0) diff --git a/tests/spiders.py b/tests/spiders.py index da923de6e..c47f2bd2b 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -68,7 +68,7 @@ class DelaySpider(MetaSpider): self.b = b self.t1 = self.t2 = self.t2_err = 0 - def start_requests(self): + async def start(self): self.t1 = time.time() url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") yield Request(url, callback=self.parse, errback=self.errback) @@ -105,7 +105,7 @@ class LogSpider(MetaSpider): class SlowSpider(DelaySpider): name = "slow" - def start_requests(self): + async def start(self): # 1st response is fast url = self.mockserver.url("/delay?n=0&b=0") yield Request(url, callback=self.parse, errback=self.errback) @@ -255,7 +255,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): callback=cb, ) - def start_requests(self): + async def start(self): for i in range(1, self.initial_reqs + 1): yield self._get_req(i) @@ -319,7 +319,7 @@ class ErrorSpider(FollowAllSpider): self.raise_exception() -class BrokenStartRequestsSpider(FollowAllSpider): +class BrokenStartSpider(FollowAllSpider): fail_before_yield = False fail_yielding = False @@ -327,7 +327,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): super().__init__(*a, **kw) self.seedsseen = [] - def start_requests(self): + async def start(self): if self.fail_before_yield: 1 / 0 @@ -338,22 +338,20 @@ class BrokenStartRequestsSpider(FollowAllSpider): if self.fail_yielding: 2 / 0 - assert self.seedsseen, ( - "All start requests consumed before any download happened" - ) + assert self.seedsseen, "All seeds consumed before any download happened" def parse(self, response): self.seedsseen.append(response.meta.get("seed")) yield from super().parse(response) -class StartRequestsItemSpider(FollowAllSpider): - def start_requests(self): +class StartItemSpider(FollowAllSpider): + async def start(self): yield {"name": "test item"} -class StartRequestsGoodAndBadOutput(FollowAllSpider): - def start_requests(self): +class StartGoodAndBadOutput(FollowAllSpider): + async def start(self): yield {"a": "a"} yield Request("data:,a") yield "data:,b" @@ -365,7 +363,7 @@ class SingleRequestSpider(MetaSpider): callback_func = None errback_func = None - def start_requests(self): + async def start(self): if isinstance(self.seed, Request): yield self.seed.replace(callback=self.parse, errback=self.on_error) else: @@ -386,13 +384,13 @@ class SingleRequestSpider(MetaSpider): return None -class DuplicateStartRequestsSpider(MockServerSpider): +class DuplicateStartSpider(MockServerSpider): dont_filter = True name = "duplicatestartrequests" distinct_urls = 2 dupe_factor = 3 - def start_requests(self): + async def start(self): for i in range(self.distinct_urls): for j in range(self.dupe_factor): url = self.mockserver.url(f"/echo?headers=1&body=test{i}") @@ -417,7 +415,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): } rules = (Rule(LinkExtractor(), callback="parse", follow=True),) - def start_requests(self): + async def start(self): test_body = b""" Page title<title></head> @@ -471,7 +469,7 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = "crawl_spider_with_errback" rules = (Rule(LinkExtractor(), callback="parse", errback="errback", follow=True),) - def start_requests(self): + async def start(self): test_body = b""" <html> <head><title>Page title<title></head> @@ -516,7 +514,7 @@ class BytesReceivedCallbackSpider(MetaSpider): crawler.signals.connect(spider.bytes_received, signals.bytes_received) return spider - def start_requests(self): + async def start(self): body = b"a" * self.full_response_length url = self.mockserver.url("/alpayload") yield Request(url, method="POST", body=body, errback=self.errback) @@ -545,7 +543,7 @@ class HeadersReceivedCallbackSpider(MetaSpider): crawler.signals.connect(spider.headers_received, signals.headers_received) return spider - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status"), errback=self.errback) def parse(self, response): diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py index 5228f6abd..5006e3689 100644 --- a/tests/test_cmdline_crawl_with_pipeline/__init__.py +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -2,17 +2,26 @@ import sys from pathlib import Path from subprocess import PIPE, Popen +from .. import TWISTED_KEEPS_TRACEBACKS + class TestCmdlineCrawlPipeline: def _execute(self, spname): args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname) cwd = Path(__file__).resolve().parent proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd) - proc.communicate() - return proc.returncode + _, stderr = proc.communicate() + return proc.returncode, stderr def test_open_spider_normally_in_pipeline(self): - assert self._execute("normal") == 0 + returncode, stderr = self._execute("normal") + assert returncode == 0 def test_exception_at_open_spider_in_pipeline(self): - assert self._execute("exception") == 1 + returncode, stderr = self._execute("exception") + # An unhandled exception in a pipeline should not stop the crawl + assert returncode == 0 + if TWISTED_KEEPS_TRACEBACKS: + assert b'RuntimeError("exception")' in stderr + else: + assert b"RuntimeError: exception" in stderr diff --git a/tests/test_commands.py b/tests/test_commands.py index f63e05628..16af97842 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -670,9 +670,10 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug("It Works!") - return [] + return + yield """ badspider = """ @@ -680,8 +681,9 @@ import scrapy class BadSpider(scrapy.Spider): name = "bad" - def start_requests(self): + async def start(self): raise Exception("oops!") + yield """ @contextmanager @@ -771,10 +773,10 @@ class MySpider(scrapy.Spider): log = self.get_log("", name="myspider.txt") assert "Unable to load" in log - def test_start_requests_errors(self): + def test_start_errors(self): log = self.get_log(self.badspider, name="badspider.py") - assert "start_requests" in log - assert "badspider.py" in log + assert "start" in log + assert "badspider.py" in log, log def test_asyncio_enabled_true(self): log = self.get_log( @@ -846,9 +848,10 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return [] + return + yield """ args = ["-o", "example.json"] log = self.get_log(spider_code, args=args) @@ -862,13 +865,14 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug( 'FEEDS: {}'.format( json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) ) ) - return [] + return + yield """ Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ["-O", "example.json"] @@ -888,8 +892,9 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): - return [] + async def start(self): + return + yield """ args = ["-o", "example1.json", "-O", "example2.json"] log = self.get_log(spider_code, args=args) @@ -904,9 +909,10 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return [] + return + yield """ args = ["-o", "-:json"] log = self.get_log(spider_code, args=args) @@ -983,9 +989,10 @@ class MySpider(scrapy.Spider): spider.settings.set("FOO", kwargs.get("foo")) return spider - def start_requests(self): + async def start(self): self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") - return [] + return + yield """ args = ["-a", "foo=42"] log = self.get_log(spider_code, args=args) @@ -1001,9 +1008,9 @@ class TestWindowsRunSpiderCommand(TestRunSpiderCommand): raise unittest.SkipTest("Windows required for .pyw files") return super().setUp() - def test_start_requests_errors(self): + def test_start_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") - assert "start_requests" in log + assert "start" in log assert "badspider.pyw" in log def test_runspider_unable_to_load(self): @@ -1053,9 +1060,10 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug('It works!') - return [] + return + yield """ log = self.get_log(spider_code) assert "[myspider] DEBUG: It works!" in log @@ -1067,9 +1075,10 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return [] + return + yield """ args = ["-o", "example.json"] log = self.get_log(spider_code, args=args) @@ -1083,13 +1092,14 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug( 'FEEDS: {}'.format( json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) ) ) - return [] + return + yield """ Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ["-O", "example.json"] @@ -1109,8 +1119,9 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): - return [] + async def start(self): + return + yield """ args = ["-o", "example1.json", "-O", "example2.json"] log = self.get_log(spider_code, args=args) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index fb961ace2..26b16a1d4 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -511,8 +511,9 @@ class TestContractsManager(unittest.TestCase): super().__init__(*args, **kwargs) self.visited = 0 - def start_requests(self_): # pylint: disable=no-self-argument - return self.conman.from_spider(self_, self.results) + async def start(self_): # pylint: disable=no-self-argument + for item_or_request in self.conman.from_spider(self_, self.results): + yield item_or_request def parse_first(self, response): self.visited += 1 diff --git a/tests/test_crawl.py b/tests/test_crawl.py index b7a8a9628..b90706027 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -36,7 +36,7 @@ from tests.spiders import ( AsyncDefDeferredMaybeWrappedSpider, AsyncDefDeferredWrappedSpider, AsyncDefSpider, - BrokenStartRequestsSpider, + BrokenStartSpider, BytesReceivedCallbackSpider, BytesReceivedErrbackSpider, CrawlSpiderWithAsyncCallback, @@ -45,14 +45,14 @@ from tests.spiders import ( CrawlSpiderWithParseMethod, CrawlSpiderWithProcessRequestCallbackKeywordArguments, DelaySpider, - DuplicateStartRequestsSpider, + DuplicateStartSpider, FollowAllSpider, HeadersReceivedCallbackSpider, HeadersReceivedErrbackSpider, SimpleSpider, SingleRequestSpider, - StartRequestsGoodAndBadOutput, - StartRequestsItemSpider, + StartGoodAndBadOutput, + StartItemSpider, ) @@ -165,9 +165,9 @@ class TestCrawl(TestCase): self._assert_retried(log) @defer.inlineCallbacks - def test_start_requests_bug_before_yield(self): + def test_start_bug_before_yield(self): with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(BrokenStartRequestsSpider) + crawler = get_crawler(BrokenStartSpider) yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) assert len(log.records) == 1 @@ -176,9 +176,9 @@ class TestCrawl(TestCase): assert record.exc_info[0] is ZeroDivisionError @defer.inlineCallbacks - def test_start_requests_bug_yielding(self): + def test_start_bug_yielding(self): with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(BrokenStartRequestsSpider) + crawler = get_crawler(BrokenStartSpider) yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) assert len(log.records) == 1 @@ -187,14 +187,14 @@ class TestCrawl(TestCase): assert record.exc_info[0] is ZeroDivisionError @defer.inlineCallbacks - def test_start_requests_items(self): + def test_start_items(self): items = [] def _on_item_scraped(item): items.append(item) with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(StartRequestsItemSpider) + crawler = get_crawler(StartItemSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) @@ -202,11 +202,11 @@ class TestCrawl(TestCase): assert items == [{"name": "test item"}] @defer.inlineCallbacks - def test_start_requests_unsupported_output(self): + def test_start_unsupported_output(self): """Anything that is not a request is assumed to be an item, avoiding a - potentially expensive call to itemadapter.is_item, and letting instead - things fail when ItemAdapter is actually used on the corresponding - non-item object.""" + potentially expensive call to itemadapter.is_item(), and letting + instead things fail when ItemAdapter is actually used on the + corresponding non-item object.""" items = [] @@ -214,7 +214,7 @@ class TestCrawl(TestCase): items.append(item) with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(StartRequestsGoodAndBadOutput) + crawler = get_crawler(StartGoodAndBadOutput) crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) @@ -223,24 +223,15 @@ class TestCrawl(TestCase): assert not any(isinstance(item, Request) for item in items) @defer.inlineCallbacks - def test_start_requests_laziness(self): + def test_start_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} - crawler = get_crawler(BrokenStartRequestsSpider, settings) - yield crawler.crawl(mockserver=self.mockserver) - assert crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index( - 99 - ), crawler.spider.seedsseen - - @defer.inlineCallbacks - def test_start_requests_dupes(self): - settings = {"CONCURRENT_REQUESTS": 1} - crawler = get_crawler(DuplicateStartRequestsSpider, settings) + crawler = get_crawler(DuplicateStartSpider, settings) yield crawler.crawl( dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver ) assert crawler.spider.visited == 6 - crawler = get_crawler(DuplicateStartRequestsSpider, settings) + crawler = get_crawler(DuplicateStartSpider, settings) yield crawler.crawl( dont_filter=False, distinct_urls=3, @@ -322,10 +313,10 @@ with multiples lines # basic asserts in case of weird communication errors assert "responses" in crawler.spider.meta assert "failures" not in crawler.spider.meta - # start requests doesn't set Referer header + # start() doesn't set Referer header echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) assert "Referer" not in echo0["headers"] - # following request sets Referer to start request url + # following request sets Referer to the source request url echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body)) assert echo1["headers"].get("Referer") == [req0.url] # next request avoids Referer header @@ -378,27 +369,6 @@ with multiples lines assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == "1" - @defer.inlineCallbacks - def test_graceful_crawl_error_handling(self): - """ - Test whether errors happening anywhere in Crawler.crawl() are properly - reported (and not somehow swallowed) after a graceful engine shutdown. - The errors should not come from within Scrapy's core but from within - spiders/middlewares/etc., e.g. raised in Spider.start_requests(), - SpiderMiddleware.process_start_requests(), etc. - """ - - class TestError(Exception): - pass - - class FaultySpider(SimpleSpider): - def start_requests(self): - raise TestError - - crawler = get_crawler(FaultySpider) - yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError) - assert not crawler.crawling - @defer.inlineCallbacks def test_open_spider_error_on_faulty_pipeline(self): settings = { diff --git a/tests/test_crawler.py b/tests/test_crawler.py index efb346dde..7a3d562e5 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -153,7 +153,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def start(self): MySpider.result = crawler.get_downloader_middleware(MySpider.cls) return yield @@ -233,7 +233,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def start(self): MySpider.result = crawler.get_extension(MySpider.cls) return yield @@ -313,7 +313,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def start(self): MySpider.result = crawler.get_item_pipeline(MySpider.cls) return yield @@ -393,7 +393,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def start(self): MySpider.result = crawler.get_spider_middleware(MySpider.cls) return yield @@ -580,8 +580,9 @@ class ExceptionSpider(scrapy.Spider): class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield @pytest.mark.usefixtures("reactor_pytest") diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 408160ccb..8ae160f8a 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -25,7 +25,7 @@ class TestManagerBase(TestCase): self.spider = self.crawler._create_spider("foo") self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler) self.crawler.engine = self.crawler._create_engine() - return self.crawler.engine.open_spider(self.spider, start_requests=()) + return self.crawler.engine.open_spider(self.spider) def tearDown(self): return self.crawler.engine.close_spider(self.spider) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 4fca9eefb..78c83ea83 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -28,7 +28,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): }, } - def start_requests(self): + async def start(self): self.times = {None: []} slots = [*self.custom_settings.get("DOWNLOAD_SLOTS", {}), None] diff --git a/tests/test_engine.py b/tests/test_engine.py index 8928e4daf..b60b510b2 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -29,7 +29,7 @@ from twisted.trial import unittest from twisted.web import server, static, util from scrapy import signals -from scrapy.core.engine import ExecutionEngine, Slot +from scrapy.core.engine import ExecutionEngine, _Slot from scrapy.core.scheduler import BaseScheduler from scrapy.exceptions import CloseSpider, IgnoreRequest from scrapy.http import Request @@ -92,8 +92,9 @@ class MySpider(Spider): class DupeFilterSpider(MySpider): - def start_requests(self): - return (Request(url) for url in self.start_urls) # no dont_filter=True + async def start(self): + for url in self.start_urls: + yield Request(url) # no dont_filter=True class DictItemsSpider(MySpider): @@ -149,7 +150,6 @@ class CrawlerRun: """A class to run the crawler and keep track of events occurred""" def __init__(self, spider_class): - self.spider = None self.respplug = [] self.reqplug = [] self.reqdropped = [] @@ -190,7 +190,6 @@ class CrawlerRun: self.response_downloaded, signals.response_downloaded ) self.crawler.crawl(start_urls=start_urls) - self.spider = self.crawler.spider self.deferred = defer.Deferred() dispatcher.connect(self.stop, signals.engine_stopped) @@ -296,7 +295,7 @@ class TestEngineBase(unittest.TestCase): assert len(run.itemerror) == 2 for item, response, spider, failure in run.itemerror: assert failure.value.__class__ is ZeroDivisionError - assert spider == run.spider + assert spider == run.crawler.spider assert item["url"] == response.url if "item1.html" in item["url"]: @@ -377,11 +376,14 @@ class TestEngineBase(unittest.TestCase): assert signals.spider_closed in run.signals_caught assert signals.headers_received in run.signals_caught - assert {"spider": run.spider} == run.signals_caught[signals.spider_opened] - assert {"spider": run.spider} == run.signals_caught[signals.spider_idle] - assert {"spider": run.spider, "reason": "finished"} == run.signals_caught[ - signals.spider_closed + assert {"spider": run.crawler.spider} == run.signals_caught[ + signals.spider_opened ] + assert {"spider": run.crawler.spider} == run.signals_caught[signals.spider_idle] + assert { + "spider": run.crawler.spider, + "reason": "finished", + } == run.signals_caught[signals.spider_closed] class TestEngine(TestEngineBase): @@ -419,9 +421,10 @@ class TestEngine(TestEngineBase): def test_crawler_change_close_reason_on_idle(self): run = CrawlerRun(ChangeCloseReasonSpider) yield run.run() - assert {"spider": run.spider, "reason": "custom_reason"} == run.signals_caught[ - signals.spider_closed - ] + assert { + "spider": run.crawler.spider, + "reason": "custom_reason", + } == run.signals_caught[signals.spider_closed] @defer.inlineCallbacks def test_close_downloader(self): @@ -471,7 +474,7 @@ class TestEngine(TestEngineBase): finally: timer.cancel() - assert b"Traceback" not in stderr + assert b"Traceback" not in stderr, stderr def test_request_scheduled_signal(caplog): @@ -491,7 +494,13 @@ def test_request_scheduled_signal(caplog): engine = ExecutionEngine(crawler, lambda _: None) engine.downloader._slot_gc_loop.stop() scheduler = TestScheduler() - engine.slot = Slot((), None, Mock(), scheduler) + + async def start(): + return + yield + + engine._start = start() + engine._slot = _Slot(False, Mock(), scheduler) crawler.signals.connect(signal_handler, request_scheduled) keep_request = Request("https://keep.example") engine._schedule_request(keep_request) diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py new file mode 100644 index 000000000..90af10f0e --- /dev/null +++ b/tests/test_engine_loop.py @@ -0,0 +1,364 @@ +from __future__ import annotations + +from collections import deque +from logging import ERROR +from typing import TYPE_CHECKING + +from testfixtures import LogCapture +from twisted.internet.defer import Deferred +from twisted.trial.unittest import TestCase + +from scrapy import Request, Spider, signals +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.test import get_crawler + +from .mockserver import MockServer +from .test_scheduler import MemoryScheduler + +if TYPE_CHECKING: + from scrapy.http import Response + + +async def sleep(seconds: float = 0.001) -> None: + from twisted.internet import reactor + + deferred: Deferred[None] = Deferred() + reactor.callLater(seconds, deferred.callback, None) + await maybe_deferred_to_future(deferred) + + +class MainTestCase(TestCase): + @deferred_f_from_coro_f + async def test_sleep(self): + """Neither asynchronous sleeps on Spider.start() nor the equivalent on + the scheduler (returning no requests while also returning True from + the has_pending_requests() method) should cause the spider to miss the + processing of any later requests.""" + seconds = 2 + + class TestSpider(Spider): + name = "test" + + async def start(self): + from twisted.internet import reactor + + yield Request("data:,a") + + await sleep(seconds) + + self.crawler.engine._slot.scheduler.pause() + self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b")) + + # During this time, the scheduler reports having requests but + # returns None. + await sleep(seconds) + + self.crawler.engine._slot.scheduler.unpause() + + # The scheduler request is processed. + await sleep(seconds) + + yield Request("data:,c") + + await sleep(seconds) + + self.crawler.engine._slot.scheduler.pause() + self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d")) + + # The last start request is processed during the time until the + # delayed call below, proving that the start iteration can + # finish before a scheduler “sleep” without causing the + # scheduler to finish. + reactor.callLater(seconds, self.crawler.engine._slot.scheduler.unpause) + + def parse(self, response): + pass + + actual_urls = [] + + def track_url(request, spider): + actual_urls.append(request.url) + + settings = {"SCHEDULER": MemoryScheduler} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_url, signals.request_reached_downloader) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + expected_urls = ["data:,a", "data:,b", "data:,c", "data:,d"] + assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" + + @deferred_f_from_coro_f + async def test_close_during_start_iteration(self): + class TestSpider(Spider): + name = "test" + + async def start(self): + assert self.crawler.engine is not None + await maybe_deferred_to_future(self.crawler.engine.close()) + yield Request("data:,a") + + def parse(self, response): + pass + + actual_urls = [] + + def track_url(request, spider): + actual_urls.append(request.url) + + settings = {"SCHEDULER": MemoryScheduler} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_url, signals.request_reached_downloader) + + with LogCapture(level=ERROR) as log: + await maybe_deferred_to_future(crawler.crawl()) + + assert not log.records, f"{log.records=}" + finish_reason = crawler.stats.get_value("finish_reason") + assert finish_reason == "shutdown", f"{finish_reason=}" + expected_urls = [] + assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" + + +class RequestSendOrderTestCase(TestCase): + seconds = 0.1 # increase if flaky + + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) # increase if flaky + + def request(self, num, response_seconds, download_slots, priority=0): + url = self.mockserver.url(f"/delay?n={response_seconds}&{num}") + meta = {"download_slot": str(num % download_slots)} + return Request(url, meta=meta, priority=priority) + + def get_num(self, request_or_response: Request | Response): + return int(request_or_response.url.rsplit("&", maxsplit=1)[1]) + + @deferred_f_from_coro_f + async def _test_request_order( + self, + start_nums, + cb_nums=None, + settings=None, + response_seconds=None, + download_slots=1, + start_fn=None, + parse_fn=None, + ): + cb_nums = cb_nums or [] + settings = settings or {} + response_seconds = response_seconds or self.seconds + + cb_requests = deque( + [self.request(num, response_seconds, download_slots) for num in cb_nums] + ) + + if start_fn is None: + + async def start_fn(spider): + for num in start_nums: + yield self.request(num, response_seconds, download_slots) + + if parse_fn is None: + + def parse_fn(spider, response): + while cb_requests: + yield cb_requests.popleft() + + class TestSpider(Spider): + name = "test" + start = start_fn + parse = parse_fn + + actual_nums = [] + + def track_num(request, spider): + actual_nums.append(self.get_num(request)) + + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_num, signals.request_reached_downloader) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + expected_nums = sorted(start_nums + cb_nums) + assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}" + + @deferred_f_from_coro_f + async def test_default(self): + """By default, start requests take priority over callback requests and + are sent in order. Priority matters, but given the same priority, a + start request takes precedence.""" + nums = [1, 2, 3, 4, 5, 6] + response_seconds = 0 + download_slots = 1 + + def _request(num, priority=0): + return self.request( + num, response_seconds, download_slots, priority=priority + ) + + async def start(spider): + # The first CONCURRENT_REQUESTS start requests are sent + # immediately. + yield _request(1) + + for request in ( + _request(4, priority=1), + _request(6), + ): + spider.crawler.engine._slot.scheduler.enqueue_request(request) + yield _request(5) + yield _request(2, priority=1) + yield _request(3, priority=1) + + def parse(spider, response): + return + yield + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=nums, + settings={"CONCURRENT_REQUESTS": 1}, + response_seconds=response_seconds, + start_fn=start, + parse_fn=parse, + ) + ) + + @deferred_f_from_coro_f + async def test_lifo_start(self): + """Changing the queues of start requests to LIFO, matching the queues + of non-start requests, does not cause all requests to be stored in the + same queue objects, it only affects the order of start requests.""" + nums = [1, 2, 3, 4, 5, 6] + response_seconds = 0 + download_slots = 1 + + def _request(num, priority=0): + return self.request( + num, response_seconds, download_slots, priority=priority + ) + + async def start(spider): + # The first CONCURRENT_REQUESTS start requests are sent + # immediately. + yield _request(1) + + for request in ( + _request(4, priority=1), + _request(6), + ): + spider.crawler.engine._slot.scheduler.enqueue_request(request) + yield _request(5) + yield _request(3, priority=1) + yield _request(2, priority=1) + + def parse(spider, response): + return + yield + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=nums, + settings={ + "CONCURRENT_REQUESTS": 1, + "SCHEDULER_START_MEMORY_QUEUE": "scrapy.squeues.LifoMemoryQueue", + }, + response_seconds=response_seconds, + start_fn=start, + parse_fn=parse, + ) + ) + + @deferred_f_from_coro_f + async def test_shared_queues(self): + """If SCHEDULER_START_*_QUEUE is falsy, start requests and other + requests share the same queue, i.e. start requests are not priorized + over other requests if their priority matches.""" + nums = list(range(1, 14)) + response_seconds = 0 + download_slots = 1 + + def _request(num, priority=0): + return self.request( + num, response_seconds, download_slots, priority=priority + ) + + async def start(spider): + # The first CONCURRENT_REQUESTS start requests are sent + # immediately. + yield _request(1) + + # Below, priority 1 requests are sent first, and requests are sent + # in LIFO order. + + for request in ( + _request(7, priority=1), + _request(6, priority=1), + _request(13), + _request(12), + ): + spider.crawler.engine._slot.scheduler.enqueue_request(request) + + yield _request(11) + yield _request(10) + yield _request(5, priority=1) + yield _request(4, priority=1) + + for request in ( + _request(3, priority=1), + _request(2, priority=1), + _request(9), + _request(8), + ): + spider.crawler.engine._slot.scheduler.enqueue_request(request) + + def parse(spider, response): + return + yield + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=nums, + settings={ + "CONCURRENT_REQUESTS": 1, + "SCHEDULER_START_MEMORY_QUEUE": None, + }, + response_seconds=response_seconds, + start_fn=start, + parse_fn=parse, + ) + ) + + # Examples from the “Start requests” section of the documentation about + # spiders. + + @deferred_f_from_coro_f + async def test_lazy(self): + start_nums = [1, 2, 4] + cb_nums = [3] + response_seconds = self.seconds * 2**1 # increase if flaky + download_slots = 1 + + async def start(spider): + for num in start_nums: + if spider.crawler.engine.needs_backout(): + await spider.crawler.signals.wait_for(signals.scheduler_empty) + request = self.request(num, response_seconds, download_slots) + yield request + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=start_nums, + cb_nums=cb_nums, + settings={ + "CONCURRENT_REQUESTS": 1, + }, + response_seconds=response_seconds, + start_fn=start, + ) + ) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 743d9774b..d658d1526 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -69,7 +69,7 @@ class AsyncDefNotAsyncioPipeline: class ItemSpider(Spider): name = "itemspider" - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index ab6baa5f0..79b53b33b 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -28,10 +28,10 @@ class InjectArgumentsSpiderMiddleware: Make sure spider middlewares are able to update the keyword arguments """ - def process_start_requests(self, start_requests, spider): - for request in start_requests: + async def process_start(self, start): + async for request in start: if request.callback.__name__ == "parse_spider_mw": - request.cb_kwargs["from_process_start_requests"] = True + request.cb_kwargs["from_process_start"] = True yield request def process_spider_input(self, response, spider): @@ -62,7 +62,7 @@ class KeywordArgumentsSpider(MockServerSpider): checks: list[bool] = [] - def start_requests(self): + async def start(self): data = {"key": "value", "number": 123, "callback": "some_callback"} yield Request(self.mockserver.url("/first"), self.parse_first, cb_kwargs=data) yield Request( @@ -138,11 +138,9 @@ class KeywordArgumentsSpider(MockServerSpider): self.checks.append(bool(from_process_response)) self.crawler.stats.inc_value("boolean_checks", 2) - def parse_spider_mw( - self, response, from_process_spider_input, from_process_start_requests - ): + def parse_spider_mw(self, response, from_process_spider_input, from_process_start): self.checks.append(bool(from_process_spider_input)) - self.checks.append(bool(from_process_start_requests)) + self.checks.append(bool(from_process_start)) self.crawler.stats.inc_value("boolean_checks", 2) return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 1d6992a32..f90293dd3 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -3,6 +3,7 @@ from __future__ import annotations import shutil import tempfile from abc import ABC, abstractmethod +from collections import deque from typing import Any, NamedTuple import pytest @@ -10,7 +11,7 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase from scrapy.core.downloader import Downloader -from scrapy.core.scheduler import Scheduler +from scrapy.core.scheduler import BaseScheduler, Scheduler from scrapy.crawler import Crawler from scrapy.http import Request from scrapy.spiders import Spider @@ -20,6 +21,38 @@ from scrapy.utils.test import get_crawler from tests.mockserver import MockServer +class MemoryScheduler(BaseScheduler): + paused = False + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.queue = deque( + Request(value) if isinstance(value, str) else value + for value in getattr(self, "queue", []) + ) + + def enqueue_request(self, request: Request) -> bool: + self.queue.append(request) + return True + + def has_pending_requests(self) -> bool: + return self.paused or bool(self.queue) + + def next_request(self) -> Request | None: + if self.paused: + return None + try: + return self.queue.pop() + except IndexError: + return None + + def pause(self) -> None: + self.paused = True + + def unpause(self) -> None: + self.paused = False + + class MockEngine(NamedTuple): downloader: MockDownloader diff --git a/tests/test_signals.py b/tests/test_signals.py index f5075fb60..663e912b7 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -1,8 +1,9 @@ import pytest from twisted.internet import defer -from twisted.trial import unittest +from twisted.trial.unittest import TestCase from scrapy import Request, Spider, signals +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler, get_from_asyncio_queue from tests.mockserver import MockServer @@ -10,7 +11,7 @@ from tests.mockserver import MockServer class ItemSpider(Spider): name = "itemspider" - def start_requests(self): + async def start(self): for index in range(10): yield Request( self.mockserver.url(f"/status?n=200&id={index}"), meta={"index": index} @@ -20,7 +21,21 @@ class ItemSpider(Spider): return {"index": response.meta["index"]} -class TestAsyncSignal(unittest.TestCase): +class MainTestCase(TestCase): + @deferred_f_from_coro_f + async def test_scheduler_empty(self): + crawler = get_crawler() + calls = [] + + def track_call(): + calls.append(object()) + + crawler.signals.connect(track_call, signals.scheduler_empty) + await maybe_deferred_to_future(crawler.crawl()) + assert len(calls) >= 1 + + +class MockServerTestCase(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() diff --git a/tests/test_spider.py b/tests/test_spider.py index aaf72390d..b4aa649a3 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,8 +1,7 @@ import gzip -import inspect import warnings from io import BytesIO -from logging import WARNING +from logging import ERROR, WARNING from pathlib import Path from typing import Any from unittest import mock @@ -27,6 +26,7 @@ from scrapy.spiders import ( XMLFeedSpider, ) from scrapy.spiders.init import InitSpider +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler, get_reactor_settings from tests import get_testdata, tests_datadir @@ -45,12 +45,6 @@ class TestSpider(unittest.TestCase): assert spider.name == "example.com" assert spider.start_urls == [] # pylint: disable=use-implicit-booleaness-not-comparison - def test_start_requests(self): - spider = self.spider_class("example.com") - start_requests = spider.start_requests() - assert inspect.isgenerator(start_requests) - assert not list(start_requests) - def test_spider_args(self): """``__init__`` method arguments are assigned to spider attributes""" spider = self.spider_class("example.com", foo="bar") @@ -152,6 +146,22 @@ class TestSpider(unittest.TestCase): class TestInitSpider(TestSpider): spider_class = InitSpider + @deferred_f_from_coro_f + async def test_start_urls(self): + responses = [] + + class TestSpider(self.spider_class): + name = "test" + start_urls = ["data:,"] + + async def parse(self, response): + responses.append(response) + + crawler = get_crawler(TestSpider) + await maybe_deferred_to_future(crawler.crawl()) + assert len(responses) == 1 + assert responses[0].url == "data:," + class TestXMLFeedSpider(TestSpider): spider_class = XMLFeedSpider @@ -454,12 +464,17 @@ class TestCrawlSpider(TestSpider): assert hasattr(spider, "_follow_links") assert not spider._follow_links + @inlineCallbacks def test_start_url(self): - spider = self.spider_class("example.com") - spider.start_url = "https://www.example.com" + class TestSpider(self.spider_class): + name = "test" + start_url = "https://www.example.com" - with pytest.raises(AttributeError, match=r"^Crawling could not start.*$"): - list(spider.start_requests()) + crawler = get_crawler(TestSpider) + with LogCapture("scrapy.core.engine", propagate=False, level=ERROR) as log: + yield crawler.crawl() + assert "Error while reading start items and requests" in str(log) + assert "did you miss an 's'?" in str(log) class TestSitemapSpider(TestSpider): @@ -776,6 +791,24 @@ Sitemap: /sitemap-relative-url.xml ), ) + @deferred_f_from_coro_f + async def test_sitemap_urls(self): + class TestSpider(self.spider_class): + name = "test" + sitemap_urls = ["https://toscrape.com/sitemap.xml"] + + crawler = get_crawler(TestSpider) + spider = TestSpider.from_crawler(crawler) + with warnings.catch_warnings(): + warnings.simplefilter("error") + requests = [request async for request in spider.start()] + + assert len(requests) == 1 + request = requests[0] + assert request.url == "https://toscrape.com/sitemap.xml" + assert request.dont_filter is False + assert request.callback == spider._parse_sitemap + class TestDeprecation: def test_crawl_spider(self): diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py new file mode 100644 index 000000000..1815aad76 --- /dev/null +++ b/tests/test_spider_start.py @@ -0,0 +1,186 @@ +import warnings +from asyncio import sleep + +import pytest +from testfixtures import LogCapture +from twisted.trial.unittest import TestCase + +from scrapy import Spider, signals +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.test import get_crawler + +from .utils import twisted_sleep + +SLEEP_SECONDS = 0.1 + +ITEM_A = {"id": "a"} +ITEM_B = {"id": "b"} + + +class MainTestCase(TestCase): + async def _test_spider(self, spider, expected_items=None): + actual_items = [] + expected_items = [] if expected_items is None else expected_items + + def track_item(item, response, spider): + actual_items.append(item) + + crawler = get_crawler(spider) + crawler.signals.connect(track_item, signals.item_scraped) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + assert actual_items == expected_items + + @deferred_f_from_coro_f + async def test_start_urls(self): + class TestSpider(Spider): + name = "test" + start_urls = ["data:,"] + + async def parse(self, response): + yield ITEM_A + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_start(self): + class TestSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_A + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_start_subclass(self): + class BaseSpider(Spider): + async def start(self): + yield ITEM_A + + class TestSpider(BaseSpider): + name = "test" + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_deprecated(self): + class TestSpider(Spider): + name = "test" + + def start_requests(self): + yield ITEM_A + + with pytest.warns(ScrapyDeprecationWarning): + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_deprecated_subclass(self): + class BaseSpider(Spider): + def start_requests(self): + yield ITEM_A + + class TestSpider(BaseSpider): + name = "test" + + # The warning must be about the base class and not the subclass. + with pytest.warns(ScrapyDeprecationWarning, match="BaseSpider"): + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_universal(self): + class TestSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_A + + def start_requests(self): + yield ITEM_B + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_universal_subclass(self): + class BaseSpider(Spider): + async def start(self): + yield ITEM_A + + def start_requests(self): + yield ITEM_B + + class TestSpider(BaseSpider): + name = "test" + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_start_deprecated_super(self): + class TestSpider(Spider): + name = "test" + + async def start(self): + for item_or_request in super().start_requests(): + yield item_or_request + + with pytest.warns( + ScrapyDeprecationWarning, match=r"use Spider\.start\(\) instead" + ) as messages: + await self._test_spider(TestSpider, []) + assert messages[0].filename.endswith("test_spider_start.py") + + async def _test_start(self, start_, expected_items=None): + class TestSpider(Spider): + name = "test" + start = start_ + + await self._test_spider(TestSpider, expected_items) + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_asyncio_delayed(self): + async def start(spider): + await sleep(SLEEP_SECONDS) + yield ITEM_A + + await self._test_start(start, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_twisted_delayed(self): + async def start(spider): + await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS)) + yield ITEM_A + + await self._test_start(start, [ITEM_A]) + + # Exceptions + + @deferred_f_from_coro_f + async def test_deprecated_non_generator_exception(self): + class TestSpider(Spider): + name = "test" + + def start_requests(self): + raise RuntimeError + + with ( + LogCapture() as log, + pytest.warns( + ScrapyDeprecationWarning, + match=r"defines the deprecated start_requests\(\) method", + ), + ): + await self._test_spider(TestSpider, []) + + assert "in start_requests\n raise RuntimeError" in str(log) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 1d671134e..db46be7dd 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import AsyncIterator, Iterable +from inspect import isasyncgen from typing import Any from unittest import mock @@ -111,7 +112,7 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): """Helpers for testing sync, async and mixed middlewares. - Should work for process_spider_output and, when it's supported, process_start_requests. + Should work for process_spider_output and, when it's supported, process_start. """ ITEM_TYPE: type | tuple @@ -200,7 +201,7 @@ class ProcessSpiderExceptionSimpleIterableMiddleware: yield {"foo": 3} -class ProcessSpiderExceptionAsyncIterableMiddleware: +class ProcessSpiderExceptionAsyncIteratorMiddleware: async def process_spider_exception(self, response, exception, spider): yield {"foo": 1} d = defer.Deferred() @@ -319,37 +320,43 @@ class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): ) -class ProcessStartRequestsSimpleMiddleware: - def process_start_requests(self, start_requests, spider): - yield from start_requests +class ProcessStartSimpleMiddleware: + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request -class TestProcessStartRequestsSimple(TestBaseAsyncSpiderMiddleware): - """process_start_requests tests for simple start_requests""" +class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware): + """process_start tests for simple start""" ITEM_TYPE = (Request, dict) - MW_SIMPLE = ProcessStartRequestsSimpleMiddleware + MW_SIMPLE = ProcessStartSimpleMiddleware - def _start_requests(self): - for i in range(2): - yield Request(f"https://example.com/{i}", dont_filter=True) - yield {"name": "test item"} + async def _get_processed_start(self, *mw_classes): + class TestSpider(Spider): + name = "test" - @defer.inlineCallbacks - def _get_middleware_result(self, *mw_classes, start_index: int | None = None): - setting = self._construct_mw_setting(*mw_classes, start_index=start_index) + async def start(self): + for i in range(2): + yield Request(f"https://example.com/{i}", dont_filter=True) + yield {"name": "test item"} + + setting = self._construct_mw_setting(*mw_classes) self.crawler = get_crawler( - Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} + TestSpider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} ) - self.spider = self.crawler._create_spider("foo") + self.spider = self.crawler._create_spider() self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - start_requests = iter(self._start_requests()) - results = yield self.mwman.process_start_requests(start_requests, self.spider) - return results + return await self.mwman.process_start(self.spider) - def test_simple(self): + @deferred_f_from_coro_f + async def test_simple(self): """Simple mw""" - return self._test_simple_base(self.MW_SIMPLE) + start = await self._get_processed_start(self.MW_SIMPLE) + assert isasyncgen(start) + start_list = await collect_asyncgen(start) + assert len(start_list) == self.RESULT_COUNT + assert isinstance(start_list[0], self.ITEM_TYPE) class UniversalMiddlewareNoSync: @@ -507,7 +514,7 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware - MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIterableMiddleware + MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIteratorMiddleware def _scrape_func(self, *args, **kwargs): 1 / 0 diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py index 46be879f3..77d055d50 100644 --- a/tests/test_spidermiddleware_base.py +++ b/tests/test_spidermiddleware_base.py @@ -27,16 +27,19 @@ def test_trivial(crawler): assert mw.crawler is crawler test_req = Request("data:,") spider_output = [test_req, {"foo": "bar"}] - processed = list( - mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) - ) - assert processed == [test_req, {"foo": "bar"}] + for processed in [ + list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ), + list(mw.process_start_requests(spider_output, crawler.spider)), + ]: + assert processed == [test_req, {"foo": "bar"}] def test_processed_request(crawler): class ProcessReqSpiderMiddleware(BaseSpiderMiddleware): def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: if request.url == "data:2,": return None @@ -49,20 +52,23 @@ def test_processed_request(crawler): test_req2 = Request("data:2,") test_req3 = Request("data:3,") spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3] - processed = list( - mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) - ) - assert len(processed) == 3 - assert isinstance(processed[0], Request) - assert processed[0].url == "data:1," - assert processed[1] == {"foo": "bar"} - assert isinstance(processed[2], Request) - assert processed[2].url == "data:30," + for processed in [ + list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ), + list(mw.process_start_requests(spider_output, crawler.spider)), + ]: + assert len(processed) == 3 + assert isinstance(processed[0], Request) + assert processed[0].url == "data:1," + assert processed[1] == {"foo": "bar"} + assert isinstance(processed[2], Request) + assert processed[2].url == "data:30," def test_processed_item(crawler): class ProcessItemSpiderMiddleware(BaseSpiderMiddleware): - def get_processed_item(self, item: Any, response: Response) -> Any: + def get_processed_item(self, item: Any, response: Response | None) -> Any: if item["foo"] == 2: return None if item["foo"] == 3: @@ -72,16 +78,19 @@ def test_processed_item(crawler): mw = ProcessItemSpiderMiddleware.from_crawler(crawler) test_req = Request("data:,") spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}] - processed = list( - mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) - ) - assert processed == [{"foo": 1}, test_req, {"foo": 30}] + for processed in [ + list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ), + list(mw.process_start_requests(spider_output, crawler.spider)), + ]: + assert processed == [{"foo": 1}, test_req, {"foo": 30}] def test_processed_both(crawler): class ProcessBothSpiderMiddleware(BaseSpiderMiddleware): def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: if request.url == "data:2,": return None @@ -89,7 +98,7 @@ def test_processed_both(crawler): return Request("data:30,") return request - def get_processed_item(self, item: Any, response: Response) -> Any: + def get_processed_item(self, item: Any, response: Response | None) -> Any: if item["foo"] == 2: return None if item["foo"] == 3: @@ -108,13 +117,16 @@ def test_processed_both(crawler): {"foo": 3}, test_req3, ] - processed = list( - mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) - ) - assert len(processed) == 4 - assert isinstance(processed[0], Request) - assert processed[0].url == "data:1," - assert processed[1] == {"foo": 1} - assert processed[2] == {"foo": 30} - assert isinstance(processed[3], Request) - assert processed[3].url == "data:30," + for processed in [ + list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ), + list(mw.process_start_requests(spider_output, crawler.spider)), + ]: + assert len(processed) == 4 + assert isinstance(processed[0], Request) + assert processed[0].url == "data:1," + assert processed[1] == {"foo": 1} + assert processed[2] == {"foo": 30} + assert isinstance(processed[3], Request) + assert processed[3].url == "data:30," diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index e306579fa..fd2fc3581 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -30,7 +30,7 @@ class _HttpErrorSpider(MockServerSpider): self.skipped = set() self.parsed = set() - def start_requests(self): + async def start(self): for url in self.start_urls: yield Request(url, self.parse, errback=self.on_error) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 6e26a85ea..20efac543 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -36,7 +36,7 @@ class RecoverySpider(Spider): }, } - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -73,7 +73,7 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): } } - def start_requests(self): + async def start(self): yield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse) def parse(self, response): @@ -83,7 +83,7 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback): name = "ProcessSpiderInputSpiderWithErrback" - def start_requests(self): + async def start(self): yield Request( self.mockserver.url("/status?n=200"), self.parse, errback=self.errback ) @@ -103,7 +103,7 @@ class GeneratorCallbackSpider(Spider): }, } - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -140,7 +140,7 @@ class NotGeneratorCallbackSpider(Spider): }, } - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -215,7 +215,7 @@ class GeneratorOutputChainSpider(Spider): }, } - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -287,8 +287,8 @@ class NotGeneratorOutputChainSpider(Spider): }, } - def start_requests(self): - return [Request(self.mockserver.url("/status?n=200"))] + async def start(self): + yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): return [ diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py new file mode 100644 index 000000000..725833a49 --- /dev/null +++ b/tests/test_spidermiddleware_process_start.py @@ -0,0 +1,352 @@ +import warnings +from asyncio import sleep + +import pytest +from twisted.trial.unittest import TestCase + +from scrapy import Spider, signals +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.test import get_crawler +from tests.test_spider_start import SLEEP_SECONDS + +from .utils import twisted_sleep + +ITEM_A = {"id": "a"} +ITEM_B = {"id": "b"} +ITEM_C = {"id": "c"} +ITEM_D = {"id": "d"} + + +class AsyncioSleepSpiderMiddleware: + async def process_start(self, start): + await sleep(SLEEP_SECONDS) + async for item_or_request in start: + yield item_or_request + + +class NoOpSpiderMiddleware: + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request + + +class TwistedSleepSpiderMiddleware: + async def process_start(self, start): + await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS)) + async for item_or_request in start: + yield item_or_request + + +class UniversalSpiderMiddleware: + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request + + def process_start_requests(self, start_requests, spider): + raise NotImplementedError + + +# Spiders and spider middlewares for MainTestCase._test_wrap + + +class ModernWrapSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_B + + +class ModernWrapSpiderSubclass(ModernWrapSpider): + name = "test" + + +class UniversalWrapSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_B + + def start_requests(self): + yield ITEM_D + + +class DeprecatedWrapSpider(Spider): + name = "test" + + def start_requests(self): + yield ITEM_B + + +class ModernWrapSpiderMiddleware: + async def process_start(self, start): + yield ITEM_A + async for item_or_request in start: + yield item_or_request + yield ITEM_C + + +class UniversalWrapSpiderMiddleware: + async def process_start(self, start): + yield ITEM_A + async for item_or_request in start: + yield item_or_request + yield ITEM_C + + def process_start_requests(self, start, spider): + yield ITEM_A + yield from start + yield ITEM_C + + +class DeprecatedWrapSpiderMiddleware: + def process_start_requests(self, start, spider): + yield ITEM_A + yield from start + yield ITEM_C + + +class MainTestCase(TestCase): + async def _test(self, spider_middlewares, spider_cls, expected_items): + actual_items = [] + + def track_item(item, response, spider): + actual_items.append(item) + + settings = { + "SPIDER_MIDDLEWARES": {cls: n for n, cls in enumerate(spider_middlewares)}, + } + crawler = get_crawler(spider_cls, settings_dict=settings) + crawler.signals.connect(track_item, signals.item_scraped) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + assert actual_items == expected_items, f"{actual_items=} != {expected_items=}" + + async def _test_wrap(self, spider_middleware, spider_cls, expected_items=None): + expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C] + await self._test([spider_middleware], spider_cls, expected_items) + + async def _test_douple_wrap(self, smw1, smw2, spider_cls, expected_items=None): + expected_items = expected_items or [ITEM_A, ITEM_A, ITEM_B, ITEM_C, ITEM_C] + await self._test([smw1, smw2], spider_cls, expected_items) + + @deferred_f_from_coro_f + async def test_modern_mw_modern_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) + + @deferred_f_from_coro_f + async def test_modern_mw_universal_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_wrap(ModernWrapSpiderMiddleware, UniversalWrapSpider) + + @deferred_f_from_coro_f + async def test_modern_mw_deprecated_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ): + await self._test_wrap(ModernWrapSpiderMiddleware, DeprecatedWrapSpider) + + @deferred_f_from_coro_f + async def test_universal_mw_modern_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_wrap(UniversalWrapSpiderMiddleware, ModernWrapSpider) + + @deferred_f_from_coro_f + async def test_universal_mw_universal_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_wrap(UniversalWrapSpiderMiddleware, UniversalWrapSpider) + + @deferred_f_from_coro_f + async def test_universal_mw_deprecated_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ): + await self._test_wrap(UniversalWrapSpiderMiddleware, DeprecatedWrapSpider) + + @deferred_f_from_coro_f + async def test_deprecated_mw_modern_spider(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.raises( + ValueError, match=r"only compatible with \(deprecated\) spiders" + ), + ): + await self._test_wrap(DeprecatedWrapSpiderMiddleware, ModernWrapSpider) + + @deferred_f_from_coro_f + async def test_deprecated_mw_modern_spider_subclass(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.raises( + ValueError, + match=r"^\S+?\.ModernWrapSpider \(inherited by \S+?.ModernWrapSpiderSubclass\) .*? only compatible with \(deprecated\) spiders", + ), + ): + await self._test_wrap( + DeprecatedWrapSpiderMiddleware, ModernWrapSpiderSubclass + ) + + @deferred_f_from_coro_f + async def test_deprecated_mw_universal_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ): + await self._test_wrap( + DeprecatedWrapSpiderMiddleware, + UniversalWrapSpider, + [ITEM_A, ITEM_D, ITEM_C], + ) + + @deferred_f_from_coro_f + async def test_deprecated_mw_deprecated_spider(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ), + ): + await self._test_wrap(DeprecatedWrapSpiderMiddleware, DeprecatedWrapSpider) + + @deferred_f_from_coro_f + async def test_modern_mw_universal_mw_modern_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + UniversalWrapSpiderMiddleware, + ModernWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_modern_mw_deprecated_mw_modern_spider(self): + with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + ModernWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_universal_mw_deprecated_mw_modern_spider(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.raises( + ValueError, match=r"only compatible with \(deprecated\) spiders" + ), + ): + await self._test_douple_wrap( + UniversalWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + ModernWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_modern_mw_universal_mw_universal_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + UniversalWrapSpiderMiddleware, + UniversalWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_modern_mw_deprecated_mw_universal_spider(self): + with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + UniversalWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_universal_mw_deprecated_mw_universal_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ): + await self._test_douple_wrap( + UniversalWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + UniversalWrapSpider, + [ITEM_A, ITEM_A, ITEM_D, ITEM_C, ITEM_C], + ) + + @deferred_f_from_coro_f + async def test_modern_mw_universal_mw_deprecated_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ): + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + UniversalWrapSpiderMiddleware, + DeprecatedWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_modern_mw_deprecated_mw_deprecated_spider(self): + with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + DeprecatedWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_universal_mw_deprecated_mw_deprecated_spider(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ), + ): + await self._test_douple_wrap( + UniversalWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + DeprecatedWrapSpider, + ) + + async def _test_sleep(self, spider_middlewares): + class TestSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_A + + await self._test(spider_middlewares, TestSpider, [ITEM_A]) + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_asyncio_sleep_single(self): + await self._test_sleep([AsyncioSleepSpiderMiddleware]) + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_asyncio_sleep_multiple(self): + await self._test_sleep( + [NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware] + ) + + @deferred_f_from_coro_f + async def test_twisted_sleep_single(self): + await self._test_sleep([TwistedSleepSpiderMiddleware]) + + @deferred_f_from_coro_f + async def test_twisted_sleep_multiple(self): + await self._test_sleep( + [NoOpSpiderMiddleware, TwistedSleepSpiderMiddleware, NoOpSpiderMiddleware] + ) diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py new file mode 100644 index 000000000..295b10ea8 --- /dev/null +++ b/tests/test_spidermiddleware_start.py @@ -0,0 +1,44 @@ +from twisted.trial.unittest import TestCase + +from scrapy.http import Request +from scrapy.spidermiddlewares.start import StartSpiderMiddleware +from scrapy.spiders import Spider +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler + + +class TestMiddleware(TestCase): + @deferred_f_from_coro_f + async def test_async(self): + crawler = get_crawler(Spider) + mw = build_from_crawler(StartSpiderMiddleware, crawler) + + async def start(): + yield Request("data:,1") + yield Request("data:,2", meta={"is_start_request": True}) + yield Request("data:,2", meta={"is_start_request": False}) + yield Request("data:,2", meta={"is_start_request": "foo"}) + + result = [ + request.meta["is_start_request"] + async for request in mw.process_start(start()) + ] + assert result == [True, True, False, "foo"] + + @deferred_f_from_coro_f + async def test_sync(self): + crawler = get_crawler(Spider) + mw = build_from_crawler(StartSpiderMiddleware, crawler) + + def start(): + yield Request("data:,1") + yield Request("data:,2", meta={"is_start_request": True}) + yield Request("data:,2", meta={"is_start_request": False}) + yield Request("data:,2", meta={"is_start_request": "foo"}) + + result = [ + request.meta["is_start_request"] + for request in mw.process_start_requests(start(), Spider("test")) + ] + assert result == [True, True, False, "foo"] diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index e69de29bb..e5e56f414 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -0,0 +1,9 @@ +from twisted.internet.defer import Deferred + + +def twisted_sleep(seconds): + from twisted.internet import reactor + + d = Deferred() + reactor.callLater(seconds, d.callback, None) + return d diff --git a/tox.ini b/tox.ini index e63e44189..92cfc3794 100644 --- a/tox.ini +++ b/tox.ini @@ -44,7 +44,7 @@ install_command = python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} [testenv:typing] -basepython = python3 +basepython = python3.9 deps = mypy==1.14.0 typing-extensions==4.12.2 From 5f6d1b464b81e9673a9639fb8f742d53831c98dc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 5 May 2025 18:08:54 +0500 Subject: [PATCH 106/491] Cover up to 373e501 in the release notes. --- docs/news.rst | 46 ++++++++++++++++++++++++++++++++++------ docs/topics/settings.rst | 3 ++- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 7bb25e6b6..d9bc572be 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -26,6 +26,9 @@ Modified requirements - Dropped support for PyPy 3.9. (:issue:`6613`) +- Added support for PyPy 3.11. + (:issue:`6697`) + Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -38,6 +41,10 @@ Backward-incompatible changes <disable-asyncio>` to use a different reactor. (:issue:`6659`, :issue:`6713`) +- The URL length limit, set by the :setting:`URLLENGTH_LIMIT` setting, is now + also enforced for start requests. + (:issue:`6777`) + - The ``from_settings()`` method of :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`, deprecated in Scrapy 2.12.0, is removed earlier than the usual deprecation @@ -174,6 +181,19 @@ Deprecations called. (:issue:`4151`) +- Passing the ``spider`` argument to the following methods of + :class:`~scrapy.core.scraper.Scraper` is deprecated: + + - ``close_spider()`` + + - ``enqueue_scrape()`` + + - ``handle_spider_error()`` + + - ``handle_spider_output()`` + + (:issue:`6764`) + New features ~~~~~~~~~~~~ @@ -189,7 +209,7 @@ New features helpful for writing :ref:`universal spider middlewares <universal-spider-middleware>` without boilerplate and code duplication. The built-in spider middlewares now inherit from this class. - (:issue:`6693`) + (:issue:`6693`, :issue:`6777`) - :ref:`Scrapy add-ons <topics-addons>` can now define a class method called ``update_pre_crawler_settings()`` to update :ref:`pre-crawler settings @@ -278,14 +298,17 @@ Bug fixes Documentation ~~~~~~~~~~~~~ +- Documented the setting values set in the default project template. + (:issue:`6762`, :issue:`6775`) + - Improved the :ref:`docs <sync-async-spider-middleware>` about asynchronous iterable support in spider middlewares. (:issue:`6688`) - Improved the :ref:`docs <coroutine-deferred-apis>` about using :class:`~twisted.internet.defer.Deferred`-based APIs in coroutine-based - code. - (:issue:`6734`) + code and included a list of such APIs. + (:issue:`6677`, :issue:`6734`, :issue:`6776`) - Improved the :ref:`contribution docs <topics-contributing>`. (:issue:`6561`, :issue:`6575`) @@ -307,7 +330,8 @@ Documentation :issue:`6623`, :issue:`6624`, :issue:`6721`, - :issue:`6723`) + :issue:`6723`, + :issue:`6780`) Packaging ~~~~~~~~~ @@ -315,6 +339,11 @@ Packaging - Switched from ``setup.py`` to ``pyproject.toml``. (:issue:`6514`, :issue:`6547`) +- Switched the build backend from setuptools_ to hatchling_. + (:issue:`6771`) + + .. _hatchling: https://pypi.org/project/hatchling/ + Quality assurance ~~~~~~~~~~~~~~~~~ @@ -384,12 +413,17 @@ Quality assurance :issue:`6722`, :issue:`6724`, :issue:`6741`, - :issue:`6743`) + :issue:`6743`, + :issue:`6766`, + :issue:`6770`, + :issue:`6772`, + :issue:`6773`) - Code cleanups. (:issue:`6600`, :issue:`6606`, - :issue:`6635`) + :issue:`6635`, + :issue:`6764`) .. _release-2.12.0: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 73ac36646..fec82f8e3 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -701,7 +701,8 @@ connections (for ``HTTP10DownloadHandler``). .. note:: - HTTP/1.0 is rarely used nowadays so you can safely ignore this setting, + HTTP/1.0 is rarely used nowadays and its Scrapy support is deprecated, + so you can safely ignore this setting, unless you really want to use HTTP/1.0 and override :setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly, i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``. From 0ce693dfa91a4ebf8418fd04c5595a570939e480 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 8 May 2025 13:23:19 +0500 Subject: [PATCH 107/491] Update VERSION strings. --- docs/topics/coroutines.rst | 4 ++-- docs/topics/spider-middleware.rst | 2 +- scrapy/core/spidermw.py | 20 ++++++++++---------- scrapy/spiders/__init__.py | 4 ++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 62cbc3d49..00812ed7f 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -21,7 +21,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be defined as an :term:`asynchronous generator`. - .. versionadded: VERSION + .. versionadded: 2.13 - :class:`~scrapy.Request` callbacks. @@ -59,7 +59,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): of :ref:`spider middlewares <custom-spider-middleware>`, which *must* be defined as an :term:`asynchronous generator`. - .. versionadded:: VERSION + .. versionadded:: 2.13 - :ref:`Signal handlers that support deferreds <signal-deferred>`. diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 638035e64..67178b8fd 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -86,7 +86,7 @@ one or more of these methods: You may yield the same type of objects as :meth:`~scrapy.Spider.start`. To write spider middlewares that work on Scrapy versions lower than - VERSION, define also a synchronous ``process_start_requests()`` method + 2.13, define also a synchronous ``process_start_requests()`` method that returns an iterable. For example: .. code-block:: python diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 4a0cd9464..310abb9b7 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -85,8 +85,8 @@ class SpiderMiddlewareManager(MiddlewareManager): "either disable or make universal 1 of those 2 sets of " "spider middlewares. Making a spider middleware universal " "means having it define both methods. See the release notes " - "of Scrapy VERSION for details: " - "https://docs.scrapy.org/en/VERSION/news.html" + "of Scrapy 2.13 for details: " + "https://docs.scrapy.org/en/2.13/news.html" ) self._use_start_requests = bool(deprecated_middlewares) @@ -103,15 +103,15 @@ class SpiderMiddlewareManager(MiddlewareManager): f"been deprecated in favor of a new method, process_start(), " f"to support asynchronous code execution. " f"process_start_requests() will stop being called in a future " - f"version of Scrapy. If you use Scrapy VERSION or higher " + f"version of Scrapy. If you use Scrapy 2.13 or higher " f"only, replace process_start_requests() with " f"process_start(); note that process_start() is a coroutine " f"(async def). If you need to maintain compatibility with " f"lower Scrapy versions, when defining " f"process_start_requests() in a spider middleware class, " f"define process_start() as well. See the release notes of " - f"Scrapy VERSION for details: " - f"https://docs.scrapy.org/en/VERSION/news.html", + f"Scrapy 2.13 for details: " + f"https://docs.scrapy.org/en/2.13/news.html", ScrapyDeprecationWarning, ) @@ -435,15 +435,15 @@ class SpiderMiddlewareManager(MiddlewareManager): f"start_requests() has been deprecated in favor of a new " f"method, start(), to support asynchronous code " f"execution. start_requests() will stop being called in a " - f"future version of Scrapy. If you use Scrapy VERSION or " + f"future version of Scrapy. If you use Scrapy 2.13 or " f"higher only, replace start_requests() with start(); " f"note that start() is a coroutine (async def). If you " f"need to maintain compatibility with lower Scrapy versions, " f"when overriding start_requests() in a spider class, " f"override start() as well; you can use super() to " f"reuse the inherited start() implementation without " - f"copy-pasting. See the release notes of Scrapy VERSION for " - f"details: https://docs.scrapy.org/en/VERSION/news.html", + f"copy-pasting. See the release notes of Scrapy 2.13 for " + f"details: https://docs.scrapy.org/en/2.13/news.html", ScrapyDeprecationWarning, ) @@ -469,8 +469,8 @@ class SpiderMiddlewareManager(MiddlewareManager): f"deprecated spider middlewares (and earlier Scrapy versions) " f"by defining a sync start_requests() method that works " f"similarly to its existing start() method. See the " - f"release notes of Scrapy VERSION for details: " - f"https://docs.scrapy.org/en/VERSION/news.html" + f"release notes of Scrapy 2.13 for details: " + f"https://docs.scrapy.org/en/2.13/news.html" ) # This method is only needed until _async compatibility methods are removed. diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 0a1d85ae6..a722dd83b 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -83,7 +83,7 @@ class Spider(object_ref): async def start(self) -> AsyncIterator[Any]: """Yield the initial :class:`~scrapy.Request` objects to send. - .. versionadded:: VERSION + .. versionadded:: 2.13 For example: @@ -115,7 +115,7 @@ class Spider(object_ref): async def start(self): yield {"foo": "bar"} - To write spiders that work on Scrapy versions lower than VERSION, + To write spiders that work on Scrapy versions lower than 2.13, define also a synchronous ``start_requests()`` method that returns an iterable. For example: From 82a32451583967a828c2e80a31930f64dbc136ac Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 8 May 2025 14:43:34 +0500 Subject: [PATCH 108/491] =?UTF-8?q?Bump=20version:=202.12.0=20=E2=86=92=20?= =?UTF-8?q?2.13.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- SECURITY.md | 4 ++-- docs/news.rst | 2 +- pyproject.toml | 2 +- scrapy/VERSION | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index bc64dec7b..a5a5c7fb3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,8 +4,8 @@ | Version | Supported | | ------- | ------------------ | -| 2.12.x | :white_check_mark: | -| < 2.12.x | :x: | +| 2.13.x | :white_check_mark: | +| < 2.13.x | :x: | ## Reporting a Vulnerability diff --git a/docs/news.rst b/docs/news.rst index d574317bb..cf1c35893 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.13.0: -Scrapy 2.13.0 (unreleased) +Scrapy 2.13.0 (2025-05-08) -------------------------- Highlights: diff --git a/pyproject.toml b/pyproject.toml index 187587eb1..85fba0f92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,7 +116,7 @@ module = "twisted" implicit_reexport = true [tool.bumpversion] -current_version = "2.12.0" +current_version = "2.13.0" commit = true tag = true tag_name = "{new_version}" diff --git a/scrapy/VERSION b/scrapy/VERSION index d8b698973..fb2c0766b 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.12.0 +2.13.0 From 128cb551eb493601d4a4cd6d7a087e09a07d7092 Mon Sep 17 00:00:00 2001 From: Felipe Benevolo <77981110+fbenevolo@users.noreply.github.com> Date: Mon, 12 May 2025 08:04:34 -0300 Subject: [PATCH 109/491] refactor tests/test_downloadermiddleware_httpcache.py (#6769) --- tests/test_downloadermiddleware_httpcache.py | 114 ++++++++++++------- 1 file changed, 72 insertions(+), 42 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 5fac88ed7..02f4f488e 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -15,8 +15,7 @@ from scrapy.utils.test import get_crawler class TestBase: - storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" - policy_class = "scrapy.extensions.httpcache.RFC2616Policy" + """Base class with common setup and helper methods.""" def setup_method(self): self.yesterday = email.utils.formatdate(time.time() - 86400) @@ -90,23 +89,10 @@ class TestBase: ) assert request1.body == request2.body - def test_dont_cache(self): - with self._middleware() as mw: - self.request.meta["dont_cache"] = True - mw.process_response(self.request, self.response, self.spider) - assert mw.storage.retrieve_response(self.spider, self.request) is None - with self._middleware() as mw: - self.request.meta["dont_cache"] = False - mw.process_response(self.request, self.response, self.spider) - if mw.policy.should_cache_response(self.response, self.request): - assert isinstance( - mw.storage.retrieve_response(self.spider, self.request), - self.response.__class__, - ) +class StorageTestMixin: + """Mixin containing storage-specific test methods.""" - -class TestDefaultStorage(TestBase): def test_storage(self): with self._storage() as storage: request2 = self.request.copy() @@ -143,31 +129,27 @@ class TestDefaultStorage(TestBase): self.assertEqualResponse(response, cached_response) -class TestDbmStorage(TestDefaultStorage): - storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" +class PolicyTestMixin: + """Mixin containing policy-specific test methods.""" + + def test_dont_cache(self): + with self._middleware() as mw: + self.request.meta["dont_cache"] = True + mw.process_response(self.request, self.response, self.spider) + assert mw.storage.retrieve_response(self.spider, self.request) is None + + with self._middleware() as mw: + self.request.meta["dont_cache"] = False + mw.process_response(self.request, self.response, self.spider) + if mw.policy.should_cache_response(self.response, self.request): + assert isinstance( + mw.storage.retrieve_response(self.spider, self.request), + self.response.__class__, + ) -class TestDbmStorageWithCustomDbmModule(TestDbmStorage): - dbm_module = "tests.mocks.dummydbm" - - def _get_settings(self, **new_settings): - new_settings.setdefault("HTTPCACHE_DBM_MODULE", self.dbm_module) - return super()._get_settings(**new_settings) - - def test_custom_dbm_module_loaded(self): - # make sure our dbm module has been loaded - with self._storage() as storage: - assert storage.dbmodule.__name__ == self.dbm_module - - -class TestFilesystemStorageGzip(TestDefaultStorage): - def _get_settings(self, **new_settings): - new_settings.setdefault("HTTPCACHE_GZIP", True) - return super()._get_settings(**new_settings) - - -class TestDummyPolicy(TestBase): - policy_class = "scrapy.extensions.httpcache.DummyPolicy" +class DummyPolicyTestMixin(PolicyTestMixin): + """Mixin containing dummy policy specific test methods.""" def test_middleware(self): with self._middleware() as mw: @@ -258,8 +240,8 @@ class TestDummyPolicy(TestBase): assert "cached" in response.flags -class TestRFC2616Policy(TestDefaultStorage): - policy_class = "scrapy.extensions.httpcache.RFC2616Policy" +class RFC2616PolicyTestMixin(PolicyTestMixin): + """Mixin containing RFC2616 policy specific test methods.""" def _process_requestresponse(self, mw, request, response): result = None @@ -562,3 +544,51 @@ class TestRFC2616Policy(TestDefaultStorage): res2 = self._process_requestresponse(mw, req0, None) self.assertEqualResponse(res1, res2) assert "cached" in res2.flags + + +# Concrete test classes that combine storage and policy mixins + + +class TestFilesystemStorageWithDummyPolicy( + TestBase, StorageTestMixin, DummyPolicyTestMixin +): + storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" + policy_class = "scrapy.extensions.httpcache.DummyPolicy" + + +class TestFilesystemStorageWithRFC2616Policy( + TestBase, StorageTestMixin, RFC2616PolicyTestMixin +): + storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" + policy_class = "scrapy.extensions.httpcache.RFC2616Policy" + + +class TestDbmStorageWithDummyPolicy(TestBase, StorageTestMixin, DummyPolicyTestMixin): + storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" + policy_class = "scrapy.extensions.httpcache.DummyPolicy" + + +class TestDbmStorageWithRFC2616Policy( + TestBase, StorageTestMixin, RFC2616PolicyTestMixin +): + storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" + policy_class = "scrapy.extensions.httpcache.RFC2616Policy" + + +class TestDbmStorageWithCustomDbmModule(TestDbmStorageWithDummyPolicy): + dbm_module = "tests.mocks.dummydbm" + + def _get_settings(self, **new_settings): + new_settings.setdefault("HTTPCACHE_DBM_MODULE", self.dbm_module) + return super()._get_settings(**new_settings) + + def test_custom_dbm_module_loaded(self): + # make sure our dbm module has been loaded + with self._storage() as storage: + assert storage.dbmodule.__name__ == self.dbm_module + + +class TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPolicy): + def _get_settings(self, **new_settings): + new_settings.setdefault("HTTPCACHE_GZIP", True) + return super()._get_settings(**new_settings) From 2442536d0f5cdd16b836aafbe2da2c668c6dc96f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 13 May 2025 12:22:28 +0400 Subject: [PATCH 110/491] Add a deepwiki badge, update other badges. (#6793) --- README.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index cf7c6043c..29488d825 100644 --- a/README.rst +++ b/README.rst @@ -17,19 +17,14 @@ Scrapy :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu :alt: Ubuntu -.. .. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg - .. :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS - .. :alt: macOS - +.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS + :alt: macOS .. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows :alt: Windows -.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg - :target: https://pypi.org/pypi/Scrapy - :alt: Wheel Status - .. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg :target: https://codecov.io/github/scrapy/scrapy?branch=master :alt: Coverage report @@ -38,6 +33,10 @@ Scrapy :target: https://anaconda.org/conda-forge/scrapy :alt: Conda Version +.. image:: https://deepwiki.com/badge.svg + :target: https://deepwiki.com/scrapy/scrapy + :alt: Ask DeepWiki + Overview ======== From b86f00327a3d113a97525f68d38f77529d090f30 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 13 May 2025 22:47:57 +0400 Subject: [PATCH 111/491] Refactor more Deferred functions. (#6795) --- scrapy/commands/parse.py | 1 + scrapy/core/downloader/__init__.py | 96 +++++++++++++--------------- scrapy/core/downloader/middleware.py | 25 ++++---- scrapy/core/engine.py | 49 ++++++-------- scrapy/core/scraper.py | 29 +++++---- scrapy/core/spidermw.py | 50 +++++++++------ scrapy/crawler.py | 38 +++++------ tests/test_downloadermiddleware.py | 33 ++++++++++ 8 files changed, 179 insertions(+), 142 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 0dd9954cb..c4b3d2af9 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -282,6 +282,7 @@ class Command(BaseRunSpiderCommand): ) -> list[Any]: items, requests, opts, depth, spider, callback = args if opts.pipelines: + assert self.pcrawler.engine itemproc = self.pcrawler.engine.scraper.itemproc for item in items: itemproc.process_item(item, spider) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 78dc16df6..5468398aa 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -5,29 +5,32 @@ import warnings from collections import deque from datetime import datetime from time import time -from typing import TYPE_CHECKING, Any, TypeVar, cast +from typing import TYPE_CHECKING, Any, cast from twisted.internet import task -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, inlineCallbacks from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.resolver import dnscache -from scrapy.utils.defer import mustbe_deferred +from scrapy.utils.defer import ( + deferred_from_coro, + maybe_deferred_to_future, + mustbe_deferred, +) from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: + from collections.abc import Generator + from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.settings import BaseSettings from scrapy.signalmanager import SignalManager -_T = TypeVar("_T") - - class Slot: """Downloader slot""" @@ -114,16 +117,17 @@ class Downloader: "DOWNLOAD_SLOTS", {} ) - def fetch(self, request: Request, spider: Spider) -> Deferred[Response | Request]: - def _deactivate(response: _T) -> _T: - self.active.remove(request) - return response - + @inlineCallbacks + def fetch( + self, request: Request, spider: Spider + ) -> Generator[Deferred[Any], Any, Response | Request]: self.active.add(request) - dfd: Deferred[Response | Request] = self.middleware.download( - self._enqueue_request, request, spider - ) - return dfd.addBoth(_deactivate) + try: + return ( + yield self.middleware.download(self._enqueue_request, request, spider) + ) + finally: + self.active.remove(request) def needs_backout(self) -> bool: return len(self.active) >= self.total_concurrency @@ -164,22 +168,23 @@ class Downloader: ) return self.get_slot_key(request) - def _enqueue_request(self, request: Request, spider: Spider) -> Deferred[Response]: + @inlineCallbacks + def _enqueue_request( + self, request: Request, spider: Spider + ) -> Generator[Deferred[Any], Any, Response]: key, slot = self._get_slot(request, spider) request.meta[self.DOWNLOAD_SLOT] = key - - def _deactivate(response: Response) -> Response: - slot.active.remove(request) - return response - slot.active.add(request) self.signals.send_catch_log( signal=signals.request_reached_downloader, request=request, spider=spider ) - deferred: Deferred[Response] = Deferred().addBoth(_deactivate) - slot.queue.append((request, deferred)) + d: Deferred[Response] = Deferred() + slot.queue.append((request, d)) self._process_queue(spider, slot) - return deferred + try: + return (yield d) + finally: + slot.active.remove(request) def _process_queue(self, spider: Spider, slot: Slot) -> None: from twisted.internet import reactor @@ -202,26 +207,23 @@ class Downloader: while slot.queue and slot.free_transfer_slots() > 0: slot.lastseen = now request, deferred = slot.queue.popleft() - dfd = self._download(slot, request, spider) + dfd = deferred_from_coro(self._download(slot, request, spider)) dfd.chainDeferred(deferred) # prevent burst if inter-request delays were configured if delay: self._process_queue(spider, slot) break - def _download( - self, slot: Slot, request: Request, spider: Spider - ) -> Deferred[Response]: - # The order is very important for the following deferreds. Do not change! - - # 1. Create the download deferred - dfd: Deferred[Response] = mustbe_deferred( - self.handlers.download_request, request, spider - ) - - # 2. Notify response_downloaded listeners about the recent download - # before querying queue for next request - def _downloaded(response: Response) -> Response: + async def _download(self, slot: Slot, request: Request, spider: Spider) -> Response: + # The order is very important for the following logic. Do not change! + slot.transferring.add(request) + try: + # 1. Download the response + response: Response = await maybe_deferred_to_future( + mustbe_deferred(self.handlers.download_request, request, spider) + ) + # 2. Notify response_downloaded listeners about the recent download + # before querying queue for next request self.signals.send_catch_log( signal=signals.response_downloaded, response=response, @@ -229,24 +231,16 @@ class Downloader: spider=spider, ) return response - - dfd.addCallback(_downloaded) - - # 3. After response arrives, remove the request from transferring - # state to free up the transferring slot so it can be used by the - # following requests (perhaps those which came from the downloader - # middleware itself) - slot.transferring.add(request) - - def finish_transferring(_: _T) -> _T: + finally: + # 3. After response arrives, remove the request from transferring + # state to free up the transferring slot so it can be used by the + # following requests (perhaps those which came from the downloader + # middleware itself) slot.transferring.remove(request) self._process_queue(spider, slot) self.signals.send_catch_log( signal=signals.request_left_downloader, request=request, spider=spider ) - return _ - - return dfd.addBoth(finish_transferring) def close(self) -> None: self._slot_gc_loop.stop() diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index db4191385..a4055849d 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -20,8 +20,6 @@ from scrapy.utils.defer import deferred_from_coro, mustbe_deferred if TYPE_CHECKING: from collections.abc import Generator - from twisted.python.failure import Failure - from scrapy import Spider from scrapy.settings import BaseSettings @@ -41,12 +39,13 @@ class DownloaderMiddlewareManager(MiddlewareManager): if hasattr(mw, "process_exception"): self.methods["process_exception"].appendleft(mw.process_exception) + @inlineCallbacks def download( self, download_func: Callable[[Request, Spider], Deferred[Response]], request: Request, spider: Spider, - ) -> Deferred[Response | Request]: + ) -> Generator[Deferred[Any], Any, Response | Request]: @inlineCallbacks def process_request( request: Request, @@ -92,9 +91,8 @@ class DownloaderMiddlewareManager(MiddlewareManager): @inlineCallbacks def process_exception( - failure: Failure, - ) -> Generator[Deferred[Any], Any, Failure | Response | Request]: - exception = failure.value + exception: Exception, + ) -> Generator[Deferred[Any], Any, Response | Request]: for method in self.methods["process_exception"]: method = cast(Callable, method) response = yield deferred_from_coro( @@ -109,11 +107,12 @@ class DownloaderMiddlewareManager(MiddlewareManager): ) if response: return response - return failure + raise exception - deferred: Deferred[Response | Request] = mustbe_deferred( - process_request, request - ) - deferred.addErrback(process_exception) - deferred.addCallback(process_response) - return deferred + try: + result: Response | Request = yield mustbe_deferred(process_request, request) + except Exception as ex: + # either returns a request or response (which we pass to process_response()) + # or reraises the exception + result = yield process_exception(ex) + return (yield process_response(result)) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 7f5dd0405..658f6e774 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -10,7 +10,7 @@ from __future__ import annotations import logging from time import time from traceback import format_exc -from typing import TYPE_CHECKING, Any, TypeVar, cast +from typing import TYPE_CHECKING, Any, cast from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall @@ -42,8 +42,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -_T = TypeVar("_T") - class _Slot: def __init__( @@ -349,28 +347,32 @@ class ExecutionEngine: signals.request_dropped, request=request, spider=self.spider ) - def download(self, request: Request) -> Deferred[Response]: + @inlineCallbacks + def download(self, request: Request) -> Generator[Deferred[Any], Any, Response]: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - d: Deferred[Response | Request] = self._download(request) - # Deferred.addBoth() overloads don't seem to support a Union[_T, Deferred[_T]] return type - d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[call-overload] - return d2 + try: + response_or_request = yield self._download(request) + finally: + assert self._slot is not None + self._slot.remove_request(request) + if isinstance(response_or_request, Request): + return (yield self.download(response_or_request)) + return response_or_request - def _downloaded( - self, result: Response | Request | Failure, request: Request - ) -> Deferred[Response] | Response | Failure: - assert self._slot is not None # typing - self._slot.remove_request(request) - return self.download(result) if isinstance(result, Request) else result - - def _download(self, request: Request) -> Deferred[Response | Request]: + @inlineCallbacks + def _download( + self, request: Request + ) -> Generator[Deferred[Any], Any, Response | Request]: assert self._slot is not None # typing + assert self.spider is not None self._slot.add_request(request) - - def _on_success(result: Response | Request) -> Response | Request: + try: + result: Response | Request = yield self.downloader.fetch( + request, self.spider + ) if not isinstance(result, (Response, Request)): raise TypeError( f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}" @@ -391,17 +393,8 @@ class ExecutionEngine: spider=self.spider, ) return result - - def _on_complete(_: _T) -> _T: - assert self._slot is not None + finally: self._slot.nextcall.schedule() - return _ - - assert self.spider is not None - dwld: Deferred[Response | Request] = self.downloader.fetch(request, self.spider) - dwld.addCallback(_on_success) - dwld.addBoth(_on_complete) - return dwld @deferred_f_from_coro_f async def open_spider( diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 9378f2651..2c48a9a81 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -111,11 +111,11 @@ class Scraper: assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter - @inlineCallbacks - def open_spider(self, spider: Spider) -> Generator[Deferred[Any], Any, None]: + @deferred_f_from_coro_f + async def open_spider(self, spider: Spider) -> None: """Open the given spider for scraping and allocate resources for it""" self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE")) - yield self.itemproc.open_spider(spider) + await maybe_deferred_to_future(self.itemproc.open_spider(spider)) def close_spider(self, spider: Spider | None = None) -> Deferred[Spider]: """Close a spider being scraped and release its resources""" @@ -191,10 +191,8 @@ class Scraper: if isinstance(result, Response): try: # call the spider middlewares and the request callback with the response - output = await maybe_deferred_to_future( - self.spidermw.scrape_response( - self.call_spider, result, request, self.crawler.spider - ) + output = await self.spidermw.scrape_response_async( + self.call_spider, result, request, self.crawler.spider ) except Exception: self.handle_spider_error(Failure(), request, result) @@ -363,12 +361,19 @@ class Scraper: self.crawler.engine.crawl(request=output) return if output is not None: - await maybe_deferred_to_future( - self.start_itemproc(output, response=response) - ) + await self.start_itemproc_async(output, response=response) - @deferred_f_from_coro_f - async def start_itemproc(self, item: Any, *, response: Response | None) -> None: + def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[None]: + """Send *item* to the item pipelines for processing. + + *response* is the source of the item data. If the item does not come + from response data, e.g. it was hard-coded, set it to ``None``. + """ + return deferred_from_coro(self.start_itemproc_async(item, response=response)) + + async def start_itemproc_async( + self, item: Any, *, response: Response | None + ) -> None: """Send *item* to the item pipelines for processing. *response* is the source of the item data. If the item does not come diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 310abb9b7..10aad7858 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -23,7 +23,6 @@ from scrapy.middleware import MiddlewareManager from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.conf import build_component_list from scrapy.utils.defer import ( - deferred_f_from_coro_f, deferred_from_coro, maybe_deferred_to_future, mustbe_deferred, @@ -169,7 +168,7 @@ class SpiderMiddlewareManager(MiddlewareManager): exception_result = cast( Union[Failure, MutableChain[_T]], self._process_spider_exception( - response, spider, Failure(ex), exception_processor_index + response, spider, ex, exception_processor_index ), ) if isinstance(exception_result, Failure): @@ -185,7 +184,7 @@ class SpiderMiddlewareManager(MiddlewareManager): exception_result = cast( Union[Failure, MutableAsyncChain[_T]], self._process_spider_exception( - response, spider, Failure(ex), exception_processor_index + response, spider, ex, exception_processor_index ), ) if isinstance(exception_result, Failure): @@ -201,13 +200,12 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - _failure: Failure, + exception: Exception, start_index: int = 0, - ) -> Failure | MutableChain[_T] | MutableAsyncChain[_T]: - exception = _failure.value + ) -> MutableChain[_T] | MutableAsyncChain[_T]: # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): - return _failure + raise exception method_list = islice( self.methods["process_spider_exception"], start_index, None ) @@ -242,7 +240,7 @@ class SpiderMiddlewareManager(MiddlewareManager): f"or an iterable, got {type(result)}" ) raise _InvalidOutput(msg) - return _failure + raise exception # This method cannot be made async def, as _process_spider_exception relies on the Deferred result # being available immediately which doesn't work when it's a wrapped coroutine. @@ -308,7 +306,7 @@ class SpiderMiddlewareManager(MiddlewareManager): except Exception as ex: exception_result: Failure | MutableChain[_T] | MutableAsyncChain[_T] = ( self._process_spider_exception( - response, spider, Failure(ex), method_index + 1 + response, spider, ex, method_index + 1 ) ) if isinstance(exception_result, Failure): @@ -369,24 +367,36 @@ class SpiderMiddlewareManager(MiddlewareManager): request: Request, spider: Spider, ) -> Deferred[MutableChain[_T] | MutableAsyncChain[_T]]: + return deferred_from_coro( + self.scrape_response_async(scrape_func, response, request, spider) + ) + + async def scrape_response_async( + self, + scrape_func: ScrapeFunc[_T], + response: Response, + request: Request, + spider: Spider, + ) -> MutableChain[_T] | MutableAsyncChain[_T]: async def process_callback_output( result: Iterable[_T] | AsyncIterator[_T], ) -> MutableChain[_T] | MutableAsyncChain[_T]: return await self._process_callback_output(response, spider, result) def process_spider_exception( - _failure: Failure, - ) -> Failure | MutableChain[_T] | MutableAsyncChain[_T]: - return self._process_spider_exception(response, spider, _failure) + exception: Exception, + ) -> MutableChain[_T] | MutableAsyncChain[_T]: + return self._process_spider_exception(response, spider, exception) - dfd: Deferred[Iterable[_T] | AsyncIterator[_T]] = mustbe_deferred( - self._process_spider_input, scrape_func, response, request, spider - ) - dfd2: Deferred[MutableChain[_T] | MutableAsyncChain[_T]] = dfd.addCallback( - deferred_f_from_coro_f(process_callback_output) - ) - dfd2.addErrback(process_spider_exception) - return dfd2 + try: + it: Iterable[_T] | AsyncIterator[_T] = await maybe_deferred_to_future( + mustbe_deferred( + self._process_spider_input, scrape_func, response, request, spider + ) + ) + return await process_callback_output(it) + except Exception as ex: + return process_spider_exception(ex) async def process_start(self, spider: Spider) -> AsyncIterator[Any] | None: self._check_deprecated_start_requests_use(spider) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 749096db5..5dbee6537 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -10,7 +10,6 @@ from twisted.internet.defer import ( Deferred, DeferredList, inlineCallbacks, - maybeDeferred, ) from zope.interface.verify import verifyClass @@ -175,7 +174,7 @@ class Crawler: if self.crawling: self.crawling = False assert self.engine - yield maybeDeferred(self.engine.stop) + yield self.engine.stop() @staticmethod def _get_component( @@ -277,12 +276,6 @@ class CrawlerRunner: process. See :ref:`run-from-script` for an example. """ - crawlers = property( - lambda self: self._crawlers, - doc="Set of :class:`crawlers <scrapy.crawler.Crawler>` started by " - ":meth:`crawl` and managed by this class.", - ) - @staticmethod def _get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol: """Get SpiderLoader instance from settings""" @@ -303,6 +296,12 @@ class CrawlerRunner: self._active: set[Deferred[None]] = set() self.bootstrap_failed = False + @property + def crawlers(self) -> set[Crawler]: + """Set of :class:`crawlers <scrapy.crawler.Crawler>` started by + :meth:`crawl` and managed by this class.""" + return self._crawlers + def crawl( self, crawler_or_spidercls: type[Spider] | str | Crawler, @@ -338,18 +337,19 @@ class CrawlerRunner: crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) - def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred[None]: + @inlineCallbacks + def _crawl( + self, crawler: Crawler, *args: Any, **kwargs: Any + ) -> Generator[Deferred[Any], Any, None]: self.crawlers.add(crawler) d = crawler.crawl(*args, **kwargs) self._active.add(d) - - def _done(result: _T) -> _T: + try: + yield d + finally: self.crawlers.discard(crawler) self._active.discard(d) self.bootstrap_failed |= not getattr(crawler, "spider", None) - return result - - return d.addBoth(_done) def create_crawler( self, crawler_or_spidercls: type[Spider] | str | Crawler @@ -501,10 +501,12 @@ class CrawlerProcess(CrawlerRunner): ) reactor.run(installSignalHandlers=install_signal_handlers) # blocking call - def _graceful_stop_reactor(self) -> Deferred[Any]: - d = self.stop() - d.addBoth(self._stop_reactor) - return d + @inlineCallbacks + def _graceful_stop_reactor(self) -> Generator[Deferred[Any], Any, None]: + try: + yield self.stop() + finally: + self._stop_reactor() def _stop_reactor(self, _: Any = None) -> None: from twisted.internet import reactor diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 8ae160f8a..61a5a7df5 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -131,6 +131,39 @@ class TestResponseFromProcessRequest(TestManagerBase): assert not download_func.called +class TestResponseFromProcessException(TestManagerBase): + """Tests middleware returning a response from process_exception.""" + + @deferred_f_from_coro_f + async def test_process_response_called(self): + resp = Response("http://example.com/index.html") + calls = [] + + def download_func(request, spider): + raise ValueError("test") + + class ResponseMiddleware: + def process_response(self, request, response, spider): + calls.append("process_response") + return resp + + def process_exception(self, request, exception, spider): + calls.append("process_exception") + return resp + + self.mwman._add_middleware(ResponseMiddleware()) + + req = Request("http://example.com/index.html") + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp + assert calls == [ + "process_exception", + "process_response", + ] + + class TestInvalidOutput(TestManagerBase): @deferred_f_from_coro_f async def test_invalid_process_request(self): From 82acef30517496d622a80f24adb5b3599e63f64a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 14 May 2025 18:21:18 +0400 Subject: [PATCH 112/491] Add AsyncCrawlerRunner. (#6796) --- conftest.py | 2 + docs/topics/api.rst | 3 + docs/topics/practices.rst | 178 +++++++++-------- scrapy/crawler.py | 188 ++++++++++++++---- tests/AsyncCrawlerRunner/multi_parallel.py | 28 +++ tests/AsyncCrawlerRunner/multi_seq.py | 27 +++ tests/AsyncCrawlerRunner/simple.py | 26 +++ .../simple_default_reactor.py | 24 +++ .../CrawlerRunner/explicit_default_reactor.py | 28 +++ tests/CrawlerRunner/multi_parallel.py | 26 +++ tests/CrawlerRunner/multi_seq.py | 27 +++ tests/CrawlerRunner/simple.py | 24 +++ tests/test_crawler.py | 143 ++++++++++++- 13 files changed, 589 insertions(+), 135 deletions(-) create mode 100644 tests/AsyncCrawlerRunner/multi_parallel.py create mode 100644 tests/AsyncCrawlerRunner/multi_seq.py create mode 100644 tests/AsyncCrawlerRunner/simple.py create mode 100644 tests/AsyncCrawlerRunner/simple_default_reactor.py create mode 100644 tests/CrawlerRunner/explicit_default_reactor.py create mode 100644 tests/CrawlerRunner/multi_parallel.py create mode 100644 tests/CrawlerRunner/multi_seq.py create mode 100644 tests/CrawlerRunner/simple.py diff --git a/conftest.py b/conftest.py index 8e0c429a0..18132b7e6 100644 --- a/conftest.py +++ b/conftest.py @@ -19,6 +19,8 @@ collect_ignore = [ "tests/mockserver.py", "tests/pipelines.py", "tests/spiders.py", + # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess + *_py_files("tests/AsyncCrawlerRunner"), # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess *_py_files("tests/CrawlerProcess"), # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 8e8f3a0c9..3e7bc45c5 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -109,6 +109,9 @@ how you :ref:`configure the downloader middlewares .. automethod:: stop +.. autoclass:: AsyncCrawlerRunner + :members: + .. autoclass:: CrawlerRunner :members: diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index db91cd073..18005aaf2 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -73,72 +73,87 @@ project as example. process.start() # the script will block here until the crawling is finished There's another Scrapy utility that provides more control over the crawling -process: :class:`scrapy.crawler.CrawlerRunner`. This class is a thin wrapper -that encapsulates some simple helpers to run multiple crawlers, but it won't -start or interfere with existing reactors in any way. +process: :class:`scrapy.crawler.AsyncCrawlerRunner` and +:class:`scrapy.crawler.CrawlerRunner`. These classes are thin wrappers +that encapsulate some simple helpers to run multiple crawlers, but they won't +start or interfere with existing reactors in any way. They have similar +functionality, differing in their asynchronous API style: +:class:`~scrapy.crawler.AsyncCrawlerRunner` returns coroutines from its +asynchronous methods while :class:`~scrapy.crawler.CrawlerRunner` returns +:class:`~twisted.internet.defer.Deferred` objects. -Using this class the reactor should be explicitly run after scheduling your -spiders. It's recommended you use :class:`~scrapy.crawler.CrawlerRunner` -instead of :class:`~scrapy.crawler.CrawlerProcess` if your application is -already using Twisted and you want to run Scrapy in the same reactor. +When using these classes the reactor should be explicitly run after scheduling +your spiders. It's recommended that you use +:class:`~scrapy.crawler.AsyncCrawlerRunner` or +:class:`~scrapy.crawler.CrawlerRunner` instead of +:class:`~scrapy.crawler.CrawlerProcess` if your application is already using +Twisted and you want to run Scrapy in the same reactor. -Note that you will also have to shutdown the Twisted reactor yourself after the -spider is finished. This can be achieved by adding callbacks to the deferred -returned by the :meth:`CrawlerRunner.crawl -<scrapy.crawler.CrawlerRunner.crawl>` method. +If you want to stop the reactor or run any other code right after the spider +finishes you can do that after the :meth:`AsyncCrawlerRunner.crawl() +<scrapy.crawler.AsyncCrawlerRunner.crawl>` coroutine completes (or the Deferred +returned from :meth:`CrawlerRunner.crawl() +<scrapy.crawler.CrawlerRunner.crawl>` fires). In the simplest case you can also +use :func:`twisted.internet.task.react` to start and stop the reactor, though +it may be easier to just use :class:`~scrapy.crawler.CrawlerProcess` instead. -Here's an example of its usage, along with a callback to manually stop the -reactor after ``MySpider`` has finished running. +Here's an example of using :class:`~scrapy.crawler.AsyncCrawlerRunner` together +with simple reactor management code: .. code-block:: python import scrapy - from scrapy.crawler import CrawlerRunner + from scrapy.crawler import AsyncCrawlerRunner + from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.log import configure_logging - - - class MySpider(scrapy.Spider): - # Your spider definition - ... - - - configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) - runner = CrawlerRunner() - - d = runner.crawl(MySpider) - - from twisted.internet import reactor - - d.addBoth(lambda _: reactor.stop()) - reactor.run() # the script will block here until the crawling is finished - -Same example but using a non-default reactor, it's only necessary call -``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically. - -.. code-block:: python - - import scrapy - from scrapy.crawler import CrawlerRunner - from scrapy.utils.log import configure_logging - - - class MySpider(scrapy.Spider): - # Your spider definition - ... - - - configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) - from scrapy.utils.reactor import install_reactor + from twisted.internet.task import react + + + class MySpider(scrapy.Spider): + # Your spider definition + ... + + + async def crawl(_): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = AsyncCrawlerRunner() + await runner.crawl(MySpider) # completes when the spider finishes + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") - runner = CrawlerRunner() - d = runner.crawl(MySpider) + react(deferred_f_from_coro_f(crawl)) - from twisted.internet import reactor +Same example but using :class:`~scrapy.crawler.CrawlerRunner` and a +different reactor (:class:`~scrapy.crawler.AsyncCrawlerRunner` only works +with :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`): - d.addBoth(lambda _: reactor.stop()) - reactor.run() # the script will block here until the crawling is finished +.. code-block:: python + + import scrapy + from scrapy.crawler import CrawlerRunner + from scrapy.utils.log import configure_logging + from scrapy.utils.reactor import install_reactor + from twisted.internet.task import react + + + class MySpider(scrapy.Spider): + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.epollreactor.EPollReactor", + } + # Your spider definition + ... + + + def crawl(_): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = CrawlerRunner() + d = runner.crawl(MySpider) + return d # this Deferred fires when the spider finishes + + + install_reactor("twisted.internet.epollreactor.EPollReactor") + react(crawl) .. seealso:: :doc:`twisted:core/howto/reactor-basics` @@ -176,14 +191,16 @@ Here is an example that runs multiple spiders simultaneously: process.crawl(MySpider2) process.start() # the script will block here until all crawling jobs are finished -Same example using :class:`~scrapy.crawler.CrawlerRunner`: +Same example using :class:`~scrapy.crawler.AsyncCrawlerRunner`: .. code-block:: python import scrapy - from scrapy.crawler import CrawlerRunner + from scrapy.crawler import AsyncCrawlerRunner + from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.log import configure_logging - from scrapy.utils.project import get_project_settings + from scrapy.utils.reactor import install_reactor + from twisted.internet.task import react class MySpider1(scrapy.Spider): @@ -196,27 +213,29 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: ... - configure_logging() - settings = get_project_settings() - runner = CrawlerRunner(settings) - runner.crawl(MySpider1) - runner.crawl(MySpider2) - d = runner.join() + async def crawl(_): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = AsyncCrawlerRunner() + runner.crawl(MySpider1) + runner.crawl(MySpider2) + await runner.join() # completes when both spiders finish - from twisted.internet import reactor - d.addBoth(lambda _: reactor.stop()) + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + react(deferred_f_from_coro_f(crawl)) - reactor.run() # the script will block here until all crawling jobs are finished -Same example but running the spiders sequentially by chaining the deferreds: +Same example but running the spiders sequentially by awaiting until each one +finishes before starting the next one: .. code-block:: python - from twisted.internet import defer - from scrapy.crawler import CrawlerRunner + import scrapy + from scrapy.crawler import AsyncCrawlerRunner + from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.log import configure_logging - from scrapy.utils.project import get_project_settings + from scrapy.utils.reactor import install_reactor + from twisted.internet.task import react class MySpider1(scrapy.Spider): @@ -229,22 +248,15 @@ Same example but running the spiders sequentially by chaining the deferreds: ... - settings = get_project_settings() - configure_logging(settings) - runner = CrawlerRunner(settings) + async def crawl(_): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = AsyncCrawlerRunner() + await runner.crawl(MySpider1) + await runner.crawl(MySpider2) - @defer.inlineCallbacks - def crawl(): - yield runner.crawl(MySpider1) - yield runner.crawl(MySpider2) - reactor.stop() - - - from twisted.internet import reactor - - crawl() - reactor.run() # the script will block here until the last crawl call is finished + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + react(deferred_f_from_coro_f(crawl)) .. note:: When running multiple spiders in the same process, :ref:`reactor settings <reactor-settings>` should not have a different value per spider. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 5dbee6537..1d6532fa9 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import contextlib import logging import pprint @@ -20,6 +21,7 @@ from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader from scrapy.settings import BaseSettings, Settings, overridden_settings from scrapy.signalmanager import SignalManager +from scrapy.utils.defer import deferred_to_future from scrapy.utils.log import ( LogCounterHandler, configure_logging, @@ -263,19 +265,7 @@ class Crawler: return self._get_component(cls, self.engine.scraper.spidermw.middlewares) -class CrawlerRunner: - """ - This is a convenient helper class that keeps track of, manages and runs - crawlers inside an already setup :mod:`~twisted.internet.reactor`. - - The CrawlerRunner object must be instantiated with a - :class:`~scrapy.settings.Settings` object. - - This class shouldn't be needed (since Scrapy is responsible of using it - accordingly) unless writing scripts that manually handle the crawling - process. See :ref:`run-from-script` for an example. - """ - +class CrawlerRunnerBase: @staticmethod def _get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol: """Get SpiderLoader instance from settings""" @@ -293,7 +283,6 @@ class CrawlerRunner: self.settings: Settings = settings self.spider_loader: SpiderLoaderProtocol = self._get_spider_loader(settings) self._crawlers: set[Crawler] = set() - self._active: set[Deferred[None]] = set() self.bootstrap_failed = False @property @@ -302,6 +291,57 @@ class CrawlerRunner: :meth:`crawl` and managed by this class.""" return self._crawlers + def create_crawler( + self, crawler_or_spidercls: type[Spider] | str | Crawler + ) -> Crawler: + """ + Return a :class:`~scrapy.crawler.Crawler` object. + + * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. + * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler + is constructed for it. + * If ``crawler_or_spidercls`` is a string, this function finds + a spider with this name in a Scrapy project (using spider loader), + then creates a Crawler instance for it. + """ + if isinstance(crawler_or_spidercls, Spider): + raise ValueError( + "The crawler_or_spidercls argument cannot be a spider object, " + "it must be a spider class (or a Crawler object)" + ) + if isinstance(crawler_or_spidercls, Crawler): + return crawler_or_spidercls + return self._create_crawler(crawler_or_spidercls) + + def _create_crawler(self, spidercls: str | type[Spider]) -> Crawler: + if isinstance(spidercls, str): + spidercls = self.spider_loader.load(spidercls) + return Crawler(spidercls, self.settings) + + def _stop(self) -> Deferred[Any]: + return DeferredList([c.stop() for c in list(self.crawlers)]) + + +class CrawlerRunner(CrawlerRunnerBase): + """ + This is a convenient helper class that keeps track of, manages and runs + crawlers inside an already setup :mod:`~twisted.internet.reactor`. + + The CrawlerRunner object must be instantiated with a + :class:`~scrapy.settings.Settings` object. + + This class shouldn't be needed (since Scrapy is responsible of using it + accordingly) unless writing scripts that manually handle the crawling + process. See :ref:`run-from-script` for an example. + + This class provides Deferred-based APIs. Use :class:`AsyncCrawlerRunner` + for modern coroutine APIs. + """ + + def __init__(self, settings: dict[str, Any] | Settings | None = None): + super().__init__(settings) + self._active: set[Deferred[None]] = set() + def crawl( self, crawler_or_spidercls: type[Spider] | str | Crawler, @@ -351,40 +391,13 @@ class CrawlerRunner: self._active.discard(d) self.bootstrap_failed |= not getattr(crawler, "spider", None) - def create_crawler( - self, crawler_or_spidercls: type[Spider] | str | Crawler - ) -> Crawler: - """ - Return a :class:`~scrapy.crawler.Crawler` object. - - * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. - * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler - is constructed for it. - * If ``crawler_or_spidercls`` is a string, this function finds - a spider with this name in a Scrapy project (using spider loader), - then creates a Crawler instance for it. - """ - if isinstance(crawler_or_spidercls, Spider): - raise ValueError( - "The crawler_or_spidercls argument cannot be a spider object, " - "it must be a spider class (or a Crawler object)" - ) - if isinstance(crawler_or_spidercls, Crawler): - return crawler_or_spidercls - return self._create_crawler(crawler_or_spidercls) - - def _create_crawler(self, spidercls: str | type[Spider]) -> Crawler: - if isinstance(spidercls, str): - spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings) - def stop(self) -> Deferred[Any]: """ Stops simultaneously all the crawling jobs taking place. Returns a deferred that is fired when they all have ended. """ - return DeferredList([c.stop() for c in list(self.crawlers)]) + return self._stop() @inlineCallbacks def join(self) -> Generator[Deferred[Any], Any, None]: @@ -398,6 +411,96 @@ class CrawlerRunner: yield DeferredList(self._active) +class AsyncCrawlerRunner(CrawlerRunnerBase): + """ + This is a convenient helper class that keeps track of, manages and runs + crawlers inside an already setup :mod:`~twisted.internet.reactor`. + + The AsyncCrawlerRunner object must be instantiated with a + :class:`~scrapy.settings.Settings` object. + + This class shouldn't be needed (since Scrapy is responsible of using it + accordingly) unless writing scripts that manually handle the crawling + process. See :ref:`run-from-script` for an example. + + This class provides coroutine APIs. It requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. + """ + + def __init__(self, settings: dict[str, Any] | Settings | None = None): + super().__init__(settings) + self._active: set[asyncio.Future[None]] = set() + + def crawl( + self, + crawler_or_spidercls: type[Spider] | str | Crawler, + *args: Any, + **kwargs: Any, + ) -> asyncio.Future[None]: + """ + Run a crawler with the provided arguments. + + It will call the given Crawler's :meth:`~Crawler.crawl` method, while + keeping track of it so it can be stopped later. + + If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler` + instance, this method will try to create one using this parameter as + the spider class given to it. + + Returns a :class:`~asyncio.Future` object which completes when the + crawling is finished. + + :param crawler_or_spidercls: already created crawler, or a spider class + or spider's name inside the project to create it + :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance, + :class:`~scrapy.spiders.Spider` subclass or string + + :param args: arguments to initialize the spider + + :param kwargs: keyword arguments to initialize the spider + """ + if isinstance(crawler_or_spidercls, Spider): + raise ValueError( + "The crawler_or_spidercls argument cannot be a spider object, " + "it must be a spider class (or a Crawler object)" + ) + if not is_asyncio_reactor_installed(): + raise RuntimeError("AsyncCrawlerRunner requires AsyncioSelectorReactor.") + crawler = self.create_crawler(crawler_or_spidercls) + return self._crawl(crawler, *args, **kwargs) + + def _crawl( + self, crawler: Crawler, *args: Any, **kwargs: Any + ) -> asyncio.Future[None]: + self.crawlers.add(crawler) + future = deferred_to_future(crawler.crawl(*args, **kwargs)) + self._active.add(future) + + def _done(_: asyncio.Future[None]) -> None: + self.crawlers.discard(crawler) + self._active.discard(future) + self.bootstrap_failed |= not getattr(crawler, "spider", None) + + future.add_done_callback(_done) + return future + + async def stop(self) -> None: + """ + Stops simultaneously all the crawling jobs taking place. + + Completes when they all have ended. + """ + await deferred_to_future(self._stop()) + + async def join(self) -> None: + """ + Completes when all managed :attr:`crawlers` have completed their + executions. + """ + while self._active: + await asyncio.gather(*self._active) + + class CrawlerProcess(CrawlerRunner): """ A class to run multiple scrapy crawlers in a process simultaneously. @@ -458,7 +561,6 @@ class CrawlerProcess(CrawlerRunner): spidercls = self.spider_loader.load(spidercls) init_reactor = not self._initialized_reactor self._initialized_reactor = True - # temporary cast until self.spider_loader is typed return Crawler(spidercls, self.settings, init_reactor=init_reactor) def start( diff --git a/tests/AsyncCrawlerRunner/multi_parallel.py b/tests/AsyncCrawlerRunner/multi_parallel.py new file mode 100644 index 000000000..f1af9f794 --- /dev/null +++ b/tests/AsyncCrawlerRunner/multi_parallel.py @@ -0,0 +1,28 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + runner.crawl(NoRequestsSpider) + runner.crawl(NoRequestsSpider) + await runner.join() + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/AsyncCrawlerRunner/multi_seq.py b/tests/AsyncCrawlerRunner/multi_seq.py new file mode 100644 index 000000000..987f7a514 --- /dev/null +++ b/tests/AsyncCrawlerRunner/multi_seq.py @@ -0,0 +1,27 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/AsyncCrawlerRunner/simple.py b/tests/AsyncCrawlerRunner/simple.py new file mode 100644 index 000000000..140777b4f --- /dev/null +++ b/tests/AsyncCrawlerRunner/simple.py @@ -0,0 +1,26 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/AsyncCrawlerRunner/simple_default_reactor.py b/tests/AsyncCrawlerRunner/simple_default_reactor.py new file mode 100644 index 000000000..ae052f188 --- /dev/null +++ b/tests/AsyncCrawlerRunner/simple_default_reactor.py @@ -0,0 +1,24 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +react(main) diff --git a/tests/CrawlerRunner/explicit_default_reactor.py b/tests/CrawlerRunner/explicit_default_reactor.py new file mode 100644 index 000000000..9eb8a39bb --- /dev/null +++ b/tests/CrawlerRunner/explicit_default_reactor.py @@ -0,0 +1,28 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": None, + } + + async def start(self): + return + yield + + +def main(reactor): + configure_logging( + {"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"} + ) + runner = CrawlerRunner() + return runner.crawl(NoRequestsSpider) + + +react(main) diff --git a/tests/CrawlerRunner/multi_parallel.py b/tests/CrawlerRunner/multi_parallel.py new file mode 100644 index 000000000..51feccd0a --- /dev/null +++ b/tests/CrawlerRunner/multi_parallel.py @@ -0,0 +1,26 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +def main(reactor): + configure_logging() + runner = CrawlerRunner() + runner.crawl(NoRequestsSpider) + runner.crawl(NoRequestsSpider) + return runner.join() + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/CrawlerRunner/multi_seq.py b/tests/CrawlerRunner/multi_seq.py new file mode 100644 index 000000000..f6549be9b --- /dev/null +++ b/tests/CrawlerRunner/multi_seq.py @@ -0,0 +1,27 @@ +from twisted.internet.defer import inlineCallbacks +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@inlineCallbacks +def main(reactor): + configure_logging() + runner = CrawlerRunner() + yield runner.crawl(NoRequestsSpider) + yield runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/CrawlerRunner/simple.py b/tests/CrawlerRunner/simple.py new file mode 100644 index 000000000..d154dcde4 --- /dev/null +++ b/tests/CrawlerRunner/simple.py @@ -0,0 +1,24 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +def main(reactor): + configure_logging() + runner = CrawlerRunner() + return runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 7a3d562e5..a1d3c02fb 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -18,11 +18,12 @@ from zope.interface.exceptions import MultipleInvalid import scrapy from scrapy import Spider -from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner +from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerProcess, CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader +from scrapy.utils.defer import deferred_from_coro from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_reactor_settings @@ -558,6 +559,26 @@ class TestCrawlerRunner(TestBaseCrawler): self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") +class TestAsyncCrawlerRunner(TestBaseCrawler): + def test_spider_manager_verify_interface(self): + settings = Settings( + { + "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, + } + ) + with pytest.raises(MultipleInvalid): + AsyncCrawlerRunner(settings) + + def test_crawler_runner_accepts_dict(self): + runner = AsyncCrawlerRunner({"foo": "bar"}) + assert runner.settings["foo"] == "bar" + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") + + def test_crawler_runner_accepts_None(self): + runner = AsyncCrawlerRunner() + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") + + class TestCrawlerProcess(TestBaseCrawler): def test_crawler_process_accepts_dict(self): runner = CrawlerProcess({"foo": "bar"}) @@ -587,20 +608,25 @@ class NoRequestsSpider(scrapy.Spider): @pytest.mark.usefixtures("reactor_pytest") class TestCrawlerRunnerHasSpider(unittest.TestCase): - def _runner(self): + @staticmethod + def _runner(): return CrawlerRunner(get_reactor_settings()) + @staticmethod + def _crawl(runner, spider): + return runner.crawl(spider) + @inlineCallbacks def test_crawler_runner_bootstrap_successful(self): runner = self._runner() - yield runner.crawl(NoRequestsSpider) + yield self._crawl(runner, NoRequestsSpider) assert not runner.bootstrap_failed @inlineCallbacks def test_crawler_runner_bootstrap_successful_for_several(self): runner = self._runner() - yield runner.crawl(NoRequestsSpider) - yield runner.crawl(NoRequestsSpider) + yield self._crawl(runner, NoRequestsSpider) + yield self._crawl(runner, NoRequestsSpider) assert not runner.bootstrap_failed @inlineCallbacks @@ -608,7 +634,7 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): runner = self._runner() try: - yield runner.crawl(ExceptionSpider) + yield self._crawl(runner, ExceptionSpider) except ValueError: pass else: @@ -621,13 +647,13 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): runner = self._runner() try: - yield runner.crawl(ExceptionSpider) + yield self._crawl(runner, ExceptionSpider) except ValueError: pass else: pytest.fail("Exception should be raised from spider") - yield runner.crawl(NoRequestsSpider) + yield self._crawl(runner, NoRequestsSpider) assert runner.bootstrap_failed @@ -643,7 +669,7 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): Exception, match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)", ): - yield runner.crawl(NoRequestsSpider) + yield self._crawl(runner, NoRequestsSpider) else: CrawlerRunner( settings={ @@ -652,6 +678,20 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): ) +@pytest.mark.only_asyncio +class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider): + @staticmethod + def _runner(): + return AsyncCrawlerRunner(get_reactor_settings()) + + @staticmethod + def _crawl(runner, spider): + return deferred_from_coro(runner.crawl(spider)) + + def test_crawler_runner_asyncio_enabled_true(self): + pytest.skip("This test is only for CrawlerRunner") + + class ScriptRunnerMixin: script_dir: Path @@ -923,6 +963,48 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): script_dir = Path(__file__).parent.resolve() / "CrawlerRunner" + def test_simple(self): + log = self.run_script("simple.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_explicit_default_reactor(self): + log = self.run_script("explicit_default_reactor.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + + def test_multi_parallel(self): + log = self.run_script("multi_parallel.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Spider opened.+Closing spider.+Closing spider", + log, + re.DOTALL, + ) + + def test_multi_seq(self): + log = self.run_script("multi_seq.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Closing spider.+Spider opened.+Closing spider", + log, + re.DOTALL, + ) + def test_response_ip_address(self): log = self.run_script("ip_address.py") assert "INFO: Spider closed (finished)" in log @@ -939,6 +1021,49 @@ class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): assert "DEBUG: Using asyncio event loop" in log +class TestAsyncCrawlerRunnerSubprocess(ScriptRunnerMixin): + script_dir = Path(__file__).parent.resolve() / "AsyncCrawlerRunner" + + def test_simple(self): + log = self.run_script("simple.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_simple_default_reactor(self): + log = self.run_script("simple_default_reactor.py") + assert "Spider closed (finished)" not in log + assert "RuntimeError: AsyncCrawlerRunner requires AsyncioSelectorReactor" in log + + def test_multi_parallel(self): + log = self.run_script("multi_parallel.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Spider opened.+Closing spider.+Closing spider", + log, + re.DOTALL, + ) + + def test_multi_seq(self): + log = self.run_script("multi_seq.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Closing spider.+Spider opened.+Closing spider", + log, + re.DOTALL, + ) + + @pytest.mark.parametrize( ("settings", "items"), [ From 1ddcb568e27cda10db4d0640aa42d020d4624a30 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 15 May 2025 14:18:01 +0500 Subject: [PATCH 113/491] Add send_catch_log_async(). --- docs/topics/signals.rst | 22 +++++++++--------- scrapy/core/engine.py | 12 +++------- scrapy/core/scraper.py | 40 ++++++++++++++------------------- scrapy/extensions/feedexport.py | 4 +--- scrapy/signalmanager.py | 20 ++++++++++++++--- scrapy/utils/signal.py | 38 +++++++++++++++++++++---------- tests/test_utils_signal.py | 39 +++++++++++++++++++++++++++++++- 7 files changed, 113 insertions(+), 62 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 66cb87fc5..59742ffeb 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -46,8 +46,8 @@ Here is a simple example showing how you can catch signals and perform some acti .. _signal-deferred: -Deferred signal handlers -======================== +Asynchronous signal handlers +============================ Some signals support returning :class:`~twisted.internet.defer.Deferred` or :term:`awaitable objects <awaitable>` from their handlers, allowing @@ -114,7 +114,7 @@ engine_started Sent when the Scrapy engine has started crawling. - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. .. note:: This signal may be fired *after* the :signal:`spider_opened` signal, depending on how the spider was started. So **don't** rely on this signal @@ -129,7 +129,7 @@ engine_stopped Sent when the Scrapy engine is stopped (for example, when a crawling process has finished). - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. scheduler_empty ~~~~~~~~~~~~~~~ @@ -164,7 +164,7 @@ item_scraped Sent when an item has been scraped, after it has passed all the :ref:`topics-item-pipeline` stages (without being dropped). - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. :param item: the scraped item :type item: :ref:`item object <item-types>` @@ -185,7 +185,7 @@ item_dropped Sent after an item has been dropped from the :ref:`topics-item-pipeline` when some stage raised a :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. :param item: the item dropped from the :ref:`topics-item-pipeline` :type item: :ref:`item object <item-types>` @@ -211,7 +211,7 @@ item_error Sent when a :ref:`topics-item-pipeline` generates an error (i.e. raises an exception), except :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. :param item: the item that caused the error in the :ref:`topics-item-pipeline` :type item: :ref:`item object <item-types>` @@ -239,7 +239,7 @@ spider_closed Sent after a spider has been closed. This can be used to release per-spider resources reserved on :signal:`spider_opened`. - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. :param spider: the spider which has been closed :type spider: :class:`~scrapy.Spider` object @@ -263,7 +263,7 @@ spider_opened reserve per-spider resources, but can be used for any task that needs to be performed when a spider is opened. - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. :param spider: the spider which has been opened :type spider: :class:`~scrapy.Spider` object @@ -332,7 +332,7 @@ feed_slot_closed Sent when a :ref:`feed exports <topics-feed-exports>` slot is closed. - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. :param slot: the slot closed :type slot: scrapy.extensions.feedexport.FeedSlot @@ -348,7 +348,7 @@ feed_exporter_closed during the handling of the :signal:`spider_closed` signal by the extension, after all feed exporting has been handled. - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. Request signals diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 658f6e774..b0d9a5452 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -127,9 +127,7 @@ class ExecutionEngine: if self.running: raise RuntimeError("Engine already running") self.start_time = time() - await maybe_deferred_to_future( - self.signals.send_catch_log_deferred(signal=signals.engine_started) - ) + await self.signals.send_catch_log_async(signal=signals.engine_started) self.running = True self._closewait: Deferred[None] = Deferred() if _start_request_processing: @@ -141,9 +139,7 @@ class ExecutionEngine: @deferred_f_from_coro_f async def _finish_stopping_engine(_: Any) -> None: - await maybe_deferred_to_future( - self.signals.send_catch_log_deferred(signal=signals.engine_stopped) - ) + await self.signals.send_catch_log_async(signal=signals.engine_stopped) self._closewait.callback(None) if not self.running: @@ -415,9 +411,7 @@ class ExecutionEngine: await maybe_deferred_to_future(self.scraper.open_spider(spider)) assert self.crawler.stats self.crawler.stats.open_spider(spider) - await maybe_deferred_to_future( - self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) - ) + await self.signals.send_catch_log_async(signals.spider_opened, spider=spider) def _spider_idle(self) -> None: """ diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 2c48a9a81..9fc1d20ed 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -392,14 +392,12 @@ class Scraper: logger.log( *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} ) - await maybe_deferred_to_future( - self.signals.send_catch_log_deferred( - signal=signals.item_dropped, - item=item, - response=response, - spider=self.crawler.spider, - exception=ex, - ) + await self.signals.send_catch_log_async( + signal=signals.item_dropped, + item=item, + response=response, + spider=self.crawler.spider, + exception=ex, ) except Exception as ex: logkws = self.logformatter.item_error( @@ -410,14 +408,12 @@ class Scraper: extra={"spider": self.crawler.spider}, exc_info=True, ) - await maybe_deferred_to_future( - self.signals.send_catch_log_deferred( - signal=signals.item_error, - item=item, - response=response, - spider=self.crawler.spider, - failure=Failure(), - ) + await self.signals.send_catch_log_async( + signal=signals.item_error, + item=item, + response=response, + spider=self.crawler.spider, + failure=Failure(), ) else: logkws = self.logformatter.scraped(output, response, self.crawler.spider) @@ -425,13 +421,11 @@ class Scraper: logger.log( *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} ) - await maybe_deferred_to_future( - self.signals.send_catch_log_deferred( - signal=signals.item_scraped, - item=output, - response=response, - spider=self.crawler.spider, - ) + await self.signals.send_catch_log_async( + signal=signals.item_scraped, + item=output, + response=response, + spider=self.crawler.spider, ) finally: self.slot.itemproc_size -= 1 diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 8bcd4e40d..c39a9c92e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -531,9 +531,7 @@ class FeedExporter: await maybe_deferred_to_future(DeferredList(self._pending_deferreds)) # Send FEED_EXPORTER_CLOSED signal - await maybe_deferred_to_future( - self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed) - ) + await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed) def _close_slot(self, slot: FeedSlot, spider: Spider) -> Deferred[None] | None: def get_file(slot_: FeedSlot) -> IO[bytes]: diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index f8c50b5e3..7fd172535 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -53,11 +53,10 @@ class SignalManager: self, signal: Any, **kwargs: Any ) -> Deferred[list[tuple[Any, Any]]]: """ - Like :meth:`send_catch_log` but supports returning - :class:`~twisted.internet.defer.Deferred` objects from signal handlers. + Like :meth:`send_catch_log` but supports asynchronous signal handlers. Returns a Deferred that gets fired once all signal handlers - deferreds were fired. Send a signal, catch exceptions and log them. + have finished. Send a signal, catch exceptions and log them. The keyword arguments are passed to the signal handlers (connected through the :meth:`connect` method). @@ -65,6 +64,21 @@ class SignalManager: kwargs.setdefault("sender", self.sender) return _signal.send_catch_log_deferred(signal, **kwargs) + async def send_catch_log_async( + self, signal: Any, **kwargs: Any + ) -> list[tuple[Any, Any]]: + """ + Like :meth:`send_catch_log` but supports asynchronous signal handlers. + + Returns a coroutine that completes once all signal handlers + have finished. Send a signal, catch exceptions and log them. + + The keyword arguments are passed to the signal handlers (connected + through the :meth:`connect` method). + """ + kwargs.setdefault("sender", self.sender) + return await _signal.send_catch_log_async(signal, **kwargs) + def disconnect_all(self, signal: Any, **kwargs: Any) -> None: """ Disconnect all receivers from the given signal. diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 5fd176a3f..d6b0a671b 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging -from collections.abc import Sequence +from collections.abc import Generator, Sequence from typing import Any as TypingAny from pydispatch.dispatcher import ( @@ -14,11 +14,11 @@ from pydispatch.dispatcher import ( liveReceivers, ) from pydispatch.robustapply import robustApply -from twisted.internet.defer import Deferred, DeferredList +from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks from twisted.python.failure import Failure from scrapy.exceptions import StopDownload -from scrapy.utils.defer import maybeDeferred_coro +from scrapy.utils.defer import maybe_deferred_to_future, maybeDeferred_coro from scrapy.utils.log import failure_to_exc_info logger = logging.getLogger(__name__) @@ -66,18 +66,19 @@ def send_catch_log( return responses +@inlineCallbacks def send_catch_log_deferred( signal: TypingAny = Any, sender: TypingAny = Anonymous, *arguments: TypingAny, **named: TypingAny, -) -> Deferred[list[tuple[TypingAny, TypingAny]]]: - """Like send_catch_log but supports returning deferreds on signal handlers. - Returns a deferred that gets fired once all signal handlers deferreds were - fired. +) -> Generator[Deferred[TypingAny], TypingAny, list[tuple[TypingAny, TypingAny]]]: + """Like send_catch_log but supports asynchronous signal handlers. + + Returns a deferred that gets fired once all signal handlers have finished. """ - def logerror(failure: Failure, recv: Any) -> Failure: + def logerror(failure: Failure, recv: TypingAny) -> Failure: if dont_log is None or not isinstance(failure.value, dont_log): logger.error( "Error caught on signal handler: %(receiver)s", @@ -103,11 +104,24 @@ def send_catch_log_deferred( ) ) dfds.append(d2) - dl = DeferredList(dfds) - d3: Deferred[list[tuple[TypingAny, TypingAny]]] = dl.addCallback( - lambda out: [x[1] for x in out] + + results = yield DeferredList(dfds) + return [result[1] for result in results] + + +async def send_catch_log_async( + signal: TypingAny = Any, + sender: TypingAny = Anonymous, + *arguments: TypingAny, + **named: TypingAny, +) -> list[tuple[TypingAny, TypingAny]]: + """Like send_catch_log but supports asynchronous signal handlers. + + Returns a coroutine that completes once all signal handlers have finished. + """ + return await maybe_deferred_to_future( + send_catch_log_deferred(signal, sender, *arguments, **named) ) - return d3 def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None: diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 751a77031..6dff321da 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -7,7 +7,12 @@ from twisted.internet import defer, reactor from twisted.python.failure import Failure from twisted.trial import unittest -from scrapy.utils.signal import send_catch_log, send_catch_log_deferred +from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.signal import ( + send_catch_log, + send_catch_log_async, + send_catch_log_deferred, +) from scrapy.utils.test import get_from_asyncio_queue @@ -85,6 +90,38 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): return await get_from_asyncio_queue("OK") +class SendCatchLogAsyncTest(TestSendCatchLog): + def _get_result(self, signal, *a, **kw): + return deferred_from_coro(send_catch_log_async(signal, *a, **kw)) + + +class SendCatchLogAsyncTest2(SendCatchLogAsyncTest): + def ok_handler(self, arg, handlers_called): + handlers_called.add(self.ok_handler) + assert arg == "test" + d = defer.Deferred() + reactor.callLater(0, d.callback, "OK") + return d + + +@pytest.mark.usefixtures("reactor_pytest") +class SendCatchLogAsyncAsyncDefTest(SendCatchLogAsyncTest): + async def ok_handler(self, arg, handlers_called): + handlers_called.add(self.ok_handler) + assert arg == "test" + await defer.succeed(42) + return "OK" + + +@pytest.mark.only_asyncio +class SendCatchLogAsyncAsyncioTest(SendCatchLogAsyncTest): + async def ok_handler(self, arg, handlers_called): + handlers_called.add(self.ok_handler) + assert arg == "test" + await asyncio.sleep(0.2) + return await get_from_asyncio_queue("OK") + + class TestSendCatchLog2: def test_error_logged_if_deferred_not_supported(self): def test_handler(): From bf1bfaaa3e584b085f78e9f89ab03c22cfbb3e59 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 15 May 2025 20:02:38 +0500 Subject: [PATCH 114/491] Slight improvements for the signal docs. --- docs/topics/signals.rst | 66 ++++++++++++++++++++++------------------- scrapy/signalmanager.py | 6 ++-- scrapy/utils/signal.py | 10 ++++--- 3 files changed, 46 insertions(+), 36 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 59742ffeb..a815ffb43 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -60,6 +60,7 @@ Let's take an example using :ref:`coroutines <topics-coroutines>`: .. code-block:: python import scrapy + import treq class SignalSpider(scrapy.Spider): @@ -103,6 +104,7 @@ Built-in signals reference Here's the list of Scrapy built-in signals and their meaning. + Engine signals -------------- @@ -114,7 +116,7 @@ engine_started Sent when the Scrapy engine has started crawling. - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. .. note:: This signal may be fired *after* the :signal:`spider_opened` signal, depending on how the spider was started. So **don't** rely on this signal @@ -129,7 +131,7 @@ engine_stopped Sent when the Scrapy engine is stopped (for example, when a crawling process has finished). - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. scheduler_empty ~~~~~~~~~~~~~~~ @@ -144,6 +146,9 @@ scheduler_empty See :ref:`start-requests-lazy` for an example. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. + + Item signals ------------ @@ -164,7 +169,7 @@ item_scraped Sent when an item has been scraped, after it has passed all the :ref:`topics-item-pipeline` stages (without being dropped). - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. :param item: the scraped item :type item: :ref:`item object <item-types>` @@ -185,7 +190,7 @@ item_dropped Sent after an item has been dropped from the :ref:`topics-item-pipeline` when some stage raised a :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. :param item: the item dropped from the :ref:`topics-item-pipeline` :type item: :ref:`item object <item-types>` @@ -211,7 +216,7 @@ item_error Sent when a :ref:`topics-item-pipeline` generates an error (i.e. raises an exception), except :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. :param item: the item that caused the error in the :ref:`topics-item-pipeline` :type item: :ref:`item object <item-types>` @@ -227,6 +232,7 @@ item_error :param failure: the exception raised :type failure: twisted.python.failure.Failure + Spider signals -------------- @@ -239,7 +245,7 @@ spider_closed Sent after a spider has been closed. This can be used to release per-spider resources reserved on :signal:`spider_opened`. - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. :param spider: the spider which has been closed :type spider: :class:`~scrapy.Spider` object @@ -263,7 +269,7 @@ spider_opened reserve per-spider resources, but can be used for any task that needs to be performed when a spider is opened. - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. :param spider: the spider which has been opened :type spider: :class:`~scrapy.Spider` object @@ -294,16 +300,16 @@ spider_idle accordingly (e.g. setting it to 'too_few_results' instead of 'finished'). - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param spider: the spider which has gone idle :type spider: :class:`~scrapy.Spider` object -.. note:: Scheduling some requests in your :signal:`spider_idle` handler does - **not** guarantee that it can prevent the spider from being closed, - although it sometimes can. That's because the spider may still remain idle - if all the scheduled requests are rejected by the scheduler (e.g. filtered - due to duplication). + .. note:: Scheduling some requests in your :signal:`spider_idle` handler does + **not** guarantee that it can prevent the spider from being closed, + although it sometimes can. That's because the spider may still remain idle + if all the scheduled requests are rejected by the scheduler (e.g. filtered + due to duplication). spider_error ~~~~~~~~~~~~ @@ -313,7 +319,7 @@ spider_error Sent when a spider callback generates an error (i.e. raises an exception). - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param failure: the exception raised :type failure: twisted.python.failure.Failure @@ -332,12 +338,11 @@ feed_slot_closed Sent when a :ref:`feed exports <topics-feed-exports>` slot is closed. - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. :param slot: the slot closed :type slot: scrapy.extensions.feedexport.FeedSlot - feed_exporter_closed ~~~~~~~~~~~~~~~~~~~~ @@ -348,7 +353,7 @@ feed_exporter_closed during the handling of the :signal:`spider_closed` signal by the extension, after all feed exporting has been handled. - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. Request signals @@ -367,7 +372,7 @@ request_scheduled Raise :exc:`~scrapy.exceptions.IgnoreRequest` to drop a request before it reaches the scheduler. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. .. versionadded:: 2.11.2 Allow dropping requests with :exc:`~scrapy.exceptions.IgnoreRequest`. @@ -387,7 +392,7 @@ request_dropped Sent when a :class:`~scrapy.Request`, scheduled by the engine to be downloaded later, is rejected by the scheduler. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param request: the request that reached the scheduler :type request: :class:`~scrapy.Request` object @@ -403,7 +408,7 @@ request_reached_downloader Sent when a :class:`~scrapy.Request` reached downloader. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param request: the request that reached downloader :type request: :class:`~scrapy.Request` object @@ -422,7 +427,7 @@ request_left_downloader Sent when a :class:`~scrapy.Request` leaves the downloader, even in case of failure. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param request: the request that reached the downloader :type request: :class:`~scrapy.Request` object @@ -433,11 +438,11 @@ request_left_downloader bytes_received ~~~~~~~~~~~~~~ -.. versionadded:: 2.2 - .. signal:: bytes_received .. function:: bytes_received(data, request, spider) + .. versionadded:: 2.2 + Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is received for a specific request. This signal might be fired multiple times for the same request, with partial data each time. For instance, @@ -449,7 +454,7 @@ bytes_received exception. Please refer to the :ref:`topics-stop-response-download` topic for additional information and examples. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param data: the data received by the download handler :type data: :class:`bytes` object @@ -463,11 +468,11 @@ bytes_received headers_received ~~~~~~~~~~~~~~~~ -.. versionadded:: 2.5 - .. signal:: headers_received .. function:: headers_received(headers, body_length, request, spider) + .. versionadded:: 2.5 + Sent by the HTTP 1.1 and S3 download handlers when the response headers are available for a given request, before downloading any additional content. @@ -476,7 +481,7 @@ headers_received exception. Please refer to the :ref:`topics-stop-response-download` topic for additional information and examples. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param headers: the headers received by the download handler :type headers: :class:`scrapy.http.headers.Headers` object @@ -490,6 +495,7 @@ headers_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.Spider` object + Response signals ---------------- @@ -502,7 +508,7 @@ response_received Sent when the engine receives a new :class:`~scrapy.http.Response` from the downloader. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param response: the response received :type response: :class:`~scrapy.http.Response` object @@ -524,9 +530,9 @@ response_downloaded .. signal:: response_downloaded .. function:: response_downloaded(response, request, spider) - Sent by the downloader right after a ``HTTPResponse`` is downloaded. + Sent by the downloader right after a :class:`~scrapy.http.Response` is downloaded. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param response: the response downloaded :type response: :class:`~scrapy.http.Response` object diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 7fd172535..283060074 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -53,7 +53,8 @@ class SignalManager: self, signal: Any, **kwargs: Any ) -> Deferred[list[tuple[Any, Any]]]: """ - Like :meth:`send_catch_log` but supports asynchronous signal handlers. + Like :meth:`send_catch_log` but supports :ref:`asynchronous signal + handlers <signal-deferred>`. Returns a Deferred that gets fired once all signal handlers have finished. Send a signal, catch exceptions and log them. @@ -68,7 +69,8 @@ class SignalManager: self, signal: Any, **kwargs: Any ) -> list[tuple[Any, Any]]: """ - Like :meth:`send_catch_log` but supports asynchronous signal handlers. + Like :meth:`send_catch_log` but supports :ref:`asynchronous signal + handlers <signal-deferred>`. Returns a coroutine that completes once all signal handlers have finished. Send a signal, catch exceptions and log them. diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index d6b0a671b..552fbaa90 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -30,7 +30,7 @@ def send_catch_log( *arguments: TypingAny, **named: TypingAny, ) -> list[tuple[TypingAny, TypingAny]]: - """Like pydispatcher.robust.sendRobust but it also logs errors and returns + """Like ``pydispatcher.robust.sendRobust()`` but it also logs errors and returns Failures instead of exceptions. """ dont_log = named.pop("dont_log", ()) @@ -73,7 +73,8 @@ def send_catch_log_deferred( *arguments: TypingAny, **named: TypingAny, ) -> Generator[Deferred[TypingAny], TypingAny, list[tuple[TypingAny, TypingAny]]]: - """Like send_catch_log but supports asynchronous signal handlers. + """Like :func:`send_catch_log` but supports :ref:`asynchronous signal handlers + <signal-deferred>`. Returns a deferred that gets fired once all signal handlers have finished. """ @@ -115,7 +116,8 @@ async def send_catch_log_async( *arguments: TypingAny, **named: TypingAny, ) -> list[tuple[TypingAny, TypingAny]]: - """Like send_catch_log but supports asynchronous signal handlers. + """Like :func:`send_catch_log` but supports :ref:`asynchronous signal handlers + <signal-deferred>`. Returns a coroutine that completes once all signal handlers have finished. """ @@ -126,7 +128,7 @@ async def send_catch_log_async( def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None: """Disconnect all signal handlers. Useful for cleaning up after running - tests + tests. """ for receiver in liveReceivers(getAllReceivers(sender, signal)): disconnect(receiver, signal=signal, sender=sender) From 3c2cd53abb0651f3ac093e71ad340626f72ad0a3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 15 May 2025 22:17:37 +0500 Subject: [PATCH 115/491] Skip the doctest. --- docs/topics/signals.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index a815ffb43..aa27e62dd 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -57,6 +57,7 @@ operation to finish. Let's take an example using :ref:`coroutines <topics-coroutines>`: +.. skip: next .. code-block:: python import scrapy From b9caaf8a63bc3280645dca2788a8c4ed1a556769 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 16 May 2025 17:13:52 +0500 Subject: [PATCH 116/491] Simplify deferred_from_coro(), add more tests. --- scrapy/utils/defer.py | 34 +++--- tests/test_utils_defer.py | 219 ++++++++++++++++++++++++++++++++++---- 2 files changed, 212 insertions(+), 41 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 6e1687f3e..d06397f50 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -10,14 +10,13 @@ import warnings from asyncio import Future from collections.abc import Awaitable, Coroutine, Iterable, Iterator from functools import wraps -from types import CoroutineType -from typing import TYPE_CHECKING, Any, Generic, TypeVar, Union, cast, overload +from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast, overload -from twisted.internet import defer from twisted.internet.defer import ( Deferred, DeferredList, - ensureDeferred, + fail, + succeed, ) from twisted.internet.task import Cooperator from twisted.python import failure @@ -315,7 +314,7 @@ def process_parallel( """Return a Deferred with the output of all successful calls to the given callbacks """ - dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] + dfds = [succeed(input).addCallback(x, *a, **kw) for x in callbacks] d: Deferred[list[tuple[bool, _T2]]] = DeferredList( dfds, fireOnOneErrback=True, consumeErrors=True ) @@ -366,27 +365,24 @@ async def aiter_errback( errback(failure.Failure(), *a, **kw) -_CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future]) +@overload +def deferred_from_coro(o: Awaitable[_T]) -> Deferred[_T]: ... @overload -def deferred_from_coro(o: _CT) -> Deferred: ... +def deferred_from_coro(o: _T2) -> _T2: ... -@overload -def deferred_from_coro(o: _T) -> _T: ... - - -def deferred_from_coro(o: _T) -> Deferred | _T: +def deferred_from_coro(o: Awaitable[_T] | _T2) -> Deferred[_T] | _T2: """Converts a coroutine or other awaitable object into a Deferred, or returns the object as is if it isn't a coroutine.""" if isinstance(o, Deferred): return o - if asyncio.isfuture(o) or inspect.isawaitable(o): + if inspect.isawaitable(o): if not is_asyncio_reactor_installed(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" - return ensureDeferred(cast(Coroutine[Deferred, Any, Any], o)) + return Deferred.fromCoroutine(cast(Coroutine[Deferred[Any], Any, _T], o)) # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor event_loop = _get_asyncio_event_loop() return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) @@ -394,7 +390,7 @@ def deferred_from_coro(o: _T) -> Deferred | _T: def deferred_f_from_coro_f( - coro_f: Callable[_P, Coroutine[Any, Any, _T]], + coro_f: Callable[_P, Awaitable[_T]], ) -> Callable[_P, Deferred[_T]]: """Converts a coroutine function into a function that returns a Deferred. @@ -403,7 +399,7 @@ def deferred_f_from_coro_f( """ @wraps(coro_f) - def f(*coro_args: _P.args, **coro_kwargs: _P.kwargs) -> Any: + def f(*coro_args: _P.args, **coro_kwargs: _P.kwargs) -> Deferred[_T]: return deferred_from_coro(coro_f(*coro_args, **coro_kwargs)) return f @@ -416,15 +412,15 @@ def maybeDeferred_coro( try: result = f(*args, **kw) except: # noqa: E722 # pylint: disable=bare-except - return defer.fail(failure.Failure(captureVars=Deferred.debug)) + return fail(failure.Failure(captureVars=Deferred.debug)) if isinstance(result, Deferred): return result if asyncio.isfuture(result) or inspect.isawaitable(result): return deferred_from_coro(result) if isinstance(result, failure.Failure): - return defer.fail(result) - return defer.succeed(result) + return fail(result) + return succeed(result) def deferred_to_future(d: Deferred[_T]) -> Future[_T]: diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 36bd8ced9..29cd5fbf2 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -1,7 +1,12 @@ +from __future__ import annotations + +import asyncio import random +from asyncio import Future +from typing import TYPE_CHECKING, Any import pytest -from twisted.internet import defer, reactor +from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.python.failure import Failure from twisted.trial import unittest @@ -9,6 +14,8 @@ from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import ( aiter_errback, deferred_f_from_coro_f, + deferred_from_coro, + deferred_to_future, iter_errback, maybe_deferred_to_future, mustbe_deferred, @@ -17,12 +24,15 @@ from scrapy.utils.defer import ( process_parallel, ) +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Awaitable, Callable, Generator + class TestMustbeDeferred(unittest.TestCase): - def test_success_function(self): - steps = [] + def test_success_function(self) -> Deferred[list[int]]: + steps: list[int] = [] - def _append(v): + def _append(v: int) -> list[int]: steps.append(v) return steps @@ -31,12 +41,14 @@ class TestMustbeDeferred(unittest.TestCase): steps.append(2) # add another value, that should be caught by assertEqual return dfd - def test_unfired_deferred(self): - steps = [] + def test_unfired_deferred(self) -> Deferred[list[int]]: + steps: list[int] = [] + + def _append(v: int) -> Deferred[list[int]]: + from twisted.internet import reactor - def _append(v): steps.append(v) - dfd = defer.Deferred() + dfd: Deferred[list[int]] = Deferred() reactor.callLater(0, dfd.callback, steps) return dfd @@ -51,7 +63,7 @@ def cb1(value, arg1, arg2): def cb2(value, arg1, arg2): - return defer.succeed(f"(cb2 {value} {arg1} {arg2})") + return succeed(f"(cb2 {value} {arg1} {arg2})") def cb3(value, arg1, arg2): @@ -67,7 +79,7 @@ def eb1(failure, arg1, arg2): class TestDeferUtils(unittest.TestCase): - @defer.inlineCallbacks + @inlineCallbacks def test_process_chain(self): x = yield process_chain([cb1, cb2, cb3], "res", "v1", "v2") assert x == "(cb3 (cb2 (cb1 res v1 v2) v1 v2) v1 v2)" @@ -75,7 +87,7 @@ class TestDeferUtils(unittest.TestCase): with pytest.raises(TypeError): yield process_chain([cb1, cb_fail, cb3], "res", "v1", "v2") - @defer.inlineCallbacks + @inlineCallbacks def test_process_parallel(self): x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2") assert x == ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"] @@ -88,7 +100,7 @@ class TestDeferUtils(unittest.TestCase): class TestIterErrback: def test_iter_errback_good(self): - def itergood(): + def itergood() -> Generator[int, None, None]: yield from range(10) errors = [] @@ -97,7 +109,7 @@ class TestIterErrback: assert not errors def test_iter_errback_bad(self): - def iterbad(): + def iterbad() -> Generator[int, None, None]: for x in range(10): if x == 5: 1 / 0 @@ -113,7 +125,7 @@ class TestIterErrback: class TestAiterErrback(unittest.TestCase): @deferred_f_from_coro_f async def test_aiter_errback_good(self): - async def itergood(): + async def itergood() -> AsyncGenerator[int, None]: for x in range(10): yield x @@ -124,7 +136,7 @@ class TestAiterErrback(unittest.TestCase): @deferred_f_from_coro_f async def test_iter_errback_bad(self): - async def iterbad(): + async def iterbad() -> AsyncGenerator[int, None]: for x in range(10): if x == 5: 1 / 0 @@ -168,10 +180,12 @@ class TestAsyncCooperator(unittest.TestCase): CONCURRENT_ITEMS = 50 @staticmethod - def callable(o, results): + def callable(o: int, results: list[int]) -> Deferred[None] | None: + from twisted.internet import reactor + if random.random() < 0.4: # simulate async processing - dfd = defer.Deferred() + dfd: Deferred[None] = Deferred() dfd.addCallback(lambda _: results.append(o)) delay = random.random() / 8 reactor.callLater(delay, dfd.callback, None) @@ -181,22 +195,24 @@ class TestAsyncCooperator(unittest.TestCase): return None @staticmethod - def get_async_iterable(length): + def get_async_iterable(length: int) -> AsyncGenerator[int, None]: # simulate a simple callback without delays between results return as_async_generator(range(length)) @staticmethod - async def get_async_iterable_with_delays(length): + async def get_async_iterable_with_delays(length: int) -> AsyncGenerator[int, None]: # simulate a callback with delays between some of the results + from twisted.internet import reactor + for i in range(length): if random.random() < 0.1: - dfd = defer.Deferred() + dfd: Deferred[None] = Deferred() delay = random.random() / 20 reactor.callLater(delay, dfd.callback, None) await maybe_deferred_to_future(dfd) yield i - @defer.inlineCallbacks + @inlineCallbacks def test_simple(self): for length in [20, 50, 100]: results = [] @@ -205,7 +221,7 @@ class TestAsyncCooperator(unittest.TestCase): yield dl assert list(range(length)) == sorted(results) - @defer.inlineCallbacks + @inlineCallbacks def test_delays(self): for length in [20, 50, 100]: results = [] @@ -213,3 +229,162 @@ class TestAsyncCooperator(unittest.TestCase): dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results) yield dl assert list(range(length)) == sorted(results) + + +class TestDeferredFromCoro(unittest.TestCase): + def test_deferred(self): + d = Deferred() + result = deferred_from_coro(d) + assert isinstance(result, Deferred) + assert result is d + + def test_object(self): + result = deferred_from_coro(42) + assert result == 42 + + @inlineCallbacks + def test_coroutine(self): + async def coroutine() -> int: + return 42 + + result = deferred_from_coro(coroutine()) + assert isinstance(result, Deferred) + coro_result = yield result + assert coro_result == 42 + + @pytest.mark.only_asyncio + @inlineCallbacks + def test_coroutine_asyncio(self): + async def coroutine() -> int: + await asyncio.sleep(0) + return 42 + + result = deferred_from_coro(coroutine()) + assert isinstance(result, Deferred) + coro_result = yield result + assert coro_result == 42 + + @pytest.mark.only_asyncio + @inlineCallbacks + def test_future(self): + future = Future() + result = deferred_from_coro(future) + assert isinstance(result, Deferred) + future.set_result(42) + future_result = yield result + assert future_result == 42 + + +class TestDeferredFFromCoroF(unittest.TestCase): + @inlineCallbacks + def _assert_result( + self, c_f: Callable[[], Awaitable[int]] + ) -> Generator[Deferred[Any], Any, None]: + d_f = deferred_f_from_coro_f(c_f) + d = d_f() + assert isinstance(d, Deferred) + result = yield d + assert result == 42 + + @inlineCallbacks + def test_coroutine(self): + async def c_f() -> int: + return 42 + + yield self._assert_result(c_f) + + @inlineCallbacks + def test_coroutine_asyncio(self): + async def c_f() -> int: + return 42 + + yield self._assert_result(c_f) + + @pytest.mark.only_asyncio + @inlineCallbacks + def test_future(self): + def c_f() -> Future[int]: + f: Future[int] = Future() + f.set_result(42) + return f + + yield self._assert_result(c_f) + + +class TestDeferredToFuture(unittest.TestCase): + @deferred_f_from_coro_f + async def test_deferred(self): + d = Deferred() + result = deferred_to_future(d) + assert isinstance(result, Future) + d.callback(42) + future_result = await result + assert future_result == 42 + + @deferred_f_from_coro_f + async def test_wrapped_coroutine(self): + async def c_f() -> int: + return 42 + + d = deferred_from_coro(c_f()) + result = deferred_to_future(d) + assert isinstance(result, Future) + future_result = await result + assert future_result == 42 + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_wrapped_coroutine_asyncio(self): + async def c_f() -> int: + await asyncio.sleep(0) + return 42 + + d = deferred_from_coro(c_f()) + result = maybe_deferred_to_future(d) + assert isinstance(result, Future) + future_result = await result + assert future_result == 42 + + +@pytest.mark.only_asyncio +class TestMaybeDeferredToFutureAsyncio(unittest.TestCase): + @deferred_f_from_coro_f + async def test_deferred(self): + d = Deferred() + result = maybe_deferred_to_future(d) + assert isinstance(result, Future) + d.callback(42) + future_result = await result + assert future_result == 42 + + @deferred_f_from_coro_f + async def test_wrapped_coroutine(self): + async def c_f() -> int: + return 42 + + d = deferred_from_coro(c_f()) + result = maybe_deferred_to_future(d) + assert isinstance(result, Future) + future_result = await result + assert future_result == 42 + + @deferred_f_from_coro_f + async def test_wrapped_coroutine_asyncio(self): + async def c_f() -> int: + await asyncio.sleep(0) + return 42 + + d = deferred_from_coro(c_f()) + result = maybe_deferred_to_future(d) + assert isinstance(result, Future) + future_result = await result + assert future_result == 42 + + +@pytest.mark.only_not_asyncio +class TestMaybeDeferredToFutureNotAsyncio: + def test_deferred(self): + d = Deferred() + result = maybe_deferred_to_future(d) + assert isinstance(result, Deferred) + assert result is d From ff7d29654a975c12bfcf6d1b7719bd861ed6d8be Mon Sep 17 00:00:00 2001 From: Keval Sakhiya <37344767+kevalsakhiya@users.noreply.github.com> Date: Tue, 20 May 2025 11:21:31 +0530 Subject: [PATCH 117/491] Fix typo in documentation and code: 'needs_backoff' -> 'needs_backout' (#6815) Corrected the typo in the code and documentation where 'needs_backoff' was incorrectly used instead of 'needs_backout'. --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 891c4da05..8240d5d4b 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -388,7 +388,7 @@ its iteration whenever there are scheduled requests: async def start(self): async for item_or_request in super().start(): - if self.crawler.engine.needs_backoff(): + if self.crawler.engine.needs_backout(): await self.crawler.signals.wait_for(signals.scheduler_empty) yield item_or_request From f2fc177f1fb954480922301749cf00ee79eebb97 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 23 May 2025 14:06:33 +0500 Subject: [PATCH 118/491] Fix a wrong versionadded usage. (#6822) --- docs/topics/coroutines.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 00812ed7f..2c0df5e0f 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -21,7 +21,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be defined as an :term:`asynchronous generator`. - .. versionadded: 2.13 + .. versionadded:: 2.13 - :class:`~scrapy.Request` callbacks. From 816d23da306e9fce0e55a933002fa7737f06fa64 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 27 May 2025 00:31:28 +0500 Subject: [PATCH 119/491] Make the release notes work better on PyPI. (#6826) --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 85fba0f92..47707e061 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,8 +58,7 @@ Homepage = "https://scrapy.org/" Documentation = "https://docs.scrapy.org/" Source = "https://github.com/scrapy/scrapy" Tracker = "https://github.com/scrapy/scrapy/issues" -Changelog = "https://github.com/scrapy/scrapy/commits/master/" -releasenotes = "https://docs.scrapy.org/en/latest/news.html" +"Release notes" = "https://docs.scrapy.org/en/latest/news.html" [project.scripts] scrapy = "scrapy.cmdline:execute" From 9d92d16510b8c82d529abf1662bce3e16ee293ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 27 May 2025 08:32:24 +0200 Subject: [PATCH 120/491] Prioritize other requests over start requests --- scrapy/core/scheduler.py | 4 ++-- scrapy/pqueues.py | 30 +++++++++++++++++------------- tests/test_engine_loop.py | 20 ++++++++++---------- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 57d27b7cf..9ac447289 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -173,8 +173,8 @@ class Scheduler(BaseScheduler): :ref:`Start requests <start-requests>` are sent in the order they are yielded from :meth:`~scrapy.Spider.start`, and given the same - :attr:`~scrapy.http.Request.priority`, start requests take precedence over - other requests. + :attr:`~scrapy.http.Request.priority`, other requests take precedence over + start requests. You can set :setting:`SCHEDULER_START_MEMORY_QUEUE` and :setting:`SCHEDULER_START_DISK_QUEUE` to ``None`` to handle start requests diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index e6c6b8bf1..34b235d83 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -160,28 +160,32 @@ class ScrapyPriorityQueue: def pop(self) -> Request | None: while self.curprio is not None: - if self._start_queues: - try: - q = self._start_queues[self.curprio] - except KeyError: - pass - else: - m = q.pop() - if not q: - del self._start_queues[self.curprio] - q.close() - return m try: q = self.queues[self.curprio] except KeyError: - self._update_curprio() + pass else: m = q.pop() if not q: del self.queues[self.curprio] q.close() - self._update_curprio() + if not self._start_queues: + self._update_curprio() return m + if self._start_queues: + try: + q = self._start_queues[self.curprio] + except KeyError: + self._update_curprio() + else: + m = q.pop() + if not q: + del self._start_queues[self.curprio] + q.close() + self._update_curprio() + return m + else: + self._update_curprio() return None def _update_curprio(self) -> None: diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 90af10f0e..c7dbc82d4 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -189,9 +189,9 @@ class RequestSendOrderTestCase(TestCase): @deferred_f_from_coro_f async def test_default(self): - """By default, start requests take priority over callback requests and + """By default, callback requests take priority over start requests and are sent in order. Priority matters, but given the same priority, a - start request takes precedence.""" + callback request takes precedence.""" nums = [1, 2, 3, 4, 5, 6] response_seconds = 0 download_slots = 1 @@ -207,13 +207,13 @@ class RequestSendOrderTestCase(TestCase): yield _request(1) for request in ( - _request(4, priority=1), - _request(6), + _request(2, priority=1), + _request(5), ): spider.crawler.engine._slot.scheduler.enqueue_request(request) - yield _request(5) - yield _request(2, priority=1) + yield _request(6) yield _request(3, priority=1) + yield _request(4, priority=1) def parse(spider, response): return @@ -249,13 +249,13 @@ class RequestSendOrderTestCase(TestCase): yield _request(1) for request in ( - _request(4, priority=1), - _request(6), + _request(2, priority=1), + _request(5), ): spider.crawler.engine._slot.scheduler.enqueue_request(request) - yield _request(5) + yield _request(6) + yield _request(4, priority=1) yield _request(3, priority=1) - yield _request(2, priority=1) def parse(spider, response): return From 05529f3017a6327ba4553c2af422f2e5f02c7d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 27 May 2025 08:44:18 +0200 Subject: [PATCH 121/491] Release notes for Scrapy 2.13.1 --- docs/news.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index cf1c35893..eb5370b6e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,22 @@ Release notes ============= +.. _release-2.13.1: + +Scrapy 2.13.1 (unreleased) +-------------------------- + +- Give callback requests precedence over start requests when priority values + are the same. + + This makes changes from 2.13.0 to start request handling more intuitive and + backward compatible. For scenarios where all requests have the same + priorities, in 2.13.0 all start requests were sent before the first + callback request. In 2.13.1, same as in 2.12 and lower, start requests are + only sent when there are not enough pending callback requests to reach + concurrency limits. + + .. _release-2.13.0: Scrapy 2.13.0 (2025-05-08) From f28be27423d720a59dcd7194df26c935fcc7e416 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 13 May 2025 12:22:28 +0400 Subject: [PATCH 122/491] Add a deepwiki badge, update other badges. (#6793) --- README.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index cf7c6043c..29488d825 100644 --- a/README.rst +++ b/README.rst @@ -17,19 +17,14 @@ Scrapy :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu :alt: Ubuntu -.. .. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg - .. :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS - .. :alt: macOS - +.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS + :alt: macOS .. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows :alt: Windows -.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg - :target: https://pypi.org/pypi/Scrapy - :alt: Wheel Status - .. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg :target: https://codecov.io/github/scrapy/scrapy?branch=master :alt: Coverage report @@ -38,6 +33,10 @@ Scrapy :target: https://anaconda.org/conda-forge/scrapy :alt: Conda Version +.. image:: https://deepwiki.com/badge.svg + :target: https://deepwiki.com/scrapy/scrapy + :alt: Ask DeepWiki + Overview ======== From 43087fe1df5b3209bcc65dabd1831067d1a29711 Mon Sep 17 00:00:00 2001 From: Keval Sakhiya <37344767+kevalsakhiya@users.noreply.github.com> Date: Tue, 20 May 2025 11:21:31 +0530 Subject: [PATCH 123/491] Fix typo in documentation and code: 'needs_backoff' -> 'needs_backout' (#6815) Corrected the typo in the code and documentation where 'needs_backoff' was incorrectly used instead of 'needs_backout'. --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 891c4da05..8240d5d4b 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -388,7 +388,7 @@ its iteration whenever there are scheduled requests: async def start(self): async for item_or_request in super().start(): - if self.crawler.engine.needs_backoff(): + if self.crawler.engine.needs_backout(): await self.crawler.signals.wait_for(signals.scheduler_empty) yield item_or_request From 06dec081254e19950eb00fcb6979fe8b7c342ee8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 23 May 2025 14:06:33 +0500 Subject: [PATCH 124/491] Fix a wrong versionadded usage. (#6822) --- docs/topics/coroutines.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 00812ed7f..2c0df5e0f 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -21,7 +21,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be defined as an :term:`asynchronous generator`. - .. versionadded: 2.13 + .. versionadded:: 2.13 - :class:`~scrapy.Request` callbacks. From 597320856776c2b6e44fc3a45a97d6524c64d464 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 27 May 2025 00:31:28 +0500 Subject: [PATCH 125/491] Make the release notes work better on PyPI. (#6826) --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 85fba0f92..47707e061 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,8 +58,7 @@ Homepage = "https://scrapy.org/" Documentation = "https://docs.scrapy.org/" Source = "https://github.com/scrapy/scrapy" Tracker = "https://github.com/scrapy/scrapy/issues" -Changelog = "https://github.com/scrapy/scrapy/commits/master/" -releasenotes = "https://docs.scrapy.org/en/latest/news.html" +"Release notes" = "https://docs.scrapy.org/en/latest/news.html" [project.scripts] scrapy = "scrapy.cmdline:execute" From e3f82afaf1ab12ac8f5915dfb0b926391bc81f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 27 May 2025 10:01:00 +0200 Subject: [PATCH 126/491] Add a test for ScrapyPriorityQueue pop order --- tests/test_pqueues.py | 54 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index d5c710ed2..b65f1b7e7 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -7,6 +7,7 @@ from scrapy.http.request import Request from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue from scrapy.spiders import Spider from scrapy.squeues import FifoMemoryQueue +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.test import get_crawler from tests.test_scheduler import MockDownloader, MockEngine @@ -155,3 +156,56 @@ class TestDownloaderAwarePriorityQueue: assert self.queue.peek().url == req3.url assert self.queue.pop().url == req3.url assert self.queue.peek() is None + + +@pytest.mark.parametrize( + ("input", "output"), + [ + # By default, start requests are FIFO, other requests are LIFO. + ([{}, {}], [2, 1]), + ([{"start": True}, {"start": True}], [1, 2]), + # Priority matters. + ([{"priority": 1}, {"start": True}], [1, 2]), + ([{}, {"start": True, "priority": 1}], [2, 1]), + # For the same priority, start requests pop last. + ([{}, {"start": True}], [1, 2]), + ([{"start": True}, {}], [2, 1]), + ], +) +def test_pop_order(input, output): + def make_url(index): + return f"https://toscrape.com/{index}" + + def make_request(index, data): + meta = {} + if data.get("start", False): + meta["is_start_request"] = True + return Request( + url=make_url(index), + priority=data.get("priority", 0), + meta=meta, + ) + + input_requests = [ + make_request(index, data) for index, data in enumerate(input, start=1) + ] + expected_output_urls = [make_url(index) for index in output] + + crawler = get_crawler(Spider) + settings = crawler.settings + queue = build_from_crawler( + ScrapyPriorityQueue, + crawler, + downstream_queue_cls=load_object(settings["SCHEDULER_MEMORY_QUEUE"]), + key="", + start_queue_cls=load_object(settings["SCHEDULER_START_MEMORY_QUEUE"]), + ) + + for request in input_requests: + queue.push(request) + + actual_output_urls = [] + while request := queue.pop(): + actual_output_urls.append(request.url) + + assert actual_output_urls == expected_output_urls From b41aea4873319df15ccb9145a4940ff1702d123a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 27 May 2025 18:19:47 +0500 Subject: [PATCH 127/491] Restructure download handler tests. (#6821) * Restructure download handler tests. * Typo. * Use mixins to reduce boilerplate. --- .../test_downloader_handler_twisted_http10.py | 46 ++ .../test_downloader_handler_twisted_http11.py | 69 ++ ... test_downloader_handler_twisted_http2.py} | 108 ++- tests/test_downloader_handlers.py | 719 +----------------- tests/test_downloader_handlers_http_base.py | 698 +++++++++++++++++ 5 files changed, 862 insertions(+), 778 deletions(-) create mode 100644 tests/test_downloader_handler_twisted_http10.py create mode 100644 tests/test_downloader_handler_twisted_http11.py rename tests/{test_downloader_handlers_http2.py => test_downloader_handler_twisted_http2.py} (73%) create mode 100644 tests/test_downloader_handlers_http_base.py diff --git a/tests/test_downloader_handler_twisted_http10.py b/tests/test_downloader_handler_twisted_http10.py new file mode 100644 index 000000000..807c8c4cb --- /dev/null +++ b/tests/test_downloader_handler_twisted_http10.py @@ -0,0 +1,46 @@ +"""Tests for scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler +from scrapy.http import Request +from scrapy.spiders import Spider +from tests.test_downloader_handlers_http_base import TestHttpBase, TestHttpProxyBase + +if TYPE_CHECKING: + from scrapy.core.downloader.handlers import DownloadHandlerProtocol + + +class HTTP10DownloadHandlerMixin: + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP10DownloadHandler + + +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class TestHttp10(HTTP10DownloadHandlerMixin, TestHttpBase): + """HTTP 1.0 test case""" + + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "HTTP/1.0") + return d + + +class TestHttps10(TestHttp10): + scheme = "https" + + +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class TestHttp10Proxy(HTTP10DownloadHandlerMixin, TestHttpProxyBase): + def test_download_with_proxy_https_timeout(self): + pytest.skip("Not implemented") + + def test_download_with_proxy_without_http_scheme(self): + pytest.skip("Not implemented") diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py new file mode 100644 index 000000000..70f55e787 --- /dev/null +++ b/tests/test_downloader_handler_twisted_http11.py @@ -0,0 +1,69 @@ +"""Tests for scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler +from tests.test_downloader_handlers_http_base import ( + TestHttp11Base, + TestHttpMockServerBase, + TestHttpProxyBase, + TestHttps11Base, + TestHttpsCustomCiphersBase, + TestHttpsInvalidDNSIdBase, + TestHttpsInvalidDNSPatternBase, + TestHttpsWrongHostnameBase, + TestSimpleHttpsBase, +) + +if TYPE_CHECKING: + from scrapy.core.downloader.handlers import DownloadHandlerProtocol + + +class HTTP11DownloadHandlerMixin: + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP11DownloadHandler + + +class TestHttp11(HTTP11DownloadHandlerMixin, TestHttp11Base): + pass + + +class TestHttps11(HTTP11DownloadHandlerMixin, TestHttps11Base): + pass + + +class TestSimpleHttps(HTTP11DownloadHandlerMixin, TestSimpleHttpsBase): + pass + + +class Https11WrongHostnameTestCase( + HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase +): + pass + + +class Https11InvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): + pass + + +class Https11InvalidDNSPattern( + HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase +): + pass + + +class Https11CustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase): + pass + + +class TestHttp11MockServer(TestHttpMockServerBase): + @property + def settings_dict(self) -> dict[str, Any] | None: + return None # default handler settings + + +class TestHttp11Proxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase): + pass diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handler_twisted_http2.py similarity index 73% rename from tests/test_downloader_handlers_http2.py rename to tests/test_downloader_handler_twisted_http2.py index c74c09cbb..46322a747 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -1,4 +1,9 @@ +"""Tests for scrapy.core.downloader.handlers.http2.H2DownloadHandler.""" + +from __future__ import annotations + import json +from typing import TYPE_CHECKING, Any from unittest import mock import pytest @@ -8,60 +13,42 @@ from twisted.web import server from twisted.web.error import SchemeNotSupported from twisted.web.http import H2_ENABLED -from scrapy.core.downloader.handlers import DownloadHandlerProtocol from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler from tests.mockserver import ssl_context_factory -from tests.test_downloader_handlers import ( +from tests.test_downloader_handlers_http_base import ( + TestHttpMockServerBase, + TestHttpProxyBase, + TestHttps11Base, + TestHttpsCustomCiphersBase, + TestHttpsInvalidDNSIdBase, + TestHttpsInvalidDNSPatternBase, + TestHttpsWrongHostnameBase, UriResource, ) +if TYPE_CHECKING: + from scrapy.core.downloader.handlers import DownloadHandlerProtocol + + pytestmark = pytest.mark.skipif( not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled" ) -class BaseTestClasses: - # A hack to prevent tests from the imported classes to run here too. - # See https://stackoverflow.com/q/1323455/113586 for other ways. - from tests.test_downloader_handlers import ( - TestHttp11MockServer as TestHttp11MockServer, - ) - from tests.test_downloader_handlers import ( - TestHttp11Proxy as TestHttp11Proxy, - ) - from tests.test_downloader_handlers import ( - TestHttps11 as TestHttps11, - ) - from tests.test_downloader_handlers import ( - TestHttps11CustomCiphers as TestHttps11CustomCiphers, - ) - from tests.test_downloader_handlers import ( - TestHttps11InvalidDNSId as TestHttps11InvalidDNSId, - ) - from tests.test_downloader_handlers import ( - TestHttps11InvalidDNSPattern as TestHttps11InvalidDNSPattern, - ) - from tests.test_downloader_handlers import ( - TestHttps11WrongHostname as TestHttps11WrongHostname, - ) - - -def _get_dh() -> type[DownloadHandlerProtocol]: - from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - - return H2DownloadHandler - - -class TestHttps2(BaseTestClasses.TestHttps11): - scheme = "https" - HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" - +class H2DownloadHandlerMixin: @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() + # the import can fail when H2_ENABLED is False + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + + return H2DownloadHandler + + +class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): + HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" def test_protocol(self): request = Request(self.getURL("host"), method="GET") @@ -179,42 +166,37 @@ class TestHttps2(BaseTestClasses.TestHttps11): return d -class Https2WrongHostnameTestCase(BaseTestClasses.TestHttps11WrongHostname): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() +class Https2WrongHostnameTestCase(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): + pass -class Https2InvalidDNSId(BaseTestClasses.TestHttps11InvalidDNSId): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() +class Https2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): + pass -class Https2InvalidDNSPattern(BaseTestClasses.TestHttps11InvalidDNSPattern): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() +class Https2InvalidDNSPattern(H2DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase): + pass -class Https2CustomCiphers(BaseTestClasses.TestHttps11CustomCiphers): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() +class Https2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase): + pass -class Http2MockServerTestCase(BaseTestClasses.TestHttp11MockServer): +class Http2MockServerTestCase(TestHttpMockServerBase): """HTTP 2.0 test case with MockServer""" - settings_dict = { - "DOWNLOAD_HANDLERS": { - "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler" + @property + def settings_dict(self) -> dict[str, Any] | None: + return { + "DOWNLOAD_HANDLERS": { + "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler" + } } - } + is_secure = True -class Https2ProxyTestCase(BaseTestClasses.TestHttp11Proxy): +class Https2ProxyTestCase(H2DownloadHandlerMixin, TestHttpProxyBase): # only used for HTTPS tests keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" @@ -224,10 +206,6 @@ class Https2ProxyTestCase(BaseTestClasses.TestHttp11Proxy): expected_http_proxy_request_body = b"/" - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() - def setUp(self): site = server.Site(UriResource(), timeout=None) self.port = reactor.listenSSL( diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index bc18e76e1..fc6ac5aee 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,52 +1,35 @@ +"""Tests for DownloadHandlers and for specific non-HTTP download handlers.""" + from __future__ import annotations import contextlib import os import shutil import sys -from abc import ABC, abstractmethod from pathlib import Path from tempfile import mkdtemp, mkstemp -from unittest import SkipTest, mock +from unittest import mock import pytest -from testfixtures import LogCapture from twisted.cred import checkers, credentials, portal -from twisted.internet import defer, error, reactor +from twisted.internet import reactor from twisted.protocols.ftp import FTPFactory, FTPRealm -from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest -from twisted.web import resource, server, static, util -from twisted.web.client import ResponseFailed -from twisted.web.http import _DataLoss from w3lib.url import path_to_file_uri -from scrapy.core.downloader.handlers import DownloadHandlerProtocol, DownloadHandlers +from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler -from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler -from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.exceptions import NotConfigured -from scrapy.http import Headers, HtmlResponse, Request +from scrapy.http import HtmlResponse, Request from scrapy.http.response.text import TextResponse from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler -from tests import NON_EXISTING_RESOLVABLE -from tests.mockserver import ( - Echo, - ForeverTakingResource, - HostHeaderResource, - MockServer, - NoLengthResource, - PayloadResource, - ssl_context_factory, -) -from tests.spiders import SingleRequestSpider class DummyDH: @@ -137,696 +120,6 @@ class TestFile(unittest.TestCase): return self.assertFailure(d, OSError) -class ContentLengthHeaderResource(resource.Resource): - """ - A testing resource which renders itself as the value of the Content-Length - header from the request. - """ - - def render(self, request): - return request.requestHeaders.getRawHeaders(b"content-length")[0] - - -class ChunkedResource(resource.Resource): - def render(self, request): - def response(): - request.write(b"chunked ") - request.write(b"content\n") - request.finish() - - reactor.callLater(0, response) - return server.NOT_DONE_YET - - -class BrokenChunkedResource(resource.Resource): - def render(self, request): - def response(): - request.write(b"chunked ") - request.write(b"content\n") - # Disable terminating chunk on finish. - request.chunked = False - closeConnection(request) - - reactor.callLater(0, response) - return server.NOT_DONE_YET - - -class BrokenDownloadResource(resource.Resource): - def render(self, request): - def response(): - request.setHeader(b"Content-Length", b"20") - request.write(b"partial") - closeConnection(request) - - reactor.callLater(0, response) - return server.NOT_DONE_YET - - -def closeConnection(request): - # We have to force a disconnection for HTTP/1.1 clients. Otherwise - # client keeps the connection open waiting for more data. - request.channel.loseConnection() - request.finish() - - -class EmptyContentTypeHeaderResource(resource.Resource): - """ - A testing resource which renders itself as the value of request body - without content-type header in response. - """ - - def render(self, request): - request.setHeader("content-type", "") - return request.content.read() - - -class LargeChunkedFileResource(resource.Resource): - def render(self, request): - def response(): - for i in range(1024): - request.write(b"x" * 1024) - request.finish() - - reactor.callLater(0, response) - return server.NOT_DONE_YET - - -class DuplicateHeaderResource(resource.Resource): - def render(self, request): - request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"]) - return b"" - - -class TestHttp(unittest.TestCase, ABC): - scheme = "http" - - # only used for HTTPS tests - keyfile = "keys/localhost.key" - certfile = "keys/localhost.crt" - - @property - @abstractmethod - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - raise NotImplementedError - - def setUp(self): - self.tmpname = Path(mkdtemp()) - (self.tmpname / "file").write_bytes(b"0123456789") - r = static.File(str(self.tmpname)) - r.putChild(b"redirect", util.Redirect(b"/file")) - r.putChild(b"wait", ForeverTakingResource()) - r.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) - r.putChild(b"nolength", NoLengthResource()) - r.putChild(b"host", HostHeaderResource()) - r.putChild(b"payload", PayloadResource()) - r.putChild(b"broken", BrokenDownloadResource()) - r.putChild(b"chunked", ChunkedResource()) - r.putChild(b"broken-chunked", BrokenChunkedResource()) - r.putChild(b"contentlength", ContentLengthHeaderResource()) - r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) - r.putChild(b"largechunkedfile", LargeChunkedFileResource()) - r.putChild(b"duplicate-header", DuplicateHeaderResource()) - r.putChild(b"echo", Echo()) - self.site = server.Site(r, timeout=None) - self.wrapper = WrappingFactory(self.site) - self.host = "localhost" - if self.scheme == "https": - # Using WrappingFactory do not enable HTTP/2 failing all the - # tests with H2DownloadHandler - self.port = reactor.listenSSL( - 0, - self.site, - ssl_context_factory(self.keyfile, self.certfile), - interface=self.host, - ) - else: - self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) - self.portno = self.port.getHost().port - self.download_handler = build_from_crawler( - self.download_handler_cls, get_crawler() - ) - self.download_request = self.download_handler.download_request - - @defer.inlineCallbacks - def tearDown(self): - yield self.port.stopListening() - if hasattr(self.download_handler, "close"): - yield self.download_handler.close() - shutil.rmtree(self.tmpname) - - def getURL(self, path): - return f"{self.scheme}://{self.host}:{self.portno}/{path}" - - def test_download(self): - request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d - - def test_download_head(self): - request = Request(self.getURL("file"), method="HEAD") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"") - return d - - def test_redirect_status(self): - request = Request(self.getURL("redirect")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.status) - d.addCallback(self.assertEqual, 302) - return d - - def test_redirect_status_head(self): - request = Request(self.getURL("redirect"), method="HEAD") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.status) - d.addCallback(self.assertEqual, 302) - return d - - @defer.inlineCallbacks - def test_timeout_download_from_spider_nodata_rcvd(self): - if self.reactor_pytest != "default" and sys.platform == "win32": - # https://twistedmatrix.com/trac/ticket/10279 - raise unittest.SkipTest( - "This test produces DirtyReactorAggregateError on Windows with asyncio" - ) - - # client connects but no data is received - spider = Spider("foo") - meta = {"download_timeout": 0.5} - request = Request(self.getURL("wait"), meta=meta) - d = self.download_request(request, spider) - yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) - - @defer.inlineCallbacks - def test_timeout_download_from_spider_server_hangs(self): - if self.reactor_pytest != "default" and sys.platform == "win32": - # https://twistedmatrix.com/trac/ticket/10279 - raise unittest.SkipTest( - "This test produces DirtyReactorAggregateError on Windows with asyncio" - ) - # client connects, server send headers and some body bytes but hangs - spider = Spider("foo") - meta = {"download_timeout": 0.5} - request = Request(self.getURL("hang-after-headers"), meta=meta) - d = self.download_request(request, spider) - yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) - - def test_host_header_not_in_request_headers(self): - def _test(response): - assert response.body == to_bytes(f"{self.host}:{self.portno}") - assert not request.headers - - request = Request(self.getURL("host")) - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_host_header_seted_in_request_headers(self): - host = self.host + ":" + str(self.portno) - - def _test(response): - assert response.body == host.encode() - assert request.headers.get("Host") == host.encode() - - request = Request(self.getURL("host"), headers={"Host": host}) - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_content_length_zero_bodyless_post_request_headers(self): - """Tests if "Content-Length: 0" is sent for bodyless POST requests. - - This is not strictly required by HTTP RFCs but can cause trouble - for some web servers. - See: - https://github.com/scrapy/scrapy/issues/823 - https://issues.apache.org/jira/browse/TS-2902 - https://github.com/kennethreitz/requests/issues/405 - https://bugs.python.org/issue14721 - """ - - def _test(response): - assert response.body == b"0" - - request = Request(self.getURL("contentlength"), method="POST") - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_content_length_zero_bodyless_post_only_one(self): - def _test(response): - import json - - headers = Headers(json.loads(response.text)["headers"]) - contentlengths = headers.getlist("Content-Length") - assert len(contentlengths) == 1 - assert contentlengths == [b"0"] - - request = Request(self.getURL("echo"), method="POST") - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_payload(self): - body = b"1" * 100 # PayloadResource requires body length to be 100 - request = Request(self.getURL("payload"), method="POST", body=body) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, body) - return d - - def test_response_header_content_length(self): - request = Request(self.getURL("file"), method=b"GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.headers[b"content-length"]) - d.addCallback(self.assertEqual, b"159") - return d - - def _test_response_class(self, filename, body, response_class): - def _test(response): - assert type(response) is response_class # pylint: disable=unidiomatic-typecheck - - request = Request(self.getURL(filename), body=body) - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_response_class_from_url(self): - return self._test_response_class("foo.html", b"", HtmlResponse) - - def test_response_class_from_body(self): - return self._test_response_class( - "foo", - b"<!DOCTYPE html>\n<title>.", - HtmlResponse, - ) - - def test_get_duplicate_header(self): - def _test(response): - assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] - - request = Request(self.getURL("duplicate-header")) - return self.download_request(request, Spider("foo")).addCallback(_test) - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestHttp10(TestHttp): - """HTTP 1.0 test case""" - - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP10DownloadHandler - - def test_protocol(self): - request = Request(self.getURL("host"), method="GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, "HTTP/1.0") - return d - - -class TestHttps10(TestHttp10): - scheme = "https" - - -class TestHttp11(TestHttp): - """HTTP 1.1 test case""" - - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP11DownloadHandler - - def test_download_without_maxsize_limit(self): - request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d - - def test_response_class_choosing_request(self): - """Tests choosing of correct response type - in case of Content-Type is empty but body contains text. - """ - body = b"Some plain text\ndata with tabs\t and null bytes\0" - - def _test_type(response): - assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck - - request = Request(self.getURL("nocontenttype"), body=body) - d = self.download_request(request, Spider("foo")) - d.addCallback(_test_type) - return d - - @defer.inlineCallbacks - def test_download_with_maxsize(self): - request = Request(self.getURL("file")) - - # 10 is minimal size for this request and the limit is only counted on - # response body. (regardless of headers) - d = self.download_request(request, Spider("foo", download_maxsize=10)) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - yield d - - d = self.download_request(request, Spider("foo", download_maxsize=9)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) - - @defer.inlineCallbacks - def test_download_with_maxsize_very_large_file(self): - with mock.patch("scrapy.core.downloader.handlers.http11.logger") as logger: - request = Request(self.getURL("largechunkedfile")) - - def check(logger): - logger.warning.assert_called_once_with(mock.ANY, mock.ANY) - - d = self.download_request(request, Spider("foo", download_maxsize=1500)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) - - # As the error message is logged in the dataReceived callback, we - # have to give a bit of time to the reactor to process the queue - # after closing the connection. - d = defer.Deferred() - d.addCallback(check) - reactor.callLater(0.1, d.callback, logger) - yield d - - @defer.inlineCallbacks - def test_download_with_maxsize_per_req(self): - meta = {"download_maxsize": 2} - request = Request(self.getURL("file"), meta=meta) - d = self.download_request(request, Spider("foo")) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) - - @defer.inlineCallbacks - def test_download_with_small_maxsize_per_spider(self): - request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo", download_maxsize=2)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) - - def test_download_with_large_maxsize_per_spider(self): - request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo", download_maxsize=100)) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d - - def test_download_chunked_content(self): - request = Request(self.getURL("chunked")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"chunked content\n") - return d - - def test_download_broken_content_cause_data_loss(self, url="broken"): - request = Request(self.getURL(url)) - d = self.download_request(request, Spider("foo")) - - def checkDataLoss(failure): - if failure.check(ResponseFailed) and any( - r.check(_DataLoss) for r in failure.value.reasons - ): - return None - return failure - - d.addCallback(lambda _: self.fail("No DataLoss exception")) - d.addErrback(checkDataLoss) - return d - - def test_download_broken_chunked_content_cause_data_loss(self): - return self.test_download_broken_content_cause_data_loss("broken-chunked") - - def test_download_broken_content_allow_data_loss(self, url="broken"): - request = Request(self.getURL(url), meta={"download_fail_on_dataloss": False}) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.flags) - d.addCallback(self.assertEqual, ["dataloss"]) - return d - - def test_download_broken_chunked_content_allow_data_loss(self): - return self.test_download_broken_content_allow_data_loss("broken-chunked") - - def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"): - crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False}) - download_handler = build_from_crawler(self.download_handler_cls, crawler) - request = Request(self.getURL(url)) - d = download_handler.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.flags) - d.addCallback(self.assertEqual, ["dataloss"]) - return d - - def test_download_broken_chunked_content_allow_data_loss_via_setting(self): - return self.test_download_broken_content_allow_data_loss_via_setting( - "broken-chunked" - ) - - def test_protocol(self): - request = Request(self.getURL("host"), method="GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, "HTTP/1.1") - return d - - -class TestHttps11(TestHttp11): - scheme = "https" - - tls_log_message = ( - 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' - 'subject "/C=IE/O=Scrapy/CN=localhost"' - ) - - @defer.inlineCallbacks - def test_tls_logging(self): - crawler = get_crawler( - settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True} - ) - download_handler = build_from_crawler(self.download_handler_cls, crawler) - try: - with LogCapture() as log_capture: - request = Request(self.getURL("file")) - d = download_handler.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - yield d - log_capture.check_present( - ("scrapy.core.downloader.tls", "DEBUG", self.tls_log_message) - ) - finally: - yield download_handler.close() - - -class TestSimpleHttps(unittest.TestCase): - """Base class for special cases tested with just one simple request""" - - keyfile = "keys/localhost.key" - certfile = "keys/localhost.crt" - cipher_string: str | None = None - - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP11DownloadHandler - - def setUp(self): - self.tmpname = Path(mkdtemp()) - (self.tmpname / "file").write_bytes(b"0123456789") - r = static.File(str(self.tmpname)) - self.site = server.Site(r, timeout=None) - self.host = "localhost" - self.port = reactor.listenSSL( - 0, - self.site, - ssl_context_factory( - self.keyfile, self.certfile, cipher_string=self.cipher_string - ), - interface=self.host, - ) - self.portno = self.port.getHost().port - if self.cipher_string is not None: - settings_dict = {"DOWNLOADER_CLIENT_TLS_CIPHERS": self.cipher_string} - else: - settings_dict = None - crawler = get_crawler(settings_dict=settings_dict) - self.download_handler = build_from_crawler(self.download_handler_cls, crawler) - self.download_request = self.download_handler.download_request - - @defer.inlineCallbacks - def tearDown(self): - yield self.port.stopListening() - if hasattr(self.download_handler, "close"): - yield self.download_handler.close() - shutil.rmtree(self.tmpname) - - def getURL(self, path): - return f"https://{self.host}:{self.portno}/{path}" - - def test_download(self): - request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d - - -class TestHttps11WrongHostname(TestSimpleHttps): - # above tests use a server certificate for "localhost", - # client connection to "localhost" too. - # here we test that even if the server certificate is for another domain, - # "www.example.com" in this case, - # the tests still pass - keyfile = "keys/example-com.key.pem" - certfile = "keys/example-com.cert.pem" - - -class TestHttps11InvalidDNSId(TestSimpleHttps): - """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" - - def setUp(self): - super().setUp() - self.host = "127.0.0.1" - - -class TestHttps11InvalidDNSPattern(TestSimpleHttps): - """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" - - keyfile = "keys/localhost.ip.key" - certfile = "keys/localhost.ip.crt" - - -class TestHttps11CustomCiphers(TestSimpleHttps): - cipher_string = "CAMELLIA256-SHA" - - -class TestHttp11MockServer(unittest.TestCase): - """HTTP 1.1 test case with MockServer""" - - settings_dict: dict | None = None - is_secure = False - - @classmethod - def setUpClass(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def tearDownClass(cls): - cls.mockserver.__exit__(None, None, None) - - @defer.inlineCallbacks - def test_download_with_content_length(self): - crawler = get_crawler(SingleRequestSpider, self.settings_dict) - # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid - # download it - yield crawler.crawl( - seed=Request( - url=self.mockserver.url("/partial", is_secure=self.is_secure), - meta={"download_maxsize": 1000}, - ) - ) - failure = crawler.spider.meta["failure"] - assert isinstance(failure.value, defer.CancelledError) - - @defer.inlineCallbacks - def test_download(self): - crawler = get_crawler(SingleRequestSpider, self.settings_dict) - yield crawler.crawl( - seed=Request(url=self.mockserver.url("", is_secure=self.is_secure)) - ) - failure = crawler.spider.meta.get("failure") - assert failure is None - reason = crawler.spider.meta["close_reason"] - assert reason == "finished" - - -class UriResource(resource.Resource): - """Return the full uri that was requested""" - - def getChild(self, path, request): - return self - - def render(self, request): - # Note: this is an ugly hack for CONNECT request timeout test. - # Returning some data here fail SSL/TLS handshake - # ToDo: implement proper HTTPS proxy tests, not faking them. - if request.method != b"CONNECT": - return request.uri - return b"" - - -class TestHttpProxy(unittest.TestCase, ABC): - expected_http_proxy_request_body = b"http://example.com" - - @property - @abstractmethod - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - raise NotImplementedError - - def setUp(self): - site = server.Site(UriResource(), timeout=None) - wrapper = WrappingFactory(site) - self.port = reactor.listenTCP(0, wrapper, interface="127.0.0.1") - self.portno = self.port.getHost().port - self.download_handler = build_from_crawler( - self.download_handler_cls, get_crawler() - ) - self.download_request = self.download_handler.download_request - - @defer.inlineCallbacks - def tearDown(self): - yield self.port.stopListening() - if hasattr(self.download_handler, "close"): - yield self.download_handler.close() - - def getURL(self, path): - return f"http://127.0.0.1:{self.portno}/{path}" - - def test_download_with_proxy(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == self.expected_http_proxy_request_body - - http_proxy = self.getURL("") - request = Request("http://example.com", meta={"proxy": http_proxy}) - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_download_without_proxy(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == b"/path/to/resource" - - request = Request(self.getURL("path/to/resource")) - return self.download_request(request, Spider("foo")).addCallback(_test) - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestHttp10Proxy(TestHttpProxy): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP10DownloadHandler - - -class TestHttp11Proxy(TestHttpProxy): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP11DownloadHandler - - @defer.inlineCallbacks - def test_download_with_proxy_https_timeout(self): - """Test TunnelingTCP4ClientEndpoint""" - if NON_EXISTING_RESOLVABLE: - raise SkipTest("Non-existing hosts are resolvable") - http_proxy = self.getURL("") - domain = "https://no-such-domain.nosuch" - request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2}) - d = self.download_request(request, Spider("foo")) - timeout = yield self.assertFailure(d, error.TimeoutError) - assert domain in timeout.osError - - def test_download_with_proxy_without_http_scheme(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == self.expected_http_proxy_request_body - - http_proxy = self.getURL("").replace("http://", "") - request = Request("http://example.com", meta={"proxy": http_proxy}) - return self.download_request(request, Spider("foo")).addCallback(_test) - - class HttpDownloadHandlerMock: def __init__(self, *args, **kwargs): pass diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py new file mode 100644 index 000000000..46e5972f7 --- /dev/null +++ b/tests/test_downloader_handlers_http_base.py @@ -0,0 +1,698 @@ +"""Base classes for HTTP download handler tests.""" + +from __future__ import annotations + +import shutil +import sys +from abc import ABC, abstractmethod +from pathlib import Path +from tempfile import mkdtemp +from typing import TYPE_CHECKING, Any +from unittest import mock + +import pytest +from testfixtures import LogCapture +from twisted.internet import defer, error, reactor +from twisted.protocols.policies import WrappingFactory +from twisted.trial import unittest +from twisted.web import resource, server, static, util +from twisted.web._newclient import ResponseFailed +from twisted.web.http import _DataLoss + +from scrapy.http import Headers, HtmlResponse, Request, TextResponse +from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.python import to_bytes +from scrapy.utils.test import get_crawler +from tests import NON_EXISTING_RESOLVABLE +from tests.mockserver import ( + Echo, + ForeverTakingResource, + HostHeaderResource, + MockServer, + NoLengthResource, + PayloadResource, + ssl_context_factory, +) +from tests.spiders import SingleRequestSpider + +if TYPE_CHECKING: + from scrapy.core.downloader.handlers import DownloadHandlerProtocol + + +class ContentLengthHeaderResource(resource.Resource): + """ + A testing resource which renders itself as the value of the Content-Length + header from the request. + """ + + def render(self, request): + return request.requestHeaders.getRawHeaders(b"content-length")[0] + + +class ChunkedResource(resource.Resource): + def render(self, request): + def response(): + request.write(b"chunked ") + request.write(b"content\n") + request.finish() + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class BrokenChunkedResource(resource.Resource): + def render(self, request): + def response(): + request.write(b"chunked ") + request.write(b"content\n") + # Disable terminating chunk on finish. + request.chunked = False + closeConnection(request) + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class BrokenDownloadResource(resource.Resource): + def render(self, request): + def response(): + request.setHeader(b"Content-Length", b"20") + request.write(b"partial") + closeConnection(request) + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +def closeConnection(request): + # We have to force a disconnection for HTTP/1.1 clients. Otherwise + # client keeps the connection open waiting for more data. + request.channel.loseConnection() + request.finish() + + +class EmptyContentTypeHeaderResource(resource.Resource): + """ + A testing resource which renders itself as the value of request body + without content-type header in response. + """ + + def render(self, request): + request.setHeader("content-type", "") + return request.content.read() + + +class LargeChunkedFileResource(resource.Resource): + def render(self, request): + def response(): + for i in range(1024): + request.write(b"x" * 1024) + request.finish() + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class DuplicateHeaderResource(resource.Resource): + def render(self, request): + request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"]) + return b"" + + +class TestHttpBase(unittest.TestCase, ABC): + scheme = "http" + + # only used for HTTPS tests + keyfile = "keys/localhost.key" + certfile = "keys/localhost.crt" + + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + def setUp(self): + self.tmpname = Path(mkdtemp()) + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) + r.putChild(b"redirect", util.Redirect(b"/file")) + r.putChild(b"wait", ForeverTakingResource()) + r.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) + r.putChild(b"nolength", NoLengthResource()) + r.putChild(b"host", HostHeaderResource()) + r.putChild(b"payload", PayloadResource()) + r.putChild(b"broken", BrokenDownloadResource()) + r.putChild(b"chunked", ChunkedResource()) + r.putChild(b"broken-chunked", BrokenChunkedResource()) + r.putChild(b"contentlength", ContentLengthHeaderResource()) + r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) + r.putChild(b"largechunkedfile", LargeChunkedFileResource()) + r.putChild(b"duplicate-header", DuplicateHeaderResource()) + r.putChild(b"echo", Echo()) + self.site = server.Site(r, timeout=None) + self.wrapper = WrappingFactory(self.site) + self.host = "localhost" + if self.scheme == "https": + # Using WrappingFactory do not enable HTTP/2 failing all the + # tests with H2DownloadHandler + self.port = reactor.listenSSL( + 0, + self.site, + ssl_context_factory(self.keyfile, self.certfile), + interface=self.host, + ) + else: + self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) + self.portno = self.port.getHost().port + self.download_handler = build_from_crawler( + self.download_handler_cls, get_crawler() + ) + self.download_request = self.download_handler.download_request + + @defer.inlineCallbacks + def tearDown(self): + yield self.port.stopListening() + if hasattr(self.download_handler, "close"): + yield self.download_handler.close() + shutil.rmtree(self.tmpname) + + def getURL(self, path): + return f"{self.scheme}://{self.host}:{self.portno}/{path}" + + def test_download(self): + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + return d + + def test_download_head(self): + request = Request(self.getURL("file"), method="HEAD") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"") + return d + + def test_redirect_status(self): + request = Request(self.getURL("redirect")) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.status) + d.addCallback(self.assertEqual, 302) + return d + + def test_redirect_status_head(self): + request = Request(self.getURL("redirect"), method="HEAD") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.status) + d.addCallback(self.assertEqual, 302) + return d + + @defer.inlineCallbacks + def test_timeout_download_from_spider_nodata_rcvd(self): + if self.reactor_pytest != "default" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) + + # client connects but no data is received + spider = Spider("foo") + meta = {"download_timeout": 0.5} + request = Request(self.getURL("wait"), meta=meta) + d = self.download_request(request, spider) + yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) + + @defer.inlineCallbacks + def test_timeout_download_from_spider_server_hangs(self): + if self.reactor_pytest != "default" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) + # client connects, server send headers and some body bytes but hangs + spider = Spider("foo") + meta = {"download_timeout": 0.5} + request = Request(self.getURL("hang-after-headers"), meta=meta) + d = self.download_request(request, spider) + yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) + + def test_host_header_not_in_request_headers(self): + def _test(response): + assert response.body == to_bytes(f"{self.host}:{self.portno}") + assert not request.headers + + request = Request(self.getURL("host")) + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_host_header_seted_in_request_headers(self): + host = self.host + ":" + str(self.portno) + + def _test(response): + assert response.body == host.encode() + assert request.headers.get("Host") == host.encode() + + request = Request(self.getURL("host"), headers={"Host": host}) + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_content_length_zero_bodyless_post_request_headers(self): + """Tests if "Content-Length: 0" is sent for bodyless POST requests. + + This is not strictly required by HTTP RFCs but can cause trouble + for some web servers. + See: + https://github.com/scrapy/scrapy/issues/823 + https://issues.apache.org/jira/browse/TS-2902 + https://github.com/kennethreitz/requests/issues/405 + https://bugs.python.org/issue14721 + """ + + def _test(response): + assert response.body == b"0" + + request = Request(self.getURL("contentlength"), method="POST") + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_content_length_zero_bodyless_post_only_one(self): + def _test(response): + import json + + headers = Headers(json.loads(response.text)["headers"]) + contentlengths = headers.getlist("Content-Length") + assert len(contentlengths) == 1 + assert contentlengths == [b"0"] + + request = Request(self.getURL("echo"), method="POST") + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_payload(self): + body = b"1" * 100 # PayloadResource requires body length to be 100 + request = Request(self.getURL("payload"), method="POST", body=body) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, body) + return d + + def test_response_header_content_length(self): + request = Request(self.getURL("file"), method=b"GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.headers[b"content-length"]) + d.addCallback(self.assertEqual, b"159") + return d + + def _test_response_class(self, filename, body, response_class): + def _test(response): + assert type(response) is response_class # pylint: disable=unidiomatic-typecheck + + request = Request(self.getURL(filename), body=body) + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_response_class_from_url(self): + return self._test_response_class("foo.html", b"", HtmlResponse) + + def test_response_class_from_body(self): + return self._test_response_class( + "foo", + b"\n.", + HtmlResponse, + ) + + def test_get_duplicate_header(self): + def _test(response): + assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] + + request = Request(self.getURL("duplicate-header")) + return self.download_request(request, Spider("foo")).addCallback(_test) + + +class TestHttp11Base(TestHttpBase): + """HTTP 1.1 test case""" + + def test_download_without_maxsize_limit(self): + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + return d + + def test_response_class_choosing_request(self): + """Tests choosing of correct response type + in case of Content-Type is empty but body contains text. + """ + body = b"Some plain text\ndata with tabs\t and null bytes\0" + + def _test_type(response): + assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck + + request = Request(self.getURL("nocontenttype"), body=body) + d = self.download_request(request, Spider("foo")) + d.addCallback(_test_type) + return d + + @defer.inlineCallbacks + def test_download_with_maxsize(self): + request = Request(self.getURL("file")) + + # 10 is minimal size for this request and the limit is only counted on + # response body. (regardless of headers) + d = self.download_request(request, Spider("foo", download_maxsize=10)) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + yield d + + d = self.download_request(request, Spider("foo", download_maxsize=9)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + @defer.inlineCallbacks + def test_download_with_maxsize_very_large_file(self): + with mock.patch("scrapy.core.downloader.handlers.http11.logger") as logger: + request = Request(self.getURL("largechunkedfile")) + + def check(logger): + logger.warning.assert_called_once_with(mock.ANY, mock.ANY) + + d = self.download_request(request, Spider("foo", download_maxsize=1500)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + # As the error message is logged in the dataReceived callback, we + # have to give a bit of time to the reactor to process the queue + # after closing the connection. + d = defer.Deferred() + d.addCallback(check) + reactor.callLater(0.1, d.callback, logger) + yield d + + @defer.inlineCallbacks + def test_download_with_maxsize_per_req(self): + meta = {"download_maxsize": 2} + request = Request(self.getURL("file"), meta=meta) + d = self.download_request(request, Spider("foo")) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + @defer.inlineCallbacks + def test_download_with_small_maxsize_per_spider(self): + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo", download_maxsize=2)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + def test_download_with_large_maxsize_per_spider(self): + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo", download_maxsize=100)) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + return d + + def test_download_chunked_content(self): + request = Request(self.getURL("chunked")) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"chunked content\n") + return d + + def test_download_broken_content_cause_data_loss(self, url="broken"): + # TODO: this one checks for Twisted-specific exceptions + request = Request(self.getURL(url)) + d = self.download_request(request, Spider("foo")) + + def checkDataLoss(failure): + if failure.check(ResponseFailed) and any( + r.check(_DataLoss) for r in failure.value.reasons + ): + return None + return failure + + d.addCallback(lambda _: self.fail("No DataLoss exception")) + d.addErrback(checkDataLoss) + return d + + def test_download_broken_chunked_content_cause_data_loss(self): + return self.test_download_broken_content_cause_data_loss("broken-chunked") + + def test_download_broken_content_allow_data_loss(self, url="broken"): + request = Request(self.getURL(url), meta={"download_fail_on_dataloss": False}) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.flags) + d.addCallback(self.assertEqual, ["dataloss"]) + return d + + def test_download_broken_chunked_content_allow_data_loss(self): + return self.test_download_broken_content_allow_data_loss("broken-chunked") + + def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"): + crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False}) + download_handler = build_from_crawler(self.download_handler_cls, crawler) + request = Request(self.getURL(url)) + d = download_handler.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.flags) + d.addCallback(self.assertEqual, ["dataloss"]) + return d + + def test_download_broken_chunked_content_allow_data_loss_via_setting(self): + return self.test_download_broken_content_allow_data_loss_via_setting( + "broken-chunked" + ) + + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "HTTP/1.1") + return d + + +class TestHttps11Base(TestHttp11Base): + scheme = "https" + + tls_log_message = ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' + 'subject "/C=IE/O=Scrapy/CN=localhost"' + ) + + @defer.inlineCallbacks + def test_tls_logging(self): + crawler = get_crawler( + settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True} + ) + download_handler = build_from_crawler(self.download_handler_cls, crawler) + try: + with LogCapture() as log_capture: + request = Request(self.getURL("file")) + d = download_handler.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + yield d + log_capture.check_present( + ("scrapy.core.downloader.tls", "DEBUG", self.tls_log_message) + ) + finally: + yield download_handler.close() + + +class TestSimpleHttpsBase(unittest.TestCase, ABC): + """Base class for special cases tested with just one simple request""" + + keyfile = "keys/localhost.key" + certfile = "keys/localhost.crt" + cipher_string: str | None = None + + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + def setUp(self): + self.tmpname = Path(mkdtemp()) + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) + self.site = server.Site(r, timeout=None) + self.host = "localhost" + self.port = reactor.listenSSL( + 0, + self.site, + ssl_context_factory( + self.keyfile, self.certfile, cipher_string=self.cipher_string + ), + interface=self.host, + ) + self.portno = self.port.getHost().port + if self.cipher_string is not None: + settings_dict = {"DOWNLOADER_CLIENT_TLS_CIPHERS": self.cipher_string} + else: + settings_dict = None + crawler = get_crawler(settings_dict=settings_dict) + self.download_handler = build_from_crawler(self.download_handler_cls, crawler) + self.download_request = self.download_handler.download_request + + @defer.inlineCallbacks + def tearDown(self): + yield self.port.stopListening() + if hasattr(self.download_handler, "close"): + yield self.download_handler.close() + shutil.rmtree(self.tmpname) + + def getURL(self, path): + return f"https://{self.host}:{self.portno}/{path}" + + def test_download(self): + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + return d + + +class TestHttpsWrongHostnameBase(TestSimpleHttpsBase): + # above tests use a server certificate for "localhost", + # client connection to "localhost" too. + # here we test that even if the server certificate is for another domain, + # "www.example.com" in this case, + # the tests still pass + keyfile = "keys/example-com.key.pem" + certfile = "keys/example-com.cert.pem" + + +class TestHttpsInvalidDNSIdBase(TestSimpleHttpsBase): + """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" + + def setUp(self): + super().setUp() + self.host = "127.0.0.1" + + +class TestHttpsInvalidDNSPatternBase(TestSimpleHttpsBase): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = "keys/localhost.ip.key" + certfile = "keys/localhost.ip.crt" + + +class TestHttpsCustomCiphersBase(TestSimpleHttpsBase): + cipher_string = "CAMELLIA256-SHA" + + +class TestHttpMockServerBase(unittest.TestCase, ABC): + """HTTP 1.1 test case with MockServer""" + + @property + @abstractmethod + def settings_dict(self) -> dict[str, Any] | None: + raise NotImplementedError + + is_secure = False + + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_download_with_content_length(self): + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid + # download it + yield crawler.crawl( + seed=Request( + url=self.mockserver.url("/partial", is_secure=self.is_secure), + meta={"download_maxsize": 1000}, + ) + ) + failure = crawler.spider.meta["failure"] + assert isinstance(failure.value, defer.CancelledError) + + @defer.inlineCallbacks + def test_download(self): + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + yield crawler.crawl( + seed=Request(url=self.mockserver.url("", is_secure=self.is_secure)) + ) + failure = crawler.spider.meta.get("failure") + assert failure is None + reason = crawler.spider.meta["close_reason"] + assert reason == "finished" + + +class UriResource(resource.Resource): + """Return the full uri that was requested""" + + def getChild(self, path, request): + return self + + def render(self, request): + # Note: this is an ugly hack for CONNECT request timeout test. + # Returning some data here fail SSL/TLS handshake + # ToDo: implement proper HTTPS proxy tests, not faking them. + if request.method != b"CONNECT": + return request.uri + return b"" + + +class TestHttpProxyBase(unittest.TestCase, ABC): + expected_http_proxy_request_body = b"http://example.com" + + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + def setUp(self): + site = server.Site(UriResource(), timeout=None) + wrapper = WrappingFactory(site) + self.port = reactor.listenTCP(0, wrapper, interface="127.0.0.1") + self.portno = self.port.getHost().port + self.download_handler = build_from_crawler( + self.download_handler_cls, get_crawler() + ) + self.download_request = self.download_handler.download_request + + @defer.inlineCallbacks + def tearDown(self): + yield self.port.stopListening() + if hasattr(self.download_handler, "close"): + yield self.download_handler.close() + + def getURL(self, path): + return f"http://127.0.0.1:{self.portno}/{path}" + + def test_download_with_proxy(self): + def _test(response): + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body + + http_proxy = self.getURL("") + request = Request("http://example.com", meta={"proxy": http_proxy}) + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_download_without_proxy(self): + def _test(response): + assert response.status == 200 + assert response.url == request.url + assert response.body == b"/path/to/resource" + + request = Request(self.getURL("path/to/resource")) + return self.download_request(request, Spider("foo")).addCallback(_test) + + @defer.inlineCallbacks + def test_download_with_proxy_https_timeout(self): + if NON_EXISTING_RESOLVABLE: + pytest.skip("Non-existing hosts are resolvable") + http_proxy = self.getURL("") + domain = "https://no-such-domain.nosuch" + request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2}) + d = self.download_request(request, Spider("foo")) + timeout = yield self.assertFailure(d, error.TimeoutError) + assert domain in timeout.osError + + def test_download_with_proxy_without_http_scheme(self): + def _test(response): + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body + + http_proxy = self.getURL("").replace("http://", "") + request = Request("http://example.com", meta={"proxy": http_proxy}) + return self.download_request(request, Spider("foo")).addCallback(_test) From 0cdb971f636fc75ed08ed13642adfa04fcc235ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 28 May 2025 10:06:04 +0200 Subject: [PATCH 128/491] Complete the release notes --- docs/news.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index eb5370b6e..76e195feb 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -18,6 +18,21 @@ Scrapy 2.13.1 (unreleased) only sent when there are not enough pending callback requests to reach concurrency limits. + (:issue:`6828`) + +- Added a deepwiki_ badge to the README. (:issue:`6793`) + + .. _deepwiki: https://deepwiki.com/scrapy/scrapy + +- Fixed a typo in the code example of :ref:`start-requests-lazy`. + (:issue:`6812`, :issue:`6815`) + +- Fixed a typo in the :ref:`coroutine-support` section of the documentation. + (:issue:`6822`) + +- Made this page more prominently listed in PyPI project links. + (:issue:`6826`) + .. _release-2.13.0: From 08ee88456f850f247117aa602f7b066356b66419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 28 May 2025 10:12:37 +0200 Subject: [PATCH 129/491] bumpversion: set the release date automatically --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 47707e061..3b8174afe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,6 +120,12 @@ commit = true tag = true tag_name = "{new_version}" +[[tool.bumpversion.files]] +filename = "docs/news.rst" +search = "\\(unreleased\\)$" +replace = "({now:%Y-%m-%d})" +regex = true + [[tool.bumpversion.files]] filename = "scrapy/VERSION" From 7fe7f1734aba5625f8c9c405bb92ef1da18cd983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 28 May 2025 10:12:46 +0200 Subject: [PATCH 130/491] =?UTF-8?q?Bump=20version:=202.13.0=20=E2=86=92=20?= =?UTF-8?q?2.13.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/news.rst | 2 +- pyproject.toml | 2 +- scrapy/VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 76e195feb..ef3b549e7 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.13.1: -Scrapy 2.13.1 (unreleased) +Scrapy 2.13.1 (2025-05-28) -------------------------- - Give callback requests precedence over start requests when priority values diff --git a/pyproject.toml b/pyproject.toml index 3b8174afe..68c1e07bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,7 +115,7 @@ module = "twisted" implicit_reexport = true [tool.bumpversion] -current_version = "2.13.0" +current_version = "2.13.1" commit = true tag = true tag_name = "{new_version}" diff --git a/scrapy/VERSION b/scrapy/VERSION index fb2c0766b..94f15e9cc 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.13.0 +2.13.1 From 7b4cf06b6e381b6210fe43bc94a1eb65b3db0583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 28 May 2025 10:54:36 +0200 Subject: [PATCH 131/491] Feature the new logo in the README (#6831) --- README.rst | 22 ++++++++++------------ docs/_static/logo.svg | 1 + 2 files changed, 11 insertions(+), 12 deletions(-) create mode 100644 docs/_static/logo.svg diff --git a/README.rst b/README.rst index 29488d825..30001e4b0 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,10 @@ -.. image:: https://scrapy.org/img/scrapylogo.png - :target: https://scrapy.org/ +.. raw:: html -====== -Scrapy -====== +

+ + Scrapy + +

.. image:: https://img.shields.io/pypi/v/Scrapy.svg :target: https://pypi.org/pypi/Scrapy @@ -37,13 +38,10 @@ Scrapy :target: https://deepwiki.com/scrapy/scrapy :alt: Ask DeepWiki - -Overview -======== - -Scrapy is a BSD-licensed fast high-level web crawling and web scraping framework, used to -crawl websites and extract structured data from their pages. It can be used for -a wide range of purposes, from data mining to monitoring and automated testing. +Scrapy is a BSD-licensed fast high-level web crawling and web scraping +framework, used to crawl websites and extract structured data from their pages. +It can be used for a wide range of purposes, from data mining to monitoring and +automated testing. Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other contributors`_. diff --git a/docs/_static/logo.svg b/docs/_static/logo.svg new file mode 100644 index 000000000..04b2d18a7 --- /dev/null +++ b/docs/_static/logo.svg @@ -0,0 +1 @@ + From f98ffc71d25ae74351df70ae03ef5299a2ab3813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 28 May 2025 10:58:34 +0200 Subject: [PATCH 132/491] 2.13.1 (#6832) --- docs/news.rst | 31 ++++++++++++++++++++++ pyproject.toml | 8 +++++- scrapy/VERSION | 2 +- scrapy/core/scheduler.py | 4 +-- scrapy/pqueues.py | 30 ++++++++++++---------- tests/test_engine_loop.py | 20 +++++++-------- tests/test_pqueues.py | 54 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 122 insertions(+), 27 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index cf1c35893..ef3b549e7 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,37 @@ Release notes ============= +.. _release-2.13.1: + +Scrapy 2.13.1 (2025-05-28) +-------------------------- + +- Give callback requests precedence over start requests when priority values + are the same. + + This makes changes from 2.13.0 to start request handling more intuitive and + backward compatible. For scenarios where all requests have the same + priorities, in 2.13.0 all start requests were sent before the first + callback request. In 2.13.1, same as in 2.12 and lower, start requests are + only sent when there are not enough pending callback requests to reach + concurrency limits. + + (:issue:`6828`) + +- Added a deepwiki_ badge to the README. (:issue:`6793`) + + .. _deepwiki: https://deepwiki.com/scrapy/scrapy + +- Fixed a typo in the code example of :ref:`start-requests-lazy`. + (:issue:`6812`, :issue:`6815`) + +- Fixed a typo in the :ref:`coroutine-support` section of the documentation. + (:issue:`6822`) + +- Made this page more prominently listed in PyPI project links. + (:issue:`6826`) + + .. _release-2.13.0: Scrapy 2.13.0 (2025-05-08) diff --git a/pyproject.toml b/pyproject.toml index 47707e061..68c1e07bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,11 +115,17 @@ module = "twisted" implicit_reexport = true [tool.bumpversion] -current_version = "2.13.0" +current_version = "2.13.1" commit = true tag = true tag_name = "{new_version}" +[[tool.bumpversion.files]] +filename = "docs/news.rst" +search = "\\(unreleased\\)$" +replace = "({now:%Y-%m-%d})" +regex = true + [[tool.bumpversion.files]] filename = "scrapy/VERSION" diff --git a/scrapy/VERSION b/scrapy/VERSION index fb2c0766b..94f15e9cc 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.13.0 +2.13.1 diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 57d27b7cf..9ac447289 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -173,8 +173,8 @@ class Scheduler(BaseScheduler): :ref:`Start requests ` are sent in the order they are yielded from :meth:`~scrapy.Spider.start`, and given the same - :attr:`~scrapy.http.Request.priority`, start requests take precedence over - other requests. + :attr:`~scrapy.http.Request.priority`, other requests take precedence over + start requests. You can set :setting:`SCHEDULER_START_MEMORY_QUEUE` and :setting:`SCHEDULER_START_DISK_QUEUE` to ``None`` to handle start requests diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index e6c6b8bf1..34b235d83 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -160,28 +160,32 @@ class ScrapyPriorityQueue: def pop(self) -> Request | None: while self.curprio is not None: - if self._start_queues: - try: - q = self._start_queues[self.curprio] - except KeyError: - pass - else: - m = q.pop() - if not q: - del self._start_queues[self.curprio] - q.close() - return m try: q = self.queues[self.curprio] except KeyError: - self._update_curprio() + pass else: m = q.pop() if not q: del self.queues[self.curprio] q.close() - self._update_curprio() + if not self._start_queues: + self._update_curprio() return m + if self._start_queues: + try: + q = self._start_queues[self.curprio] + except KeyError: + self._update_curprio() + else: + m = q.pop() + if not q: + del self._start_queues[self.curprio] + q.close() + self._update_curprio() + return m + else: + self._update_curprio() return None def _update_curprio(self) -> None: diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 90af10f0e..c7dbc82d4 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -189,9 +189,9 @@ class RequestSendOrderTestCase(TestCase): @deferred_f_from_coro_f async def test_default(self): - """By default, start requests take priority over callback requests and + """By default, callback requests take priority over start requests and are sent in order. Priority matters, but given the same priority, a - start request takes precedence.""" + callback request takes precedence.""" nums = [1, 2, 3, 4, 5, 6] response_seconds = 0 download_slots = 1 @@ -207,13 +207,13 @@ class RequestSendOrderTestCase(TestCase): yield _request(1) for request in ( - _request(4, priority=1), - _request(6), + _request(2, priority=1), + _request(5), ): spider.crawler.engine._slot.scheduler.enqueue_request(request) - yield _request(5) - yield _request(2, priority=1) + yield _request(6) yield _request(3, priority=1) + yield _request(4, priority=1) def parse(spider, response): return @@ -249,13 +249,13 @@ class RequestSendOrderTestCase(TestCase): yield _request(1) for request in ( - _request(4, priority=1), - _request(6), + _request(2, priority=1), + _request(5), ): spider.crawler.engine._slot.scheduler.enqueue_request(request) - yield _request(5) + yield _request(6) + yield _request(4, priority=1) yield _request(3, priority=1) - yield _request(2, priority=1) def parse(spider, response): return diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index d5c710ed2..b65f1b7e7 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -7,6 +7,7 @@ from scrapy.http.request import Request from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue from scrapy.spiders import Spider from scrapy.squeues import FifoMemoryQueue +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.test import get_crawler from tests.test_scheduler import MockDownloader, MockEngine @@ -155,3 +156,56 @@ class TestDownloaderAwarePriorityQueue: assert self.queue.peek().url == req3.url assert self.queue.pop().url == req3.url assert self.queue.peek() is None + + +@pytest.mark.parametrize( + ("input", "output"), + [ + # By default, start requests are FIFO, other requests are LIFO. + ([{}, {}], [2, 1]), + ([{"start": True}, {"start": True}], [1, 2]), + # Priority matters. + ([{"priority": 1}, {"start": True}], [1, 2]), + ([{}, {"start": True, "priority": 1}], [2, 1]), + # For the same priority, start requests pop last. + ([{}, {"start": True}], [1, 2]), + ([{"start": True}, {}], [2, 1]), + ], +) +def test_pop_order(input, output): + def make_url(index): + return f"https://toscrape.com/{index}" + + def make_request(index, data): + meta = {} + if data.get("start", False): + meta["is_start_request"] = True + return Request( + url=make_url(index), + priority=data.get("priority", 0), + meta=meta, + ) + + input_requests = [ + make_request(index, data) for index, data in enumerate(input, start=1) + ] + expected_output_urls = [make_url(index) for index in output] + + crawler = get_crawler(Spider) + settings = crawler.settings + queue = build_from_crawler( + ScrapyPriorityQueue, + crawler, + downstream_queue_cls=load_object(settings["SCHEDULER_MEMORY_QUEUE"]), + key="", + start_queue_cls=load_object(settings["SCHEDULER_START_MEMORY_QUEUE"]), + ) + + for request in input_requests: + queue.push(request) + + actual_output_urls = [] + while request := queue.pop(): + actual_output_urls.append(request.url) + + assert actual_output_urls == expected_output_urls From c480c77f54e1b417468847ab5437458cf5beffe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 28 May 2025 11:35:18 +0200 Subject: [PATCH 133/491] Shorten the README and remove broken links to scrapy.org (#6833) --- README.rst | 68 ++++++------------------------------------- docs/contributing.rst | 9 ++++-- 2 files changed, 16 insertions(+), 61 deletions(-) diff --git a/README.rst b/README.rst index 30001e4b0..5dc994570 100644 --- a/README.rst +++ b/README.rst @@ -38,74 +38,24 @@ :target: https://deepwiki.com/scrapy/scrapy :alt: Ask DeepWiki -Scrapy is a BSD-licensed fast high-level web crawling and web scraping -framework, used to crawl websites and extract structured data from their pages. -It can be used for a wide range of purposes, from data mining to monitoring and -automated testing. - -Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other -contributors`_. +Scrapy_ is a web scraping framework to extract structured data from websites. +It is cross-platform, and requires Python 3.9+. It is maintained by Zyte_ +(formerly Scrapinghub) and `many other contributors`_. .. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors +.. _Scrapy: https://scrapy.org/ .. _Zyte: https://www.zyte.com/ -Check the Scrapy homepage at https://scrapy.org for more information, -including a list of features. - - -Requirements -============ - -* Python 3.9+ -* Works on Linux, Windows, macOS, BSD - -Install -======= - -The quick way: +Install with: .. code:: bash pip install scrapy -See the install section in the documentation at -https://docs.scrapy.org/en/latest/intro/install.html for more details. +And follow the documentation_ to learn how to use it. -Documentation -============= +.. _documentation: https://docs.scrapy.org/en/latest/ -Documentation is available online at https://docs.scrapy.org/ and in the ``docs`` -directory. +If you wish to contribute, see Contributing_. -Releases -======== - -You can check https://docs.scrapy.org/en/latest/news.html for the release notes. - -Community (blog, twitter, mail list, IRC) -========================================= - -See https://scrapy.org/community/ for details. - -Contributing -============ - -See https://docs.scrapy.org/en/master/contributing.html for details. - -Code of Conduct ---------------- - -Please note that this project is released with a Contributor `Code of Conduct `_. - -By participating in this project you agree to abide by its terms. -Please report unacceptable behavior to opensource@zyte.com. - -Companies using Scrapy -====================== - -See https://scrapy.org/companies/ for a list. - -Commercial Support -================== - -See https://scrapy.org/support/ for details. +.. _Contributing: https://docs.scrapy.org/en/master/contributing.html diff --git a/docs/contributing.rst b/docs/contributing.rst index f5c1c74b8..0172887d6 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -6,8 +6,13 @@ Contributing to Scrapy .. important:: - Double check that you are reading the most recent version of this document at - https://docs.scrapy.org/en/master/contributing.html + Double check that you are reading the most recent version of this document + at https://docs.scrapy.org/en/master/contributing.html + + By participating in this project you agree to abide by the terms of our + `Code of Conduct + `_. Please + report unacceptable behavior to opensource@zyte.com. There are many ways to contribute to Scrapy. Here are some of them: From dceb85bf3e41d06dfac81c037c69fd0f1ab61156 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 28 May 2025 14:46:39 +0500 Subject: [PATCH 134/491] Add is_asyncio_available(). (#6827) * Add is_asyncio_available(). * Print unexpected warnings in test_install_asyncio_reactor(). * Fix printing warnings. * Fix printing warnings - 2. * Skip TestDeferredToFuture on non-asyncio. * Test the is_asyncio_available() exception. --- docs/topics/asyncio.rst | 11 +++--- scrapy/utils/asyncio.py | 38 +++++++++++++++++++ scrapy/utils/defer.py | 7 ++-- scrapy/utils/reactor.py | 6 +++ .../CrawlerProcess/asyncio_enabled_reactor.py | 10 +++++ tests/test_utils_asyncio.py | 33 ++-------------- tests/test_utils_defer.py | 4 +- tests/test_utils_deprecate.py | 2 +- tests/test_utils_reactor.py | 35 +++++++++++++++++ 9 files changed, 106 insertions(+), 40 deletions(-) create mode 100644 scrapy/utils/asyncio.py create mode 100644 tests/test_utils_reactor.py diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 0490129b3..473ef7bfa 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -105,25 +105,26 @@ Enforcing asyncio as a requirement ================================== If you are writing a :ref:`component ` that requires asyncio -to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to +to work, use :func:`scrapy.utils.asyncio.is_asyncio_available` to :ref:`enforce it as a requirement `. For example: .. code-block:: python - from scrapy.utils.reactor import is_asyncio_reactor_installed + from scrapy.utils.asyncio import is_asyncio_available class MyComponent: def __init__(self): - if not is_asyncio_reactor_installed(): + if not is_asyncio_available(): raise ValueError( - f"{MyComponent.__qualname__} requires the asyncio Twisted " - f"reactor. Make sure you have it configured in the " + f"{MyComponent.__qualname__} requires the asyncio support. " + f"Make sure you have configured the asyncio reactor in the " f"TWISTED_REACTOR setting. See the asyncio documentation " f"of Scrapy for more information." ) +.. autofunction:: scrapy.utils.asyncio.is_asyncio_available .. autofunction:: scrapy.utils.reactor.is_asyncio_reactor_installed diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py new file mode 100644 index 000000000..4469369fa --- /dev/null +++ b/scrapy/utils/asyncio.py @@ -0,0 +1,38 @@ +"""Utilities related to asyncio and its support in Scrapy.""" + +from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed + + +def is_asyncio_available() -> bool: + """Check if it's possible to call asyncio code that relies on the asyncio event loop. + + .. versionadded:: VERSION + + Currently this function is identical to + :func:`scrapy.utils.reactor.is_asyncio_reactor_installed`: it returns + ``True`` if the Twisted reactor that is installed is + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, returns + ``False`` if a different reactor is installed, and raises a + :exc:`RuntimeError` if no reactor is installed. In a future Scrapy version, + when Scrapy supports running without a Twisted reactor, this function will + also return ``True`` when running in that mode, so code that doesn't + directly require a Twisted reactor should use this function instead of + :func:`~scrapy.utils.reactor.is_asyncio_reactor_installed`. + + When this returns ``True``, an asyncio loop is installed and used by + Scrapy. It's possible to call functions that require it, such as + :func:`asyncio.sleep`, and await on :class:`asyncio.Future` objects in + Scrapy-related code. + + When this returns ``False``, a non-asyncio Twisted reactor is installed. + It's not possible to use asyncio features that require an asyncio event + loop or await on :class:`asyncio.Future` objects in Scrapy-related code, + but it's possible to await on :class:`~twisted.internet.defer.Deferred` + objects. + """ + if not is_reactor_installed(): + raise RuntimeError( + "is_asyncio_available() called without an installed reactor." + ) + + return is_asyncio_reactor_installed() diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index d06397f50..4649c4daa 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -22,7 +22,8 @@ from twisted.internet.task import Cooperator from twisted.python import failure from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning -from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed +from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.reactor import _get_asyncio_event_loop if TYPE_CHECKING: from collections.abc import AsyncIterator, Callable @@ -379,7 +380,7 @@ def deferred_from_coro(o: Awaitable[_T] | _T2) -> Deferred[_T] | _T2: if isinstance(o, Deferred): return o if inspect.isawaitable(o): - if not is_asyncio_reactor_installed(): + if not is_asyncio_available(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" return Deferred.fromCoroutine(cast(Coroutine[Deferred[Any], Any, _T], o)) @@ -471,6 +472,6 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: deferred = self.crawler.engine.download(additional_request) additional_response = await maybe_deferred_to_future(deferred) """ - if not is_asyncio_reactor_installed(): + if not is_asyncio_available(): return d return deferred_to_future(d) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 9c2754394..5e76da37b 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -202,6 +202,12 @@ def is_asyncio_reactor_installed() -> bool: """Check whether the installed reactor is :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. Raise a :exc:`RuntimeError` if no reactor is installed. + + In a future Scrapy version, when Scrapy supports running without a Twisted + reactor, this function won't be useful for checking if it's possible to use + asyncio features, so the code that that doesn't directly require a Twisted + reactor should use :func:`scrapy.utils.asyncio.is_asyncio_available` + instead of this function. """ if not is_reactor_installed(): raise RuntimeError( diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index f3dab12fe..4e8d3db12 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,5 +1,6 @@ import scrapy from scrapy.crawler import CrawlerProcess +from scrapy.utils.asyncio import is_asyncio_available from scrapy.utils.reactor import ( install_reactor, is_asyncio_reactor_installed, @@ -18,6 +19,13 @@ except RuntimeError: else: raise RuntimeError("is_asyncio_reactor_installed() did not raise RuntimeError.") +try: + is_asyncio_available() +except RuntimeError: + pass +else: + raise RuntimeError("is_asyncio_available() did not raise RuntimeError.") + if is_reactor_installed(): raise RuntimeError( "Reactor already installed after is_asyncio_reactor_installed()." @@ -33,6 +41,8 @@ class ReactorCheckExtension: def __init__(self): if not is_asyncio_reactor_installed(): raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") + if not is_asyncio_available(): + raise RuntimeError("ReactorCheckExtension requires asyncio support.") class NoRequestsSpider(scrapy.Spider): diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 901e03d59..fe44748f9 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,35 +1,10 @@ -import asyncio -import warnings - import pytest -from twisted.trial.unittest import TestCase -from scrapy.utils.defer import deferred_f_from_coro_f -from scrapy.utils.reactor import ( - install_reactor, - is_asyncio_reactor_installed, - set_asyncio_event_loop, -) +from scrapy.utils.asyncio import is_asyncio_available @pytest.mark.usefixtures("reactor_pytest") -class TestAsyncio(TestCase): - def test_is_asyncio_reactor_installed(self): +class TestAsyncio: + def test_is_asyncio_available(self): # the result should depend only on the pytest --reactor argument - assert is_asyncio_reactor_installed() == (self.reactor_pytest != "default") - - def test_install_asyncio_reactor(self): - from twisted.internet import reactor as original_reactor - - with warnings.catch_warnings(record=True) as w: - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") - assert len(w) == 0 - from twisted.internet import reactor # pylint: disable=reimported - - assert original_reactor == reactor - - @pytest.mark.only_asyncio - @deferred_f_from_coro_f - async def test_set_asyncio_event_loop(self): - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") - assert set_asyncio_event_loop(None) is asyncio.get_running_loop() + assert is_asyncio_available() == (self.reactor_pytest != "default") diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 29cd5fbf2..c565c1c4e 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -311,6 +311,7 @@ class TestDeferredFFromCoroF(unittest.TestCase): yield self._assert_result(c_f) +@pytest.mark.only_asyncio class TestDeferredToFuture(unittest.TestCase): @deferred_f_from_coro_f async def test_deferred(self): @@ -332,7 +333,6 @@ class TestDeferredToFuture(unittest.TestCase): future_result = await result assert future_result == 42 - @pytest.mark.only_asyncio @deferred_f_from_coro_f async def test_wrapped_coroutine_asyncio(self): async def c_f() -> int: @@ -340,7 +340,7 @@ class TestDeferredToFuture(unittest.TestCase): return 42 d = deferred_from_coro(c_f()) - result = maybe_deferred_to_future(d) + result = deferred_to_future(d) assert isinstance(result, Future) future_result = await result assert future_result == 42 diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 52c165bb4..662de0dc3 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -243,7 +243,7 @@ class TestWarnWhenSubclassed: ) w = self._mywarnings(w) - assert len(w) == 0, str(map(str, w)) + assert len(w) == 0, [str(warning) for warning in w] with warnings.catch_warnings(record=True) as w: AlsoDeprecated() diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py new file mode 100644 index 000000000..99f175c60 --- /dev/null +++ b/tests/test_utils_reactor.py @@ -0,0 +1,35 @@ +import asyncio +import warnings + +import pytest +from twisted.trial.unittest import TestCase + +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + set_asyncio_event_loop, +) + + +@pytest.mark.usefixtures("reactor_pytest") +class TestAsyncio(TestCase): + def test_is_asyncio_reactor_installed(self): + # the result should depend only on the pytest --reactor argument + assert is_asyncio_reactor_installed() == (self.reactor_pytest != "default") + + def test_install_asyncio_reactor(self): + from twisted.internet import reactor as original_reactor + + with warnings.catch_warnings(record=True) as w: + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + assert len(w) == 0, [str(warning) for warning in w] + from twisted.internet import reactor # pylint: disable=reimported + + assert original_reactor == reactor + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_set_asyncio_event_loop(self): + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + assert set_asyncio_event_loop(None) is asyncio.get_running_loop() From 916fe509744c39be63b35c859e50a35a2ac4333d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 28 May 2025 15:53:52 +0500 Subject: [PATCH 135/491] Fix and ban the top-level twisted.internet.reactor imports. (#6835) --- extras/qps-bench-server.py | 2 +- pyproject.toml | 7 +++++++ tests/CrawlerProcess/reactor_default.py | 2 +- .../reactor_default_twisted_reactor_select.py | 2 +- tests/CrawlerRunner/change_reactor.py | 2 +- tests/CrawlerRunner/ip_address.py | 3 ++- tests/mockserver.py | 6 +++++- .../__init__.py | 2 +- tests/test_core_downloader.py | 5 ++++- .../test_downloader_handler_twisted_http2.py | 6 +++++- tests/test_downloader_handlers.py | 5 ++++- tests/test_downloader_handlers_http_base.py | 19 ++++++++++++++++++- tests/test_downloadermiddleware_robotstxt.py | 12 +++++++++++- tests/test_engine.py | 6 +++++- tests/test_http2_client_protocol.py | 3 ++- tests/test_pipeline_media.py | 3 ++- tests/test_utils_signal.py | 6 +++++- tests/test_webclient.py | 6 +++++- 18 files changed, 80 insertions(+), 17 deletions(-) diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py index 70c9003e5..734614aa5 100755 --- a/extras/qps-bench-server.py +++ b/extras/qps-bench-server.py @@ -2,7 +2,7 @@ from collections import deque from time import time -from twisted.internet import reactor +from twisted.internet import reactor # noqa: TID253 from twisted.web.resource import Resource from twisted.web.server import NOT_DONE_YET, Site diff --git a/pyproject.toml b/pyproject.toml index 68c1e07bb..871da8020 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -299,6 +299,8 @@ extend-select = [ "T10", # flake8-type-checking "TC", + # flake8-tidy-imports + "TID", # pyupgrade "UP", # pycodestyle warnings @@ -398,6 +400,11 @@ ignore = [ "SIM112", ] +[tool.ruff.lint.flake8-tidy-imports] +banned-module-level-imports = [ + "twisted.internet.reactor", +] + [tool.ruff.lint.per-file-ignores] # Circular import workarounds "scrapy/linkextractors/__init__.py" = ["E402"] diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index 8f59c035c..cbe6427ea 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -1,4 +1,4 @@ -from twisted.internet import reactor # noqa: F401 +from twisted.internet import reactor # noqa: F401,TID253 from twisted.python import log import scrapy diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index 9901dd634..f7802fbc6 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -1,4 +1,4 @@ -from twisted.internet import reactor # noqa: F401 +from twisted.internet import reactor # noqa: F401,TID253 from twisted.python import log import scrapy diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py index 6c0102241..c275e0583 100644 --- a/tests/CrawlerRunner/change_reactor.py +++ b/tests/CrawlerRunner/change_reactor.py @@ -26,7 +26,7 @@ runner = CrawlerRunner() d = runner.crawl(NoRequestsSpider) -from twisted.internet import reactor # noqa: E402 +from twisted.internet import reactor # noqa: E402,TID253 d.addBoth(callback=lambda _: reactor.stop()) reactor.run() diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 5e2184afb..207fc86ad 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -6,7 +6,6 @@ install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") from urllib.parse import urlparse -from twisted.internet import reactor from twisted.names import cache, resolve from twisted.names import hosts as hostsModule from twisted.names.client import Resolver @@ -44,6 +43,8 @@ class LocalhostSpider(Spider): if __name__ == "__main__": + from twisted.internet import reactor + with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: port = urlparse(mock_http_server.http_address).port url = f"http://not.a.real.domain:{port}/echo" diff --git a/tests/mockserver.py b/tests/mockserver.py index f5c12787a..e0ac127f2 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -13,7 +13,7 @@ from typing import TYPE_CHECKING from urllib.parse import urlencode from OpenSSL import SSL -from twisted.internet import defer, reactor, ssl +from twisted.internet import defer, ssl from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory @@ -114,6 +114,8 @@ class LeafResource(resource.Resource): isLeaf = True def deferRequest(self, request, delay, f, *a, **kw): + from twisted.internet import reactor + def _cancelrequest(_): # silence CancelledError d.addErrback(lambda _: None) @@ -378,6 +380,8 @@ def ssl_context_factory( if __name__ == "__main__": + from twisted.internet import reactor + parser = argparse.ArgumentParser() parser.add_argument( "-t", "--type", type=str, choices=("http", "dns"), default="http" diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py index 5006e3689..c6fdb13ea 100644 --- a/tests/test_cmdline_crawl_with_pipeline/__init__.py +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -2,7 +2,7 @@ import sys from pathlib import Path from subprocess import PIPE, Popen -from .. import TWISTED_KEEPS_TRACEBACKS +from tests import TWISTED_KEEPS_TRACEBACKS class TestCmdlineCrawlPipeline: diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 1bffd69ed..ef77f7843 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -8,7 +8,6 @@ from typing import Any import OpenSSL.SSL import pytest -from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest @@ -40,6 +39,8 @@ class TestContextFactoryBase(unittest.TestCase): context_factory = None def _listen(self, site): + from twisted.internet import reactor + return reactor.listenSSL( 0, site, @@ -71,6 +72,8 @@ class TestContextFactoryBase(unittest.TestCase): client_context_factory: BrowserLikePolicyForHTTPS, body: str | None = None, ) -> bytes: + from twisted.internet import reactor + agent = Agent(reactor, contextFactory=client_context_factory) body_producer = _RequestBodyProducer(body.encode()) if body else None response: TxResponse = await maybe_deferred_to_future( diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 46322a747..159f403d0 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -8,7 +8,7 @@ from unittest import mock import pytest from testfixtures import LogCapture -from twisted.internet import defer, error, reactor +from twisted.internet import defer, error from twisted.web import server from twisted.web.error import SchemeNotSupported from twisted.web.http import H2_ENABLED @@ -59,6 +59,8 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): @defer.inlineCallbacks def test_download_with_maxsize_very_large_file(self): + from twisted.internet import reactor + with mock.patch("scrapy.core.http2.stream.logger") as logger: request = Request(self.getURL("largechunkedfile")) @@ -207,6 +209,8 @@ class Https2ProxyTestCase(H2DownloadHandlerMixin, TestHttpProxyBase): expected_http_proxy_request_body = b"/" def setUp(self): + from twisted.internet import reactor + site = server.Site(UriResource(), timeout=None) self.port = reactor.listenSSL( 0, diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index fc6ac5aee..dacadb075 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -12,7 +12,6 @@ from unittest import mock import pytest from twisted.cred import checkers, credentials, portal -from twisted.internet import reactor from twisted.protocols.ftp import FTPFactory, FTPRealm from twisted.trial import unittest from w3lib.url import path_to_file_uri @@ -310,6 +309,8 @@ class TestFTPBase(unittest.TestCase): ) def setUp(self): + from twisted.internet import reactor + # setup dirs and test file self.directory = Path(mkdtemp()) userdir = self.directory / self.username @@ -451,6 +452,8 @@ class TestAnonymousFTP(TestFTPBase): req_meta = {} def setUp(self): + from twisted.internet import reactor + # setup dir and test file self.directory = Path(mkdtemp()) for filename, content in self.test_files: diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 46e5972f7..5eaf66966 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -12,7 +12,7 @@ from unittest import mock import pytest from testfixtures import LogCapture -from twisted.internet import defer, error, reactor +from twisted.internet import defer, error from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest from twisted.web import resource, server, static, util @@ -52,6 +52,8 @@ class ContentLengthHeaderResource(resource.Resource): class ChunkedResource(resource.Resource): def render(self, request): + from twisted.internet import reactor + def response(): request.write(b"chunked ") request.write(b"content\n") @@ -63,6 +65,8 @@ class ChunkedResource(resource.Resource): class BrokenChunkedResource(resource.Resource): def render(self, request): + from twisted.internet import reactor + def response(): request.write(b"chunked ") request.write(b"content\n") @@ -76,6 +80,8 @@ class BrokenChunkedResource(resource.Resource): class BrokenDownloadResource(resource.Resource): def render(self, request): + from twisted.internet import reactor + def response(): request.setHeader(b"Content-Length", b"20") request.write(b"partial") @@ -105,6 +111,8 @@ class EmptyContentTypeHeaderResource(resource.Resource): class LargeChunkedFileResource(resource.Resource): def render(self, request): + from twisted.internet import reactor + def response(): for i in range(1024): request.write(b"x" * 1024) @@ -133,6 +141,8 @@ class TestHttpBase(unittest.TestCase, ABC): raise NotImplementedError def setUp(self): + from twisted.internet import reactor + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) @@ -365,6 +375,9 @@ class TestHttp11Base(TestHttpBase): @defer.inlineCallbacks def test_download_with_maxsize_very_large_file(self): + from twisted.internet import reactor + + # TODO: the logger check is specific to scrapy.core.downloader.handlers.http11 with mock.patch("scrapy.core.downloader.handlers.http11.logger") as logger: request = Request(self.getURL("largechunkedfile")) @@ -501,6 +514,8 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC): raise NotImplementedError def setUp(self): + from twisted.internet import reactor + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) @@ -639,6 +654,8 @@ class TestHttpProxyBase(unittest.TestCase, ABC): raise NotImplementedError def setUp(self): + from twisted.internet import reactor + site = server.Site(UriResource(), timeout=None) wrapper = WrappingFactory(site) self.port = reactor.listenTCP(0, wrapper, interface="127.0.0.1") diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 9518f1835..04800896c 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING from unittest import mock import pytest -from twisted.internet import error, reactor +from twisted.internet import error from twisted.internet.defer import Deferred, maybeDeferred from twisted.python import failure from twisted.trial import unittest @@ -53,6 +53,8 @@ Disallow: /some/randome/page.html response = TextResponse("http://site.local/robots.txt", body=ROBOTS) def return_response(request): + from twisted.internet import reactor + deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -102,6 +104,8 @@ Disallow: /some/randome/page.html ) def return_response(request): + from twisted.internet import reactor + deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -124,6 +128,8 @@ Disallow: /some/randome/page.html response = Response("http://site.local/robots.txt") def return_response(request): + from twisted.internet import reactor + deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -145,6 +151,8 @@ Disallow: /some/randome/page.html err = error.DNSLookupError("Robotstxt address not found") def return_failure(request): + from twisted.internet import reactor + deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(err)) return deferred @@ -178,6 +186,8 @@ Disallow: /some/randome/page.html self.crawler.settings.set("ROBOTSTXT_OBEY", True) def ignore_request(request): + from twisted.internet import reactor + deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest())) return deferred diff --git a/tests/test_engine.py b/tests/test_engine.py index b60b510b2..1f79a081d 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -24,7 +24,7 @@ from urllib.parse import urlparse import attr from itemadapter import ItemAdapter from pydispatch import dispatcher -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.trial import unittest from twisted.web import server, static, util @@ -130,6 +130,8 @@ class ChangeCloseReasonSpider(MySpider): def start_test_site(debug=False): + from twisted.internet import reactor + root_dir = Path(tests_datadir, "test_site") r = static.File(str(root_dir)) r.putChild(b"redirect", util.Redirect(b"/redirected")) @@ -514,6 +516,8 @@ def test_request_scheduled_signal(caplog): if __name__ == "__main__": + from twisted.internet import reactor # pylint: disable=ungrouped-imports + if len(sys.argv) > 1 and sys.argv[1] == "runserver": start_test_site(debug=True) reactor.run() diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 7c1b38877..0605c2438 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -13,7 +13,6 @@ from unittest import mock, skipIf from urllib.parse import urlencode import pytest -from twisted.internet import reactor from twisted.internet.defer import ( CancelledError, Deferred, @@ -209,6 +208,8 @@ class TestHttps2ClientProtocol(TestCase): @inlineCallbacks def setUp(self): + from twisted.internet import reactor + # Initialize resource tree root = self._init_resource() self.site = Site(root, timeout=None) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index d915fc2a3..2d0db6e25 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -4,7 +4,6 @@ import warnings import pytest from testfixtures import LogCapture -from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from twisted.trial import unittest @@ -329,6 +328,8 @@ class TestMediaPipeline(TestBaseMediaPipeline): rsp1 = Response("http://url") def rsp1_func(): + from twisted.internet import reactor + dfd = Deferred().addCallback(_check_downloading) reactor.callLater(0.1, dfd.callback, rsp1) return dfd diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 6dff321da..9b3ce6b0b 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -3,7 +3,7 @@ import asyncio import pytest from pydispatch import dispatcher from testfixtures import LogCapture -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.python.failure import Failure from twisted.trial import unittest @@ -65,6 +65,8 @@ class SendCatchLogDeferredTest(TestSendCatchLog): class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): def ok_handler(self, arg, handlers_called): + from twisted.internet import reactor + handlers_called.add(self.ok_handler) assert arg == "test" d = defer.Deferred() @@ -97,6 +99,8 @@ class SendCatchLogAsyncTest(TestSendCatchLog): class SendCatchLogAsyncTest2(SendCatchLogAsyncTest): def ok_handler(self, arg, handlers_called): + from twisted.internet import reactor + handlers_called.add(self.ok_handler) assert arg == "test" d = defer.Deferred() diff --git a/tests/test_webclient.py b/tests/test_webclient.py index c3c03d6c3..e580d51ca 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -12,7 +12,7 @@ from urllib.parse import urlparse import OpenSSL.SSL import pytest -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.internet.defer import inlineCallbacks from twisted.internet.testing import StringTransport from twisted.protocols.policies import WrappingFactory @@ -205,6 +205,8 @@ class EncodingResource(resource.Resource): @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestWebClient(unittest.TestCase): def _listen(self, site): + from twisted.internet import reactor + return reactor.listenTCP(0, site, interface="127.0.0.1") def setUp(self): @@ -318,6 +320,8 @@ class TestWebClient(unittest.TestCase): assert b"404 - No Such Resource" in pageData def testFactoryInfo(self): + from twisted.internet import reactor + url = self.getURL("file") parsed = urlparse(url) factory = client.ScrapyHTTPClientFactory(Request(url)) From 05b3b205ce296c72063e05d15f5cae8047476ca2 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 28 May 2025 18:55:44 +0500 Subject: [PATCH 136/491] Add `AsyncCrawlerProcess` and `Crawler.crawl_async()` (#6817) * Add a basic Crawler.crawl_async(). * Add custom loop tests for *CrawlerRunner. * Add AsyncCrawlerProcess. * Update related docs. * Update practices.rst. * Address test failures. * Add a note about AsyncCrawler* to the docs about switching reactors. * Address feedback. * Update for TID253. * Simplify test_crawler_crawl_async_twice_parallel_unsupported(). --- conftest.py | 2 + docs/topics/api.rst | 13 +- docs/topics/asyncio.rst | 9 +- docs/topics/practices.rst | 55 +-- docs/topics/settings.rst | 32 +- scrapy/core/engine.py | 16 +- scrapy/crawler.py | 301 ++++++++++++---- scrapy/utils/defer.py | 17 +- scrapy/utils/reactor.py | 35 +- tests/AsyncCrawlerProcess/args_settings.py | 25 ++ .../asyncio_custom_loop.py | 20 ++ ...o_custom_loop_custom_settings_different.py | 23 ++ ...syncio_custom_loop_custom_settings_same.py | 23 ++ .../asyncio_deferred_signal.py | 48 +++ .../asyncio_enabled_no_reactor.py | 27 ++ .../asyncio_enabled_reactor.py | 53 +++ .../asyncio_enabled_reactor_different_loop.py | 29 ++ .../asyncio_enabled_reactor_same_loop.py | 31 ++ .../caching_hostname_resolver.py | 35 ++ .../caching_hostname_resolver_ipv6.py | 22 ++ .../default_name_resolver.py | 18 + tests/AsyncCrawlerProcess/multi.py | 17 + tests/AsyncCrawlerProcess/reactor_default.py | 18 + tests/AsyncCrawlerProcess/simple.py | 16 + tests/AsyncCrawlerProcess/sleeping.py | 20 ++ .../twisted_reactor_asyncio.py | 15 + .../twisted_reactor_custom_settings.py | 14 + .../twisted_reactor_custom_settings_same.py | 22 ++ .../twisted_reactor_custom_settings_select.py | 30 ++ .../custom_loop_different.py | 31 ++ tests/AsyncCrawlerRunner/custom_loop_same.py | 31 ++ .../asyncio_enabled_reactor_different_loop.py | 8 +- .../asyncio_enabled_reactor_same_loop.py | 8 +- tests/CrawlerRunner/custom_loop_different.py | 29 ++ tests/CrawlerRunner/custom_loop_same.py | 29 ++ tests/test_crawler.py | 323 +++++++++++------- tests/test_utils_reactor.py | 5 +- 37 files changed, 1201 insertions(+), 249 deletions(-) create mode 100644 tests/AsyncCrawlerProcess/args_settings.py create mode 100644 tests/AsyncCrawlerProcess/asyncio_custom_loop.py create mode 100644 tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_different.py create mode 100644 tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_same.py create mode 100644 tests/AsyncCrawlerProcess/asyncio_deferred_signal.py create mode 100644 tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py create mode 100644 tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py create mode 100644 tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py create mode 100644 tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py create mode 100644 tests/AsyncCrawlerProcess/caching_hostname_resolver.py create mode 100644 tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py create mode 100644 tests/AsyncCrawlerProcess/default_name_resolver.py create mode 100644 tests/AsyncCrawlerProcess/multi.py create mode 100644 tests/AsyncCrawlerProcess/reactor_default.py create mode 100644 tests/AsyncCrawlerProcess/simple.py create mode 100644 tests/AsyncCrawlerProcess/sleeping.py create mode 100644 tests/AsyncCrawlerProcess/twisted_reactor_asyncio.py create mode 100644 tests/AsyncCrawlerProcess/twisted_reactor_custom_settings.py create mode 100644 tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_same.py create mode 100644 tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py create mode 100644 tests/AsyncCrawlerRunner/custom_loop_different.py create mode 100644 tests/AsyncCrawlerRunner/custom_loop_same.py create mode 100644 tests/CrawlerRunner/custom_loop_different.py create mode 100644 tests/CrawlerRunner/custom_loop_same.py diff --git a/conftest.py b/conftest.py index 18132b7e6..ed7d14166 100644 --- a/conftest.py +++ b/conftest.py @@ -19,6 +19,8 @@ collect_ignore = [ "tests/mockserver.py", "tests/pipelines.py", "tests/spiders.py", + # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess + *_py_files("tests/AsyncCrawlerProcess"), # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess *_py_files("tests/AsyncCrawlerRunner"), # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 3e7bc45c5..b11de2914 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -99,13 +99,11 @@ how you :ref:`configure the downloader middlewares provided while constructing the crawler, and it is created after the arguments given in the :meth:`crawl` method. - .. method:: crawl(*args, **kwargs) + .. automethod:: crawl_async - Starts the crawler by instantiating its spider class with the given - ``args`` and ``kwargs`` arguments, while setting the execution engine in - motion. Should be called only once. + .. automethod:: crawl - Returns a deferred that is fired when the crawl is finished. + .. automethod:: stop_async .. automethod:: stop @@ -115,6 +113,11 @@ how you :ref:`configure the downloader middlewares .. autoclass:: CrawlerRunner :members: +.. autoclass:: AsyncCrawlerProcess + :show-inheritance: + :members: + :inherited-members: + .. autoclass:: CrawlerProcess :show-inheritance: :members: diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 473ef7bfa..ad5c71fbf 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -20,7 +20,8 @@ To enable :mod:`asyncio` support, your :setting:`TWISTED_REACTOR` setting needs to be set to ``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, which is the default value. -If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to +If you are using :class:`~scrapy.crawler.AsyncCrawlerRunner` or +:class:`~scrapy.crawler.CrawlerRunner`, you also need to install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` reactor manually. You can do that using :func:`~scrapy.utils.reactor.install_reactor`: @@ -169,4 +170,8 @@ Switching to a non-asyncio reactor If for some reason your code doesn't work with the asyncio reactor, you can use a different reactor by setting the :setting:`TWISTED_REACTOR` setting to its import path (e.g. ``'twisted.internet.epollreactor.EPollReactor'``) or to -``None``, which will use the default reactor for your platform. +``None``, which will use the default reactor for your platform. If you are +using :class:`~scrapy.crawler.AsyncCrawlerRunner` or +:class:`~scrapy.crawler.AsyncCrawlerProcess` you also need to switch to their +Deferred-based counterparts: :class:`~scrapy.crawler.CrawlerRunner` or +:class:`~scrapy.crawler.CrawlerProcess` respectively. diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 18005aaf2..56177ba4e 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -21,16 +21,21 @@ Remember that Scrapy is built on top of the Twisted asynchronous networking library, so you need to run it inside the Twisted reactor. The first utility you can use to run your spiders is -:class:`scrapy.crawler.CrawlerProcess`. This class will start a Twisted reactor -for you, configuring the logging and setting shutdown handlers. This class is -the one used by all Scrapy commands. +:class:`scrapy.crawler.AsyncCrawlerProcess` or +:class:`scrapy.crawler.CrawlerProcess`. These classes will start a Twisted +reactor for you, configuring the logging and setting shutdown handlers. These +classes are the ones used by all Scrapy commands. They have similar +functionality, differing in their asynchronous API style: +:class:`~scrapy.crawler.AsyncCrawlerProcess` returns coroutines from its +asynchronous methods while :class:`~scrapy.crawler.CrawlerProcess` returns +:class:`~twisted.internet.defer.Deferred` objects. Here's an example showing how to run a single spider with it. .. code-block:: python import scrapy - from scrapy.crawler import CrawlerProcess + from scrapy.crawler import AsyncCrawlerProcess class MySpider(scrapy.Spider): @@ -38,7 +43,7 @@ Here's an example showing how to run a single spider with it. ... - process = CrawlerProcess( + process = AsyncCrawlerProcess( settings={ "FEEDS": { "items.json": {"format": "json"}, @@ -49,53 +54,57 @@ Here's an example showing how to run a single spider with it. process.crawl(MySpider) process.start() # the script will block here until the crawling is finished -Define settings within dictionary in CrawlerProcess. Make sure to check :class:`~scrapy.crawler.CrawlerProcess` +You can define :ref:`settings ` within the dictionary passed +to :class:`~scrapy.crawler.AsyncCrawlerProcess`. Make sure to check the +:class:`~scrapy.crawler.AsyncCrawlerProcess` documentation to get acquainted with its usage details. If you are inside a Scrapy project there are some additional helpers you can use to import those components within the project. You can automatically import -your spiders passing their name to :class:`~scrapy.crawler.CrawlerProcess`, and -use ``get_project_settings`` to get a :class:`~scrapy.settings.Settings` -instance with your project settings. +your spiders passing their name to +:class:`~scrapy.crawler.AsyncCrawlerProcess`, and use +:func:`scrapy.utils.project.get_project_settings` to get a +:class:`~scrapy.settings.Settings` instance with your project settings. What follows is a working example of how to do that, using the `testspiders`_ project as example. .. code-block:: python - from scrapy.crawler import CrawlerProcess + from scrapy.crawler import AsyncCrawlerProcess from scrapy.utils.project import get_project_settings - process = CrawlerProcess(get_project_settings()) + process = AsyncCrawlerProcess(get_project_settings()) # 'followall' is the name of one of the spiders of the project. process.crawl("followall", domain="scrapy.org") process.start() # the script will block here until the crawling is finished There's another Scrapy utility that provides more control over the crawling -process: :class:`scrapy.crawler.AsyncCrawlerRunner` and +process: :class:`scrapy.crawler.AsyncCrawlerRunner` or :class:`scrapy.crawler.CrawlerRunner`. These classes are thin wrappers that encapsulate some simple helpers to run multiple crawlers, but they won't -start or interfere with existing reactors in any way. They have similar -functionality, differing in their asynchronous API style: -:class:`~scrapy.crawler.AsyncCrawlerRunner` returns coroutines from its -asynchronous methods while :class:`~scrapy.crawler.CrawlerRunner` returns -:class:`~twisted.internet.defer.Deferred` objects. +start or interfere with existing reactors in any way. Just like +:class:`scrapy.crawler.AsyncCrawlerProcess` and +:class:`scrapy.crawler.CrawlerProcess` they differ in their asynchronous API +style. When using these classes the reactor should be explicitly run after scheduling your spiders. It's recommended that you use :class:`~scrapy.crawler.AsyncCrawlerRunner` or :class:`~scrapy.crawler.CrawlerRunner` instead of +:class:`~scrapy.crawler.AsyncCrawlerProcess` or :class:`~scrapy.crawler.CrawlerProcess` if your application is already using Twisted and you want to run Scrapy in the same reactor. If you want to stop the reactor or run any other code right after the spider -finishes you can do that after the :meth:`AsyncCrawlerRunner.crawl() -` coroutine completes (or the Deferred -returned from :meth:`CrawlerRunner.crawl() +finishes you can do that after the task returned from +:meth:`AsyncCrawlerRunner.crawl() ` +completes (or the Deferred returned from :meth:`CrawlerRunner.crawl() ` fires). In the simplest case you can also use :func:`twisted.internet.task.react` to start and stop the reactor, though -it may be easier to just use :class:`~scrapy.crawler.CrawlerProcess` instead. +it may be easier to just use :class:`~scrapy.crawler.AsyncCrawlerProcess` or +:class:`~scrapy.crawler.CrawlerProcess` instead. Here's an example of using :class:`~scrapy.crawler.AsyncCrawlerRunner` together with simple reactor management code: @@ -171,7 +180,7 @@ Here is an example that runs multiple spiders simultaneously: .. code-block:: python import scrapy - from scrapy.crawler import CrawlerProcess + from scrapy.crawler import AsyncCrawlerProcess from scrapy.utils.project import get_project_settings @@ -186,7 +195,7 @@ Here is an example that runs multiple spiders simultaneously: settings = get_project_settings() - process = CrawlerProcess(settings) + process = AsyncCrawlerProcess(settings) process.crawl(MySpider1) process.crawl(MySpider2) process.start() # the script will block here until all crawling jobs are finished diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0f81a0c0a..68c5079cf 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -330,7 +330,8 @@ exception is raised. These settings are: -- :setting:`ASYNCIO_EVENT_LOOP` +- :setting:`ASYNCIO_EVENT_LOOP` (not possible to set per-spider when using + :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below) - :setting:`DNS_RESOLVER` and settings used by the corresponding component, e.g. :setting:`DNSCACHE_ENABLED`, :setting:`DNSCACHE_SIZE` @@ -338,12 +339,25 @@ These settings are: - :setting:`REACTOR_THREADPOOL_MAXSIZE` -- :setting:`TWISTED_REACTOR` +- :setting:`TWISTED_REACTOR` (ignored when using + :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below) :setting:`ASYNCIO_EVENT_LOOP` and :setting:`TWISTED_REACTOR` are used upon installing the reactor. The rest of the settings are applied when starting the reactor. +There is an additional restriction for :setting:`TWISTED_REACTOR` and +:setting:`ASYNCIO_EVENT_LOOP` when using +:class:`~scrapy.crawler.AsyncCrawlerProcess`: when this class is instantiated, +it installs :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, +ignoring the value of :setting:`TWISTED_REACTOR` and using the value of +:setting:`ASYNCIO_EVENT_LOOP` that was passed to +:meth:`AsyncCrawlerProcess.__init__() +`. If a different value for +:setting:`TWISTED_REACTOR` or :setting:`ASYNCIO_EVENT_LOOP` is provided later, +e.g. in :ref:`per-spider settings `, an exception will be +raised. + .. _topics-settings-ref: @@ -1977,9 +1991,11 @@ Import path of a given :mod:`~twisted.internet.reactor`. Scrapy will install this reactor if no other reactor is installed yet, such as when the ``scrapy`` CLI program is invoked or when using the +:class:`~scrapy.crawler.AsyncCrawlerProcess` class or the :class:`~scrapy.crawler.CrawlerProcess` class. -If you are using the :class:`~scrapy.crawler.CrawlerRunner` class, you also +If you are using the :class:`~scrapy.crawler.AsyncCrawlerRunner` class or the +:class:`~scrapy.crawler.CrawlerRunner` class, you also need to install the correct reactor manually. You can do that using :func:`~scrapy.utils.reactor.install_reactor`: @@ -1988,12 +2004,12 @@ need to install the correct reactor manually. You can do that using If a reactor is already installed, :func:`~scrapy.utils.reactor.install_reactor` has no effect. -:meth:`CrawlerRunner.__init__ ` raises -:exc:`Exception` if the installed reactor does not match the +:class:`~scrapy.crawler.AsyncCrawlerRunner` and other similar classes raise an +exception if the installed reactor does not match the :setting:`TWISTED_REACTOR` setting; therefore, having top-level :mod:`~twisted.internet.reactor` imports in project files and imported -third-party libraries will make Scrapy raise :exc:`Exception` when -it checks which reactor is installed. +third-party libraries will make Scrapy raise an exception when it checks which +reactor is installed. In order to use the reactor installed by Scrapy: @@ -2025,7 +2041,7 @@ In order to use the reactor installed by Scrapy: self.crawler.engine.close_spider(self, "timeout") -which raises :exc:`Exception`, becomes: +which raises an exception, becomes: .. code-block:: python diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b0d9a5452..721c81d81 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -22,6 +22,7 @@ from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response from scrapy.utils.defer import ( deferred_f_from_coro_f, + deferred_from_coro, maybe_deferred_to_future, ) from scrapy.utils.log import failure_to_exc_info, logformatter_adapter @@ -122,8 +123,10 @@ class ExecutionEngine: ) return scheduler_cls - @deferred_f_from_coro_f - async def start(self, _start_request_processing=True) -> None: + def start(self, _start_request_processing=True) -> Deferred[None]: + return deferred_from_coro(self.start_async(_start_request_processing)) + + async def start_async(self, _start_request_processing=True) -> None: if self.running: raise RuntimeError("Engine already running") self.start_time = time() @@ -392,10 +395,15 @@ class ExecutionEngine: finally: self._slot.nextcall.schedule() - @deferred_f_from_coro_f - async def open_spider( + def open_spider(self, spider: Spider, close_if_idle: bool = True) -> Deferred[None]: + return deferred_from_coro( + self.open_spider_async(spider, close_if_idle=close_if_idle) + ) + + async def open_spider_async( self, spider: Spider, + *, close_if_idle: bool = True, ) -> None: if self._slot is not None: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1d6532fa9..c22b8603b 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -21,7 +21,8 @@ from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader from scrapy.settings import BaseSettings, Settings, overridden_settings from scrapy.signalmanager import SignalManager -from scrapy.utils.defer import deferred_to_future +from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.defer import deferred_from_coro, deferred_to_future from scrapy.utils.log import ( LogCounterHandler, configure_logging, @@ -33,8 +34,10 @@ from scrapy.utils.log import ( from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.reactor import ( + _asyncio_reactor_path, install_reactor, is_asyncio_reactor_installed, + is_reactor_installed, verify_installed_asyncio_event_loop, verify_installed_reactor, ) @@ -142,6 +145,12 @@ class Crawler: # this method. @inlineCallbacks def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred[Any], Any, None]: + """Start the crawler by instantiating its spider class with the given + *args* and *kwargs* arguments, while setting the execution engine in + motion. Should be called only once. + + Return a deferred that is fired when the crawl is finished. + """ if self.crawling: raise RuntimeError("Crawling already taking place") if self._started: @@ -163,6 +172,42 @@ class Crawler: yield self.engine.close() raise + async def crawl_async(self, *args: Any, **kwargs: Any) -> None: + """Start the crawler by instantiating its spider class with the given + *args* and *kwargs* arguments, while setting the execution engine in + motion. Should be called only once. + + .. versionadded:: VERSION + + Complete when the crawl is finished. + + This function requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be + installed. + """ + if not is_asyncio_available(): + raise RuntimeError("Crawler.crawl_async() requires AsyncioSelectorReactor.") + if self.crawling: + raise RuntimeError("Crawling already taking place") + if self._started: + raise RuntimeError( + "Cannot run Crawler.crawl_async() more than once on the same instance." + ) + self.crawling = self._started = True + + try: + self.spider = self._create_spider(*args, **kwargs) + self._apply_settings() + self._update_root_log_handler() + self.engine = self._create_engine() + await self.engine.open_spider_async(self.spider) + await self.engine.start_async() + except Exception: + self.crawling = False + if self.engine is not None: + await deferred_to_future(self.engine.close()) + raise + def _create_spider(self, *args: Any, **kwargs: Any) -> Spider: return self.spidercls.from_crawler(self, *args, **kwargs) @@ -171,13 +216,26 @@ class Crawler: @inlineCallbacks def stop(self) -> Generator[Deferred[Any], Any, None]: - """Starts a graceful stop of the crawler and returns a deferred that is + """Start a graceful stop of the crawler and return a deferred that is fired when the crawler is stopped.""" if self.crawling: self.crawling = False assert self.engine yield self.engine.stop() + async def stop_async(self) -> None: + """Start a graceful stop of the crawler and complete when the crawler is stopped. + + .. versionadded:: VERSION + + This function requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be + installed. + """ + if not is_asyncio_available(): + raise RuntimeError("Crawler.stop_async() requires AsyncioSelectorReactor.") + await deferred_to_future(self.stop()) + @staticmethod def _get_component( component_class: type[_T], components: Iterable[Any] @@ -318,9 +376,6 @@ class CrawlerRunnerBase: spidercls = self.spider_loader.load(spidercls) return Crawler(spidercls, self.settings) - def _stop(self) -> Deferred[Any]: - return DeferredList([c.stop() for c in list(self.crawlers)]) - class CrawlerRunner(CrawlerRunnerBase): """ @@ -397,7 +452,7 @@ class CrawlerRunner(CrawlerRunnerBase): Returns a deferred that is fired when they all have ended. """ - return self._stop() + return DeferredList(c.stop() for c in self.crawlers) @inlineCallbacks def join(self) -> Generator[Deferred[Any], Any, None]: @@ -429,14 +484,14 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): def __init__(self, settings: dict[str, Any] | Settings | None = None): super().__init__(settings) - self._active: set[asyncio.Future[None]] = set() + self._active: set[asyncio.Task[None]] = set() def crawl( self, crawler_or_spidercls: type[Spider] | str | Crawler, *args: Any, **kwargs: Any, - ) -> asyncio.Future[None]: + ) -> asyncio.Task[None]: """ Run a crawler with the provided arguments. @@ -447,7 +502,7 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): instance, this method will try to create one using this parameter as the spider class given to it. - Returns a :class:`~asyncio.Future` object which completes when the + Returns a :class:`~asyncio.Task` object which completes when the crawling is finished. :param crawler_or_spidercls: already created crawler, or a spider class @@ -465,24 +520,27 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): "it must be a spider class (or a Crawler object)" ) if not is_asyncio_reactor_installed(): - raise RuntimeError("AsyncCrawlerRunner requires AsyncioSelectorReactor.") + raise RuntimeError( + f"{type(self).__name__} requires AsyncioSelectorReactor." + ) crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) - def _crawl( - self, crawler: Crawler, *args: Any, **kwargs: Any - ) -> asyncio.Future[None]: + def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> asyncio.Task[None]: + # At this point the asyncio loop has been installed either by the user + # or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()). + loop = asyncio.get_event_loop() self.crawlers.add(crawler) - future = deferred_to_future(crawler.crawl(*args, **kwargs)) - self._active.add(future) + task = loop.create_task(crawler.crawl_async(*args, **kwargs)) + self._active.add(task) - def _done(_: asyncio.Future[None]) -> None: + def _done(_: asyncio.Task[None]) -> None: self.crawlers.discard(crawler) - self._active.discard(future) + self._active.discard(task) self.bootstrap_failed |= not getattr(crawler, "spider", None) - future.add_done_callback(_done) - return future + task.add_done_callback(_done) + return task async def stop(self) -> None: """ @@ -490,7 +548,10 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): Completes when they all have ended. """ - await deferred_to_future(self._stop()) + if self.crawlers: + await asyncio.wait( + [asyncio.create_task(c.stop_async()) for c in self.crawlers] + ) async def join(self) -> None: """ @@ -498,33 +559,10 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): executions. """ while self._active: - await asyncio.gather(*self._active) + await asyncio.wait(self._active) -class CrawlerProcess(CrawlerRunner): - """ - A class to run multiple scrapy crawlers in a process simultaneously. - - This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support - for starting a :mod:`~twisted.internet.reactor` and handling shutdown - signals, like the keyboard interrupt command Ctrl-C. It also configures - top-level logging. - - This utility should be a better fit than - :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another - :mod:`~twisted.internet.reactor` within your application. - - The CrawlerProcess object must be instantiated with a - :class:`~scrapy.settings.Settings` object. - - :param install_root_handler: whether to install root logging handler - (default: True) - - This class shouldn't be needed (since Scrapy is responsible of using it - accordingly) unless writing scripts that manually handle the crawling - process. See :ref:`run-from-script` for an example. - """ - +class CrawlerProcessBase(CrawlerRunnerBase): def __init__( self, settings: dict[str, Any] | Settings | None = None, @@ -533,7 +571,6 @@ class CrawlerProcess(CrawlerRunner): super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) - self._initialized_reactor: bool = False def _signal_shutdown(self, signum: int, _: Any) -> None: from twisted.internet import reactor @@ -556,6 +593,75 @@ class CrawlerProcess(CrawlerRunner): ) reactor.callFromThread(self._stop_reactor) + def _setup_reactor(self, install_signal_handlers: bool) -> None: + from twisted.internet import reactor + + resolver_class = load_object(self.settings["DNS_RESOLVER"]) + # We pass self, which is CrawlerProcess, instead of Crawler here, + # which works because the default resolvers only use crawler.settings. + resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[arg-type] + resolver.install_on_reactor() + tp = reactor.getThreadPool() + tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) + reactor.addSystemEventTrigger("before", "shutdown", self._stop_dfd) + if install_signal_handlers: + reactor.addSystemEventTrigger( + "after", "startup", install_shutdown_handlers, self._signal_shutdown + ) + + def _stop_dfd(self) -> Deferred[Any]: + raise NotImplementedError + + @inlineCallbacks + def _graceful_stop_reactor(self) -> Generator[Deferred[Any], Any, None]: + try: + yield self._stop_dfd() + finally: + self._stop_reactor() + + def _stop_reactor(self, _: Any = None) -> None: + from twisted.internet import reactor + + # raised if already stopped or in shutdown stage + with contextlib.suppress(RuntimeError): + reactor.stop() + + +class CrawlerProcess(CrawlerProcessBase, CrawlerRunner): + """ + A class to run multiple scrapy crawlers in a process simultaneously. + + This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support + for starting a :mod:`~twisted.internet.reactor` and handling shutdown + signals, like the keyboard interrupt command Ctrl-C. It also configures + top-level logging. + + This utility should be a better fit than + :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another + :mod:`~twisted.internet.reactor` within your application. + + The CrawlerProcess object must be instantiated with a + :class:`~scrapy.settings.Settings` object. + + :param install_root_handler: whether to install root logging handler + (default: True) + + This class shouldn't be needed (since Scrapy is responsible of using it + accordingly) unless writing scripts that manually handle the crawling + process. See :ref:`run-from-script` for an example. + + This class provides Deferred-based APIs. Use :class:`AsyncCrawlerProcess` + for modern coroutine APIs. + """ + + def __init__( + self, + settings: dict[str, Any] | Settings | None = None, + install_root_handler: bool = True, + ): + super().__init__(settings, install_root_handler) + self._initialized_reactor: bool = False + def _create_crawler(self, spidercls: type[Spider] | str) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) @@ -563,6 +669,9 @@ class CrawlerProcess(CrawlerRunner): self._initialized_reactor = True return Crawler(spidercls, self.settings, init_reactor=init_reactor) + def _stop_dfd(self) -> Deferred[Any]: + return self.stop() + def start( self, stop_after_crawl: bool = True, install_signal_handlers: bool = True ) -> None: @@ -589,30 +698,86 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) - resolver_class = load_object(self.settings["DNS_RESOLVER"]) - # We pass self, which is CrawlerProcess, instead of Crawler here, - # which works because the default resolvers only use crawler.settings. - resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[arg-type] - resolver.install_on_reactor() - tp = reactor.getThreadPool() - tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) - reactor.addSystemEventTrigger("before", "shutdown", self.stop) - if install_signal_handlers: - reactor.addSystemEventTrigger( - "after", "startup", install_shutdown_handlers, self._signal_shutdown - ) + self._setup_reactor(install_signal_handlers) reactor.run(installSignalHandlers=install_signal_handlers) # blocking call - @inlineCallbacks - def _graceful_stop_reactor(self) -> Generator[Deferred[Any], Any, None]: - try: - yield self.stop() - finally: - self._stop_reactor() - def _stop_reactor(self, _: Any = None) -> None: +class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): + """ + A class to run multiple scrapy crawlers in a process simultaneously. + + This class extends :class:`~scrapy.crawler.AsyncCrawlerRunner` by adding support + for starting a :mod:`~twisted.internet.reactor` and handling shutdown + signals, like the keyboard interrupt command Ctrl-C. It also configures + top-level logging. + + This utility should be a better fit than + :class:`~scrapy.crawler.AsyncCrawlerRunner` if you aren't running another + :mod:`~twisted.internet.reactor` within your application. + + The AsyncCrawlerProcess object must be instantiated with a + :class:`~scrapy.settings.Settings` object. + + :param install_root_handler: whether to install root logging handler + (default: True) + + This class shouldn't be needed (since Scrapy is responsible of using it + accordingly) unless writing scripts that manually handle the crawling + process. See :ref:`run-from-script` for an example. + + This class provides coroutine APIs. It requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. + """ + + def __init__( + self, + settings: dict[str, Any] | Settings | None = None, + install_root_handler: bool = True, + ): + super().__init__(settings, install_root_handler) + # We want the asyncio event loop to be installed early, so that it's + # always the correct one. And as we do that, we can also install the + # reactor here. + # The ASYNCIO_EVENT_LOOP setting cannot be overridden by add-ons and + # spiders when using AsyncCrawlerProcess. + loop_path = self.settings["ASYNCIO_EVENT_LOOP"] + if is_reactor_installed(): + # The user could install a reactor before this class is instantiated. + # We need to make sure the reactor is the correct one and the loop + # type matches the setting. + verify_installed_reactor(_asyncio_reactor_path) + if loop_path: + verify_installed_asyncio_event_loop(loop_path) + else: + install_reactor(_asyncio_reactor_path, loop_path) + self._initialized_reactor = True + + def _stop_dfd(self) -> Deferred[Any]: + return deferred_from_coro(self.stop()) + + def start( + self, stop_after_crawl: bool = True, install_signal_handlers: bool = True + ) -> None: + """ + This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool + size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache + based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. + + If ``stop_after_crawl`` is True, the reactor will be stopped after all + crawlers have finished, using :meth:`join`. + + :param bool stop_after_crawl: stop or not the reactor when all + crawlers have finished + + :param bool install_signal_handlers: whether to install the OS signal + handlers from Twisted and Scrapy (default: True) + """ from twisted.internet import reactor - # raised if already stopped or in shutdown stage - with contextlib.suppress(RuntimeError): - reactor.stop() + if stop_after_crawl: + loop = asyncio.get_event_loop() + join_task = loop.create_task(self.join()) + join_task.add_done_callback(self._stop_reactor) + + self._setup_reactor(install_signal_handlers) + reactor.run(installSignalHandlers=install_signal_handlers) # blocking call diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 4649c4daa..2df82c8f2 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -23,7 +23,6 @@ from twisted.python import failure from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.utils.asyncio import is_asyncio_available -from scrapy.utils.reactor import _get_asyncio_event_loop if TYPE_CHECKING: from collections.abc import AsyncIterator, Callable @@ -385,8 +384,7 @@ def deferred_from_coro(o: Awaitable[_T] | _T2) -> Deferred[_T] | _T2: # that use asyncio, e.g. "await asyncio.sleep(1)" return Deferred.fromCoroutine(cast(Coroutine[Deferred[Any], Any, _T], o)) # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor - event_loop = _get_asyncio_event_loop() - return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) + return Deferred.fromFuture(asyncio.ensure_future(o)) return o @@ -430,6 +428,10 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]: Return an :class:`asyncio.Future` object that wraps *d*. + This function requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be + installed. + When :ref:`using the asyncio reactor `, you cannot await on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy callables defined as coroutines `, you can only await on @@ -442,8 +444,15 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]: additional_request = scrapy.Request('https://example.org/price') deferred = self.crawler.engine.download(additional_request) additional_response = await deferred_to_future(deferred) + + .. versionchanged:: VERSION + This function no longer installs an asyncio loop if called before the + Twisted asyncio reactor is installed. A :exc:`RuntimeError` is raised + in this case. """ - return d.asFuture(_get_asyncio_event_loop()) + if not is_asyncio_available(): + raise RuntimeError("deferred_to_future() requires AsyncioSelectorReactor.") + return d.asFuture(asyncio.get_event_loop()) def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 5e76da37b..2fb1e0ce7 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -10,6 +10,7 @@ from twisted.internet import asyncioreactor, error from twisted.internet.defer import Deferred from scrapy.utils.misc import load_object +from scrapy.utils.python import global_object_name if TYPE_CHECKING: from asyncio import AbstractEventLoop, AbstractEventLoopPolicy @@ -87,6 +88,9 @@ class CallLaterOnce(Generic[_T]): await maybe_deferred_to_future(d) +_asyncio_reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" + + def set_asyncio_event_loop_policy() -> None: """The policy functions from asyncio often behave unexpectedly, so we restrict their use to the absolutely essential case. @@ -161,21 +165,34 @@ def set_asyncio_event_loop(event_loop_path: str | None) -> AbstractEventLoop: def verify_installed_reactor(reactor_path: str) -> None: - """Raises :exc:`Exception` if the installed + """Raise :exc:`RuntimeError` if the installed :mod:`~twisted.internet.reactor` does not match the specified import - path.""" + path or if no reactor is installed.""" + if not is_reactor_installed(): + raise RuntimeError( + "verify_installed_reactor() called without an installed reactor." + ) + from twisted.internet import reactor - reactor_class = load_object(reactor_path) - if not reactor.__class__ == reactor_class: + expected_reactor_type = load_object(reactor_path) + reactor_type = type(reactor) + if not reactor_type == expected_reactor_type: raise RuntimeError( - "The installed reactor " - f"({reactor.__module__}.{reactor.__class__.__name__}) does not " - f"match the requested one ({reactor_path})" + f"The installed reactor ({global_object_name(reactor_type)}) " + f"does not match the requested one ({reactor_path})" ) def verify_installed_asyncio_event_loop(loop_path: str) -> None: + """Raise :exc:`RuntimeError` if the even loop of the installed + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` + does not match the specified import path or if no reactor is installed.""" + if not is_reactor_installed(): + raise RuntimeError( + "verify_installed_asyncio_event_loop() called without an installed reactor." + ) + from twisted.internet import reactor loop_class = load_object(loop_path) @@ -185,16 +202,16 @@ def verify_installed_asyncio_event_loop(loop_path: str) -> None: f"{reactor._asyncioEventloop.__class__.__module__}" f".{reactor._asyncioEventloop.__class__.__qualname__}" ) - specified = f"{loop_class.__module__}.{loop_class.__qualname__}" raise RuntimeError( "Scrapy found an asyncio Twisted reactor already " f"installed, and its event loop class ({installed}) does " "not match the one specified in the ASYNCIO_EVENT_LOOP " - f"setting ({specified})" + f"setting ({global_object_name(loop_class)})" ) def is_reactor_installed() -> bool: + """Check whether a :mod:`~twisted.internet.reactor` is installed.""" return "twisted.internet.reactor" in sys.modules diff --git a/tests/AsyncCrawlerProcess/args_settings.py b/tests/AsyncCrawlerProcess/args_settings.py new file mode 100644 index 000000000..5e162e782 --- /dev/null +++ b/tests/AsyncCrawlerProcess/args_settings.py @@ -0,0 +1,25 @@ +from typing import Any + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess, Crawler + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + @classmethod + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("FOO", kwargs.get("foo")) + return spider + + async def start(self): + self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") + return + yield + + +process = AsyncCrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider, foo=42) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_custom_loop.py b/tests/AsyncCrawlerProcess/asyncio_custom_loop.py new file mode 100644 index 000000000..172e36b7b --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_custom_loop.py @@ -0,0 +1,20 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_different.py b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_different.py new file mode 100644 index 000000000..d76da51a1 --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_different.py @@ -0,0 +1,23 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + custom_settings = { + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": None, + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_same.py b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_same.py new file mode 100644 index 000000000..bd4a99e14 --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_same.py @@ -0,0 +1,23 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + custom_settings = { + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py new file mode 100644 index 000000000..c32aaf37d --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import asyncio +import sys + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.defer import deferred_from_coro + + +class UppercasePipeline: + async def _open_spider(self, spider): + spider.logger.info("async pipeline opened!") + await asyncio.sleep(0.1) + + def open_spider(self, spider): + return deferred_from_coro(self._open_spider(spider)) + + def process_item(self, item, spider): + return {"url": item["url"].upper()} + + +class UrlSpider(Spider): + name = "url_spider" + start_urls = ["data:,"] + custom_settings = { + "ITEM_PIPELINES": {UppercasePipeline: 100}, + } + + def parse(self, response): + yield {"url": response.url} + + +if __name__ == "__main__": + ASYNCIO_EVENT_LOOP: str | None + try: + ASYNCIO_EVENT_LOOP = sys.argv[1] + except IndexError: + ASYNCIO_EVENT_LOOP = None + + process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP, + } + ) + process.crawl(UrlSpider) + process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py b/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py new file mode 100644 index 000000000..3c47eb826 --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py @@ -0,0 +1,27 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.reactor import is_asyncio_reactor_installed + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py new file mode 100644 index 000000000..e025e17d1 --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py @@ -0,0 +1,53 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + is_reactor_installed, +) + +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed before is_asyncio_reactor_installed()." + ) + +try: + is_asyncio_reactor_installed() +except RuntimeError: + pass +else: + raise RuntimeError("is_asyncio_reactor_installed() did not raise RuntimeError.") + +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed after is_asyncio_reactor_installed()." + ) + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + +if not is_asyncio_reactor_installed(): + raise RuntimeError("Wrong reactor installed after install_reactor().") + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py new file mode 100644 index 000000000..4257bc0ac --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -0,0 +1,29 @@ +import asyncio +import sys + +from twisted.internet import asyncioreactor + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + +if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +asyncioreactor.install(asyncio.get_event_loop()) + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py new file mode 100644 index 000000000..9c6fd090b --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -0,0 +1,31 @@ +import asyncio +import sys + +from twisted.internet import asyncioreactor +from uvloop import Loop + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + +if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +asyncio.set_event_loop(Loop()) +asyncioreactor.install(asyncio.get_event_loop()) + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py new file mode 100644 index 000000000..5f75d5e17 --- /dev/null +++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py @@ -0,0 +1,35 @@ +import sys + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class CachingHostnameResolverSpider(scrapy.Spider): + """ + Finishes in a finite amount of time (does not hang indefinitely in the DNS resolution) + """ + + name = "caching_hostname_resolver_spider" + + async def start(self): + yield scrapy.Request(self.url) + + def parse(self, response): + for _ in range(10): + yield scrapy.Request( + response.url, dont_filter=True, callback=self.ignore_response + ) + + def ignore_response(self, response): + self.logger.info(repr(response.ip_address)) + + +if __name__ == "__main__": + process = AsyncCrawlerProcess( + settings={ + "RETRY_ENABLED": False, + "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + } + ) + process.crawl(CachingHostnameResolverSpider, url=sys.argv[1]) + process.start() diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py new file mode 100644 index 000000000..c43f0a9c2 --- /dev/null +++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class CachingHostnameResolverSpider(scrapy.Spider): + """ + Finishes without a twisted.internet.error.DNSLookupError exception + """ + + name = "caching_hostname_resolver_spider" + start_urls = ["http://[::1]"] + + +if __name__ == "__main__": + process = AsyncCrawlerProcess( + settings={ + "RETRY_ENABLED": False, + "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + } + ) + process.crawl(CachingHostnameResolverSpider) + process.start() diff --git a/tests/AsyncCrawlerProcess/default_name_resolver.py b/tests/AsyncCrawlerProcess/default_name_resolver.py new file mode 100644 index 000000000..af56ccd01 --- /dev/null +++ b/tests/AsyncCrawlerProcess/default_name_resolver.py @@ -0,0 +1,18 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class IPv6Spider(scrapy.Spider): + """ + Raises a twisted.internet.error.DNSLookupError: + the default name resolver does not handle IPv6 addresses. + """ + + name = "ipv6_spider" + start_urls = ["http://[::1]"] + + +if __name__ == "__main__": + process = AsyncCrawlerProcess(settings={"RETRY_ENABLED": False}) + process.crawl(IPv6Spider) + process.start() diff --git a/tests/AsyncCrawlerProcess/multi.py b/tests/AsyncCrawlerProcess/multi.py new file mode 100644 index 000000000..2eede5471 --- /dev/null +++ b/tests/AsyncCrawlerProcess/multi.py @@ -0,0 +1,17 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactor_default.py b/tests/AsyncCrawlerProcess/reactor_default.py new file mode 100644 index 000000000..9638652bd --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactor_default.py @@ -0,0 +1,18 @@ +from twisted.internet import reactor # noqa: F401,TID253 + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess(settings={}) + +d = process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/simple.py b/tests/AsyncCrawlerProcess/simple.py new file mode 100644 index 000000000..d24b4f193 --- /dev/null +++ b/tests/AsyncCrawlerProcess/simple.py @@ -0,0 +1,16 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/sleeping.py b/tests/AsyncCrawlerProcess/sleeping.py new file mode 100644 index 000000000..88caf5032 --- /dev/null +++ b/tests/AsyncCrawlerProcess/sleeping.py @@ -0,0 +1,20 @@ +import asyncio +import sys + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class SleepingSpider(scrapy.Spider): + name = "sleeping" + + start_urls = ["data:,;"] + + async def parse(self, response): + await asyncio.sleep(int(sys.argv[1])) + + +process = AsyncCrawlerProcess(settings={}) + +process.crawl(SleepingSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_asyncio.py b/tests/AsyncCrawlerProcess/twisted_reactor_asyncio.py new file mode 100644 index 000000000..dc820ea3a --- /dev/null +++ b/tests/AsyncCrawlerProcess/twisted_reactor_asyncio.py @@ -0,0 +1,15 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class AsyncioReactorSpider(scrapy.Spider): + name = "asyncio_reactor" + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } +) +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings.py new file mode 100644 index 000000000..5fd48274a --- /dev/null +++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings.py @@ -0,0 +1,14 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class AsyncioReactorSpider(scrapy.Spider): + name = "asyncio_reactor" + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = AsyncCrawlerProcess() +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_same.py new file mode 100644 index 000000000..c205c3cd2 --- /dev/null +++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_same.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class AsyncioReactorSpider1(scrapy.Spider): + name = "asyncio_reactor1" + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +class AsyncioReactorSpider2(scrapy.Spider): + name = "asyncio_reactor2" + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = AsyncCrawlerProcess() +process.crawl(AsyncioReactorSpider1) +process.crawl(AsyncioReactorSpider2) +process.start() diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py new file mode 100644 index 000000000..68239e651 --- /dev/null +++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + +if TYPE_CHECKING: + from asyncio import Task + + +class AsyncioReactorSpider(scrapy.Spider): + name = "asyncio_reactor" + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", + } + + +def log_task_exception(task: Task) -> None: + try: + task.result() + except Exception: + logging.exception("Crawl task failed") + + +process = AsyncCrawlerProcess() +task = process.crawl(AsyncioReactorSpider) +task.add_done_callback(log_task_exception) +process.start() diff --git a/tests/AsyncCrawlerRunner/custom_loop_different.py b/tests/AsyncCrawlerRunner/custom_loop_different.py new file mode 100644 index 000000000..89cf0e536 --- /dev/null +++ b/tests/AsyncCrawlerRunner/custom_loop_different.py @@ -0,0 +1,31 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/AsyncCrawlerRunner/custom_loop_same.py b/tests/AsyncCrawlerRunner/custom_loop_same.py new file mode 100644 index 000000000..43d0dc053 --- /dev/null +++ b/tests/AsyncCrawlerRunner/custom_loop_same.py @@ -0,0 +1,31 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor", "uvloop.Loop") +react(main) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index d8c467f40..7c50277b9 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -4,12 +4,12 @@ import sys from twisted.internet import asyncioreactor from twisted.python import log +import scrapy +from scrapy.crawler import CrawlerProcess + if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -asyncioreactor.install(asyncio.get_event_loop()) - -import scrapy # noqa: E402 -from scrapy.crawler import CrawlerProcess # noqa: E402 +asyncioreactor.install() class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index e7d3ca9cc..578e0029d 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -4,13 +4,13 @@ import sys from twisted.internet import asyncioreactor from uvloop import Loop +import scrapy +from scrapy.crawler import CrawlerProcess + if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncio.set_event_loop(Loop()) -asyncioreactor.install(asyncio.get_event_loop()) - -import scrapy # noqa: E402 -from scrapy.crawler import CrawlerProcess # noqa: E402 +asyncioreactor.install() class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerRunner/custom_loop_different.py b/tests/CrawlerRunner/custom_loop_different.py new file mode 100644 index 000000000..86ba1ed47 --- /dev/null +++ b/tests/CrawlerRunner/custom_loop_different.py @@ -0,0 +1,29 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +def main(reactor): + configure_logging() + runner = CrawlerRunner() + return runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/CrawlerRunner/custom_loop_same.py b/tests/CrawlerRunner/custom_loop_same.py new file mode 100644 index 000000000..98b8dde87 --- /dev/null +++ b/tests/CrawlerRunner/custom_loop_same.py @@ -0,0 +1,29 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +def main(reactor): + configure_logging() + runner = CrawlerRunner() + return runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor", "uvloop.Loop") +react(main) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index a1d3c02fb..ce5963cd7 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,3 +1,4 @@ +import asyncio import logging import platform import re @@ -5,6 +6,7 @@ import signal import subprocess import sys import warnings +from abc import ABC, abstractmethod from pathlib import Path from typing import Any @@ -18,12 +20,18 @@ from zope.interface.exceptions import MultipleInvalid import scrapy from scrapy import Spider -from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerProcess, CrawlerRunner +from scrapy.crawler import ( + AsyncCrawlerProcess, + AsyncCrawlerRunner, + Crawler, + CrawlerProcess, + CrawlerRunner, +) from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader -from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.defer import deferred_f_from_coro_f, deferred_from_coro from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_reactor_settings @@ -88,12 +96,39 @@ class TestCrawler(TestBaseCrawler): Crawler(DefaultSpider()) @inlineCallbacks - def test_crawler_crawl_twice_unsupported(self): + def test_crawler_crawl_twice_seq_unsupported(self): crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) yield crawler.crawl() with pytest.raises(RuntimeError, match="more than once on the same instance"): yield crawler.crawl() + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_crawler_crawl_async_twice_seq_unsupported(self): + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) + await crawler.crawl_async() + with pytest.raises(RuntimeError, match="more than once on the same instance"): + await crawler.crawl_async() + + @inlineCallbacks + def test_crawler_crawl_twice_parallel_unsupported(self): + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) + d1 = crawler.crawl() + d2 = crawler.crawl() + yield d1 + with pytest.raises(RuntimeError, match="Crawling already taking place"): + yield d2 + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_crawler_crawl_async_twice_parallel_unsupported(self): + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) + t1 = asyncio.create_task(crawler.crawl_async()) + t2 = asyncio.create_task(crawler.crawl_async()) + await t1 + with pytest.raises(RuntimeError, match="Crawling already taking place"): + await t2 + def test_get_addon(self): class ParentAddon: pass @@ -590,6 +625,18 @@ class TestCrawlerProcess(TestBaseCrawler): self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") +@pytest.mark.only_asyncio +class TestAsyncCrawlerProcess(TestBaseCrawler): + def test_crawler_process_accepts_dict(self): + runner = AsyncCrawlerProcess({"foo": "bar"}) + assert runner.settings["foo"] == "bar" + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") + + def test_crawler_process_accepts_None(self): + runner = AsyncCrawlerProcess() + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") + + class ExceptionSpider(scrapy.Spider): name = "exception" @@ -692,8 +739,15 @@ class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider): pytest.skip("This test is only for CrawlerRunner") -class ScriptRunnerMixin: - script_dir: Path +class ScriptRunnerMixin(ABC): + @property + @abstractmethod + def script_dir(self) -> Path: + raise NotImplementedError + + @staticmethod + def get_script_dir(name: str) -> Path: + return Path(__file__).parent.resolve() / name def get_script_args(self, script_name: str, *script_args: str) -> list[str]: script_path = self.script_dir / script_name @@ -711,8 +765,10 @@ class ScriptRunnerMixin: return stderr.decode("utf-8") -class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): - script_dir = Path(__file__).parent.resolve() / "CrawlerProcess" +class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin, unittest.TestCase): + """Common tests between CrawlerProcess and AsyncCrawlerProcess, + with the same file names and expectations. + """ def test_simple(self): log = self.run_script("simple.py") @@ -739,48 +795,6 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" ) in log - def test_reactor_default_twisted_reactor_select(self): - log = self.run_script("reactor_default_twisted_reactor_select.py") - if platform.system() in ["Windows", "Darwin"]: - # The goal of this test function is to test that, when a reactor is - # installed (the default one here) and a different reactor is - # configured (select here), an error raises. - # - # In Windows the default reactor is the select reactor, so that - # error does not raise. - # - # If that ever becomes the case on more platforms (i.e. if Linux - # also starts using the select reactor by default in a future - # version of Twisted), then we will need to rethink this test. - assert "Spider closed (finished)" in log - else: - assert "Spider closed (finished)" not in log - assert ( - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ) in log - - def test_reactor_select(self): - log = self.run_script("reactor_select.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the requested one " - "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" - ) in log - - def test_reactor_select_twisted_reactor_select(self): - log = self.run_script("reactor_select_twisted_reactor_select.py") - assert "Spider closed (finished)" in log - assert "ReactorAlreadyInstalledError" not in log - - def test_reactor_select_subclass_twisted_reactor_select(self): - log = self.run_script("reactor_select_subclass_twisted_reactor_select.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ) in log - def test_asyncio_enabled_no_reactor(self): log = self.run_script("asyncio_enabled_no_reactor.py") assert "Spider closed (finished)" in log @@ -829,19 +843,6 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): assert "TimeoutError" not in log assert "twisted.internet.error.DNSLookupError" not in log - def test_twisted_reactor_select(self): - log = self.run_script("twisted_reactor_select.py") - assert "Spider closed (finished)" in log - assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log - - @pytest.mark.skipif( - platform.system() == "Windows", reason="PollReactor is not supported on Windows" - ) - def test_twisted_reactor_poll(self): - log = self.run_script("twisted_reactor_poll.py") - assert "Spider closed (finished)" in log - assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log - def test_twisted_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") assert "Spider closed (finished)" in log @@ -866,14 +867,6 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): in log ) - def test_twisted_reactor_asyncio_custom_settings_conflict(self): - log = self.run_script("twisted_reactor_custom_settings_conflict.py") - assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log - assert ( - "(twisted.internet.selectreactor.SelectReactor) does not match the requested one" - in log - ) - @pytest.mark.requires_uvloop def test_custom_loop_asyncio(self): log = self.run_script("asyncio_custom_loop.py") @@ -960,8 +953,113 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.wait() -class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): - script_dir = Path(__file__).parent.resolve() / "CrawlerRunner" +class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("CrawlerProcess") + + def test_reactor_default_twisted_reactor_select(self): + log = self.run_script("reactor_default_twisted_reactor_select.py") + if platform.system() in ["Windows", "Darwin"]: + # The goal of this test function is to test that, when a reactor is + # installed (the default one here) and a different reactor is + # configured (select here), an error raises. + # + # In Windows the default reactor is the select reactor, so that + # error does not raise. + # + # If that ever becomes the case on more platforms (i.e. if Linux + # also starts using the select reactor by default in a future + # version of Twisted), then we will need to rethink this test. + assert "Spider closed (finished)" in log + else: + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + def test_reactor_select(self): + log = self.run_script("reactor_select.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + ) in log + + def test_reactor_select_twisted_reactor_select(self): + log = self.run_script("reactor_select_twisted_reactor_select.py") + assert "Spider closed (finished)" in log + assert "ReactorAlreadyInstalledError" not in log + + def test_reactor_select_subclass_twisted_reactor_select(self): + log = self.run_script("reactor_select_subclass_twisted_reactor_select.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + def test_twisted_reactor_select(self): + log = self.run_script("twisted_reactor_select.py") + assert "Spider closed (finished)" in log + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + + @pytest.mark.skipif( + platform.system() == "Windows", reason="PollReactor is not supported on Windows" + ) + def test_twisted_reactor_poll(self): + log = self.run_script("twisted_reactor_poll.py") + assert "Spider closed (finished)" in log + assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log + + def test_twisted_reactor_asyncio_custom_settings_conflict(self): + log = self.run_script("twisted_reactor_custom_settings_conflict.py") + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + assert ( + "(twisted.internet.selectreactor.SelectReactor) does not match the requested one" + in log + ) + + +class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("AsyncCrawlerProcess") + + def test_twisted_reactor_custom_settings_select(self): + log = self.run_script("twisted_reactor_custom_settings_select.py") + assert "Spider closed (finished)" not in log + assert ( + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor) " + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_same_loop(self): + log = self.run_script("asyncio_custom_loop_custom_settings_same.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_different_loop(self): + log = self.run_script("asyncio_custom_loop_custom_settings_different.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ) in log + + +class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): + """Common tests between CrawlerRunner and AsyncCrawlerRunner, + with the same file names and expectations. + """ def test_simple(self): log = self.run_script("simple.py") @@ -971,14 +1069,6 @@ class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): in log ) - def test_explicit_default_reactor(self): - log = self.run_script("explicit_default_reactor.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log - ) - def test_multi_parallel(self): log = self.run_script("multi_parallel.py") assert "Spider closed (finished)" in log @@ -1005,6 +1095,39 @@ class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): re.DOTALL, ) + @pytest.mark.requires_uvloop + def test_custom_loop_same(self): + log = self.run_script("custom_loop_same.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_custom_loop_different(self): + log = self.run_script("custom_loop_different.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ) in log + + +class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("CrawlerRunner") + + def test_explicit_default_reactor(self): + log = self.run_script("explicit_default_reactor.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + def test_response_ip_address(self): log = self.run_script("ip_address.py") assert "INFO: Spider closed (finished)" in log @@ -1021,48 +1144,16 @@ class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): assert "DEBUG: Using asyncio event loop" in log -class TestAsyncCrawlerRunnerSubprocess(ScriptRunnerMixin): - script_dir = Path(__file__).parent.resolve() / "AsyncCrawlerRunner" - - def test_simple(self): - log = self.run_script("simple.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) +class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("AsyncCrawlerRunner") def test_simple_default_reactor(self): log = self.run_script("simple_default_reactor.py") assert "Spider closed (finished)" not in log assert "RuntimeError: AsyncCrawlerRunner requires AsyncioSelectorReactor" in log - def test_multi_parallel(self): - log = self.run_script("multi_parallel.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert re.search( - r"Spider opened.+Spider opened.+Closing spider.+Closing spider", - log, - re.DOTALL, - ) - - def test_multi_seq(self): - log = self.run_script("multi_seq.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert re.search( - r"Spider opened.+Closing spider.+Spider opened.+Closing spider", - log, - re.DOTALL, - ) - @pytest.mark.parametrize( ("settings", "items"), diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py index 99f175c60..eb00ab193 100644 --- a/tests/test_utils_reactor.py +++ b/tests/test_utils_reactor.py @@ -6,6 +6,7 @@ from twisted.trial.unittest import TestCase from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.reactor import ( + _asyncio_reactor_path, install_reactor, is_asyncio_reactor_installed, set_asyncio_event_loop, @@ -22,7 +23,7 @@ class TestAsyncio(TestCase): from twisted.internet import reactor as original_reactor with warnings.catch_warnings(record=True) as w: - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + install_reactor(_asyncio_reactor_path) assert len(w) == 0, [str(warning) for warning in w] from twisted.internet import reactor # pylint: disable=reimported @@ -31,5 +32,5 @@ class TestAsyncio(TestCase): @pytest.mark.only_asyncio @deferred_f_from_coro_f async def test_set_asyncio_event_loop(self): - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + install_reactor(_asyncio_reactor_path) assert set_asyncio_event_loop(None) is asyncio.get_running_loop() From e0b9f2d8f6f0feec9626e314166d5ae320d83be1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 28 May 2025 19:57:33 +0500 Subject: [PATCH 137/491] Don't use CrawlerProcess in the commands that don't need it. (#6824) * Don't use CrawlerProcess in the commands that don't need it. * Use a dummy spider loader in runspider. --- docs/topics/api.rst | 2 + docs/topics/settings.rst | 9 -- scrapy/cmdline.py | 3 +- scrapy/commands/__init__.py | 9 +- scrapy/commands/check.py | 1 + scrapy/commands/edit.py | 7 +- scrapy/commands/genspider.py | 14 ++- scrapy/commands/list.py | 7 +- scrapy/commands/runspider.py | 3 +- scrapy/commands/settings.py | 7 +- scrapy/commands/startproject.py | 4 +- scrapy/commands/version.py | 3 +- scrapy/crawler.py | 20 +--- scrapy/spiderloader.py | 31 +++++- tests/test_crawler.py | 5 - tests/test_spiderloader/__init__.py | 145 ++++++++++++++-------------- 16 files changed, 150 insertions(+), 120 deletions(-) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index b11de2914..d90eb0bad 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -213,6 +213,8 @@ SpiderLoader API :param request: queried request :type request: :class:`~scrapy.Request` instance +.. autoclass:: DummySpiderLoader + .. _topics-api-signals: Signals API diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 68c5079cf..65f2e5ebd 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1868,15 +1868,6 @@ it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception But you can choose to silence this exception and turn it into a simple warning by setting ``SPIDER_LOADER_WARN_ONLY = True``. -.. note:: - Some :ref:`scrapy commands ` run with this setting to ``True`` - already (i.e. they will only issue a warning and will not fail) - since they do not actually need to load spider classes to work: - :command:`scrapy runspider `, - :command:`scrapy settings `, - :command:`scrapy startproject `, - :command:`scrapy version `. - .. setting:: SPIDER_MIDDLEWARES SPIDER_MIDDLEWARES diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index b08fd3409..81e507a4e 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -201,7 +201,8 @@ def execute(argv: list[str] | None = None, settings: Settings | None = None) -> opts, args = parser.parse_known_args(args=argv[1:]) _run_print_help(parser, cmd.process_options, args, opts) - cmd.crawler_process = CrawlerProcess(settings) + if cmd.requires_crawler_process: + cmd.crawler_process = CrawlerProcess(settings) _run_print_help(parser, _run_command, cmd, args, opts) sys.exit(cmd.exitcode) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 56199cc01..2818ead77 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -19,11 +19,13 @@ if TYPE_CHECKING: from collections.abc import Iterable from scrapy.crawler import Crawler, CrawlerProcess + from scrapy.settings import Settings class ScrapyCommand: requires_project: bool = False - crawler_process: CrawlerProcess | None = None + requires_crawler_process: bool = True + crawler_process: CrawlerProcess | None = None # set in scrapy.cmdline # default settings to be used for this command instead of global defaults default_settings: dict[str, Any] = {} @@ -31,7 +33,7 @@ class ScrapyCommand: exitcode: int = 0 def __init__(self) -> None: - self.settings: Any = None # set in scrapy.cmdline + self.settings: Settings | None = None # set in scrapy.cmdline def set_crawler(self, crawler: Crawler) -> None: if hasattr(self, "_crawler"): @@ -68,6 +70,7 @@ class ScrapyCommand: """ Populate option parse with options available for this command """ + assert self.settings is not None group = parser.add_argument_group(title="Global Options") group.add_argument( "--logfile", metavar="FILE", help="log file. if omitted stderr will be used" @@ -100,6 +103,7 @@ class ScrapyCommand: group.add_argument("--pdb", action="store_true", help="enable pdb on failure") def process_options(self, args: list[str], opts: argparse.Namespace) -> None: + assert self.settings is not None try: self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline") except ValueError: @@ -170,6 +174,7 @@ class BaseRunSpiderCommand(ScrapyCommand): except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.output or opts.overwrite_output: + assert self.settings is not None feeds = feed_process_params_from_cli( self.settings, opts.output, diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 56dc1ea55..e9ada0fb6 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -69,6 +69,7 @@ class Command(ScrapyCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: # load contracts + assert self.settings is not None contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS")) conman = ContractsManager(load_object(c) for c in contracts) runner = TextTestRunner(verbosity=2 if opts.verbose else 1) diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index d153a5271..f2d52673a 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -4,10 +4,12 @@ import sys from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError +from scrapy.spiderloader import get_spider_loader class Command(ScrapyCommand): requires_project = True + requires_crawler_process = False default_settings = {"LOG_ENABLED": False} def syntax(self) -> str: @@ -30,10 +32,11 @@ class Command(ScrapyCommand): if len(args) != 1: raise UsageError + assert self.settings is not None editor = self.settings["EDITOR"] - assert self.crawler_process + spider_loader = get_spider_loader(self.settings) try: - spidercls = self.crawler_process.spider_loader.load(args[0]) + spidercls = spider_loader.load(args[0]) except KeyError: self._err(f"Spider not found: {args[0]}") return diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 6d4aec3d8..c4abfc4c9 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -11,6 +11,7 @@ from urllib.parse import urlparse import scrapy from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError +from scrapy.spiderloader import get_spider_loader from scrapy.utils.template import render_templatefile, string_camelcase if TYPE_CHECKING: @@ -46,6 +47,7 @@ def verify_url_scheme(url: str) -> str: class Command(ScrapyCommand): requires_project = False + requires_crawler_process = False default_settings = {"LOG_ENABLED": False} def syntax(self) -> str: @@ -92,6 +94,7 @@ class Command(ScrapyCommand): ) def run(self, args: list[str], opts: argparse.Namespace) -> None: + assert self.settings is not None if opts.list: self._list_templates() return @@ -127,6 +130,7 @@ class Command(ScrapyCommand): url: str, template_name: str, ) -> dict[str, Any]: + assert self.settings is not None capitalized_module = "".join(s.capitalize() for s in module.split("_")) return { "project_name": self.settings.get("BOT_NAME"), @@ -147,6 +151,7 @@ class Command(ScrapyCommand): template_file: str | os.PathLike, ) -> None: """Generate the spider module, based on the given template""" + assert self.settings is not None tvars = self._generate_template_variables(module, name, url, template_name) if self.settings.get("NEWSPIDER_MODULE"): spiders_module = import_module(self.settings["NEWSPIDER_MODULE"]) @@ -180,6 +185,7 @@ class Command(ScrapyCommand): print(f" {file.stem}") def _spider_exists(self, name: str) -> bool: + assert self.settings is not None if not self.settings.get("NEWSPIDER_MODULE"): # if run as a standalone command and file with same filename already exists path = Path(name + ".py") @@ -188,12 +194,9 @@ class Command(ScrapyCommand): return True return False - assert self.crawler_process is not None, ( - "crawler_process must be set before calling run" - ) - + spider_loader = get_spider_loader(self.settings) try: - spidercls = self.crawler_process.spider_loader.load(name) + spidercls = spider_loader.load(name) except KeyError: pass else: @@ -215,6 +218,7 @@ class Command(ScrapyCommand): @property def templates_dir(self) -> str: + assert self.settings is not None return str( Path( self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"), diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 3b2f127c2..b4dc97f3d 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -3,6 +3,7 @@ from __future__ import annotations from typing import TYPE_CHECKING from scrapy.commands import ScrapyCommand +from scrapy.spiderloader import get_spider_loader if TYPE_CHECKING: import argparse @@ -10,12 +11,14 @@ if TYPE_CHECKING: class Command(ScrapyCommand): requires_project = True + requires_crawler_process = False default_settings = {"LOG_ENABLED": False} def short_desc(self) -> str: return "List available spiders" def run(self, args: list[str], opts: argparse.Namespace) -> None: - assert self.crawler_process - for s in sorted(self.crawler_process.spider_loader.list()): + assert self.settings is not None + spider_loader = get_spider_loader(self.settings) + for s in sorted(spider_loader.list()): print(s) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 357ca8b37..3e826456e 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError +from scrapy.spiderloader import DummySpiderLoader from scrapy.utils.spider import iter_spider_classes if TYPE_CHECKING: @@ -30,7 +31,7 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType: class Command(BaseRunSpiderCommand): requires_project = False - default_settings = {"SPIDER_LOADER_WARN_ONLY": True} + default_settings = {"SPIDER_LOADER_CLASS": DummySpiderLoader} def syntax(self) -> str: return "[options] " diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 59f86b9a7..e63031f2d 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -7,7 +7,8 @@ from scrapy.settings import BaseSettings class Command(ScrapyCommand): requires_project = False - default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} + requires_crawler_process = False + default_settings = {"LOG_ENABLED": False} def syntax(self) -> str: return "[options]" @@ -46,8 +47,8 @@ class Command(ScrapyCommand): ) def run(self, args: list[str], opts: argparse.Namespace) -> None: - assert self.crawler_process - settings = self.crawler_process.settings + assert self.settings is not None + settings = self.settings if opts.get: s = settings.get(opts.get) if isinstance(s, BaseSettings): diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 1adc1530f..323979193 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -34,7 +34,8 @@ def _make_writable(path: Path) -> None: class Command(ScrapyCommand): requires_project = False - default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} + requires_crawler_process = False + default_settings = {"LOG_ENABLED": False} def syntax(self) -> str: return " [project_dir]" @@ -132,6 +133,7 @@ class Command(ScrapyCommand): @property def templates_dir(self) -> str: + assert self.settings is not None return str( Path( self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"), diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 713a78ad9..30b0e9fd7 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -6,7 +6,8 @@ from scrapy.utils.versions import get_versions class Command(ScrapyCommand): - default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} + requires_crawler_process = False + default_settings = {"LOG_ENABLED": False} def syntax(self) -> str: return "[-v]" diff --git a/scrapy/crawler.py b/scrapy/crawler.py index c22b8603b..8e3223a5c 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -5,22 +5,21 @@ import contextlib import logging import pprint import signal -from typing import TYPE_CHECKING, Any, TypeVar, cast +from typing import TYPE_CHECKING, Any, TypeVar from twisted.internet.defer import ( Deferred, DeferredList, inlineCallbacks, ) -from zope.interface.verify import verifyClass from scrapy import Spider, signals from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine from scrapy.extension import ExtensionManager -from scrapy.interfaces import ISpiderLoader -from scrapy.settings import BaseSettings, Settings, overridden_settings +from scrapy.settings import Settings, overridden_settings from scrapy.signalmanager import SignalManager +from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader from scrapy.utils.asyncio import is_asyncio_available from scrapy.utils.defer import deferred_from_coro, deferred_to_future from scrapy.utils.log import ( @@ -46,7 +45,6 @@ if TYPE_CHECKING: from collections.abc import Generator, Iterable from scrapy.logformatter import LogFormatter - from scrapy.spiderloader import SpiderLoaderProtocol from scrapy.statscollectors import StatsCollector from scrapy.utils.request import RequestFingerprinterProtocol @@ -324,22 +322,12 @@ class Crawler: class CrawlerRunnerBase: - @staticmethod - def _get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol: - """Get SpiderLoader instance from settings""" - cls_path = settings.get("SPIDER_LOADER_CLASS") - loader_cls = load_object(cls_path) - verifyClass(ISpiderLoader, loader_cls) - return cast( - "SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy()) - ) - def __init__(self, settings: dict[str, Any] | Settings | None = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) AddonManager.load_pre_crawler_settings(settings) self.settings: Settings = settings - self.spider_loader: SpiderLoaderProtocol = self._get_spider_loader(settings) + self.spider_loader: SpiderLoaderProtocol = get_spider_loader(settings) self._crawlers: set[Crawler] = set() self.bootstrap_failed = False diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index f537e0593..8eac188c8 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -3,12 +3,13 @@ from __future__ import annotations import traceback import warnings from collections import defaultdict -from typing import TYPE_CHECKING, Protocol +from typing import TYPE_CHECKING, Protocol, cast from zope.interface import implementer +from zope.interface.verify import verifyClass from scrapy.interfaces import ISpiderLoader -from scrapy.utils.misc import walk_modules +from scrapy.utils.misc import load_object, walk_modules from scrapy.utils.spider import iter_spider_classes if TYPE_CHECKING: @@ -21,6 +22,14 @@ if TYPE_CHECKING: from scrapy.settings import BaseSettings +def get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol: + """Get SpiderLoader instance from settings""" + cls_path = settings.get("SPIDER_LOADER_CLASS") + loader_cls = load_object(cls_path) + verifyClass(ISpiderLoader, loader_cls) + return cast("SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy())) + + class SpiderLoaderProtocol(Protocol): @classmethod def from_settings(cls, settings: BaseSettings) -> Self: @@ -120,3 +129,21 @@ class SpiderLoader: Return a list with the names of all spiders available in the project. """ return list(self._spiders.keys()) + + +@implementer(ISpiderLoader) +class DummySpiderLoader: + """A dummy spider loader that does not load any spiders.""" + + @classmethod + def from_settings(cls, settings: BaseSettings) -> Self: + return cls() + + def load(self, spider_name: str) -> type[Spider]: + raise KeyError("DummySpiderLoader doesn't load any spiders") + + def list(self) -> list[str]: + return [] + + def find_by_request(self, request: Request) -> __builtins__.list[str]: + return [] diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ce5963cd7..56cb21650 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -30,7 +30,6 @@ from scrapy.crawler import ( from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings -from scrapy.spiderloader import SpiderLoader from scrapy.utils.defer import deferred_f_from_coro_f, deferred_from_coro from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider @@ -570,10 +569,6 @@ class SpiderLoaderWithWrongInterface: pass -class CustomSpiderLoader(SpiderLoader): - pass - - class TestCrawlerRunner(TestBaseCrawler): def test_spider_manager_verify_interface(self): settings = Settings( diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 476487a04..245507c0b 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,10 +1,8 @@ import contextlib import shutil import sys -import tempfile import warnings from pathlib import Path -from tempfile import mkdtemp from unittest import mock import pytest @@ -17,7 +15,7 @@ from scrapy.crawler import CrawlerRunner from scrapy.http import Request from scrapy.interfaces import ISpiderLoader from scrapy.settings import Settings -from scrapy.spiderloader import SpiderLoader +from scrapy.spiderloader import DummySpiderLoader, SpiderLoader, get_spider_loader module_dir = Path(__file__).resolve().parent @@ -27,73 +25,76 @@ def _copytree(source: Path, target: Path): shutil.copytree(source, target) +@pytest.fixture +def spider_loader_env(tmp_path): + orig_spiders_dir = module_dir / "test_spiders" + spiders_dir = tmp_path / "test_spiders_xxx" + _copytree(orig_spiders_dir, spiders_dir) + sys.path.append(str(tmp_path)) + settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) + + yield settings, spiders_dir + + sys.modules.pop("test_spiders_xxx", None) + sys.path.remove(str(tmp_path)) + + +@pytest.fixture +def spider_loader(spider_loader_env): + settings, _ = spider_loader_env + return SpiderLoader.from_settings(settings) + + class TestSpiderLoader: - def setup_method(self): - orig_spiders_dir = module_dir / "test_spiders" - self.tmpdir = Path(tempfile.mkdtemp()) - self.spiders_dir = self.tmpdir / "test_spiders_xxx" - _copytree(orig_spiders_dir, self.spiders_dir) - sys.path.append(str(self.tmpdir)) - settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) - self.spider_loader = SpiderLoader.from_settings(settings) + def test_interface(self, spider_loader): + verifyObject(ISpiderLoader, spider_loader) - def teardown_method(self): - del self.spider_loader - del sys.modules["test_spiders_xxx"] - sys.path.remove(str(self.tmpdir)) - - def test_interface(self): - verifyObject(ISpiderLoader, self.spider_loader) - - def test_list(self): - assert set(self.spider_loader.list()) == { + def test_list(self, spider_loader): + assert set(spider_loader.list()) == { "spider1", "spider2", "spider3", "spider4", } - def test_load(self): - spider1 = self.spider_loader.load("spider1") + def test_load(self, spider_loader): + spider1 = spider_loader.load("spider1") assert spider1.__name__ == "Spider1" - def test_find_by_request(self): - assert self.spider_loader.find_by_request( - Request("http://scrapy1.org/test") - ) == ["spider1"] - assert self.spider_loader.find_by_request( - Request("http://scrapy2.org/test") - ) == ["spider2"] + def test_find_by_request(self, spider_loader): + assert spider_loader.find_by_request(Request("http://scrapy1.org/test")) == [ + "spider1" + ] + assert spider_loader.find_by_request(Request("http://scrapy2.org/test")) == [ + "spider2" + ] assert set( - self.spider_loader.find_by_request(Request("http://scrapy3.org/test")) + spider_loader.find_by_request(Request("http://scrapy3.org/test")) ) == {"spider1", "spider2"} - assert ( - self.spider_loader.find_by_request(Request("http://scrapy999.org/test")) - == [] - ) - assert self.spider_loader.find_by_request(Request("http://spider3.com")) == [] - assert self.spider_loader.find_by_request( + assert spider_loader.find_by_request(Request("http://scrapy999.org/test")) == [] + assert spider_loader.find_by_request(Request("http://spider3.com")) == [] + assert spider_loader.find_by_request( Request("http://spider3.com/onlythis") ) == ["spider3"] def test_load_spider_module(self): module = "tests.test_spiderloader.test_spiders.spider1" settings = Settings({"SPIDER_MODULES": [module]}) - self.spider_loader = SpiderLoader.from_settings(settings) - assert len(self.spider_loader._spiders) == 1 + spider_loader = SpiderLoader.from_settings(settings) + assert len(spider_loader._spiders) == 1 def test_load_spider_module_multiple(self): prefix = "tests.test_spiderloader.test_spiders." module = ",".join(prefix + s for s in ("spider1", "spider2")) settings = Settings({"SPIDER_MODULES": module}) - self.spider_loader = SpiderLoader.from_settings(settings) - assert len(self.spider_loader._spiders) == 2 + spider_loader = SpiderLoader.from_settings(settings) + assert len(spider_loader._spiders) == 2 def test_load_base_spider(self): module = "tests.test_spiderloader.test_spiders.spider0" settings = Settings({"SPIDER_MODULES": [module]}) - self.spider_loader = SpiderLoader.from_settings(settings) - assert len(self.spider_loader._spiders) == 0 + spider_loader = SpiderLoader.from_settings(settings) + assert len(spider_loader._spiders) == 0 def test_load_spider_module_from_addons(self): module = "tests.test_spiderloader.spiders_from_addons.spider0" @@ -183,27 +184,14 @@ class TestSpiderLoader: class TestDuplicateSpiderNameLoader: - def setup_method(self): - orig_spiders_dir = module_dir / "test_spiders" - self.tmpdir = Path(mkdtemp()) - self.spiders_dir = self.tmpdir / "test_spiders_xxx" - _copytree(orig_spiders_dir, self.spiders_dir) - sys.path.append(str(self.tmpdir)) - self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) + def test_dupename_warning(self, spider_loader_env): + settings, spiders_dir = spider_loader_env - def teardown_method(self): - del sys.modules["test_spiders_xxx"] - sys.path.remove(str(self.tmpdir)) - - def test_dupename_warning(self): # copy 1 spider module so as to have duplicate spider name - shutil.copyfile( - self.tmpdir / "test_spiders_xxx" / "spider3.py", - self.tmpdir / "test_spiders_xxx" / "spider3dupe.py", - ) + shutil.copyfile(spiders_dir / "spider3.py", spiders_dir / "spider3dupe.py") with warnings.catch_warnings(record=True) as w: - spider_loader = SpiderLoader.from_settings(self.settings) + spider_loader = SpiderLoader.from_settings(settings) assert len(w) == 1 msg = str(w[0].message) @@ -218,20 +206,15 @@ class TestDuplicateSpiderNameLoader: spiders = set(spider_loader.list()) assert spiders == {"spider1", "spider2", "spider3", "spider4"} - def test_multiple_dupename_warning(self): + def test_multiple_dupename_warning(self, spider_loader_env): + settings, spiders_dir = spider_loader_env # copy 2 spider modules so as to have duplicate spider name # This should issue 2 warning, 1 for each duplicate spider name - shutil.copyfile( - self.tmpdir / "test_spiders_xxx" / "spider1.py", - self.tmpdir / "test_spiders_xxx" / "spider1dupe.py", - ) - shutil.copyfile( - self.tmpdir / "test_spiders_xxx" / "spider2.py", - self.tmpdir / "test_spiders_xxx" / "spider2dupe.py", - ) + shutil.copyfile(spiders_dir / "spider1.py", spiders_dir / "spider1dupe.py") + shutil.copyfile(spiders_dir / "spider2.py", spiders_dir / "spider2dupe.py") with warnings.catch_warnings(record=True) as w: - spider_loader = SpiderLoader.from_settings(self.settings) + spider_loader = SpiderLoader.from_settings(settings) assert len(w) == 1 msg = str(w[0].message) @@ -247,3 +230,25 @@ class TestDuplicateSpiderNameLoader: spiders = set(spider_loader.list()) assert spiders == {"spider1", "spider2", "spider3", "spider4"} + + +class CustomSpiderLoader(SpiderLoader): + pass + + +def test_custom_spider_loader(): + settings = Settings( + { + "SPIDER_LOADER_CLASS": CustomSpiderLoader, + } + ) + spider_loader = get_spider_loader(settings) + assert isinstance(spider_loader, CustomSpiderLoader) + + +def test_dummy_spider_loader(spider_loader_env): + settings, _ = spider_loader_env + spider_loader = DummySpiderLoader.from_settings(settings) + assert not spider_loader.list() + with pytest.raises(KeyError): + spider_loader.load("spider1") From a724541a715bb9fc5428ba630f24e1036ca0a896 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 29 May 2025 00:46:04 +0500 Subject: [PATCH 138/491] Split tests/test_commands.py. (#6836) --- tests/test_command_crawl.py | 93 +++ tests/test_command_genspider.py | 208 +++++++ tests/test_command_runspider.py | 375 +++++++++++ tests/test_command_startproject.py | 318 ++++++++++ tests/test_commands.py | 966 +---------------------------- 5 files changed, 998 insertions(+), 962 deletions(-) create mode 100644 tests/test_command_crawl.py create mode 100644 tests/test_command_genspider.py create mode 100644 tests/test_command_runspider.py create mode 100644 tests/test_command_startproject.py diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py new file mode 100644 index 000000000..3d5e17977 --- /dev/null +++ b/tests/test_command_crawl.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from pathlib import Path + +from tests.test_commands import TestCommandBase + + +class TestCrawlCommand(TestCommandBase): + def crawl(self, code, args=()): + Path(self.proj_mod_path, "spiders", "myspider.py").write_text( + code, encoding="utf-8" + ) + return self.proc("crawl", "myspider", *args) + + def get_log(self, code, args=()): + _, _, stderr = self.crawl(code, args=args) + return stderr + + def test_no_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('It works!') + return + yield +""" + log = self.get_log(spider_code) + assert "[myspider] DEBUG: It works!" in log + + def test_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return + yield +""" + args = ["-o", "example.json"] + log = self.get_log(spider_code, args=args) + assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log + + def test_overwrite_output(self): + spider_code = """ +import json +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug( + 'FEEDS: {}'.format( + json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) + ) + ) + return + yield +""" + Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") + args = ["-O", "example.json"] + log = self.get_log(spider_code, args=args) + assert ( + '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' + in log + ) + with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: + first_line = f2.readline() + assert first_line != "not empty" + + def test_output_and_overwrite_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + return + yield +""" + args = ["-o", "example1.json", "-O", "example2.json"] + log = self.get_log(spider_code, args=args) + assert ( + "error: Please use only one of -o/--output and -O/--overwrite-output" in log + ) diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py new file mode 100644 index 000000000..18ec81fed --- /dev/null +++ b/tests/test_command_genspider.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import os +from pathlib import Path + +from tests.test_commands import TestCommandBase, TestProjectBase + + +class TestGenspiderCommand(TestCommandBase): + def test_arguments(self): + # only pass one argument. spider script shouldn't be created + assert self.call("genspider", "test_name") == 2 + assert not Path(self.proj_mod_path, "spiders", "test_name.py").exists() + # pass two arguments . spider script should be created + assert self.call("genspider", "test_name", "test.com") == 0 + assert Path(self.proj_mod_path, "spiders", "test_name.py").exists() + + def test_template(self, tplname="crawl"): + args = [f"--template={tplname}"] if tplname else [] + spname = "test_spider" + spmodule = f"{self.project_name}.spiders.{spname}" + p, out, err = self.proc("genspider", spname, "test.com", *args) + assert ( + f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}" + in out + ) + assert Path(self.proj_mod_path, "spiders", "test_spider.py").exists() + modify_time_before = ( + Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime + ) + p, out, err = self.proc("genspider", spname, "test.com", *args) + assert f"Spider {spname!r} already exists in module" in out + modify_time_after = ( + Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime + ) + assert modify_time_after == modify_time_before + + def test_template_basic(self): + self.test_template("basic") + + def test_template_csvfeed(self): + self.test_template("csvfeed") + + def test_template_xmlfeed(self): + self.test_template("xmlfeed") + + def test_list(self): + assert self.call("genspider", "--list") == 0 + + def test_dump(self): + assert self.call("genspider", "--dump=basic") == 0 + assert self.call("genspider", "-d", "basic") == 0 + + def test_same_name_as_project(self): + assert self.call("genspider", self.project_name) == 2 + assert not Path( + self.proj_mod_path, "spiders", f"{self.project_name}.py" + ).exists() + + def test_same_filename_as_existing_spider(self, force=False): + file_name = "example" + file_path = Path(self.proj_mod_path, "spiders", f"{file_name}.py") + assert self.call("genspider", file_name, "example.com") == 0 + assert file_path.exists() + + # change name of spider but not its file name + with file_path.open("r+", encoding="utf-8") as spider_file: + file_data = spider_file.read() + file_data = file_data.replace('name = "example"', 'name = "renamed"') + spider_file.seek(0) + spider_file.write(file_data) + spider_file.truncate() + modify_time_before = file_path.stat().st_mtime + file_contents_before = file_data + + if force: + p, out, err = self.proc("genspider", "--force", file_name, "example.com") + assert ( + f"Created spider {file_name!r} using template 'basic' in module" in out + ) + modify_time_after = file_path.stat().st_mtime + assert modify_time_after != modify_time_before + file_contents_after = file_path.read_text(encoding="utf-8") + assert file_contents_after != file_contents_before + else: + p, out, err = self.proc("genspider", file_name, "example.com") + assert f"{file_path.resolve()} already exists" in out + modify_time_after = file_path.stat().st_mtime + assert modify_time_after == modify_time_before + file_contents_after = file_path.read_text(encoding="utf-8") + assert file_contents_after == file_contents_before + + def test_same_filename_as_existing_spider_force(self): + self.test_same_filename_as_existing_spider(force=True) + + def test_url(self, url="test.com", domain="test.com"): + assert self.call("genspider", "--force", "test_name", url) == 0 + assert ( + self.find_in_file( + Path(self.proj_mod_path, "spiders", "test_name.py"), + r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]", + ).group(1) + == domain + ) + assert ( + self.find_in_file( + Path(self.proj_mod_path, "spiders", "test_name.py"), + r"start_urls\s*=\s*\[['\"](.+)['\"]\]", + ).group(1) + == f"https://{domain}" + ) + + def test_url_schema(self): + self.test_url("https://test.com", "test.com") + + def test_template_start_urls( + self, url="test.com", expected="https://test.com", template="basic" + ): + assert self.call("genspider", "-t", template, "--force", "test_name", url) == 0 + assert ( + self.find_in_file( + Path(self.proj_mod_path, "spiders", "test_name.py"), + r"start_urls\s*=\s*\[['\"](.+)['\"]\]", + ).group(1) + == expected + ) + + def test_genspider_basic_start_urls(self): + self.test_template_start_urls("https://test.com", "https://test.com", "basic") + self.test_template_start_urls("http://test.com", "http://test.com", "basic") + self.test_template_start_urls( + "http://test.com/other/path", "http://test.com/other/path", "basic" + ) + self.test_template_start_urls( + "test.com/other/path", "https://test.com/other/path", "basic" + ) + + def test_genspider_crawl_start_urls(self): + self.test_template_start_urls("https://test.com", "https://test.com", "crawl") + self.test_template_start_urls("http://test.com", "http://test.com", "crawl") + self.test_template_start_urls( + "http://test.com/other/path", "http://test.com/other/path", "crawl" + ) + self.test_template_start_urls( + "test.com/other/path", "https://test.com/other/path", "crawl" + ) + self.test_template_start_urls("test.com", "https://test.com", "crawl") + + def test_genspider_xmlfeed_start_urls(self): + self.test_template_start_urls( + "https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" + ) + self.test_template_start_urls( + "http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed" + ) + self.test_template_start_urls( + "test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" + ) + + def test_genspider_csvfeed_start_urls(self): + self.test_template_start_urls( + "https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" + ) + self.test_template_start_urls( + "http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed" + ) + self.test_template_start_urls( + "test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" + ) + + +class TestGenspiderStandaloneCommand(TestProjectBase): + def test_generate_standalone_spider(self): + self.call("genspider", "example", "example.com") + assert Path(self.temp_path, "example.py").exists() + + def test_same_name_as_existing_file(self, force=False): + file_name = "example" + file_path = Path(self.temp_path, file_name + ".py") + p, out, err = self.proc("genspider", file_name, "example.com") + assert f"Created spider {file_name!r} using template 'basic' " in out + assert file_path.exists() + modify_time_before = file_path.stat().st_mtime + file_contents_before = file_path.read_text(encoding="utf-8") + + if force: + # use different template to ensure contents were changed + p, out, err = self.proc( + "genspider", "--force", "-t", "crawl", file_name, "example.com" + ) + assert f"Created spider {file_name!r} using template 'crawl' " in out + modify_time_after = file_path.stat().st_mtime + assert modify_time_after != modify_time_before + file_contents_after = file_path.read_text(encoding="utf-8") + assert file_contents_after != file_contents_before + else: + p, out, err = self.proc("genspider", file_name, "example.com") + assert ( + f"{Path(self.temp_path, file_name + '.py').resolve()} already exists" + in out + ) + modify_time_after = file_path.stat().st_mtime + assert modify_time_after == modify_time_before + file_contents_after = file_path.read_text(encoding="utf-8") + assert file_contents_after == file_contents_before + + def test_same_name_as_existing_file_force(self): + self.test_same_name_as_existing_file(force=True) diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py new file mode 100644 index 000000000..664de16f8 --- /dev/null +++ b/tests/test_command_runspider.py @@ -0,0 +1,375 @@ +from __future__ import annotations + +import inspect +import platform +import sys +from contextlib import contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory, mkdtemp +from typing import TYPE_CHECKING +from unittest import skipIf + +import pytest +from twisted.trial import unittest + +from tests.test_commands import TestCommandBase +from tests.test_crawler import ExceptionSpider, NoRequestsSpider + +if TYPE_CHECKING: + from collections.abc import Iterator + + +class TestRunSpiderCommand(TestCommandBase): + spider_filename = "myspider.py" + + debug_log_spider = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug("It Works!") + return + yield +""" + + badspider = """ +import scrapy + +class BadSpider(scrapy.Spider): + name = "bad" + async def start(self): + raise Exception("oops!") + yield + """ + + @contextmanager + def _create_file(self, content: str, name: str | None = None) -> Iterator[str]: + with TemporaryDirectory() as tmpdir: + if name: + fname = Path(tmpdir, name).resolve() + else: + fname = Path(tmpdir, self.spider_filename).resolve() + fname.write_text(content, encoding="utf-8") + yield str(fname) + + def runspider(self, code, name=None, args=()): + with self._create_file(code, name) as fname: + return self.proc("runspider", fname, *args) + + def get_log(self, code, name=None, args=()): + p, stdout, stderr = self.runspider(code, name, args=args) + return stderr + + def test_runspider(self): + log = self.get_log(self.debug_log_spider) + assert "DEBUG: It Works!" in log + assert "INFO: Spider opened" in log + assert "INFO: Closing spider (finished)" in log + assert "INFO: Spider closed (finished)" in log + + def test_run_fail_spider(self): + proc, _, _ = self.runspider( + "import scrapy\n" + inspect.getsource(ExceptionSpider) + ) + ret = proc.returncode + assert ret != 0 + + def test_run_good_spider(self): + proc, _, _ = self.runspider( + "import scrapy\n" + inspect.getsource(NoRequestsSpider) + ) + ret = proc.returncode + assert ret == 0 + + def test_runspider_log_level(self): + log = self.get_log(self.debug_log_spider, args=("-s", "LOG_LEVEL=INFO")) + assert "DEBUG: It Works!" not in log + assert "INFO: Spider opened" in log + + def test_runspider_dnscache_disabled(self): + # see https://github.com/scrapy/scrapy/issues/2811 + # The spider below should not be able to connect to localhost:12345, + # which is intended, + # but this should not be because of DNS lookup error + # assumption: localhost will resolve in all cases (true?) + dnscache_spider = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + start_urls = ['http://localhost:12345'] + + def parse(self, response): + return {'test': 'value'} +""" + log = self.get_log(dnscache_spider, args=("-s", "DNSCACHE_ENABLED=False")) + assert "DNSLookupError" not in log + assert "INFO: Spider opened" in log + + def test_runspider_log_short_names(self): + log1 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=1")) + assert "[myspider] DEBUG: It Works!" in log1 + assert "[scrapy]" in log1 + assert "[scrapy.core.engine]" not in log1 + + log2 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=0")) + assert "[myspider] DEBUG: It Works!" in log2 + assert "[scrapy]" not in log2 + assert "[scrapy.core.engine]" in log2 + + def test_runspider_no_spider_found(self): + log = self.get_log("from scrapy.spiders import Spider\n") + assert "No spider found in file" in log + + def test_runspider_file_not_found(self): + _, _, log = self.proc("runspider", "some_non_existent_file") + assert "File not found: some_non_existent_file" in log + + def test_runspider_unable_to_load(self): + log = self.get_log("", name="myspider.txt") + assert "Unable to load" in log + + def test_start_errors(self): + log = self.get_log(self.badspider, name="badspider.py") + assert "start" in log + assert "badspider.py" in log, log + + def test_asyncio_enabled_true(self): + log = self.get_log( + self.debug_log_spider, + args=[ + "-s", + "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", + ], + ) + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_asyncio_enabled_default(self): + log = self.get_log(self.debug_log_spider, args=[]) + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_asyncio_enabled_false(self): + log = self.get_log( + self.debug_log_spider, + args=["-s", "TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor"], + ) + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + + @pytest.mark.requires_uvloop + def test_custom_asyncio_loop_enabled_true(self): + log = self.get_log( + self.debug_log_spider, + args=[ + "-s", + "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "-s", + "ASYNCIO_EVENT_LOOP=uvloop.Loop", + ], + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + def test_custom_asyncio_loop_enabled_false(self): + log = self.get_log( + self.debug_log_spider, + args=[ + "-s", + "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", + ], + ) + import asyncio + + if sys.platform != "win32": + loop = asyncio.new_event_loop() + else: + loop = asyncio.SelectorEventLoop() + assert ( + f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}" + in log + ) + + def test_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return + yield +""" + args = ["-o", "example.json"] + log = self.get_log(spider_code, args=args) + assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log + + def test_overwrite_output(self): + spider_code = """ +import json +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug( + 'FEEDS: {}'.format( + json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) + ) + ) + return + yield +""" + Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") + args = ["-O", "example.json"] + log = self.get_log(spider_code, args=args) + assert ( + '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' + in log + ) + with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: + first_line = f2.readline() + assert first_line != "not empty" + + def test_output_and_overwrite_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + return + yield +""" + args = ["-o", "example1.json", "-O", "example2.json"] + log = self.get_log(spider_code, args=args) + assert ( + "error: Please use only one of -o/--output and -O/--overwrite-output" in log + ) + + def test_output_stdout(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return + yield +""" + args = ["-o", "-:json"] + log = self.get_log(spider_code, args=args) + assert "[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}" in log + + @skipIf(platform.system() == "Windows", reason="Linux only") + def test_absolute_path_linux(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} + """ + temp_dir = mkdtemp() + + args = ["-o", f"{temp_dir}/output1.json:json"] + log = self.get_log(spider_code, args=args) + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json" + in log + ) + + args = ["-o", f"{temp_dir}/output2.json"] + log = self.get_log(spider_code, args=args) + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json" + in log + ) + + @skipIf(platform.system() != "Windows", reason="Windows only") + def test_absolute_path_windows(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} + """ + temp_dir = mkdtemp() + + args = ["-o", f"{temp_dir}\\output1.json:json"] + log = self.get_log(spider_code, args=args) + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json" + in log + ) + + args = ["-o", f"{temp_dir}\\output2.json"] + log = self.get_log(spider_code, args=args) + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json" + in log + ) + + def test_args_change_settings(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("FOO", kwargs.get("foo")) + return spider + + async def start(self): + self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") + return + yield +""" + args = ["-a", "foo=42"] + log = self.get_log(spider_code, args=args) + assert "Spider closed (finished)" in log + assert "The value of FOO is 42" in log + + +class TestWindowsRunSpiderCommand(TestRunSpiderCommand): + spider_filename = "myspider.pyw" + + def setUp(self): + if platform.system() != "Windows": + raise unittest.SkipTest("Windows required for .pyw files") + return super().setUp() + + def test_start_errors(self): + log = self.get_log(self.badspider, name="badspider.pyw") + assert "start" in log + assert "badspider.pyw" in log + + def test_runspider_unable_to_load(self): + raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py new file mode 100644 index 000000000..08bf9b0fd --- /dev/null +++ b/tests/test_command_startproject.py @@ -0,0 +1,318 @@ +from __future__ import annotations + +import os +import subprocess +import sys +from contextlib import contextmanager +from itertools import chain +from pathlib import Path +from shutil import copytree +from stat import S_IWRITE as ANYONE_WRITE_PERMISSION +from tempfile import mkdtemp + +import scrapy +from scrapy.commands.startproject import IGNORE +from tests.test_commands import TestProjectBase + + +class TestStartprojectCommand(TestProjectBase): + def test_startproject(self): + p, out, err = self.proc("startproject", self.project_name) + print(out) + print(err, file=sys.stderr) + assert p.returncode == 0 + + assert Path(self.proj_path, "scrapy.cfg").exists() + assert Path(self.proj_path, "testproject").exists() + assert Path(self.proj_mod_path, "__init__.py").exists() + assert Path(self.proj_mod_path, "items.py").exists() + assert Path(self.proj_mod_path, "pipelines.py").exists() + assert Path(self.proj_mod_path, "settings.py").exists() + assert Path(self.proj_mod_path, "spiders", "__init__.py").exists() + + assert self.call("startproject", self.project_name) == 1 + assert self.call("startproject", "wrong---project---name") == 1 + assert self.call("startproject", "sys") == 1 + + def test_startproject_with_project_dir(self): + project_dir = mkdtemp() + assert self.call("startproject", self.project_name, project_dir) == 0 + + assert Path(project_dir, "scrapy.cfg").exists() + assert Path(project_dir, "testproject").exists() + assert Path(project_dir, self.project_name, "__init__.py").exists() + assert Path(project_dir, self.project_name, "items.py").exists() + assert Path(project_dir, self.project_name, "pipelines.py").exists() + assert Path(project_dir, self.project_name, "settings.py").exists() + assert Path(project_dir, self.project_name, "spiders", "__init__.py").exists() + + assert self.call("startproject", self.project_name, project_dir + "2") == 0 + + assert self.call("startproject", self.project_name, project_dir) == 1 + assert self.call("startproject", self.project_name + "2", project_dir) == 1 + assert self.call("startproject", "wrong---project---name") == 1 + assert self.call("startproject", "sys") == 1 + assert self.call("startproject") == 2 + assert ( + self.call("startproject", self.project_name, project_dir, "another_params") + == 2 + ) + + def test_existing_project_dir(self): + project_dir = mkdtemp() + project_name = self.project_name + "_existing" + project_path = Path(project_dir, project_name) + project_path.mkdir() + + p, out, err = self.proc("startproject", project_name, cwd=project_dir) + print(out) + print(err, file=sys.stderr) + assert p.returncode == 0 + + assert Path(project_path, "scrapy.cfg").exists() + assert Path(project_path, project_name).exists() + assert Path(project_path, project_name, "__init__.py").exists() + assert Path(project_path, project_name, "items.py").exists() + assert Path(project_path, project_name, "pipelines.py").exists() + assert Path(project_path, project_name, "settings.py").exists() + assert Path(project_path, project_name, "spiders", "__init__.py").exists() + + +def get_permissions_dict( + path: str | os.PathLike, renamings=None, ignore=None +) -> dict[str, str]: + def get_permissions(path: Path) -> str: + return oct(path.stat().st_mode) + + path_obj = Path(path) + + renamings = renamings or () + permissions_dict = { + ".": get_permissions(path_obj), + } + for root, dirs, files in os.walk(path_obj): + nodes = list(chain(dirs, files)) + if ignore: + ignored_names = ignore(root, nodes) + nodes = [node for node in nodes if node not in ignored_names] + for node in nodes: + absolute_path = Path(root, node) + relative_path = str(absolute_path.relative_to(path)) + for search_string, replacement in renamings: + relative_path = relative_path.replace(search_string, replacement) + permissions = get_permissions(absolute_path) + permissions_dict[relative_path] = permissions + return permissions_dict + + +class TestStartprojectTemplates(TestProjectBase): + maxDiff = None + + def setUp(self): + super().setUp() + self.tmpl = str(Path(self.temp_path, "templates")) + self.tmpl_proj = str(Path(self.tmpl, "project")) + + def test_startproject_template_override(self): + copytree(Path(scrapy.__path__[0], "templates"), self.tmpl) + Path(self.tmpl_proj, "root_template").write_bytes(b"") + assert Path(self.tmpl_proj, "root_template").exists() + + args = ["--set", f"TEMPLATES_DIR={self.tmpl}"] + p, out, err = self.proc("startproject", self.project_name, *args) + assert ( + f"New Scrapy project '{self.project_name}', using template directory" in out + ) + assert self.tmpl_proj in out + assert Path(self.proj_path, "root_template").exists() + + def test_startproject_permissions_from_writable(self): + """Check that generated files have the right permissions when the + template folder has the same permissions as in the project, i.e. + everything is writable.""" + scrapy_path = scrapy.__path__[0] + project_template = Path(scrapy_path, "templates", "project") + project_name = "startproject1" + renamings = ( + ("module", project_name), + (".tmpl", ""), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + destination = mkdtemp() + process = subprocess.Popen( + ( + sys.executable, + "-m", + "scrapy.cmdline", + "startproject", + project_name, + ), + cwd=destination, + env=self.env, + ) + process.wait() + + project_dir = Path(destination, project_name) + actual_permissions = get_permissions_dict(project_dir) + + assert actual_permissions == expected_permissions + + def test_startproject_permissions_from_read_only(self): + """Check that generated files have the right permissions when the + template folder has been made read-only, which is something that some + systems do. + + See https://github.com/scrapy/scrapy/pull/4604 + """ + scrapy_path = scrapy.__path__[0] + templates_dir = Path(scrapy_path, "templates") + project_template = Path(templates_dir, "project") + project_name = "startproject2" + renamings = ( + ("module", project_name), + (".tmpl", ""), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + def _make_read_only(path: Path): + current_permissions = path.stat().st_mode + path.chmod(current_permissions & ~ANYONE_WRITE_PERMISSION) + + read_only_templates_dir = str(Path(mkdtemp()) / "templates") + copytree(templates_dir, read_only_templates_dir) + + for root, dirs, files in os.walk(read_only_templates_dir): + for node in chain(dirs, files): + _make_read_only(Path(root, node)) + + destination = mkdtemp() + process = subprocess.Popen( + ( + sys.executable, + "-m", + "scrapy.cmdline", + "startproject", + project_name, + "--set", + f"TEMPLATES_DIR={read_only_templates_dir}", + ), + cwd=destination, + env=self.env, + ) + process.wait() + + project_dir = Path(destination, project_name) + actual_permissions = get_permissions_dict(project_dir) + + assert actual_permissions == expected_permissions + + def test_startproject_permissions_unchanged_in_destination(self): + """Check that preexisting folders and files in the destination folder + do not see their permissions modified.""" + scrapy_path = scrapy.__path__[0] + project_template = Path(scrapy_path, "templates", "project") + project_name = "startproject3" + renamings = ( + ("module", project_name), + (".tmpl", ""), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + destination = mkdtemp() + project_dir = Path(destination, project_name) + + existing_nodes = { + oct(permissions)[2:] + extension: permissions + for extension in ("", ".d") + for permissions in ( + 0o444, + 0o555, + 0o644, + 0o666, + 0o755, + 0o777, + ) + } + project_dir.mkdir() + for node, permissions in existing_nodes.items(): + path = project_dir / node + if node.endswith(".d"): + path.mkdir(mode=permissions) + else: + path.touch(mode=permissions) + expected_permissions[node] = oct(path.stat().st_mode) + + process = subprocess.Popen( + ( + sys.executable, + "-m", + "scrapy.cmdline", + "startproject", + project_name, + ".", + ), + cwd=project_dir, + env=self.env, + ) + process.wait() + + actual_permissions = get_permissions_dict(project_dir) + + assert actual_permissions == expected_permissions + + def test_startproject_permissions_umask_022(self): + """Check that generated files have the right permissions when the + system uses a umask value that causes new files to have different + permissions than those from the template folder.""" + + @contextmanager + def umask(new_mask): + cur_mask = os.umask(new_mask) + yield + os.umask(cur_mask) + + scrapy_path = scrapy.__path__[0] + project_template = Path(scrapy_path, "templates", "project") + project_name = "umaskproject" + renamings = ( + ("module", project_name), + (".tmpl", ""), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + with umask(0o002): + destination = mkdtemp() + process = subprocess.Popen( + ( + sys.executable, + "-m", + "scrapy.cmdline", + "startproject", + project_name, + ), + cwd=destination, + env=self.env, + ) + process.wait() + + project_dir = Path(destination, project_name) + actual_permissions = get_permissions_dict(project_dir) + + assert actual_permissions == expected_permissions diff --git a/tests/test_commands.py b/tests/test_commands.py index 16af97842..6e59f561d 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,38 +1,29 @@ from __future__ import annotations import argparse -import inspect import json -import os -import platform import re import subprocess import sys -from contextlib import contextmanager from io import StringIO -from itertools import chain from pathlib import Path -from shutil import copytree, rmtree -from stat import S_IWRITE as ANYONE_WRITE_PERMISSION -from tempfile import TemporaryDirectory, TemporaryFile, mkdtemp +from shutil import rmtree +from tempfile import TemporaryFile, mkdtemp from threading import Timer from typing import TYPE_CHECKING -from unittest import mock, skipIf +from unittest import mock -import pytest from twisted.trial import unittest import scrapy from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view -from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv -from tests.test_crawler import ExceptionSpider, NoRequestsSpider if TYPE_CHECKING: - from collections.abc import Iterator + import os class TestCommandSettings: @@ -125,309 +116,6 @@ class TestProjectBase(unittest.TestCase): return None -class TestStartprojectCommand(TestProjectBase): - def test_startproject(self): - p, out, err = self.proc("startproject", self.project_name) - print(out) - print(err, file=sys.stderr) - assert p.returncode == 0 - - assert Path(self.proj_path, "scrapy.cfg").exists() - assert Path(self.proj_path, "testproject").exists() - assert Path(self.proj_mod_path, "__init__.py").exists() - assert Path(self.proj_mod_path, "items.py").exists() - assert Path(self.proj_mod_path, "pipelines.py").exists() - assert Path(self.proj_mod_path, "settings.py").exists() - assert Path(self.proj_mod_path, "spiders", "__init__.py").exists() - - assert self.call("startproject", self.project_name) == 1 - assert self.call("startproject", "wrong---project---name") == 1 - assert self.call("startproject", "sys") == 1 - - def test_startproject_with_project_dir(self): - project_dir = mkdtemp() - assert self.call("startproject", self.project_name, project_dir) == 0 - - assert Path(project_dir, "scrapy.cfg").exists() - assert Path(project_dir, "testproject").exists() - assert Path(project_dir, self.project_name, "__init__.py").exists() - assert Path(project_dir, self.project_name, "items.py").exists() - assert Path(project_dir, self.project_name, "pipelines.py").exists() - assert Path(project_dir, self.project_name, "settings.py").exists() - assert Path(project_dir, self.project_name, "spiders", "__init__.py").exists() - - assert self.call("startproject", self.project_name, project_dir + "2") == 0 - - assert self.call("startproject", self.project_name, project_dir) == 1 - assert self.call("startproject", self.project_name + "2", project_dir) == 1 - assert self.call("startproject", "wrong---project---name") == 1 - assert self.call("startproject", "sys") == 1 - assert self.call("startproject") == 2 - assert ( - self.call("startproject", self.project_name, project_dir, "another_params") - == 2 - ) - - def test_existing_project_dir(self): - project_dir = mkdtemp() - project_name = self.project_name + "_existing" - project_path = Path(project_dir, project_name) - project_path.mkdir() - - p, out, err = self.proc("startproject", project_name, cwd=project_dir) - print(out) - print(err, file=sys.stderr) - assert p.returncode == 0 - - assert Path(project_path, "scrapy.cfg").exists() - assert Path(project_path, project_name).exists() - assert Path(project_path, project_name, "__init__.py").exists() - assert Path(project_path, project_name, "items.py").exists() - assert Path(project_path, project_name, "pipelines.py").exists() - assert Path(project_path, project_name, "settings.py").exists() - assert Path(project_path, project_name, "spiders", "__init__.py").exists() - - -def get_permissions_dict( - path: str | os.PathLike, renamings=None, ignore=None -) -> dict[str, str]: - def get_permissions(path: Path) -> str: - return oct(path.stat().st_mode) - - path_obj = Path(path) - - renamings = renamings or () - permissions_dict = { - ".": get_permissions(path_obj), - } - for root, dirs, files in os.walk(path_obj): - nodes = list(chain(dirs, files)) - if ignore: - ignored_names = ignore(root, nodes) - nodes = [node for node in nodes if node not in ignored_names] - for node in nodes: - absolute_path = Path(root, node) - relative_path = str(absolute_path.relative_to(path)) - for search_string, replacement in renamings: - relative_path = relative_path.replace(search_string, replacement) - permissions = get_permissions(absolute_path) - permissions_dict[relative_path] = permissions - return permissions_dict - - -class TestStartprojectTemplates(TestProjectBase): - maxDiff = None - - def setUp(self): - super().setUp() - self.tmpl = str(Path(self.temp_path, "templates")) - self.tmpl_proj = str(Path(self.tmpl, "project")) - - def test_startproject_template_override(self): - copytree(Path(scrapy.__path__[0], "templates"), self.tmpl) - Path(self.tmpl_proj, "root_template").write_bytes(b"") - assert Path(self.tmpl_proj, "root_template").exists() - - args = ["--set", f"TEMPLATES_DIR={self.tmpl}"] - p, out, err = self.proc("startproject", self.project_name, *args) - assert ( - f"New Scrapy project '{self.project_name}', using template directory" in out - ) - assert self.tmpl_proj in out - assert Path(self.proj_path, "root_template").exists() - - def test_startproject_permissions_from_writable(self): - """Check that generated files have the right permissions when the - template folder has the same permissions as in the project, i.e. - everything is writable.""" - scrapy_path = scrapy.__path__[0] - project_template = Path(scrapy_path, "templates", "project") - project_name = "startproject1" - renamings = ( - ("module", project_name), - (".tmpl", ""), - ) - expected_permissions = get_permissions_dict( - project_template, - renamings, - IGNORE, - ) - - destination = mkdtemp() - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scrapy.cmdline", - "startproject", - project_name, - ), - cwd=destination, - env=self.env, - ) - process.wait() - - project_dir = Path(destination, project_name) - actual_permissions = get_permissions_dict(project_dir) - - assert actual_permissions == expected_permissions - - def test_startproject_permissions_from_read_only(self): - """Check that generated files have the right permissions when the - template folder has been made read-only, which is something that some - systems do. - - See https://github.com/scrapy/scrapy/pull/4604 - """ - scrapy_path = scrapy.__path__[0] - templates_dir = Path(scrapy_path, "templates") - project_template = Path(templates_dir, "project") - project_name = "startproject2" - renamings = ( - ("module", project_name), - (".tmpl", ""), - ) - expected_permissions = get_permissions_dict( - project_template, - renamings, - IGNORE, - ) - - def _make_read_only(path: Path): - current_permissions = path.stat().st_mode - path.chmod(current_permissions & ~ANYONE_WRITE_PERMISSION) - - read_only_templates_dir = str(Path(mkdtemp()) / "templates") - copytree(templates_dir, read_only_templates_dir) - - for root, dirs, files in os.walk(read_only_templates_dir): - for node in chain(dirs, files): - _make_read_only(Path(root, node)) - - destination = mkdtemp() - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scrapy.cmdline", - "startproject", - project_name, - "--set", - f"TEMPLATES_DIR={read_only_templates_dir}", - ), - cwd=destination, - env=self.env, - ) - process.wait() - - project_dir = Path(destination, project_name) - actual_permissions = get_permissions_dict(project_dir) - - assert actual_permissions == expected_permissions - - def test_startproject_permissions_unchanged_in_destination(self): - """Check that preexisting folders and files in the destination folder - do not see their permissions modified.""" - scrapy_path = scrapy.__path__[0] - project_template = Path(scrapy_path, "templates", "project") - project_name = "startproject3" - renamings = ( - ("module", project_name), - (".tmpl", ""), - ) - expected_permissions = get_permissions_dict( - project_template, - renamings, - IGNORE, - ) - - destination = mkdtemp() - project_dir = Path(destination, project_name) - - existing_nodes = { - oct(permissions)[2:] + extension: permissions - for extension in ("", ".d") - for permissions in ( - 0o444, - 0o555, - 0o644, - 0o666, - 0o755, - 0o777, - ) - } - project_dir.mkdir() - for node, permissions in existing_nodes.items(): - path = project_dir / node - if node.endswith(".d"): - path.mkdir(mode=permissions) - else: - path.touch(mode=permissions) - expected_permissions[node] = oct(path.stat().st_mode) - - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scrapy.cmdline", - "startproject", - project_name, - ".", - ), - cwd=project_dir, - env=self.env, - ) - process.wait() - - actual_permissions = get_permissions_dict(project_dir) - - assert actual_permissions == expected_permissions - - def test_startproject_permissions_umask_022(self): - """Check that generated files have the right permissions when the - system uses a umask value that causes new files to have different - permissions than those from the template folder.""" - - @contextmanager - def umask(new_mask): - cur_mask = os.umask(new_mask) - yield - os.umask(cur_mask) - - scrapy_path = scrapy.__path__[0] - project_template = Path(scrapy_path, "templates", "project") - project_name = "umaskproject" - renamings = ( - ("module", project_name), - (".tmpl", ""), - ) - expected_permissions = get_permissions_dict( - project_template, - renamings, - IGNORE, - ) - - with umask(0o002): - destination = mkdtemp() - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scrapy.cmdline", - "startproject", - project_name, - ), - cwd=destination, - env=self.env, - ) - process.wait() - - project_dir = Path(destination, project_name) - actual_permissions = get_permissions_dict(project_dir) - - assert actual_permissions == expected_permissions - - class TestCommandBase(TestProjectBase): def setUp(self): super().setUp() @@ -436,208 +124,6 @@ class TestCommandBase(TestProjectBase): self.env["SCRAPY_SETTINGS_MODULE"] = f"{self.project_name}.settings" -class TestGenspiderCommand(TestCommandBase): - def test_arguments(self): - # only pass one argument. spider script shouldn't be created - assert self.call("genspider", "test_name") == 2 - assert not Path(self.proj_mod_path, "spiders", "test_name.py").exists() - # pass two arguments . spider script should be created - assert self.call("genspider", "test_name", "test.com") == 0 - assert Path(self.proj_mod_path, "spiders", "test_name.py").exists() - - def test_template(self, tplname="crawl"): - args = [f"--template={tplname}"] if tplname else [] - spname = "test_spider" - spmodule = f"{self.project_name}.spiders.{spname}" - p, out, err = self.proc("genspider", spname, "test.com", *args) - assert ( - f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}" - in out - ) - assert Path(self.proj_mod_path, "spiders", "test_spider.py").exists() - modify_time_before = ( - Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime - ) - p, out, err = self.proc("genspider", spname, "test.com", *args) - assert f"Spider {spname!r} already exists in module" in out - modify_time_after = ( - Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime - ) - assert modify_time_after == modify_time_before - - def test_template_basic(self): - self.test_template("basic") - - def test_template_csvfeed(self): - self.test_template("csvfeed") - - def test_template_xmlfeed(self): - self.test_template("xmlfeed") - - def test_list(self): - assert self.call("genspider", "--list") == 0 - - def test_dump(self): - assert self.call("genspider", "--dump=basic") == 0 - assert self.call("genspider", "-d", "basic") == 0 - - def test_same_name_as_project(self): - assert self.call("genspider", self.project_name) == 2 - assert not Path( - self.proj_mod_path, "spiders", f"{self.project_name}.py" - ).exists() - - def test_same_filename_as_existing_spider(self, force=False): - file_name = "example" - file_path = Path(self.proj_mod_path, "spiders", f"{file_name}.py") - assert self.call("genspider", file_name, "example.com") == 0 - assert file_path.exists() - - # change name of spider but not its file name - with file_path.open("r+", encoding="utf-8") as spider_file: - file_data = spider_file.read() - file_data = file_data.replace('name = "example"', 'name = "renamed"') - spider_file.seek(0) - spider_file.write(file_data) - spider_file.truncate() - modify_time_before = file_path.stat().st_mtime - file_contents_before = file_data - - if force: - p, out, err = self.proc("genspider", "--force", file_name, "example.com") - assert ( - f"Created spider {file_name!r} using template 'basic' in module" in out - ) - modify_time_after = file_path.stat().st_mtime - assert modify_time_after != modify_time_before - file_contents_after = file_path.read_text(encoding="utf-8") - assert file_contents_after != file_contents_before - else: - p, out, err = self.proc("genspider", file_name, "example.com") - assert f"{file_path.resolve()} already exists" in out - modify_time_after = file_path.stat().st_mtime - assert modify_time_after == modify_time_before - file_contents_after = file_path.read_text(encoding="utf-8") - assert file_contents_after == file_contents_before - - def test_same_filename_as_existing_spider_force(self): - self.test_same_filename_as_existing_spider(force=True) - - def test_url(self, url="test.com", domain="test.com"): - assert self.call("genspider", "--force", "test_name", url) == 0 - assert ( - self.find_in_file( - Path(self.proj_mod_path, "spiders", "test_name.py"), - r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]", - ).group(1) - == domain - ) - assert ( - self.find_in_file( - Path(self.proj_mod_path, "spiders", "test_name.py"), - r"start_urls\s*=\s*\[['\"](.+)['\"]\]", - ).group(1) - == f"https://{domain}" - ) - - def test_url_schema(self): - self.test_url("https://test.com", "test.com") - - def test_template_start_urls( - self, url="test.com", expected="https://test.com", template="basic" - ): - assert self.call("genspider", "-t", template, "--force", "test_name", url) == 0 - assert ( - self.find_in_file( - Path(self.proj_mod_path, "spiders", "test_name.py"), - r"start_urls\s*=\s*\[['\"](.+)['\"]\]", - ).group(1) - == expected - ) - - def test_genspider_basic_start_urls(self): - self.test_template_start_urls("https://test.com", "https://test.com", "basic") - self.test_template_start_urls("http://test.com", "http://test.com", "basic") - self.test_template_start_urls( - "http://test.com/other/path", "http://test.com/other/path", "basic" - ) - self.test_template_start_urls( - "test.com/other/path", "https://test.com/other/path", "basic" - ) - - def test_genspider_crawl_start_urls(self): - self.test_template_start_urls("https://test.com", "https://test.com", "crawl") - self.test_template_start_urls("http://test.com", "http://test.com", "crawl") - self.test_template_start_urls( - "http://test.com/other/path", "http://test.com/other/path", "crawl" - ) - self.test_template_start_urls( - "test.com/other/path", "https://test.com/other/path", "crawl" - ) - self.test_template_start_urls("test.com", "https://test.com", "crawl") - - def test_genspider_xmlfeed_start_urls(self): - self.test_template_start_urls( - "https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" - ) - self.test_template_start_urls( - "http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed" - ) - self.test_template_start_urls( - "test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" - ) - - def test_genspider_csvfeed_start_urls(self): - self.test_template_start_urls( - "https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" - ) - self.test_template_start_urls( - "http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed" - ) - self.test_template_start_urls( - "test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" - ) - - -class TestGenspiderStandaloneCommand(TestProjectBase): - def test_generate_standalone_spider(self): - self.call("genspider", "example", "example.com") - assert Path(self.temp_path, "example.py").exists() - - def test_same_name_as_existing_file(self, force=False): - file_name = "example" - file_path = Path(self.temp_path, file_name + ".py") - p, out, err = self.proc("genspider", file_name, "example.com") - assert f"Created spider {file_name!r} using template 'basic' " in out - assert file_path.exists() - modify_time_before = file_path.stat().st_mtime - file_contents_before = file_path.read_text(encoding="utf-8") - - if force: - # use different template to ensure contents were changed - p, out, err = self.proc( - "genspider", "--force", "-t", "crawl", file_name, "example.com" - ) - assert f"Created spider {file_name!r} using template 'crawl' " in out - modify_time_after = file_path.stat().st_mtime - assert modify_time_after != modify_time_before - file_contents_after = file_path.read_text(encoding="utf-8") - assert file_contents_after != file_contents_before - else: - p, out, err = self.proc("genspider", file_name, "example.com") - assert ( - f"{Path(self.temp_path, file_name + '.py').resolve()} already exists" - in out - ) - modify_time_after = file_path.stat().st_mtime - assert modify_time_after == modify_time_before - file_contents_after = file_path.read_text(encoding="utf-8") - assert file_contents_after == file_contents_before - - def test_same_name_as_existing_file_force(self): - self.test_same_name_as_existing_file(force=True) - - class TestMiscCommands(TestCommandBase): def test_list(self): assert self.call("list") == 0 @@ -661,362 +147,6 @@ Unknown command: abc assert out.getvalue().strip() == message.strip() -class TestRunSpiderCommand(TestCommandBase): - spider_filename = "myspider.py" - - debug_log_spider = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug("It Works!") - return - yield -""" - - badspider = """ -import scrapy - -class BadSpider(scrapy.Spider): - name = "bad" - async def start(self): - raise Exception("oops!") - yield - """ - - @contextmanager - def _create_file(self, content: str, name: str | None = None) -> Iterator[str]: - with TemporaryDirectory() as tmpdir: - if name: - fname = Path(tmpdir, name).resolve() - else: - fname = Path(tmpdir, self.spider_filename).resolve() - fname.write_text(content, encoding="utf-8") - yield str(fname) - - def runspider(self, code, name=None, args=()): - with self._create_file(code, name) as fname: - return self.proc("runspider", fname, *args) - - def get_log(self, code, name=None, args=()): - p, stdout, stderr = self.runspider(code, name, args=args) - return stderr - - def test_runspider(self): - log = self.get_log(self.debug_log_spider) - assert "DEBUG: It Works!" in log - assert "INFO: Spider opened" in log - assert "INFO: Closing spider (finished)" in log - assert "INFO: Spider closed (finished)" in log - - def test_run_fail_spider(self): - proc, _, _ = self.runspider( - "import scrapy\n" + inspect.getsource(ExceptionSpider) - ) - ret = proc.returncode - assert ret != 0 - - def test_run_good_spider(self): - proc, _, _ = self.runspider( - "import scrapy\n" + inspect.getsource(NoRequestsSpider) - ) - ret = proc.returncode - assert ret == 0 - - def test_runspider_log_level(self): - log = self.get_log(self.debug_log_spider, args=("-s", "LOG_LEVEL=INFO")) - assert "DEBUG: It Works!" not in log - assert "INFO: Spider opened" in log - - def test_runspider_dnscache_disabled(self): - # see https://github.com/scrapy/scrapy/issues/2811 - # The spider below should not be able to connect to localhost:12345, - # which is intended, - # but this should not be because of DNS lookup error - # assumption: localhost will resolve in all cases (true?) - dnscache_spider = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - start_urls = ['http://localhost:12345'] - - def parse(self, response): - return {'test': 'value'} -""" - log = self.get_log(dnscache_spider, args=("-s", "DNSCACHE_ENABLED=False")) - assert "DNSLookupError" not in log - assert "INFO: Spider opened" in log - - def test_runspider_log_short_names(self): - log1 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=1")) - assert "[myspider] DEBUG: It Works!" in log1 - assert "[scrapy]" in log1 - assert "[scrapy.core.engine]" not in log1 - - log2 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=0")) - assert "[myspider] DEBUG: It Works!" in log2 - assert "[scrapy]" not in log2 - assert "[scrapy.core.engine]" in log2 - - def test_runspider_no_spider_found(self): - log = self.get_log("from scrapy.spiders import Spider\n") - assert "No spider found in file" in log - - def test_runspider_file_not_found(self): - _, _, log = self.proc("runspider", "some_non_existent_file") - assert "File not found: some_non_existent_file" in log - - def test_runspider_unable_to_load(self): - log = self.get_log("", name="myspider.txt") - assert "Unable to load" in log - - def test_start_errors(self): - log = self.get_log(self.badspider, name="badspider.py") - assert "start" in log - assert "badspider.py" in log, log - - def test_asyncio_enabled_true(self): - log = self.get_log( - self.debug_log_spider, - args=[ - "-s", - "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", - ], - ) - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - - def test_asyncio_enabled_default(self): - log = self.get_log(self.debug_log_spider, args=[]) - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - - def test_asyncio_enabled_false(self): - log = self.get_log( - self.debug_log_spider, - args=["-s", "TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor"], - ) - assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log - ) - - @pytest.mark.requires_uvloop - def test_custom_asyncio_loop_enabled_true(self): - log = self.get_log( - self.debug_log_spider, - args=[ - "-s", - "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "-s", - "ASYNCIO_EVENT_LOOP=uvloop.Loop", - ], - ) - assert "Using asyncio event loop: uvloop.Loop" in log - - def test_custom_asyncio_loop_enabled_false(self): - log = self.get_log( - self.debug_log_spider, - args=[ - "-s", - "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", - ], - ) - import asyncio - - if sys.platform != "win32": - loop = asyncio.new_event_loop() - else: - loop = asyncio.SelectorEventLoop() - assert ( - f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}" - in log - ) - - def test_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return - yield -""" - args = ["-o", "example.json"] - log = self.get_log(spider_code, args=args) - assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log - - def test_overwrite_output(self): - spider_code = """ -import json -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug( - 'FEEDS: {}'.format( - json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) - ) - ) - return - yield -""" - Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") - args = ["-O", "example.json"] - log = self.get_log(spider_code, args=args) - assert ( - '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' - in log - ) - with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: - first_line = f2.readline() - assert first_line != "not empty" - - def test_output_and_overwrite_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - return - yield -""" - args = ["-o", "example1.json", "-O", "example2.json"] - log = self.get_log(spider_code, args=args) - assert ( - "error: Please use only one of -o/--output and -O/--overwrite-output" in log - ) - - def test_output_stdout(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return - yield -""" - args = ["-o", "-:json"] - log = self.get_log(spider_code, args=args) - assert "[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}" in log - - @skipIf(platform.system() == "Windows", reason="Linux only") - def test_absolute_path_linux(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - start_urls = ["data:,"] - - def parse(self, response): - yield {"hello": "world"} - """ - temp_dir = mkdtemp() - - args = ["-o", f"{temp_dir}/output1.json:json"] - log = self.get_log(spider_code, args=args) - assert ( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json" - in log - ) - - args = ["-o", f"{temp_dir}/output2.json"] - log = self.get_log(spider_code, args=args) - assert ( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json" - in log - ) - - @skipIf(platform.system() != "Windows", reason="Windows only") - def test_absolute_path_windows(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - start_urls = ["data:,"] - - def parse(self, response): - yield {"hello": "world"} - """ - temp_dir = mkdtemp() - - args = ["-o", f"{temp_dir}\\output1.json:json"] - log = self.get_log(spider_code, args=args) - assert ( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json" - in log - ) - - args = ["-o", f"{temp_dir}\\output2.json"] - log = self.get_log(spider_code, args=args) - assert ( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json" - in log - ) - - def test_args_change_settings(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - @classmethod - def from_crawler(cls, crawler, *args, **kwargs): - spider = super().from_crawler(crawler, *args, **kwargs) - spider.settings.set("FOO", kwargs.get("foo")) - return spider - - async def start(self): - self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") - return - yield -""" - args = ["-a", "foo=42"] - log = self.get_log(spider_code, args=args) - assert "Spider closed (finished)" in log - assert "The value of FOO is 42" in log - - -class TestWindowsRunSpiderCommand(TestRunSpiderCommand): - spider_filename = "myspider.pyw" - - def setUp(self): - if platform.system() != "Windows": - raise unittest.SkipTest("Windows required for .pyw files") - return super().setUp() - - def test_start_errors(self): - log = self.get_log(self.badspider, name="badspider.pyw") - assert "start" in log - assert "badspider.pyw" in log - - def test_runspider_unable_to_load(self): - raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") - - class TestBenchCommand(TestCommandBase): def test_run(self): _, _, log = self.proc( @@ -1042,94 +172,6 @@ class TestViewCommand(TestCommandBase): assert "URL using the Scrapy downloader and show its" in command.long_desc() -class TestCrawlCommand(TestCommandBase): - def crawl(self, code, args=()): - Path(self.proj_mod_path, "spiders", "myspider.py").write_text( - code, encoding="utf-8" - ) - return self.proc("crawl", "myspider", *args) - - def get_log(self, code, args=()): - _, _, stderr = self.crawl(code, args=args) - return stderr - - def test_no_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug('It works!') - return - yield -""" - log = self.get_log(spider_code) - assert "[myspider] DEBUG: It works!" in log - - def test_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return - yield -""" - args = ["-o", "example.json"] - log = self.get_log(spider_code, args=args) - assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log - - def test_overwrite_output(self): - spider_code = """ -import json -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug( - 'FEEDS: {}'.format( - json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) - ) - ) - return - yield -""" - Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") - args = ["-O", "example.json"] - log = self.get_log(spider_code, args=args) - assert ( - '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' - in log - ) - with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: - first_line = f2.readline() - assert first_line != "not empty" - - def test_output_and_overwrite_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - return - yield -""" - args = ["-o", "example1.json", "-O", "example2.json"] - log = self.get_log(spider_code, args=args) - assert ( - "error: Please use only one of -o/--output and -O/--overwrite-output" in log - ) - - class TestHelpMessage(TestCommandBase): def setUp(self): super().setUp() From 8f92a26636b3cabf64fd29a65ef0eb554bdf4f05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 30 May 2025 09:33:17 +0200 Subject: [PATCH 139/491] Avoid raw HTML in the README (#6839) --- README.rst | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index 5dc994570..536dec7f0 100644 --- a/README.rst +++ b/README.rst @@ -1,40 +1,41 @@ -.. raw:: html +|logo| -

- - Scrapy - -

+.. |logo| image:: https://raw.githubusercontent.com/scrapy/scrapy/master/docs/_static/logo.svg + :target: https://scrapy.org + :alt: Scrapy + :width: 480px -.. image:: https://img.shields.io/pypi/v/Scrapy.svg +|version| |python_version| |ubuntu| |macos| |windows| |coverage| |conda| |deepwiki| + +.. |version| image:: https://img.shields.io/pypi/v/Scrapy.svg :target: https://pypi.org/pypi/Scrapy :alt: PyPI Version -.. image:: https://img.shields.io/pypi/pyversions/Scrapy.svg +.. |python_version| image:: https://img.shields.io/pypi/pyversions/Scrapy.svg :target: https://pypi.org/pypi/Scrapy :alt: Supported Python Versions -.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg +.. |ubuntu| image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu :alt: Ubuntu -.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg +.. |macos| image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS :alt: macOS -.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg +.. |windows| image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows :alt: Windows -.. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg +.. |coverage| image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg :target: https://codecov.io/github/scrapy/scrapy?branch=master :alt: Coverage report -.. image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg +.. |conda| image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg :target: https://anaconda.org/conda-forge/scrapy :alt: Conda Version -.. image:: https://deepwiki.com/badge.svg +.. |deepwiki| image:: https://deepwiki.com/badge.svg :target: https://deepwiki.com/scrapy/scrapy :alt: Ask DeepWiki From 8ae418df44b7a107a4abe1a718721742d7e33fc0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 2 Jun 2025 19:02:08 +0500 Subject: [PATCH 140/491] Rewrite download handler tests to coroutines. (#6846) --- .../test_downloader_handler_twisted_http10.py | 10 +- .../test_downloader_handler_twisted_http2.py | 112 +++-- tests/test_downloader_handlers.py | 123 +++--- tests/test_downloader_handlers_http_base.py | 410 +++++++++--------- 4 files changed, 329 insertions(+), 326 deletions(-) diff --git a/tests/test_downloader_handler_twisted_http10.py b/tests/test_downloader_handler_twisted_http10.py index 807c8c4cb..bc306aa07 100644 --- a/tests/test_downloader_handler_twisted_http10.py +++ b/tests/test_downloader_handler_twisted_http10.py @@ -9,6 +9,7 @@ import pytest from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.defer import deferred_f_from_coro_f from tests.test_downloader_handlers_http_base import TestHttpBase, TestHttpProxyBase if TYPE_CHECKING: @@ -25,12 +26,11 @@ class HTTP10DownloadHandlerMixin: class TestHttp10(HTTP10DownloadHandlerMixin, TestHttpBase): """HTTP 1.0 test case""" - def test_protocol(self): + @deferred_f_from_coro_f + async def test_protocol(self): request = Request(self.getURL("host"), method="GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, "HTTP/1.0") - return d + response = await self.download_request(request, Spider("foo")) + assert response.protocol == "HTTP/1.0" class TestHttps10(TestHttp10): diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 159f403d0..e058cedae 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -15,6 +15,10 @@ from twisted.web.http import H2_ENABLED from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + maybe_deferred_to_future, +) from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler from tests.mockserver import ssl_context_factory @@ -50,15 +54,14 @@ class H2DownloadHandlerMixin: class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" - def test_protocol(self): + @deferred_f_from_coro_f + async def test_protocol(self): request = Request(self.getURL("host"), method="GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, "h2") - return d + response = await self.download_request(request, Spider("foo")) + assert response.protocol == "h2" - @defer.inlineCallbacks - def test_download_with_maxsize_very_large_file(self): + @deferred_f_from_coro_f + async def test_download_with_maxsize_very_large_file(self): from twisted.internet import reactor with mock.patch("scrapy.core.http2.stream.logger") as logger: @@ -67,8 +70,10 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): def check(logger): logger.error.assert_called_once_with(mock.ANY) - d = self.download_request(request, Spider("foo", download_maxsize=1500)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + with pytest.raises((defer.CancelledError, error.ConnectionAborted)): + await self.download_request( + request, Spider("foo", download_maxsize=1500) + ) # As the error message is logged in the dataReceived callback, we # have to give a bit of time to the reactor to process the queue @@ -76,13 +81,13 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): d = defer.Deferred() d.addCallback(check) reactor.callLater(0.1, d.callback, logger) - yield d + await maybe_deferred_to_future(d) - @defer.inlineCallbacks - def test_unsupported_scheme(self): + @deferred_f_from_coro_f + async def test_unsupported_scheme(self): request = Request("ftp://unsupported.scheme") - d = self.download_request(request, Spider("foo")) - yield self.assertFailure(d, SchemeNotSupported) + with pytest.raises(SchemeNotSupported): + await self.download_request(request, Spider("foo")) def test_download_broken_content_cause_data_loss(self, url="broken"): pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) @@ -102,70 +107,60 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): def test_download_broken_chunked_content_allow_data_loss_via_setting(self): pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) - def test_concurrent_requests_same_domain(self): + @deferred_f_from_coro_f + async def test_concurrent_requests_same_domain(self): spider = Spider("foo") request1 = Request(self.getURL("file")) - d1 = self.download_request(request1, spider) - d1.addCallback(lambda r: r.body) - d1.addCallback(self.assertEqual, b"0123456789") + response1 = await self.download_request(request1, spider) + assert response1.body == b"0123456789" request2 = Request(self.getURL("echo"), method="POST") - d2 = self.download_request(request2, spider) - d2.addCallback(lambda r: r.headers["Content-Length"]) - d2.addCallback(self.assertEqual, b"79") - - return defer.DeferredList([d1, d2]) + response2 = await self.download_request(request2, spider) + assert response2.headers["Content-Length"] == b"79" @pytest.mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247") - def test_connect_request(self): + @deferred_f_from_coro_f + async def test_connect_request(self): request = Request(self.getURL("file"), method="CONNECT") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"" - def test_custom_content_length_good(self): + @deferred_f_from_coro_f + async def test_custom_content_length_good(self): request = Request(self.getURL("contentlength")) custom_content_length = str(len(request.body)) request.headers["Content-Length"] = custom_content_length - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.text) - d.addCallback(self.assertEqual, custom_content_length) - return d + response = await self.download_request(request, Spider("foo")) + assert response.text == custom_content_length - def test_custom_content_length_bad(self): + @deferred_f_from_coro_f + async def test_custom_content_length_bad(self): request = Request(self.getURL("contentlength")) actual_content_length = str(len(request.body)) bad_content_length = str(len(request.body) + 1) request.headers["Content-Length"] = bad_content_length - log = LogCapture() - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.text) - d.addCallback(self.assertEqual, actual_content_length) - d.addCallback( - lambda _: log.check_present( - ( - "scrapy.core.http2.stream", - "WARNING", - f"Ignoring bad Content-Length header " - f"{bad_content_length!r} of request {request}, sending " - f"{actual_content_length!r} instead", - ) + with LogCapture() as log: + response = await self.download_request(request, Spider("foo")) + assert response.text == actual_content_length + log.check_present( + ( + "scrapy.core.http2.stream", + "WARNING", + f"Ignoring bad Content-Length header " + f"{bad_content_length!r} of request {request}, sending " + f"{actual_content_length!r} instead", ) ) - d.addCallback(lambda _: log.uninstall()) - return d - def test_duplicate_header(self): + @deferred_f_from_coro_f + async def test_duplicate_header(self): request = Request(self.getURL("echo")) header, value1, value2 = "Custom-Header", "foo", "bar" request.headers.appendlist(header, value1) request.headers.appendlist(header, value2) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: json.loads(r.text)["headers"][header]) - d.addCallback(self.assertEqual, [value1, value2]) - return d + response = await self.download_request(request, Spider("foo")) + assert json.loads(response.text)["headers"][header] == [value1, value2] class Https2WrongHostnameTestCase(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): @@ -222,12 +217,13 @@ class Https2ProxyTestCase(H2DownloadHandlerMixin, TestHttpProxyBase): self.download_handler = build_from_crawler( self.download_handler_cls, get_crawler() ) - self.download_request = self.download_handler.download_request def getURL(self, path): return f"{self.scheme}://{self.host}:{self.portno}/{path}" - @defer.inlineCallbacks - def test_download_with_proxy_https_timeout(self): + @deferred_f_from_coro_f + async def test_download_with_proxy_https_timeout(self): with pytest.raises(NotImplementedError): - yield super().test_download_with_proxy_https_timeout() + await maybe_deferred_to_future( + super().test_download_with_proxy_https_timeout() + ) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index dacadb075..09cdbaf35 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -22,10 +22,14 @@ from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.exceptions import NotConfigured -from scrapy.http import HtmlResponse, Request +from scrapy.http import HtmlResponse, Request, Response from scrapy.http.response.text import TextResponse from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + maybe_deferred_to_future, +) from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler @@ -95,28 +99,33 @@ class TestFile(unittest.TestCase): # add a special char to check that they are handled correctly self.fd, self.tmpname = mkstemp(suffix="^") Path(self.tmpname).write_text("0123456789", encoding="utf-8") - handler = build_from_crawler(FileDownloadHandler, get_crawler()) - self.download_request = handler.download_request + self.download_handler = build_from_crawler(FileDownloadHandler, get_crawler()) def tearDown(self): os.close(self.fd) Path(self.tmpname).unlink() - def test_download(self): - def _test(response): - assert response.url == request.url - assert response.status == 200 - assert response.body == b"0123456789" - assert response.protocol is None + async def download_request(self, request: Request, spider: Spider) -> Response: + return await maybe_deferred_to_future( + self.download_handler.download_request(request, spider) + ) + @deferred_f_from_coro_f + async def test_download(self): request = Request(path_to_file_uri(self.tmpname)) assert request.url.upper().endswith("%5E") - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.url == request.url + assert response.status == 200 + assert response.body == b"0123456789" + assert response.protocol is None - def test_non_existent(self): + @deferred_f_from_coro_f + async def test_non_existent(self): request = Request(path_to_file_uri(mkdtemp())) - d = self.download_request(request, Spider("foo")) - return self.assertFailure(d, OSError) + # the specific exception differs between platforms + with pytest.raises(OSError): # noqa: PT011 + await self.download_request(request, Spider("foo")) class HttpDownloadHandlerMock: @@ -479,69 +488,65 @@ class TestDataURI(unittest.TestCase): def setUp(self): crawler = get_crawler() self.download_handler = build_from_crawler(DataURIDownloadHandler, crawler) - self.download_request = self.download_handler.download_request self.spider = Spider("foo") - def test_response_attrs(self): + async def download_request(self, request: Request, spider: Spider) -> Response: + return await maybe_deferred_to_future( + self.download_handler.download_request(request, spider) + ) + + @deferred_f_from_coro_f + async def test_response_attrs(self): uri = "data:,A%20brief%20note" - - def _test(response): - assert response.url == uri - assert not response.headers - request = Request(uri) - return self.download_request(request, self.spider).addCallback(_test) - - def test_default_mediatype_encoding(self): - def _test(response): - assert response.text == "A brief note" - assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck - assert response.encoding == "US-ASCII" + response = await self.download_request(request, self.spider) + assert response.url == uri + assert not response.headers + @deferred_f_from_coro_f + async def test_default_mediatype_encoding(self): request = Request("data:,A%20brief%20note") - return self.download_request(request, self.spider).addCallback(_test) - - def test_default_mediatype(self): - def _test(response): - assert response.text == "\u038e\u03a3\u038e" - assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck - assert response.encoding == "iso-8859-7" + response = await self.download_request(request, self.spider) + assert response.text == "A brief note" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "US-ASCII" + @deferred_f_from_coro_f + async def test_default_mediatype(self): request = Request("data:;charset=iso-8859-7,%be%d3%be") - return self.download_request(request, self.spider).addCallback(_test) - - def test_text_charset(self): - def _test(response): - assert response.text == "\u038e\u03a3\u038e" - assert response.body == b"\xbe\xd3\xbe" - assert response.encoding == "iso-8859-7" + response = await self.download_request(request, self.spider) + assert response.text == "\u038e\u03a3\u038e" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "iso-8859-7" + @deferred_f_from_coro_f + async def test_text_charset(self): request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") - return self.download_request(request, self.spider).addCallback(_test) - - def test_mediatype_parameters(self): - def _test(response): - assert response.text == "\u038e\u03a3\u038e" - assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck - assert response.encoding == "utf-8" + response = await self.download_request(request, self.spider) + assert response.text == "\u038e\u03a3\u038e" + assert response.body == b"\xbe\xd3\xbe" + assert response.encoding == "iso-8859-7" + @deferred_f_from_coro_f + async def test_mediatype_parameters(self): request = Request( "data:text/plain;foo=%22foo;bar%5C%22%22;" "charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22" ",%CE%8E%CE%A3%CE%8E" ) - return self.download_request(request, self.spider).addCallback(_test) - - def test_base64(self): - def _test(response): - assert response.text == "Hello, world." + response = await self.download_request(request, self.spider) + assert response.text == "\u038e\u03a3\u038e" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "utf-8" + @deferred_f_from_coro_f + async def test_base64(self): request = Request("data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D") - return self.download_request(request, self.spider).addCallback(_test) - - def test_protocol(self): - def _test(response): - assert response.protocol is None + response = await self.download_request(request, self.spider) + assert response.text == "Hello, world." + @deferred_f_from_coro_f + async def test_protocol(self): request = Request("data:,") - return self.download_request(request, self.spider).addCallback(_test) + response = await self.download_request(request, self.spider) + assert response.protocol is None diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 5eaf66966..14e12a3e6 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import shutil import sys from abc import ABC, abstractmethod @@ -13,14 +14,20 @@ from unittest import mock import pytest from testfixtures import LogCapture from twisted.internet import defer, error +from twisted.internet.defer import maybeDeferred from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest from twisted.web import resource, server, static, util from twisted.web._newclient import ResponseFailed from twisted.web.http import _DataLoss -from scrapy.http import Headers, HtmlResponse, Request, TextResponse +from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse from scrapy.spiders import Spider +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + deferred_from_coro, + maybe_deferred_to_future, +) from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler @@ -178,7 +185,6 @@ class TestHttpBase(unittest.TestCase, ABC): self.download_handler = build_from_crawler( self.download_handler_cls, get_crawler() ) - self.download_request = self.download_handler.download_request @defer.inlineCallbacks def tearDown(self): @@ -190,36 +196,37 @@ class TestHttpBase(unittest.TestCase, ABC): def getURL(self, path): return f"{self.scheme}://{self.host}:{self.portno}/{path}" - def test_download(self): + async def download_request(self, request: Request, spider: Spider) -> Response: + return await maybe_deferred_to_future( + self.download_handler.download_request(request, spider) + ) + + @deferred_f_from_coro_f + async def test_download(self): request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"0123456789" - def test_download_head(self): + @deferred_f_from_coro_f + async def test_download_head(self): request = Request(self.getURL("file"), method="HEAD") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"" - def test_redirect_status(self): + @deferred_f_from_coro_f + async def test_redirect_status(self): request = Request(self.getURL("redirect")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.status) - d.addCallback(self.assertEqual, 302) - return d + response = await self.download_request(request, Spider("foo")) + assert response.status == 302 - def test_redirect_status_head(self): + @deferred_f_from_coro_f + async def test_redirect_status_head(self): request = Request(self.getURL("redirect"), method="HEAD") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.status) - d.addCallback(self.assertEqual, 302) - return d + response = await self.download_request(request, Spider("foo")) + assert response.status == 302 - @defer.inlineCallbacks - def test_timeout_download_from_spider_nodata_rcvd(self): + @deferred_f_from_coro_f + async def test_timeout_download_from_spider_nodata_rcvd(self): if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 raise unittest.SkipTest( @@ -230,11 +237,12 @@ class TestHttpBase(unittest.TestCase, ABC): spider = Spider("foo") meta = {"download_timeout": 0.5} request = Request(self.getURL("wait"), meta=meta) - d = self.download_request(request, spider) - yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) + d = deferred_from_coro(self.download_request(request, spider)) + with pytest.raises((defer.TimeoutError, error.TimeoutError)): + await maybe_deferred_to_future(d) - @defer.inlineCallbacks - def test_timeout_download_from_spider_server_hangs(self): + @deferred_f_from_coro_f + async def test_timeout_download_from_spider_server_hangs(self): if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 raise unittest.SkipTest( @@ -244,28 +252,27 @@ class TestHttpBase(unittest.TestCase, ABC): spider = Spider("foo") meta = {"download_timeout": 0.5} request = Request(self.getURL("hang-after-headers"), meta=meta) - d = self.download_request(request, spider) - yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) - - def test_host_header_not_in_request_headers(self): - def _test(response): - assert response.body == to_bytes(f"{self.host}:{self.portno}") - assert not request.headers + d = deferred_from_coro(self.download_request(request, spider)) + with pytest.raises((defer.TimeoutError, error.TimeoutError)): + await maybe_deferred_to_future(d) + @deferred_f_from_coro_f + async def test_host_header_not_in_request_headers(self): request = Request(self.getURL("host")) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.body == to_bytes(f"{self.host}:{self.portno}") + assert not request.headers - def test_host_header_seted_in_request_headers(self): + @deferred_f_from_coro_f + async def test_host_header_set_in_request_headers(self): host = self.host + ":" + str(self.portno) - - def _test(response): - assert response.body == host.encode() - assert request.headers.get("Host") == host.encode() - request = Request(self.getURL("host"), headers={"Host": host}) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.body == host.encode() + assert request.headers.get("Host") == host.encode() - def test_content_length_zero_bodyless_post_request_headers(self): + @deferred_f_from_coro_f + async def test_content_length_zero_bodyless_post_request_headers(self): """Tests if "Content-Length: 0" is sent for bodyless POST requests. This is not strictly required by HTTP RFCs but can cause trouble @@ -276,105 +283,93 @@ class TestHttpBase(unittest.TestCase, ABC): https://github.com/kennethreitz/requests/issues/405 https://bugs.python.org/issue14721 """ - - def _test(response): - assert response.body == b"0" - request = Request(self.getURL("contentlength"), method="POST") - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_content_length_zero_bodyless_post_only_one(self): - def _test(response): - import json - - headers = Headers(json.loads(response.text)["headers"]) - contentlengths = headers.getlist("Content-Length") - assert len(contentlengths) == 1 - assert contentlengths == [b"0"] + response = await self.download_request(request, Spider("foo")) + assert response.body == b"0" + @deferred_f_from_coro_f + async def test_content_length_zero_bodyless_post_only_one(self): request = Request(self.getURL("echo"), method="POST") - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + headers = Headers(json.loads(response.text)["headers"]) + contentlengths = headers.getlist("Content-Length") + assert len(contentlengths) == 1 + assert contentlengths == [b"0"] - def test_payload(self): + @deferred_f_from_coro_f + async def test_payload(self): body = b"1" * 100 # PayloadResource requires body length to be 100 request = Request(self.getURL("payload"), method="POST", body=body) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, body) - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == body - def test_response_header_content_length(self): + @deferred_f_from_coro_f + async def test_response_header_content_length(self): request = Request(self.getURL("file"), method=b"GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.headers[b"content-length"]) - d.addCallback(self.assertEqual, b"159") - return d - - def _test_response_class(self, filename, body, response_class): - def _test(response): - assert type(response) is response_class # pylint: disable=unidiomatic-typecheck + response = await self.download_request(request, Spider("foo")) + assert response.headers[b"content-length"] == b"159" + async def _test_response_class( + self, filename: str, body: bytes, response_class: type[Response] + ) -> None: request = Request(self.getURL(filename), body=body) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert type(response) is response_class # pylint: disable=unidiomatic-typecheck - def test_response_class_from_url(self): - return self._test_response_class("foo.html", b"", HtmlResponse) + @deferred_f_from_coro_f + async def test_response_class_from_url(self): + await self._test_response_class("foo.html", b"", HtmlResponse) - def test_response_class_from_body(self): - return self._test_response_class( + @deferred_f_from_coro_f + async def test_response_class_from_body(self): + await self._test_response_class( "foo", b"\n.", HtmlResponse, ) - def test_get_duplicate_header(self): - def _test(response): - assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] - + @deferred_f_from_coro_f + async def test_get_duplicate_header(self): request = Request(self.getURL("duplicate-header")) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] class TestHttp11Base(TestHttpBase): """HTTP 1.1 test case""" - def test_download_without_maxsize_limit(self): + @deferred_f_from_coro_f + async def test_download_without_maxsize_limit(self): request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"0123456789" - def test_response_class_choosing_request(self): + @deferred_f_from_coro_f + async def test_response_class_choosing_request(self): """Tests choosing of correct response type in case of Content-Type is empty but body contains text. """ body = b"Some plain text\ndata with tabs\t and null bytes\0" - - def _test_type(response): - assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck - request = Request(self.getURL("nocontenttype"), body=body) - d = self.download_request(request, Spider("foo")) - d.addCallback(_test_type) - return d + response = await self.download_request(request, Spider("foo")) + assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck - @defer.inlineCallbacks - def test_download_with_maxsize(self): + @deferred_f_from_coro_f + async def test_download_with_maxsize(self): request = Request(self.getURL("file")) # 10 is minimal size for this request and the limit is only counted on # response body. (regardless of headers) - d = self.download_request(request, Spider("foo", download_maxsize=10)) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - yield d + response = await self.download_request( + request, Spider("foo", download_maxsize=10) + ) + assert response.body == b"0123456789" - d = self.download_request(request, Spider("foo", download_maxsize=9)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + with pytest.raises((defer.CancelledError, error.ConnectionAborted)): + await self.download_request(request, Spider("foo", download_maxsize=9)) - @defer.inlineCallbacks - def test_download_with_maxsize_very_large_file(self): + @deferred_f_from_coro_f + async def test_download_with_maxsize_very_large_file(self): from twisted.internet import reactor # TODO: the logger check is specific to scrapy.core.downloader.handlers.http11 @@ -384,8 +379,10 @@ class TestHttp11Base(TestHttpBase): def check(logger): logger.warning.assert_called_once_with(mock.ANY, mock.ANY) - d = self.download_request(request, Spider("foo", download_maxsize=1500)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + with pytest.raises((defer.CancelledError, error.ConnectionAborted)): + await self.download_request( + request, Spider("foo", download_maxsize=1500) + ) # As the error message is logged in the dataReceived callback, we # have to give a bit of time to the reactor to process the queue @@ -393,84 +390,81 @@ class TestHttp11Base(TestHttpBase): d = defer.Deferred() d.addCallback(check) reactor.callLater(0.1, d.callback, logger) - yield d + await maybe_deferred_to_future(d) - @defer.inlineCallbacks - def test_download_with_maxsize_per_req(self): + @deferred_f_from_coro_f + async def test_download_with_maxsize_per_req(self): meta = {"download_maxsize": 2} request = Request(self.getURL("file"), meta=meta) - d = self.download_request(request, Spider("foo")) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + with pytest.raises((defer.CancelledError, error.ConnectionAborted)): + await self.download_request(request, Spider("foo")) - @defer.inlineCallbacks - def test_download_with_small_maxsize_per_spider(self): + @deferred_f_from_coro_f + async def test_download_with_small_maxsize_per_spider(self): request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo", download_maxsize=2)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + with pytest.raises((defer.CancelledError, error.ConnectionAborted)): + await self.download_request(request, Spider("foo", download_maxsize=2)) - def test_download_with_large_maxsize_per_spider(self): + @deferred_f_from_coro_f + async def test_download_with_large_maxsize_per_spider(self): request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo", download_maxsize=100)) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d + response = await self.download_request( + request, Spider("foo", download_maxsize=100) + ) + assert response.body == b"0123456789" - def test_download_chunked_content(self): + @deferred_f_from_coro_f + async def test_download_chunked_content(self): request = Request(self.getURL("chunked")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"chunked content\n") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"chunked content\n" - def test_download_broken_content_cause_data_loss(self, url="broken"): + @deferred_f_from_coro_f + async def test_download_broken_content_cause_data_loss( + self, url: str = "broken" + ) -> None: # TODO: this one checks for Twisted-specific exceptions request = Request(self.getURL(url)) - d = self.download_request(request, Spider("foo")) - - def checkDataLoss(failure): - if failure.check(ResponseFailed) and any( - r.check(_DataLoss) for r in failure.value.reasons - ): - return None - return failure - - d.addCallback(lambda _: self.fail("No DataLoss exception")) - d.addErrback(checkDataLoss) - return d + with pytest.raises(ResponseFailed) as exc_info: + await self.download_request(request, Spider("foo")) + assert any(r.check(_DataLoss) for r in exc_info.value.reasons) def test_download_broken_chunked_content_cause_data_loss(self): return self.test_download_broken_content_cause_data_loss("broken-chunked") - def test_download_broken_content_allow_data_loss(self, url="broken"): + @deferred_f_from_coro_f + async def test_download_broken_content_allow_data_loss( + self, url: str = "broken" + ) -> None: request = Request(self.getURL(url), meta={"download_fail_on_dataloss": False}) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.flags) - d.addCallback(self.assertEqual, ["dataloss"]) - return d + response = await self.download_request(request, Spider("foo")) + assert response.flags == ["dataloss"] def test_download_broken_chunked_content_allow_data_loss(self): return self.test_download_broken_content_allow_data_loss("broken-chunked") - def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"): + @deferred_f_from_coro_f + async def test_download_broken_content_allow_data_loss_via_setting( + self, url: str = "broken" + ) -> None: crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False}) download_handler = build_from_crawler(self.download_handler_cls, crawler) request = Request(self.getURL(url)) - d = download_handler.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.flags) - d.addCallback(self.assertEqual, ["dataloss"]) - return d + response = await maybe_deferred_to_future( + download_handler.download_request(request, Spider("foo")) + ) + assert response.flags == ["dataloss"] def test_download_broken_chunked_content_allow_data_loss_via_setting(self): return self.test_download_broken_content_allow_data_loss_via_setting( "broken-chunked" ) - def test_protocol(self): + @deferred_f_from_coro_f + async def test_protocol(self): request = Request(self.getURL("host"), method="GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, "HTTP/1.1") - return d + response = await self.download_request(request, Spider("foo")) + assert response.protocol == "HTTP/1.1" class TestHttps11Base(TestHttp11Base): @@ -481,8 +475,8 @@ class TestHttps11Base(TestHttp11Base): 'subject "/C=IE/O=Scrapy/CN=localhost"' ) - @defer.inlineCallbacks - def test_tls_logging(self): + @deferred_f_from_coro_f + async def test_tls_logging(self): crawler = get_crawler( settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True} ) @@ -490,15 +484,15 @@ class TestHttps11Base(TestHttp11Base): try: with LogCapture() as log_capture: request = Request(self.getURL("file")) - d = download_handler.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - yield d + response = await maybe_deferred_to_future( + download_handler.download_request(request, Spider("foo")) + ) + assert response.body == b"0123456789" log_capture.check_present( ("scrapy.core.downloader.tls", "DEBUG", self.tls_log_message) ) finally: - yield download_handler.close() + await maybe_deferred_to_future(maybeDeferred(download_handler.close)) class TestSimpleHttpsBase(unittest.TestCase, ABC): @@ -536,7 +530,6 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC): settings_dict = None crawler = get_crawler(settings_dict=settings_dict) self.download_handler = build_from_crawler(self.download_handler_cls, crawler) - self.download_request = self.download_handler.download_request @defer.inlineCallbacks def tearDown(self): @@ -548,12 +541,16 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC): def getURL(self, path): return f"https://{self.host}:{self.portno}/{path}" - def test_download(self): + async def download_request(self, request: Request, spider: Spider) -> Response: + return await maybe_deferred_to_future( + self.download_handler.download_request(request, spider) + ) + + @deferred_f_from_coro_f + async def test_download(self): request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"0123456789" class TestHttpsWrongHostnameBase(TestSimpleHttpsBase): @@ -604,25 +601,29 @@ class TestHttpMockServerBase(unittest.TestCase, ABC): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - @defer.inlineCallbacks - def test_download_with_content_length(self): + @deferred_f_from_coro_f + async def test_download_with_content_length(self): crawler = get_crawler(SingleRequestSpider, self.settings_dict) # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid # download it - yield crawler.crawl( - seed=Request( - url=self.mockserver.url("/partial", is_secure=self.is_secure), - meta={"download_maxsize": 1000}, + await maybe_deferred_to_future( + crawler.crawl( + seed=Request( + url=self.mockserver.url("/partial", is_secure=self.is_secure), + meta={"download_maxsize": 1000}, + ) ) ) failure = crawler.spider.meta["failure"] assert isinstance(failure.value, defer.CancelledError) - @defer.inlineCallbacks - def test_download(self): + @deferred_f_from_coro_f + async def test_download(self): crawler = get_crawler(SingleRequestSpider, self.settings_dict) - yield crawler.crawl( - seed=Request(url=self.mockserver.url("", is_secure=self.is_secure)) + await maybe_deferred_to_future( + crawler.crawl( + seed=Request(url=self.mockserver.url("", is_secure=self.is_secure)) + ) ) failure = crawler.spider.meta.get("failure") assert failure is None @@ -663,7 +664,6 @@ class TestHttpProxyBase(unittest.TestCase, ABC): self.download_handler = build_from_crawler( self.download_handler_cls, get_crawler() ) - self.download_request = self.download_handler.download_request @defer.inlineCallbacks def tearDown(self): @@ -674,42 +674,44 @@ class TestHttpProxyBase(unittest.TestCase, ABC): def getURL(self, path): return f"http://127.0.0.1:{self.portno}/{path}" - def test_download_with_proxy(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == self.expected_http_proxy_request_body + async def download_request(self, request: Request, spider: Spider) -> Response: + return await maybe_deferred_to_future( + self.download_handler.download_request(request, spider) + ) + @deferred_f_from_coro_f + async def test_download_with_proxy(self): http_proxy = self.getURL("") request = Request("http://example.com", meta={"proxy": http_proxy}) - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_download_without_proxy(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == b"/path/to/resource" + response = await self.download_request(request, Spider("foo")) + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body + @deferred_f_from_coro_f + async def test_download_without_proxy(self): request = Request(self.getURL("path/to/resource")) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.status == 200 + assert response.url == request.url + assert response.body == b"/path/to/resource" - @defer.inlineCallbacks - def test_download_with_proxy_https_timeout(self): + @deferred_f_from_coro_f + async def test_download_with_proxy_https_timeout(self): if NON_EXISTING_RESOLVABLE: pytest.skip("Non-existing hosts are resolvable") http_proxy = self.getURL("") domain = "https://no-such-domain.nosuch" request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2}) - d = self.download_request(request, Spider("foo")) - timeout = yield self.assertFailure(d, error.TimeoutError) - assert domain in timeout.osError - - def test_download_with_proxy_without_http_scheme(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == self.expected_http_proxy_request_body + with pytest.raises(error.TimeoutError) as exc_info: + await self.download_request(request, Spider("foo")) + assert domain in exc_info.value.osError + @deferred_f_from_coro_f + async def test_download_with_proxy_without_http_scheme(self): http_proxy = self.getURL("").replace("http://", "") request = Request("http://example.com", meta={"proxy": http_proxy}) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body From 9cc23641ccc988d1f623902397496db5d9fe499b Mon Sep 17 00:00:00 2001 From: Rodrigosnrocha <83819959+Rodrigosnrocha@users.noreply.github.com> Date: Mon, 2 Jun 2025 17:00:17 +0200 Subject: [PATCH 141/491] Deprecate _parse_response and implement parse_with_rules (#6804) --- docs/conf.py | 1 - pyproject.toml | 3 +++ scrapy/spiders/crawl.py | 32 +++++++++++++++++++++++++++--- tests/test_spider.py | 44 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 1167ce050..493a62976 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -3,7 +3,6 @@ # For the full list of built-in configuration values, see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html -# pylint: disable=import-error import os import sys from collections.abc import Sequence diff --git a/pyproject.toml b/pyproject.toml index 871da8020..02ab7858d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,6 +161,9 @@ extension-pkg-allow-list=[ enable = [ "useless-suppression", ] +# Make INFO checks like useless-suppression also cause pylint to return a +# non-zero exit code. +fail-on = "I" disable = [ # Ones we want to ignore "attribute-defined-outside-init", diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 171d8479c..f44f70e40 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -8,6 +8,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import copy +import warnings from collections.abc import AsyncIterator, Awaitable, Callable from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast @@ -18,6 +19,8 @@ from scrapy.link import Link from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen +from scrapy.utils.deprecate import method_is_overridden +from scrapy.utils.python import global_object_name from scrapy.utils.spider import iterate_spider_output if TYPE_CHECKING: @@ -95,9 +98,17 @@ class CrawlSpider(Spider): def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) self._compile_rules() + if method_is_overridden(self.__class__, CrawlSpider, "_parse_response"): + warnings.warn( + f"The CrawlSpider._parse_response method, which the " + f"{global_object_name(self.__class__)} class overrides, is " + f"deprecated: it will be removed in future Scrapy releases. " + f"Please override the CrawlSpider.parse_with_rules method " + f"instead." + ) def _parse(self, response: Response, **kwargs: Any) -> Any: - return self._parse_response( + return self.parse_with_rules( response=response, callback=self.parse_start_url, cb_kwargs=kwargs, @@ -137,7 +148,7 @@ class CrawlSpider(Spider): def _callback(self, response: Response, **cb_kwargs: Any) -> Any: rule = self._rules[cast(int, response.meta["rule"])] - return self._parse_response( + return self.parse_with_rules( response, cast("CallbackT", rule.callback), {**rule.cb_kwargs, **cb_kwargs}, @@ -150,7 +161,7 @@ class CrawlSpider(Spider): failure, cast(Callable[[Failure], Any], rule.errback) ) - async def _parse_response( + async def parse_with_rules( self, response: Response, callback: CallbackT | None, @@ -171,6 +182,21 @@ class CrawlSpider(Spider): for request_or_item in self._requests_to_follow(response): yield request_or_item + def _parse_response( + self, + response: Response, + callback: CallbackT | None, + cb_kwargs: dict[str, Any], + follow: bool = True, + ) -> AsyncIterator[Any]: + warnings.warn( + "The CrawlSpider._parse_response method is deprecated: " + "it will be removed in future Scrapy releases. " + "Please use the CrawlSpider.parse_with_rules method instead.", + stacklevel=2, + ) + return self.parse_with_rules(response, callback, cb_kwargs, follow) + def _handle_failure( self, failure: Failure, errback: Callable[[Failure], Any] | None ) -> Iterable[Any]: diff --git a/tests/test_spider.py b/tests/test_spider.py index b4aa649a3..4e4a99638 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -476,6 +476,50 @@ class TestCrawlSpider(TestSpider): assert "Error while reading start items and requests" in str(log) assert "did you miss an 's'?" in str(log) + def test_parse_response_use(self): + class _CrawlSpider(CrawlSpider): + name = "test" + start_urls = "https://www.example.com" + _follow_links = False + + with warnings.catch_warnings(record=True) as w: + spider = _CrawlSpider() + assert len(w) == 0 + spider._parse_response( + TextResponse(spider.start_urls, body=b""), None, None + ) + assert len(w) == 1 + + def test_parse_response_override(self): + class _CrawlSpider(CrawlSpider): + def _parse_response(self, response, callback, cb_kwargs, follow=True): + pass + + name = "test" + start_urls = "https://www.example.com" + _follow_links = False + + with warnings.catch_warnings(record=True) as w: + assert len(w) == 0 + spider = _CrawlSpider() + assert len(w) == 1 + spider._parse_response( + TextResponse(spider.start_urls, body=b""), None, None + ) + assert len(w) == 1 + + def test_parse_with_rules(self): + class _CrawlSpider(CrawlSpider): + name = "test" + start_urls = "https://www.example.com" + + with warnings.catch_warnings(record=True) as w: + spider = _CrawlSpider() + spider.parse_with_rules( + TextResponse(spider.start_urls, body=b""), None, None + ) + assert len(w) == 0 + class TestSitemapSpider(TestSpider): spider_class = SitemapSpider From d400aa3e2d8c5e3d1a7d8a483fb1a5a6f1c66d50 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Jun 2025 14:19:15 +0500 Subject: [PATCH 142/491] Add _parallel_asyncio(). (#6852) --- scrapy/core/scraper.py | 30 ++++++++++-- scrapy/utils/asyncio.py | 54 +++++++++++++++++++++ tests/test_utils_asyncio.py | 93 ++++++++++++++++++++++++++++++++++++- tests/test_utils_defer.py | 49 +++++++++++++++++-- 4 files changed, 217 insertions(+), 9 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 9fc1d20ed..9fd68bce5 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -21,6 +21,7 @@ from scrapy.exceptions import ( ScrapyDeprecationWarning, ) from scrapy.http import Request, Response +from scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available from scrapy.utils.defer import ( _defer_sleep, aiter_errback, @@ -328,11 +329,21 @@ class Scraper: response: Response, ) -> None: """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" + it: Iterable[_T] | AsyncIterator[_T] + if is_asyncio_available(): + if isinstance(result, AsyncIterator): + it = aiter_errback(result, self.handle_spider_error, request, response) + else: + it = iter_errback(result, self.handle_spider_error, request, response) + await _parallel_asyncio( + it, self.concurrent_items, self._process_spidermw_output_async, response + ) + return if isinstance(result, AsyncIterator): - ait = aiter_errback(result, self.handle_spider_error, request, response) + it = aiter_errback(result, self.handle_spider_error, request, response) await maybe_deferred_to_future( parallel_async( - ait, + it, self.concurrent_items, self._process_spidermw_output, response, @@ -349,8 +360,19 @@ class Scraper: ) ) - @deferred_f_from_coro_f - async def _process_spidermw_output(self, output: Any, response: Response) -> None: + def _process_spidermw_output( + self, output: Any, response: Response + ) -> Deferred[None]: + """Process each Request/Item (given in the output parameter) returned + from the given spider. + + Items are sent to the item pipelines, requests are scheduled. + """ + return deferred_from_coro(self._process_spidermw_output_async(output, response)) + + async def _process_spidermw_output_async( + self, output: Any, response: Response + ) -> None: """Process each Request/Item (given in the output parameter) returned from the given spider. diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index 4469369fa..a3f27bcc9 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -1,7 +1,23 @@ """Utilities related to asyncio and its support in Scrapy.""" +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING, Any, TypeVar + +from scrapy.utils.asyncgen import as_async_generator from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable, Coroutine, Iterable + + # typing.Concatenate and typing.ParamSpec require Python 3.10 + from typing_extensions import Concatenate, ParamSpec + + _P = ParamSpec("_P") + +_T = TypeVar("_T") + def is_asyncio_available() -> bool: """Check if it's possible to call asyncio code that relies on the asyncio event loop. @@ -36,3 +52,41 @@ def is_asyncio_available() -> bool: ) return is_asyncio_reactor_installed() + + +async def _parallel_asyncio( + iterable: Iterable[_T] | AsyncIterator[_T], + count: int, + callable: Callable[Concatenate[_T, _P], Coroutine[Any, Any, None]], + *args: _P.args, + **kwargs: _P.kwargs, +) -> None: + """Execute a callable over the objects in the given iterable, in parallel, + using no more than ``count`` concurrent calls. + + This function is only used in + :meth:`scrapy.core.scraper.Scraper.handle_spider_output_async` and so it + assumes that neither *callable* nor iterating *iterable* will raise an + exception. + """ + queue: asyncio.Queue[_T | None] = asyncio.Queue() + + async def worker() -> None: + while True: + item = await queue.get() + if item is None: + break + try: + await callable(item, *args, **kwargs) + finally: + queue.task_done() + + async def fill_queue() -> None: + async for item in as_async_generator(iterable): + await queue.put(item) + for _ in range(count): + await queue.put(None) + + fill_task = asyncio.create_task(fill_queue()) + work_tasks = [asyncio.create_task(worker()) for _ in range(count)] + await asyncio.wait([fill_task, *work_tasks]) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index fe44748f9..6c47965a3 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,6 +1,18 @@ -import pytest +from __future__ import annotations -from scrapy.utils.asyncio import is_asyncio_available +import asyncio +import random +from typing import TYPE_CHECKING + +import pytest +from twisted.trial import unittest + +from scrapy.utils.asyncgen import as_async_generator +from scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available +from scrapy.utils.defer import deferred_f_from_coro_f + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator @pytest.mark.usefixtures("reactor_pytest") @@ -8,3 +20,80 @@ class TestAsyncio: def test_is_asyncio_available(self): # the result should depend only on the pytest --reactor argument assert is_asyncio_available() == (self.reactor_pytest != "default") + + +@pytest.mark.only_asyncio +class TestParallelAsyncio(unittest.TestCase): + """Test for scrapy.utils.asyncio.parallel_asyncio(), based on tests.test_utils_defer.TestParallelAsync.""" + + CONCURRENT_ITEMS = 50 + + @staticmethod + async def callable(o: int, results: list[int]) -> None: + if random.random() < 0.4: + # simulate async processing + await asyncio.sleep(random.random() / 8) + # simulate trivial sync processing + results.append(o) + + async def callable_wrapped( + self, + o: int, + results: list[int], + parallel_count: list[int], + max_parallel_count: list[int], + ) -> None: + parallel_count[0] += 1 + max_parallel_count[0] = max(max_parallel_count[0], parallel_count[0]) + await self.callable(o, results) + assert parallel_count[0] > 0, parallel_count[0] + parallel_count[0] -= 1 + + @staticmethod + def get_async_iterable(length: int) -> AsyncGenerator[int, None]: + # simulate a simple callback without delays between results + return as_async_generator(range(length)) + + @staticmethod + async def get_async_iterable_with_delays(length: int) -> AsyncGenerator[int, None]: + # simulate a callback with delays between some of the results + for i in range(length): + if random.random() < 0.1: + await asyncio.sleep(random.random() / 20) + yield i + + @deferred_f_from_coro_f + async def test_simple(self): + for length in [20, 50, 100]: + parallel_count = [0] + max_parallel_count = [0] + results = [] + ait = self.get_async_iterable(length) + await _parallel_asyncio( + ait, + self.CONCURRENT_ITEMS, + self.callable_wrapped, + results, + parallel_count, + max_parallel_count, + ) + assert list(range(length)) == sorted(results) + assert max_parallel_count[0] <= self.CONCURRENT_ITEMS + + @deferred_f_from_coro_f + async def test_delays(self): + for length in [20, 50, 100]: + parallel_count = [0] + max_parallel_count = [0] + results = [] + ait = self.get_async_iterable_with_delays(length) + await _parallel_asyncio( + ait, + self.CONCURRENT_ITEMS, + self.callable_wrapped, + results, + parallel_count, + max_parallel_count, + ) + assert list(range(length)) == sorted(results) + assert max_parallel_count[0] <= self.CONCURRENT_ITEMS diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index c565c1c4e..98962f74c 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -164,7 +164,7 @@ class TestAsyncDefTestsuite(unittest.TestCase): raise RuntimeError("This is expected to be raised") -class TestAsyncCooperator(unittest.TestCase): +class TestParallelAsync(unittest.TestCase): """This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage. parallel_async is called with the results of a callback (so an iterable of items, requests and None, @@ -194,6 +194,27 @@ class TestAsyncCooperator(unittest.TestCase): results.append(o) return None + def callable_wrapped( + self, + o: int, + results: list[int], + parallel_count: list[int], + max_parallel_count: list[int], + ) -> Deferred[None] | None: + parallel_count[0] += 1 + max_parallel_count[0] = max(max_parallel_count[0], parallel_count[0]) + dfd = self.callable(o, results) + + def decrement(_: Any = None) -> None: + assert parallel_count[0] > 0, parallel_count[0] + parallel_count[0] -= 1 + + if dfd is not None: + dfd.addBoth(decrement) + else: + decrement() + return dfd + @staticmethod def get_async_iterable(length: int) -> AsyncGenerator[int, None]: # simulate a simple callback without delays between results @@ -215,20 +236,42 @@ class TestAsyncCooperator(unittest.TestCase): @inlineCallbacks def test_simple(self): for length in [20, 50, 100]: + parallel_count = [0] + max_parallel_count = [0] results = [] ait = self.get_async_iterable(length) - dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results) + dl = parallel_async( + ait, + self.CONCURRENT_ITEMS, + self.callable_wrapped, + results, + parallel_count, + max_parallel_count, + ) yield dl assert list(range(length)) == sorted(results) + assert parallel_count[0] == 0 + assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0] @inlineCallbacks def test_delays(self): for length in [20, 50, 100]: + parallel_count = [0] + max_parallel_count = [0] results = [] ait = self.get_async_iterable_with_delays(length) - dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results) + dl = parallel_async( + ait, + self.CONCURRENT_ITEMS, + self.callable_wrapped, + results, + parallel_count, + max_parallel_count, + ) yield dl assert list(range(length)) == sorted(results) + assert parallel_count[0] == 0 + assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0] class TestDeferredFromCoro(unittest.TestCase): From 3aa5e757871c6288f0a1e7031ad74b48d6321f5c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Jun 2025 17:47:52 +0500 Subject: [PATCH 143/491] Use AsyncCrawlerProcess in commands. (#6845) * Use AsyncCrawlerProcess in commands. * Ignore coverage of abstract methods. * Address feedback. --- docs/news.rst | 12 ++ docs/topics/commands.rst | 38 +++++ docs/topics/settings.rst | 20 +++ pyproject.toml | 6 +- scrapy/cmdline.py | 10 +- scrapy/commands/__init__.py | 4 +- scrapy/commands/crawl.py | 20 +-- scrapy/commands/fetch.py | 2 - scrapy/commands/genspider.py | 1 - scrapy/commands/runspider.py | 1 - scrapy/commands/settings.py | 1 - scrapy/commands/shell.py | 1 - scrapy/commands/startproject.py | 1 - scrapy/crawler.py | 21 ++- scrapy/settings/default_settings.py | 2 + tests/test_command_crawl.py | 25 ++++ tests/test_command_runspider.py | 17 ++- tests/test_command_shell.py | 4 +- tests/test_commands.py | 216 ++++++++++++++++++++++++++++ tests/test_utils_defer.py | 6 +- 20 files changed, 370 insertions(+), 38 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index ef3b549e7..d3e6c6774 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,18 @@ Release notes ============= +Scrapy VERSION (unreleased) +--------------------------- + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- If you set the :setting:`TWISTED_REACTOR` setting to a :ref:`non-asyncio + value ` at the :ref:`spider level `, you + may now need to set the :setting:`FORCE_CRAWLER_PROCESS` setting to + ``True`` when running Scrapy via :ref:`its command-line tool + ` to avoid a reactor mismatch exception. + .. _release-2.13.1: Scrapy 2.13.1 (2025-05-28) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 6ffb8ae93..4994fe1d6 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -587,6 +587,44 @@ bench Run a quick benchmark test. :ref:`benchmarking`. +.. _topics-commands-crawlerprocess: + +Commands that run a crawl +========================= + +Many commands need to run a crawl of some kind, running either a user-provided +spider or a special internal one: + +* :command:`bench` +* :command:`check` +* :command:`crawl` +* :command:`fetch` +* :command:`parse` +* :command:`runspider` +* :command:`shell` +* :command:`view` + +They use an internal instance of :class:`scrapy.crawler.AsyncCrawlerProcess` or +:class:`scrapy.crawler.CrawlerProcess` for this. In most cases this detail +shouldn't matter to the user running the command, but when the user :ref:`needs +a non-default Twisted reactor `, it may be important. + +Scrapy decides which of these two classes to use based on the value of the +:setting:`TWISTED_REACTOR` setting. If the setting value is the default one +(``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``), +:class:`~scrapy.crawler.AsyncCrawlerProcess` will be used, otherwise +:class:`~scrapy.crawler.CrawlerProcess` will be used. The :ref:`spider settings +` are not taken into account when doing this, as they are +loaded after this decision is made. This may cause an error if the +project-level setting is set to :ref:`the asyncio reactor ` +(:ref:`explicitly ` or :ref:`by using the Scrapy default +`) and :ref:`the setting of the spider being run +` is set to :ref:`a different one `, because +:class:`~scrapy.crawler.AsyncCrawlerProcess` only supports the asyncio reactor. +In this case you should set the :setting:`FORCE_CRAWLER_PROCESS` setting to +``True`` (at the project level or via the command line) so that Scrapy uses +:class:`~scrapy.crawler.CrawlerProcess` which supports all reactors. + Custom project commands ======================= diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 65f2e5ebd..2a1be5f88 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1263,6 +1263,26 @@ FEED_STORAGE_GCS_ACL The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage `. For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation `_. +.. setting:: FORCE_CRAWLER_PROCESS + +FORCE_CRAWLER_PROCESS +--------------------- + +Default: ``False`` + +If ``False``, :ref:`Scrapy commands that need a CrawlerProcess +` will decide between using +:class:`scrapy.crawler.AsyncCrawlerProcess` and +:class:`scrapy.crawler.CrawlerProcess` based on the value of the +:setting:`TWISTED_REACTOR` setting, but ignoring its value in :ref:`per-spider +settings `. + +If ``True``, these commands will always use +:class:`~scrapy.crawler.CrawlerProcess`. + +Set this to ``True`` if you want to set :setting:`TWISTED_REACTOR` to a +non-default value in :ref:`per-spider settings `. + .. setting:: FTP_PASSIVE_MODE FTP_PASSIVE_MODE diff --git a/pyproject.toml b/pyproject.toml index 02ab7858d..8ec0c1056 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -147,8 +147,10 @@ source = [ ] [tool.coverage.report] -# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185 -exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"] +exclude_also = [ + "if TYPE_CHECKING:", + "@(abc\\.)?abstractmethod", +] [tool.pylint.MASTER] persistent = "no" diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 81e507a4e..3d448532b 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -10,11 +10,12 @@ from typing import TYPE_CHECKING import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter -from scrapy.crawler import CrawlerProcess +from scrapy.crawler import AsyncCrawlerProcess, CrawlerProcess from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect +from scrapy.utils.reactor import _asyncio_reactor_path if TYPE_CHECKING: from collections.abc import Callable, Iterable @@ -202,7 +203,12 @@ def execute(argv: list[str] | None = None, settings: Settings | None = None) -> _run_print_help(parser, cmd.process_options, args, opts) if cmd.requires_crawler_process: - cmd.crawler_process = CrawlerProcess(settings) + if settings[ + "TWISTED_REACTOR" + ] == _asyncio_reactor_path and not settings.getbool("FORCE_CRAWLER_PROCESS"): + cmd.crawler_process = AsyncCrawlerProcess(settings) + else: + cmd.crawler_process = CrawlerProcess(settings) _run_print_help(parser, _run_command, cmd, args, opts) sys.exit(cmd.exitcode) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 2818ead77..4ce070e6e 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -18,14 +18,14 @@ from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli if TYPE_CHECKING: from collections.abc import Iterable - from scrapy.crawler import Crawler, CrawlerProcess + from scrapy.crawler import Crawler, CrawlerProcessBase from scrapy.settings import Settings class ScrapyCommand: requires_project: bool = False requires_crawler_process: bool = True - crawler_process: CrawlerProcess | None = None # set in scrapy.cmdline + crawler_process: CrawlerProcessBase | None = None # set in scrapy.cmdline # default settings to be used for this command instead of global defaults default_settings: dict[str, Any] = {} diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 184bd5ca4..866ba9f6b 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,8 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, cast - -from twisted.python.failure import Failure +from typing import TYPE_CHECKING from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError @@ -30,17 +28,7 @@ class Command(BaseRunSpiderCommand): spname = args[0] assert self.crawler_process - crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) - - if getattr(crawl_defer, "result", None) is not None and issubclass( - cast(Failure, crawl_defer.result).type, Exception - ): + self.crawler_process.crawl(spname, **opts.spargs) + self.crawler_process.start() + if self.crawler_process.bootstrap_failed: self.exitcode = 1 - else: - self.crawler_process.start() - - if self.crawler_process.bootstrap_failed or ( - hasattr(self.crawler_process, "has_exception") - and self.crawler_process.has_exception - ): - self.exitcode = 1 diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index ef6e13de2..e5eedffb5 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -18,8 +18,6 @@ if TYPE_CHECKING: class Command(ScrapyCommand): - requires_project = False - def syntax(self) -> str: return "[options] " diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index c4abfc4c9..0e90c3188 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -46,7 +46,6 @@ def verify_url_scheme(url: str) -> str: class Command(ScrapyCommand): - requires_project = False requires_crawler_process = False default_settings = {"LOG_ENABLED": False} diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 3e826456e..eeb1303e2 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -30,7 +30,6 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType: class Command(BaseRunSpiderCommand): - requires_project = False default_settings = {"SPIDER_LOADER_CLASS": DummySpiderLoader} def syntax(self) -> str: diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index e63031f2d..704cc500d 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -6,7 +6,6 @@ from scrapy.settings import BaseSettings class Command(ScrapyCommand): - requires_project = False requires_crawler_process = False default_settings = {"LOG_ENABLED": False} diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 9dabfcd9c..eedaeb263 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -22,7 +22,6 @@ if TYPE_CHECKING: class Command(ScrapyCommand): - requires_project = False default_settings = { "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", "KEEP_ALIVE": True, diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 323979193..8f4427580 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -33,7 +33,6 @@ def _make_writable(path: Path) -> None: class Command(ScrapyCommand): - requires_project = False requires_crawler_process = False default_settings = {"LOG_ENABLED": False} diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 8e3223a5c..d6fb9972e 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -5,6 +5,7 @@ import contextlib import logging import pprint import signal +from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any, TypeVar from twisted.internet.defer import ( @@ -42,7 +43,7 @@ from scrapy.utils.reactor import ( ) if TYPE_CHECKING: - from collections.abc import Generator, Iterable + from collections.abc import Awaitable, Generator, Iterable from scrapy.logformatter import LogFormatter from scrapy.statscollectors import StatsCollector @@ -321,7 +322,7 @@ class Crawler: return self._get_component(cls, self.engine.scraper.spidermw.middlewares) -class CrawlerRunnerBase: +class CrawlerRunnerBase(ABC): def __init__(self, settings: dict[str, Any] | Settings | None = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -364,6 +365,15 @@ class CrawlerRunnerBase: spidercls = self.spider_loader.load(spidercls) return Crawler(spidercls, self.settings) + @abstractmethod + def crawl( + self, + crawler_or_spidercls: type[Spider] | str | Crawler, + *args: Any, + **kwargs: Any, + ) -> Awaitable[None]: + raise NotImplementedError + class CrawlerRunner(CrawlerRunnerBase): """ @@ -560,6 +570,12 @@ class CrawlerProcessBase(CrawlerRunnerBase): configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) + @abstractmethod + def start( + self, stop_after_crawl: bool = True, install_signal_handlers: bool = True + ) -> None: + raise NotImplementedError + def _signal_shutdown(self, signum: int, _: Any) -> None: from twisted.internet import reactor @@ -597,6 +613,7 @@ class CrawlerProcessBase(CrawlerRunnerBase): "after", "startup", install_shutdown_handlers, self._signal_shutdown ) + @abstractmethod def _stop_dfd(self) -> Deferred[Any]: raise NotImplementedError diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 01443fa17..4a27017a6 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -179,6 +179,8 @@ FEED_STORAGE_S3_ACL = "" FILES_STORE_S3_ACL = "private" FILES_STORE_GCS_ACL = "" +FORCE_CRAWLER_PROCESS = False + FTP_USER = "anonymous" FTP_PASSWORD = "guest" # noqa: S105 FTP_PASSIVE_MODE = True diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py index 3d5e17977..0ab0659b2 100644 --- a/tests/test_command_crawl.py +++ b/tests/test_command_crawl.py @@ -30,6 +30,11 @@ class MySpider(scrapy.Spider): """ log = self.get_log(spider_code) assert "[myspider] DEBUG: It works!" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Spider closed (finished)" in log def test_output(self): spider_code = """ @@ -91,3 +96,23 @@ class MySpider(scrapy.Spider): assert ( "error: Please use only one of -o/--output and -O/--overwrite-output" in log ) + + def test_default_reactor(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('It works!') + return + yield +""" + log = self.get_log(spider_code, args=("-s", "TWISTED_REACTOR=")) + assert "[myspider] DEBUG: It works!" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + assert "Spider closed (finished)" in log diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index 664de16f8..c57c09249 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -65,8 +65,10 @@ class BadSpider(scrapy.Spider): def test_runspider(self): log = self.get_log(self.debug_log_spider) assert "DEBUG: It Works!" in log - assert "INFO: Spider opened" in log - assert "INFO: Closing spider (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) assert "INFO: Spider closed (finished)" in log def test_run_fail_spider(self): @@ -88,6 +90,17 @@ class BadSpider(scrapy.Spider): assert "DEBUG: It Works!" not in log assert "INFO: Spider opened" in log + def test_runspider_default_reactor(self): + log = self.get_log(self.debug_log_spider, args=("-s", "TWISTED_REACTOR=")) + assert "DEBUG: It Works!" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + assert "INFO: Spider opened" in log + assert "INFO: Closing spider (finished)" in log + assert "INFO: Spider closed (finished)" in log + def test_runspider_dnscache_disabled(self): # see https://github.com/scrapy/scrapy/issues/2811 # The spider below should not be able to connect to localhost:12345, diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 0f45a7ee8..8041e7cb1 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -7,6 +7,7 @@ from pexpect.popen_spawn import PopenSpawn from twisted.internet import defer from twisted.trial import unittest +from scrapy.utils.reactor import _asyncio_reactor_path from tests import NON_EXISTING_RESOLVABLE, tests_datadir from tests.mockserver import MockServer from tests.utils.testproc import ProcessTest @@ -132,10 +133,9 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_shell_fetch_async(self): - reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" url = self.url("/html") code = f"fetch('{url}')" - args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"] + args = ["-c", code, "--set", f"TWISTED_REACTOR={_asyncio_reactor_path}"] _, _, err = yield self.execute(args, check_code=True) assert b"RuntimeError: There is no current event loop in thread" not in err diff --git a/tests/test_commands.py b/tests/test_commands.py index 6e59f561d..8ca5d51e5 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -20,6 +20,7 @@ from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view from scrapy.settings import Settings from scrapy.utils.python import to_unicode +from scrapy.utils.reactor import _asyncio_reactor_path from scrapy.utils.test import get_testenv if TYPE_CHECKING: @@ -124,6 +125,221 @@ class TestCommandBase(TestProjectBase): self.env["SCRAPY_SETTINGS_MODULE"] = f"{self.project_name}.settings" +class TestCommandCrawlerProcess(TestCommandBase): + """Test that the command uses the expected kind of *CrawlerProcess + and produces expected errors when needed.""" + + name = "crawltest" + + NORMAL_MSG = "Type of self.crawler_process: " + ASYNC_MSG = ( + "Type of self.crawler_process: " + ) + + def setUp(self): + super().setUp() + (self.cwd / self.project_name / "commands").mkdir(exist_ok=True) + (self.cwd / self.project_name / "commands" / "__init__.py").touch() + (self.cwd / self.project_name / "commands" / f"{self.name}.py").write_text(""" +from scrapy.commands.crawl import Command + +class CrawlerProcessCrawlCommand(Command): + requires_project = True + + def run(self, args, opts): + print(f"Type of self.crawler_process: {type(self.crawler_process)}") + super().run(args, opts) +""") + + self._append_settings(f"COMMANDS_MODULE = '{self.project_name}.commands'\n") + + (self.cwd / self.project_name / "spiders" / "sp.py").write_text(""" +import scrapy + +class MySpider(scrapy.Spider): + name = 'sp' + + custom_settings = {} + + async def start(self): + self.logger.debug('It works!') + return + yield +""") + + (self.cwd / self.project_name / "spiders" / "aiosp.py").write_text(""" +import asyncio + +import scrapy + +class MySpider(scrapy.Spider): + name = 'aiosp' + + custom_settings = {} + + async def start(self): + await asyncio.sleep(0.01) + self.logger.debug('It works!') + return + yield +""") + + def _append_settings(self, text: str) -> None: + """Add text to the end of the project settings.py.""" + with (self.cwd / self.project_name / "settings.py").open( + "a", encoding="utf-8" + ) as f: + f.write(text) + + def _replace_custom_settings(self, spider_name: str, text: str) -> None: + """Replace custom_settings in the given spider file with the given text.""" + spider_path = self.cwd / self.project_name / "spiders" / f"{spider_name}.py" + with spider_path.open("r+", encoding="utf-8") as f: + content = f.read() + content = content.replace( + "custom_settings = {}", f"custom_settings = {text}" + ) + f.seek(0) + f.write(content) + f.truncate() + + def _assert_spider_works(self, msg: str, *args: str) -> None: + """The command uses the expected *CrawlerProcess, the spider works.""" + _, out, err = self.proc(self.name, *args) + assert msg in out, out + assert "It works!" in err, err + assert "Spider closed (finished)" in err, err + + def _assert_spider_asyncio_fail(self, msg: str, *args: str) -> None: + """The command uses the expected *CrawlerProcess, the spider fails to use asyncio.""" + _, out, err = self.proc(self.name, *args) + assert msg in out, out + assert "no running event loop" in err, err + + def test_project_settings(self): + """The reactor is set via the project default settings (to the asyncio value). + + AsyncCrawlerProcess, the asyncio reactor, both spiders work.""" + for spider in ["sp", "aiosp"]: + self._assert_spider_works(self.ASYNC_MSG, spider) + + def test_cmdline_asyncio(self): + """The reactor is set via the command line to the asyncio value. + AsyncCrawlerProcess, the asyncio reactor, both spiders work.""" + for spider in ["sp", "aiosp"]: + self._assert_spider_works( + self.ASYNC_MSG, spider, "-s", f"TWISTED_REACTOR={_asyncio_reactor_path}" + ) + + def test_project_settings_explicit_asyncio(self): + """The reactor explicitly is set via the project settings to the asyncio value. + + AsyncCrawlerProcess, the asyncio reactor, both spiders work.""" + self._append_settings(f"TWISTED_REACTOR = '{_asyncio_reactor_path}'\n") + + for spider in ["sp", "aiosp"]: + self._assert_spider_works(self.ASYNC_MSG, spider) + + def test_cmdline_empty(self): + """The reactor is set via the command line to the empty value. + + CrawlerProcess, the default reactor, only the normal spider works.""" + self._assert_spider_works(self.NORMAL_MSG, "sp", "-s", "TWISTED_REACTOR=") + self._assert_spider_asyncio_fail( + self.NORMAL_MSG, "aiosp", "-s", "TWISTED_REACTOR=" + ) + + def test_project_settings_empty(self): + """The reactor is set via the project settings to the empty value. + + CrawlerProcess, the default reactor, only the normal spider works.""" + self._append_settings("TWISTED_REACTOR = None\n") + + self._assert_spider_works(self.NORMAL_MSG, "sp") + self._assert_spider_asyncio_fail( + self.NORMAL_MSG, "aiosp", "-s", "TWISTED_REACTOR=" + ) + + def test_spider_settings_asyncio(self): + """The reactor is set via the spider settings to the asyncio value. + + AsyncCrawlerProcess, the asyncio reactor, both spiders work.""" + for spider in ["sp", "aiosp"]: + self._replace_custom_settings( + spider, f"{{'TWISTED_REACTOR': '{_asyncio_reactor_path}'}}" + ) + self._assert_spider_works(self.ASYNC_MSG, spider) + + def test_spider_settings_asyncio_cmdline_empty(self): + """The reactor is set via the spider settings to the asyncio value + and via command line to the empty value. The command line value takes + precedence so the spider settings don't matter. + + CrawlerProcess, the default reactor, only the normal spider works.""" + for spider in ["sp", "aiosp"]: + self._replace_custom_settings( + spider, f"{{'TWISTED_REACTOR': '{_asyncio_reactor_path}'}}" + ) + + self._assert_spider_works(self.NORMAL_MSG, "sp", "-s", "TWISTED_REACTOR=") + self._assert_spider_asyncio_fail( + self.NORMAL_MSG, "aiosp", "-s", "TWISTED_REACTOR=" + ) + + def test_project_empty_spider_settings_asyncio(self): + """The reactor is set via the project settings to the empty value + and via the spider settings to the asyncio value. CrawlerProcess is + chosen based on the project settings, but the asyncio reactor is chosen + based on the spider settings. + + CrawlerProcess, the asyncio reactor, both spiders work.""" + self._append_settings("TWISTED_REACTOR = None\n") + for spider in ["sp", "aiosp"]: + self._replace_custom_settings( + spider, f"{{'TWISTED_REACTOR': '{_asyncio_reactor_path}'}}" + ) + self._assert_spider_works(self.NORMAL_MSG, spider) + + def test_project_asyncio_spider_settings_select(self): + """The reactor is set via the project settings to the asyncio value + and via the spider settings to the select value. AsyncCrawlerProcess + is chosen based on the project settings, and the conflicting reactor + setting in the spider settings causes an exception. + + AsyncCrawlerProcess, the asyncio reactor, both spiders produce a + mismatched reactor exception.""" + self._append_settings(f"TWISTED_REACTOR = '{_asyncio_reactor_path}'\n") + for spider in ["sp", "aiosp"]: + self._replace_custom_settings( + spider, + "{'TWISTED_REACTOR': 'twisted.internet.selectreactor.SelectReactor'}", + ) + _, out, err = self.proc(self.name, spider) + assert self.ASYNC_MSG in out, out + assert ( + "The installed reactor (twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + " does not match the requested one" + " (twisted.internet.selectreactor.SelectReactor)" + ) in err, err + + def test_project_asyncio_spider_settings_select_forced(self): + """The reactor is set via the project settings to the asyncio value + and via the spider settings to the select value, CrawlerProcess is + forced via the project settings. The reactor is chosen based on the + spider settings. + + CrawlerProcess, the select reactor, only the normal spider works.""" + self._append_settings("FORCE_CRAWLER_PROCESS = True\n") + for spider in ["sp", "aiosp"]: + self._replace_custom_settings( + spider, + "{'TWISTED_REACTOR': 'twisted.internet.selectreactor.SelectReactor'}", + ) + + self._assert_spider_works(self.NORMAL_MSG, "sp") + self._assert_spider_asyncio_fail(self.NORMAL_MSG, "aiosp") + + class TestMiscCommands(TestCommandBase): def test_list(self): assert self.call("list") == 0 diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 98962f74c..372213319 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -299,7 +299,7 @@ class TestDeferredFromCoro(unittest.TestCase): @inlineCallbacks def test_coroutine_asyncio(self): async def coroutine() -> int: - await asyncio.sleep(0) + await asyncio.sleep(0.01) return 42 result = deferred_from_coro(coroutine()) @@ -379,7 +379,7 @@ class TestDeferredToFuture(unittest.TestCase): @deferred_f_from_coro_f async def test_wrapped_coroutine_asyncio(self): async def c_f() -> int: - await asyncio.sleep(0) + await asyncio.sleep(0.01) return 42 d = deferred_from_coro(c_f()) @@ -414,7 +414,7 @@ class TestMaybeDeferredToFutureAsyncio(unittest.TestCase): @deferred_f_from_coro_f async def test_wrapped_coroutine_asyncio(self): async def c_f() -> int: - await asyncio.sleep(0) + await asyncio.sleep(0.01) return 42 d = deferred_from_coro(c_f()) From 8fb8d2c6b8a83181ef6564fbc9afa9cfeea2bd05 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Jun 2025 23:21:33 +0500 Subject: [PATCH 144/491] Add AsyncioLoopingCall. (#6855) --- scrapy/core/downloader/__init__.py | 8 ++- scrapy/core/engine.py | 11 ++- scrapy/extensions/closespider.py | 5 +- scrapy/extensions/logstats.py | 12 ++-- scrapy/extensions/memusage.py | 16 +++-- scrapy/extensions/periodic_log.py | 14 ++-- scrapy/utils/asyncio.py | 106 ++++++++++++++++++++++++++++- tests/test_utils_asyncio.py | 48 ++++++++++++- 8 files changed, 195 insertions(+), 25 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 5468398aa..501c669ce 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -7,7 +7,6 @@ from datetime import datetime from time import time from typing import TYPE_CHECKING, Any, cast -from twisted.internet import task from twisted.internet.defer import Deferred, inlineCallbacks from scrapy import Request, Spider, signals @@ -15,6 +14,7 @@ from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.resolver import dnscache +from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call from scrapy.utils.defer import ( deferred_from_coro, maybe_deferred_to_future, @@ -25,6 +25,8 @@ from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: from collections.abc import Generator + from twisted.internet.task import LoopingCall + from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.settings import BaseSettings @@ -111,7 +113,9 @@ class Downloader: self.middleware: DownloaderMiddlewareManager = ( DownloaderMiddlewareManager.from_crawler(crawler) ) - self._slot_gc_loop: task.LoopingCall = task.LoopingCall(self._slot_gc) + self._slot_gc_loop: AsyncioLoopingCall | LoopingCall = create_looping_call( + self._slot_gc + ) self._slot_gc_loop.start(60) self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict( "DOWNLOAD_SLOTS", {} diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 721c81d81..d9361a674 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -13,13 +13,16 @@ from traceback import format_exc from typing import TYPE_CHECKING, Any, cast from twisted.internet.defer import Deferred, inlineCallbacks, succeed -from twisted.internet.task import LoopingCall from twisted.python.failure import Failure from scrapy import signals from scrapy.core.scraper import Scraper from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + create_looping_call, +) from scrapy.utils.defer import ( deferred_f_from_coro_f, deferred_from_coro, @@ -32,6 +35,8 @@ from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: from collections.abc import AsyncIterator, Callable, Generator + from twisted.internet.task import LoopingCall + from scrapy.core.downloader import Downloader from scrapy.core.scheduler import BaseScheduler from scrapy.crawler import Crawler @@ -56,7 +61,9 @@ class _Slot: self.close_if_idle: bool = close_if_idle self.nextcall: CallLaterOnce[None] = nextcall self.scheduler: BaseScheduler = scheduler - self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule) + self.heartbeat: AsyncioLoopingCall | LoopingCall = create_looping_call( + nextcall.schedule + ) def add_request(self, request: Request) -> None: self.inprogress.add(request) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index dff8bc97e..a649a86e2 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured +from scrapy.utils.asyncio import create_looping_call if TYPE_CHECKING: from twisted.python.failure import Failure @@ -118,9 +119,7 @@ class CloseSpider: task_no_item.stop() def spider_opened_no_item(self, spider: Spider) -> None: - from twisted.internet import task - - self.task_no_item = task.LoopingCall(self._count_items_produced, spider) + self.task_no_item = create_looping_call(self._count_items_produced, spider) self.task_no_item.start(self.timeout_no_item, now=False) logger.info( diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index f2e1f57b8..387cfddb3 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -3,12 +3,16 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING -from twisted.internet import task - from scrapy import Spider, signals from scrapy.exceptions import NotConfigured +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + create_looping_call, +) if TYPE_CHECKING: + from twisted.internet.task import LoopingCall + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -29,7 +33,7 @@ class LogStats: self.stats: StatsCollector = stats self.interval: float = interval self.multiplier: float = 60.0 / self.interval - self.task: task.LoopingCall | None = None + self.task: AsyncioLoopingCall | LoopingCall | None = None @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -46,7 +50,7 @@ class LogStats: self.pagesprev: int = 0 self.itemsprev: int = 0 - self.task = task.LoopingCall(self.log, spider) + self.task = create_looping_call(self.log, spider) self.task.start(self.interval) def log(self, spider: Spider) -> None: diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index d7f810107..2ef322f1a 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -13,14 +13,18 @@ from importlib import import_module from pprint import pformat from typing import TYPE_CHECKING -from twisted.internet import task - from scrapy import signals from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + create_looping_call, +) from scrapy.utils.engine import get_engine_status if TYPE_CHECKING: + from twisted.internet.task import LoopingCall + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -66,16 +70,16 @@ class MemoryUsage: def engine_started(self) -> None: assert self.crawler.stats self.crawler.stats.set_value("memusage/startup", self.get_virtual_size()) - self.tasks: list[task.LoopingCall] = [] - tsk = task.LoopingCall(self.update) + self.tasks: list[AsyncioLoopingCall | LoopingCall] = [] + tsk = create_looping_call(self.update) self.tasks.append(tsk) tsk.start(self.check_interval, now=True) if self.limit: - tsk = task.LoopingCall(self._check_limit) + tsk = create_looping_call(self._check_limit) self.tasks.append(tsk) tsk.start(self.check_interval, now=True) if self.warning: - tsk = task.LoopingCall(self._check_warning) + tsk = create_looping_call(self._check_warning) self.tasks.append(tsk) tsk.start(self.check_interval, now=True) diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index f97577442..9158482fa 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -4,16 +4,20 @@ import logging from datetime import datetime, timezone from typing import TYPE_CHECKING, Any -from twisted.internet import task - from scrapy import Spider, signals from scrapy.exceptions import NotConfigured +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + create_looping_call, +) from scrapy.utils.serialize import ScrapyJSONEncoder if TYPE_CHECKING: - # typing.Self requires Python 3.11 from json import JSONEncoder + from twisted.internet.task import LoopingCall + + # typing.Self requires Python 3.11 from typing_extensions import Self from scrapy.crawler import Crawler @@ -37,7 +41,7 @@ class PeriodicLog: self.stats: StatsCollector = stats self.interval: float = interval self.multiplier: float = 60.0 / self.interval - self.task: task.LoopingCall | None = None + self.task: AsyncioLoopingCall | LoopingCall | None = None self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) self.ext_stats_enabled: bool = bool(ext_stats) self.ext_stats_include: list[str] = ext_stats.get("include", []) @@ -97,7 +101,7 @@ class PeriodicLog: self.delta_prev: dict[str, int | float] = {} self.stats_prev: dict[str, int | float] = {} - self.task = task.LoopingCall(self.log) + self.task = create_looping_call(self.log) self.task.start(self.interval) def log(self) -> None: diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index a3f27bcc9..cae2dc033 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -3,22 +3,30 @@ from __future__ import annotations import asyncio +import logging +import time +from collections.abc import AsyncIterator, Callable, Coroutine, Iterable from typing import TYPE_CHECKING, Any, TypeVar +from twisted.internet.defer import Deferred +from twisted.internet.task import LoopingCall + from scrapy.utils.asyncgen import as_async_generator from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed if TYPE_CHECKING: - from collections.abc import AsyncIterator, Callable, Coroutine, Iterable - # typing.Concatenate and typing.ParamSpec require Python 3.10 from typing_extensions import Concatenate, ParamSpec _P = ParamSpec("_P") + _T = TypeVar("_T") +logger = logging.getLogger(__name__) + + def is_asyncio_available() -> bool: """Check if it's possible to call asyncio code that relies on the asyncio event loop. @@ -90,3 +98,97 @@ async def _parallel_asyncio( fill_task = asyncio.create_task(fill_queue()) work_tasks = [asyncio.create_task(worker()) for _ in range(count)] await asyncio.wait([fill_task, *work_tasks]) + + +class AsyncioLoopingCall: + """A simple implementation of a periodic call using asyncio, keeping + some API and behavior compatibility with the Twisted ``LoopingCall``. + + The function is called every *interval* seconds, independent of the finish + time of the previous call. If the function is still running when it's time + to call it again, calls are skipped until the function finishes. + + The function must not return a coroutine or a ``Deferred``. + """ + + def __init__(self, func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs): + self._func: Callable[_P, _T] = func + self._args: tuple[Any, ...] = args + self._kwargs: dict[str, Any] = kwargs + self._task: asyncio.Task | None = None + self.interval: float | None = None + self._start_time: float | None = None + + @property + def running(self) -> bool: + return self._start_time is not None + + def start(self, interval: float, now: bool = True) -> None: + """Start calling the function every *interval* seconds. + + :param interval: The interval in seconds between calls. + :type interval: float + + :param now: If ``True``, also call the function immediately. + :type now: bool + """ + if self.running: + raise RuntimeError("AsyncioLoopingCall already running") + + if interval <= 0: + raise ValueError("Interval must be greater than 0") + + self.interval = interval + self._start_time = time.time() + if now: + self._call() + loop = asyncio.get_event_loop() + self._task = loop.create_task(self._loop()) + + def _to_sleep(self) -> float: + """Return the time to sleep until the next call.""" + assert self.interval is not None + assert self._start_time is not None + now = time.time() + running_for = now - self._start_time + return self.interval - (running_for % self.interval) + + async def _loop(self) -> None: + """Run an infinite loop that calls the function periodically.""" + while self.running: + await asyncio.sleep(self._to_sleep()) + self._call() + + def stop(self) -> None: + """Stop the periodic calls.""" + self.interval = self._start_time = None + if self._task is not None: + self._task.cancel() + self._task = None + + def _call(self) -> None: + """Execute the function.""" + try: + result = self._func(*self._args, **self._kwargs) + except Exception: + logger.exception("Error calling the AsyncioLoopingCall function") + self.stop() + else: + if isinstance(result, (Coroutine, Deferred)): + self.stop() + raise TypeError( + "The AsyncioLoopingCall function must not return a coroutine or a Deferred" + ) + + +def create_looping_call( + func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs +) -> AsyncioLoopingCall | LoopingCall: + """Create an instance of a looping call class. + + This creates an instance of :class:`AsyncioLoopingCall` or + :class:`LoopingCall`, depending on whether asyncio support is available. + """ + if is_asyncio_available(): + return AsyncioLoopingCall(func, *args, **kwargs) + return LoopingCall(func, *args, **kwargs) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 6c47965a3..a6e52eb26 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -3,12 +3,18 @@ from __future__ import annotations import asyncio import random from typing import TYPE_CHECKING +from unittest import mock import pytest +from twisted.internet.defer import Deferred from twisted.trial import unittest from scrapy.utils.asyncgen import as_async_generator -from scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + _parallel_asyncio, + is_asyncio_available, +) from scrapy.utils.defer import deferred_f_from_coro_f if TYPE_CHECKING: @@ -97,3 +103,43 @@ class TestParallelAsyncio(unittest.TestCase): ) assert list(range(length)) == sorted(results) assert max_parallel_count[0] <= self.CONCURRENT_ITEMS + + +@pytest.mark.only_asyncio +class TestAsyncioLoopingCall: + def test_looping_call(self): + func = mock.MagicMock() + looping_call = AsyncioLoopingCall(func) + looping_call.start(1, now=False) + assert looping_call.running + looping_call.stop() + assert not looping_call.running + assert not func.called + + def test_looping_call_now(self): + func = mock.MagicMock() + looping_call = AsyncioLoopingCall(func) + looping_call.start(1) + looping_call.stop() + assert func.called + + def test_looping_call_already_running(self): + looping_call = AsyncioLoopingCall(lambda: None) + looping_call.start(1) + with pytest.raises(RuntimeError): + looping_call.start(1) + looping_call.stop() + + def test_looping_call_interval(self): + looping_call = AsyncioLoopingCall(lambda: None) + with pytest.raises(ValueError, match="Interval must be greater than 0"): + looping_call.start(0) + with pytest.raises(ValueError, match="Interval must be greater than 0"): + looping_call.start(-1) + assert not looping_call.running + + def test_looping_call_bad_function(self): + looping_call = AsyncioLoopingCall(Deferred) + with pytest.raises(TypeError): + looping_call.start(0.1) + assert not looping_call.running From c6698b9fe8d3e1f27f2982b484ba69486a8cbc3d Mon Sep 17 00:00:00 2001 From: Mehraz Hossain Rumman <59512321+MehrazRumman@users.noreply.github.com> Date: Wed, 4 Jun 2025 17:02:29 +0600 Subject: [PATCH 145/491] fixing settings order (#6849) * fixing issue #6838 * Reorder some more settings. * Clarify the header. --------- Co-authored-by: Andrey Rakhmatullin --- scrapy/settings/default_settings.py | 146 ++++++++++++++-------------- 1 file changed, 71 insertions(+), 75 deletions(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 4a27017a6..7cd470f11 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -1,16 +1,16 @@ -""" -This module contains the default values for all settings used by Scrapy. +"""This module contains the default values for all settings used by Scrapy. For more information about these settings you can read the settings documentation in docs/topics/settings.rst Scrapy developers, if you add a setting here remember to: -* add it in alphabetical order +* add it in alphabetical order, with the exception that enabling flags and + other high-level settings for a group should come first in their group + and pairs like host/port and user/password should be in the usual order * group similar settings without leaving blank lines * add its documentation to the available settings documentation (docs/topics/settings.rst) - """ import sys @@ -31,10 +31,10 @@ AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 BOT_NAME = "scrapybot" -CLOSESPIDER_TIMEOUT = 0 -CLOSESPIDER_PAGECOUNT = 0 -CLOSESPIDER_ITEMCOUNT = 0 CLOSESPIDER_ERRORCOUNT = 0 +CLOSESPIDER_ITEMCOUNT = 0 +CLOSESPIDER_PAGECOUNT = 0 +CLOSESPIDER_TIMEOUT = 0 COMMANDS_MODULE = "" @@ -59,8 +59,8 @@ DEFAULT_REQUEST_HEADERS = { } DEPTH_LIMIT = 0 -DEPTH_STATS_VERBOSE = False DEPTH_PRIORITY = 0 +DEPTH_STATS_VERBOSE = False DNSCACHE_ENABLED = True DNSCACHE_SIZE = 10000 @@ -69,6 +69,8 @@ DNS_TIMEOUT = 60 DOWNLOAD_DELAY = 0 +DOWNLOAD_FAIL_ON_DATALOSS = True + DOWNLOAD_HANDLERS = {} DOWNLOAD_HANDLERS_BASE = { "data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler", @@ -79,18 +81,13 @@ DOWNLOAD_HANDLERS_BASE = { "ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler", } -DOWNLOAD_TIMEOUT = 180 # 3mins - DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m -DOWNLOAD_FAIL_ON_DATALOSS = True +DOWNLOAD_TIMEOUT = 180 # 3mins DOWNLOADER = "scrapy.core.downloader.Downloader" -DOWNLOADER_HTTPCLIENTFACTORY = ( - "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory" -) DOWNLOADER_CLIENTCONTEXTFACTORY = ( "scrapy.core.downloader.contextfactory.ScrapyClientContextFactory" ) @@ -99,8 +96,11 @@ DOWNLOADER_CLIENT_TLS_CIPHERS = "DEFAULT" DOWNLOADER_CLIENT_TLS_METHOD = "TLS" DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False -DOWNLOADER_MIDDLEWARES = {} +DOWNLOADER_HTTPCLIENTFACTORY = ( + "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory" +) +DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50, @@ -130,7 +130,6 @@ if sys.platform == "win32": EDITOR = "%s -m idlelib.idle" EXTENSIONS = {} - EXTENSIONS_BASE = { "scrapy.extensions.corestats.CoreStats": 0, "scrapy.extensions.telnet.TelnetConsole": 0, @@ -143,22 +142,11 @@ EXTENSIONS_BASE = { "scrapy.extensions.throttle.AutoThrottle": 0, } -FEED_TEMPDIR = None FEEDS = {} -FEED_URI_PARAMS = None # a function to extend uri arguments -FEED_STORE_EMPTY = True +FEED_EXPORT_BATCH_ITEM_COUNT = 0 FEED_EXPORT_ENCODING = None FEED_EXPORT_FIELDS = None -FEED_STORAGES = {} -FEED_STORAGES_BASE = { - "": "scrapy.extensions.feedexport.FileFeedStorage", - "file": "scrapy.extensions.feedexport.FileFeedStorage", - "ftp": "scrapy.extensions.feedexport.FTPFeedStorage", - "gs": "scrapy.extensions.feedexport.GCSFeedStorage", - "s3": "scrapy.extensions.feedexport.S3FeedStorage", - "stdout": "scrapy.extensions.feedexport.StdoutFeedStorage", -} -FEED_EXPORT_BATCH_ITEM_COUNT = 0 +FEED_EXPORT_INDENT = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { "json": "scrapy.exporters.JsonItemExporter", @@ -170,59 +158,69 @@ FEED_EXPORTERS_BASE = { "marshal": "scrapy.exporters.MarshalItemExporter", "pickle": "scrapy.exporters.PickleItemExporter", } -FEED_EXPORT_INDENT = 0 - +FEED_STORE_EMPTY = True +FEED_STORAGES = {} +FEED_STORAGES_BASE = { + "": "scrapy.extensions.feedexport.FileFeedStorage", + "file": "scrapy.extensions.feedexport.FileFeedStorage", + "ftp": "scrapy.extensions.feedexport.FTPFeedStorage", + "gs": "scrapy.extensions.feedexport.GCSFeedStorage", + "s3": "scrapy.extensions.feedexport.S3FeedStorage", + "stdout": "scrapy.extensions.feedexport.StdoutFeedStorage", +} FEED_STORAGE_FTP_ACTIVE = False FEED_STORAGE_GCS_ACL = "" FEED_STORAGE_S3_ACL = "" +FEED_TEMPDIR = None +FEED_URI_PARAMS = None # a function to extend uri arguments -FILES_STORE_S3_ACL = "private" FILES_STORE_GCS_ACL = "" +FILES_STORE_S3_ACL = "private" FORCE_CRAWLER_PROCESS = False +FTP_PASSIVE_MODE = True FTP_USER = "anonymous" FTP_PASSWORD = "guest" # noqa: S105 -FTP_PASSIVE_MODE = True GCS_PROJECT_ID = None HTTPCACHE_ENABLED = False -HTTPCACHE_DIR = "httpcache" -HTTPCACHE_IGNORE_MISSING = False -HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" -HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_ALWAYS_STORE = False -HTTPCACHE_IGNORE_HTTP_CODES = [] -HTTPCACHE_IGNORE_SCHEMES = ["file"] -HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = [] HTTPCACHE_DBM_MODULE = "dbm" -HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy" +HTTPCACHE_DIR = "httpcache" +HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_GZIP = False +HTTPCACHE_IGNORE_HTTP_CODES = [] +HTTPCACHE_IGNORE_MISSING = False +HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = [] +HTTPCACHE_IGNORE_SCHEMES = ["file"] +HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy" +HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = "latin-1" -IMAGES_STORE_S3_ACL = "private" IMAGES_STORE_GCS_ACL = "" - -ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager" +IMAGES_STORE_S3_ACL = "private" ITEM_PIPELINES = {} ITEM_PIPELINES_BASE = {} +ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager" + JOBDIR = None LOG_ENABLED = True -LOG_ENCODING = "utf-8" -LOG_FORMATTER = "scrapy.logformatter.LogFormatter" -LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s" LOG_DATEFORMAT = "%Y-%m-%d %H:%M:%S" -LOG_STDOUT = False -LOG_LEVEL = "DEBUG" +LOG_ENCODING = "utf-8" LOG_FILE = None LOG_FILE_APPEND = True +LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s" +LOG_FORMATTER = "scrapy.logformatter.LogFormatter" +LOG_LEVEL = "DEBUG" LOG_SHORT_NAMES = False +LOG_STDOUT = False LOG_VERSIONS = [ "lxml", "libxml2", @@ -236,21 +234,19 @@ LOG_VERSIONS = [ "Platform", ] -SCHEDULER_DEBUG = False - LOGSTATS_INTERVAL = 60.0 +MAIL_FROM = "scrapy@localhost" MAIL_HOST = "localhost" MAIL_PORT = 25 -MAIL_FROM = "scrapy@localhost" -MAIL_PASS = None MAIL_USER = None +MAIL_PASS = None MEMDEBUG_ENABLED = False # enable memory debugging MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown -MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0 MEMUSAGE_ENABLED = True +MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0 MEMUSAGE_LIMIT_MB = 0 MEMUSAGE_NOTIFY_MAIL = [] MEMUSAGE_WARNING_MB = 0 @@ -280,9 +276,6 @@ REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter" REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL" RETRY_ENABLED = True -RETRY_TIMES = 2 # initial response + 2 retries = 3 requests -RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] -RETRY_PRIORITY_ADJUST = -1 RETRY_EXCEPTIONS = [ "twisted.internet.defer.TimeoutError", "twisted.internet.error.TimeoutError", @@ -298,12 +291,16 @@ RETRY_EXCEPTIONS = [ OSError, "scrapy.core.downloader.handlers.http11.TunnelError", ] +RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] +RETRY_PRIORITY_ADJUST = -1 +RETRY_TIMES = 2 # initial response + 2 retries = 3 requests ROBOTSTXT_OBEY = False ROBOTSTXT_PARSER = "scrapy.robotstxt.ProtegoRobotParser" ROBOTSTXT_USER_AGENT = None SCHEDULER = "scrapy.core.scheduler.Scheduler" +SCHEDULER_DEBUG = False SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleLifoDiskQueue" SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.LifoMemoryQueue" SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue" @@ -312,11 +309,19 @@ SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000 +SPIDER_CONTRACTS = {} +SPIDER_CONTRACTS_BASE = { + "scrapy.contracts.default.UrlContract": 1, + "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1, + "scrapy.contracts.default.MetadataContract": 1, + "scrapy.contracts.default.ReturnsContract": 2, + "scrapy.contracts.default.ScrapesContract": 3, +} + SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader" SPIDER_LOADER_WARN_ONLY = False SPIDER_MIDDLEWARES = {} - SPIDER_MIDDLEWARES_BASE = { # Engine side "scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25, @@ -334,27 +339,18 @@ STATS_DUMP = True STATSMAILER_RCPTS = [] +TELNETCONSOLE_ENABLED = 1 +TELNETCONSOLE_HOST = "127.0.0.1" +TELNETCONSOLE_PORT = [6023, 6073] +TELNETCONSOLE_USERNAME = "scrapy" +TELNETCONSOLE_PASSWORD = None + TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve()) +TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" + URLLENGTH_LIMIT = 2083 USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)" -TELNETCONSOLE_ENABLED = 1 -TELNETCONSOLE_PORT = [6023, 6073] -TELNETCONSOLE_HOST = "127.0.0.1" -TELNETCONSOLE_USERNAME = "scrapy" -TELNETCONSOLE_PASSWORD = None - -TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" - -SPIDER_CONTRACTS = {} -SPIDER_CONTRACTS_BASE = { - "scrapy.contracts.default.UrlContract": 1, - "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1, - "scrapy.contracts.default.MetadataContract": 1, - "scrapy.contracts.default.ReturnsContract": 2, - "scrapy.contracts.default.ScrapesContract": 3, -} - WARN_ON_GENERATOR_RETURN_VALUE = True From 5902aab25ce2ef0b26e158e0455ee6f0846636bb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 4 Jun 2025 20:37:36 +0500 Subject: [PATCH 146/491] Add the call_later() wrapper. (#6858) --- scrapy/core/downloader/__init__.py | 25 +++++++----- scrapy/extensions/closespider.py | 33 ++++++++++------ scrapy/utils/asyncio.py | 63 +++++++++++++++++++++++++++++- scrapy/utils/reactor.py | 13 +++--- 4 files changed, 107 insertions(+), 27 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 501c669ce..9293d7b78 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -14,7 +14,12 @@ from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.resolver import dnscache -from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + CallLaterResult, + call_later, + create_looping_call, +) from scrapy.utils.defer import ( deferred_from_coro, maybe_deferred_to_future, @@ -50,7 +55,7 @@ class Slot: self.queue: deque[tuple[Request, Deferred[Response]]] = deque() self.transferring: set[Request] = set() self.lastseen: float = 0 - self.latercall = None + self.latercall: CallLaterResult | None = None def free_transfer_slots(self) -> int: return self.concurrency - len(self.transferring) @@ -61,8 +66,9 @@ class Slot: return self.delay def close(self) -> None: - if self.latercall and self.latercall.active(): + if self.latercall: self.latercall.cancel() + self.latercall = None def __repr__(self) -> str: cls_name = self.__class__.__name__ @@ -191,9 +197,8 @@ class Downloader: slot.active.remove(request) def _process_queue(self, spider: Spider, slot: Slot) -> None: - from twisted.internet import reactor - - if slot.latercall and slot.latercall.active(): + if slot.latercall: + # block processing until slot.latercall is called return # Delay queue processing if a download_delay is configured @@ -202,9 +207,7 @@ class Downloader: if delay: penalty = delay - now + slot.lastseen if penalty > 0: - slot.latercall = reactor.callLater( - penalty, self._process_queue, spider, slot - ) + slot.latercall = call_later(penalty, self._latercall, spider, slot) return # Process enqueued requests if there are free slots to transfer for this slot @@ -218,6 +221,10 @@ class Downloader: self._process_queue(spider, slot) break + def _latercall(self, spider: Spider, slot: Slot) -> None: + slot.latercall = None + self._process_queue(spider, slot) + async def _download(self, slot: Slot, request: Request, spider: Spider) -> Response: # The order is very important for the following logic. Do not change! slot.transferring.add(request) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index a649a86e2..b4c6c73a0 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -12,9 +12,15 @@ from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured -from scrapy.utils.asyncio import create_looping_call +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + CallLaterResult, + call_later, + create_looping_call, +) if TYPE_CHECKING: + from twisted.internet.task import LoopingCall from twisted.python.failure import Failure # typing.Self requires Python 3.11 @@ -31,6 +37,12 @@ class CloseSpider: def __init__(self, crawler: Crawler): self.crawler: Crawler = crawler + # for CLOSESPIDER_TIMEOUT + self.task: CallLaterResult | None = None + + # for CLOSESPIDER_TIMEOUT_NO_ITEM + self.task_no_item: AsyncioLoopingCall | LoopingCall | None = None + self.close_on: dict[str, Any] = { "timeout": crawler.settings.getfloat("CLOSESPIDER_TIMEOUT"), "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"), @@ -92,14 +104,12 @@ class CloseSpider: self.crawler.engine.close_spider(spider, "closespider_pagecount_no_item") def spider_opened(self, spider: Spider) -> None: - from twisted.internet import reactor - assert self.crawler.engine - self.task = reactor.callLater( + self.task = call_later( self.close_on["timeout"], self.crawler.engine.close_spider, spider, - reason="closespider_timeout", + "closespider_timeout", ) def item_scraped(self, item: Any, spider: Spider) -> None: @@ -110,13 +120,14 @@ class CloseSpider: self.crawler.engine.close_spider(spider, "closespider_itemcount") def spider_closed(self, spider: Spider) -> None: - task = getattr(self, "task", None) - if task and task.active(): - task.cancel() + if self.task: + self.task.cancel() + self.task = None - task_no_item = getattr(self, "task_no_item", None) - if task_no_item and task_no_item.running: - task_no_item.stop() + if self.task_no_item: + if self.task_no_item.running: + self.task_no_item.stop() + self.task_no_item = None def spider_opened_no_item(self, spider: Spider) -> None: self.task_no_item = create_looping_call(self._count_items_produced, spider) diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index cae2dc033..8c5b843cb 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -15,10 +15,14 @@ from scrapy.utils.asyncgen import as_async_generator from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed if TYPE_CHECKING: + from twisted.internet.base import DelayedCall + # typing.Concatenate and typing.ParamSpec require Python 3.10 - from typing_extensions import Concatenate, ParamSpec + # typing.Self, typing.TypeVarTuple and typing.Unpack require Python 3.11 + from typing_extensions import Concatenate, ParamSpec, Self, TypeVarTuple, Unpack _P = ParamSpec("_P") + _Ts = TypeVarTuple("_Ts") _T = TypeVar("_T") @@ -192,3 +196,60 @@ def create_looping_call( if is_asyncio_available(): return AsyncioLoopingCall(func, *args, **kwargs) return LoopingCall(func, *args, **kwargs) + + +def call_later( + delay: float, func: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts] +) -> CallLaterResult: + """Schedule a function to be called after a delay. + + This uses either ``loop.call_later()`` or ``reactor.callLater()``, depending + on whether asyncio support is available. + """ + if is_asyncio_available(): + loop = asyncio.get_event_loop() + return CallLaterResult.from_asyncio(loop.call_later(delay, func, *args)) + + from twisted.internet import reactor + + return CallLaterResult.from_twisted(reactor.callLater(delay, func, *args)) + + +class CallLaterResult: + """An universal result for :func:`call_later`, wrapping either + :class:`asyncio.TimerHandle` or :class:`twisted.internet.base.DelayedCall`. + + The provided API is close to the :class:`asyncio.TimerHandle` one: there is + no ``active()`` (as there is no such public API in + :class:`asyncio.TimerHandle`) but ``cancel()`` can be called on already + called or cancelled instances. + """ + + _timer_handle: asyncio.TimerHandle | None = None + _delayed_call: DelayedCall | None = None + + @classmethod + def from_asyncio(cls, timer_handle: asyncio.TimerHandle) -> Self: + """Create a CallLaterResult from an asyncio TimerHandle.""" + o = cls() + o._timer_handle = timer_handle + return o + + @classmethod + def from_twisted(cls, delayed_call: DelayedCall) -> Self: + """Create a CallLaterResult from a Twisted DelayedCall.""" + o = cls() + o._delayed_call = delayed_call + return o + + def cancel(self) -> None: + """Cancel the underlying delayed call. + + Does nothing if the delayed call was already called or cancelled. + """ + if self._timer_handle: + self._timer_handle.cancel() + self._timer_handle = None + elif self._delayed_call and self._delayed_call.active(): + self._delayed_call.cancel() + self._delayed_call = None diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 2fb1e0ce7..76f42392b 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -16,13 +16,14 @@ if TYPE_CHECKING: from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from collections.abc import Callable - from twisted.internet.base import DelayedCall from twisted.internet.protocol import ServerFactory from twisted.internet.tcp import Port # typing.ParamSpec requires Python 3.10 from typing_extensions import ParamSpec + from scrapy.utils.asyncio import CallLaterResult + _P = ParamSpec("_P") _T = TypeVar("_T") @@ -55,27 +56,27 @@ class CallLaterOnce(Generic[_T]): self._func: Callable[_P, _T] = func self._a: tuple[Any, ...] = a self._kw: dict[str, Any] = kw - self._call: DelayedCall | None = None + self._call: CallLaterResult | None = None self._deferreds: list[Deferred] = [] def schedule(self, delay: float = 0) -> None: - from twisted.internet import reactor + from scrapy.utils.asyncio import call_later if self._call is None: - self._call = reactor.callLater(delay, self) + self._call = call_later(delay, self) def cancel(self) -> None: if self._call: self._call.cancel() def __call__(self) -> _T: - from twisted.internet import reactor + from scrapy.utils.asyncio import call_later self._call = None result = self._func(*self._a, **self._kw) for d in self._deferreds: - reactor.callLater(0, d.callback, None) + call_later(0, d.callback, None) self._deferreds = [] return result From d602f13e8cd22154936ded9c9356e28fe3be4cd4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 5 Jun 2025 18:02:31 +0500 Subject: [PATCH 147/491] Fix a regression in errback result handling. (#6863) --- scrapy/core/scraper.py | 19 ++++++++----- scrapy/logformatter.py | 4 +-- tests/test_crawl.py | 61 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 10 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 9fd68bce5..1f0d57c63 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -189,6 +189,7 @@ class Scraper: ) assert self.crawler.spider + output: Iterable[Any] | AsyncIterator[Any] if isinstance(result, Response): try: # call the spider middlewares and the request callback with the response @@ -203,7 +204,7 @@ class Scraper: try: # call the request errback with the downloader error - await self.call_spider_async(result, request) + output = await self.call_spider_async(result, request) except Exception as spider_exc: # the errback didn't silence the exception if not result.check(IgnoreRequest): @@ -218,6 +219,8 @@ class Scraper: if spider_exc is not result.value: # the errback raised a different exception, handle it self.handle_spider_error(Failure(), request, result) + else: + await self.handle_spider_output_async(output, request, result) def call_spider( self, result: Response | Failure, request: Request, spider: Spider | None = None @@ -308,7 +311,7 @@ class Scraper: self, result: Iterable[_T] | AsyncIterator[_T], request: Request, - response: Response, + response: Response | Failure, spider: Spider | None = None, ) -> Deferred[None]: """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" @@ -326,7 +329,7 @@ class Scraper: self, result: Iterable[_T] | AsyncIterator[_T], request: Request, - response: Response, + response: Response | Failure, ) -> None: """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" it: Iterable[_T] | AsyncIterator[_T] @@ -361,7 +364,7 @@ class Scraper: ) def _process_spidermw_output( - self, output: Any, response: Response + self, output: Any, response: Response | Failure ) -> Deferred[None]: """Process each Request/Item (given in the output parameter) returned from the given spider. @@ -371,7 +374,7 @@ class Scraper: return deferred_from_coro(self._process_spidermw_output_async(output, response)) async def _process_spidermw_output_async( - self, output: Any, response: Response + self, output: Any, response: Response | Failure ) -> None: """Process each Request/Item (given in the output parameter) returned from the given spider. @@ -385,7 +388,9 @@ class Scraper: if output is not None: await self.start_itemproc_async(output, response=response) - def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[None]: + def start_itemproc( + self, item: Any, *, response: Response | Failure | None + ) -> Deferred[None]: """Send *item* to the item pipelines for processing. *response* is the source of the item data. If the item does not come @@ -394,7 +399,7 @@ class Scraper: return deferred_from_coro(self.start_itemproc_async(item, response=response)) async def start_itemproc_async( - self, item: Any, *, response: Response | None + self, item: Any, *, response: Response | Failure | None ) -> None: """Send *item* to the item pipelines for processing. diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 4f08918ae..e81a9ec93 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -116,7 +116,7 @@ class LogFormatter: self, item: Any, exception: BaseException, - response: Response | None, + response: Response | Failure | None, spider: Spider, ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" @@ -137,7 +137,7 @@ class LogFormatter: self, item: Any, exception: BaseException, - response: Response | None, + response: Response | Failure | None, spider: Spider, ) -> LogFormatterResult: """Logs a message when an item causes an error while it is passing diff --git a/tests/test_crawl.py b/tests/test_crawl.py index b90706027..8289b2243 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -5,6 +5,7 @@ import logging import unittest from ipaddress import IPv4Address from socket import gethostbyname +from typing import Any from urllib.parse import urlparse import pytest @@ -419,6 +420,8 @@ with multiples lines class TestCrawlSpider(TestCase): + mockserver: MockServer + @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -756,6 +759,34 @@ class TestCrawlSpider(TestCase): ) assert "Spider error processing" in str(log) + @defer.inlineCallbacks + def test_spider_errback_item(self): + def eb(failure: Failure) -> Any: + return {"foo": "bar"} + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "'item_scraped_count': 1" in str(log) + + @defer.inlineCallbacks + def test_spider_errback_request(self): + def eb(failure: Failure) -> Request: + return Request(self.mockserver.url("/")) + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "Crawled (200)" in str(log) + @defer.inlineCallbacks def test_spider_errback_downloader_error(self): failures = [] @@ -774,7 +805,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) @defer.inlineCallbacks - def test_spider_errback_exception_downloader_error(self): + def test_spider_errback_downloader_error_exception(self): def eb(failure: Failure) -> None: raise ValueError("foo") @@ -786,6 +817,34 @@ class TestCrawlSpider(TestCase): assert "Error downloading" in str(log) assert "Spider error processing" in str(log) + @defer.inlineCallbacks + def test_spider_errback_downloader_error_item(self): + def eb(failure: Failure) -> Any: + return {"foo": "bar"} + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "'item_scraped_count': 1" in str(log) + + @defer.inlineCallbacks + def test_spider_errback_downloader_error_request(self): + def eb(failure: Failure) -> Request: + return Request(self.mockserver.url("/")) + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "Crawled (200)" in str(log) + @defer.inlineCallbacks def test_raise_closespider(self): def cb(response): From 105c0afb6ee12a5d1664b311582caa90bcc6c6bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 28 May 2025 10:54:36 +0200 Subject: [PATCH 148/491] Feature the new logo in the README (#6831) --- README.rst | 22 ++++++++++------------ docs/_static/logo.svg | 1 + 2 files changed, 11 insertions(+), 12 deletions(-) create mode 100644 docs/_static/logo.svg diff --git a/README.rst b/README.rst index 29488d825..30001e4b0 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,10 @@ -.. image:: https://scrapy.org/img/scrapylogo.png - :target: https://scrapy.org/ +.. raw:: html -====== -Scrapy -====== +

+ + Scrapy + +

.. image:: https://img.shields.io/pypi/v/Scrapy.svg :target: https://pypi.org/pypi/Scrapy @@ -37,13 +38,10 @@ Scrapy :target: https://deepwiki.com/scrapy/scrapy :alt: Ask DeepWiki - -Overview -======== - -Scrapy is a BSD-licensed fast high-level web crawling and web scraping framework, used to -crawl websites and extract structured data from their pages. It can be used for -a wide range of purposes, from data mining to monitoring and automated testing. +Scrapy is a BSD-licensed fast high-level web crawling and web scraping +framework, used to crawl websites and extract structured data from their pages. +It can be used for a wide range of purposes, from data mining to monitoring and +automated testing. Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other contributors`_. diff --git a/docs/_static/logo.svg b/docs/_static/logo.svg new file mode 100644 index 000000000..04b2d18a7 --- /dev/null +++ b/docs/_static/logo.svg @@ -0,0 +1 @@ + From b8cd079014f0e31c609d5fd7fd5f52b89283b1c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 28 May 2025 11:35:18 +0200 Subject: [PATCH 149/491] Shorten the README and remove broken links to scrapy.org (#6833) --- README.rst | 68 ++++++------------------------------------- docs/contributing.rst | 9 ++++-- 2 files changed, 16 insertions(+), 61 deletions(-) diff --git a/README.rst b/README.rst index 30001e4b0..5dc994570 100644 --- a/README.rst +++ b/README.rst @@ -38,74 +38,24 @@ :target: https://deepwiki.com/scrapy/scrapy :alt: Ask DeepWiki -Scrapy is a BSD-licensed fast high-level web crawling and web scraping -framework, used to crawl websites and extract structured data from their pages. -It can be used for a wide range of purposes, from data mining to monitoring and -automated testing. - -Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other -contributors`_. +Scrapy_ is a web scraping framework to extract structured data from websites. +It is cross-platform, and requires Python 3.9+. It is maintained by Zyte_ +(formerly Scrapinghub) and `many other contributors`_. .. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors +.. _Scrapy: https://scrapy.org/ .. _Zyte: https://www.zyte.com/ -Check the Scrapy homepage at https://scrapy.org for more information, -including a list of features. - - -Requirements -============ - -* Python 3.9+ -* Works on Linux, Windows, macOS, BSD - -Install -======= - -The quick way: +Install with: .. code:: bash pip install scrapy -See the install section in the documentation at -https://docs.scrapy.org/en/latest/intro/install.html for more details. +And follow the documentation_ to learn how to use it. -Documentation -============= +.. _documentation: https://docs.scrapy.org/en/latest/ -Documentation is available online at https://docs.scrapy.org/ and in the ``docs`` -directory. +If you wish to contribute, see Contributing_. -Releases -======== - -You can check https://docs.scrapy.org/en/latest/news.html for the release notes. - -Community (blog, twitter, mail list, IRC) -========================================= - -See https://scrapy.org/community/ for details. - -Contributing -============ - -See https://docs.scrapy.org/en/master/contributing.html for details. - -Code of Conduct ---------------- - -Please note that this project is released with a Contributor `Code of Conduct `_. - -By participating in this project you agree to abide by its terms. -Please report unacceptable behavior to opensource@zyte.com. - -Companies using Scrapy -====================== - -See https://scrapy.org/companies/ for a list. - -Commercial Support -================== - -See https://scrapy.org/support/ for details. +.. _Contributing: https://docs.scrapy.org/en/master/contributing.html diff --git a/docs/contributing.rst b/docs/contributing.rst index f5c1c74b8..0172887d6 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -6,8 +6,13 @@ Contributing to Scrapy .. important:: - Double check that you are reading the most recent version of this document at - https://docs.scrapy.org/en/master/contributing.html + Double check that you are reading the most recent version of this document + at https://docs.scrapy.org/en/master/contributing.html + + By participating in this project you agree to abide by the terms of our + `Code of Conduct + `_. Please + report unacceptable behavior to opensource@zyte.com. There are many ways to contribute to Scrapy. Here are some of them: From 3d382aa650735827647093dfb157d5bd2f15efc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 30 May 2025 09:33:17 +0200 Subject: [PATCH 150/491] Avoid raw HTML in the README (#6839) --- README.rst | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index 5dc994570..536dec7f0 100644 --- a/README.rst +++ b/README.rst @@ -1,40 +1,41 @@ -.. raw:: html +|logo| -

- - Scrapy - -

+.. |logo| image:: https://raw.githubusercontent.com/scrapy/scrapy/master/docs/_static/logo.svg + :target: https://scrapy.org + :alt: Scrapy + :width: 480px -.. image:: https://img.shields.io/pypi/v/Scrapy.svg +|version| |python_version| |ubuntu| |macos| |windows| |coverage| |conda| |deepwiki| + +.. |version| image:: https://img.shields.io/pypi/v/Scrapy.svg :target: https://pypi.org/pypi/Scrapy :alt: PyPI Version -.. image:: https://img.shields.io/pypi/pyversions/Scrapy.svg +.. |python_version| image:: https://img.shields.io/pypi/pyversions/Scrapy.svg :target: https://pypi.org/pypi/Scrapy :alt: Supported Python Versions -.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg +.. |ubuntu| image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu :alt: Ubuntu -.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg +.. |macos| image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS :alt: macOS -.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg +.. |windows| image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows :alt: Windows -.. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg +.. |coverage| image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg :target: https://codecov.io/github/scrapy/scrapy?branch=master :alt: Coverage report -.. image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg +.. |conda| image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg :target: https://anaconda.org/conda-forge/scrapy :alt: Conda Version -.. image:: https://deepwiki.com/badge.svg +.. |deepwiki| image:: https://deepwiki.com/badge.svg :target: https://deepwiki.com/scrapy/scrapy :alt: Ask DeepWiki From 54474ceb0d1467d90bf047415d1c1f135263d983 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 5 Jun 2025 18:02:31 +0500 Subject: [PATCH 151/491] Fix a regression in errback result handling. (#6863) --- scrapy/core/scraper.py | 17 ++++++++---- scrapy/logformatter.py | 4 +-- tests/test_crawl.py | 61 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 8 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 9378f2651..975344103 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -188,6 +188,7 @@ class Scraper: ) assert self.crawler.spider + output: Iterable[Any] | AsyncIterator[Any] if isinstance(result, Response): try: # call the spider middlewares and the request callback with the response @@ -204,7 +205,7 @@ class Scraper: try: # call the request errback with the downloader error - await self.call_spider_async(result, request) + output = await self.call_spider_async(result, request) except Exception as spider_exc: # the errback didn't silence the exception if not result.check(IgnoreRequest): @@ -219,6 +220,8 @@ class Scraper: if spider_exc is not result.value: # the errback raised a different exception, handle it self.handle_spider_error(Failure(), request, result) + else: + await self.handle_spider_output_async(output, request, result) def call_spider( self, result: Response | Failure, request: Request, spider: Spider | None = None @@ -309,7 +312,7 @@ class Scraper: self, result: Iterable[_T] | AsyncIterator[_T], request: Request, - response: Response, + response: Response | Failure, spider: Spider | None = None, ) -> Deferred[None]: """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" @@ -327,7 +330,7 @@ class Scraper: self, result: Iterable[_T] | AsyncIterator[_T], request: Request, - response: Response, + response: Response | Failure, ) -> None: """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" if isinstance(result, AsyncIterator): @@ -352,7 +355,9 @@ class Scraper: ) @deferred_f_from_coro_f - async def _process_spidermw_output(self, output: Any, response: Response) -> None: + async def _process_spidermw_output( + self, output: Any, response: Response | Failure + ) -> None: """Process each Request/Item (given in the output parameter) returned from the given spider. @@ -368,7 +373,9 @@ class Scraper: ) @deferred_f_from_coro_f - async def start_itemproc(self, item: Any, *, response: Response | None) -> None: + async def start_itemproc( + self, item: Any, *, response: Response | Failure | None + ) -> None: """Send *item* to the item pipelines for processing. *response* is the source of the item data. If the item does not come diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 4f08918ae..e81a9ec93 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -116,7 +116,7 @@ class LogFormatter: self, item: Any, exception: BaseException, - response: Response | None, + response: Response | Failure | None, spider: Spider, ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" @@ -137,7 +137,7 @@ class LogFormatter: self, item: Any, exception: BaseException, - response: Response | None, + response: Response | Failure | None, spider: Spider, ) -> LogFormatterResult: """Logs a message when an item causes an error while it is passing diff --git a/tests/test_crawl.py b/tests/test_crawl.py index b90706027..8289b2243 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -5,6 +5,7 @@ import logging import unittest from ipaddress import IPv4Address from socket import gethostbyname +from typing import Any from urllib.parse import urlparse import pytest @@ -419,6 +420,8 @@ with multiples lines class TestCrawlSpider(TestCase): + mockserver: MockServer + @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -756,6 +759,34 @@ class TestCrawlSpider(TestCase): ) assert "Spider error processing" in str(log) + @defer.inlineCallbacks + def test_spider_errback_item(self): + def eb(failure: Failure) -> Any: + return {"foo": "bar"} + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "'item_scraped_count': 1" in str(log) + + @defer.inlineCallbacks + def test_spider_errback_request(self): + def eb(failure: Failure) -> Request: + return Request(self.mockserver.url("/")) + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "Crawled (200)" in str(log) + @defer.inlineCallbacks def test_spider_errback_downloader_error(self): failures = [] @@ -774,7 +805,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) @defer.inlineCallbacks - def test_spider_errback_exception_downloader_error(self): + def test_spider_errback_downloader_error_exception(self): def eb(failure: Failure) -> None: raise ValueError("foo") @@ -786,6 +817,34 @@ class TestCrawlSpider(TestCase): assert "Error downloading" in str(log) assert "Spider error processing" in str(log) + @defer.inlineCallbacks + def test_spider_errback_downloader_error_item(self): + def eb(failure: Failure) -> Any: + return {"foo": "bar"} + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "'item_scraped_count': 1" in str(log) + + @defer.inlineCallbacks + def test_spider_errback_downloader_error_request(self): + def eb(failure: Failure) -> Request: + return Request(self.mockserver.url("/")) + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "Crawled (200)" in str(log) + @defer.inlineCallbacks def test_raise_closespider(self): def cb(response): From b20995c9d8dd00618ae71d0f64cdc53f6d669cf2 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 6 Jun 2025 13:16:48 +0500 Subject: [PATCH 152/491] Silence a typing error. --- scrapy/core/scraper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 975344103..ac720e03f 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -193,7 +193,7 @@ class Scraper: try: # call the spider middlewares and the request callback with the response output = await maybe_deferred_to_future( - self.spidermw.scrape_response( + self.spidermw.scrape_response( # type: ignore[arg-type] self.call_spider, result, request, self.crawler.spider ) ) From d99234a33f02f7dd5fb06d167ec78266b7f4dfeb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 6 Jun 2025 14:54:21 +0500 Subject: [PATCH 153/491] Install the reactor explicitly in CrawlerRunner examples. (#6865) --- docs/topics/practices.rst | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index db91cd073..b3c881b81 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -95,6 +95,7 @@ reactor after ``MySpider`` has finished running. import scrapy from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging + from scrapy.utils.reactor import install_reactor class MySpider(scrapy.Spider): @@ -102,6 +103,7 @@ reactor after ``MySpider`` has finished running. ... + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) runner = CrawlerRunner() @@ -112,26 +114,26 @@ reactor after ``MySpider`` has finished running. d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished -Same example but using a non-default reactor, it's only necessary call -``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically. +Same example but using a different reactor. .. code-block:: python import scrapy from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging + from scrapy.utils.reactor import install_reactor class MySpider(scrapy.Spider): + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.epollreactor.EPollReactor", + } # Your spider definition ... + install_reactor("twisted.internet.epollreactor.EPollReactor") configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) - - from scrapy.utils.reactor import install_reactor - - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") runner = CrawlerRunner() d = runner.crawl(MySpider) @@ -184,6 +186,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging from scrapy.utils.project import get_project_settings + from scrapy.utils.reactor import install_reactor class MySpider1(scrapy.Spider): @@ -196,6 +199,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: ... + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") configure_logging() settings = get_project_settings() runner = CrawlerRunner(settings) @@ -217,6 +221,7 @@ Same example but running the spiders sequentially by chaining the deferreds: from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging from scrapy.utils.project import get_project_settings + from scrapy.utils.reactor import install_reactor class MySpider1(scrapy.Spider): @@ -229,6 +234,7 @@ Same example but running the spiders sequentially by chaining the deferreds: ... + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") settings = get_project_settings() configure_logging(settings) runner = CrawlerRunner(settings) From 405d9bc8a247cfec4d698310c425112c456f134f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 6 Jun 2025 15:59:49 +0500 Subject: [PATCH 154/491] More docs for the is_asyncio_reactor_installed() behavior change. (#6866) --- docs/news.rst | 9 +++++++++ scrapy/utils/reactor.py | 5 +++++ scrapy/utils/test.py | 7 ++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index ef3b549e7..8b1d51674 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -126,6 +126,15 @@ Backward-incompatible changes also enforced for start requests. (:issue:`6777`) +- Calling :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` without + an installed reactor now raises an exception instead of installing a + reactor. This shouldn't affect normal Scrapy use cases, but it may affect + 3rd-party test suites that use Scrapy internals such as + :class:`~scrapy.crawler.Crawler` and don't install a reactor explicitly. If + you are affected by this change, you most likely need to install the + reactor before running Scrapy code that expects it to be installed. + (:issue:`6732`, :issue:`6735`) + - The ``from_settings()`` method of :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`, deprecated in Scrapy 2.12.0, is removed earlier than the usual deprecation diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 9c2754394..1b179f988 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -202,6 +202,11 @@ def is_asyncio_reactor_installed() -> bool: """Check whether the installed reactor is :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. Raise a :exc:`RuntimeError` if no reactor is installed. + + .. versionchanged:: 2.13 + In earlier Scrapy versions this function silently installed the default + reactor if there was no reactor installed. Now it raises an exception to + prevent silent problems in this case. """ if not is_reactor_installed(): raise RuntimeError( diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 2da526cd8..4a732bd72 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -18,7 +18,7 @@ from twisted.trial.unittest import SkipTest from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.boto import is_botocore_available from scrapy.utils.deprecate import create_deprecated_class -from scrapy.utils.reactor import is_asyncio_reactor_installed +from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed from scrapy.utils.spider import DefaultSpider if TYPE_CHECKING: @@ -117,6 +117,11 @@ def get_reactor_settings() -> dict[str, Any]: settings, so tests that run the crawler in the current process may need to pass a correct ``"TWISTED_REACTOR"`` setting value when creating it. """ + if not is_reactor_installed(): + raise RuntimeError( + "get_reactor_settings() called without an installed reactor," + " you may need to install a reactor explicitly when running your tests." + ) settings: dict[str, Any] = {} if not is_asyncio_reactor_installed(): settings["TWISTED_REACTOR"] = None From 657e6cb2b57d52005740e92543c1270dbaf61ded Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 6 Jun 2025 16:02:15 +0500 Subject: [PATCH 155/491] Don't try to close ExecutionEngine.downloader when it doesn't exist. (#6867) --- scrapy/core/engine.py | 3 ++- tests/test_engine.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index d9361a674..fe635dc82 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -174,7 +174,8 @@ class ExecutionEngine: return self.close_spider( self.spider, reason="shutdown" ) # will also close downloader - self.downloader.close() + if hasattr(self, "downloader"): + self.downloader.close() return succeed(None) def pause(self) -> None: diff --git a/tests/test_engine.py b/tests/test_engine.py index 1f79a081d..9f618437c 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -22,6 +22,7 @@ from unittest.mock import Mock from urllib.parse import urlparse import attr +import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher from twisted.internet import defer @@ -433,6 +434,19 @@ class TestEngine(TestEngineBase): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) yield e.close() + def test_close_without_downloader(self): + class CustomException(Exception): + pass + + class BadDownloader: + def __init__(self, crawler): + raise CustomException + + with pytest.raises(CustomException): + ExecutionEngine( + get_crawler(MySpider, {"DOWNLOADER": BadDownloader}), lambda _: None + ) + @defer.inlineCallbacks def test_start_already_running_exception(self): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) From d329eedfefd9a1fa7006e6d0a214e9d5e01a8e0c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 6 Jun 2025 16:02:15 +0500 Subject: [PATCH 156/491] Don't try to close ExecutionEngine.downloader when it doesn't exist. (#6867) --- scrapy/core/engine.py | 3 ++- tests/test_engine.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 7f5dd0405..0df9ad2b2 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -170,7 +170,8 @@ class ExecutionEngine: return self.close_spider( self.spider, reason="shutdown" ) # will also close downloader - self.downloader.close() + if hasattr(self, "downloader"): + self.downloader.close() return succeed(None) def pause(self) -> None: diff --git a/tests/test_engine.py b/tests/test_engine.py index b60b510b2..b2e436425 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -22,6 +22,7 @@ from unittest.mock import Mock from urllib.parse import urlparse import attr +import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher from twisted.internet import defer, reactor @@ -431,6 +432,19 @@ class TestEngine(TestEngineBase): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) yield e.close() + def test_close_without_downloader(self): + class CustomException(Exception): + pass + + class BadDownloader: + def __init__(self, crawler): + raise CustomException + + with pytest.raises(CustomException): + ExecutionEngine( + get_crawler(MySpider, {"DOWNLOADER": BadDownloader}), lambda _: None + ) + @defer.inlineCallbacks def test_start_already_running_exception(self): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) From 744edb9ba9e293ddccfcfa03e0aef0a7c0da14b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Jun 2025 16:09:51 +0200 Subject: [PATCH 157/491] Make scrapy fetch work with scrapy-poet (#6872) --- scrapy/commands/fetch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index ef6e13de2..0aaff6c25 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,6 +1,7 @@ from __future__ import annotations import sys +from argparse import Namespace # noqa: TC003 from typing import TYPE_CHECKING from w3lib.url import is_url @@ -12,7 +13,7 @@ from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import DefaultSpider, spidercls_for_request if TYPE_CHECKING: - from argparse import ArgumentParser, Namespace + from argparse import ArgumentParser from scrapy import Spider From d8251332845d48d2418f0055c27686cee04b5b9a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 7 Jun 2025 01:59:09 +0500 Subject: [PATCH 158/491] Reduce deps on unittest, unify inlineCallbacks imports in tests. (#6873) --- tests/test_closespider.py | 16 +- tests/test_command_fetch.py | 10 +- tests/test_command_parse.py | 42 ++-- tests/test_command_runspider.py | 5 +- tests/test_command_shell.py | 39 ++-- tests/test_command_version.py | 6 +- tests/test_contracts.py | 4 +- tests/test_crawl.py | 135 +++++++------ tests/test_dependencies.py | 12 -- tests/test_downloader_handlers.py | 2 +- tests/test_downloader_handlers_http_base.py | 12 +- ...st_downloadermiddleware_httpcompression.py | 35 ++-- tests/test_downloadermiddleware_robotstxt.py | 8 +- tests/test_downloaderslotssettings.py | 4 +- tests/test_engine.py | 24 +-- tests/test_engine_stop_download_bytes.py | 4 +- tests/test_engine_stop_download_headers.py | 4 +- tests/test_exporters.py | 3 +- tests/test_extension_periodic_log.py | 3 +- tests/test_extension_telnet.py | 12 +- tests/test_feedexport.py | 179 +++++++++--------- tests/test_http2_client_protocol.py | 4 +- tests/test_logformatter.py | 6 +- tests/test_pipeline_crawl.py | 12 +- tests/test_pipeline_files.py | 22 +-- tests/test_pipelines.py | 13 +- tests/test_proxy_connect.py | 8 +- tests/test_request_attribute_binding.py | 16 +- tests/test_request_cb_kwargs.py | 4 +- tests/test_request_left.py | 10 +- tests/test_scheduler.py | 6 +- tests/test_scheduler_base.py | 5 +- tests/test_signals.py | 4 +- tests/test_spidermiddleware.py | 15 +- tests/test_spidermiddleware_httperror.py | 8 +- tests/test_spidermiddleware_output_chain.py | 26 +-- tests/test_squeues_request.py | 13 +- tests/test_utils_log.py | 13 +- tests/test_utils_signal.py | 3 +- tests/test_utils_trackref.py | 4 +- tests/test_utils_url.py | 3 +- tests/test_webclient.py | 35 ++-- 42 files changed, 380 insertions(+), 409 deletions(-) diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 4a17b254b..c6ec690a1 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -1,4 +1,4 @@ -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial.unittest import TestCase from scrapy.utils.test import get_crawler @@ -22,7 +22,7 @@ class TestCloseSpider(TestCase): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_itemcount(self): close_on = 5 crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on}) @@ -32,7 +32,7 @@ class TestCloseSpider(TestCase): itemcount = crawler.stats.get_value("item_scraped_count") assert itemcount >= close_on - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_pagecount(self): close_on = 5 crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on}) @@ -42,7 +42,7 @@ class TestCloseSpider(TestCase): pagecount = crawler.stats.get_value("response_received_count") assert pagecount >= close_on - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_pagecount_no_item(self): close_on = 5 max_items = 5 @@ -62,7 +62,7 @@ class TestCloseSpider(TestCase): itemcount = crawler.stats.get_value("item_scraped_count") assert pagecount <= close_on + itemcount - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_pagecount_no_item_with_pagecount(self): close_on_pagecount_no_item = 5 close_on_pagecount = 20 @@ -79,7 +79,7 @@ class TestCloseSpider(TestCase): pagecount = crawler.stats.get_value("response_received_count") assert pagecount < close_on_pagecount - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_errorcount(self): close_on = 5 crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on}) @@ -91,7 +91,7 @@ class TestCloseSpider(TestCase): assert crawler.stats.get_value("spider_exceptions/count") >= close_on assert errorcount >= close_on - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_timeout(self): close_on = 0.1 crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on}) @@ -101,7 +101,7 @@ class TestCloseSpider(TestCase): total_seconds = crawler.stats.get_value("elapsed_time_seconds") assert total_seconds >= close_on - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_timeout_no_item(self): timeout = 1 crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index a31cada85..89f664336 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -1,4 +1,4 @@ -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from tests.utils.testproc import ProcessTest @@ -8,17 +8,17 @@ from tests.utils.testsite import SiteTest class TestFetchCommand(ProcessTest, SiteTest, unittest.TestCase): command = "fetch" - @defer.inlineCallbacks + @inlineCallbacks def test_output(self): _, out, _ = yield self.execute([self.url("/text")]) assert out.strip() == b"Works" - @defer.inlineCallbacks + @inlineCallbacks def test_redirect_default(self): _, out, _ = yield self.execute([self.url("/redirect")]) assert out.strip() == b"Redirected here" - @defer.inlineCallbacks + @inlineCallbacks def test_redirect_disabled(self): _, out, err = yield self.execute( ["--no-redirect", self.url("/redirect-no-meta-refresh")] @@ -27,7 +27,7 @@ class TestFetchCommand(ProcessTest, SiteTest, unittest.TestCase): assert b"downloader/response_status_count/302" in err, err assert b"downloader/response_status_count/200" not in err, err - @defer.inlineCallbacks + @inlineCallbacks def test_headers(self): _, out, _ = yield self.execute([self.url("/text"), "--headers"]) out = out.replace(b"\r", b"") # required on win32 diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 9e66d319c..6681aba17 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -3,7 +3,7 @@ import os import re from pathlib import Path -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from scrapy.commands import parse from scrapy.settings import Settings @@ -171,7 +171,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} """ ) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_arguments(self): _, _, stderr = yield self.execute( [ @@ -187,7 +187,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "DEBUG: It Works!" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_request_with_meta(self): raw_json_string = '{"foo" : "baz"}' _, _, stderr = yield self.execute( @@ -218,7 +218,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "DEBUG: It Works!" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_request_with_cb_kwargs(self): raw_json_string = '{"foo" : "bar", "key": "value"}' _, _, stderr = yield self.execute( @@ -239,7 +239,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} "DEBUG: request.callback signature: (response, foo=None, key=None)" in log ) - @defer.inlineCallbacks + @inlineCallbacks def test_request_without_meta(self): _, _, stderr = yield self.execute( [ @@ -253,7 +253,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "DEBUG: It Works!" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_pipelines(self): _, _, stderr = yield self.execute( [ @@ -268,7 +268,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "INFO: It Works!" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse_items_list(self): status, out, stderr = yield self.execute( [ @@ -283,7 +283,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} assert "{'id': 1}" in _textmode(out) assert "{'id': 2}" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse_items_single_element(self): status, out, stderr = yield self.execute( [ @@ -297,7 +297,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} assert "INFO: Got response 200" in _textmode(stderr) assert "{'foo': 42}" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse_loop(self): status, out, stderr = yield self.execute( [ @@ -312,7 +312,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} for i in range(10): assert f"{{'foo': {i}}}" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse_exc(self): status, out, stderr = yield self.execute( [ @@ -327,7 +327,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} for i in range(7): assert f"{{'foo': {i}}}" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse(self): _, _, stderr = yield self.execute( [ @@ -340,21 +340,21 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "DEBUG: Got response 200" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_parse_items(self): status, out, stderr = yield self.execute( ["--spider", self.spider_name, "-c", "parse", self.url("/html")] ) assert "[{}, {'foo': 'bar'}]" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_parse_items_no_callback_passed(self): status, out, stderr = yield self.execute( ["--spider", self.spider_name, self.url("/html")] ) assert "[{}, {'foo': 'bar'}]" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_wrong_callback_passed(self): status, out, stderr = yield self.execute( ["--spider", self.spider_name, "-c", "dummy", self.url("/html")] @@ -362,7 +362,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) assert "Cannot find callback" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_matching_rule_callback_set(self): """If a rule matches the URL, use it's defined callback.""" status, out, stderr = yield self.execute( @@ -370,7 +370,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "[{}, {'foo': 'bar'}]" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_matching_rule_default_callback(self): """If a rule match but it has no callback set, use the 'parse' callback.""" status, out, stderr = yield self.execute( @@ -378,7 +378,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "[{}, {'nomatch': 'default'}]" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_with_no_rules_attribute(self): """Using -r with a spider with no rule should not produce items.""" status, out, stderr = yield self.execute( @@ -387,14 +387,14 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) assert "No CrawlSpider rules found" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_missing_callback(self): status, out, stderr = yield self.execute( ["--spider", "badcrawl" + self.spider_name, "-r", self.url("/html")] ) assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_no_matching_rule(self): """The requested URL has no matching rule, so no items should be scraped""" status, out, stderr = yield self.execute( @@ -403,12 +403,12 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) assert "Cannot find a rule that matches" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_not_exists_with_not_matched_url(self): status, out, stderr = yield self.execute([self.url("/invalid_url")]) assert status == 0 - @defer.inlineCallbacks + @inlineCallbacks def test_output_flag(self): """Checks if a file was created successfully having correct format containing correct data in it. diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index c57c09249..7f8d9fb61 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -10,7 +10,6 @@ from typing import TYPE_CHECKING from unittest import skipIf import pytest -from twisted.trial import unittest from tests.test_commands import TestCommandBase from tests.test_crawler import ExceptionSpider, NoRequestsSpider @@ -376,7 +375,7 @@ class TestWindowsRunSpiderCommand(TestRunSpiderCommand): def setUp(self): if platform.system() != "Windows": - raise unittest.SkipTest("Windows required for .pyw files") + pytest.skip("Windows required for .pyw files") return super().setUp() def test_start_errors(self): @@ -385,4 +384,4 @@ class TestWindowsRunSpiderCommand(TestRunSpiderCommand): assert "badspider.pyw" in log def test_runspider_unable_to_load(self): - raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") + pytest.skip("Already Tested in 'RunSpiderCommandTest' ") diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 8041e7cb1..d9f17d76b 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -3,8 +3,9 @@ import sys from io import BytesIO from pathlib import Path +import pytest from pexpect.popen_spawn import PopenSpawn -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from scrapy.utils.reactor import _asyncio_reactor_path @@ -17,52 +18,52 @@ from tests.utils.testsite import SiteTest class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): command = "shell" - @defer.inlineCallbacks + @inlineCallbacks def test_empty(self): _, out, _ = yield self.execute(["-c", "item"]) assert b"{}" in out - @defer.inlineCallbacks + @inlineCallbacks def test_response_body(self): _, out, _ = yield self.execute([self.url("/text"), "-c", "response.body"]) assert b"Works" in out - @defer.inlineCallbacks + @inlineCallbacks def test_response_type_text(self): _, out, _ = yield self.execute([self.url("/text"), "-c", "type(response)"]) assert b"TextResponse" in out - @defer.inlineCallbacks + @inlineCallbacks def test_response_type_html(self): _, out, _ = yield self.execute([self.url("/html"), "-c", "type(response)"]) assert b"HtmlResponse" in out - @defer.inlineCallbacks + @inlineCallbacks def test_response_selector_html(self): xpath = "response.xpath(\"//p[@class='one']/text()\").get()" _, out, _ = yield self.execute([self.url("/html"), "-c", xpath]) assert out.strip() == b"Works" - @defer.inlineCallbacks + @inlineCallbacks def test_response_encoding_gb18030(self): _, out, _ = yield self.execute( [self.url("/enc-gb18030"), "-c", "response.encoding"] ) assert out.strip() == b"gb18030" - @defer.inlineCallbacks + @inlineCallbacks def test_redirect(self): _, out, _ = yield self.execute([self.url("/redirect"), "-c", "response.url"]) assert out.strip().endswith(b"/redirected") - @defer.inlineCallbacks + @inlineCallbacks def test_redirect_follow_302(self): _, out, _ = yield self.execute( [self.url("/redirect-no-meta-refresh"), "-c", "response.status"] ) assert out.strip().endswith(b"200") - @defer.inlineCallbacks + @inlineCallbacks def test_redirect_not_follow_302(self): _, out, _ = yield self.execute( [ @@ -74,7 +75,7 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): ) assert out.strip().endswith(b"302") - @defer.inlineCallbacks + @inlineCallbacks def test_fetch_redirect_follow_302(self): """Test that calling ``fetch(url)`` follows HTTP redirects by default.""" url = self.url("/redirect-no-meta-refresh") @@ -84,7 +85,7 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): assert b"Redirecting (302)" in errout assert b"Crawled (200)" in errout - @defer.inlineCallbacks + @inlineCallbacks def test_fetch_redirect_not_follow_302(self): """Test that calling ``fetch(url, redirect=False)`` disables automatic redirects.""" url = self.url("/redirect-no-meta-refresh") @@ -93,27 +94,27 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): assert errcode == 0, out assert b"Crawled (302)" in errout - @defer.inlineCallbacks + @inlineCallbacks def test_request_replace(self): url = self.url("/text") code = f"fetch('{url}') or fetch(response.request.replace(method='POST'))" errcode, out, _ = yield self.execute(["-c", code]) assert errcode == 0, out - @defer.inlineCallbacks + @inlineCallbacks def test_scrapy_import(self): url = self.url("/text") code = f"fetch(scrapy.Request('{url}'))" errcode, out, _ = yield self.execute(["-c", code]) assert errcode == 0, out - @defer.inlineCallbacks + @inlineCallbacks def test_local_file(self): filepath = Path(tests_datadir, "test_site", "index.html") _, out, _ = yield self.execute([str(filepath), "-c", "item"]) assert b"{}" in out - @defer.inlineCallbacks + @inlineCallbacks def test_local_nofile(self): filepath = "file:///tests/sample_data/test_site/nothinghere.html" errcode, out, err = yield self.execute( @@ -122,16 +123,16 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): assert errcode == 1, out or err assert b"No such file or directory" in err - @defer.inlineCallbacks + @inlineCallbacks def test_dns_failures(self): if NON_EXISTING_RESOLVABLE: - raise unittest.SkipTest("Non-existing hosts are resolvable") + pytest.skip("Non-existing hosts are resolvable") url = "www.somedomainthatdoesntexi.st" errcode, out, err = yield self.execute([url, "-c", "item"], check_code=False) assert errcode == 1, out or err assert b"DNS lookup failed" in err - @defer.inlineCallbacks + @inlineCallbacks def test_shell_fetch_async(self): url = self.url("/html") code = f"fetch('{url}')" diff --git a/tests/test_command_version.py b/tests/test_command_version.py index a61a6a32b..87dfb16df 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -1,6 +1,6 @@ import sys -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest import scrapy @@ -10,13 +10,13 @@ from tests.utils.testproc import ProcessTest class TestVersionCommand(ProcessTest, unittest.TestCase): command = "version" - @defer.inlineCallbacks + @inlineCallbacks def test_output(self): encoding = sys.stdout.encoding or "utf-8" _, out, _ = yield self.execute([]) assert out.strip().decode(encoding) == f"Scrapy {scrapy.__version__}" - @defer.inlineCallbacks + @inlineCallbacks def test_verbose_output(self): encoding = sys.stdout.encoding or "utf-8" _, out, _ = yield self.execute(["-v"]) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 26b16a1d4..ad3efa042 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,7 +1,7 @@ from unittest import TextTestResult import pytest -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.python import failure from twisted.trial import unittest @@ -502,7 +502,7 @@ class TestContractsManager(unittest.TestCase): assert not self.results.failures assert self.results.errors - @defer.inlineCallbacks + @inlineCallbacks def test_same_url(self): class TestSameUrlSpider(Spider): name = "test_same_url" diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 8289b2243..4c1f6216b 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -2,7 +2,6 @@ from __future__ import annotations import json import logging -import unittest from ipaddress import IPv4Address from socket import gethostbyname from typing import Any @@ -10,7 +9,7 @@ from urllib.parse import urlparse import pytest from testfixtures import LogCapture -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.internet.ssl import Certificate from twisted.python.failure import Failure from twisted.trial.unittest import TestCase @@ -67,21 +66,21 @@ class TestCrawl(TestCase): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - @defer.inlineCallbacks + @inlineCallbacks def test_follow_all(self): crawler = get_crawler(FollowAllSpider) yield crawler.crawl(mockserver=self.mockserver) assert len(crawler.spider.urls_visited) == 11 # 10 + start_url - @defer.inlineCallbacks + @inlineCallbacks def test_fixed_delay(self): yield self._test_delay(total=3, delay=0.2) - @defer.inlineCallbacks + @inlineCallbacks def test_randomized_delay(self): yield self._test_delay(total=3, delay=0.1, randomize=True) - @defer.inlineCallbacks + @inlineCallbacks def _test_delay(self, total, delay, randomize=False): crawl_kwargs = { "maxlatency": delay * 2, @@ -110,7 +109,7 @@ class TestCrawl(TestCase): average = total_time / (len(times) - 1) assert average <= delay / tolerance, "test total or delay values are too small" - @defer.inlineCallbacks + @inlineCallbacks def test_timeout_success(self): crawler = get_crawler(DelaySpider) yield crawler.crawl(n=0.5, mockserver=self.mockserver) @@ -118,7 +117,7 @@ class TestCrawl(TestCase): assert crawler.spider.t2 > 0 assert crawler.spider.t2 > crawler.spider.t1 - @defer.inlineCallbacks + @inlineCallbacks def test_timeout_failure(self): crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) yield crawler.crawl(n=0.5, mockserver=self.mockserver) @@ -135,7 +134,7 @@ class TestCrawl(TestCase): assert crawler.spider.t2_err > 0 assert crawler.spider.t2_err > crawler.spider.t1 - @defer.inlineCallbacks + @inlineCallbacks def test_retry_503(self): crawler = get_crawler(SimpleSpider) with LogCapture() as log: @@ -144,7 +143,7 @@ class TestCrawl(TestCase): ) self._assert_retried(log) - @defer.inlineCallbacks + @inlineCallbacks def test_retry_conn_failed(self): crawler = get_crawler(SimpleSpider) with LogCapture() as log: @@ -153,10 +152,10 @@ class TestCrawl(TestCase): ) self._assert_retried(log) - @defer.inlineCallbacks + @inlineCallbacks def test_retry_dns_error(self): if NON_EXISTING_RESOLVABLE: - raise unittest.SkipTest("Non-existing hosts are resolvable") + pytest.skip("Non-existing hosts are resolvable") crawler = get_crawler(SimpleSpider) with LogCapture() as log: # try to fetch the homepage of a nonexistent domain @@ -165,7 +164,7 @@ class TestCrawl(TestCase): ) self._assert_retried(log) - @defer.inlineCallbacks + @inlineCallbacks def test_start_bug_before_yield(self): with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(BrokenStartSpider) @@ -176,7 +175,7 @@ class TestCrawl(TestCase): assert record.exc_info is not None assert record.exc_info[0] is ZeroDivisionError - @defer.inlineCallbacks + @inlineCallbacks def test_start_bug_yielding(self): with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(BrokenStartSpider) @@ -187,7 +186,7 @@ class TestCrawl(TestCase): assert record.exc_info is not None assert record.exc_info[0] is ZeroDivisionError - @defer.inlineCallbacks + @inlineCallbacks def test_start_items(self): items = [] @@ -202,7 +201,7 @@ class TestCrawl(TestCase): assert len(log.records) == 0 assert items == [{"name": "test item"}] - @defer.inlineCallbacks + @inlineCallbacks def test_start_unsupported_output(self): """Anything that is not a request is assumed to be an item, avoiding a potentially expensive call to itemadapter.is_item(), and letting @@ -223,7 +222,7 @@ class TestCrawl(TestCase): assert len(items) == 3 assert not any(isinstance(item, Request) for item in items) - @defer.inlineCallbacks + @inlineCallbacks def test_start_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = get_crawler(DuplicateStartSpider, settings) @@ -241,7 +240,7 @@ class TestCrawl(TestCase): ) assert crawler.spider.visited == 3 - @defer.inlineCallbacks + @inlineCallbacks def test_unbounded_response(self): # Completeness of responses without Content-Length or Transfer-Encoding # can not be determined, we treat them as valid but flagged as "partial" @@ -275,7 +274,7 @@ with multiples lines ) assert str(log).count("Got response 200") == 1 - @defer.inlineCallbacks + @inlineCallbacks def test_retry_conn_lost(self): # connection lost after receiving data crawler = get_crawler(SimpleSpider) @@ -285,7 +284,7 @@ with multiples lines ) self._assert_retried(log) - @defer.inlineCallbacks + @inlineCallbacks def test_retry_conn_aborted(self): # connection lost before receiving data crawler = get_crawler(SimpleSpider) @@ -299,7 +298,7 @@ with multiples lines assert str(log).count("Retrying") == 2 assert str(log).count("Gave up retrying") == 1 - @defer.inlineCallbacks + @inlineCallbacks def test_referer_header(self): """Referer header is set by RefererMiddleware unless it is already set""" req0 = Request(self.mockserver.url("/echo?headers=1&body=0"), dont_filter=1) @@ -327,7 +326,7 @@ with multiples lines echo3 = json.loads(to_unicode(crawler.spider.meta["responses"][3].body)) assert echo3["headers"].get("Referer") == ["http://example.com"] - @defer.inlineCallbacks + @inlineCallbacks def test_engine_status(self): from scrapy.utils.engine import get_engine_status @@ -345,7 +344,7 @@ with multiples lines assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == 1 - @defer.inlineCallbacks + @inlineCallbacks def test_format_engine_status(self): from scrapy.utils.engine import format_engine_status @@ -370,7 +369,7 @@ with multiples lines assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == "1" - @defer.inlineCallbacks + @inlineCallbacks def test_open_spider_error_on_faulty_pipeline(self): settings = { "ITEM_PIPELINES": { @@ -378,15 +377,13 @@ with multiples lines } } crawler = get_crawler(SimpleSpider, settings) - yield self.assertFailure( - crawler.crawl( + with pytest.raises(ZeroDivisionError): + yield crawler.crawl( self.mockserver.url("/status?n=200"), mockserver=self.mockserver - ), - ZeroDivisionError, - ) + ) assert not crawler.crawling - @defer.inlineCallbacks + @inlineCallbacks def test_crawlerrunner_accepts_crawler(self): crawler = get_crawler(SimpleSpider) runner = CrawlerRunner() @@ -398,7 +395,7 @@ with multiples lines ) assert "Got response 200" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_crawl_multiple(self): runner = CrawlerRunner(get_reactor_settings()) runner.crawl( @@ -431,7 +428,7 @@ class TestCrawlSpider(TestCase): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - @defer.inlineCallbacks + @inlineCallbacks def _run_spider(self, spider_cls): items = [] @@ -446,7 +443,7 @@ class TestCrawlSpider(TestCase): ) return log, items, crawler.stats - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_with_parse(self): crawler = get_crawler(CrawlSpiderWithParseMethod) with LogCapture() as log: @@ -456,7 +453,7 @@ class TestCrawlSpider(TestCase): assert "[parse] status 201 (foo: None)" in str(log) assert "[parse] status 202 (foo: bar)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_with_async_callback(self): crawler = get_crawler(CrawlSpiderWithAsyncCallback) with LogCapture() as log: @@ -466,7 +463,7 @@ class TestCrawlSpider(TestCase): assert "[parse_async] status 201 (foo: None)" in str(log) assert "[parse_async] status 202 (foo: bar)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_with_async_generator_callback(self): crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback) with LogCapture() as log: @@ -476,7 +473,7 @@ class TestCrawlSpider(TestCase): assert "[parse_async_gen] status 201 (foo: None)" in str(log) assert "[parse_async_gen] status 202 (foo: bar)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_with_errback(self): crawler = get_crawler(CrawlSpiderWithErrback) with LogCapture() as log: @@ -489,7 +486,7 @@ class TestCrawlSpider(TestCase): assert "[errback] status 500" in str(log) assert "[errback] status 501" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_process_request_cb_kwargs(self): crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments) with LogCapture() as log: @@ -499,7 +496,7 @@ class TestCrawlSpider(TestCase): assert "[parse] status 201 (foo: process_request)" in str(log) assert "[parse] status 202 (foo: bar)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_parse(self): crawler = get_crawler(AsyncDefSpider) with LogCapture() as log: @@ -509,7 +506,7 @@ class TestCrawlSpider(TestCase): assert "Got response 200" in str(log) @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse(self): crawler = get_crawler( AsyncDefAsyncioSpider, @@ -524,7 +521,7 @@ class TestCrawlSpider(TestCase): assert "Got response 200" in str(log) @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse_items_list(self): log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider) assert "Got response 200" in str(log) @@ -532,7 +529,7 @@ class TestCrawlSpider(TestCase): assert {"id": 2} in items @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse_items_single_element(self): items = [] @@ -549,7 +546,7 @@ class TestCrawlSpider(TestCase): assert {"foo": 42} in items @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse(self): log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider) assert "Got response 200" in str(log) @@ -557,7 +554,7 @@ class TestCrawlSpider(TestCase): assert itemcount == 1 @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse_loop(self): log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider) assert "Got response 200" in str(log) @@ -567,7 +564,7 @@ class TestCrawlSpider(TestCase): assert {"foo": i} in items @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse_exc(self): log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider) log = str(log) @@ -579,7 +576,7 @@ class TestCrawlSpider(TestCase): assert {"foo": i} in items @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse_complex(self): _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider) itemcount = stats.get_value("item_scraped_count") @@ -591,37 +588,37 @@ class TestCrawlSpider(TestCase): assert {"index2": i} in items @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse_reqs_list(self): log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider) for req_id in range(3): assert f"Got response 200, req_id {req_id}" in str(log) @pytest.mark.only_not_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_deferred_direct(self): _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider) assert items == [{"code": 200}] @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_deferred_wrapped(self): log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider) assert items == [{"code": 200}] - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_deferred_maybe_wrapped(self): _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider) assert items == [{"code": 200}] - @defer.inlineCallbacks + @inlineCallbacks def test_response_ssl_certificate_none(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test", is_secure=False) yield crawler.crawl(seed=url, mockserver=self.mockserver) assert crawler.spider.meta["responses"][0].certificate is None - @defer.inlineCallbacks + @inlineCallbacks def test_response_ssl_certificate(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test", is_secure=True) @@ -634,7 +631,7 @@ class TestCrawlSpider(TestCase): @pytest.mark.xfail( reason="Responses with no body return early and contain no certificate" ) - @defer.inlineCallbacks + @inlineCallbacks def test_response_ssl_certificate_empty_response(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/status?n=200", is_secure=True) @@ -644,7 +641,7 @@ class TestCrawlSpider(TestCase): assert cert.getSubject().commonName == b"localhost" assert cert.getIssuer().commonName == b"localhost" - @defer.inlineCallbacks + @inlineCallbacks def test_dns_server_ip_address_none(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/status?n=200") @@ -652,7 +649,7 @@ class TestCrawlSpider(TestCase): ip_address = crawler.spider.meta["responses"][0].ip_address assert ip_address is None - @defer.inlineCallbacks + @inlineCallbacks def test_dns_server_ip_address(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test") @@ -662,7 +659,7 @@ class TestCrawlSpider(TestCase): assert isinstance(ip_address, IPv4Address) assert str(ip_address) == gethostbyname(expected_netloc) - @defer.inlineCallbacks + @inlineCallbacks def test_bytes_received_stop_download_callback(self): crawler = get_crawler(BytesReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -676,7 +673,7 @@ class TestCrawlSpider(TestCase): < crawler.spider.full_response_length ) - @defer.inlineCallbacks + @inlineCallbacks def test_bytes_received_stop_download_errback(self): crawler = get_crawler(BytesReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -692,7 +689,7 @@ class TestCrawlSpider(TestCase): < crawler.spider.full_response_length ) - @defer.inlineCallbacks + @inlineCallbacks def test_headers_received_stop_download_callback(self): crawler = get_crawler(HeadersReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -702,7 +699,7 @@ class TestCrawlSpider(TestCase): "headers_received" ) - @defer.inlineCallbacks + @inlineCallbacks def test_headers_received_stop_download_errback(self): crawler = get_crawler(HeadersReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -714,7 +711,7 @@ class TestCrawlSpider(TestCase): "failure" ].value.response.headers == crawler.spider.meta.get("headers_received") - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback(self): failures = [] @@ -731,7 +728,7 @@ class TestCrawlSpider(TestCase): assert "HTTP status code is not handled or not allowed" in str(log) assert "Spider error processing" not in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_silence(self): failures = [] @@ -747,7 +744,7 @@ class TestCrawlSpider(TestCase): assert "HTTP status code is not handled or not allowed" not in str(log) assert "Spider error processing" not in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_exception(self): def eb(failure: Failure) -> None: raise ValueError("foo") @@ -759,7 +756,7 @@ class TestCrawlSpider(TestCase): ) assert "Spider error processing" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_item(self): def eb(failure: Failure) -> Any: return {"foo": "bar"} @@ -773,7 +770,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) assert "'item_scraped_count': 1" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_request(self): def eb(failure: Failure) -> Request: return Request(self.mockserver.url("/")) @@ -787,7 +784,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) assert "Crawled (200)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_downloader_error(self): failures = [] @@ -804,7 +801,7 @@ class TestCrawlSpider(TestCase): assert "Error downloading" in str(log) assert "Spider error processing" not in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_downloader_error_exception(self): def eb(failure: Failure) -> None: raise ValueError("foo") @@ -817,7 +814,7 @@ class TestCrawlSpider(TestCase): assert "Error downloading" in str(log) assert "Spider error processing" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_downloader_error_item(self): def eb(failure: Failure) -> Any: return {"foo": "bar"} @@ -831,7 +828,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) assert "'item_scraped_count': 1" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_downloader_error_request(self): def eb(failure: Failure) -> Request: return Request(self.mockserver.url("/")) @@ -845,7 +842,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) assert "Crawled (200)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_raise_closespider(self): def cb(response): raise CloseSpider @@ -856,7 +853,7 @@ class TestCrawlSpider(TestCase): assert "Closing spider (cancelled)" in str(log) assert "Spider error processing" not in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_raise_closespider_reason(self): def cb(response): raise CloseSpider("my_reason") diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index c2df67c66..4436efd9b 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,25 +1,13 @@ import os import re from configparser import ConfigParser -from importlib import import_module from pathlib import Path import pytest from twisted import version as twisted_version -from twisted.trial import unittest class TestScrapyUtils: - def test_required_openssl_version(self): - try: - module = import_module("OpenSSL") - except ImportError: - raise unittest.SkipTest("OpenSSL is not available") - - if hasattr(module, "__version__"): - installed_version = [int(x) for x in module.__version__.split(".")[:2]] - assert installed_version >= [0, 6], "OpenSSL >= 0.6 required" - def test_pinned_twisted_version(self): """When running tests within a Tox environment with pinned dependencies, make sure that the version of Twisted is the pinned diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 09cdbaf35..2c8e96040 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -440,7 +440,7 @@ class TestFTPBase(unittest.TestCase): class TestFTP(TestFTPBase): def test_invalid_credentials(self): if self.reactor_pytest != "default" and sys.platform == "win32": - raise unittest.SkipTest( + pytest.skip( "This test produces DirtyReactorAggregateError on Windows with asyncio" ) from twisted.protocols.ftp import ConnectionLost diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 14e12a3e6..9b2c49fd4 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -14,7 +14,7 @@ from unittest import mock import pytest from testfixtures import LogCapture from twisted.internet import defer, error -from twisted.internet.defer import maybeDeferred +from twisted.internet.defer import inlineCallbacks, maybeDeferred from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest from twisted.web import resource, server, static, util @@ -186,7 +186,7 @@ class TestHttpBase(unittest.TestCase, ABC): self.download_handler_cls, get_crawler() ) - @defer.inlineCallbacks + @inlineCallbacks def tearDown(self): yield self.port.stopListening() if hasattr(self.download_handler, "close"): @@ -229,7 +229,7 @@ class TestHttpBase(unittest.TestCase, ABC): async def test_timeout_download_from_spider_nodata_rcvd(self): if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 - raise unittest.SkipTest( + pytest.skip( "This test produces DirtyReactorAggregateError on Windows with asyncio" ) @@ -245,7 +245,7 @@ class TestHttpBase(unittest.TestCase, ABC): async def test_timeout_download_from_spider_server_hangs(self): if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 - raise unittest.SkipTest( + pytest.skip( "This test produces DirtyReactorAggregateError on Windows with asyncio" ) # client connects, server send headers and some body bytes but hangs @@ -531,7 +531,7 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC): crawler = get_crawler(settings_dict=settings_dict) self.download_handler = build_from_crawler(self.download_handler_cls, crawler) - @defer.inlineCallbacks + @inlineCallbacks def tearDown(self): yield self.port.stopListening() if hasattr(self.download_handler, "close"): @@ -665,7 +665,7 @@ class TestHttpProxyBase(unittest.TestCase, ABC): self.download_handler_cls, get_crawler() ) - @defer.inlineCallbacks + @inlineCallbacks def tearDown(self): yield self.port.stopListening() if hasattr(self.download_handler, "close"): diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index e7427c5ac..3c26b242f 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -2,7 +2,6 @@ from gzip import GzipFile from io import BytesIO from logging import WARNING from pathlib import Path -from unittest import SkipTest import pytest from testfixtures import LogCapture @@ -130,7 +129,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") response = self._getresponse("br") request = response.request assert response.headers["Content-Encoding"] == b"br" @@ -146,11 +145,11 @@ class TestHttpCompression: try: import brotli # noqa: F401 - raise SkipTest("Requires not having brotli support") + pytest.skip("Requires not having brotli support") except ImportError: import brotlicffi # noqa: F401 - raise SkipTest("Requires not having brotli support") + pytest.skip("Requires not having brotli support") except ImportError: pass response = self._getresponse("br") @@ -180,7 +179,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") raw_content = None for check_key in FORMAT: if not check_key.startswith("zstd-"): @@ -201,7 +200,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 - raise SkipTest("Requires not having zstandard support") + pytest.skip("Requires not having zstandard support") except ImportError: pass response = self._getresponse("zstd-static-content-size") @@ -520,7 +519,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_compression_bomb_setting("br") def test_compression_bomb_setting_deflate(self): @@ -533,7 +532,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_compression_bomb_setting("zstd") def _test_compression_bomb_spider_attr(self, compression_id): @@ -556,7 +555,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_compression_bomb_spider_attr("br") def test_compression_bomb_spider_attr_deflate(self): @@ -569,7 +568,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_compression_bomb_spider_attr("zstd") def _test_compression_bomb_request_meta(self, compression_id): @@ -590,7 +589,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_compression_bomb_request_meta("br") def test_compression_bomb_request_meta_deflate(self): @@ -603,7 +602,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_compression_bomb_request_meta("zstd") def _test_download_warnsize_setting(self, compression_id): @@ -639,7 +638,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_download_warnsize_setting("br") def test_download_warnsize_setting_deflate(self): @@ -652,7 +651,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_download_warnsize_setting("zstd") def _test_download_warnsize_spider_attr(self, compression_id): @@ -690,7 +689,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_download_warnsize_spider_attr("br") def test_download_warnsize_spider_attr_deflate(self): @@ -703,7 +702,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_download_warnsize_spider_attr("zstd") def _test_download_warnsize_request_meta(self, compression_id): @@ -739,7 +738,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_download_warnsize_request_meta("br") def test_download_warnsize_request_meta_deflate(self): @@ -752,5 +751,5 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_download_warnsize_request_meta("zstd") diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 04800896c..146b0057e 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -235,12 +235,10 @@ Disallow: /some/randome/page.html self, request: Request, middleware: RobotsTxtMiddleware ) -> None: spider = None # not actually used - await maybe_deferred_to_future( - self.assertFailure( - middleware.process_request(request, spider), # type: ignore[arg-type] - IgnoreRequest, + with pytest.raises(IgnoreRequest): + await maybe_deferred_to_future( + maybeDeferred(middleware.process_request, request, spider) # type: ignore[call-overload] ) - ) def assertRobotsTxtRequested(self, base_url: str) -> None: calls = self.crawler.engine.download.call_args_list diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 78c83ea83..9b7c09448 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -1,6 +1,6 @@ import time -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial.unittest import TestCase from scrapy import Request @@ -62,7 +62,7 @@ class CrawlTestCase(TestCase): def setUp(self): self.runner = CrawlerRunner() - @defer.inlineCallbacks + @inlineCallbacks def test_delay(self): crawler = get_crawler(DownloaderSlotsSettingsTestSpider) yield crawler.crawl(mockserver=self.mockserver) diff --git a/tests/test_engine.py b/tests/test_engine.py index 9f618437c..e181a36cf 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -26,6 +26,7 @@ import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from twisted.web import server, static, util @@ -390,7 +391,7 @@ class TestEngineBase(unittest.TestCase): class TestEngine(TestEngineBase): - @defer.inlineCallbacks + @inlineCallbacks def test_crawler(self): for spider in ( MySpider, @@ -407,20 +408,20 @@ class TestEngine(TestEngineBase): self._assert_signals_caught(run) self._assert_bytes_received(run) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_dupefilter(self): run = CrawlerRun(DupeFilterSpider) yield run.run() self._assert_scheduled_requests(run, count=8) self._assert_dropped_requests(run) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_itemerror(self): run = CrawlerRun(ItemZeroDivisionErrorSpider) yield run.run() self._assert_items_error(run) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_change_close_reason_on_idle(self): run = CrawlerRun(ChangeCloseReasonSpider) yield run.run() @@ -429,7 +430,7 @@ class TestEngine(TestEngineBase): "reason": "custom_reason", } == run.signals_caught[signals.spider_closed] - @defer.inlineCallbacks + @inlineCallbacks def test_close_downloader(self): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) yield e.close() @@ -447,19 +448,14 @@ class TestEngine(TestEngineBase): get_crawler(MySpider, {"DOWNLOADER": BadDownloader}), lambda _: None ) - @defer.inlineCallbacks + @inlineCallbacks def test_start_already_running_exception(self): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) yield e.open_spider(MySpider(), []) e.start() - - def cb(exc: BaseException) -> None: - assert str(exc), "Engine already running" - - try: - yield self.assertFailure(e.start(), RuntimeError).addBoth(cb) - finally: - yield e.stop() + with pytest.raises(RuntimeError, match="Engine already running"): + yield e.start() + yield e.stop() def test_short_timeout(self): args = ( diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index f09b0e091..2662e45e1 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -1,5 +1,5 @@ from testfixtures import LogCapture -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from scrapy.exceptions import StopDownload from tests.test_engine import ( @@ -19,7 +19,7 @@ class BytesReceivedCrawlerRun(CrawlerRun): class TestBytesReceivedEngine(TestEngineBase): - @defer.inlineCallbacks + @inlineCallbacks def test_crawler(self): for spider in ( MySpider, diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index dbb0ea0d2..142715927 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -1,5 +1,5 @@ from testfixtures import LogCapture -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from scrapy.exceptions import StopDownload from tests.test_engine import ( @@ -19,7 +19,7 @@ class HeadersReceivedCrawlerRun(CrawlerRun): class TestHeadersReceivedEngine(TestEngineBase): - @defer.inlineCallbacks + @inlineCallbacks def test_crawler(self): for spider in ( MySpider, diff --git a/tests/test_exporters.py b/tests/test_exporters.py index f55cb6c97..05e8865bc 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -4,7 +4,6 @@ import marshal import pickle import re import tempfile -import unittest from datetime import datetime from io import BytesIO from typing import Any @@ -662,7 +661,7 @@ class TestCustomExporterItem: def setup_method(self): if self.item_class is None: - raise unittest.SkipTest("item class is None") + pytest.skip("item class is None") def test_exporter_custom_serializer(self): class CustomItemExporter(BaseItemExporter): diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 85bd42857..b86f3c7f2 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -1,7 +1,6 @@ from __future__ import annotations import datetime -import unittest from typing import Any, Callable from scrapy.extensions.periodic_log import PeriodicLog @@ -66,7 +65,7 @@ def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog: return CustomPeriodicLog.from_crawler(crawler) -class TestPeriodicLog(unittest.TestCase): +class TestPeriodicLog: def test_extension_enabled(self): # Expected that settings for this extension loaded successfully # And on certain conditions - extension raising NotConfigured diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 8c897c223..2ac4d7830 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -1,6 +1,7 @@ +import pytest from twisted.conch.telnet import ITelnetProtocol from twisted.cred import credentials -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from scrapy.extensions.telnet import TelnetConsole @@ -21,15 +22,16 @@ class TelnetExtensionTest(unittest.TestCase): return console, portal - @defer.inlineCallbacks + @inlineCallbacks def test_bad_credentials(self): console, portal = self._get_console_and_portal() creds = credentials.UsernamePassword(b"username", b"password") d = portal.login(creds, None, ITelnetProtocol) - yield self.assertFailure(d, ValueError) + with pytest.raises(ValueError, match="Invalid credentials"): + yield d console.stop_listening() - @defer.inlineCallbacks + @inlineCallbacks def test_good_credentials(self): console, portal = self._get_console_and_portal() creds = credentials.UsernamePassword( @@ -39,7 +41,7 @@ class TelnetExtensionTest(unittest.TestCase): yield d console.stop_listening() - @defer.inlineCallbacks + @inlineCallbacks def test_custom_credentials(self): settings = { "TELNETCONSOLE_USERNAME": "user", diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 44cd10ec3..cdf03ca76 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -26,6 +26,7 @@ import lxml.etree import pytest from testfixtures import LogCapture from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from w3lib.url import file_uri_to_path, path_to_file_uri from zope.interface import implementer @@ -131,7 +132,7 @@ class TestFileFeedStorage(unittest.TestCase): FileFeedStorage(str(path), feed_options={"overwrite": True}), path ) - @defer.inlineCallbacks + @inlineCallbacks def _assert_stores(self, storage, path: Path, expected_content=b"content"): spider = scrapy.Spider("default") file = storage.open(spider) @@ -172,7 +173,7 @@ class TestFTPFeedStorage(unittest.TestCase): finally: path.unlink() - @defer.inlineCallbacks + @inlineCallbacks def test_append(self): with MockFTPServer() as ftp_server: filename = "file" @@ -182,7 +183,7 @@ class TestFTPFeedStorage(unittest.TestCase): yield self._store(url, b"bar", feed_options=feed_options) self._assert_stored(ftp_server.path / filename, b"foobar") - @defer.inlineCallbacks + @inlineCallbacks def test_overwrite(self): with MockFTPServer() as ftp_server: filename = "file" @@ -191,7 +192,7 @@ class TestFTPFeedStorage(unittest.TestCase): yield self._store(url, b"bar") self._assert_stored(ftp_server.path / filename, b"bar") - @defer.inlineCallbacks + @inlineCallbacks def test_append_active_mode(self): with MockFTPServer() as ftp_server: settings = {"FEED_STORAGE_FTP_ACTIVE": True} @@ -202,7 +203,7 @@ class TestFTPFeedStorage(unittest.TestCase): yield self._store(url, b"bar", feed_options=feed_options, settings=settings) self._assert_stored(ftp_server.path / filename, b"foobar") - @defer.inlineCallbacks + @inlineCallbacks def test_overwrite_active_mode(self): with MockFTPServer() as ftp_server: settings = {"FEED_STORAGE_FTP_ACTIVE": True} @@ -290,7 +291,7 @@ class TestS3FeedStorage(unittest.TestCase): assert storage.access_key == "uri_key" assert storage.secret_key == "uri_secret" - @defer.inlineCallbacks + @inlineCallbacks def test_store(self): settings = { "AWS_ACCESS_KEY_ID": "access_key", @@ -431,7 +432,7 @@ class TestS3FeedStorage(unittest.TestCase): assert storage.region_name == region_name assert storage.s3_client._client_config.region_name == region_name - @defer.inlineCallbacks + @inlineCallbacks def test_store_without_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", @@ -451,7 +452,7 @@ class TestS3FeedStorage(unittest.TestCase): ) assert acl is None - @defer.inlineCallbacks + @inlineCallbacks def test_store_with_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" @@ -489,7 +490,7 @@ class TestGCSFeedStorage(unittest.TestCase): try: from google.cloud.storage import Client # noqa: F401 except ImportError: - raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + pytest.skip("GCSFeedStorage requires google-cloud-storage") settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"} crawler = get_crawler(settings_dict=settings) @@ -503,7 +504,7 @@ class TestGCSFeedStorage(unittest.TestCase): try: from google.cloud.storage import Client # noqa: F401 except ImportError: - raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + pytest.skip("GCSFeedStorage requires google-cloud-storage") settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""} crawler = get_crawler(settings_dict=settings) @@ -515,12 +516,12 @@ class TestGCSFeedStorage(unittest.TestCase): storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") assert storage.acl is None - @defer.inlineCallbacks + @inlineCallbacks def test_store(self): try: from google.cloud.storage import Client # noqa: F401 except ImportError: - raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + pytest.skip("GCSFeedStorage requires google-cloud-storage") uri = "gs://mybucket/export.csv" project_id = "myproject-123" @@ -556,7 +557,7 @@ class TestGCSFeedStorage(unittest.TestCase): class TestStdoutFeedStorage(unittest.TestCase): - @defer.inlineCallbacks + @inlineCallbacks def test_store(self): out = BytesIO() storage = StdoutFeedStorage("stdout:", _stdout=out) @@ -669,7 +670,7 @@ class TestFeedExportBase(ABC, unittest.TestCase): def tearDown(self): shutil.rmtree(self.temp_dir, ignore_errors=True) - @defer.inlineCallbacks + @inlineCallbacks def exported_data(self, items, settings): """ Return exported data which a spider yielding ``items`` would return. @@ -684,7 +685,7 @@ class TestFeedExportBase(ABC, unittest.TestCase): data = yield self.run_and_export(TestSpider, settings) return data - @defer.inlineCallbacks + @inlineCallbacks def exported_no_data(self, settings): """ Return exported data which a spider yielding no ``items`` would return. @@ -699,7 +700,7 @@ class TestFeedExportBase(ABC, unittest.TestCase): data = yield self.run_and_export(TestSpider, settings) return data - @defer.inlineCallbacks + @inlineCallbacks def assertExported(self, items, header, rows, settings=None): yield self.assertExportedCsv(items, header, rows, settings) yield self.assertExportedJsonLines(items, rows, settings) @@ -770,7 +771,7 @@ class ExceptionJsonItemExporter(JsonItemExporter): class TestFeedExport(TestFeedExportBase): - @defer.inlineCallbacks + @inlineCallbacks def run_and_export(self, spider_cls, settings): """Run spider with specified settings; return exported data.""" @@ -800,7 +801,7 @@ class TestFeedExport(TestFeedExportBase): return content - @defer.inlineCallbacks + @inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update( @@ -815,7 +816,7 @@ class TestFeedExport(TestFeedExportBase): assert reader.fieldnames == list(header) assert rows == list(reader) - @defer.inlineCallbacks + @inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -830,7 +831,7 @@ class TestFeedExport(TestFeedExportBase): rows = [{k: v for k, v in row.items() if v} for row in rows] assert rows == parsed - @defer.inlineCallbacks + @inlineCallbacks def assertExportedXml(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -846,7 +847,7 @@ class TestFeedExport(TestFeedExportBase): got_rows = [{e.tag: e.text for e in it} for it in root.findall("item")] assert rows == got_rows - @defer.inlineCallbacks + @inlineCallbacks def assertExportedMultiple(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -867,7 +868,7 @@ class TestFeedExport(TestFeedExportBase): json_rows = json.loads(to_unicode(data["json"])) assert rows == json_rows - @defer.inlineCallbacks + @inlineCallbacks def assertExportedPickle(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -884,7 +885,7 @@ class TestFeedExport(TestFeedExportBase): result = self._load_until_eof(data["pickle"], load_func=pickle.load) assert result == expected - @defer.inlineCallbacks + @inlineCallbacks def assertExportedMarshal(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -901,7 +902,7 @@ class TestFeedExport(TestFeedExportBase): result = self._load_until_eof(data["marshal"], load_func=marshal.load) assert result == expected - @defer.inlineCallbacks + @inlineCallbacks def test_stats_file_success(self): settings = { "FEEDS": { @@ -915,7 +916,7 @@ class TestFeedExport(TestFeedExportBase): assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 - @defer.inlineCallbacks + @inlineCallbacks def test_stats_file_failed(self): settings = { "FEEDS": { @@ -933,7 +934,7 @@ class TestFeedExport(TestFeedExportBase): assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1 - @defer.inlineCallbacks + @inlineCallbacks def test_stats_multiple_file(self): settings = { "FEEDS": { @@ -955,7 +956,7 @@ class TestFeedExport(TestFeedExportBase): crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage") == 1 ) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items(self): # feed exporters use field names from Item items = [ @@ -969,7 +970,7 @@ class TestFeedExport(TestFeedExportBase): header = self.MyItem.fields.keys() yield self.assertExported(items, header, rows) - @defer.inlineCallbacks + @inlineCallbacks def test_export_no_items_not_store_empty(self): for fmt in ("json", "jsonlines", "xml", "csv"): settings = { @@ -981,7 +982,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_no_data(settings) assert data[fmt] is None - @defer.inlineCallbacks + @inlineCallbacks def test_start_finish_exporting_items(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1001,7 +1002,7 @@ class TestFeedExport(TestFeedExportBase): assert not listener.start_without_finish assert not listener.finish_without_start - @defer.inlineCallbacks + @inlineCallbacks def test_start_finish_exporting_no_items(self): items = [] settings = { @@ -1019,7 +1020,7 @@ class TestFeedExport(TestFeedExportBase): assert not listener.start_without_finish assert not listener.finish_without_start - @defer.inlineCallbacks + @inlineCallbacks def test_start_finish_exporting_items_exception(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1040,7 +1041,7 @@ class TestFeedExport(TestFeedExportBase): assert not listener.start_without_finish assert not listener.finish_without_start - @defer.inlineCallbacks + @inlineCallbacks def test_start_finish_exporting_no_items_exception(self): items = [] settings = { @@ -1059,7 +1060,7 @@ class TestFeedExport(TestFeedExportBase): assert not listener.start_without_finish assert not listener.finish_without_start - @defer.inlineCallbacks + @inlineCallbacks def test_export_no_items_store_empty(self): formats = ( ("json", b"[]"), @@ -1079,7 +1080,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_no_data(settings) assert expctd == data[fmt] - @defer.inlineCallbacks + @inlineCallbacks def test_export_no_items_multiple_feeds(self): """Make sure that `storage.store` is called for every feed.""" settings = { @@ -1097,7 +1098,7 @@ class TestFeedExport(TestFeedExportBase): assert str(log).count("Storage.store is called") == 0 - @defer.inlineCallbacks + @inlineCallbacks def test_export_multiple_item_classes(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1119,7 +1120,7 @@ class TestFeedExport(TestFeedExportBase): yield self.assertExportedCsv(items, header, rows_csv) yield self.assertExportedJsonLines(items, rows_jl) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_empty_field_list(self): # FEED_EXPORT_FIELDS==[] means the same as default None items = [{"foo": "bar"}] @@ -1129,7 +1130,7 @@ class TestFeedExport(TestFeedExportBase): yield self.assertExportedCsv(items, header, rows) yield self.assertExportedJsonLines(items, rows, settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] @@ -1137,7 +1138,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": header} yield self.assertExported(items, header, rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_comma_separated_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] @@ -1145,7 +1146,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": ",".join(header)} yield self.assertExported(items, header, rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_json_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] @@ -1153,7 +1154,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} yield self.assertExported(items, header, rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_field_names(self): items = [{"foo": "bar"}] header = {"foo": "Foo"} @@ -1161,7 +1162,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": header} yield self.assertExported(items, list(header.values()), rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_dict_field_names(self): items = [{"foo": "bar"}] header = { @@ -1172,7 +1173,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": header} yield self.assertExported(items, ["Baz", "Foo"], rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_json_field_names(self): items = [{"foo": "bar"}] header = {"foo": "Foo"} @@ -1180,7 +1181,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} yield self.assertExported(items, list(header.values()), rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_based_on_item_classes(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1226,7 +1227,7 @@ class TestFeedExport(TestFeedExportBase): for fmt, expected in formats.items(): assert data[fmt] == expected - @defer.inlineCallbacks + @inlineCallbacks def test_export_based_on_custom_filters(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1285,7 +1286,7 @@ class TestFeedExport(TestFeedExportBase): for fmt, expected in formats.items(): assert data[fmt] == expected - @defer.inlineCallbacks + @inlineCallbacks def test_export_dicts(self): # When dicts are used, only keys from the first row are used as # a header for CSV, and all fields are used for JSON Lines. @@ -1298,7 +1299,7 @@ class TestFeedExport(TestFeedExportBase): yield self.assertExportedCsv(items, ["foo", "egg"], rows_csv) yield self.assertExportedJsonLines(items, rows_jl) - @defer.inlineCallbacks + @inlineCallbacks def test_export_tuple(self): items = [ {"foo": "bar1", "egg": "spam1"}, @@ -1309,7 +1310,7 @@ class TestFeedExport(TestFeedExportBase): rows = [{"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}] yield self.assertExported(items, ["foo", "baz"], rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_feed_export_fields(self): # FEED_EXPORT_FIELDS option allows to order export fields # and to select a subset of fields to export, both for Items and dicts. @@ -1335,7 +1336,7 @@ class TestFeedExport(TestFeedExportBase): rows = [{"egg": "spam1", "baz": ""}, {"egg": "spam2", "baz": "quux2"}] yield self.assertExported(items, ["egg", "baz"], rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_encoding(self): items = [{"foo": "Test\xd6"}] @@ -1380,7 +1381,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_data(items, settings) assert data[fmt] == expected - @defer.inlineCallbacks + @inlineCallbacks def test_export_multiple_configs(self): items = [{"foo": "FOO", "bar": "BAR"}] @@ -1420,7 +1421,7 @@ class TestFeedExport(TestFeedExportBase): for fmt, expected in formats.items(): assert data[fmt] == expected - @defer.inlineCallbacks + @inlineCallbacks def test_export_indentation(self): items = [ {"foo": ["bar"]}, @@ -1576,7 +1577,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_data(items, settings) assert data[row["format"]] == row["expected"] - @defer.inlineCallbacks + @inlineCallbacks def test_init_exporters_storages_with_crawler(self): settings = { "FEED_EXPORTERS": {"csv": FromCrawlerCsvItemExporter}, @@ -1589,7 +1590,7 @@ class TestFeedExport(TestFeedExportBase): assert FromCrawlerCsvItemExporter.init_with_crawler assert FromCrawlerFileFeedStorage.init_with_crawler - @defer.inlineCallbacks + @inlineCallbacks def test_str_uri(self): settings = { "FEED_STORE_EMPTY": True, @@ -1598,7 +1599,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_no_data(settings) assert data["csv"] == b"" - @defer.inlineCallbacks + @inlineCallbacks def test_multiple_feeds_success_logs_blocking_feed_storage(self): settings = { "FEEDS": { @@ -1619,7 +1620,7 @@ class TestFeedExport(TestFeedExportBase): for fmt in ["json", "xml", "csv"]: assert f"Stored {fmt} feed (2 items)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_multiple_feeds_failing_logs_blocking_feed_storage(self): settings = { "FEEDS": { @@ -1640,7 +1641,7 @@ class TestFeedExport(TestFeedExportBase): for fmt in ["json", "xml", "csv"]: assert f"Error storing {fmt} feed (2 items)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_extend_kwargs(self): items = [{"foo": "FOO", "bar": "BAR"}] @@ -1677,7 +1678,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_data(items, settings) assert data[feed_options["format"]] == row["expected"] - @defer.inlineCallbacks + @inlineCallbacks def test_storage_file_no_postprocessing(self): @implementer(IFeedStorage) class Storage: @@ -1699,7 +1700,7 @@ class TestFeedExport(TestFeedExportBase): yield self.exported_no_data(settings) assert Storage.open_file is Storage.store_file - @defer.inlineCallbacks + @inlineCallbacks def test_storage_file_postprocessing(self): @implementer(IFeedStorage) class Storage: @@ -1752,7 +1753,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): def _named_tempfile(self, name) -> str: return str(Path(self.temp_dir, name)) - @defer.inlineCallbacks + @inlineCallbacks def run_and_export(self, spider_cls, settings): """Run spider with specified settings; return exported data with filename.""" @@ -1796,7 +1797,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data_stream.seek(0) return data_stream.read() - @defer.inlineCallbacks + @inlineCallbacks def test_gzip_plugin(self): filename = self._named_tempfile("gzip_file") @@ -1815,7 +1816,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): except OSError: pytest.fail("Received invalid gzip data.") - @defer.inlineCallbacks + @inlineCallbacks def test_gzip_plugin_compresslevel(self): filename_to_compressed = { self._named_tempfile("compresslevel_0"): self.get_gzip_compressed( @@ -1852,7 +1853,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_gzip_plugin_mtime(self): filename_to_compressed = { self._named_tempfile("mtime_123"): self.get_gzip_compressed( @@ -1887,7 +1888,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_gzip_plugin_filename(self): filename_to_compressed = { self._named_tempfile("filename_FILE1"): self.get_gzip_compressed( @@ -1922,7 +1923,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_lzma_plugin(self): filename = self._named_tempfile("lzma_file") @@ -1941,7 +1942,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): except lzma.LZMAError: pytest.fail("Received invalid lzma data.") - @defer.inlineCallbacks + @inlineCallbacks def test_lzma_plugin_format(self): filename_to_compressed = { self._named_tempfile("format_FORMAT_XZ"): lzma.compress( @@ -1974,7 +1975,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_lzma_plugin_check(self): filename_to_compressed = { self._named_tempfile("check_CHECK_NONE"): lzma.compress( @@ -2007,7 +2008,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_lzma_plugin_preset(self): filename_to_compressed = { self._named_tempfile("preset_PRESET_0"): lzma.compress( @@ -2040,11 +2041,11 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_lzma_plugin_filters(self): if "PyPy" in sys.version: # https://foss.heptapod.net/pypy/pypy/-/issues/3527 - raise unittest.SkipTest("lzma filters doesn't work in PyPy") + pytest.skip("lzma filters doesn't work in PyPy") filters = [{"id": lzma.FILTER_LZMA2}] compressed = lzma.compress(self.expected, filters=filters) @@ -2065,7 +2066,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): result = lzma.decompress(data[filename]) assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_bz2_plugin(self): filename = self._named_tempfile("bz2_file") @@ -2084,7 +2085,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): except OSError: pytest.fail("Received invalid bz2 data.") - @defer.inlineCallbacks + @inlineCallbacks def test_bz2_plugin_compresslevel(self): filename_to_compressed = { self._named_tempfile("compresslevel_1"): bz2.compress( @@ -2117,7 +2118,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_custom_plugin(self): filename = self._named_tempfile("csv_file") @@ -2133,7 +2134,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data = yield self.exported_data(self.items, settings) assert data[filename] == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_custom_plugin_with_parameter(self): expected = b"foo\r\n\nbar\r\n\n" filename = self._named_tempfile("newline") @@ -2151,7 +2152,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data = yield self.exported_data(self.items, settings) assert data[filename] == expected - @defer.inlineCallbacks + @inlineCallbacks def test_custom_plugin_with_compression(self): expected = b"foo\r\n\nbar\r\n\n" @@ -2196,7 +2197,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): result = decompressor(data[filename]) assert result == expected - @defer.inlineCallbacks + @inlineCallbacks def test_exports_compatibility_with_postproc(self): import marshal import pickle @@ -2254,7 +2255,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): class TestBatchDeliveries(TestFeedExportBase): _file_mark = "_%(batch_time)s_#%(batch_id)02d_" - @defer.inlineCallbacks + @inlineCallbacks def run_and_export(self, spider_cls, settings): """Run spider with specified settings; return exported data.""" @@ -2276,7 +2277,7 @@ class TestBatchDeliveries(TestFeedExportBase): content[feed["format"]].append(file.read_bytes()) return content - @defer.inlineCallbacks + @inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -2298,7 +2299,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update( @@ -2318,7 +2319,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert list(got_batch) == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def assertExportedXml(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -2339,7 +2340,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def assertExportedMultiple(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -2371,7 +2372,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def assertExportedPickle(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -2393,7 +2394,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def assertExportedMarshal(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -2415,7 +2416,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def test_export_items(self): """Test partial deliveries in all supported formats""" items = [ @@ -2444,7 +2445,7 @@ class TestBatchDeliveries(TestFeedExportBase): with pytest.raises(NotConfigured): FeedExporter(crawler) - @defer.inlineCallbacks + @inlineCallbacks def test_export_no_items_not_store_empty(self): for fmt in ("json", "jsonlines", "xml", "csv"): settings = { @@ -2460,7 +2461,7 @@ class TestBatchDeliveries(TestFeedExportBase): data = dict(data) assert len(data[fmt]) == 0 - @defer.inlineCallbacks + @inlineCallbacks def test_export_no_items_store_empty(self): formats = ( ("json", b"[]"), @@ -2484,7 +2485,7 @@ class TestBatchDeliveries(TestFeedExportBase): data = dict(data) assert data[fmt][0] == expctd - @defer.inlineCallbacks + @inlineCallbacks def test_export_multiple_configs(self): items = [ {"foo": "FOO", "bar": "BAR"}, @@ -2540,7 +2541,7 @@ class TestBatchDeliveries(TestFeedExportBase): for expected_batch, got_batch in zip(expected, data[fmt]): assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def test_batch_item_count_feeds_setting(self): items = [{"foo": "FOO"}, {"foo": "FOO1"}] formats = { @@ -2564,7 +2565,7 @@ class TestBatchDeliveries(TestFeedExportBase): for expected_batch, got_batch in zip(expected, data[fmt]): assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def test_batch_path_differ(self): """ Test that the name of all batch files differ from each other. @@ -2586,7 +2587,7 @@ class TestBatchDeliveries(TestFeedExportBase): data = yield self.exported_data(items, settings) assert len(items) == len(data["json"]) - @defer.inlineCallbacks + @inlineCallbacks def test_stats_batch_file_success(self): settings = { "FEEDS": { @@ -2604,7 +2605,7 @@ class TestBatchDeliveries(TestFeedExportBase): assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12 @pytest.mark.requires_boto3 - @defer.inlineCallbacks + @inlineCallbacks def test_s3_export(self): bucket = "mybucket" items = [ diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 0605c2438..ef1806cc0 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -9,7 +9,7 @@ from ipaddress import IPv4Address from pathlib import Path from tempfile import mkdtemp from typing import TYPE_CHECKING -from unittest import mock, skipIf +from unittest import mock from urllib.parse import urlencode import pytest @@ -183,7 +183,7 @@ def get_client_certificate( return PrivateCertificate.loadPEM(pem) -@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") +@pytest.mark.skipif(not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled") class TestHttps2ClientProtocol(TestCase): scheme = "https" key_file = Path(__file__).parent / "keys" / "localhost.key" diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 3c9f97631..047f8c610 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -2,7 +2,7 @@ import logging import pytest from testfixtures import LogCapture -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.python.failure import Failure from twisted.trial.unittest import TestCase @@ -272,7 +272,7 @@ class TestShowOrSkipMessages(TestCase): }, } - @defer.inlineCallbacks + @inlineCallbacks def test_show_messages(self): crawler = get_crawler(ItemSpider, self.base_settings) with LogCapture() as lc: @@ -281,7 +281,7 @@ class TestShowOrSkipMessages(TestCase): assert "Crawled (200) Date: Sat, 7 Jun 2025 09:17:36 +0200 Subject: [PATCH 159/491] Remove the old artwork (#6874) --- artwork/README.rst | 20 -------------------- artwork/qlassik.zip | Bin 120204 -> 0 bytes artwork/scrapy-blog-logo.xcf | Bin 52428 -> 0 bytes artwork/scrapy-logo.jpg | Bin 23398 -> 0 bytes 4 files changed, 20 deletions(-) delete mode 100644 artwork/README.rst delete mode 100644 artwork/qlassik.zip delete mode 100644 artwork/scrapy-blog-logo.xcf delete mode 100644 artwork/scrapy-logo.jpg diff --git a/artwork/README.rst b/artwork/README.rst deleted file mode 100644 index c1880ef6c..000000000 --- a/artwork/README.rst +++ /dev/null @@ -1,20 +0,0 @@ -============== -Scrapy artwork -============== - -This folder contains the Scrapy artwork resources such as logos and fonts. - -scrapy-logo.jpg ---------------- - -The main Scrapy logo, in JPEG format. - -qlassik.zip ------------ - -The font used for the Scrapy logo. Homepage: https://www.dafont.com/qlassik.font - -scrapy-blog.logo.xcf --------------------- - -The logo used in the Scrapy blog, in Gimp format. diff --git a/artwork/qlassik.zip b/artwork/qlassik.zip deleted file mode 100644 index 2885c06ef4bab2fd9027bf748bd5ad2a69eb857f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 120204 zcmV(+K;6GkO9KQH000080H&bGI*MgI(|wQt0Pr0F01f~E08wmVb8~5HUsOUabaZCy zy?LA@MVU7q5qaO0m06W_SJvHKUENh(-Cfo9HQjUH1H%l*a1Q4TH;5M^iU%U#feN0i zB7(Z2;_HYgq9U&Ax*m8dBI1h2x+-|X)cZtaRZq`=``gd&^ZvO*=Tli(RT=R-@f_dh ziHHPYgir_wkI=g9+jf2LW50Mk!l@?^!s@nN_2!3ub^U%W*@qDl zpF~J_?979wUp)QD+@BEAzl@N)^~@_T_mK^nF7>3W^jL=`^&)t9eS^qWhid*3`9lqZ>7cPje;ywlMm*9Qz+=G{2m3?3R9(YgN zLl^A7^g`iD;VOjfJ#ejj;G#26-}jWi7Gd)dXztpBr(bn3w-f&nK06nF?_YTO!Tn#o z|K{H!9KHsAFI{}mWtR{C$Nm<=9`GaVz4+4o7au}s0Qz$Sq7tC8z-V#J)B2OATGM|- z3il-C=1V*Gd?0xJMXU3f4&v_P-UM$IL{RV#en)=W*^M&XkKyxW?ml*>RsZljy#n;* zNb3Cp{;BXgkPAEK@V)qHU=or0$@TF5hTs+d7&Xz~WJ{2G_qqm=k8DT&#@_pjUwA3I z!{14_4{yTAM>xWo`7n71UVLcgXWg&=EI9D}CdHx2_?{{Q4Wuz88`~~^Jo}9 zg&OFSNWn!^fY%bA70DoKkZHK*ooJlQ zqda~HK6@4P`zy#GeW;CpjSP-MQT%(P;jh8*E$HLf=mNYJ#o;xDKMB&Ne7z2?KZM5d z#mL7G!*L@V5@`J$NCIBl=qqRxeXa8ud;@Bse?e_>C#vHw!trU=Rs(+-zF&j)C&K#~ zXzyV(3Kn6LQ`tQ%{A#4)^FS6&bP0M84FI1T@SmZ-_ahCx1=`;Y#~0yq~t2dZ*B$n+AFhI_a1*HMYw3;q5*jM<|wPIOFefHvssQ)mlW z1Ub7%eZGT^6@EF4)2ai;sf7N$^E)`6fa4F{G4i`du6ItyiH_0ggN;#doL=`l$Lry^ z0)BhL(FgD`guc@GBOJei$`vo4`iO%x(g?Pf(dNnOuT$OlLtJ@V-fs$iN?k@jnUlgxVO@b$Y-# z`6lpAZF~@H4GwBwqBrKdYld4 ze+K%)gOB+*yr(wzMKr?fj@s*iZu<{|%&DzC%>2d#IT6}E8P4B}A|Uf|kjXgc_XNmt zg0*)kdK;dEdzC;|4}$(spNr`~q@%yc9V8>U^DOvq1*nQtpc>ME>Oh}C2GW6=$N*{~ z6KDuooxdX+g@A^U4b(wlpe}Mce?t-E0*#^wP!C0c#(@46#gPXzfnq?DC=N7*5}m(* zZ6<+cPzq=krGfe=)A=*>H48M4e4quC16l<7C$PCZ&@w6jt)L>%K2!pW=tpIsRa5~w zfciR5qZ;Z5I*6)3htL4fI;wT3H>Y$M4FYYVA)q6u-uVM)Q3L208V1@zO`vTw()m3a zN25R|&=}B3)B-vM^mk|)wSmr{aiFtk0_Yr??EDv+M^iu-&@|9RGy`-En(h1+EulG} z%V-|xTC@Ok9nk+o>(L_64QLI}jc5tzCbZo74cd&>0^Nev0o{t$1Kox;bbgJtqm4j! zpiMw`qRl{e0sR#^4s8K?JlYC$H`>yMjrMhZj?O`+0zDU< z2J}31I?(fhK7lSkX8=8b&IEc8odxtlw7>H+bP+lm=*8$9pqHR?fnJKv>pYGwL+1m% z99;nP3UmPIl|X-rUVsh)y$W3j^o8gmpf5rfcOFA8MwbA63Az;MOVMROuSS=5eu7?x zt^j%sx)SKi(F;0{qH7s_1-c68E71#qUWZ=P`7wGGdNI&fqn7}E4SFfi*P^RCKSI}| zmjQhpx(4VC=;c6PkFM?f5Z%b=P3RRsZ$_^K`UZ4e=LhJG=v6@9gkBBw&FD2iZ$Yo^ zd>>-n^+4Z>UI+AT=mwy-0{tGk4ZR-d+tH0c-+^ud`c8Cn=ey`#=nX*Mjot|KJ?Kq9 zZ%1$LdUHZ|C38L+BGgzl8n|(1+0{fqoh2R{?Dt0{Rv7DWLy~?gRQ&^y$vO zqJKm81N|EM3{YqX=-1H$ov)y8pw9vQCVCL)x6tQ1|AHQ2^xNnQK)-{&2=u$?q0X1l z_t2Msejhyy^atq6K!1q-rSmZQ5k!rL(T~x;0(}&H73fdUzjeNZ9%J;UfP=n-9*0Qt zB|wQ^?>vN_K;HoRbM#H1|AD>*^cU!n&KJ>>=-WVliM|8$SLnMye~rG^`2zY4qyLG% z5A?U_2SEP|{jl?S^gHw;pua~y2Koo|DA1?SPdX2xr_p0T{|K@8LG&l|IM6?%pLITm z{=(>A(Gx)bhJFt8@8~}|51?nzFMvLao&PG=wWO?dGPH;@abjn<$d78tKhq9;IoIoS2w^%H^DcLf=_ONFCGUUJPE#c z8gyj_e90Vm+6C~iYrwNEgGXHlo^%6v&`sbuw}8jo2A*;Uc*tGg8IK2#cmjCB2f+`1 z1@sGny}u4@`$n+TTfy$$2exzvSk+x%O?QL+{4-e3hrwPx2G;UXu#bDe7Cr%1aR@Bn zGob(XgNA(%{OGqq_s#*`x(0mAtH9@63|{O9;HzE)I`$*btw+JXLPiE!fIzz*1AY1# zXwXZ*UtI{k_zBRXpTZ~9=UxE*5`q7@0ldSD!ISL=zjrG5(^-hm*Mg2c0KVX4_{Qnr zlg|WS{9W+D=YrQg2>$CP@MJfGM|vZ8qBnyFdMo&xZ-TFRC-{~3fQP&U{Om75H!lP4 zdL`)PufenO9L-O!0vQS*Sp?%&0ol$%e@}znUkyIz4v@h|f#XjDAJ3u=9>$A!7d{$p4S&*K@;{Wz=gN68AI>N8 znfyThwEX_kSN_uZJLzG3)=V@&;@FB zpT!E6nSHziBGP*y)?5O*9BB5hLCb#wKK9SxZT}9Q@)_`%f5B<=Cy?GB@DzFqGS;WT zzf#MhR&_43uk*l?zY8|}pD+^tYZL!#6aQ-yN81E6hiaP+A!YlXk79h|-b1)^-66D; z{uD%f?$mvQhY+s${_=TC_u$ju4XMF}Y7Wl0n!olQuC#XNp2A-LdjI+jXI<~F_0K*1 ztb6zpdxa17U%$8R--CATIS=S@dvf>8?@g|p?%%t267Iv(`|#{O*YAZpoZr2}`RopG zgJqo_jvGl+5kj3*Qn~dJ3;EJulYn zvjlw>t8gL*ciFY)p84cGXz%sc2k#2Gd#<_u`sDS%NB8|9^tod$V07$-`R)af1id3y zT0VrY+0H(?rjSe03x!-E2mRc;1Z^v|O}q9iLr-&i2NA2qQeQ*>G3~^+l0HB$rvjcj zgx1y%p&f7};3(A_!|l;_d2F=R9+}1M!Wf*+lJ-ceJc~!75m%H}pQf?ocqfLcUy?1*4!}fcyDaP`PPA64vmXz~e;@Yv~-X@uv{1WpbRxO=3ae zCwZBNFYr8G3cjEVr-BDGRM|5HS=a*~@8c!mUoAo6i6>^JGLFP!MN>$qoQ?2OwCPzT zKgvnoa77Dv<5$$4PYlF^Ki%$II{b(7 zLOmJohqtGODr;^K6;1F?x$;%lUfO5NqAoi-55D}`7dK;{of)5)T^O6j{LJ{|?EKgi zvxP+GucV*c3f%97eBuzQfCTDrEY$A;-+wI^`ugAu3DYR;O#^p zm^VO{nxN_Bv88IGDVvVQ3*lU9sGO+Jm9y=^vNdpgEjLzly|7r&Oubx9Z$18mVtIPF zFD%n}5R7v8ujp3teaNjWkO&9T5a6(>UV!1#5s$c%$w{KbY`aY@pe<&@qG+3#Xdzdz zLb5H`VJu>fkbA8tA?c{hyM3js8sY>oyOblm?9#SoI?Fgp{s7q=)jtQ?k!d?rQXtDA zZUzUY2Jc=UDG#TGSSUQ%?k}}w;~|%u-hzu8u060j70W0~SHAxOy#L)-ZVq$l$nw|V z-i6LG`uobg#U2l%w9_LQE{*Bf!N|_y8T`SGvqH?WC&%H|GjZD`6WegPtWzvIO_!d%{6{N{}UwVz=;iQhxs2@%VI2w#D`Qi3)ldVdqwaTzQli(46DVX<8C zqR>#K+-gg$s3(b_C%};eQUl=~CYs}5ON&G=S9A&E>lBroA&MbSUE~FOt!(IvLddp- zGi6@AUg1SUBH#vv*Qv64Ig|WhDYYLAsu|U7lc*GS4a0K9x9*G<8SF9T`7E=kCf4^+M z{xmBj;d=`Hu5iVYILur2wfWp@85}}N=Vrhp2O&R~*jPd&p<0sf>Btz>p6DBi5lw~k zoaUy9j5$pLJQ-<5YSn@+tNthi2+dclXd<|00eu@I+=SepMk)jPZFTig4(?CmON&O9 zD5%Jf?CL-@ENY_UR8K2t8nJ)Q2?FTgGOr0lNcFLMmO6g|1a*K_7XV)L?llUXlc)ig zOH+9VW3Q0*nvSPm!J&3;4Bd5j6(3~@r_ z5@C*DjFp5T%Cab&^w^c0sw!9tDVD8-Lpl$BtoqU$-x!R4r-N^UKF)xZ9YQ*bPlr%H z95wLJY1EH&=Aj)XsIghD7pB;_aDW8BWsd}71e(Hh99%FBQzqFU6A=J}0&X>fYty9= z#zH<}<|>VJZbP}SXhMt)w+d^nXz+x{+UV|?++fuy=TrFSq7k-vD-qUW{o}r9h@}lh zx34i-#=Ku9gu|SwSy5~0oSo~6wVJG%l$ToP0emy?vIQ(8koPk1fB+*eGaeYq0E5gk zV*w=?^(e-eC6)4M03-r$#TXbO^gZPYKAOR^T%a$Kh?V(u<%K;XeGN?&K-I!6zcf{G zLE_@p3E7ch95_*k50@b9WTJ)XC=m$g=S z5#Tu1jjv7Myak^`fb;txBM#1}6}HA`Af=27RNtCWfKm~_nbc?F2#lndAHcx?7o658 zRY;K1EbidU%y>Oy=tR!+4G#8)%%Z7Az@xe)oj?q9L`5(ZTPxTCCf;CwUiUXvK_{)E zgF$2UjOom-+qBfOj1b{Th*G-*IZrQ#u4$7L~!Rr+^ z_JP$Aj)>9HSV;*hpr>bDj3Y`cstH)r;sevg_5{QX&=P;^mbEdexMM2gE8Tet?Cx!l z!BM`bUzi5Is%RQ%!8lE0=I}u6V1T4@U~d6L>TyFAl&Ud9iCkB;(?sac$J?OW$$mjt zJXrLyGAGWDMk^V+FT=}%nbK4(meGX#aMtu18=A-M|D}}+A9qd7uf|o)vBP|RIK88l z7)zhE&#g?9#>YVzBIkw^qK3JAb1v6BekOlq;Ik{8uj4n7Q&FCI*ib!)OMt@KU57wB z2t$yg-4~?1IE2@DvkP-xYR8m6XcJkqYjG@t;%k+m;aWk_GYLC8bIaRq8pm*%7qzS@ zkdSYd_g{S3c_$<>M==yf=N0kW$&IK6659+>EU@=996rb~3^t!;nm^DzTV?$eKZ4NO zD>(&-cqDMf(8*}Uje1dU7I%G^(-n6)0H}C(t{88u9c-;1bZjd-?&Xb8C_Xq}mJO-Y zD%xSwD8~Fx+hH)$`dBHPi--0)rL^t1v1Z@I9B!l*221meWOjHqn;mxh3ueZ0hGON} zff(lE{bQNTP}t7ck&k%3?Z&;y;^2T5$sZmy;^l;2bYiL4Vs&&PkbSE2S$qq~ewoRB z7;hOFFlBmV|)2+#V zU(`iC2I0d{5;mw$u9>&v!y6hS%T>L4GJvR=NSpBwUZ#vC+q335knnFdTvok;g4o(&37yT2i{$7cMn@Q*+{b zGB)`1Y$Ad$_QxXN_+VheZZw=u+mQWE_yu1#aU!WZy66^$Q}JrKaQGL=WFnu+j0b%l z=$t`5PR>Rbp_j2fOVDQ*`YccgDFuj&LNJacZ`33!EElOTTn5P>_%m>St+JGdZ-beR z2K-T12>-bek^V`b%PYRUoBfS2AylIDK69Aqek6lQ-5^>-JM;K8m+TmTC?m8_zIyX( zZp)1tKh4qHn4Kq@PG7jUg)dr6yU$tb5?xSKN}F z**n=cn{o!Z@nm!clW5zG%O>wvPr7NpY{@6S=A`-?Z#m)eT>Kk`$ z3jhMKgmR78qUcB*SO{+35AdC4)c(4i)rbz!-^4r@nVfFff*@#u zYzn4eSNh7KNXEdtk~fs_-|x`EYRHj{RK)1tI?~*mmn5E3ZF%m!ciz^DfN(fzZi|9-#^v)JHDOVii&6yt!2Jn>T;U{E+Z(1RL0v__YACM z22+JPg8G@K?*&a@qg!m6W(c8*rLk7C-K6#uTmh4s!eS&cl^I@Ji)6!6NRNf}q!-Fh z?HDT0miLH7ZFm_=^TV1iNo6w>cCuI)tth@gl=BpU0!6}meeLYnlm+iZRf^>5{>09) z1TZ*9xb<5_RcKepA~39s)&!Za0`K7C^PS)0&yYJHhoF{3u{ol75%t@&J|fb<6m;3- zg8B(0YJrodagJt8quEA2!pR~pm`WreTM%l6!t7A$v^~W&HegfNobR7GeYv!||1@%k zX4#S*m$AsJJXN>+(z)wj`Qoq!F3{Fpf6LYTK7UHh5A1vz#^FC<9Ac=C^^RtZVdm$! z6*05?(H?DK8bS+s$qr0*z2aA@&aMOV@&0DCF5*L8GBQyd=-fRuVvTHM#H!XL(THh1{Y)9mT{Rg6G;lcFY`#4e2 zl{c*ezktgrkPk;RN5enK&1Q&>iwS|lUyY?K;preg(|(-?{mL-uDBZF1@*t1$b6a<* zcRV|30Rl}qf8`WSOL^BkKm^rP-U^;4!O6tUjm4m$QjF*D_dRH6=NpzufG2&>lOK@x zp#@e`D%5+~0O-mnHDl6b6`4Q=RZwbfY$Xd`(WIUi1tp?M0$z`aq!mhe4dj58lo_2a z$+kSUXRgpH=1-qnrB{`qlh4VFEEbakvf|nX3E)K?$W-|pQw;*#D%40T@Sb$@|KJ9L`iI!D@ zAOV5VX0pWB%l2^fT0s`Gebjl&g_6ksR+WABPAYQ{J`IcyhvS+|iYK)2hWduI%qz*n zcfV2aB(8YE*x`E{YvB^~YohZkx&`_b)OXC4u>>iO(eT82!D^8-=D?qcmlWELSSg7d zeJyj&#GNNkTr}9lX2j>bz&kN<4A+^uZ02 z16G)Zo~W)H0*4rnf5YE^J}g3f2>hwdB6rXiItGJvTC=|onSgOz=3}S3XCRBAqeg=w z&?ZIJQCiGowStV`v2h!~Gs?6+JQWo5eB{8Gpo*#^$0c47oZ8S(Bvv1&htqE4fZq#zL8sT@yc6T}XgKO(cG^?(OSA(1v7)mXMg9n@M?#8#c@hDFV+C zo~y^AqpeXdk`umIosP*|V52au_*F;26<%nZ)D}V zNvDut$a<42vyh`^~rJS%&NltDxSxi5q;L5F-^ zj@KiE=gn#uunp+CtZ+i50=Xmn1XJHh|KzX}mh6Hp*lJ8Os!Ns@|gh_Kze6Znr>HydsY@^p9Qc8tveMr@WBH}W-~z3wgKGtkOI=Y8EZN-N&iKQVaoGu} ziWn|u+;FBpN^y-UMqM9d`XbSiMG&($ysh2ZP>*s%wk=+?3X=oHX4W8w@4qN+N?dLK zbbxj8om0r)K^84QoDvGWvcSxU)(lyBhQ?GDR#!U9?2tN-Ag>H$RtbVE`D84o(Qzy# z{q42d4nJK>n-e=mVz0b<{?wUFvbk;mi@F<9Y&9{SO%8inT*Hl$68U}=?3xY^qa@~#V&tAyjSHEG+ znTt8f6ZQAs0sJ9y1j(c-A!DnH-Ans=#r$`%l<&YJPS&%lDnM zV+JoO^V2gk1a~m*)lGg6KK~BLm#9`}>p?7cSfTBC=9JldoVdD}_1tOc?@Z52mLQ6{ z*}rN!+03TS*wM? z`QQ)+JMUAj@f<90MB)TZwySB}EDW6s}N5YaI z$cmQUe&VSE=iTTk3gP5zZt*R^&Tx+9}Aq~C6b10=?Z{Z(~KINDDq~>gO9<3IQ>yNiFFMt${6X! zZRd=g+-U3_J#go;q(W{4;6n&ondRE-8(y*Bku*X?kT?9-xL`6qH8-E}rWfW@QmHXk zF9@;Ol>nHxATm&cLwBE zXhDpPBAC(>0IB+2)ltoiZI{BKiW|=9f+|TVQ&xrY8BJ%f)@N}-s6Q^MvK~?GT*xlC z21N*#28kS>IIgj{8*l)Y2rr~3{ldmvdQ;h-ib*`CxNY5s_mp?-*p%T!3eRja(&v$5uO3w-o->WHH*de}cC;d+2S7(hQLc-td-%3i59V-2=^FO{Idw~W0WPBau(W_H zlxf)}kV_rYNp#2{@I|tz$Wa@HRO=t?Qv?Z+p&Cv{WysW>)M5hLnWh+xCu5F4-T)zi zKuQ*2kyD2D%IFW8hyuM{Yw?sE&d@ zQ#?Xz3VuDn<$bKqR6=5RG=no%;iV=<=Q~&$CQr2A4Qzo1NjQ`~pl_I1f$q@!dX3?WCSf;nCh zy?E_~uY2=af_ZE>W{lbee4@#~DJODk-gMJ-ZBZnYw+lK~;Eh}c`mA-wB^Ts5Ue_N` z;fs+JiAI>`fS5>87dV1Zmg>%SOSeByjKt)STeggd2CiDQ%dw0@cuAZ&Kidt$Jc;K^ zT!z=hq)A~v5rpiTFTHmDl%?T_pJW&8h%SP@JsTNUxEG(GH5axr0U&Vo(>C(m6R~eV?$M@SDUQv?25sf7N(~!2MI_vNB{yC zlR(AGII=c#L1s1ouj4f-VF9lgYm1M({eeQG$9vx8dYt3>#aCW2V$fn7V;%Y^b>4;V z>HHA3m zDP|coLrozmu!ZiV#3XfD6bM8_IBAP4kIt7XT`$*N%|&YkUKcCUn!nQ;ox$Zin}er? zO)r4a6PSc{Hg(k)uhJ^Iu46IjhV$UOWxMFc1`Cnm=%gnbl4FMa#6&c#N#cV>YkY@j~n1*PI^lW92Lf8$=57gYnK&_)hW#DJYtZdu>mw^^0(185 z)pHrr8d?#N`?CTUnJVU|Yw=LB;AGck`WMGlKO7k>dgH_HfNkn28}hWFu&rv@{CyUP zXQ-Ovi|tx}votAF{RI!h;&g8E;u6reXi zy-FcIMS@6b%IT~k)Mui_p;&w{M0l(#WGj2EIc4Mf)Re{(1M|C2Y%P^d zO_nV|i%P2QjPBdC{j6jvqUdHIJ6gNE17tUZmRN4ZR>!PT7Lh57xT5enC&#hwm9Zh(*V(tsiMDRBX$jfyN6lexAaYft+h- zo=G}drxiU~#iMfdR>FI^JNX;z zNQa1KO|&fOMrgS`2qj{^kitFg3P++yd5jgnTPxFm5RRD%> z(Z?Hv`d~qmGBerPYnHHm;VDJ8|*mI@K)>VS{>eT?am87wbyTF<`$dyG1RO1ER~gs=x;xMs-(O^bOENf9MZN|!1jP9xzMPbJMkI<6I>vSDU*cWtA(FzD%! zy$cX41xvL(Ox%Ip?TO~({Mc|S@K0^fwbz3mXt9;TYB%2JnHNP}FY3)6TV)PBQyQn( zBHg-jLtB@eX5RwmrPr=|SEAIdwBAC*rS`e>oyhUB})+1xtl zXXi7WY4mnzs|=a9gO1E{(Ta8dQEhhT2D;E?q#2}s&+W$z0>DGV$WDm55Y;mMt#U&_LNyU8(%q_?VeGzGmA&*XxH&`gJy>FqHGI5xtpGL3)Q{VhoZ#B z9n0m6D5hrfao>@1wL#r#O^@5MuGx+iu|+qLG6dVPc>-yWU6}ByStn%MgDz*Zroo9_ zm>6_ZVO`CZ@AaH;Djyn{tHpVBdfi&^k-VT9V$9E3g5!pHqH=H00nYwSP3J37+e@bc zoqig95Og}tP(hsK5%(bhbR{03f}=qZEt0I7=ecPmhA9qsJUS#az8J z6nzC!>>5!axKd%O&FJzdc%AGeqroW`*dU!apo2z#oW}TCkgw5C(&f}Z2<%L?<1Us2 zF;DY)E?fv@a=aqs`;BnP&SU|o<*IRoDA_8dH1T$3!*o;SiQBnE&~n|TM4X8*4Z8+x^@}H&xHKBH@9w3TokPG z3nGTFvTnJ|mM!CVrAOCRPdO_+x_;oZhks}#%Tc$O47CP{TTF$1JGXDnzx-NSUH;SP zx&g--SUY;gPG?|=ty$_c@dxqGKwGBSiVBM7gDiq#dJd(SZC1NyV4JhCAND8-)09!kQZx+%v0B0-;oExGOWTwTWmb8nr_2h6l+4jihM`@4`bnW_K*d&BKqI;V!Oh;9FtLefs1O zbfiiuuvNhVN;01C%9+|d11XPDJz!d}8Ui!kfb$3hZg4jYbFw8D`1Mw55aVz=o1Ys_ z*UH{xL2;A~%Yw9>V|R77fFDBx3>9nC!{?7E=vez01*N%q*CUqmj}2!yh-z$-h%T@6K{*ZH zF3<%=6<$(!K`=x?!8iJcA2b{p@=nmqsk=bHirJN~K=_E3GifcMS2uqO&x1}*vmCQm z)5&&g=Rpa9Cd?~~4!t7_uIYBSW)BJB00{vM0=eEVTjF3p2hc8RcmU?iuruZP$;8l1 zzP?tLWnK$~DiJ{ul3CYtYj&)59DW?|MM_bVG5`uC?HMuC>UR<&eqpo_jyZZtl_dqf z;>Yc*ZRX6#z{uo6z(W-5s!Dd#b+)Ki)1hneF~a??M6NmlI7Dbx{q4qape|HzT^y@FqcnmLyG4 z)J$VK^JhWOHId^KSvb-1NMSBpW)+;mvY(rbSI2l!3i>|Uc^bb3KLN3l`Y>9{ru)fS zpv?u;k6M8b15PCB7NV`Ty{ZgP3xz$zYO=+4N7UygK~z-;bBSStm%0;+$M~~a3;dZX zi%Oa2?GXOF-CzD>z8X_VB!p-Gk!K^ZftkK5hy+SEhIZ_bizQSvkt!S<+SxU?Sy-!UR<(SIxMx3+z z(wr4Cuu$j|6;84>*)p}W)W+P}TxGp4Kn@|RQR}1!FOekzdF=FcCn!3AP@Qz*@1t6&SZ1-bS{C$)Y*ElCVjbnq`tWmj^lxsBoe!8B*s?G_I1zb&Kkzg zP;VgwtJFt(#a*sV!|m+zP2;oKMXwo&_G+cwy9C8!PWq)F%3{iuAvD>JtJ)Dk=&PuL zWm!=fSR#h0gK~wUQBf3mIihKfpvO(sq>CLP61z@_B6MC$rqjmCtspL&p2q8zEu}*t zS_~GGMtpwlhQi!tO;eTV^xXP`>G0+;WpsK}Ap%x0Eno=YP-%Cgu{Bb7p2gcmi%xnr|ZGqpr4fJG|A;(-j$TxfbCa7x#V=xMG*63<* zCf&a78Q3#LQ>zQ=RBc$U#ZfFdg^w00RkA>m8JxxVHaTlMDLJf$ie|XsL$agAG$p&X zWabMwofC3qPH_w~Z)!E$fzYFZ6_BJxjhXHa?n`w!l~H! z#F(pas+^6)r#+|XSt(1Ecuq9Lf&Gn%t0r`2<7#IDUkFInBas|Y0#Uvq)x|L)l|EB?qyev zkFBeQT|-s`y4XVya=8EuV4FVG{}sEB{SL;9-7dOW>#&uMkgV-Sc;t3{Y_mZ6%3-6x;ik5w`r z_~lDFS@a-hf+qH23f=qXQPgbHWgrgy1UDyS6Oo~6GbiXKwqADqNNRav2Nhe#4#fBVB)G7&@kC`FY6C*?Acy_4m(F>L2np(33Ue-yt=z!be4)9)M+swkAiL_|vZ0g9E{PUgf;O~-KfD4LC z%&yI(SWTa+9KDUYkY5KGm5TMTwc9o?1^b~50^wlumJJz8ApJy=Rl2|Vx@#}%3(JD3 zIy(+tbL~r-@nEcb^ID56hSB&%t9!k9zc!l}YIk!xhixlcJ$dCkXQrzi6)k*Zx=Sht zOV>XQpzYY%tXCX8@x_Vw0FlCbyR%;x()krl+KG#VBgf8b;l$AsVWKN($ud#SSeXsG zPjiYQUH-%75}rX~+thQ>)K*J^MhN>EZLuS_E~A=bO#I z*_G)GxEGaWiFs$*?@B$$DQ4;c zW?@Upd8xOuoU4?db5&vyKd08}Rav$)$%-apHsN#SjHFqLlGgyHFNgEK8nAD*1?LRx5MVG_Pz7`&V< z_2*-pP@in+oM~#2>4Zy& zlSsuZ+G(X&9kA8k+w*H@O{_tpA?Nz45FofjaiHSMfFpHGH`~R+w9P(1K~FumO}g`y zIuyP=Bm@tD;JoNl+F)}~g%nW>#nTA~+KfS)nq_iI0oqiawC9(hO;Hcgg}wc(&C)<$ zPL@qg3fi=Lb?n!ZhtVE3a>`M$CfFNxq*|9Z@qb^phQ9z>w}Ls+ZV^1cx(ME$#Q3^y zna{2D`|@)u*b1bAK?ytE2!~7|D1i%6&8#^=3H(k-mya$A{s!ZJRL_Q7S|HtmHZeqo z?}Dnh1ktQp6emm;Pvbj57N@Yh@&wBN=>MVnh5C z$kFvL63z`rIR1ZF-#V8L$ky6HRaes#Weh_^Dy%SIT<2~<#D|_BMtM|M&(8tkN@d@2_sW)VK zK3zl@4G>+UGEh%KLKgOcZ7lZn_m!QjXY>WXw>ob|H=!3m-sG(ErnDj)W_FCb&yxsN z@Y0w*OZU2!2gIx+01h{!37r>tNQ?|O0&Y>*X?le7Bi?YWsL(b1s=w?L-le|V?%aY0 zpe+xLu=$bRYUO0tlISWcn$1-i=dE7;#u}$Bx7xuf`RD7<5dIfKg{YxhZkYB-(?dp? z={ z9Xhpg?DIRYAiL_32YVF#qaN%*z=!JipCA?$ARnb8Pvt+?<%!M(l#cbLbWu3N%Dt=H zijT$rqzyEx3W4AN(Iyrq#E!+IrUu!+llJxTPr zJ=}fd-N*(_X`nUeRCEElirFU3>zAlxyZdHWo*l93xjg?5AL~Qz4j0A>>?uBLG9yKI zgYTycel}BhcH-Cv{%k+yfj>X_ZyxyLuRQSQqbm>mx&5dI{EYOT<0Je(dkhd;H?k2@ z>z_lO(Pq*dsdt}+#fpJ5N0xvE*+sXmN>?Fp-TJF@Bn>*{zum?m1am4{fK-IMOjbvZY*#p1Y%2C*pM<1|7p>$lr+eYV;bf&d0)aY zQkAmBf<|~%*2CFq%_5xE%^l7s6-XMt-QC^1a$l++EFI{g+V9F?3!>apI7rolDppXj7{YgXCM1ofE@Nw`6iJv8AWWM0 z37RTlVtKw3kD0Pc&KGi;q{w)1IA%&pB&^D+N-RHNCrw!lWPC1}MGun?9TSIF?f5>o znFb)@W?~MxvXw|A{3O%QHfiG~xt;kQXO-s(yv|W~SnY9|DY%)Fj=Aq#IU@&0M(| z0|+aPF)#O}6UCSTX``faW(?wv!Yke911*pou8@BP?ds{5)77se=$8TC>FzxWDzeR$ zvf4ZPiZPI5i{avJ7Q+C})F4@vYm7sdTuj700k z9E)(ckPFrW5ya4!Ik1PwQeVE~t_h5B!hS`78Nn@)pS4f?Yk$`jHAW z%{%ETgY|$6nyk*Wv3tg51}Gf%R;q|WO|8JP_wM`PK`QXa9&&*PXmxQUc;(>A2oU^j zdnoBI$3dGzZmT@9DNdJ1i{!3|?}a2z4QZwdT{LOHu$(2wdf~3og3g%s$=K1QF+x0^!Ea8LOtJ zufkgz+sZiY%}|+esM>jql*zpi=ca(;O1CaVy0)8OYjg=pFp11{&p^M#m0gEJxcm5# zU_!Xf>Zr6D*PHb*vExQPhb_+%AjhB&45G6p=LnK2Ng9#TW@7cVzG_<=pZGlOxAv8y6lpt6WJgoX_S6pC-Q} z{{*OTE5n6eR~KABxp~HUseAVPx-hnSevtAH`NA<9drPaHF2&ZV5ziX*{Pk(NX;AMg zw037e8PX?>FYTL3jy3&__4e^8F>e0`G9@h=yW-PCQViuyhtH$is;C%Z0~5L2)IfYa zO?kRnAgDsUS{XXw>g@;LP))>(r8?1YA=%rJEUi6jE|Xb2Z{wD8<`Vw$`5Wl?r#nxP z9r$q=`x61>Q2U;$2lE+TH-5|j?^C^sI^}p}b>I;p1eMC}3Z}7On>Q`QcAXEngkVNE zSYRwPsn(S9Y{xq2NxK^l+TAH^BIqeAyO6N6wNzmuOxL@mOmjHhw?3NKG1Zt)49ym- zBp8&mkf=M3Dao9@XL48ETT?Byi?rY)_aA=CXpZCA{H9QD|1>$BFkD$7**`}<Cf{n_z7&Y)cFDaApR4oqA~DU zS=3vvaUW!w0;@05(Ghy9NFb6PF-e9mAuA6?X?6PUdh9DOK*3{eD$H81fY7UQT9j~! zP$V)twqtF-Waug}B1DKAkx1V2maFASB~;#2)_r$i`aHLk3WdUbeU*@rcjZV}BVj|T z#?re+U%D_=P+VIjj;Yq%^!CQ$NohY9dUDL4ymBgR9)I{rXJG9}yPTezor-%F<2BX! zJ$W1cDH;dgd@>qFJ^c%+YeQ`PR~l6q&xw_@f$kYyE1Lhuy4iaSLU(z>YE4{alJHxm ziK`TG^YbfWZEMooG&HnsIKCWTDBDquTneTH3){PA>RUKzwX?SdHE&K$> zbD~2PZZwFCh0fFXQSwp9m?&bQcDtq?>`&NN51!{S6zosyO}R76q(M0nb-kWau!2r^ zi79nn)2k-Mx=%&oo->`_V!Nf-1o*YY((Fn&k{<0>%z|&4f+m*B0^N&8LSPz>0%YGS(nvEXM4tr@h?H7;f2ihqL9WgqjTH z`sp3Tndy;<6W_bTifbYd>9ZS7Po$?`I7b%<$bw`@BkLlT0S2C^G?M9}rn%9$1(07$ znz6}palGtCjhL;Ars^aET#&}^#fQjS(N1&`+R?S#-Y&7NaBQ#B3fUoaVm+u8(8{4+ zKT@xzJGIZP2fMUs9b`XfYa6nh?o;!aJS$@@`cRu==5j~dO?s6sB=EW$_R6D`GJ6O~ zxB52F-81a62?IBn{SLOw)5!w5R(OovY>F@^^C+hZ7^`wf;3ZMe1zw?Bn-oVDI8F|U zbbl{TxRA@Um2ra3OQtLnTd-wL6ENTeQRcP1STJ0!Ey(O!JmK%+=+2Wn8W+=RY(mhs ztc<0lcrK-R^(Zah2}XV~-zf7$Rzq1^jT=Txu?lLq;&RqtM9!Oh)=mw_6p6!Mdn=3{ zk0tFhSQ8|k=X71N}}zXM3NHzP*tSdnfWD+e%Hdfq2NHLzmh7b~Hkjnb^o z@e}UK$uH>hny&&R9?K^1*5DP<|65)TpNuac^#5@dW6$U9;c{iA`gRN~XIq^oyZZn$ zbQ3o#UIi$i03XvqVG|>*k(S3`d?7*|N^bAAZM#m`w*9!qOlxFmx`oFU_sWJeV*|<+ zL2x|g6rK;sA`dTyO^cR<@H3VP($--tiGKV^C+yjA+6jA?=hrOFFD+y+ZaZ;F=}!`# zZKhB7D!-1y_tQ7yMhMSwa<{hl3y>#$0$OiR2&xBrM7yB5kPAe|9zg}7%IDedwxFH( z1Nc7pJ@qS8>qA{l?_+r|?wWrW=m)io-ZS`u?HiR|{TE`=!`bO9uZEn=OeWDxh#^7J zWLxkT{8T$7NSv&M!G*Q5rX)M8qAtU0?`^mAS2yxnx3Tq1K$+V!+XV~Nx ze19RQ>ar#K^Po3HbUR6tPeO0YV8gvR1vX!`QmY9b{MPP0KCkQcOK$He(iMOt6pwWC zQ^rWwJu+ldEU!6rV*KQVLTT}o@rhH`6sOKTvAS*~ofuwQtF0eP;{>EDoG3!dAsX`k zrS8gO>?*JO{l2@eZ-4jA+qZe!EN|8~i)ZoJ#ybU!V~P!!V1o%zT0$t4s!EfBL{ha$ znv?_;rB0i?xC;O{$Psmy@xzebLdWM7)H)zJC5dF~9%(#@e}qqxl2p z_FZ@=y?E2?^o>jD)WVIkzbS=90y+b4wZ5pU<5`6l4c^oQb?>p-QqgY`J<&Gb#|nRz zVm^I{-k%?>d!s!a4Y+7e>}kn)x(2i-Nc5=9;YmS}6hYBfhlYPjiYgcYMe!%DbovN8tbCAf+21^Y&5M+}M0N zMKU%FX~dPE zrMXvz78_r0E9$-f(%|B1p0yKHFV9h}Y_)OZyq$Ek@<>xFk2JON$hB!@A7$QaP*Goh z72+YZ_N87mWrF=OPBHl6&ejTsFxREh8X5qe?}uJh*;D;P_H{1c;=k`3%nvgRJ%}mB zoZgl#1if!@OZ7oe^x6W1K2i1Z-oZdmus!s-&c5H}>pbvvobokZ9~K?3t0V7|$BkZ2 zyC=7}+RI1t4$p^x=LY09cFWtG9QciTJKR25KH}kAdvb{`x89*Jo*v;65OsuFJn4jc zJJ;$b3HWW_T)s!$qKezxqI#h}PVv7`A1L!`(+3I)q_?Jis4noLi}n+g~FRZ0UG}u^eO3}Cy_cxPb9>ys?$oZ(o07= zQ_f%Lp&b0?+d%(+^49_VRgkhf$C7nUQFlqt(HSQvnAlb$XyxF^xQ$|V+_p!akH}A;b=M?wo@Y`&hOOcZ2k=Be1V?#MJi`<-jkl^JW9hsz8mGACB1Sq;xn5^TjL?% zyP<`0=Ln19))7_|7R9AHqgS%|v0tE{K>Tr>d$OAchY98N5 z@NsRxdaL~fAiIXYhd$mbF5wAo0#5RGQH%-JRYAO)O-@M{guaDVgYk(DkbNh;4M!&= zzglNE+@$parFEUsYS*3Fv2L_>GaQ}hexjEF>BPbJx>^r`;UR)wB|KCev_ir~jz-zt zIVpSk&`xNHiigweMD>F^U1d|&zX9=T)j^FX)?=pbl}%E7AHECW=%`1!sEFVe|Dgl% z_=jLbMl_lW`edCL5zsAggIDkDgaPT&e1gjMIx1O$HPPtjJonc9PABDPpGw_9@)x?Q zC}uc={DWMUyMsnyq$qm?A4Zr%nyfWh#8(Bt!Ex>&cUqL2SyWYuT1s{FQ|DNx&cuK@#nkc zA&kd!z~hM~k26E?*tMG2C%=1p8OQ-S=WD%l62`wcd*cc=jN$-Y-cG9btJ z&DQ*c@%IqKYu7g|?);X;UBlbMHK3OtjAOJ!t+0kcy(C_7LpWJfOZ_p8e?TKUNf1m+ zYesgWp%_(5?g5%HMS|F%}jKEK!2}Esvt=^ zQ%cKK%0CzY{qH4gC}7A{y+t8c2Sx?t9exji*Ix%KMdq&wmd3vN7M9@xi)RO6vBNnY z0O9aSkFz{5>LGfzC4`{;x>wM5`Hz6_Iz!RmTYc9?eX9mbFp(J+`7O><55Fd}8}Rf% zFp2YT_i||883IP}^)3$W(SacrnXRK3dH79=k@p}yE4S8nBIL2$=L12=<6GN_cMCEP zx3?wlB8nb<%)7&ROP+5~c|J<}6qziUTz`F=OfPZwe!$Z0+(x}ya-Qhkj=h^Qf3#a? zx8TVAB9;3&ILV|#@C-=&o|e;m<^z!a(roLQY|mB?9DA;H5;8)HX2J z3uYl=Q~uM7r2M1L~m<3pSOLG=d-aj;Ird~aktLsmlwxTbTn=)k5!jvQc-f_XeV~_ zkQQot0?A@%Y^gD6XR?t9**xc-({E6@ew=Fl-5}9lYmw~TAE0D=r}euf;(Pln1MH&E zmG0F&(w9E&V6`5h`2@M=SgsF-*2;IfpX?vT^;=%Ill@`4UM+SWE#VnPD%e4Sk$S2* z54BOhW|Y*najKzVQ+@3L&yYx}Xt6_Zvr`Udsr53P==zR2=?F2n4XR_;D3va8%h2*k zb`MmBmR0w0GqCwgW*4ce6eFT`xH)?{Mt66$#|rPp$1*`+vT)cALrw*3JyImso3Vw(d|p5Vi^N(?m4xXupKlKiR;~PE6(q-7l9BP7F@58BYW?oR zA74m?WL`}A)R=7Q==@ks)TOXVfOpW5gCatU-hlw-CO5Be{|IsjI=cu-pLWQ50;A1B zY#+&(_Q-O)rZf=hl&|2;{{~v#qO?p?jb^m4&1rc~*vwpVoRMcWrzk2T2fr(~XH+xyhq_n9Z% z*EGnW8XK4^wV%{aWyXMER7;Bnf1gKWdRg&@rwZSkyrl$LZ77dUW|PS15MsjFqS&Dj z3(az;X(U7@EsoW9B8akWdiCty zPkt=Be#h$cfw9nuyXFoOQV-#rr~13JeuL6_xG7n;hpPKY>8;(}#XR4iN_Uq0^}Tus zFvE1kf6J|aZ6g{%kSxi}Q>M4CybU9pO^BwgNJt}k zea{wPt0Li{0b6kd>$0r!64CCEuBd`SH2#1K<2p7P7o^0HFs^j&fXM11cZ561ou&Tq zc%4nm(v1{D%fuVjX_ruJBVKy48_2ewYtx=XS&vNOepEdDtHDrd-s zoiC4$3Rv`OvJU5`RFDkOFJrPVRX}GarlS^UEos=4{K**nC7i28RkGL3P|WC9C9;?z zUE%_KLp1`?gkQqL4T;w<&K34& zHnO2YKndw;&}U#|X{2Yv3`R9MsQB~2%u1rV5>O2+>b#_q-7SQ6F%${2*pA!$U-V=A z4A|*88fB9njV9t0%TnRk1PTNkfeM3Xs1U5W9H9_4ku-Y!<_GqURA-`7)AjK9k@5;s zt)if)*>Y_pyFQz_?m!}!j{Fu<{O5f|{LJLd5ADD3*{N{cN1%#V3c0^aCv*hFUHLP6 z{iCNp`hYI+FX!{Wz&yw5-2a7Mz~2P>aT`Us2-A#=AliqWiNQtZ34CgUx(pw~D2_M{ zGwz!t-N0-i6Wc~bp=nr&tM0<4)5E~CqWSzBEr*d0L67)FS+=4|0M{Csc>HkfbR(87 zW+Syy&M0QXwv{xKX$3?Kp(w-|F)_Kns01~iuQo9j&?CAY`EjgJ4b0YbeD~`>Mzv4rmtI*S^*MvYpAvnZKf|?e%Qx zyR}rF;+Rl+x%a@Ars>-yv*gh_*|Z^Z(E!8qU{8cjXK(Q;@*?ds5CWA8av7+ZbN1j| zPpDb{%iO-WA|ys%ivkVlx7cm&Dd($WnQffu_(= zH#>?R6Q%tP@B%i*c5&Qp?3c){Uj6F3lBOPA9ht2pcvCU(sG5-HDuH|=h;SgD4OFI- z9DEzAuZX4^)fXp2aoQn_?JXwKvI z_znud3!#vtX~$OLGu1e6sK!YxuG|JcKY$VW`Q>T&`6i-2Ux{f;li^YL`8w4zwE6z# zPw5OBLpdrq5sd_Twn5*+yz>NFMv-|KFttV2V%YT2IYiCjwK9RKZmgry-24h!aK3PF zo|?mNXs@V?^QS`ZU^)2D2%*1#-=FrMKDaf;Hg{^@gjxeLsrbNrmgqwuxTkvO?6w)W zB0i18<2S_;C2X5=@^h!WAV+Q*I|H}G!c54dG&(6aH z`a^UQzrdAfHIGyl6LmHRLhdy~a5`PgJkiNh#4^MCVx5-2>PwQHNGISZ#dJj%{h?66 z}Pi&$h;ucDgOc1=?X2PgCbEZ}~}z!^Di?d}AGLy9n6wflW`ihQ18m zED;SDsz_ZIk5h}0%2?DSt_~wLTFYTs^VbvL*@>gGqSF&+!Do?1EZGl!t3ckL4u1t) z`sipmZ-XO+&%yW0;YdiN&JFXB?(^VM4K=V*O{`>sdA|~rwP?^piUQ#R5_3U4qy!3~ z>_%pPA&0TXgYPA-jWm{Gxt0Ql>B|T0sW`aMO5L)iYw1;Rtfm~HvV^P*y@l?F7>Z1a zk>(D5*#fS zf{~1+!CM!VZ-aAN*jF@gCKf4_a-{TJrh%fk* zM|&S1YVAEeg&XK5h`|T@>{DLYg%9lfY{b2aUqQdmpM^|sM+DK@YW*Yc7Has%9ebh? z_Zj>;dX_&iJZ|KV`YVv}Mz1;;;y#9JD2y*V*qqw7NNH!9`h0udi7$I5l9;<4=g}yB zhqJ)Tx%e^$eIK2gMsD|N_P|eibZBt@&-MZ4HouKt#BXu4+&xql#KN7SlO1Vc+kzk3 z%wBRQ8rTYEXY1?24E*^5F>O4%sgz3OyA7hXeA6s8$jn5PSUnKuP1%{XJu_woGXAMp zcs5~1$1PP5!h4eC1{M;zlSgMGmL8RL-ikboqh{vVz6oAPW_i$dg%`Dvd?;9}77g|R zmDO}r*DN_&87Vi=<-*45!c^5)u8N9So7TZSE23BqDQcmBWg(u(?K^&Yd2~5w8b}E@ z?u@0(%<}%58i8PZLPr8`j97`K`AMHIo6Gtt6$mVAQ+fo$am{M*8Prd0uA!&VWyl*z ztVi}t5FBK(MR5;}p`u)g`tG3fMC#EkwWd-z;>5^hGB{Umt2=Gm5rsb8tTBahq{dW> z3#KLTff0ceURbFK>7mq)MN#+ftSRYVH3JCTg})F(+80}kOFMQ&QN3$yzK6b#zs{X( zR+b3vqyic<-@GNIVH!CRYj`0_LpCRjr7wJr_%kQ|09QuhDJKGv96v`7OPGY0G?IBk zfLu$eWN*mog>se51mb||kIrNpPaH$}vIy^Sg@qx!Al0Wo}46B`E^W(G+8`K(~`6u_TAx z%&5%5vw5QYL^LGPTSsqMIaA^ZB*0e$lA%fhmW^Vy;wPPb#UO-A5| zvV=BUn~iOZF~%egcFblBcAVHOAtVqUVT(fy0Ybp+;6OqWKc3@&V+@ZiJmOe0@0_~T zJtJ#D2EXsidoLsP?V8(n`c~CFbIu#!GMP{Pd0E5x<1?&veRA*H}RR@?gDi`{5!XhlFI;WF=0;MDI|dy52SuMr>Jcen~x< zZA>Mbmjy&6^wW) zSQ;(%jTgL=4PT$>c7Mhw2D06aqBk9}yjov6S+zWR&Yom6nIfC0LH`CQvX5DB5kqvh zIZYl3lZlKWRKptqnDAi867o1Er(_x%_~7-Mb~|D;mVK_R&{t_&p+`IrnmDT;AC#47z|Kfg#tuXs?fzy^?lbXlSq8=Wffx0N zX7!ln_qZd17;<}TuI69C)O`FNS*LVypZ#Qz^|_N{#8KBeot|W6Q&8<^opx+m3R{n@ zc2HJvvS(LLc1wcSXZQ4#ZN*e&(*!0A{gcb#SXMWbn?c3Y77Yda4cb_zZV$^N`b*(jVAqq3N9Q@lsB*c3`4@QYx!fB%n9Kezc*gqHJnXb=BoNuUOku zLDG1{t0S>|N|uMru+Q}62hwzG?wh?65iGs!je`;*sb)k|G~m2e%w=kH?lak_*V%JJ z=bxDGrR#)e`0l%Uo3ny2OmuwIynM}2N@S`;6L(}W`RJ%J2pqYMY*9V?fJFoEqFSUk zAMUeQK)UD)1(cM{Dxye~E z%Dz&dk>0s)SI+3^E$OTXy(a2duSYQ?W(USHH(H^m$j5gG#`o7`2OFRBY=E-PJF>G% z^mIn2EE?R|#eS5)JI_7V7ZvE?Lspc6$dSJ zyMSH--A{%7?nGufs;i1FsFnas!>fwAsFnJXg|IHrd=T?N$kt!Tme~hWFGZ3OKdet_T>9X?N`uoocYbczeF&gsS5> z@Ik0e*=6nz7LxWIufpy7cB&LS##s87oHbw(fWpZs33Y8?V33UP-Y2zB+;?*N%<889K3*fXM!>Ub(PrpkMBG#$@xE-BevGCjn%)BGd zGH|HC{d_RnCYQ4~%MD+t6>+bf@F{Z7<#jC|9v(S7 zAgJ=(zXQsoub2HcHB{kV3LMfCNqO(2p5G3}|t3Kwkv+WN1WT@jj8mjqSx

za(*i1v0_81P`mbzcx`NMq-0S_s zqh7C57iE?%7v2PG80Am97HB)uGRxyS$|Y#N1IHLy-(q zcq0IFA)dtL3fv?YrYxte&_(wNkHPBkaL(bYwryF=9@q|E;ua?Qo!S~dRSi?tB)d1C zFWcO#C$U1hh+ z^jCV~-r`h6^J(Q~!Z;F$SfT-VP;^JSV!=W#=aWn&kWQtY=XQxO0R4Xg=BSrxg*Z!r zs$&(v98rG9T(2rFiVAR$(~deoJ(FEK#w204oX}<9Ae9m7TM0IBPp+g|7<% zODm6Bm@`js7rVP$vM!$E9C7T|r|BW#KETars|a)Kfb(otk)|{fX4IACEugM5b8@x( zB_}IayNE_uObK=1(DWcnytS4m-g@klREybEadK}Re(W(dmQ1A{|AJr$?_hZdCU{TVzW8(HFHPF*Oo>iwgtvZ!Oo{Xlf9*7gFFn8E z|C{;XC>fbQP2MWpPZAxZ{VTJOy*N$R`EzppUy~>7zxKi`Y#N{>JO5L*XRg`Fu+oK` z5eEfwRdvAeLg2vSt#aYcwz@L?UU^viV?p)3RSf6*!jXy>?x|U|7;cQ~O*tFmynuWa zwu4So=mXNS>3ES=MzgMSZ-tYc{c;S)0bSiMJle@{AFN)?4vWRxDi+(|&#{@JV|D*e zx2J>Dfe+nIe+2k_4dX&yF2llACsw^xbt=y~!g9)q6|1xVfuIs#k(Bt1CMa%xWQip5 z$fY>vYH=$|Xiy8qa{)ON_C>mfGX5?tq{)6sQJE7XUXj(jE?TjWU)4eZ-RCmBAD3;- z2+4}y%Jd{cTJOk`q!{k1M01U-&Hf0TkDB1g27b}IB_k-N{XXjL?&heDYjX%?^?_Q*7Yr5(2&;Zl3{pGHRC*iSRW~y;R z2xSUh+s?Zv^>hbxLjnCOmGETZKKkhqP`@?F8yAXaDc_X#p0a|9tOhG9>z^N9SC2|k z029+FFt=)<)(F4XvR&1zWji+QI1%Yvgj>k6j`lL_*s^&@f$C_c!FFtpqoOLRqt`MX z+enG+tpJW@`*?QJ#3%x@n)JlAtf`pMp5)+YPeySmu82-mw;n^-maXUP=)me)W4i38 zG~MMJ6?V(Ok^)KF@WQ&ZhSyXFs%u+`jmrl|(T*?3zoFhR4xRjRbr0023GIz!MFn?+BX$2NrYu&mGhiA!sy z2fgJl#Y$WK*79grva}DZ{7bi>DK{L`48ir6@qyQT@9wbB6+8Ve^oN#BU8fHG$&3|7 zj3tt>`Tw8~3vUMvqDf|$j-_(cl4TljmKZ#i!)d|%qKRXu&E$Az91j{+r>3O^M_Ywu zDQ*qchtNC7&%gs1qe?J1mKvDp3#NUl*A?-(;$d%YY-6Q3S=_F4nFG^QT{2*1RJCY% zef~6+n2 zIxPc+VKK))qkjl^vILkV$azJYS%|$P?ToEGC>Bxd;FT8Ol?qm|3GTAkb?Y8Tyx2C|sW%YHpn9wSpeY zZ6nj7QrW4K{MMx?CS@u!IQVMS*Mxtm??h+M8`fIv1(-mU3|&cxT9M#Z!h6&lUpyv_|D( zQI@o{KTycV4cX-lx-EA|)_*!AgZy3RI(--+) zy6o?(ROsRGvem1nD!MF*f+SWWq2^#S9L&m}EssYuQ47F;Gno< z%|tSyzQIgktQ0_Gy?#XVYXBBSMUD;>!kRA?%8!QxkQE{bsATU@rhhc&4n^q^*ZQSH zTSk%|T??8zIwnjDF@vnjl^(QM%cfs5;<@gK6%V)q9+(v)Y|5^%ZAJ_IfxfO@NAG{0 zJ`c3Gj%B_@SzqCFs4+Q*R5rjfWUb*XmMP3JsWA8%VhP}o5wtVFO>(;|)8QQkhg(KR zog|P+W+p;V1lLj7idl)O?eQpqpe<2dE4r?@(!#Z)T}SS1kgm5b>KE)k~lK2N_6bheE7Op)iI{@zB=sGnlYJnk~LNR0&8?2hvm=&Z=GF2st| z^D?^?I0ffnEx`-$FUDBDdtPplxFAz<#%#_1QrHS(?(6Km?B=qRws#(~4J1r$6*;TT zT29_Xj;pEeDK2Z|%eW`R2wO(bh$cNK_wxn13DLb;ENDNXAG_vVr#pGI(;^hMR zFQREJ980)?Eh=sW1c>ZXHIEjp20^E>%05vNR1YW)qFgGOSYcE|^_wjP&FA;2cHWk4 zBVxMCA;D{yewodrDWdbya_b>Wlw2OK@M_tGc369KK<1*&FplJ+aBWCt)R4=Tg1E&Z8s4O@HO;nhgu9eDhMZo+*sFw|I+% zynD%wRm-nS5~ybr_r3f54lh1U|B=y)pL3o(UmF(Cj9f&a-P<++OrEhK8cw~^4(-sp zG<%?YyR0ecZX6qsiz5BV&@$Z)&4|FEKqAihqNXNvT{=itGiuS}v2TApA6CT$vSIOW zKM|6`yIFqYcJ^y#*}gz#@$2cM9(#8=blK^ zw@=eI3lH;b{K#EU->xxT8zUYYi>w6Bs_GPg1a&q^F?yY4JZgnNpcGc5vVeS)0 zqn0Z6?HzZpLXPjfsWQ%AZ1dv9`((I!ZJnn`6m4F7K`7zf7hU#=*(y)d&w#whF?rFp zRbo^H)(NHdd90J6dfRwGy^eL_wnf%PyPTwTX1Wm8FIXwgHX<@T{-^YW@D{R~-M@#+ zYNrpi-(dgn%d}Hic+0Q9D_HCAY?Q`4U8jX7V4gE9LkrSDtlEj-P#u#(ZlayB6Yv}x zDrWqb;SsPP=axlfSlhTG3&>S?;`i?bzE;rheXwW!(HZVny6^3$?>l^w(&I;k%#j=E zjfd%~n{HB)Syhw#lg6z#PF+5pk-|wColBN}`-v4BwBl^Jd$vp6HaWFfyT5m1LESzD z&js^cqYZmCSxOweVyf!k3FPUR^8hmvPE?(|2VSC*ejYv0ZVmDlaC~@R3y%9rJ9B;K z3_MuiTaKCcOzb;FS@t9z;76YZ5Y`hg{DzgX?Sju+3iz@v*-+Jlr5SQ@cirFL*XF{~+2kd~%vc1A6~dBU`Tp(2 z%^NRCi3-A4+X{C33oDYzHN|i>q^pu>xm$19aQtCs?TpSJ7KVg}c%9s~CmHKjC@AL~ zw;LNVJi2@qL6$?npnSqZ3}!owJZOi60pNR777|7DaV536*R>1O0N^$|IBZ3&yelue zBq?BrN0S29mZ6_SG*;MJsqC!&KyU*blN1KW=5BX!z40{tUE%jYYs9E2)ESFaM^7Zs zcbmt0un4~i0s0dP+9_smuX9<$i}E5xxv_ zI!fZiiT_yaA;OAZgU;Nt&@Sk7wmG;ktx2H)OJ(t|i>6rkt2T%|{by~2`?>ZUf37pb zImED-@D?6NXLcFZO)u+O^c(Pqg_+*U=|ME1?~|9sk=+Rwo#Mt_?R}#2awLUd!!D*z|8R6`xNx)46**p}EbS zfqq8TI)0STfZVAvYb+1sDTq@}+0R7aQFA-vmB#duztn z^R=@(Mj`V4BQnZMm{)ryg68E>`=IxLZ7ZMIR{&vRj+x~Wi+X$H% z9+Z(Y9PXx>>utoG{4Ic3CM! zi~IVAi1EAU{c@$Eei?I2J39H;{PT1J{UPA{ zCf`ri#`sQrC3<6UKP8T>O6_qvC9oj2Iz!nxB_ArG%a5t-+&Po?x07dp#u5bP?Kgrx z*FgW8L|@zhQU=QGi?1x?N&sM_yX2TDyHq!5b%lfy77O_SD);BLkSs;1rTAi9kqwn@ zic0zyB~UxD2461ST9Fn+4>PkB(wdZt3v+igh7Gxt5I%+p75*#lt&^(?_fNh+*V5-m z{*2h!9AD{CC(dV#89Xsqv6pFdK{nbF<{%wpDEkt3#Qjygo2AlsQYz`OlqzbHDB$yW z4)CWQFbzYP`$DqMRzyuytPs6d4=9Q%iVm-wn17hQkNz7-m0lM88bPEJR2vGj>Ew~Z z7Q!PMn8*bKB=TxIK!Uz+fTFGWJGy38QdvFq4jQrTm6lQU*@B_mvgQF(R771~HEpV@ zU_M|cFMIf?EqFcAxx48m!=-Oob5q^$s_mRI$AK3hZ6IZYsxM0{+^-$=_G?DCG1-WI z?j-#_xs6=IpD*t2zpUz1IoMfsa`GR8*R54YZ^t+z^tRQij?RYqSqXR|VOm+U)4I0P zI>lSS3&!X|A`<3_NITnaduhj`kfdRUd@9ba!9{Xm!tqovMZE3X9TU=&$rSDhZw+DG zf4J>}Lh=*n-+4{T1=M6DvN^9O6_1%#SQS;p6)?k5)vCs2AWT6?B{2Y8f@;WeMH5L+YH1WyciH7JJ+>%B>rq9KT?(dqi{=#pK5@&69IHoQ#=lQpif$OX zRqCw-z2*LDFxCUKtf_EW$xhE^WUq3=NM6t+N%QHCDqcA=o7Vld3(&U;JV5P<1*`pK zZ}*Oz4oqBDTtEfcRe4R*Y;9Ar;>RdlSqu5z0KaQmepObar0pADMxQ1dIyQg=kAs?Q z7xuH9&1{Sy6~2pikbms#-Djte6Eo#`icX59ir{RE##wKG)QOHI!@VB2A!H)iUP&}P zR&T6OjD!FNskpgSg?@SMNFW`T#H8Q$f>I5nC#bUM$;CaO==y_tBBXg$fF*xwSh8gZ zg4gHi@@ET6&%XMXbK=l0ia1hwNZU`fFUM2!-;}v z^9arhJ^)dMv<8biEjUtMg^3aef!2n+CUyI424tz3jOsG_kI)F@)lx+pH~MN}!3@CC zrDh^%ff^>fKT**^?_hD6etIKGhM68qR{r0-y?jLtS9@K6&N$;?K;OST>7%k0;8aF z`Nx05$=P^*2l>l2Yn#vI9(R1>*Ml6~$g=FA{OjU85d9B`b}>h1o7NBs&%(^QfgYV~ zrro?%8<2C{_kAX>LmjDAItTL_tdH?rI}S5k#5Q3cWPcmRT-y&AD5=O0AA*{m%Kgw)ld|;o2+SqAUQsD`e|5y6ATZwGlrnt z=&w4vRUOpCShW!3euf3mkXHg2sp+U@nAH;}p%JbRUaAL?Urdn?%>;n<1a4;E&3&0XsGnk5iu{ zYKMm8u8<4JO$ijOeUq>XHJ1wC4&Q35d~ElX)HHC;sT#O^oh~X;+`<}ylBi-}R3Pfz zm}rKf6MzSy!^7=5Pf8-7t8aw}NuIWb-1bH2}=dL7i@?!s#J_HyNeb*-UEV+M` zubMt|AusfKJ_C?z>@{kYfMkt z4)lcQKCOt7j;J=Xx|^zV_gO(UC$;%H{bTyiq@Rql=+Z>Bot*^rYmp>uCv&OOsttrK zCu)zImsknN#kCOH8!?zqW9ZC4T{&rk5=#|X40w?UEf4cW#_g`jYJ8v=9||X{?lpFI zF4U;SDqgeuuO(S@4;1M)BV8WZ(Au8GNORNZRj1M+kJosQEBIqyW#i<=>8LgLl;z~0 z%J4lSj_nC8oaO3EtFmJ#;f3jd@d?9Qj;l&!4Hr;iDJ0sg>=U za5O%Jj_-D~IZ)a*0JmYm?I=}4?quL;%-i(Lxlkc%dBRm6T@lFZ%JlNd*S%jH4ab|F_6td-40d20}(wKLe@&g5Kakuwn|mYH?LDw8E=#?K)!a&QJho4!L! z+x~>+GrV1vua=R0VKZXt>6wC+%V%ApoVBvL-)-eAGj8cVT{XRC#si=&xipa_$c6 zD2igypQL~aB(*ER&;m8Ht9$4QQ8bXb$Hud-dw?Er#WbCXg1P4yKPFA4okXjSjiHY}CzU-PnP=rG1Qr_xKq9tH0wAxMD|<|+54E#9acNJNQoSpv zilA}!#k@ma;7wXA1CpHTs>hcz71;=;Caw* zXo)yz{S3=XHb~gH%~X~HE_?#Q4OEu+!#E|;e>Oxm!A~fP#D%S3PJyKDJPQ!CbN?Rb z37vU@zQT$*G$YK?pOBSoJZ)<}?sdrXwZm$tD=(${LyJcbvLv9gM0-VnYHs;OBas8` zk;qyi!N>{_qUH;^WTYuJkmXLl)y(Q19LS4K(1WhHf|Iba7pTl-g(tcV8E#Qh^ zh>i!FvGjnAYHPplf#Fd!a)L0S{@&15-L$$wqnR#uFm{zE0WvagrCnZ+#|=PO)Pi2R z7o?5^;!%+`!GYkWuS;I58O8GYp2_}H(48CZ8?PG1Q<~#rq<;ZpDzHi#S-##!_{eb0 zPqP>=Hy@ebk(q7J8;;C=vBhkWV-dK3>|EK=O_|gM^+g4N11s){WWU{I1!JyIGnT5` zAo)@0=<(rlv-5^08FVM|p0ufks10a2DVvy(t}DjEhT!Kwz*nB_ndndHmItT6PSADX zv?U0BRn?-=jhZp{St~ozJ6_ez{~^g;up~7vy|EMp0=E@gQhe?myLZk65}HZ4FLE=V zk0_rFXC{8LS$GMoF5<|y> zs+962rnrpFG_NdlmAYqkwknlCNL7r1$;?n+Qu&dbK0OI4Fb%%@w6QGYXRGZ#@v`Cie^BM1dFAaV%>DZ z;bg`ap}MW>R!Glgx@O8vjk*AjD>{8d0Mz66=tjWhiduqb`iy8JyQEn18K&Ee>!$8< zr;~wfh`Kz6<_@{am1d?Ga*1NlPX!hmPhX(2@Ga1Mt4zYloG&1Mz=#Yd-UN5@i}@y~ zUW{Im5m;jQ>{B}hVA&FAgL>AZDVj_{m$~~K=LAgvVswu#9dQ9S7lM`+tXJ~txlR(S zFZsX7+l6~O`}Vb}<6YxLHc&Oh-525ZrukNLi6Ehsw9$t?AO&=rUcJ3>|I3A=H=Zn8 zhM5Rtc3rzO8T9U9oAy(~Wbw~EAY$$myuyrdLU=@YUhEUMiMNRlil3DvX`A#3xhcO# z@hPX252;P{LG^PQ(Tdt_`k?+nqiCEkK5k0psQHljtSjca!u2h8!2Mlok9EKGtY?eo zYu=joI`60Ln*C$n?Y{5&*Zbe)|5_jy7!4c=JRe*eygx)j`$GR1elYxtNMGdks1jWk zeLR+nT^9Rnyeobv{*^>u;th#slYfv(rQVeKWjc^vlm2-6N13ZL_hg>SZp+@AeLS}z z_nqAH`A~i^e<=TWSH0`Ot|tq36~0?^6$gtq79T5qr=*nzOZ!UiEj`}7rTbewTY8=^ z=gRl@HhaI-_h8>Q`d9Q{-T(1Qt@8Qmk(yL{WMFjQnfhh*&o!1b-q%btk2Js1%C&A9 zY!2Qv_}tKvp?3^R!}pIEBX^BFKYDER*|FW@WcPk527gl3VhorJE~{>+L~E50;6Jbmx<_hz=uJhSrZl|Ndwf7N$apIW1=dGL}Im)y6uvi8ok zpI>LJTeI$K>tpLbHEYZc&+eOj%j|bHNE@0P?%LS3@xD!&O?Pf~Z64qJrp-@o*|z18 zEiY`{zV*><&eH?|ww*Kfan`{UbR*iqhb(~hrPw(7DEUG~)FgO`8e z3hRm$SG?g$a^+oD{&;8K&PR66T{V8yhpxWu>R;}-@TnWSZ+yoq zrB}Y;mCxKXeA7cW&D~tS`Tm=qy(L2&PDW^sVC{rY0lwb?vTthsZa6Mw$7SreoE=xp zKL^KY*1m)t*Rta}c3jVnv+THm9XHP31;P)AP5yao5Q~0+w?(p+ew()?k`VH|EtB=a6mKgeCfv>2D%mIeA#ZCWC%SoChjx{> zP0}rH!MNX2pTS2TmM2kXgR#IJ{kda&M+y zs}1#KZn)uw3S5ZS!NnU7?K*z^z`@MAy?YLvJiKY|wI>hlI>!IB^OvKUGcOy(uNpb? z+_t^PjvqLBBr{MMs11y5*qo`=1}g9x+kf=LHAjz}f)8qy#xVaQ=WovZhRlKEnO&I^ z$9C=6dwAEegPEiI&K^VMtTShwnAtQvoym0XIeKK*3Fy!byN>Pc*?;21b)(hlnL}q6 zo_QVlBa$H}$bPaH7V2&|9wB?+%r0^eUKxOc2gqS~ogl~Hm;o}}MUL~|t|uqq`V6U) z8vGj~ebByv{j2bAY=58g%^S%f_SeVZZx6z?>)_lTXrF{TY=ZW+(84Q@z38rI-*XiH z`a<6}+UeEE1+Uu%?PKhdqtMR`sE!qQ)!;QoHh{vFVc&sgYy9kfxbg(KhFyP(|3wX6 z4P5$X+-dQBF4Tt%ydP&Xo?&C$1)uDN^EeN9?NPGtqMk$LBCh$>SIoeZn1+8D_}2|R z!Ex?_E1Vg+f%R%H>4AAT0spRpnXbaW3qAFPZ@kd=5rJ_W0z9!!&hbN+z)OIuMBpGQ za0{J$40wYJ_<%(`#0%Wa2WUS4s5t~EFap|P43J;~uvQ9godfN#3$(@}aLI1CU77R( zx%NXBtL(wn;Q=;*-wpy94+GJR!lTCQZj)q+EFsItM`7xwVJIufD!|Wc$R%VgJjC^6 z7RY2H3~)2Zx2-@u&$sObqMg;a`ytQIWi! ze3tw*`3QL*`D5}q@(4VNyUE{?zbF4l?jip~o+ST+{295Iypep6`~mqkd6@h+^4sJM zfEw>2|C9Wf{0{j7`5Tzt_X8<@fDz2+$>ZdUKr&w=Uxx+tP4cJYY4Qa5 zTX-zauazK$ckIZZR9oN z)#N$yDEU6wN&b~QOP(VCOwxe3C9;%U4G(bf&o{`O=p+Zbu3 zq#+un5gMg28m9@0s!Ez>XqM(^o_5g!Ez%P0raiO_N?af9r|5#MQBYoKgEnc44$>hy zOh@P_D1PHC3&0dzLYLBIbU9r?r|Aq`NmtR;bPc_PuBGefdOAxt(2aBx-AuOt-E9R4 zCdri}Cl4L6uG@R;z|lR|>^*W~@3Fmmr0J8#j>;?c9y+m0IlSwdV@HpuCyyMcH3w?W zdu@Qf5AydR=e@DgnRak-?pCd+sVbhIe%a0^HAsg ztn+@?d4KDCJayjhI`4O#_q)#fUFZF-^M2QPzw5l;b>8nf?{}TgUxVMj!SCB@|IY8< z;P-Fv`#1Rg8~pwae*Xr)e}mt@!SCPT_iym~H~9S<{QgaTza}4lllP;^@89J2Z}R&$ z`Td)1I^g$j^7}XW{hR#$O@99-zkie8zs0}5)nd#OUIy^NK0*)eI&lQJCss=&^YiQz zdT{SCJm*}8yRMxW2@PJ46Q_Q$1uH$k@H?V+#O{dP5w|01N6e0h9q~G%b;Rn3)Dfp6 zN=J;22p#b`qI1ONh|FUQnIkSoRF0S&5jo;Ur{;%!9Rh_w-EBhJQrP$;6w{{~P?0|XQR000O8#GuGJY~r*@vUvaikmdjY z5&!@IQEXvzb7^ZrZ){{=R6;IvbY|?lcbFqrl{b9vt(>c*3W}vHb*qEArIysWC-mg; z%y`E3csw|aJ>!HUIDie<#zY%1X2F0LmSisrOBe%Y2QR^7Ff1(DBx5jOSYUU-nC#Ja z?k%ai#|EC~ecpe*Ki;0Hqmopry7$E2IX57T5VC>r2yNN3d*6fad-UZ9{p1A*VRiSu zM)Mtiy8I$Ie+gW5&S}dhoqoyuGe3%ua1@S*&%5-{6+el6Z!1FDD-n{+^RBrnk6gGC zxd|ZT{3|ZFR6Ov8E<&+4!#$pP!6nZ=fBrq)+u{2RLZ8(yJaXvp|4iL{9KO@x`p$*$ zfp`;l9lT$L_vH&Oz3N%7dvErG@cw5Aai4qok!M^c{7iTjLf^OHv+5<6pLd8n@a14_Xs^whUT4p z#WRjvaSWjmXb){Qk0>uN+VBbW+n+t#ntvQA+|OulKC}1quSc(sUe>#}$8#Uypub2# z1Vw-FJM!PX(@@Ag3g2(%KEm#__U||Bli#4bkTm!O{HgFe;0rtF@KN$%B)}&hCbz)* zmqoAm`>2WjBwK}1FZy&tB;@QujcgX*l^xN=C^p`xs4tvb&(Bt$wc+4a4 zN{=s~3a+3e{xI_JXV5tL462e{Ahmx-3A_Qd;Yj08pmF>d8b%+7JCsoxUNgW^6Rr>N zOVJ#>X5gs6p~Eo*hmVhoxHH%V-i`2j^dbzP}C*2g&%o@VhUf3G{W) zfd_l{;yX|SeGPbW548JB(63JekG>A)Uxn*CsD=;2J+}ZKzJkW^tB^`ALMq%t!ncF; zo{2QvL>B^oi_p*Y1Vg(&13GmFJo7a8{ilF89=v+Mr|%*Ke+`bGfet-@%Ak=obRY2N zG-wmO_LHbUZbwb-QF!)Nu+H_sx83Lveg!%WA47Za2)Ynof*QcX^Kl&Q#7EG2ZW0CL z^GG2FQJFJ=$Je6(?%l=T0X}{dFDwCJNGwADMXfxRW--n3*YX_ATz7=@B z<^VYjqi^?q2FLf{_&Jl&Jj$~JWYvF7Q8|$z)MW?AioP-#4dnEizjC}5jvIhqcR>5r z93Y<|^!45!;P??7zh$!d0`%$02bIrPK+bCqDjRyd0L6fZAxZZB1Ts(Jn|qI;U-TZs z4}e@hi{|lndynH6f=aQ!jC}zzJjXwmmq`V zr~pS1pNCFm=fGF;7toI~@EU9fjud$X=-Z9^S=^xV3ALJgAk=%O%^hg1!A{D5HG@v@r zzaRtYKuu%-wU7zaMpp07$U!#H7;=EdQ4FYy;=MnC?YcmHiN2Z0_&PwV{@9YJRTJs+I~ z^a4gNL}&MYf-XYm0KFI;0{V1xF3?NRdA%Q_OVMGVm!Ts-FGuGCy#ig(djwsHE(H1v zMxTi;0(up?7_9SZ^mL%tpi6*Wi!KHFEOc4#hv?bpa-i3tD}Y{)t_1oV^o-ua=(&tO z4?Pp;4d|-g4q=~xc43O zYDQm!UIO$E^irU&MYr_6jb4Xt1^RmQGN5;&mjnGr^orht=nd#Lpl?L41o}_tcA#%U zuj+jZy_wOspnm}RR`hD1Z$q!?eG?-89YF6!uLb(g=ygEfj$YsU2KpCtC(w7Ge+2qY z^ah~+3iRvfUFeNK-;MqW=zGwcfW8;Kx%V}66ukxLJ?O1K--q4?^!@0r-dE8F(A_}q zMgI)+gN*(gdVB9H=otDJpdUi-0D2#KC(wUK|JwUc^kMWapdUf+2KrI-9-tpX@9ljV z-H(m}{W!V@=qDKcBzj-(OXySR{XjpBJ^=JH=w6_oMIY?_2l^cPH=v(K$ACV7J_Pg& z=)T?;(HGIb1N{$1zl1&v^vmcYy)Phm8qlwxj{*HEx*zD*(8qfZfRFzK&~Knm0{tfX z6wq&>Pxn5L9%S^}=rcgSgFXxNyXbSh&!O+3&jbBFdI0D{=nFu9fWFxKEP5FI2hbm) zF9H1#`ZCZ*(0}$mgMQ5DPtaF@{uF%`=%eUsy-%Z`K}7j9`Z@Xr(Emc;1o{i~t=^~5 zF98vJ3jGRV+^5igqwfIyAE2K^|BJp0^w;QnK!1b25A-qgQ127yx9A5zA4d-Z{T=!t z(BGpU^*)aNfF1$*NAzQ$e`55{=qJ7V(O=L{fj)sA1$rF)toJe0Lq7)!NH9A1>-s1$ z1Me`!w}22HU})}Pcu(L{9`K?Fi1IA(|0?)>9sIose%=QE9s|GbfZRj zvR(jRumqlRJ$T5C;2Af8N8AFQa2t5Q9pL#+0*|*FJl)CQ;r4-NI~6?Ie(+=;2S4@z z=m!FOy%lWp6<}$v2fMioY~gOOig$oDyc6{QJ)r&X27P}YX!|{&-yZ;7z8AFm7-;ZE zKz~0B8uB3c!Eb=>TmZUpBlw6HfzP-CywG>R*IWlW@;%UvhrmBl&W=SAQjUjlyR#o$$L2XFEZ;6+{oe&cnZ*>{3(_$TlN&jnxk4A8It0zY{* zc*b9Xeq9Ur>eXE+ku^20+ieYi2cuw(1<$*u+{!Ds3 zrZ-f|^B}pipdZ(RkGdQ9{vP1&ebB!ju&1oYd+<5<3V6zM;VGXX55iO8`9waG&$Fk@ zta*yF@|45uDY(}IzmC}P12`U{%G*1J*8KJE@9pf}%+7A?)q(D&=TGDY-#@(h;f05X z9!@-*`N7#g_}D{mr-#1y&_^D+_o3?^diHli>1)Ed%qGr-0JYIeF12} zC;GSg16URP`wKg0jwxap>)1w5`osK_KY_oq`j!&u`ao~b=5Gf)BEu3KDQizJ6znq!^P|laDyk{B4Kn#{wP=7bNb1rAH8uoeRSROfpnpe-*oi; zJ*OYNe>q(^Z~(3nS6T|M&%G$oZ?gn#7VB^#M|atG`q6diqv*gbw?ywsg`+p#a!dLa z=tuwkF?9cvKEUY7AFS(t0GyzA zWV4xan4f3QPvXb1LB0vtr3=R~WW)1gaE!t+4u=3o0ooHcxzbb>n$(6SHG>dmvB^on zU;Xp~;&`seL2C%sD~=|r3Km3}d-u^_7@C0NARL##aXIDV6mYUV)#^s(!7t`X#Vws?(A}9I&5K zK8^zwvv9Xy@E`BWhufEkK*lE2YFC#OUQ&g`QgL`PQc+pW2QaTym=MNJT6Zd`KwLNzs(1qN{% z<*yx@*~!4KS2B`24A(xBPpE2E=fUViIiKh3lo=0gUgESkf_Pr-JwomvZ-Ts_1gQNa zh#izaP2f)%_*0?U)eH$RsZ<;%_{6tyINFOsx5G>*Fa@b4mMX3v@eVkG*PdS!t9Glo zsj|CuYrb9ejvtLxrYpIrio10iymz02P1`iuoA>NmP8f<}Y9`Sp*6rN2BgD3WN7Csa zmrka_HQtT?`Ih^TM01ti@!> zP-A;9z2TK zPHYDB1!Uiv?Q2bmnOY=pYEI*X>Izi|U<*~0NMBk_YOE4d5zh??y?hb3=W&f|cDn2r zp6laxiem{$iJMA1X%jt_kW8^Wq;h85Rb)*wR3#R#?w@FEok+O(Aum@-jxJVOiH?8;Rd!<|#dKqR zHEfSpt@^3;LZ{*;U2({A%u+3}v}JoyleOj33X&ure`7jdJ++jWs|?M6EhX@8!1sR} zxMBgPxW2cIQE%IKr(Om;3MYvY*hVL^t6IAu6@5jF#WB%sS9Ri=FBib4VgkB(j}s6f z=W`N?4^{GNjFaVsjiAl4DbW!ijM}}eI3<e@XRI9z2io6F3fzJThB&^IESmnGv&I zY<{L*?JlK)BsaeUSGL~p^iwkFys~`FKVN*_o37p!6T)EA1MJ@E-WG7}zXrL}dy5Sk zP2k>O{xpnnOw7=MMQ#DVW!r)ny0N*LdS!YknMjh%WZA>j7kp)1I<0KD^34|@ z1@CLVC;{i_McqzFEG$-Q_i1s5GtI-2-j=l;Adm7LTwRWn%RHsa^cIp*PX6@Z5NJbB4x+Q(z85$jl$)?Ji_V76+crbDO zA3;Rp2w$yWUK6m$hjotiE7SWOpyaDq1qL7}dj^f{tpOOyEY|{zQ^TcHXE6-3m3+hx zv-df?i+l|9h3bk)<5U!(>3^=&iP83S0=z5JPP-$X6xRhk9ws_Z@9>(NERkbp9y@F6 zE4L=cG+rS1UVPaHb%Su2`MIF?8+9I@h%;!g#<_RFv|7zY@MPlzJXbJP z_oj36J4c*zFC5?2OkZjw3U0iRG>sr1A8zBJi(i;`V$Ho5O~2)>r5)GoUVnN!2&T{3 zK7Vr4dph)KuJ;7Kf*b_>FR(sEh%F9B0**HDy@C>`9eG)k^O#$ueiIr4|4m#hfu{8V z4EzVRpe70D9Eb)C45F!|%VvVtK<5SGw_1~N9J+!gf*Ucb)ld>-ESO4iW@=QI9kOfq zIk(=n04@@OZ7MEs7~`a5NE%$vku$z}HLof%R>Ao?iV}AWj!1o(_V8PPf9oLQI)+qe zdku~{9AnUjG^!(&^&yIlT%V!Riv0X0M8s~V*@|#6sMkQj7Qu%>U!~T1Sra*>+?>yk zud6y33#F7kr(%R}7VwIW;d8WddSSV` z@!(W_O2e8Y2Q%4XryK`C2wU26!T`h~3CTvq&sOu(z9cJ>z-wo1Y%G*b9lyw_#LZoc z-BV{mOI3Z%Nt=#eYlk3EnX5x@H6jX{E32tOEl3T=Z9!E7MMw=NYh#{2!fZkB{QSHu`WR;b_6p0S+v|w{gaS<^CD9;7*%?z8ZiZ=w#D}^ah+?7K6WHnNQ$* zz$LLb(ubENFt7GJ>9j#@eTuUm$J1AoER)E&+Su5zZB!jSE)f~}9;;{|DPHtE5!68U zv|(Ss*sT_`T4yVKbD0AElQ%;%zPM%Qa>q0*4$yeePMke<79cdRcv-enBDvgh0b=3{ zHCI)2P191d^Rr1VJ(rGr%n-=?4zR~a-dk9Jhr0GPXD6~J5qqojO+kW6(RUusNrXA`0>$Xe3U5?a`e6Yz7qCm-?PT-q znm=Ao&-zYL;}k^|IFY3Hp0+!|V_7CK4qR-0O9|6#LkudN3gibd!4z?GPd5ydv?r*B zX$JM#V!I6>8gk0S-27b1&S_NVX9bzCe*3-0zJeROzxc8f6*_@qI2@iBl(e}YIv%z@cBk&S4WTpe>6DfjO|-HQ=l1q zCZ3Tr%oSTp#n!0{rE8%t`QBIY8_0R!b0ns{6O%#2c(J%vG6k02Kmtg6QU{dLowq8( zBc5NfQ#+>$lQCYA5@P`d6XCIvuh!jc!?enIk0gc;o_pRzb8i*XtVAn05@&)d=5V<0 ztRoW}o;dOppP-=2*O=IwhkF)2Z0X*+~DapMg>4QOped z>{e?luz3zs&N2aj=1F}Gt44sCbNxP&j?#-WN|n^)w#M|1aW`hAr$>FqPBk_R@e*%V zXDYF{sg*s)m!IPllC~31)Jv_Is^zFzCqLB4=bE^k-7p3zL?+){%7sm@R0)%ABT-u% zOJFWl?}XXOn3=P*nDA>oQBH(KCy@*mtIan4S3k5gSM%G&`p^*5F|YSAkjpt}GmEe^ zvtOWb-|xq+sKL+$&>Wg)c6~SM+Nd_$3kKict4Dt4{L&bPeGBYwH?tEc#eYv%7u8`k!@s1;J>&DZ7)@0et>I6sm5yLY01At|`6*LoPxM2m& zI8O#P=~{911Yfh5%VsON<3H>}9l~%Qy=MXJkKWVjXPtio`PBxR;j;;q#} zwR)^vs#Y2A{NC^J82JaVj~Uj_8bdsAKc)m=0DT|;fs|Hb1{xyK&JGX+w^~4Zv;htu z3u(j2i-IA0n(SmjsVjjQ$hm4gUh5RB`lYm*7F2RceCfil;>dt*wxYe#X90V2L<=<_#y^^2TC&V`0RzrizJrK8`b~ zV9KSaK#&E@d6_V8dNL>oXKWd%P8mAoCH%>PkyLku4B{qE%UfIjBQB) z)8rtW3uY!rdPOs|&6FXVvIKfJp^IMYl%;1~@}|4@T)sI+G=Qk&u9!hvMEsc7@kq4| z;f~5f?EQt<3AmXLwU%*U=nUo!z-n?QiSX`Wm)i;MmQX{;`3zGIbEvdPcSR(Y}M(NZE9Oc_ny+1>~{d!bF zZM2=`cVgd)^9_n*^GJy>`us}QS5P{1K8ohh(S8|lo_4uZ2gK8%;2ro>q9!^3x>$Bp zZ$8`F+VBc-K^HAk&m`^A{Jx3m=IX^tN*)D|70;A?IU$P%Kmgv15hsQDv6AS^e+{jz8m;C@3{yJbKrrN;wJL41!y3GuB9r*3m#t6L zTmEEo!W%oOE+}}x&Q0V~In9%krEs>9CAVXaknw$2ZoJ@?6K={9M1gF~W{l(C=$_mO z1rZwUV=FHQqbBPrK8KMG+ z-=b0aO=isv_v8e}jnCA_c4q;u5(Q+dTYm)eS0sNdv+EcqX}132-2~V30O~ow()c_1 zbs;pFCp>o3w!-oF{&da}2JO2N+85PDjJ|9G#vp3C5Qe%1*0w<-vXGozXi_i5YgkS= z7;%QFWfNC@n25Tg-zr1UlZa86N)QPY3D+|`hrgK0JHlt5#;Np~NzkVslDp7G7X6Cw zOc#!T>J%#wERtr64VN&M)TwE-S4sx~9h4ARt$f9Y1HNg~((wXxQR4N=a9IWG&YAu~ zO?Kp|Gd5JF%cb+yG40b7QCBe}aFzOL=VTfS#h|39uC3!sAjc7CaLK>&RT7NZ%676zxw7c=!nDTm!|SKPl)-r4iMbXhbhm z+s;asMvBy~oqpMm*0Y#p00!|Kh||zre1l?*m#z^NUiS(Ks`To}h%9_lQ->z#J7uJn z$UrN87fTuf&y^0g@uu-@Syj}tsV_fJ4ivt8aO(Jn8k@432yL6_J%RoS+E!%pbynpI zA(G}9@WR^CgoMkbSOH#q%KBx6#Xh+{WQgpQQ_X8f54FO%?9+xCQd}*TpHtjaEUwR; zo2_alJdEml?|*wAMn7bJk@7-ka*cSxbe(4KE4_y?QLSmFXr2$z@eNyAG)Ow}@};}?u=ogK9tkqAOkGaad~(|^JbLK}9V zR=i6=vMl&2J~BG$di9~Q?G`lXooR&yO-<*^kV42< z8Xm@~XW;MD*QF%c$fUj$8;VUHZh-64B+1}3Rl=#c`PsBTyM9Ahp3^jm@ccx=pXxL{ zFD~4Zs5)bs5ar;LpqtNFk;kAOMk^2{>J_OFRuY627Zk*Qx;2`8d4sIrOYGH(v>`9;JfNQd>aMJj=$ClyBlxL@C!DetVp{jkdul0A9U8H!8IT+9-AF`>lNbSJ)a%{V8+4Fm}jY4Vvj zzGPE=L!=MGV57eWeb~*aVzgGf42RcPt<`dn1)BXjk=L)5)6q{_;RByZn#^ggC@hGs zfmdV%*Y!g^b+`SWaM2iI^dIkz6)l2Q#fWDU6jL}-sJUYaBV`V0;Aa#X3kWTCiKdrJ zP`FJ**Y`!ow;O5M01cP$-qPlL@s!f9iyi>Xv5V)9Zy_-vRCd*I&Cqyqy#w2Kw0Lksg<(2dA}&cisaqZ z@;L;>SL|KCX=hOD=GO0i^@c-Bx#z!d)42=T7vtsUKkvp{Hl4dJeE!$R_usJR?H`&r zaKoN=zjxIq+)3U7-qBivTO0IAwEn$XqzCUDJ!YjyPoYxZ=>cP#SWgD+^wi#&u+W-M zseW9y6uX#?+3B*U$7QT5VVDLsUSseIPdO1ZcTab=Hv*0*aYsP9;6y!z|jmGLl7eLTYc+PN_LEhs&|1^;=4_3CgDd-1aLG&uXj=LhenK zb&lg@qm~MH*GKmh!m+%KXLh%Om)uA>mzvx>0(pa@W7mk)ebJIq%|Hu|KWllXTWJ?I z?b^3d$^f)hZPg(}Q#Fg{tIeh(%^tXV!@J*0xxD+@Q<|ER2wX@TAp0YNNM3N`Et}6< z%Ho?}e$x#sPp7fz`=Ixm&_3|V(fIEu<9<{R?lxAvW15YEL41S=Lwj2*RtLpi#(s(u zqx6FM!6u!?F^8M}U{*||=iExF?*Vi0HznJ!Jkttdrl6*Cr5F)Ok(cCB(WP!i)TwF` z8PKGsOo*0v@Sq>&)$F9_<&^*Oi_Ia!mxr{tt3Y~$Z7-#JQ&TMue@BliDmQxJ96l*M z1|W+LRxvMzg)`2{%`}Q~{a~#+t>CTl{Os&3#=pgpF1mmPp8>L7W*T2?MEE353#0vd z{t2@lR>o{jC=0Jnqv*Q;THgBR(z5I*s-^;Hrqdius~*FgmDTg%`R7h;GA&z6F6DnYb8+d}Tn>p@BsB&AX@;yKG1azJyUi|4!TSx3Y?l!L=~Wilhrkq zAcp^bjeiGiX);@<_=jSEVX9eRDi9Rg@u<9FVhRBJ=^7hGSp(B(x{ecLV;Zk27&v3N zA$UZ?%s+j8ZqH;&0G$E2Gpg_qKUy0MnQL7)kF8h^0v3mBhq`av^Ze;*CO^BYk!)rE zePU-$lk8&l`2U36#KK8T52W57SUfm~gUH`qZvfJk$rYCi(A`CqF6ef2+g zLhP5;yrJ+d@J|>iW7!p!jD(_5IjVxa>;=P6eRDvT1M(C|Ho&!^cK0mRhXLUw6r= zzM=r{oZcVEAIa;`+t@5MntKevG17=;sl5=67dN7LB~RwpDzlcbH1M0WY}O2($|nYQ z`Vt8I0ZNQ2H7nP!3)-t#3HO0LgK4zV^As83OCXX;mM$rr?CAg&DE~xJ|2tiY@v+O3GK%=}CyYLlr6deaPYe!rHi}B@82D0aJm6dfWu%mlJ9%r%RS9 zg$7_K&Mxd95lq1y3A`bbmt*5uS(UXwaROVmWyzLQN9Mq(5UjXHEkQVv+Se#djg(@% z0IZYJoh*DN$CTtypahyT7R${9su&^Zg*$g@ajm*pzC-gL{eaxvzH zGI=GP+6s7Hwp2jyseSd*Mo;t2S0;v>aRG9TMDk~n#<_FF^h`Flm<7e%UYl4SEzbtO z@dWhgPKdoDYid)q6-06$N}y3z$D}pi(LqI%ik22FXy_Gaot&n`{YeJ^M8c@Uq}W#( zR%7TaU`UQiwCqr2xSHsio^|mAHr$16yQ?EtcG6XqXf&6Bd|39fLz#ouR|p-a9@@KW zQ~JR|njC~4NwO^3w3Mk6i3?IA&wA-?8))>=9NEf{=+>4DA_S*Qq>Zn*_4yrH=A!>$@F9RxXU<#-~WT&nqqfhKPxZ94>aptv(M0;9=EzDGzZYrtwj(%yas%weFs) zuUp!gwZapaM>rm5Hk4jDG~k{eagWXqdlUY*-cKOk(E9dG`{RLhDInui{7z0il*wtC zeANF0rgPK3hjNV9bk17j6X7 zViMW`=E8uJx!NvYRqTZgbbcZ&ZxP5HoZjR3YWx$l9i6u_Gl*7f>DUaN85E-OpgKZj zOQ$W((x z5^>S3(Hi6HAz2V)$pIIJ`K=p9Gcy?_k?F1=YbZjZ-*9unVoy*h0ktd!Nkb z9hv8~gnaxfN?>h(R%mYMmomhm<7X__n(RG>-$Oo$3TT4O#Gw2NS=BG>^QrsfTHi`G z8AUvnT?~c`r3LDjmC33gc&$QtaXe*b%JKZxTz%O;sHDvJaKUSi+EvRj(+>V)e%x_& zqfmUOZU?PB)2)*iMoix}&1qhUjqjY>vrgBj9`$~MkAO@knBI*v`Z)p}SLx6Bmgrz) zRODvExGOSSsfgnvvaHLtqGt?I811_Gx*tqJ5{gaDn#|UB%oN7LaCojXvqe3E6p|-iU zK<@|b)x3d@Dbp^PWY7bt)9y^Q@$G$`k#j*jrwW?vRK41UiF75G*;);92C%tV1w?g$ z3OK43#?4g8b&D#P_qkLnI1QYpjftz#F;E2nV6wA+{l@c4xlj}?9x(p+y;nG0oRZ}D z$nxa)V%0Wv4C)uyQQscJ*8$(h(1w+H0V!6Yrfg=ezBNES{iz}Y_sIkx)X717Fw>y0v&53iR!j6lMdV4Mz(GjAsJR0ez2tkhC^oD^q7!IbuC&F1jL)k zZZ>Q)Bg?j_ahwznvrk*xc4R4g(gpxc#iVQlCdGKi3_g`#zsyR0f~^2pRflTSAIT8Q zPbHs5r?Q-rRuky_dRph++lXesP=QGzAh<{(61W4}cnIBH`c<)%Sy`z7d3O}yXcE{4 zadMF_?|RXkCA^0^E5N*lCh&4xvC=-IQ6tWf0?1tjamd6h`CyWYT~Ta`fa`I>O8~JA z0J(f+Y9U=1lF3m)5e3L@AglvqmxEepf#Cp6K(fDJH)Y9m<`yRPL+71U6X@hK0LAj0 znX{i=SSS+Hicv&IIJa!$*7}NNMD~t*JiZ%bdL?=;v-dOl_I^3o`xy~BiwXzKq;jiE zDOL@>%4T>&C<>ygIGv9jMN1m>qughzJHWsL#i-J^rAX=F33Xb3tQE!R9`F5rJPaWAz|6(N#nX3?#+a(aY`A=?i9)TqNtk`YULWaP*Kes*ruaWyTI`>1RP zg5-^FZQVY+DJ_8WPNe?FjLW0Bgnj%QR#F#4j_J)v?+Nr4av$>hqoA}}qYXw!7`Sam zra)U=Tb&qmyU3ZG>J?JZJY}MI*5Y7w8GcaD9QwhhemCqJXDXFfvnl4<2wmQvo#}Q@|L1OV(3;1XIG~+ z-2_-`wC6e|z9=>7raeg$P6Yc0<3oYz74GotXPd9n#$R+*{_ zb}FkJ530trVodXWI$MtB`&4_k(MavG z$-xxDMT{GmQ}ju#6AfwxbR2DVkiWCjQpgR=F={vcb=SeSxYr4rHf^GRMv3b4oOE-uidkW;SK1ff1*VNcz^`9#nQ(&i%(QDZ3js`Y&{H|zj@e}y9_{XTj z@OPy~-=<6DHvr>eesy>>4mhq6~SiTLmEDBmqkcjCBR%qI}SYx)J zLFhKTLQ)Iyt9*w${zdRGkT>9=!!!R9HCKIi)8ay=HSZ)~2mD?YS=&M)k>@%!*k!R}_6Cr&j6V+!}78l6iUjdQa& zzoMAk4jnrP)~wKofB>B~*ImWHPKaMC=Hd>mQUDINWz19AhgLRLSwER+4yC74lUcIf zE{E=T-5;{F;>URa3e771kDzSHkOLTUTmw$E_syPlO+K)kI}BI-i!*l8{7D-;$8uW=@9V>J(j!mXY z>sz_;YA|0>;>z}&f^;D<)4Dg5bFEa_b*kDsKs1GY%}LElbWdNm_31@7D`1?n%1KF9 z!KwR=t25&Q^hFqEwKb&d7)9iFM`49g3!?hGSP^ydnfi$Ky2ORLtb~>G&s* z?1h>d(vlD1I3?xkz9HJO=BuerzT7TDJlAr%g6SMgk`0`!39wSn>h5@JegnfBCfHSz zoDMN*usWR1X{dy3Y2aWegW$W(NGj1QRVeJU8g@x z#0F~U%(_T@qHc_HwY6Gyry#0|leB_iou|uF{)$%{pULZxI*DS!6-DBC_!G`>^#l2# zKqipaOgo;ctoS$0B!;3{;NM7MbaD7gnR-mHR6^Idsl7kp=i-OZGFx*+D-qR3v{s4E zHJSlyC8&;OqL|p%{+0hLl~S5S_RFM2sU;3dhpoXX{}sjjcS$A^PiGW>nW%!SYjQde zpmVY%yCW$j$<`-}#Ln{0a0c_1WniJ+fRvFF>3D%;exaP!GG=a5-YJ@H%@KIWW59j_ z2dAb|8*baB(A>k+=AM6sGCnyh*rp8V+c%u5o0(<({Ta}OKJf9tEX3o&K!=$7N5Xx0UbRAGKp#|J63$* zV65>zWJ6RLiN-<(n63Z6&r((p8l-+T>IC~^ilq>Ij2BI14p_zW1Pxyii^iH7lwBq6S>WLyqm&IjrCu@XyhQ8k2|%5J1n`8KrUFW4a5@V zxLU7=4}<;BvoRC8{)o<5SdFc81yYgq{XRf0#Q_%}xX{{fe-%zNS^~j_f&gh{aBYP{ z5RK_N9jZv}SS|f|I`P>WYP8dCGvQYAJ`dU-cP1JW7gTwbPEnHMYVFk0nUMB*xsWqV z{Cyr$44E!&lwxLf`>AJ+V_N&x5^|o#Bu6Mz2Oz>v7}`~y8rNny&`&mGeCNS zcuG-fKywC%&bN9%v;d+|JfHYa04Ut|M4CVoRP60QZvw0~*r$b#IBJ7Y2QFZ>|TeX*pPRT>8P(3RX(Q>B}AL`(iVQ(d|vN)%f= zEj$gaPS@!)2wjsL@SJ3N%Fb3jznRihtw_rO1iWie@W(4bIM$)4j>H16bTC_TT2K!J zPZu(}#6eDpPu)E}ThJ|I3fctunUjy%CgCm-6O$*+E$*Mm$+9T}y0YmUfJg_^y>H=% z$Q__v6fZb^-7ECRU#KE+YvXIsADJ%|H&R#a5xui%ceT7R5OlB)QL~BGTHZ5MTuj@# zTm$U@+X~LS_C@O!udK?tz>Bu#?!WrQ8)u$AmO5c9bT?!Z(enqZM7FRo!!0Oe`ajh_ zV`GL(cy(65>X_$=FyO=iQ~1tjZ`X{R!nxA z(BtTNY_A|Hlfh0wLgR1k4s`zC*azru@ArdHv?tN;@dEiC%76!=vpwi+-bQ0}L^>Mn zXL`P76pCYuNKLx*#HZ8f{_X%3uC0tvVY-V(Ea6#>1MWj0cd~CdckS2}oiD}>#;$S! z2l#QQoY!1CyD>-jn2-PNm{tnu5SzaOk$DHJgUOAkvOzU!U4zvN+4B2nhYWFYuvG?E ztG4^zvj5TQH%!Ue895&}vdK7z+*VShqRffKat0uet`vucZTSQGcrqBKbN`iWr7Vi2 za#pcDT@2Epq4|m={N8A!;=|>PjCIT8w25vLz;tt_qXVkcvDuuP4QvBDOT}=;Aev?J zT5Ebjkte!S8pEAhp<0VsSwnI9b2O5OM~~qL!DHv@| zKb>-Uu~H2c%hRP)p=`^x#0!r@6N>9iqFWX(=PJYHv>>f&2hinrCxHpQ)WRk5O> z3J$VZU84gPt@(6*+yTQk4S@}LaH0^C6}rA<*s;<8nQ(1a)yXPPt@u6yclxWt7uX78 z6dKdPi{|>OMRbvt=P9C-MM0G%70@>Opg$r)Fyq&6@q2+^hggl_AmtHVr~H3BlJit% zQC2^tSV^zjFB^Ayu0 zR${h|2b2&;X#`yr> z3<*#w@ijd|hH_fKkL20!%jkae&v@}kd#J2gbzENALj|LiO;j-0@lo^*{IvdaxdG~- zbCUf&svVh_0<5>>sXkI4atjHg9@&7{y92!r-GD-71A|$=G@iyIzYy(nMEyah zB$ilVd=azN>$Q?p3>49gnO+(gZ3&zTS%C|=0OmP-x|t+=A@Cjx$|3 ziXn3EwYbHeyU1o+4xT*Hk6N9lKAkNbkG5H&Y7|fn>XukRr4=pA?Za4>9F?JQvgpQ5 zk1i9FR8IF}LsZ@%=r`E9NUI=VV~N1>0R`!-=`qObqjvjIyCHqqxWJP6nB%)?T7RKy zOUgme94*tG8|3_QRxnsh{0+p!Z-X3{*-jIKSV(6y(V37G1=9NAbfZ6esTmc#*X|%S z;A?e?4Nk1tU@EeSLDEG)bc7;HmjgPmuW^8(GOA$D>rn^d_FMO8&5vK(&6N840EUkgaV?|^tYjP|U= zOFGg^vk+GGH(8^Z1LwO@4JF#Yj{`?8t@)O)4y`zPi2HBX>IpsY9G-Jr3uKpw(<}lj zg?z$t~cr`iSBBbZmrf@fO({=%^3DZxyN18Bcfm6jhLxBUZC&8`@oebi98Tm`H!m>brs$TN*~#rVH%K}$ znj%H?6Y^kxKR2p}N$Oqt<@9J#C!6BNO6l(x%O0e!SW(ipkS`Y{q8N}+l;uKP4_!i+ zYsGW*F$-{&X(%){93ml(lW+9*qC-sAX*C73r*Ff3WdAM2i|GJM%zPoM+9A=4#Vq-T zYSW=L_%N4M`J@R6zcV(Rf%~3D(%>sU4Zc6y_a0s&x<9SkSV&%}Sg^#sxH7-H6cs2s z-Og8(Fqc)Ngr=x6~d5$FgW z1u0gLk0%qRtdfg`0$pm3$0ie|1bk9uRV7wF<)lqHTCa*OB#Y>?q7YaCKzuHb^&6|;$dTsb2bO{deTVm2GS zz`f5Wvv`CY2V9YSDzfOik7{>~M>xMb=9g0D%xG<9GV2BDbiz-QJBR0-bSYk+ZjT0G z-t!6t+TWeMQS`Ikk0I}pP;}Rz;}7CIEz@-AFwZzCE#kARf|Vah50A$+-KdxCWM~p0 zOH0T@*;F}UaKwkuZYEsS0>_<-__>}GaE*Kcv}>SW@xFegLBDAIP>%E~D(yE{ivS_p z{EmXwcPZI?MiD{5G`l1lDFf2rNWD~5(n(>UU-1d^1nnYx)P?~4fJ@=V8eFDEk+D>W zW~mjxAauLzKCzEiovKUTiMihxVk#?XQYstyC8hUA@_zDKh_X*(mA=8M{lQLbi_n`> z00AztO2--fvuK6llNR)^ls!reO>|GF_t8pCzhFFAX79U{DRlN34dyLofcUGNNbI2? zw>wEnF<=5wT1MRb+1JrQ7QV_wUKU zCyFhL3a;iv!X=uC#AK?Ge|=+1r8wQ4DLNxGUYC0hlTq>kh?C3cWL75{)Z4|rubGCc zvW$CM`)Bc!Xaew!#!LSi?ew?&AJPGW4mn z6JzTxu5Seqi@Yo=F-v1EIG@d}J8yYpM+Ra5-Vr#}cx6jio+wNY`Ntnrwax{dLYr070~xShF59kTV<8AG^#q$ir46jjb@&>I|SYI z4*1hvJK{}Q#Ht~!z6wP9+RbOA++22i|MT`- z@~V0|k<26j&Ela@=!7puJeO28QT3Q8bq!l`edcL3}3Lhr)-)+TkJbCy5_eZidjM|G^t*(ZAw6m zaFN!%D1$rQsVd?0jFp{9;@M_l(WTop<{Yyb*0uzxJu_3w>526f)1m9dI>A^xh|3zU z6h_z3n3lcmPhb|S|)$|UB$ReaCQIqJ*5fP zmLr@U_P&kZhyR2|!9KSj6hyNnPq0YlwwXl+kst=eh8oi~YL9-+9T+Vuu>Ybw*_fwZ|}fYzz>*7W)0hJB5wuHnnygFRKkB-aOKS{Li>%^ zI|MG*aPV40%WvKx1%(#bT2~z-H4_UYOR>yCgwWY!Ol`uco=!wh*RDKxBDPYwHX2*U zS2z_Uv3~PuGZQn$%AqV?e>4k*u`I==Z5%fv82>9QOXcaT9#plYf`y3_^G<5$@#jCi z8i^mxWT@B-2SoFW!Rqnx%cp~tV|CIc1TG(5!!fv^Q`t#wy$k*g{yy_b>cKSX*aVNq z>J1(zGEhqnU2wZUsB-aCcWfLV`|RXX-k^tA0>zMOcztJ_IFWMZ$E}!_W{JScU+N6k~HKd@Yq` z3n5F_VZ2J}nLMi}#CSSBaIbe$xWO5+Kqs0wB=Mw(Q3 zs6@orT(K})vf_Hg5_w)9qITZ~uYhmEFEaNtpJG;;dsTqu!= zPMIhzI(42a=6y<&;sTX^*H@XNgk8dSA z*f43>CdC32jk9!82XX)@JTIUaGDXUD0RfIrSpls9F_c6#q$uRPRqkupPVnG2oYlNl zA-!V=f3qCY1j5PB0+GydN;Wnw8ia%(ir2WzY9>+(3LG}Xq9KY{F0iYjuzFgDtUh8{ zMH8PRD*-N4(L^5%cdCdp^l_Qt>mz~k@_@ozfH zobfYR5k!SRR0XLsz(JD@O+Gl^97^gg)B|3GL&rOj`C{@PEiRtkO z{jPi%e^Yomc6npt{KbtsFZ`-#;2#O7Vg%n@D0h(!8@2FAKefQZoqihc71j9sZne{PLA2YwHHi&>zyi*!ws?m&_W zJM7g|MjE#k8A-(Iumq{a&Npm1fy$<(0^X$!Y$U%=I`fY1=s5a;ykI4r!;TVGCRpCU zQ94jeWaP|1!qlT$GQneMRl@jNQpwq*9#I7J+4QkgAZ3_YLkvif%#n-|Q>IuR%XL^Z zhfFhN>Tx}p!f5h_9LdaOQj2L>7X$R}Wxfpa@EdqXL)d2~@`mazPkG4C9Jw^6GnU=d zDjF^{a{Up~PDmTtBEF~OTWD{HrMPx=?#Sikp~C8wBXd_)3umw0MMiHfZ&X&t62Rsm z5?M(Aii9*FT4Yf^tqv&&VY4gcV#iKIN|Q;aiokC#Z5|)WuWTM$+*~Wi zJ9hQ4_JCUZ_0u>J?#(Q z66+QY-E_?aaUw_{>k^rG-3b$H3)AENAb(nnISBMRA=Q>uGb523Ei8$NR0`i9@BR92;z-Urlhah@?`1y7 zj5VFgMw8@k_>#ZjOa8{bPo`D!*|u4#eV)ZG@xN_682}*UM6K8Tr@f%QOZLAv8H>fz z@ud5S-k~nOPhfm6GLKT3zrDWATzUzkcss@Ovj6mu_q0D`ORW3D#m0D8qcfA_)ectH zpA^awt85aE9bJT~#AnMDowTdS3U~b=q#M2oZC#0qydcX$fEDDBlhLpbz-gKSZr0lH zp*H0DC5-zW%ry`BhFlYt|8p2+m*s!Mf4by7?HBo$vPz3_Z?2;35BQ%4lI3W$m^2nf z!^NbTZ^c@(&5&hTW_T~4v+;1z37E-(Rh!l0h4+5aD(wGmC7fU=m13IY{BCMTK2EL3 zMU3==O|nj`$VK0ZT=cET#eE~b>;CMWvEL{1yG)AEUj<+<2eHe_{CW2(0?FsqJbv7B zeimlAz88jEwo@*CaO&J>yX^gde7f%~cFFcbm)jQmZzIeJ>V?nOn;C8gVz%Aof(<}* zn`rM{^}~P*Uhd>QZSU_syuW+s{_few{ydKebn%D=*z&v>Pg3^-r8MH~1j0?ujuG>n zoqjNY)rK#lr`mfmZvCBLahmeWuhOm&!k@B_6L0T&41mEMZM=$p_6z=br`OOp=a1sO zou=o!wD(;#3c!ttBWeWmHuVAq*xOxVC2G}7o-{_0CVqOw0~HCfiu&5)m#`^N;v$Ew zB@zwRBsaJsvbv#q?^>vAy~q3z>u%1gyOTFa^vMj&w(9C}*`*2|_Nan~tL-(i7lSbT zZYOPoTR+6>eH-gyj#;PcUGIOr{pem}3)PD1vK=dZYFMv5v75cM3+va`Kj8hn%dX)4 zjd*il{562iER+l}tqc z@-wSf=9ACe)!qt$)#so6^yf}Ia4h}&Up8~&Ry(@_{#qZTYh9viB{idTueJJJGz~Jn zCZp47-OtIHTv|a9MU?|(B{ZY~(me;HcJPLf%%){6E@@Rcl9MT=b5O)) z#!-2MV>HSyjPG#5FI@K6f62T6sXf>`c9L)1NU7~@-HCm;&CT42k@)e}5Z=f71`{V4 zc&5S5L+l{o!F40-2`|E?W`pf=1Yn(XB?bxiqAuY37NtvNoscm~f6)Pd_^(_j8j}$x zi=rVYAt$Rz0;_7u&8zKkXXyK>-hnwBj}dHFDYB&Fv|9Jx!qXYqncf!Q#r81Y(Wg>( zK)zRV6m+?H*+=kx9;F^1=`p>d?!H6%GXA($A3YGJ-Wh-Vb)1$3)5NdJPd7?h)WOQ{ zna(#V#*#^TlRNs(n;c-b$-Bx`6czTx`&;9~@i_S{hQlF^T=QA?nwQBnukfUIcLwA6 zHhTp}>#h$&Di%w1mx*_=eFVfMT~#E25HAO?ssT-RuFuV$8I4h5UcOiqSe8=` z%k2%~tS(M`VhIE-N8>mMN|z3Q`P_4}`9x;+>_}|V`PSH(w20Jf>gIoDren)z;TO8a zOzs}<$9vrH_c*?XJ$5sk{qFZ@FGD)u&e>{L150feejCg#&^4Dq1s%2iMz29op)p}F!RiL%Jy@rzRmi2 z%Jm2v#(UQgV^7vsk~gND`>s0EA0=&E1GB+8q9fu6QP2X_a!!;2ng~@fSPcWaD7WW{ zjp)D9gC`bDum-+X6irLVO{kfcAV^5G6(z*EIY^AlX^g`OtnGxQC0}===XDRArG2B* zUI7NhCM7=IMiutTo(o=Vby}dyrHlQ;73M?qtuv(Ra0!20tv5635>7goRoAUOEO}3( z`()Lf>gx*KGxP2@TdvUU+FH{UN}}PtQ}liehso~!iNj=ke=3bI*}p$om;V&LMfp#k zqVnc4pMI0sth?-|n=Y2Ol>M|#5B|XtqAG1V`fmwID!kEQOu)zbpD5PP(Ei;`jPuj= zoout4REi&epO9xe&meeWmim?LIP#WYue2ESo$`N!^6e4%C-3keo!Sh>-j)5D_vEr&jChN+Tg1OwG8hc=SZ_q}dLCf%S%o zc%k8BjX^4IIX$HT=S(l|M5Mlu2bs!EJ67@5b15+AFiMHYS= z|G!kS_)t`{odCeN%coF0lU0GGC{6~)yHGZQ;AJtY;CR8zWX3RHW?|t_ej*z{63M3p zfhnT|MqHFM!Hz~Hd=9ncllVH84O0;D8KIkhjYRzGO_u{^GiD{{DYosKlr}LHsQi+H)wOP^8M1`8n6p-{m5|T$hc#Z zxt&Pj)(H4ebnx z*^$~iN>X>HX+MBGw~!kl*sP(IM!y#yQf(;AjWc9 zR9QabEPe3Q;zcJF5?EHz1EQGJ0~)xyRF?7Hr2wKc1FOe4u+rYEfP^QQ`Fo6o&5Za} zn~vPxZLG%U2~CXRb9{%}#bzDYz5&}Qykg+i|KfGMMb|Y&HJz+$vhK1-7x79X4WDxN zD!YV1EsW%Lk2t3D!n zzKhrOGR3sE^Q`G47Rfp~BwsR38IO01@@KMRr1%`qh84Ul{^2$r@%N>WHJba~=!m^R zRFJc?;Py;X1$+mm7P>su@axPpbBXqrxGdGKMxOQ7HW*uVskg5*2%+`)K0Tp9d9VMm z*X%u+$NLF*9iJEB#h&%&9dh*^%-C(T6ujPzEc^3ql(sX0Hh80>+P19uDAind%_Q$h z7#|UnbNeg1?x3>Aq^r4ApAnX?4BViFC-)cHPP$s^WIO zS=qa+|MV_>-+2}@{(KgdpT-!bI~DFOx-mQx+NJs6_51EWv_74>es%cNbn50?Ml=(w zj_UZoRyL{^b{m5KzVV6F;+3P%K9Me1$A`73Mm{w~h6Fm9BT)N0?ZH zgm5|ITq(My3RSCdAJRgq4^)9Dy zSOv&IIQEf1eIlqxycE!blA=H}n>1~m){K7&PQo{tHD;4)2g!qz_plTwUqYdg?LH9x z@tH0h5(9IS2W+34e6lP5#6FqJ=en_1kat$S3tqu8J;|J=5}mAjwY9$3zey3T!57Z9 zF_V?c)W)3Sn})xVMAK3yECVIz@fca9t?sv+9hEJ z+;?kwb;sGey%14P&I>vykvs<34RzbXuD+suaf_o;=epbI-Qi=d(*}6HRd+v9xpFKxC$=R-M0@BuJN-i zNbV8Dw)%2V4CL{4inMQ5@-OV_pMy+rYYRMr@8EgnUivm*HxJUxR_?NtA2%B;^nRq{$(EnrC~C=R6{3!T|mbulQa z5NYK794h2ko+Pv&joW{=^?%@}@Yk3Wj*=T}di2e_ZVkp$T9cN-T*yKms}Xh_t3V*{ zav26;`+IkP_*lF$VULZEg%6#s&H*hH=OsB;9*(Bw>P{d%nn|P-TYL)0kqZg<|D9b6 zcpTMrzW2^^cOJVlJF~Oew|2D;t+WrlW!aV=vWy?{Tb2#BF|sYNjV*yJ1K~{|#1J5a z0x2o4Q1T*y5DE#UltP-&gaQFVfDn?V2@OfgM_o!&`hCIbxp!t|8ym1IX|L|i&d%J& zx##@n%(>^>R#|uMz*#retY|D`-^kgETK?lhP4~6HBZ|qsqodhZIRKD0WP$gMQ5NMo`WoEOxRmN{ay{Eb+?mclh39R7Pjr5h&r+>GsoZJ^vcSPyauSiXH^S*$pqx`kAZs=>89{^G=J$(cn1(hG zWYT@t#yLB3*?d!DdbrkoYBn6zd5#Ok6-g82pe(}5)3l)N^U0D%LJdMR7EtY|4wFqp z8YfCn4h>c3HU-8u^;U?X+O|C(;pzXH{s5l~^WV<=4Cw6B_~|rmToFb$ih`)|sC-Da^E#e?`7(uXSW2B?rRbC8i?&(lmi z%y%eFLQ{aF;mdeoL1(xn3wS1-FIlxNN#GSZs1JO0k*x5-@;MFd%~4L)WWP$9|!_C zif(VaDdT>W(?a_e^E_VA6>iNqM2t7J*zI#b?gmT+4_vbd8nJ9nw5=E=LEWc~$O%7f zPSn7>6-x!1TIGyC;I&5gHK@JiP(EoB;6t=6)BipFpKR~u2%9(RskIp9O@wh<%;v4v ztHFw9F~g*Onw21cb$6!1-tuO|IJ1|NF6un(3ruT|R|m+^<2{*kw}e))UBP&s;1_Bu zLp|qr6JO*dj7z7Nppu)P3l7iA42I~Q&cMQ!u8oVzKE?47PSAkY`fLempr09YOQT)h z&QQ?GWigZ|Yq2me1P~?F!r7{$JFvXdE!6Hlv^1=^`#YWP4f8OE!$nahNi{?jSysn8 z=a%go z7~4;zNGzdg(6ofEgX(OBd@^tdRnp_KXbO5vARyWr8X`VH@yiDJhTK%^jzkJRH6DXHbg`c%MVx*~)%yQiz73VxA4Qhr~^o-8wvLL!_AH>#Z^&vx~e-}0nh2Cpx|_tW7&X$09ppJ zsW1(cM6m%OXNe&zD7egx6KQ-Nl}26ybfd~&#zc?c z-{Nync_uowQ+4;UvFA^e4mb)4-8IfESUv5`xSe6*oW--x&JWmrInk6i3v;40n{X%> zw88@C*IN`ZoNjikTrO)CEm_x9K@C3&sIhl2r;@ZoqlL#y*+58Qn-*( zsr)`beu=N=PC3b#tp|?dIm(4TyoYpzbmKMT?vuRs$Z>Q6vghb)WAqd9EWUx;iXum& zqYrNb`2F$S`{BzU>93E`=gH6T?cCb`e=OeLpV;#QPW`0k0T@k@61K=|jHA@^2fHZn zX#M_c$13dPwd1Pb1Z^Q1oC98<1CFRigJl^SKhno%%kjK&$Y+jqpkbIWI{i5QA^8R9 zrI#@YLOq~|f!pjwSv1Tf$R%Elj;0q-!Q(Mv)@3(cI?LS9Qr{6%-zsz1CE81`-qY?d z9~=-u4psA+YX=2_E}x=*Fg?tx-uu(>^TblkWFX$t8UtpN&Tzaa0c!|4F%8=t2`y6A z0oswsylA#ch^_=1%~XGTIvh}fEoTzFh%BMpObH))0upu{`P5RQA*eS?Olz7Cc(5> z3`!PhLaZy3w98Bm)A)ZLRDri0Kzc5bxe!J4AT^^@%XR~LhDE$3H7Ch~4k+f5r^J;^ zvDF4iM#4l1L^>LK4h-SO0!_%MXEeYlQb3jnFSa!-E;qI}g?XQ(gcZ@z{SjYNo(E`M zJhec4coipcW`_k5%`f$~0D%&o_s4ve?&AeP*XNo8Wq>iERtrR#-7LbW>MiD2UnToswz>3%Uk@^{=kbg za+;`Sa!oo7UD<4lAggJPCycN2Lz`7jmM;TIPS#N{;1@tEDq2~e=955K1CXYJY=_~- zg)Q5+S2*6u2?CZQ@uaqJywxt4jc|iPR1m*TLwP`7Gt(F2SIHW*i|yE>+$di?9AR>l z`GC+(@Ge7vc4zXmQ`Eb!o2CF}j$(AfL48=_%*g6m6&WeeEtc8(0XN{VJ4iq?o|5a~ zpKb*}3V~tZ{g-4HX9OPsm?|olm>e)1o+wejB5~9xc?%#Kj^@@O!39>@U~hR0@3A6k zY~*P2$(-W{whq;OG!>QA807fD3h~*=8EFLmb$vQ3Gi~o^)8gNX1DBTiKiv@ z+5h|$tM+fZ?B>YQ)39dk8hRDVxAdf@hN=U@@Y#-+^7+-t@j~w0=4`lZ2hPgpseQPN zHj?M?t*8Yw%Vlg_8@zEfP=hzFd9UV1<+0g#F$6=&cP5y&>l4Pny43H*F816kh5EG9 zFN8@fx1bisT(aDotPO$s5|E;U<)(p-RHl1DvSUM7OKG9@7yWU?o||p$Qk{$)O=Hwl zEaXBlP(zg#e8A3kr#X@Hrz^=orA3uP5{@VB;_Q66KNsw*2C_a~eM-)m>1-_ON%LMxLn8?S>4ZL!8Do+xi-U; z@P+vt2EGMaLb92MmCX?r6A9WT;}>Dv-S{GIR%6#Y$kUmRHA3`wx{?#-j&329!@uFh zNXSU=(0wy3GlPVd8b}JVuQBQr0Cz~{9Up2iv@{UJe=jHeMnH3Da%?SFw@x^14nR~# zjaWt!Kr`kJy>fIfcpxsCpN{t91=NK47^0wjfkut)X8m~#uw6G2Ty)pz=`x4y!gOsG zvK}*}T`O=O9{fTk-vw(eBn1?{;6xjFL6I&JgEZ(qFSs4vloA5L;%G)PBoYck&q^kM zUK!sg$NWJ~kolxxMMOjO>Cz^erOYSFS-+vHHn-@|j|+J^Ilu|MZgE}?$;9;gWG@i_ zC()dlIZsB=_^GsqHe1(1Jr=f?XT``G)O5xCkPZtC3LVh-s&qC?1`yI+hyB zwx@CXp*KqXjsP;%j^e8b5k<}i&=iOe8wpkMlS9uYoDhI9mD+Qjo37wt@|L$Vl}1!% z@zCjd_k0TK?M`w`nPEFHK$`bsmJ_X=g@mQTUzBqxBXW+crr3stP1b;Ju)-qTU{2Na!fk7ymwt1nWz4Pd;3y!;WfubRA z;{sf20nE^>4782Tx<+`*P%V*@L?hYQ70DI?Ce@AgMqL1PT)-C)PWD?dRpI?M49YqX z(mbcwn5&fHMw8Jk^9rX2h@@gr-g#asm6Q31ABZU)vS4gDDJVQH=>7uX!cjX&WEu&M z^?xZ~m(PIw4bSMfK6k$W>kTOE(veXR%S&eUFI>`h*~3Z4PR1MTB>v;-HOtnmU2%JV zwK})2t&Q>rv=={zuLQXgK!WxHrT75tSVClqYX;0}FL(eahi>$a*2 z1pla^pu4=Lj4+u1eWk?qQMuQlGWsz@AGrCMd^Wuri`_ii4La}PF3Ed@m@Rl)7+hZ= zsVo@Arsg@Hy`09x=r5fq8Zxo%mxWpVZmf)XojfiTnlC!-{%_N`7hA35Ab*ixw&HKUM`y&_RO9R_KcwgizI->`eRQ}8@j{S zWLITzAQ5;PLo1hBIji4F$+IRpcmr@J7pvkwtiFFWwcuTfg-d-toN2h_&}+9yq#-!< z=gZw}b_v&p07WO<--H&VJUNaS-6Ev0(P{K*wYCllLZMk85S_5;gvBh*7ex95OcKK# zjWyc9ouQN^3xzE;UoqO>J6`=4s4Hpue!NG&4Ezg)XTH{{OE<#;R= z$q!f3e0j7M$Qj|EwX7*=`hBJ0lBT(|wVSNM1LP(cE2{6*bIcqWy)D$|$=mD^53Zf> zrsqXy{^Ot#RGnjRCsDWVC-y(KZQHhO+qONiZQJ%Fnb@|IiEW>p_ujhao=@GqSM~m| zYIjxdy`J^_CN*DDHz$J~K&=ewwF(DSbs|jRZWbQx-MgvB4C#|-?M=mlh z#kGfT&Ra|gXn2!aN6tkthk{A5;Au!Sg~jfWA(4m4i9Swnz^vKLq}o zkp!-Ha#IN|3iD3xmY~t4dpu=`;x&~Fh}q{5DA zs6&p899EUR%Rt1YXZr8_naM(-YA(a3yS!GASXlYmAk*1wasf7usoAfvuBc0P|L^dn zbdbig#fQhUxFia3Dg|)jsw)!l?>>8$V|>1OllO$HgqacP(qA-8L1U89M4uUPSAVf@ zvrNc2|2{gIJGD7o-+F$*evy3veZdhD>Ck-WZIXRyt<(7Gqa9%%KKo+s;O*4(27igC zq>RHamUqhfO89DHFJH~+tsK1ZzFh-fgVIC#QNNfQx82rutaXokD4&>}t9dtk8nk14 zP2AkewKdIiB1npp5zDiwVA;ip71j-L86Py19Z(-YAbA;y??3%5ICQwXQej|C@uKXh zZm76jn{1pc%1CQcd+NMsk|ma1Ve84-$mpVfmisyjQaz!mrps1-QJbUE+n`~ixAbdk zZgjCc?VaDCy`-hqrnBp)J9=F9u=YQ%8}}ZEpk`Oat3s=SUv_JqvhT{j+;Z#r40;O2 zb$7qIigfFl_Z6OTn1SiJ_s+{*;ye%JYq;+2JvLu4!_RK=m|i*l@Vpw`B!72*>f^g^ z_MDm4XZa2VBZfH&m50UASnNCb?1whE$?H`MN{?Dd(=r@XDxs(tOZMmxQidw4hbK7t=_AM4^!#KpvF zu{Ye5uHWUDjZ%%4}1}89T^+qRDYV{_ksfQMY+MF-AoYdMD!K z9)}(A{A_sl)6lMkxBtA4s8fc1C3haR##vTREUnRzoHC)&k)6Z*cO11^XKY-xT4!pU zwOWV!Q<-9=;f;ZMrO7EI1zV#7QR)qO=o8ZWyl?`SU9IwFx&~Q538tG1V$>@_{dg~% zS5NnsiFcqnx}baKM3poH^s}ccCgw?+-WJ58dv_hjDF4BdCkkPyI{K`X(%l&mbP`N=XUH@OyN4^L>8Q8fRydQ+ zY85rb%%hhpYD)V;Pbc^Yybm#Ni*t3Ot&NYk*Tco!L!qv>yR7V$6nykmu>bKDzI5A) z6>rVy{S1E}F!}ub?jftU*z@IvtDD!}-&|blk?Hk5bC>z|{mJLIdGXxE5jy9?goK7M z-!8J@1>2NXt&NMN^Z6DnZ7{ticR3l`JPdw#zH(JAub#SD`^CN!D8r*3hy-8Dby~E_u&aI?edg@*rmA{r2!y zp>MAjZTq?ne!S1))&XLtZO{GgGhxo?g-{%gAT^;g(?QmA^5pz3BgWS&5IY~`)Q`Ax znQ>g^ptX!+v2p?%^W|__E8sg8!cBsyb-~1}Pqq7>d2qZL0cbYn$9c!TMc{RS&hfZT z=}$cT-e>6cV?gVp{%fLCc;J^%$ZkP! zz8hYRZ>nlnT$V%v2O4CK6fj6$*i~v^EHf0AHx8+Ml1fn7iP(tF>M94=k$xJei~ZU4#nXIE4_d}K}NgEW`*+=ebGLD zeIoDze>`WMSh~PZ4nCf+J~%&scLKfK)Xnx4_6f2V(&t0WZJnq-m_FdTz;yx`j#^Im zX9XWPAB-R589JXq;Q0QOABmq}{t);yA8@?zK0!~8z#fRI^YaSri`eHTPRtqT>+=1@ z{loo6PVC{=;fngu|Lx+4y&exlXG1hK;2-`in-x7Vav|&l{lxA>(T%MYNGoWYGd`i6 z1^x>7`t29&7o}8==uF@~9q`J1cTDQS`bP8y`^NId*NKEHkSESw&?mrNq&=%&ura4T zhkGLDLO+^UXMp!fi#AI?TQ}F2zfu32l*TyEZ+i8xHUf8JWVmdYY{cy*_??_QMAX>3 zRBWiHXoH-5xFsUzePjkJXS2_J=H!dgc1iJi8UNAlJK=Z)FLeyjCWRXJw!8th3 z%@mu@+a?jud1mki)9~BWgLiNmR1>_X1w*cYBGd>aQh|c6;3UKdIZ}p#uAn@`2q{v6 z0AfYOZhQP?iNE$NN%`Qj| z1+pvC3=`RWQg;(RGBTdcJ}%alWNc&jSAPY>l9N+lMWDh&(`MsPVwn#@_nJXRFM}ex zH&sC~L{o|NZP;|yN~5GVP9-I?Yhb|m#n}Y^)OnUqIHow$=j8zMYk^Y6(jQOea^1+0 zut`lZWKYT@PunR%y=M)xi8uQ?4Syk^uMl|5!>mkDp-dA<1GJb}OIV9;r-fc>_7KR~ z>@?r9yjC;gDGEmcTUyTiK>w3&#F9a-wqNqmvp;Dz|4BLe&s9xS zNs~ZQRzY4#MNm$KKu%so^gpSW@3$IIK2vVgkm_5jBV~o1( zx!D%Gz3wV_0$I=DotnOwHY=KB7{q75FnH|X@-h7E-N79{YW?e`UYidzF$M0spN`fy z&@y6efmEsqk)QyF%VKzPG~?fd_&IUM>?t|2vF@B*t}zeat2{Q~?X;r~*_|C=ao?_yTn z(NEi~^Ul6hlRS*^nh(J-RSN^D6Yn|HfH z(Y@r&;vmku#s)~FCLBSrX`aN??k0bK+Z#aL-97Q@eh2mGyn?4nk@Xv*dQ++Ru{1$n%6*Xf2qSnu+ zrNRqwS_IM38zPv)*otmVglbwP4?xSU8 zW_k7096g_1TnM52z2&TU;m@Yc!m&7@YbiRE@#@*&Y&M<2Yks?o+t6e_lL57`Py9+s z`RZv}Z}c?z0tVj+c>b7iG-8f2PBM~?OjKHMN{VbWzF+tzUt#_hA6vh)Uj94Gmy(&6 zrUHDLMvht{%d4cMRBy3d{hhs`F!*f~?jo#9BN$6yIVwYf$#);GqLCP4GKdmYEyk3f zlP(oc^b#G~C@&O=e+ZqRw91F3HE83(A1o8@@jAyVVyj zwM3{JMQIgq3;P?cmEz4_<(qs;wEp>quD7D1kFRi?iMg(1?Nc z2GC&greMDZ(t+ii313tSTTkOXVZ<)=FzNR%db7^_#^!H8K!dNbx|rojG?2xj!OuQ7 z*zh6w{I;qw|I%;PgjsQd>o(r|L@8$Gh$2}v^7uLuW?6`@a7ZP+p$o8Ph1vej+Wgu< z{BFAiuR|Q9Jez~qHX3CLzhjIXjXwqww*6Xq`hLeM&?wb_<{gnZAx-{~PXU8p`m;2< zvfeQ~$Qb9Be1pF+z%cF$L+A1vDE@<11WlND+aWU0L?g? z0KXSb6XfJuGZUD_LR*JWHv)PB?t<1!u_pq0LhppG(W-cVfFl7B5g`ys2#b*YW<)bF z?(DzLc)VhCPJsJ}u>}5nT(*E5QCa-LDCZ%p6B=injz}#*TKvSQ_b9}u-=XfItp#ph zFm?=B10fkj3(}#mNYqJuTzp)De7t;u$RV!8{vlPoumoBoikCL?CX}9tRy@ckwL<0CEU?lZUjl&DNa0xH zZ^SAh`o0*?Z}eM2zxJ6$0(-(13Euet7DBIbf6>vlQFjG=>4GhvxDO~R@drqs-$_Rb zK{G_b?^2ubC}_fw5%KuAhYhH;V~QN#_YT8(1pLH(6#ppHsN5(LQ9>i3MSpBuUAfPa z5LXI|3RC9d#1`h7ijKwXW4FXDEh&){{ED`cv&4WHV!YR|3H0Y{4UMq643>! zp!R#^4*Iy`yn~e)rSA(rT@Va2kh!lqPqXf`c^hI1S^BzT@v+1F>tBj^_~p*v>-_Eg z^6|Cu<74x_mN=99X7gbl_x&|{t^a`%`*~`er}z3WS=K4vg0DE*2%EZ+>B*B%>yyKB+RPN<#2`sl@ZSBOUL* zc3sa3*jVVZG2Q{`p$o+N{3!|4<=gqgyuM&qJ##CpED+Y|3;9k3uUmz$lf$khqG=^z zuO}f}6@CYj>4h(=au>(A&W>Ynw3QO(*n~8)3azyYLb(f9od2*Llz$WJVbFLI-9aKTQW2Gw|F#r4Y&dyryw1z|<}^J#Ft+3Wcv+2RY>B)6 ze%BxYc?#1*4&A;(JRuwruET)o*;P5YMicUH#eA#bodt(@E(dO}6>A?y(pg~0ww#G) z?S(vg;z}M8SK!+r@(*?XWhtcui*TtTs?=2P*heG$7wAL2*h7Z(+;C>$1({T&&TiNwlm<+s?RSXIa- zW*fDR>`Fn!OT}JEQpq%qGbGwk7^DcNc&R8}Bqy2^)rs|}PmIERFXil{W~;BPmF#bE zb>Yb{6;26MYHpzQYhj=`C@8zR`y;V=V7@qevio7OdlJc3PRsFDK~PLlOG#2|4nZ$I zBSSqgRWUiaiIkm|7%EIBgHn>SYQae=(do|B(i78isHCGqI8ez_@85-|vl)Egnsb=o z0E34a!5*0GUs5?q$UF_ z8Xy>T(BOzcgB4Z{KA3gT!1gO)o`<7uelxpxK9RO>`r^a2^nsr#Soat(<8sBI?i}u! z%{{rfX*hfgxaV+BAL$t6na2I+c*PL+e?kYFhIs!kB#q23L+_V9`vK8m=yf~zupfRp z1iui4UyA;palH?Qo|pd}V7E^n=-zvs!Y1RkPv7n~^tul_ox(0e1l}shaD$6R}BG)3BQa!GgQeLTAEv59Tno~JZ$zA!Zs$cb0`c`(9HdiDbhZz%> z%jMO7f?C8LyUpfxd=gv~7d?%s%g|}n?AUDF?CfIUa(+@&G!xy+@HT#uW+!H5EfTDJ z&Rn3^cL+V(U%OCn@K;lPPSC4Mi3K~LGw*#04s?&by)x&1qROCUcPTrtMIm}?Aj>nk zO5&e#L^H<9Gx7qz$O=w3?o zHz!_Gx!^jjT@k&*2W@9G)D!1Q7?2XR#eR#a_UR=p&jb$lN1Akx96u-?U$E+{U&IPy z^l#nmY4`PZQWPsNWQB=))o5DZKCexkrf*YJH;dXy45%-=P(QW^3#MKx@p9K5{ykOG zG=;mjj!lxt9!K6&ayBq#iRiLJDp_O15S5KvrS%=Lf~0vP#|7uwgS30K0;1+#%7|9C z?U&y5wDzd3`DZ-y((kXe=XH1Z{$C|GpW)67QmU94&m^UZAbqA~NtLMnv4`^gDl_pZ z8M&;*6WrVEG8#3TllEJ|J3kVqIQ+-Z;!ZRipCX>Y6)oXpsV2K%w+%V3jna?PM)0%} z{5&uydfoCW>wG0f75&aOD}KT_t#3jNI5)qSxfMOFfXIJ@L4LW<3y(HAHDOTeK5TIT zk$`9k;-&LZXs9LHovd6D?P|8F7{Lq!yOc*;w)CyCt*?zD={<1W%ot( z1pzSM-g*QfUYmeN<6(IuVWDweln zbP}@#pv0q_@p`tuN;$anJ*z9nZqkVq1#q4@CCOTc82w_&$J#ZjE~awR&`T@3LMDZ! zKi|LjMv@=-D9|xt1hb zGqTip0DE9!k#5*1{^9e|S?*n^{~ z)0Ce5y8>R+jEl9^VE^5lkN0IeWWk-j+xMuM<(tvPXB~VNs7<+s5T3 zm+yy>E&ajDE7cZ-c|+z@bFGGU;RG}5VIGCGaF1G565#{tkr&KXqMNs?HBcp6pk#t> zCRbNQZ!%LmZ{IHl4%UFkvCRzq* z@%4ad9a9|rZWN_k6DPZLT&9$Cx)V2beaP0hYL=sih(|Y|Upmjlm7VP= zjJtazyG+5n&*L93U|Z?QnUE|#LiCOWJhh^}te|_a|6rNHI~zJQgliw4QTwRe(`hpn zla1N>h_-OrJn@3U`;~S?cmGJPe@*6{R9?VJY$!nCV1#qRa05WZs)=xS;{In9{@3<5 zw|)=GZcllha~E8CH#J$$>>gFA>QX)vldf|Yp+6<<3N^K8&xXWSb&8K)CjXFrXKXO0 z6hq@-Z8cR528vNdi%49Cb#p{iPcodViPsC~JUr52;0ZBulorC2>hmS3YTqK-)9Bss*&d*EE|lN(T9fH*O-2;)hxK z^10Z;6^`#!p;gg}3W3KU=UF6;pk>jasY#~V`H<}Oll6RQ=AQWf##(>y_5*>P6s0r_p6cK zPz(v=6ot0?vYuONcF-F0Kd84RDacT&tD8Zm7R>=Q`WySxb2kDAhzU6odu6d$bxI?>_4e2oVA$9hYQ;j>INyJgvL)C zrO_3-*_gwIS5zkqCD5~MFuWe6F9x+>cszU!rSqDY)G5`hdGQ^UWF1d$cV~Ke!swLP zN=DGns)Nd!8+*Ik3k&-Bc`DlJSxUo~4AbAlr22~5bc7^mEt{>dQh%jd0TNU=$Bdh~O8X}EY~T!Pf2{i|=RF1zS45v5bUK!2ZQxbH zn&}q%IW=;F6d@<>JI@C83w3;DpKco&&RVrdsGBF0aE0B`Hn$cP11oErq5a^uCi1!i zgL%gTS~}Wkzt>ZvW?XFGOv4x6F3Kx24K$X==PRH10=Wn^umDJvp(%hG0KE%}!_Xpt zJ1J904hb%HpKq31KIs?W(|0(~SubfpY)ijX zXR6Zd~3~E1b4-Ru+zzz zEyd(a9mV(}XMpZ_Y9xC*Improiyqe{!WdOOt5Rvkz0dyqq=K9~ z-j>#~(wz3Z9-|;BnRG|8!n~^Q09i*iU;m)4!OmV?k(KT-*n;%m3Ti4@AlQKIIiJn# z9eF6mPuevo7?=g6B5a4zvD*$2`-!! zsXRgA4hcb0p1Cm0gjAgJp-(0J-c_0POqH=w>v3a*G4|D@czoQH4 zk!)6c7TwMsPXArJ!obWg0MZ_PWh!*UFlkB>pl-5SMglt?^L7KyHD~vuvn3F=5 zDLY7-%Vb5ngq6y1A;e0kN+qrM=w~Zw8Z{I$E=Yo-s*_wB(o{scX7lG?ir)Zvh{~J} z!{b<04#6w?ElO0}D`YMmyvUn!btlLw=-V&c9yT)_LsTWwSf?&4Zz^t?rXFHP1eB^R zb8_pu^_4lOHZ4G6K?(!uxL%Y)LC%2HH z1Pv&S*#bz2hxNdciUi`x5{N;`VqOuoWO@een(`KR zc8%xvkdQ7eg-7Wv^8B0UUjn19vDZj8j2S$k(;?KM*L|yl^@G^qoX9Se=Y!%^NoF$I z%x0tcH%V>s%Ul-=rQzZM$#uz6$yCWn62nCIM1Lg@rSsx#$uctC?1%Ycb+R7b(?Q8R z`8T)uxFmF0Jx0$1*V|-Hp}%XOFq{EV-ohikKooH=git4Uwxt)^McnW@HUFY8v%r;N3{DNRQ4%=?j!nVFVp z*U28X=ci5mebigxedGN(4;ns7K2ko~udnVP&z+eaM*m}RZ!W111J?C z(;&Y7cmuf;$rJhZcn8|Ol#ni{I?_#PkLQf7nG4_2s|%?^{##K02NiD`QMUZ2ams);(DoqG&x5@&Nm-rlA5uIo$V$VWdDro+Yp&4kNXL$z zDWa=G@CmT8!Goc;Ar9|Pac_q*VIxwiUPEIh;hAX1$Co(wJvzU78@<8%@al&!jAc72 zBS(x3DYnY}+(gNS5+=%%RE(}69l(TWmr0pNRxsm%0o7{$E1Ow;C@okH;(b7OC#(-J z7wh`l-FjkJcj`dt{J*>5GcRxCGZj6KIj^d+oCX%9>X9jKsImn@sb>B|%Zh&5!0Iqr zIevkWYx?Hnh0<4IbCnX?tw?hSr`S?uIMYj0-J!>nN7WY9laj^G0T&-}r?u42a3R^( zWJ&w!+JTzHDf5znx1_i8Y{^(W|B|IO$#;MLJ9yVLJKqrfrCf=tqQyTBbV_OxLH2hH z9Tw=|bXJnD=RQaj{l^B`{{*xup9mWGKKEnJam(l*=g6P7wr}QPPM&v3V=P=8$UXuL z-*d?;z;-4;e|zKiIW|5T$%nTT=0JyZZ|QZjQg&MohvnREtheTE$z4jUH6CI5jBlat z7I4=!cRQtMeIZ)g5o39MDCBebyIsGj&#m-Bj%f3?Z|v>ss%XoYj~)Z|$f47fa>tLD zY|sFz6TB-B%hpQuv9)P&26wOwh}kOvE$l#_b1TU`$~!9mMA$FFH%@6fFppF2pcuq* z{To88pxK|jPE4=u!I}H*bl|w@K+svbgAW`$&(S>R8$FXX5U6Y#it6-1i49C)=qLos zc+e&#Y)ZymJ87st$3D^yDets&bisHM%w#ckiMN(g6GxG8=J#nT>?Z1%VgphTWu0NxJ?tw{ujHdbXEn{z1V+^oe1$n9=SOI8m6&R@bIOOn)Z*^e}X36Z+l`{O1XsL+4j z0Ug`2{#a#VcM>N`4*`mC8DbSA{yCxtGE^6|zaoIRcwpxsNqpSHiCz9Dgj)cJsecjH zBRVG`QBhax*H9Ltd~Wo2!csuW@l_9SQEXP3+j!Jba0LCuYN8KJbhFB}6_skRM5uT!?T zzUhVEZ<-O0$|+zt>Jl*SIQBhG;xCuFIxOm3RT0pwf6AA5+hp;`)emVBsCZ2Gq!xdi zlb(nRf74Tq=;lig;{S7ZgqXv92R&ujdjG;j_#E%jNhFSt4+D-MNI1YsBj^B4yI=Qk z3+7pCO=Pg6NgcsFF8UJ7wJz{(cYfkS_UdZ99+mQW3u|dgWa=((8t)%eW^&PH1Fs7b zAD|lFy_(x5{zd5XSpY8+0JD4J8SKOGJIl}GYH_}kszRz7$+-kcykxicU)O$@gZQuC zE(7o#C%|4Y-7;R@oWIF_`Mt~k`er(~*W}Zx7i^PA9vrks)ipJ29#Vra9?B^oQ|H}t zyzP$SGrizz1opSunXa*h?*=BzcENSWEBml#?_2b}o43AZ^|2H#3D7|Pvr*kq3u`nH zv>)>L&~NK`kKqYAj>;`H;rO`c&55Pvt$Tv{_6xW)5S~83@(NtOsS7G)kWsTEs4JvZ zg@M#-lWr9(wvo-9(L7>D7dZs^;r>p=vzW#%H4oEeekU+1vl7cz)>GEr*;=?Ati~SB zHE)1l%Ez`coGA+o`Sx!noV*!CV^-vegQv7+8(R%WU41tKEj@z?`2h_5G= zJ-4}2VR8yi)QiPUz8%loyVLHP_qgKAnS#4>DeFhV=JvJK)fj(Q|01)AFJ=2p4)-f9 z`mHM4m1WgJT+#kJ5}&Q@$t{JkZ8H`JFfeRg+LKN2acy1e^Ib*#$qIL@+O0xH zfrA~Vp{FjY8NHJlL0VK7s7D?x^%P_lv|2y`r5 zSs`Mr>vHt73Nmrx6#>VZr+U(mr?W`rKuGJRU#WC8sX>*C^Er$JOj z(#vPw6%g+#J@prfeLVbFGLKlR%0hPpd!|E22GD^CXf>(bHWW*b*h;6>M+;DIYDF8U zA}74eECy*!+NFSQ_d=+@N9`?KWR?vzMHNl6Ji{PiwE|hx=;^A|#qFwJ#LdP@#&P=vfMvOpgO}MeaRum8IQ{Bwprs<~M-Cfq}M^ zcDL)G_z0H(od;_{_u3)sBz^+bK~Kz-vGg}Rd#0w!C>j&|-g8&748h+}s7KcuN6x;b&$PN)prhn;7zb)_W0%+glCStBu6L4H zAnf3ozCW7?rbTW81BeB5_%2~w<{r086=uVMlN!^yCRr3;t32uA*f?nbT0N23Q0`T> zIYGlVNFvNUt!uPCB`TP+tg8yk_=i?dc{_yQCukM@+UNrId3W6~Vh0HxM~r+Xa7y0N zt9LX^;v65qbl-xf2`kmErMKl?jA-yi5T({0cYaHJG)iR`$7dN@QoJzQ0+q)JXOAQp zj5nKHF(&0m4LTy&H?=$P~t8B_KV@FBdwFauXn#Qf@J^ZNAOOtxv zU>Y!P+h6+-Z{Alw)5z&;x4M3*NZI}_24nTinF%aY`vR$X{=ORI-#55O#b!Dyg$_*+ zG9|4euB`OzjbyuL+aAZ}3~OsD$H%h-V;`OH8nC@(r>y2%)}Tjr#KvTXq2g5Fhy{sM1EY4_#woZH_#v;nEn12lr8 z@kQ~9%MCAH<}X)TqI}kB`EN{}8+?M**MHu3s#{GLPJ?TNd3&H!D{wVP%|cU5aA^z- zb8t}GGB=QUw-9!4RXAnYIS+ANhD9E&AH#ORwY9Xl+M5KXZ}y@2E4ohOv4JV%q#ifB zaGIN5gb5fvBbS-JwO8{{^UpP^FiA(72;=I8<`BV}>&J?~@l-N81*i4+_~b_ao3NN8 z`Q%!VI|n+rJj;6X?$}Kz#HaR~@s3e15Iube7FKsaYns}vfuy&|^^+NvAB)mD=BUaR zO9&)Jcacyj!ng?muKnk15L{NZw^jNLy=>GgCsdp zW_9~dTmFaZ2sXph7DeDlZ;)atKD?)pl7>c^@urLAI+K zP@YUgAbNv?0RnQB2b%wa)_1}&k(XbbTyiBUlCotd`&amht2Xd@Z~fJ5y2e=zlC(Vj zdQ?`%#YXba5s6yv=uA;%jaOGJ{V_}jFo5=m^B?54HJ=%$o$yjRGUa>$38v+TzW9FC7GMv|(OziHqzWP`uIs1sL? zQ@z8MjgH>ibZp;hIiPoG4F|FvypQs|YC~Ep)Q4u{toz@eypCE*XO&y<@nKER*!k1@ z=#Ibtg3J1g0=j*Vj;6^ZVGSu{Bwm77?Ew?6AL)};B zJvs5IA-NR4DUe`wSDrGvTWwI%;T#T%^w4iA4T5X1>yDN+T6TvT=C~m~p5A?A{9RPd zIy{lBd7nm^?;PKskAz;dYS6~i8MUV6Dig=;DRp+M%(Xw#elc?!z|4|ArWONJW7u#! zp;)P#FLblTQx>v#REjUj7*6sX=^Sj)G56ekJ*0nTrM4v-3*ChDeCDJw_>)EXrHxu62!m0yudyu2kDQTsf5r;t8?W_s1WZWh8CO*Yn#+r|g}nqU``&J8W66=nJ>|YgmR*ua z6j^vLaGLEnbi=r5z8!>UU*11?4mzEYeiYL5c>mASLZvKK=@f0Lha*%9jzXTnB?cg> zidj9Y{7@u9dZ<2rwGM}qm=-T;N{F;1Yr?5Go>)9Ao&uLOUo?ykY+&ig z*D0XVgz^eRQvOZKV7(&tTT)Cg`7)RMo!%N+5U4;@@e-6q<_d85tEQ};irW$%-sG;ip@sL_p1+vbZCFEJ!?WeTGzEo zBY-=q%+M+x6%&2_S1^(Rkz01VR$5<3DtEHi*kQ8*yHFy%gtKq#y}H9q^&P!w-Hp5pja}jYO-e3V|qy`Q9|+xWg2(x@w37Tu<*ns z`lmzQi@;ww1oeU7X0gGysZG2y7a5LS1fFJf7tYUiFVYISm`PtGWqQJ?gouQ2mY!NP zdEzAphdzIN!vO8XK)yf4s)vUK-teY%epqQ~KCMUaW$)J7S{0Aua36MG1ghHS<)m@~ zpB96}%*mnSuXhr%lB#a3&1oRV0Z4zheP7wO{e;@AIE(FLI|_J_S)hq9*0zU(^&5OE|m-|KiG6{S%U`hqbhKw(N&*eS=`;{ z@lDf!=pL|Z>o8&Sf?geg$7%c_Ktf$X6WLjd1LPb~Z^~ueVLSaoWeuoe&qsr~p6t&s z$mvOo#~l~l^W1sBxd+9jpl3mxi-P70-wF5jq`xj*oRD~wa=4+#QkGwfGtE1r$C%FQ zx^$Hp-4}IF(QJXetmdKd`!(1Ql03oMflXdwS1TwXSZ zS=pZh_%RYHE~M%gz_8`nB#D-M^L^VIx-DTc8W&$80C4!vvh;2|nSZyev$jy5t+Y$? zg5?%UU2rJmf=qPz3+X;i|GE|@-5ygU@*XWAw{e58F4OE9oYt%npM>~vN;`0kqJGxo zuCiCzs_CrqU-MX7{tHf%D|O6AJW@zTdR>1{V@f@p>R8(D6yujxvg*=!7JPRtV=hxB zTecd%Wq+IB6;#IqUwDzt2K)Z#;_Kaf^9qdvV!Soh~B#68g-i$Y9HY&jIPAQC)@ojg{;QT~ zL)2v!0kK&Baiy~txy_%$k;8){bF?7YPYp}9=h7QnI={|7k#&vrlawejCIyXX5aLg*q;KaJgd2KOUF?S017an?4 zlrwm3xK(`67g$v$E*a&{GyVkBSdDXN1$*W2{A{=gr3goGU%!mJ9ppvjDL2!4C-}0SILxciG@;HGQU+`z3vR)Uw44^g*^iBzzm1Gx zha0|pQ!I2rqEa5jb^*J$XPQ8bnYZwsbluXEE@DMhP=hgYLl1>NI!i3GIsY8wl2J>O z6oWC+4E{1c1xK`&Zy?p(g?HU<&c9_Pa?;hypco3OIW-M~G$?{wHlDOhUp8nqc^WL6 zEA!8g8QRVs7yFG2_v_u9uvnpT&+3duXPS{ai6a;()nO1m3Hh-)33a zWEdq=PB1$DieO;t!zUz}Tdt^SxwBqZ-mNl(J^gosjRI67mGsR;-)Kl#CKG0EJgWjk ztZJu5WzDz!Zi8Jph`~WjKYHcvD96iaD7WRz_Z$>heAP&6D4wEgn?=$k6GQ4pwJA@h{ISt8y<4`K@e-U z7*55!+3Xp49Xv%Yf%+#0!~L(tk;|QbHQ~7+Koi8~sfoK~fntBGeC!J3P6Yk+u#{CS z>gdX*VD&R%;H!UI+9IA~S+X*SGwsaRDT-_FDP}`yzuWGLCeL zR73zebz25JRrt`AhtXGIqh)7Qw5invo?ps0+q80z+8{3M0H&XObRsOAynN)k{+*9+ zvJBv0(niIyavo6%;zL%+++Ha?;yS0uyMEh;WKB;TiNB+51Zhf+%!!IiZc5n@-oQFJE~U`^?CrAC+6D#$D>C-L0nBZGIfws^Dz}NI zJZd3>iNaIRliOMDO!V;cKS~&U2t%LG!}D=Yde!!%(fGdFdOH7H1S+ve!I+C|EYth9 zO+?o$m?y&CIkw0fJNUEXCP-O$E(4Mzrs&UY^ZQ;(v*#fvAA4t`$ze6Wd^dpMDVvE3 z5hyLy<-)r8A>K8tNcYBTB%}-HRq)I==4fk zY@WyvLrpn4X@acBK(X>OVGQ1yTvgbPl2kbO`QyZkwpv7SeaxywSfOvpJ}vNm+pzZ! z+*#Tg<(CZQ#)|8(UVR(a${B?`T3aV-X8o2h;BPn=(9|N!@oBd?5c09L@HWR%P1Vh{ zWtZ-(?@hhGiy^uf({}%v-KzhG}%jX(ezDA)QZu<4CAFVD-P+jXuoPX=!-KTu0U^EIwA~n@go<;A^&pE)XLwU2qY{sjtX1mpj z=6ed7%BW|AMbD=4X=%mwYCFycnAVm?(Ho$okR#lP^Z#%Fga<10q|hfGqud|0_Fsh| zxz{6j2)~c#eDoQnE$A|swtt=Q6`k3R?#Zjsdk^>a?*So*Twd7=ecw2dDP=@b1uqa6Ld@G|4kXOkeQ^EiLudX-fAxcAcz@?Oo`dYdC)@*+%^ru-MZcvS{8wkUlV>H` z9pkUADR&(~IhO~<*t{S*M8fNGz=ahAhbZEr($ZJnWeTEozM60we!RRfS)c^7*4yPOzjFQP~LMuZs zU?fhRJ9YNd+0%zlA3k*m4Fuh_HMGa$^0<6ADG1*O=84d@Bgd5DrzQ}Toyy~yBz;y8R zkADLT!OE#$%TLSGsF%kXF!1;%9|9DNdixn@agQ%rC`Zexd>8fztay&Vi{4Vm(nb>49G15H)y9X{0l$l8v z8avA0UyJlkZNEqEl&A|QvJ{>4` z4)R<|n$0Y2HdY(S!uJzfO{koBKmG|!y$sv)w>WG$*fo&1oow9oEH)Y2H!uax9div_ zg>%Q`@dJ4?urI*t1Nj<{==Qk-sM8~PBqIhnLi;in->N1U#$vY9Rn<8rdhnmv+&(hv z=Y&b5ngyu?2iB405~^FVH~lbewK9YM-tBPk4$3YvPAlayxlLXU?Ues87gIlY>nn%B zxEYuTDh4vQlk#)2Y6%vLu>;BkG8Ri5h?xd~yd@HP0>XrWnAxyWj*s+jBG<)|agksg zHfug9Z@!-mL$kFL$;;ri1QJ3wcx@UPkIl!XVfDbDKu*JofPWgf2Iijx{@K_{au-OC z!)B0RR>YJ*La2@5wf!b)x~C7&j@KR-q^s2t`b`U9T)?wD%i_?_IBk^2 z>^1v2bk?>fN&WLol|5MfO6Bc+q+f8*9rF&9#ZDv0mmkwj!MZvSp|z=5z$0nL3ckM5Iv%wayNQge62M- z!Cv<MLAbzc=dtW{EVtrHZb^KJtnaY!txzXHc3v)iMP6(jtJ0NplhGyt%AU*tf zP~M-R)Y{4p@bqo@jhJkVa^L@xX6W-E4Wz#)n+Dw*Esp_*VszNhr$b6z6&VEwse-;p zZ>}=y&3X%Z>d?{?$%N5jvoN%_$Xj}exln#0e}a+OknX^(Cab9guXI|SECo%J3%DEn zje&+h*xwLn@Yhy^ib9#Fx2)7zMC5F-Y=41HI#{@6CH`FVw%%mw(DBQ~w~6al+b$lX zk6mi{?Z-G6oeW}Lc#HD5&a^x5hM-u}i3d23O`_~hD{r!+vT*--&6S!9&0SP?Yin&k z(b;J!ZlXPT&U6^~j44{a?%=B9;4w-aFOsX0#)B6geB9$nd8cw`PNmsh){;EobjwIN zXCDScrCPR%59J7=Q5`4xU%4t&!1@ z3mUtZ@t9#i$FOEAsyE1u@<9s;I^@IUm}~*7OdiVb0amclf%|ODOpsEWKKQ-HY4KRS zgML=>deBf1G=hT;+~bh~b$GzywRkAG6CAd>;fBvou=0$}W~U~-nZt5LNByc79BW3@VXE4_x+4}fUNJX8e!6m`tZ2Hk=NsbEmD*4uiI+v+@r9&gwFFXz+(tw`HQKc<`52wmsM^ux{|D zR?vRuLi^?F4X3w3`_)1FwFry=1EIkmj7cazWJONNEqN)?DfvYAN$!-n(|}$p?H;sW zZL%(zG8LOkm@>}9=>@Y;P?z12T(a>Nl?xa&MUP``9+&tF@#)fFGvB0Az>=wJ3t48-Qx|~cRH@B-Zo6hV_ zYdIjIhqjl3ErLAgavv=C-qXVL4>isH(YeYiXm%vkhQJQdajw2~GTZ4MiY zCngu~PCxAN7f`k$CMRfZu=X-tJnC{foJ8MoPpE_Lt?ln|-9Vw=b$Rd(V`q6Awfl~o z4Mq!qLV*%Irnzvs^~IxYR#O+{tEp?(*PsD?ejtTN5i*$qeRfr9u5AY@kc<^C4CFT! zQavee9=McpCHZ24ewAIz=$J~*?qr?9nNawc>li8qg&lVsxcg`ZHmur@6AqH0rw z`iExu;IEQ&W$BffBDz1X;SaR11ry|HBjpBVwyo4!M!A@JGmIE`X)t`O_tu16G+3yt zw=~;3dGsLJ9Bf;Fsly-_e5e8AL;rfCx>PsP-pDpL;xH5!Je2GGuB+F4wNjX?*A zNm4+P3D>mix{A82ZusB2!Q1}D|9H20@O}Hg?|(FBs=E8u4JX`l zPSrW5J%~5T3~RibTPjPd%S>y%%kg@#($_|rSqJB2<8VzF;644957l5a&}|7Tpo)WAeI1ln^f4kiIMBB%K?99Qi|4`HZxT~z%*pJS zbN~(M>lLGUqFMwzBT~FqU?^!G)T^d0@{{Crql7Qv>l0&n0&*lohNnb1$yb`EIwZb! zh)6S0Z-f#V^cmUN>O~Wgy(KS)yI6+x64C<6pmrlii=kacS4$(o07m2x?&M5ccrT}v z^wSL()WGCwX&_kO&``(*Kv>mDInyEszg($5c%b{Fco`q%$9E7JtvbjD1i!C4l_#RT z#Ygz$@ZvI+^mCPzd&?qZ_TTjJ((mX)zJ1}T5dMT(Qr5h%s)cg3nY^u}wNvo5)5sTd zi0Bn`hgsa)+w1KkM@NJGV>EJMY{<3y4C<5+vlt#yz4s4gh02lD>G=;RCAX^0D~EQ$ zM%6+|$1+MusggrVIr>}GVrix_rt0g7cE#v7ey@afi$SVU>EQ9q?o6J(rov%vqK%z3 z6{aTq-G&thmyu82vqjR}^~IWRP95moIZ4OkecKZ${KfuLCFjVGKivIUkVbBGnE3qZ z)Av!5S$aj^++?(uk}I}O?LWGI&s!%B)fR52%?+)!9Tpr~8}$XHeS2%@RDaiCB8K<$ zbcaXDlVfFNB~{yArghR{sDu^z>8RJ&Pfy3iE;ww|tyGa8d zgj0GsM9G1aiJO&Gw?$IVLz?^gxow(<&u7*4S8oF%;^~!@wzT*hoU4~}g4{G4mUDs* zWpjcC#K?AGmq)aVE`GE78OK@=4!N1&tK|_FWp_Qi5)d|Deb7evodFM0_gqj21>p66 z%nJoWp%{^5qV6CCB=x1vIEAPsxRHH{41Wjiwt8CagbqHN`QZ&@yfVNQ!tKTucN1CB zkvKrZi+@(H)@->_yG(QJ%B`wvxmLbX1AHBpO7E%{46Ogd-u)BXMKtr<;eyO^eINgI ztAUNsAw~`-@cTvN{WTeaULQ3bdQhU7TmK+0y|~WsJr9#mN$8~YaDI;-y^|CB{y)ic zeW0(^@NYp1(|r|a&<6AG*;MNcI%&}dnZM|fd20)}aeqJ%5%s%}_nAE3MVbl1tbetqv3EQ5l8K=s2-wD8AiK@d5ev z4pLW~k1kYq0tf5LqU}(|rvF>%u_u1Dexrm**#3e~E*?AaPfA+B>3=smHbDuhU4zvN z=hfC$Zml2vlGaU}%G{`LZ}&9SAT#YN5T$HDS_7zIVK*;^aUmka;{>c#R|S+hSOoX; zdf)rXrTv?}fWIW*+Vfw2L-DFVy!W`YT3WdIk*AHcf0+^%srtpph&VZ&sD?`_zv|Q* zN57CWgDaoNMw>l09c`vNvha-jx=8EIswH^kCT7ZR`_X59Iwe^Ec)1RlSk;J`ruET%3}Yt5#h}C|NVtXhK|w zW%l0`^aq11{XE>Z2pSi`$~FGHng(N?on9_2e?eNgR$7V|TgHnHlJI#%=Z^3e(MiGE zuaqj0oyK@csaw_3-rF0bA(>gE6hon1X+r18+zT7^fruDO5CiUhYcEyyPSLr7Go`+~ z#%7zblP+De^`&hu;f{8X(L@;irbs)rclo{-4zHYYD_PcJcBPgs}Imkrg`mhJ2fpc=U~T5$5{>i{yk5%DAMY80Gfz;qkTqiS;L`OQOH0=fOFwHm z=A;q=GT?}iS9A!}ny!}kFoA}w`l0L|!$NrVqMj{X4@$_YS9k%rTQ9*~s6(3)Dpnh$ zuj zBJM3})|8f4HfeTD%8C2Ya3q}|f=rZ^^Y$wpRpbl#L-2YR4e#NV1H$M|hL)aIMK15# z|ArseM`Q@_OL^F>s71qiulPl!bu8holk?eWS{7G*`%&l!ts&6e5*7I&z? z3yE+PIUdldX995$CF5Oy576)#HSe_WCbFoeY4z&9#(oE_`w)8IRlRuh;9hc9j~wUI zE;x;Z^n{vQ#msANt2fs9+pIWXje?W129wS~G7{y*IK4k|{sa&25#XjgIGb;Je6KHY7k4&o+qld)my>4T z(gxw+0!WhZXYX8w#Gea3c{obD102l{qJx_UxA;vWA*EE(UT>2Pea6}oX9q%t+{>oYsm zJ8DKMknvH}tdwq1OLIDXn%O&S^1G=Y}b@6%Y}?U_q@=vAThkEjay8jrmh z=bQMELgGq~daGe$so7XZow@Rh3WN1z=6~t0oK?$N(9o;SUa+Nir3Uo^Bbi`Ep?^vv z-LHP`+eJ{Z^iSEKBB!O3vy*1diHvl&;PE?<&I{TCu47Bzn{#sUy>lLvnzvTqE}n~c zDJGa=(quxl)0f_z7$1J~@aHGa436{-M1#0=Ky4B%NMo)sBvgeOIjEs4le17WU!$9^ zIiH!eMDw@?Nl{m3gXrOvehX!HG_e~;u`IX@jgYJ(|^TU05 z^DBk~AB%Jv;C|RE+0+}rtLuqHUTu zc-nxhk%4;*&l#T4tS~J8>koI|rP0BIKUBcq>EYblzLqw|C?D|qUof~q8()n}FWM#5 zGt%RP^e|M+hifcUGeF$cDS86+nh+J{diS7BA_QlC3Lhn|{BK@=>4WodPMF%ruU$vH z4Aye%DQRwG$;m$unSFwO{<7z#M;sJ)1iW$TC^&wUfDhZ>Up1f5J*t6t=JacNKG6~` z2;rHZx-=HfYLm-nbT$xI`lKFcRdeUSg|oz2a9TL>vgQeH8JACGu2)6-cri_HsWw>P z1sKq`HqR3`aQH8|H{fjgKh$r4L%WDbBr2RG{Z^p^iR%t^n~?74Nu;RyiEhgPjHa>+E&dDFvPU-{8fuG(E8DJ>Ay}_L zoiEl;*CBux)#V0Sx7+}4h3!8q)UU`l*St#Cg_Nxz8G z(if_7CQ&;;!lNpQ=)C7vi6&RR)Vs5Q_Ba^3o!Hh@HD;wc_DnerkZ=v0i(YHs%hFtU zzqE$d!Qo5+8bx~O!JDNm4tgE0{FMLV;)5EQspJASZ*cGanmBar65;`B zh*$C{nexdv$0g}7=SQY*lnrn`2IobChu{Jl{y9FM+R)0y?6fPwWW&-6)h-XmdFWU2 z3+C77cUGCUlry;3i|7Yd3b7OgvtjAsG)R$M19CYKIV|076q?9YB_)qNzP)7JO6zXc z=GAF9sn?0ODn=U8lyob6RPXB%LgcPtbE%K!voTI3rbaW8 zHiwgQk)2!%aC7)B$)dNnh`fh}E7Stonxf>PiH=tSG}546H?g>^vVhXf(>$-y^=4*$ zsP6^v@5t|UCv6eR7U}SJa5$%u?l&u?8S0zu^V?QY?zYa378~wuK<>IUSAEDPyApoi z?0{bk1OYgaLYAef9Uybl<#YSIlponS0c6gojzpDkt{RksPHKr&FWsODg@aKsj`zJC z|Lm{C=jy{@WjyMzOa~*smD!YO*UNX=s|vSm*tTglFxGdpPG=)L`PrSFd!-qZ#zcoYraEeQ?+p}yOVJH+@g!} zx%@7lXJYf==l8BFEh<}5{gSDoy~g2Sm^?;p^D`j^X@@wA%n#L=m>7@rcki2cbLUhr z7>R{LcomTD2b7h!A_XCMm44NpEtv-9e5pcLT2h+~Bg*(EXX;%_4 zeKN_W$e}@hcz~XaZSU_M>lqL3_3rW4J{rUjAPnWfS1CWw$TWDmG^mQ3b%7t1snXK>CZA`4fy|I4#_ z#$`v6WapI*2Vfnvhh;rVj`3QZ1gqxM#=x+32MO^MTnOg_;kSI6PQ#g?i);hZ_q-;A zSVD+|Y0;)%55w@NozB+@-+w#)AgaugGnOX?)T zXcwU6=d!*q9}*(71zr>gaj*Z4;Cr~x1xAJlfIQ|9#mfnh-bihzUU1={`(;2SisM_t z^@1}mq>99OAxg)^V89p0Idx<>zB9f%J~=zwADxWuz*)et1mGJ1)yOySZIoc*TYRr5 zfmi`3-XgznT@5^*Y1F%|Y^M<*MQJ2V(oA3`y8*l&P(y6o6{WoL4pL#_Y%&cR`HIbi zWKpl@O12TUsCr*qe#8=q3yCxU@R!|0>MgY4jd$~Wl5Pdk!@MS_+LPv^(P^3>@*X0P z$%4o`N$^pnO;y&)S)BB>pmYn&i|a#CKHiNw`*SzZ1@jP}7=gq9Y8_b56;n=i5ba`- zULcUkM93eaeWEg;hVQGl^TWdg-v@eoh$!NkUKLxyxez&6&Z~?XJRIQ>n4pt#0_h;2 zUPeY`5rw9|D+N*!TK8l&kK|7HUS_@?JPu40V$B19+QQc%L&vQ~8>iiB;YvxjiU}y) zVK$mV^ZUyQIST3UR57tPAW$N)GwMofZT+#bqotZLfgs`iux%17~6_m#^<$v)|R zeQc5%+i{@lu;ZX(PyN6a{37>fE=moA2K-0J5YM;1+Q8lAAO`mA89o?396n$gtH8&$ zb+3JjkbYYJ)T<>_d2N}y+P@`OHDK<>FV5^S51B_C_`!~M_Q1!8uNw{*I_#|1PE}hg z8=4yAh;KW*D(E@8o%4{=L{!Be;E%jcz)#dZxt2?+RpV516`-`R2yEVr81EOxYqahG z@LNEoKsRzeW3d>&hm6`(ToJcvji_lO_*$`O9pUmgSuf>vyZDVkXAc*f&Gn&IKdyTa zemJa8aXs4!$f!^AhxbpjSg9Q#ed%&}o7>S)FWL!@vx99RO{xx+!xap1v;g3GG$G9|9wzGV0_qIR0tWO@I42`W7MVPk`zYdjcV3Y15eq zFJm3)ak<-W(A!VQ*Q0JfZa28H-LAE0E+K>L$B(a7*J;(bvfZwo?zT7EZA$D#-O7vE zh15G*Ofk3VB~fD{YJ?JD9p&^o+;+;zZwxql5gGbeL)xgj zYOu8`zLl(PblOey$LeIjKRQ4}A63_KHOLx4yD!TkedY16F5K&6I;;f47z9_4Q?ii= zh>>6nBq+3(d=02p(c-tsrKpsazDegka>ve4&j`J6yWZR``t3AqRmW3(!EW+!y6y=H zFRL!4b(4!VFjl3Hs-s-jFmX}Meart<`L#L?VqA#wb6mHngt&;94&l*(A^$LW>FreyC+KEr zE7}t{JKA(uhG_S9gdCJR81jZm0DlGgsfRn3uX>a$DT@zQ(e{wrrbNu_a1b@Ah9Ns; z+dk$VBax?}Mz34mtAA_bYxHXyRtzsA3kt2xTabHz!GC7F`ut6;rImE`R@2MgCAfIW<3@lW65Ub!(If>`e>Nu)Mm2nu`L2 znloy^9ZM5jPy|6zd^WXiWGh}g_FBt1^1_jU!xQxQfz*Z12v~Oefk&t->t-(BbQazS z4F}J@ec{MU5+M=AFD|2X&ud^cH0d8O+y@W*P==1uz0jC>RL^9?Vf}tVj;up!PZ^5K z=`Es~6O}1jdDqM2=B<{-Cc3rNR#Qn-Me6%&saSimv&$TI=7F#sP!UJi5p>`@ry8$~ z80(3$=9aRW+HlH9Il?ZR(rIaA`QM~jj&B31kD!FSx%XrOu*mR_B89o+o0Oqo7;Bi1Gt zXHsGU_P%0lR+O__`OaGhH`+-CLSFHS;ygH0v~0O<2>X*`fJF z^E=I7HInv0?Go)OZHd;e?bZ%zcW4i2FK9p1eyY7Z3!9ZY>(*Hh&w6s!>RE-eDrPxn zfmz{M!?X6yIyCFuS-+X})vUkHg1IwuiQKul&*ZMl-Im*&+n(#pWpabLvD{SdQ0|W0 z1Gy)1&*XlX`)TeMxnJl0E%$q!LN`lygO1Qqy1R69bPww0Exp;G)pw?%OpHtqi@q2w z1T`Yy1fDP-GoG#cZ85xYC0xB3{=(~biB17`+W~8%T~6fGA=}YuVd%}$ZIVJduts{^ zYh!Gj1^0H1FbT3R;17#*@?h_|{m&Z$E zg^2=OTBy?MBLS~HLL0lIEfeIi$s_v$w4X5zzHDf6**jWEhcg&<&|SvPo;q?vZAE?? z&4|4%Y0@wH10o&njVF>KuG2$C4#5tE&j)GgMOgIpYJ=q2tPx5^Ts6`E@| zjjQCF#@a98#Hrg55~OF}p7+gP-@F7@!_~_^eX!oJa-Ou7hL6C1>EWW^Nz%=fHOY(0zOVjhgdzl>F|*g zl%P7ZXM^-}i7F^qmtS-kQndC?L&r)_Cll^G-+p$3L3`7!2AyzAe?G)Lvo{<1V*?8` zi!@sOtp+VpRMq;DYPq&&17BT0NJ}3yXj|VD^~XFr*<&0IZ}{^&@ZL+Cc2sxM)}Y02 z_N4-mU^s{m^bGF0NPNETE$Qc_#SBwNSvUvhVeyE;%>afWZdHFJ*GLTyfh76Xp|-_g zns))WleleN!Sfw8qTr-5Lo@lnh!7)>?`~R?^|0M;Vs&Fhqm^Q!wO@|_EP9{xhy0nYOK z@g8uXhX|Mx#mNov;#isJ7@arvaC!}%Ov`t!V?x01i}zpXf2;p&_wntQd*8-+pdt7M zP%o77o54D9D{m5ve4DR8EEX&9Hn9WQ-)7MovQozfgjkUFo_GJ=e%S`Ex5Mil@CLSz ziF)IBQx(IQXpfh5F|N8rRrgmuU~jZ-v)4IVoJ}s1tIOTv9CQviMx6ulgKj2*SA16j z@s@*ltBGUHw2O1ezK5IP>~2D*Pn=s&Fi!J`DlE}%Hgz&o-SFDu8L^ETE@T%%(HUaKQ~+c0#hgqJej+g;by`kXo+M zF1y7b{R|e~VX(pr8Ka(MS*EkTWlQab*`Tc)P^Ky+Xet6!M_p%~wI1h3`H2w%KBMkt zVxAbqiHx8Ocep?q^0ifbA-{1gAt9p&nfrmHBS{Lqz!#x?f*^M9=-b`5XZG%QlwkYo zNHg?C2crEr@P&OLimTx&xpG{3LTwUEK?h}x#W-X`BI9L0ID8UW?FU0dAS8%BLI8eF zq&S~fPzJ4~$PRPb8Lxwu7f}k|dO$OtF-G;JzgzqFhvA&_|2p%}9T&xj5E8<;udYe# zByDcS%g|m&M{RqRxq5cP`leS}wzTePIn;c_GT1rXK4R~7%PEnSbzW~P?aEdp-~0Hl zrJo+VbGd{sm2~Up1d$gR<Nhdjb?PzVR;)Oi2CfvMr6!zX(83=|H&(qGV5+*8)Q z1$V{cY@8gL0O1JD1Hg*_ZK$;Fk1&zBMc+M<9_t$0b$;S>-xMB*M8zl=RoY|+D0xFWN-`sb6^8LxTaBUB6_Ju87 za$aj+Bp&n!a959~f1J?jcMlE@rbqEae`sioc)j`qN&D#Y)m0soN8Kr!gBFSr1Wq7D zIX#o^8J$WVB2V{Jl&xG|@N5I^wtG9hopRVZ@5cEk@;u3pPc^(4qIoxP^Dcb8u;#hD z$l|g{vWeypk{E(@uy!laDRjhblr=GKJwZb9``^MlFRefEXo}YH)CBKJYi;i``ZWfh zTpF}c^OWXF^R)*WI=!S{|KyWfqvaFDba&HG(`a3nsmq)+0XQm5zScw%SVyPJaTs%NA-rCg~_!ldVL zFJdS!@AVLZE$9?joIUyp3w7j2qrA{ZCxv)V5Sgza%0_U`UgIqxpL&27R@482FTz={ z{shGFP}-MDhPZ&=s*#SV8jOO_PTN?<#(EhG+sU-!x|1Ki_wBKZfnYG|_Zz~3KY%=# zfSIT4oxFu~x&TtwOYd7GDW02yBR-nslCC(lzR~M)(tHJ9yoS(z5A|}@@~70F@E7>k zskNMqx7u-sg|)O2TBPm%OW)Y!vbE5I;3S|X*n}rK?S3^?fd(70T$QYvH}P#Xgmg*W zCRqGdN=`0RTKXrQdy(xQ=^mvg2KOF^3gl<(Lw?3UOLd}*td-tF{(#T#pka$Tk?aff zkcZOsk4x&URVB0*z_ULre8|w&OjNcsmwPGB?c&^IU41azO8e~&(aFjgUv`VTwVh~; zoG_gu|M=12Z>HKOiYDor!Gi6LJ$O^MGnppFhI{vQr5I!crrk*{!Qi22us2S4V~qo= zd^j)H9wN2+^T)@&8KixTY3OCcy5`3E0&-L1@QG$R$S|Utw3r&uCNg;c@tO-9uDe$w zJq+)@!?05Gf1h<|U7AkLYX9+hPrtMQVLD&45fpMIl>A1OuX%!9mQQ3ZAzQ~~W7=3N zzUwu6U44sXjfr+3Vmb*&xGlj_$3mYCTqJf^jTS|yHh(i{<8bZ3sc+$&k$3Sx!XJqd z0vF(Xl+Y~K=C>jj+eT~e`zT}6GYn!i)Y0r|@9fI$>JHKBcT9V~d*gE1x$IpSuAdv( zgSObIKB5QShP)tN6&9jFF^=;p5wT>9h)Shub34zQX|0^E*W(W*{Kt?@4*;bFd<$)J z@ov)V^=vd~)k3FW5j$~a{5X4#JgE(L#l8?u;vu9hl6Er>udGpP&8_9_ z6#~AouKN#kL@TY)`vXBa4g3bU5!PmA?*E95I)YZe!RZoNlJ$73F8W^SA&I$i=lZ$4 zWzsAnD#+VSpp!Zi7o%Z%jN5mT&<4n4S5G2LH}BTg7{Wi6CZ-5@w_KW7Et& znPyHaC;R$r(+SCi)=kZBO$;fpNOVw}hy_zZ7Z*n+N>JMtGNx>lD;jsl$U056TRaX3 ztvAxgg~{&T1lsOowW8&FsY||nJqJl`hsM#{l1&pvfDY_1~&*OFMRDH2RGi zt^4>P?`iVHN3qvW({G<2{1Y?~@Y#HLlhmMXF*xOn5N#jP<=8yFfYeGKWK(-T{e#<3 zZ`icx4q7|JcSk}ov=Mjl2}*lpK!|AdJJa2}_FB6tO6-jl99_-@O?ydQWtVyT_U_)X zVeRv`5YnbUmO@NBoKJ3CZ{50u)_R;>G!uz1L2Wo|qIe_O;>aO)kPY&)1AM^m58=V6 zZ#YT-t*Va=+x^rw&cr#9E8E7`H4)ltx#T4^Omn57hTpb<&t?N z(tU*1Z;rP{oI`w%5aJ^~ArkD>`i|qg5BPk4O5 zM~hZirDa4`o6%VeS9Gd2uU7pj|1N);TBj|+oxvCrCAE9;kjo+3NiAn-0WG99v~TAD zl2@JR+VT|5Y5PE5xkg)VHg47OfY%};PO$i_l;~1w)&0Gr-SW_^nHco+yR`I`>s3at zoV}o>Y7E*sP*YAwm(@>b1wqi-J2^ACtw_`&HbRI346mgV0WFsOy*v5eSJR+4g8c%) zqd7;g`*N0G1oj$si(&!xB!b%%Z(t88{((J!9l)+r+>YIbj&~?FV|l1OrLbbZLdV+> z+=t*s1P>rED2B1S(D4?zU%7quTGR*XkIMNBo0EgQWAxqQii_B-IoD%Ppne`xv|*3s z;MmPMH(`%q?_vuTKf9Vyyort`Y++6uyFF(P_CU@PsLv5>LCzxVR>f)bod?bPJeHet zH#T4KXDoNd4D2SwcbHc34uW5!Ic~+O6w9#L==cl80Zg4;7eJq%#ug~*v6~fV5QqrW z*bRyyOoP@sA3Kjdg1vL~sDj5H#m-^#b4IYaic<()%Z~Mc;w)-^A9}w5y?+vocLsYz z!C*hhDa`h9y~2z7*@kfO5Y~Xbj@^saxm*F!e2-vSEQ-cohu{7h5pnGlb)((A*azx~#$~(Y)6w4r2L=d$9^dHCjK5 zZB^WetyGj@i)TEDkvYe)>vL9Pznn1>;j{rWpuXoT-ox(5*@x!+9irLq5uIe3@MsMA z_z&!boM#Z8e}-xPCy;5Sup>JC8z4H}fxUb6ZwS6d@b_ypy7?O1^5eNoCz(e74p|!g zNGI?=fPkO{UCaM>KyiNyDV*9N4)s|f=r(?h|d2GGHqlqV}xQUb|>OJ z4AJ~%h3V=Q?9$aMh`#rucF!m-Ui~kH8=E7yQT!8|jn>bgvDcyFyJ(F*G{0s91&V(n zKKTv8-x<{QOYBk9_HG3CC`#q;&^f|u&J`p-x1;$W*^wc~g^r7_exR6z;10w)ixK=C z;k^rcB0{`81(K0FvvM#zk_LvlR7l3R^e>!ycQFhpxX7$ztR5ct{===6r=@^z_w#Wa(Lj z<|5Cx5#i}Fnj4b4oS&lkRHL{e*4`=00me=ZQOkANf}vH5i56 zhbgLyn`!K>MC6jf!7@@Lb(FfCVm&&eKf1n09Yb1)rNg#8SA`l^5}M(_-R zUm;j>^)J{`bXj9e5v)e=($!z1k*3f{ zQ)r|qG}06rX$p-rg;tW2{e@oa{{T=+0|XQR000O8G%v_HX%^Im1Y7_B9BKdn5&!@I zQEXvzb7^ZrZ){{=R6;IqbY`Tx2bkQ{xj(FR*%1PUmH}G=X)(rhS1@2>1I9Ks#=YR~ zt=`^ecgjv5X;eniXnHT(d$HHOcVg2+O+rsk$R#%k(e9OV`OYlPz5MTezVG=z&o|nm z^PZwp-|~BpziUnmRcZa>R%ve@}aVlg5qyytB-)P0T33Jfs9=HWFw$| z4Z_T_YOOvpd>?4LWEH6Y)9M0!9rV`?peP3hx7?b7>JkWFlK|;D2%2K6t8LVNe+-%r z#?t}H^Prq;)#G0VAqw>|eAGZ;bYD*4fA#+q1l^!>pj$xd z2YM2uf?m8qPljeft8UO!epF{F^bk~agU*F+hWs~Z6|?|K-JoIU4(Pob^i4maZ-(xI zzP>@<0%^e5exPstQU2{e>g$e)@l1wtrrZQl6* zp7NtQQ=xS^U*Di}p}Qx|y+Ny>{gYPSpkZjnq=6gsO+TV_6|?m~ z<&9MZjg94%dCO|6iZ_*%HdPfg+<5T6o?Ddn<4TJrn%P#;&{$qulQ(bnyt(rhZ`hnS zckaB|AS^Dc)fUy(G=qYf`Md4T8Vg6nDMaX{-PuGu$Q{8xKFw7e1U zV)p4q@N6Zhy9}xYImOT>P*w`s%9b|#j~@Sbe~ZAA|8FfV`q5~%fw~P7HEO{a@}PNO z6!SpIJZLcx< zkO$(%iIvQo7=HmMDFOM}IcDqDLi_)()_3;*MV*yk4Of6G4|*8%k{y2mDF0#Rx``f3 zphv*`wBV`(t275(|6eo;uNw0XGesUO+J_RlRPvJ&zq4qD{pSz zqP*pK1NYCkf7XoL8F$V=XW%m)n~^`GFJN~Pe;)z~z zzW@IFub`(vfWH3%gwL}(32f_+e&3(_{?+fD6O`@yN5Or4mi{O4uktTfe!1k!hrgWm z<-K3*`{L~9pry~>{QUIihd(!dZuljxdVlJFTeZC)(Etb36O#F&<^Mt^aivO+68?Gy$^bF z0SlH7@P-rm1M~^>DUgW|pq~SIY=(XXWU~h1AR1yI9%3OkWPk+73yF{ilAs{u12!}O zU4TMR3<^ULC=Nv-3XD7hC4ogvgFUfXYli`U907B-K&QbDo`g<8XP|GPe*-&fg%Z$lz(wbPeCwb;0%>o8{sg4>8z7r{ zz-KU6vwdLA%At>-kAd#|Gg!CDKTQ1l!&xl!zkfM^Wucs$oLhcmXR_%B#%_nk=I4wp zm^8Lv^4L$Nyr4+`vGG44N zq_J@#qqL!xaVkSXC4kYD3{z-r<05d?HIe-PAzD3Y(2=EJF48T)>g#d3t1e8NVK$M}qIXO%& z;61=hKo+379Kb_h!*e`2fC+NEle6mB4FUE8{FW1(4B2u3a{>;|iB5t3Iw`xa02;jn zZOPd*37NEc()W|`$y+CjQ|_3ua>}MD)+zLq^HZUzHB*0^YgXN{Y^LDB;9oI zrZ;c;5?Ot7{mtfEZn@>QTejY^~6>rX#9`;!Ylxvbf!iQRqQ-Sh9>boaKqyYKG5XYM^u-a}1uPW#KW@22mX z{u}fr^bC3)eI0ul>$~^Xdo}mQ?oHme07voP;eW)xC31<)#2dXM8u~KOaCJ*!{q-XRe$1?#!D(mz3`QVsfB-d`kAMtXI4Jb@$Bd#^&-t8Y|)HGk1U$IXyKw~7C*B%`CRpqyrrt8 z3zqgSm7gb{KmWpgFGOGX+lvJ+URYMKEVk_8OZZFXm%d)UZaKC5=<;ut|9!>ND_&T! zZpF40dlg=t{p)x`09Zh$zuv1E9S-(;2tF-z3;oisee%uxRk*{ICX@K8e6oopglD}9 zF3k{1_)5MWKKevIaxxr8lvn;%h{zA{B}k3aVX8sYRbR@xHPn7`KUE3~0Zzo|c#H{R zy@3mdKT^>rJ zq=rmms=mXRv?k~bEUFIox1R0lpqy!(@%lM0=9S!%p9sbMtr64_sZ6hxg0gDb`U+PS z!PL_=Y(2a{pTA)vS{<$&D#jD~c3o$)ga|Ar2&_a09UeMj^N}GNe6l!RUxsck)otCo zFWy#(n+?|eCY1D%0Vf`}95kP%?aHKS*LtNsLhKSd#ZLJ1%=s%Hqi6I7^M`S@Y#l@7 zXPt=hYxy0Ga_^Tc0T_gF5h+Cs|ou3$q>Qi%4&JLTs=1FNEhM|U6{*($QR_T znvXMS#4z{?u6%7pWYyThu@^Oh&CNQ{eYS1+^KfgF5iC*ox!*h{PrZho9cVq&)$M5N z#bfEvfFJGJn|%4?!$##-&nT-Jsn6!c(|Vov7`1!S9@ur^+wLX;kh~H1ZR>+wr<)L#dPabm2MWLFFY> z-5{?U+oO4WRrH{?ljx%|j}d8V?Uu-MX6i*GI{ zS)kmH$~lZiUOf8d`7^_>DNicf3%Bjqw*R91BCdW`{^{7)8mDBD>|%vue%;FrR;%4c z*?6nh?6V|2KT946vdSD*d4N@Jb}~+uV&OWcmNGkdhXZX;o&AjGFB783doW*+7eYkT z%Lj%~pWdhU;x>^K9U?C;6l9wqPlDB-jWuex+<5!x*5lae&f2Z3H!fK=pQtIXEGpY; zYN9QSo(E5O>@a=pEBDvfiD6M_CsIPJ-*W<{v~JE!aM~KS7TaCq<_n3DQysmXy`BB7 z9jUItlbvV1$6?;j2Lk9 z?owF=qtAkqlEip1pGOcqM5gCp^e}cNUQxbo&AwIj1Zk(uj0t9Vpd4|Y&5E>;Nb*7K zy|dcQK7wc5G|lh~gDw_cTQdhMDU2km36~LYCTbE*F^e0q#0ISwF`1Baeknin*{*Yo zQp9jaTSu%7s7h9zq)`x6PaiKtiEc@X0@0W(s{B5;;3vBM;gmlEdpQr|#W{iVuo7&p z<=94o|Ec?p*U>+!`uUMQlp}ct6C4(?1#6;f>_+37O^QY_R4x$>{^qnEb{+qm`4F>k z4l63jE={wFE9DB=Qg~MZFO(8JZ=e3{xp!f%*3H*iII|g5k3nNynvfv*c(3R3H0BlM zDc4IFW2Uuqv-Rbc$DOahv38DaBYN1w^spViU>c}gfzDmGdSOX%Fjz-TRn~^MCrKUq*Wn-tP|j(4@-C`Mjv(+m}b5OPItK!6fJequb2G&Rp;Dz-j;C zYy7vKAwR5cm8}}(1@WWjWf&~r)o*0%Hzi*^8X)*w-bJ}dtVu_cCPMEZbq#2(N=kM6 zVwfaxoChfCVMkZRa`f?i+qbRFzbyZxPwesZ;K2Awyf^r>%-Qad<5xdQ{>lY=!a_WP zc9Uh^V(it0jZdv2_HL?~M=B7^YTbGj)qJ3ma5RziCX^80IY~UdJ>8`gQ^x@`Qmy35{_b*#DvociCT2I7`ZTfC~ zX=6=;K@00@Efq!RGbd-ukH~kt_w&EJ-f(F5L0q{_p)}j}jl6Yc^ziXsqN}CN7;k|) znnL;_bj{w%J@}Hb=QS5cI}aThuG_M=x~!lE*G6?wV3wNF_GAyL{?Mawx!j_OXc3wN z2Ha>`l`hI$Ebk&4O99;x+ybODmRW^#;9!j{~fWvq2^t zM!VyKeWUmyMe0XRh2n^u89TpE(=0ZU^`r?Ybrd>w0^2)l@h^ZzuKV_H@?!b!cULM4 zl;^f=FQr|gNE3XVKiG=$0e2)2XfP5i z!_hQKt2mxvQNb=+f>t;Z7Nj5%OlQ0qY=nV@vGS zTDYuEU$PAqW}VYrW8q`Nu~3>YM>Av(_N#w;TQFvVDzj_pD42HrmgRpE>^|Pl*0MC|iY!zX5SlopA z-LZQa&Dkz78YZ{{+_9^u{7col{8j!WzNVS8S_r3+F*KseUZp}VQVlV2dmP_X&sZ!3 zU&8NPhbli;t>D&jyYV;D?r@L@C&XAgDswWTI^OH%(}WUH=|!Wr1y?JfcNon%En?hO zw;Bn9^v0f$#%UEqGeJ8j8)pzrQGF{3Gr=eu$NUku5F|Q0X@5KfUp?giP`()*dqOqD z#T+qweI4BxM5_3Lt*G*q%BL(@>nZ18we4q&=6t6R4G_mvtz1VxDz8w5NXgz&duq#x zgezEy)n=YUG*;~Ip^B%(_I=p^Z3J8)H z4cG!rJZ{V-NuIj4lyTDzBRaf$YmH@0a!luXsfHb?l z7{5)!6w-U>0;96lpuFN#?uKcR@#9P)D-oHHkqun-i2u*dD;McngK|7ZR$gbw#X zei)AMK#8@hyhbV!+Zid0l!%T%gReHAO>azX441&EHeLj%Q}lbKST7=lJz*)5Je@k5 zI2j$uyq>(k+uc^)3R`)b73C}3B|;rc@)n-Nc@^&jMRuW4XyBW8gJ9%Mpqibx<2=bb zov6`c37YVq_6YF+L0_f5u>ZOTvB_$?e5*rNv+Zoy6@zQ`GjucIAc2qU;`wPDcL3p9 z@{KzTJFI0b1(q_hg=!-8w4Jdt4wuDcc3H``X*@SoVEh0r(P1V`N9ZsWW5QGd~OYE(@P)ri?`f*3?{ zE5f#;OuG#CeDGIJE`{Za#qyoX3T5dN%FT*FS#d?llONOJ!oDL9{K9tBbs=&&dNFb# zFv<)vU6zC;lyQq5zv!RVlF+u61>oxs%Bbe)wO;`5Yr};venX8f&7T&6HabE{&d%-Xl15o zl!l|Z?vDjSJFZye&os2d>^5UXGLs9`DWAa3jTA3kT31qIBh-r#H959u>=&8{qP&?~ zQfEOdDD7qiC(a}jTnv+Mx;~U^@RHw0uc5q zW6-!pV`3PFra`Yxj;c{|>y+GvnT|qoKUL`*NR8flZ+&EIY-4m&`nmMe(QTqtvI`W_ zPrU0Nu{Wf&7!$mai}Clvm*dX`3Sz6{YvZp(wh4AHN;@pLL_2U$_-Q=NvpfyEhidx- zJR^hx(E!g2!$|V*QF-~WJj>VX>+^LZDt^!NF?eI2lr<|AIzr}Ee>f)(RQ2TuZr zipI=Nx64JiNWf{pBu;a+=;k&)v|pJqovNejT#a_aB(&5 zY9wr6TU=x3XX1j$nZ<@7M$=SCE4tm|32&}Wn`8k+Lv~vM05)>l= zBHoud*nTL|o*W3Z`D6aLFPy9jHO4EvR>|tM@vOk`Y`P>>nka=Gi8vF(x`*A7C?UDs zoCjAwkEowR)OoUD{D7wTMKA-x<=_l9lvOZRS*_f*mD~Cns?1dFaqr?waUsZgFy`<_ zu8**^SMr}mjZ(9y=WV>r?c`f{tJ}i6B20!$!`ylIJMt3r8~$7VU7SDU{<`;{!av3_RQ=O41>e;hkWRHNFaer33&x@$L8~jXrlfm^|yId79{YxJSw9dU!<1 z8&w`Wpya`?$Ap-l;3c=mgQn;V*@@T4ldI%O)$-(;iFO41`;V>$u7m)B${ZD`(`iToVgV0Oe|yA&byK`-o%21*=QPmOiS9 zMBM2S^!n0Vi`lK;kE)eJa^BczC&Gj{fe|O!RA!_RzF^!?; zg#99R=xm_pFmd+4iI3#_fKhny^~d&ASX*jwd8z6clLUr*M-sU%sI-F7Z@~WmJe63w z+n>R%4DHWT=B~|uO>gi>Rzm&gdPHp>d+^ViB!9eh#C>1{mA_COW3vrxPa^lLd`6w8 z8r!j5SGb=j+*7x5KB~O%@#ep|@NTZt(;DtUf|B3sl_KU=V~5bow?=|dSmHbQC|*U1 z$K2@5^O|P2iL>EAw{slM^HW#+PeC z_PXFcY}um~3T7iHq=hCqlGRa^%e5EjY3oM3qR96mDW5kao`*SWr5*7MrAD!E)Q9*z z9*=-{VZU2)`Ed2Vv0LOfHT`6JO9#${y#PtKhf|$VI!yZsdo#xY44K=^)pAWhI(eXI z;E=W|T!zhjag8!{ZD#k8I)bKo&PAKd`WCYpcG_JeiSpGQb*D_QyVri4#=XFE!`ir3 zu9X?2gGPTd%($Hl$`@{bb+rLz3^kMvv)BbcOVG)oq5iI9f*Ndyyjn2pso)N{B6!?( z8tWhONS%ayd(Usb{T0ew$sbx9b_iyGcv|=tp@lCNr&WlCKmhgdUXKS)r>C;vFqg#s z@#fWk5X#E$G{-th*LjIoCByE$*v`g-*IdNU$nL>|nA&uS(Oe9>qe&tV7vgDDKC8;K z_N4}~-b~8|kr3=|ivd-fDudVp;PKC5Vgw-YVfTsisQjQx#%eA+P^Wc~2HZp%oh`Nz zM0JRXI%D{bN>@EtxF)`)5WRk@Y7w^zKpFKb$b$w%3CqM-t!8fp(-1@|_|k2t;!)Lc z)m#(KY;jc9Sz$+W6J=ysp2h6Bl#MYtP*E4Nw$U&X2y=1F8*uY};*ivz$@C|Bf^UeLI)Tb>sSdIcDuT~6AS*93ny>xtVX$T@iHw?~=CCznlcx+LykLoWj%P*rE%5Zt1>Z~gckp0do zvNnj+P37zOnsQY6r>c}wZsW@Fzsvur8SLmvq_DWpq<<~HV0#^*Rub~#f7XlzRA;z0 z%8Ok0sdPbI%7QyXF)E4yao`cPTrozZH1c0m3^O&8?n?GzMfW!6>5H zQQQowyu$9lPv@q>Zr)3T0W1%r>J^B(Y3$j@0XUN9D4R4w1T+A&jLMekpa!wQgZ|!O z;@shpPvkpLdG6w?4>uZUQitnEFdPdQjwNWo+%C71#hV=_lCr}-okv@RUN2B>cUOo; zoQX!+FxJr~Nq*v8*{^AA2*=EXJg7>fyAwUw#s0zvl?CgI3JA4h?BTInH3L$cFO8>n z?g(witd=ZT+({KSSJ%PY_g1|07^*z~>-E1M912I<@tvyG+-7bcej)d$=<{}=56Eea z&QxpDVq5bM%adgE(xnSToJ(>k04`meg9du{S-ElSUX3WaSs%gqgIov;xZQ$}h^9T= zF_eiJf;FOEY@&^%o@{pLEpU`_c$+br4j|5>h!?Chr!1bjlzW-mhQAr(!EOeC4+NI_ z9aSpT>g&SPwMe%JRd~ zk4Ss`5fQlWcjX@7`}xCoT(zFA*E_XbEmOtSQVhVTtc5nwCOhnOa_$;5Mkc5(;KDmp z&!*;o+kYSsYr}nB$t(C^-tP_t&;Wo4UR=EdQ7@C%%lkCW5&NiPgzO?a$yQf_jseFe zM#gL*cx{8DnIt^sL1l8VK|&5Fw{$LUeGWEDZMF<1NN!#v*j!!^+&mWZBt7Bq;NQCc z)b({}C^!@x>_L33o>&+a-5$!1d#YM^8)l|h5~y-5S?{c)^i(6&WX-qlvKPAaEqAn{pBieE?yAYG!~9BFh{F)}7|n zmRBvi=p^tP2VH5p)zxacYQADS2@HME7Sv1YO$-e?RGizzQn;(JO1X<_wCCFksRpu> z3tBI`+FV^!m;I3au;U=`1*3L9yspO4;vl@1*tXVZI_CD|C0BbU#ylbY9ag0ARu^PsiD@>g4NM3vz?$E zq=T}93*0Tu=2~kL%mHwLTGLd}+~EX+AVVgE6T(Jaw+XRk>@iaq-k+>; zpUr3WI=l|aE;(VUHN{4-4!%_gd0_P-ME$_2Wq`An$a}|IHLtX|O-ABB>r`(f+|dM3 zkR9?pa;K(vzrC6vxMsG7t#r*Nm7BSBaJ{7S8S%G|cv}U6{fv@v*LUm)H^6nk`~ILg zoX#9PY)TjIvexbAh!QTKJAtX?3uE7EDBv7nl+(>oL^W+A*=87+G0u*2cDJ3g!)tW& zHzv!>F;m%vOVK z+TA6q|2}x-L&3{&Ko@c$CP>4ql(q4`fFK2cAr)JFs54#_EA&mR0g=J!jddC1m!)lGEy8G0Nevkwo?OU5j@;`=Ii?a+9*B zVx3@1+QK9cAiRf$)gQ|P8q&^I3$^{?5ou6_m6dyt->bUCHlbS_5QPC|w1O$7%NfBh zhD0gU)p@e}WZ#8>tFghgNb!^TPZmFEYtwH;I(8>YLbW}}a;FXWDRy$YTKV~33&BoJ zmrst3YPcvHp(CsS`~X2X6h6^<5DwX`k{L6ZNQ;%IuddldZwLH!45?xfWj5c5uoJkH zC6W4?n*YM3k)}=>=R7{Qg!OhLGhRZi?UY%KQ?vooc6j|qd2RdQexYk9hz=}}?`&xW z2$17M58T@SsSJ0c{<^_x5x3|H)|bB4P^K-n8DV)jqoG7gz>0^BF_Fi_>O7|M54?5*5evD@5~Z9w+O7LAVy zxI#D?v`3ubeOK3gZRmw<^e$ZkTKOjhi|KHVqg^Zm*A_3%t3hi{>ii7u55?Nzt%onY z{&~6|R=dYE019!|W_Q!}*VxyvR(FC=H)Zw@;N;QUk}3al)&;@e@!r-UB*4mAU39%c9C>Z#GZ2XfC9)v8W&?Nkb90uHGnto2uh3PH<{F^HXM z%C@{0>2A*gY(byJ)CaL}jAfz(U(46+L)DS78FHznKasp75fY!`qWJZDa;q8*+c<)8 zYx(RO*3Mx$F!9-0e^8EVHni5h{B^qh7*!hTUwu)u3&xI%M&{l08Nbi^aMMa<)qhLHY=W_qk0bw4!Yy3ve0r%_`U903w%i>#&MIM%V(|uIAzd5)mW5V5pkE7L;y$}Xi7r^*Fju*$3auN_hekM;AW4^ZA*$a)x&z$`=#oT`m@W{igwBSN6Qi zt=GMp5YUmq?0J{lemL)PxC?QLE}$3@`BL`0OFE4h2S35(Gk1TS85tN%RV`l9R90E8 z2YLUz$%-fseI@5=_V2FPyBb|GvgvPU-bxH-@B@aVUcmL$4P-M`(U3ZPu&uYPpAbEQ zNA$#6W0_zp9LRW5DKyGPnF!wPY=|^r#wK7&2@>oYi36Bnv7)stPkJ=`Up!rWa^r@=JzMcpyrw(d+0&n@ zudUTLRMj@M^#T#x{fC-0h=Ry3j2&Kz$bTHq$SzI9?@sbV@IRO4*4NSICc@(23}xu} zi@BX0LLx(`Wyjcl`E`waUv6(Oe$7iL>vFk4c7W|?``v?FKm1MZP>&#Hh@{__;7`NK z>D(r+26%*7B6{bt%7Y=rtW)W?_*UfgLIPgvM>`2unDY73IrtpEJQCa zBoDTqKP#(X`H@_4t!KSwEl_eaPlC(AlQ7%=I{g-QdW7dYiB4ZA4iGR`vCdtHRn@tL zW`g;c{)qkrRvwd|(1g1qeZf|7uCPU%2P;!^GpEu=d#(ab{fV4guL;B=$#5F38#!yb zgvqNvmG71tex?7c;B_L=*Lp;X!cnfKO^exRnsPauHq!1e!QCZ0!<(_?o0|*riS?T* z9#y8H^W#q+TZ=6-Xjt6SMv87C4cRj`IU8{_6B_P+Noci z`fbPY&XaA2;Mjiu&W)&Y%A}DuU;X@xp>Dgj4Sz|wdzFH`q}&6W4p4_LqiXr}u~GRA zjf~~?h66wI5z4FbmcQo?q}@`Ii2M98;T){cV7<+xgSJxB8dW3vx5ZauYj&DS^NE)? zmprOGfGXwhDR;_i-@cSObP(66ct#|726hW1@4}hUx0&~_V_iHSBRajIM0Q1s*1HR^ zhB{u*5!W9C{K2c_&2N4B>8EeJs_axYuX|zPvui$-SK(^;$+57!U(+KFN5Vl^$_V{E z=%3hKKbwJV-)k(ZB8K#3iOtw^tE-zE3C6_%?l<~dLM?cdk4jNsz(O5;kv?oF(^4)G zyqz~&QLReUldK+lc*%>iA3am}E<=2DKGQu&l=Yo9pT$OpecmL|>hEj?<|WK#s5oq{ z+h^W~tu7e)2e2{&quc>3o#Od8q5M{vsi|tooVYg9dfZ3wUdipnqPd0Qc260u)z#Z- zF*zWAqd5^RT}E(r#tGm&=d`#jSapdYmJ@&Q4-XswGTkZPH&&%7tYVt&1Ygfr?a8{$ z>$nZu(TjZ4?F+(zs2FcU<=?9gb1^cCt2xnikhE{n&IHYJ%5 z6HSCu;pDXD?gP$~*oRZJ`e2)h7`=Mt7biY|y*g1_iz;^)?Ae7sAurd+w;w)zx-~)S zqqx=JY_S?jx0Y`y+Xhn!wyhH|ly2;ExkS?~jzmJC?56eouxSrq8((WGs34A3>`Je} zw(c=hRS>)PHm-aURbGC7g?#7b^PQvpc=7PLhKtz25ucO*8`0IqC%Gt-aV6lE>U?uP zwz_cWyX;1cxJR*c+MV5q|5P5;>?k@aqw*c2$4{Rd-lyEDp!s|E5b}WhSIzNQcKsa_ z>(AQlZWC5hEO;u2-}{FK4r1#5F+{#cQ{*kHFT!dY!hSPhwlM%AJ1s6V*8=M!N1ETj zzWw0*zn#_|EFU0jA$?pQqea?7izY;uY)QAG0|V_tZ8)17^-WF2;~fW3?)}|wJP?F= zyN$PEMTHK11+k;LVi&b4dxn8>n}CkyM4kl_<)kSJuJ_lZoA3g@s1Q|Pm5$RJ&`l~6 zYvV|qIe?&S)ZW|KCWfN87>&{AF~AKPdF$-uK%_5Bp(0;O9jw5a8GV zG308kFF<2+Q<6`}o1m<~`0DojRTLoVgyT$r7ufbNGW)ei?C*aoJC zt*7_UyP0CxD(}?zg@7dZVZqObLa4GT*JWdz9KpH3MZqQ&6Y~P0J*uqEB?YJG#7BJY zK!|8l0Vh6`eGjibgmin`&vhX3Bl4@7l8FBBsZhtD8A%9M-F`y- zT*p{Squg?JgFH#jdGkGRN7h|Ya+I7^uP9S-`N4nJ(5@`%q5+)4>@J1@eyOr$YJbS> z4-(gazwhR8Z*IUEG)ko~2An%y!UEpxJ0YFE-d=;PDIWd$tD}dmy*09Tagn3i z*hF|#&Ot+uEegj&VW2LC8(a4ZIPbK%?bwm^8WU}&7@SwJ6wT2XdoEW1yM~2*b~6Fy z*yHIq%Xc8^_V;C6!&^mrzy=#)Q7VOv96ES-WOTG(xa43yoJjg(QNri-1wFt3xdKjq z4`N6gQu>rJ*^)FQTiX(;Olxafdq=vJ3R>f)kO}cRogN2fwcDIdf?>9AGn>ptlWBVK z;nI=fBe1T;VYU(uD{W)#X!mNV!Y7=lSOzP$sYVM7{@?d*wSdL8>xR zg1xY+P?^ZU{FdOQ0?JP>em zKEemQBQMMG(-_IiNtnkgie5tWN(n!zeh4`fhg0*f7^s4jhTH&*wjiRiFO;Z z{djX6e);;rrx8$Z%UI8xY`w|jyEQ>iTXzs;0+g5Z!G{s2gCT6R-9kH#ATCs-y)GZ! zYU?(3nfeU9#sC8^z5WPj13WB_4`?==EV_8@_>s%!3QoPcX5a2j_=W3+r;&lN0G zds}C;v7yndZD=xPI&si#*I0Tc@-njYpWD_Ti)SHS<99E}#^=Yow`AkR<9p_0*BU zuOUar$G3p!_3<`MX~r--)YdxKW9%p{X>O>(m1plRZ8Z!HxBif~zX{}JkHPFjem-8M z3H`7dKFaU(nUJCV&s;WRR_8?TjNRmNW_$OzfUadS_I_i#so&6Jj8L%p8RX%y%`=g2 zWR3jEa-{I^f`5@ddD{-;#yJN?Bdg|sg3rfwnxL5I@q-D{66J-5_dj>pf{~O1EWX2L z0lfkANKvT3co>N}S^x2#b~~)>xxQd|_RPfn*|v6lHxpTm=#FRt8OiS@`sxs^H9XA{ zvBa7aCdB8o33kk&ZHywjTTQU~6M5j$d}QbNLko}}NNN1c4rCsp{A9dh4#;|IyjJ5A zya8_r_V`4fi1Kc?8)w1tQxZH}x)_X(qMQzzFj=bXM0QwS`eeh$yTPz3{<&~IqGn|4 z7@@JX>0d}(Y^>ZWS3H>|D&&u!Le?SvZAe&-Eks^Hs^!>YAkxV*HPwhR=kBaKQjZiJ z2mXCrzIUuvv#)GJw)*{Y4D^ZYkn3k6PayKFF>)b@|1m~B2BN)V>IpyQc&G}bM#jje zK=f1jKQvy~RDCjL?8PpPoH^20wp5j_ z1s;IdQQwM1M4!(?1X96pQ1oyT?RUcJTBJuGSc&MaEJYqc^zsJ_L9|i+;4u&>+vL(G zLHwBf!Bfa=q{v?AC_I2r98Fb{)l?(Q@oru~sdi^0^5>J9?yhrZhK41??~^1!g1wB_ z<;CMRL%azytX{aMieOzF&7q`7c_`d(_d8-t_C;um{LquglNXmE@~v+#MS#oa9ABc5 zCeDBa`K&3I5MZxG-UExGn-@`ml^7q+32uQCSPva!eS4XG&N>^sxv?^T3tAB`P88!j z%QHOd*ZH+RtzZ|-9LY7Bs-6T`isBqD)ZuYRPQXiHGRj?W`5OgLi3Qpm<~Wu^sU}Cg zvl%uN==T(%`%15f<1nZ-!F1JkFp+0bO>f~njQFIG!N(d8+(RP zKN!3ZAQ-SV-n1{{Z;uXy2fTf-7!tz~^rW_bUm8E2P7j<$hYg2J!#F2$U@m0BnWEBc zfQ|D3ArT!p5r)-Q=i3^h!*Q5!1UsKe^CG^0kz z>2={=fO8W5R4^4vg`)m=AQlLjLWWQS>@n#C16Hxtxp6a*zo%~fOX#a@8~gL|eZyxO zu3?wYb)Fa^22ZuW^Gj5|sYsr(^?lqY4Y#_{wx|&4LBovS>A{6u(r4q$6s)vdU!yq@ zK9=dmyIb3$16X@IrA-s8Fx6M%Dcz4Mw=})9zVMYn`Ht)fKKs~YOIZ zZ&v+w)vs3t$sikSMWn6}>Q*+$_?A&T=#x$IN;ev&+gu@Bt$Xh)jpPV8!%T<`c!GX7 z5|$J4E;kwoh~YRIb_VPLeBRr$-=6=r332-Df{n9rPKt8Cq{UThL)~(zGJBvAcTkE& z0nPFju~D+aYPbCU*l#r?<>pA@RccdV!>kYdpZnhTiG^%sC9Ujlp2oFsCIfI#FM8kd zUi69uAfMUkDst^~!Lu@#cYT2!JDZGk5Ppkf;w`M++0tTxjk?_}n=s}6M=#4u8VECI zWUPZoPz>>KuPV-kgcxq+T3E**5(0Svep=|AuFIo`;VT#3PQHiz>O{FNNN7oe-GV!E z?HncAB$m%!z;kY>m$aDrDS7w!^BTJYplredW=WFTihYmPZ*m%)Eez>1r|l^U_C@`H z>_56?jwTNU(*CqRCQ@#9u&Hf#|LT|?cKH2N2uo$!;~|0!8@rADg=+?7aRMt znyRJPSd4S*0fee$cf1L_j7V4v1c)yp=~x19ZV+AkN5{Yi6Vr1e;Dm(x0khvMlzYY2`SDx&#V zAzw<>nRtF$SxZqBwJG}r8z3j1VcZO6<#Z0S0oK(Ui*^A=_V>?y_A=hCYt?0RQB%+) zw!nhx7*bH$w5NP`AY6$vCfaC5EuuBx!c$~F)z86^+yg_tSU16OL$N~x(xvH~iudsz zKbmmGoe4bqy;mfT#nMC45G;iFXb_d5#c#d%Q(TwO#~a&P;w>?u7gnY!8qH|9Kh}Xu zxld9wpdMTMHjr}4Q0~T{5Np+Fp@k; zvma^5-OJev5amr(i@wTSBfy3A$u9}M7zrY!eDNlfS00(F#8j28CbAj#kqJk;4Q9RB zuW;?bSkg<#PpD#anr-Fa_O!>BCVz@wAkvZ>+0(2eapT;!j?cJEV3^GDx#<`45R2b>MY+s^UaJ?@hktI|8~p#&piM0<$2z| zm8x6!+o+?7T)^qHr6cJKPlu6M^;B3WQaVqYEgHb8jg|*u^?0ErS;x1?^ zD=Ob+T;-V$*9+x>k#K=dY@;0%828GB>ZG^F+vbJ+HDW~_MrER9sU4- zehKSy54pRE1HJ>|K_9H!E7y#Dr?;_LPR8kWVajynUaj5jraYvx+1yaP`iAEAegd%M`^EahqwQyeZ{b7Q1fNV{ zV`{C)i-O?o>B#qjAxp$0hv*=bS2fB+MfHaAr1HScIX{rY!d_{Q1nf#66LOALfhrcS zZ6YY6$z#H;CPAzxdCF(;wfexs8l_=RN4uvR9~h9j2T5JGd|m!R{}&dSrupb^Sp}Nb zjE&S>-B@9&0(Od~hH3;xnq01VL&?f|M3W9tWy#-bW-IEfZcT4jxGhSayqP5%SVY;b z(a9rY`^FyBmu+{N>d0nmLrt>*zF4(r|2+KF$y=42$NppapDrBk*uRHlwZULG777O= z`OAAwmtVqvbA9;B02$KyfN_f5o_>A`0VeQO`o@N4M=8F%Yd08ShASMZ$1fDjDp-kM)(7ZmTZeKFu3ePd?7H0OK^|_^&nVn*j^jlgwKAmJiY0Y*U}B_v$Y%f zx8`EXDmM~@>DD&HCAtHM+@k%S|Kk}v7__2}h=+K>1T&S2@s)}Wsl5n*5Ev9*Lq zcejE@wQ);TG+p%F{iJ5v14ija%HwetWgeun@FsriW=wg^jf7y@+=fsh6#zc&g3Dk~ z6|dxj8<<`^$c$f&YR z4@M#evpZTS3+~ahdL6-jY;Q-T*GKSTkoV!jR*i*oFfJnfSn|}EMgOca)MK0kH(2SS<8>c8mP)xW#Fa>lC-HPsc3eYXf**Qv1w^hOg^Q-;ZZPzp4@I4G{) zJHY!B>6O}09FwcG#Zs0|UM-*G^}ZWfbGtU)k{`q5rRRQk`^(ESl!eNRo2Sh*65<io8bdkEN z@-N0(_2z*co!i|Wmf`?dGwo(SQeMBV%=%n;Ce+l~)F)h&@_XfVJ#qo~g|BJ)qOnnd zD(**Hty{*IWx zqpGReN-k1nEl^gjQD(rKE&DdUhsz7bcIjFEu=4nO%GATk6O&k;;|L|JX|W}fe)5p~ z(5U7Qy1tVQq(svKnD|V!9-8cV z=0xRXFfA(ua;~D*pJ?xu$Dbr@0c*(ShLzk6#lsgusw(cJc7eZtVlIR z`!Gmq{eb#ub+y{4PN@&7Z;pF# z-12d2$9**JvvD`Zzcqev{I&5nvs77+WzEa_vu3j9Wz7fL4O&5aMfKbjxd(Dj z=U&del6x!nPq{zlY4YyPBk~^0o0|7}-rISr^0wra=QZZp^0>TkUMjCMuQzXB-r>A6 zc^~F|k#{ri%e-&$zRUY--j5U26XA(D6YrV$z(jl^d2H+%z1QYzj+xl_ z#CQ7|3(56hlyh+o++pKb2f4_lj4xGkuxCCfywcy}jdT-!Usw`-aBH;N+=UPDov~mP z_VZoo?f;Z8%BA?xE{xyL4e!SeX}_+0?=@h)w$MN`-(nV3HyU8Z&ARE#R6`N+3gxsO z3=}u0v9%w#dIBiCnkCn*dnl9GesJq_yp)24JZC=>b%CUBm^02Ij1S1E_ z;#S($>LKfLUbf$CtD z)F3qk8f~3-G@Reo$7jZ9gNYu|qxTXuM(=e<)M(M86DC9_QGSV@AToL{K@cru7z7Ey z=tlI23^52s4PJTg@80*f?t0I&*7NLjp7YszpYzu_Yn}Do4hJO-KiIWRu#x7M@RjBJ zOfQVc!a*tl^FLWQgHI1DIPxeS`tjGxVZki?ok4Xf`}B406{xD3NZzZBglyw6`r$y> z@pAOib^d$9st&=|m%8IOf~u)}=h}zgGx()l7INrM)iLUIO!rQ(Q$8+mzy(kEjtuuC z75ydJ_eSPoX@?EV-&3i)b~LL|+8{dL#-h^sV^&Q?aJ)PpuZtS;_3dGU3*tCU`=<{* z!z0W$?h@%yuqErU8jeK9PzGr_KLk!^Y9dJ!)a^P#oaA=&vzD)F0>E_5E(oJG3?%B^ z#-P`=5IO0y;DA*)<}Hea+E238mEw>NH+)r1PvcO(DE5-_3!~`D^Iu5j5ig!Go$Y@2 z5_{rJd#<1JFUK$Lk-pv|x)izKTLoU+Bft9w`(+M7Gbz?}bpI6OBJ6bYBrCD$~ajG2e-!?P8&6N5S5hAes!ki(KF?;Cz zu#{V^Tipbr?@|rJX3pP`J$!|j?1q2d`9<`qONo((B|aXGZ(yN>-UY=TRo)(A31IZg!%y``u7uM8EUW!LdIN}tcICuNrt) z@n={67o-e$^FAEt@4}1>OxBvA>kpGfx-{EYoX#(h5>baFt@!UBz$=Q z-YKjSj#Rv<5@ehIm0zQtdXJ~nA>V250Ptg@5n6m%!B2A4>D7^k$yi6 z`abGb@tO7Fc~PWqA9ktb#kQK;CD3qt3i{4x8tQZ}vx0%?p4RX7xa>yFj^Pds0q%?s zv*|z9e*DUrq1#DLs~UaM8iih~n)CXd@SEJJD7k@I@70)d<$zTg?5E78)=+p4zYBJ_ zxrJB0y1+hiokeP6ad?ZRd(Cc+$UBEyt>#SyendL2nL@(#cA+V<{3Q!Zo{&oN(y?Ns zOSWMI`)})biIbVR(c{Eu3^S>h+)J}}16CAd1Xq+H3Zsx(-#Te^9Uv68YMGD{)B`NI zpCMr!Hso6ZkQ=qNysHZeoBdwyAawTzfx2D#dbx4}&iVTNMfvQ=R}hIZYot-F^qqy) z&5wr>83iPr-19B(y69YF#lk(DZ`@9j5OLd-M6^BY?3~~~!oTF{{<+`FRz~!QlWwTx zzJ6XJ+iQ9ehFms^($FZo!s!_yq^49&8uJ0{$Evx<8gr((`NKjtspA%Zqqv2e&0jv43(f5bG|^*XoH4#vW}@C{lP+PWxpd z&$AxiD^|1!aj=N2<}%N(i4DOo55`QPemV-}ryTy=G=eAxY_Z{~E|{b8t#GSfs7i+n zTPK|%WecCWzAb1nE&9k?M=>GDT&K^(mo5$`ch=(edgA2DQEgkUu7Wp>)1xp$fst?K zOELrr;)0>Bb&c*Vx#F(|%XeJO^3{q*Ic@MZtbKI>Vb9*gZ$6yj#u>if8n8Tow1l1> zywLreFY3~R@xaxkTJ3AK;%IP{yiQa$%VRy7t~4z!XDQ}3vaN(nn%d;7=Zs%gN_v8N zCWAg6NQx~*5u`%qZ}b-RXt)@RV|;bH74fbi#hve*UKLGD`uZFl0HiKv9Y&vZD!(d8 zEECA% z`J9${8O{C@-wpc1YXlrD_iZXEKd zXe3ixIFuA?>*wiS*Vt;lS`E20@`CS1xG zlb)sL7b;`cVe8gdR_w&sPPws|Aq1I0e@F;akm&nBqc-44;A@@0VIi$8VNhB&W^9kfYPvZ#l9X z$D(<<>w*ev_Lakbon0zM+1T@m=Y$0cB`ChbX}K~8cBNU&yzV-=XIBAylvnnpVE%z< zxT6`(AhYxeC4;7I0`+}`XwUY;WJ5FKx9ij#>rOQC^!wThk|*Z~3CjmzegV`v$;nn_ zRaGzetOonqf1O$j&T2MVcMU^XA9eK4t$Is1Po+!6U)xn#l0O}w>0RtWxdb*z`X;r; z4m;WaeD&`(fTw4$EpqG?Rq`~4v9Be#8y))xdRDCJVb4RHGEs?|*}>SyEGYqd#2CUN z($Bj6g>8G`rpI@7!(Zpl?Y{@HCC^+~(*Gi%U6hsGOo5fuzmIJ8psTB7jJP?P_9`;y zJScjNSA~D2h-`@D6PPVDR8p1&2{bN#Bi4!cfJ$ zr&~p1cP>5&@cr;ey_Hv4F#_L(c81Fw)&#NKw^c02>!LI@sr_tRlt|+tlU-{kJBPMf zIoH8Y$s}hn@X;C$HPHovsU+j&+1sWf7+8ds-t%fQeIywHe4v&T>*qR04V*%+ejBQ)cn$x1IYNq|>8t9pvRZ>K!brx8H zM0;jx8?xwM_VJeasem%JjkWgLR+PG}U^Em7FT+{06v=~g>nF43gm%k@f(K)3lpe3< z>h2Yz$sac^;Pun;b+3x#&=?79kT$ALbSF25_wgCNMiJ=+;EIq21pZC<*U%MLBo%y4 zt;Of({P>;8&Sqzu0NAN4Ew9OuZv{{J-7x5u!c$o;pJ9AXdgc4xr={)QpW1tq$*o#> z7ZJy1O{{|Mo60G{i<6og&V=48L51y!I?hZ6vXcCP#R=eGP;)!A`(sGYe>b<%P zc0#O$XYcd#ro|kKe{JyrawtlJ3o*FXUC6o+khB zb+c3JmFFC~zDQ+Ej>E&=jtVlvWHqQEDKUd9WpWr{61EXW`QNeI6ut#s6Kc{Q{>cCE zzKA;ebWj88DU&s8b8E+0E2pDkqZC^jwTGGsWF52t`O99VGWstbE|eYK&LQ+KYfMya z+nRgMEFX7^nb)^1MJcoIri&JG>3P+o%g$!O1q)cv#i5_^MirIbP9d;Hn#mRF?`r=7J4-DYdTJ09|dJ{7c# zD%}%s2W6S!Jp}&xZ-xt=7;FFM<#y7~HeaPM7AS<_7TyiCvfK3B9D>Nz@hlB(y<3!- zwJ^2wseDu8xy8(hmkFR!s|mb%V9&%sg8{?S3PyTE@nlO%voxjnmaN6{x-7jU zxwXA9iT0nzRW0z~$!zIk$C1xtA}ekJNY`QA#O9X8=oIZi>I_3Ybi6!qNf>KKGRM@# zVhbreWj*bW*k2{xM;^XG#DX@zU(;H|E2fc$HZJqAxHhB-ye-R_J!Phx zvheRSMk})2(o`d6WD86r5Ixfmb6}3^&a?YC zhetdjt6B~AMw?_o)1RPkfi@nfD+=AIfA96ynflV;Yt%f~SV3>3D11_XuTo`RkY|OH zUotP2bjqgrgGN0?P(BUQv{*;wydtTV>9s6z{ey~NRS5XJ zQbjjFai9@AxWDWha=Wd+n6j^E&9as|F+-4PF2Yt8wC;sLeZ8;zpuZMQUoO}mqu}^C zIv|5oBYC4vhzz8kbA$erwa}>$+zmnqPLkk+Q^pBoyhT!>$fDoC zHAbs1pi0oZC!A9y)X;HK^RhJPT9j&^>f4wDwdl_PEWi}7ZHZ4{2MLlC8vhubrx*l$ zUwC@~MHWwMF#F?)r;NxfjX?+|Tu3HcsC~pWX}xpm11}uOLGJUmeaoqVg^|F z1Zmtuf~`rx&E3nd7;!U8vOlTB% z?h$ks$1HyVUW+0>&GZx&2xMRGHOF~mf^TlJQG*6C)xR>5h!%;O*rtMN>IhnmQ6kU;n0Z&x`mNe)J4KO8Xl;;oy(D6IwNx{4$%7gm3Yf38H zFx4>Sj0z<2aSK?;j2z=RCD3F>;4!PN2HbKgq@6-xxTa*?r`xJ9M4O2iFo6U&_D})+ zjS&YlIh3CA18}4gy%$t<>mmx!ABrWF9^Y6hdcTQXRsben7NlimJ||MVlmpyWf-bb} zE1~^>7isBZ2wV|mbj^0Tl{I`*5Yu#Jk9%lqU%oKKMX7?(kYffC!kHZ4#4x6A;Rew$ z^D@T`tSyPp>1BG>s5$44C>?8AkG8@vz%_D0DeRKUMOp9rhkaw5aqvY|J9sS=h^%W5 z`9aD+j9^~7b57K*6TzWLq+Cm~!9bWuyVisPI&imnD>%NTZ&$C~Dp zwS%lgoz+pG(_aG1GB_9@+Zxc`u{)AbeFo&yFb0Mwmz}a4P&|T!TAxc~)aH}A>;nsx zAemKAjUB-2PDsji&)`;!Fesa9mk*r_i;7sY(-qp>_COK22f_mgNDInUqBn)(Q?4`XxJ5G>mSK3pMPyv77dDy zTmH=z9SmS35&>F4>M0#g@{ECHG^!q86J}tBHsF8FCe6GPF~~s_(8K@;q6g4l>4TF2 z$_qa!Ja<)^h)4qfcG-Xc=vD8(8t32NJKm0gfgWCe^urMdw|}x9uUPNZCcVZG03dDs zUy9@Z$5vf={G0u^{`i0KNU=`}S~R?I+CTun0|Wq2{2dVp_!B!30t6u3U7cKgUBnQN z5&sNKQ`W1{3NTK>Tn&401z-OU_{TV<00Jzu?%w0nGcqy0Yo-P_0j&OG@Csv;SK;{htZ^e_HE*sDHID!T(|ZJ#)W5D@RQ7=Re8S N6Lgg{VDiuFe*lPvsYCz( diff --git a/artwork/scrapy-blog-logo.xcf b/artwork/scrapy-blog-logo.xcf deleted file mode 100644 index 320102604f4511d26094cd4c964ae767151d880c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 52428 zcmeFZ2XtK3)jxXgNS1BcR<&hGqcZBvy{K8eSDR{MTb3-@>dg%T1Y$@?LQ4bkg^&c2 z5C|b3ACM3rgc=ATKnTUgOd?wch&HdjGec;cCw8a?U>c z?6dbi`^-#pYv(b|%}1IvO|2b`f*^Qk1VKzgga7>Th$8+3;#pDfLqXt!e{1o0;!)!X zT4Xwd1&B^YL)}u+@iA~K%x;wg0 zXcCo|jVGEL>NQ(7ZBhPxNn`5*DhaF&U&6YG*L-z@tT&%LR)3_mt68&kaV!9D`7+=F zY~zhxt*11tT}N6PPm;g8sp(|nsU-o){huH=625@oqUNe3sr*hA-Z&1?A=M+@Umo}w6|3S^+(#9PjuhdrI?r(Kr~Ak&x(aM$yP43@dDA^G;FWBtiAfO_L|Gu zzG$7CmByBo(en%@U-E%70Mc8o>%a^kLMyD6P`KD*BehLo@6|kcq;H5z|)53Ry_AoAE*8P4}V}Kr!Rm0|M~pC zYfUd-8@B(ywmxg#6i)x2{CEurr!RYuE-=~nncG(i@8Pw2p-pEWztG->SEeWUC;k`y zFrw1uufF!kiTY!=XbL*IoAFjq@els}Z^npkqW{BqzAQ#CO+tDk?!&7)Jv!BR^Qny; z^|v&h07yxXb|n&~iM~wqXe1X+-CY1)-BEw?WNW*oy76f1jh!17UKjBjPqyCLh=k^{ zjD~Cq*qwtMYrVO#gXv34YZp@w%`HkYQ;#?G9j)~z7v6W&cQxOLyiF6`*tPM-lkC5t z{$%6Eqpc?zkDO}l?$UIeIsxFnCvM1wc5K<6v2EK@TC~()&ee9~$y2ROw`e*Vn@+LI z?wdDub{}m-+Ie#$GeH8r)_U|*i)P1`)WtN<4Q^>{ZEiWG*}8S-m2f2~TbvbDxbV;W zg1fO~4hX^Cv&bTTj^`;nFIN*anUZWWEk9Y4toQ-zEO<1YRv92-EAXzvsoX*4cS%LDx4Zj?Q0*5$3&H z-*w5$WDh|KcM|%d3i3L|b|WV1DYzcRi|bw?Qjo&_DtiMWh1&@_E`$rxHm$r$UM9Fs zD2h3ZbBMHM3`$;xNxeasbU+9JhY4iT)EF{YDO69mbHA`&kUZZ-dw%l4J>f|0q>LGb zqaG(2*ViG5({-O8@?q%)iEvjZ(mY8s#pzB|BiD5`LLApO;B;J1C^{%a>MXUpurRJ4 zuo*8=35_6-8VQviQIYKzguV@S4vq8)pm<-$6HkTARcM(9o&`Zg{%ItTE50@Gm)u7?TxEn)8vmJ65^<6^Uy zz*pHS$CiOzVIk&uKyv+wz&ZlIN$xPh9w+Q(!uArDNZ5UZJxW*}VOt10N!a^@We}E3 z*j0c@^Q$RN6@ifi#yS?>|}q2&_~%C=Dhz*{9Gg_u2RIA&B>UFqU9x-!LeL%4Tuo^WfKKnub8>*} z`VlngT=}pIE@R%Al=L|_hs499)1%EKua#XrRCPQyrSygkyGeO31^iVGqTymL7fJLR z64l5)vP;?Qw-NCO5!VrM2#7f$xad0<>6lW8tfzSMrU}eUy1;WRyv<5^uJ@(KfAkn8 zGoN=lded`)*WnlEVVqBT3$=7XhL+gvz^<>LkhV~xJ?IobgExj{jKbKOI;*d+k@yFu^;2S(vd z010$M;4@SYLiK>GbwdK6gkOWk>FjxjqXsR4Vf1if!{5(e1*hX$?*_yD#4q3Bs-=ENqGHJ8b0=7P$m_kgW{DqLyF}uvPjWd*w|*l;_6Yd$f8P{FzPM zcJm9pcF8{c*9Z5mfv<$M8uz0gXTp6WupsBmlUuMYusa?`Sz>PYKc@e-1<92&@=1wy z==!-I`d8w}D(LDXSm~V3Pw28EDoZ^Z;1J>Sr`N-UO6$4tq^~~*+m=SR!b0Kn?shxe zaEuuS`vhc!kSF8_S$a>$N% z0Hw1<+@oUK1Q?|;A8wgiQanhCtu*=hTQHO$Z6}567WT1fp3CtvDPln@>2@(<6>#aL z0g+vNOoD%sKv^3q10e|f`NTd)?0I5?Ih2BpBli2m{)*V)%dlA2cS(1K*q;(>xDtCi z2X?LD=6?7~(1q+e^Q*Ayh;8^%EKF2!wO?Pt>Mt7@*b5ZvAuRB`+D}l1)?G>P9tj=? z0lbfWg0hWVNuY$5r$K{FJh1JFSg;L37OW1#0#tUo? zu^X1Kf4dUfj-DlH)e`!xE73|~@F`70S%BBC#12!epD$tmav2sTxs3xiE@A(CCHA`< z*cHBne(^Fik|$3c`+k?f<8dnUg)7i-Zs_|MLd@waLhswcbjm%i^^VOtuMu`472-BW zD&U7u`nDbZfEx($)n}nU>R#LokB4H_ny++topp)iTA{%v`QJMh= z;;#G2{dagSB)_1nA?!WE?j|geu-|d>de0G9f$Yrn3Ax9YG36^@)9FK~l7$x#c$$w$PyM6-Wr%zDu-@*@^2h0*?~7 zgWTH)Q&=LHi478~A&i z8wtU=(@02}ut9M3q2>HsNW=?G0FWPoMq1ZQiYJ7Pf~%14ul_CHUx)Y7vT@S>&?tK# za;?JBTw(6wR$(UR{uj=h>@U#TJsu~qO=LFIz%^LM*3E2fgc+H|u>6KQAA9+uk+CUh zc4F}FPu)_qoJXB*Qe?sc=Dq&ZJ0trcN&S6=$V9A#n0(sCdL)X{r^*?Lxw5N-z+fC@ zrKflmfus?@ayKvpX-5!GI{OH$a05eS>62A7=4%P8bOY70>sN#^J9kyNVP3N1ldG~f zMQOHFfAb8FW4VmNd#<2>H&`x6d7mOn!uABZ`rpww`z`w&(Q!2Qs((i#;;P=J3%T)2 z(b!)JjJK{oekmFf|Akm3Zv02zz_tR*^cy<*GISb8X0Gz3SZ`tjzZAQg*^T#?qA)5m zTS*eX3az-cuM~MJC!{$4FOzsbu|N1KW&DiT-Cv4@CcPL8CCiGhz+=~e`@3Al{}>s| z6l7Mo{k{~>RaQcB4Xx-u*kL37e`fE$qBUNLw!fBETD50yL)YoY%pj5ewEW*`)+0Il zCJ*vgaITY`(z6>FIxg>TojgDLhx*lw6qkd8WvBHnCbCyRu@#03|6O1;mhR^NoTRh= z>m4m(%iPI3fBxPScf);Os>bCw2ww9d)6m8*U4AS>dN1rsnu(npHQB+Dk!n2NssKe(Z@ci8vlufqO-*tW029wheO zufnDi`?IgY?jZKRzY42V51#odtWs}y{Hw6nvge1t3foES?O%ocF|oB@g?*RUv@gXX zZCW!=P0#R}_&+s`b*dxxJ72L*2y4Ke{8B7z_$pwTv3+3z%dU(^ULOe)TaW+t1Ea$^ zY54Hzn|FC2iJJDzzw=M$#^z>VxX!@Sy*M~I4L^*;S=AYSp9PzHh2UI%hh6r?@&Ccq zIOaRO?wZC(*u2T-C=V_DT+hOH(g>#!qmVk(+Oc~x?xW1~r- zEYHkM{^eJ{dxxn(ybVEM#`cE??|K-gH=O_Uf!Jh{>P-yn9B*EU$w*$GVmPPn2yf%1 znBOpJJk6chM$e;*;fA`IQ-7D&?b5+Hv|U+%N^2|(xXx{C2k|T;o%QARJV1)K$afrF z!2AHai|Dxx-1bU6Y$G?Cl6|I;eT5Zxcv#H9Uuix-HE*H>qbl}vox&eVcXTgFhloGM z?v;f88N!~S|53`lTe$vD68qom-BYPV?xol2Az=s69}xW*`_5qBPJf~wBl-!VjYR*J zXyw@dG4lbXo~WD#rigu>ScT(JV)NO}l*rsosVICzv{DAViRegbC6rhPnQg=xIov+3 zua^>Uw-T)YW4JON^czS3Py`ugtAyx}*lOhTSu;Xv1MDDSdx?FDtv?W}RQIBYJxkaj zVp$rLoNCTb(Nh@F%E9ai(OcNzgTyLlmk?r=v%*neCGn!dy`7YsN%;kL4S7xiCh23C zd?pdCjLkw3ayg#sbpSf8M?INJMw37ho%V`}LEwY1@VU|7+}RmGWHtw%BvLVzM^QPF zMDA`>-l5GEq*+IU^!fv~lqBeh>Lf}zIE=&Sq-76dgGhpMzW+W^N%da}(~ST~7Li8r9C6oUy(G^kVDVG;bwB*#yYK&1$fgV_+=(7zSW?NQ zY>?&8fkyEn(#lzK-bAycak$>5yenTJ{x&YZ5}ZFGT1g?AxaQuk11)`~Li_ZxJDTzE zRXDA>Mwl~v`pecZNaP`pyZ(BO=o6K)vtZAm_FKMl7jz17&pNPqowz>aJq&K+@G{>Z z2(!p(XYs)VpN=uqiKLkTr2UfmBFmuP=fZ~Zd@*}lWnD{~vJi(knf{fEqv)lrc2i@g z*gC_OavDL-adu%CZj5pX)&LYpSbq}#WWa}li znv{V@*<`s3ew=;?MGV39I)M+c)lHz32rW8SeutMWNOF(`oPKt+td8!YfVD{k<5 z^EFsmSf`~NyzqRkr$RBDhwL1EWHVgK+#i3^jWYGs!s*p-Hsj$dYlJmi;G1Z?O1)l! z+e4;M6bgfp{QHPZq~X4fmZS^E5cA7NjC*MiD+t3m7EbOuy4`^dT<;nbQl0OVaVk>G zM|8A@Xc6MaT(+s(Rw>dj=D7<2{IC||54G!2K*CtfTQ2vd4T!7 z7X|iy3g!T_2>BP{i_>{ph3zL@H;8^wlV5qqB#P|Po7+)$onKb_8gDt42~NM`AEO_q z?KfGdQ969{kN@iL=X8$wQOs#9{O}L|e8%i>Iw!U7KYeQfjCtjgSEv_zj^#ZZGO@$H z!&VzBrtwR#R<@obENKarjg~YWf+ceXbbX)dQjY5ELt3faW4%cwI=q8ZuDkPXr##_7 z$F#-Gj}f1Q;LdYbFT-|m`t1EJ*C>$`+ZQRM4ZInfcYN}h1ygqg5SdEk*XfQP?muFxJ9?;RTbES(n(Qmz4drA82VVf=(gwIg*}%Q7dsq zdW;{?lWOJ(y@zUgtHykGXW1jHl}g(!KLsD^od32`nBnB5m*1l2dv`NdY^M5^iHU#RPuGEwkt{tdl>Cv? za?bs!g0-3jZHa>2r~KQGzxei_etC2LYF1np5GX4A@3oZ6j)z&eSjPoYT7gR~zonLP zd81&Ji=T&406{rf@s4Qquyq??Q`?^%mF)em+_qLY?b&yQ3BekuCd63$d0yDeyloW| z@6B*-$~Eq+udqdMy$QQSAz-WE>SU`HEh$*J)&_o?;QAd`;RRli$4*nc8h6zNHb1q7 z&F@i`rzzh**}4iaDVRkiX?!{HN;T~l#Q%Z#G~)edDbz;3%|=CUjD%|74StcV*J*^Mnq2Q&hjd5PU5*+LU*0c|m?f<74B5Ji{u~ zL(DZs0@*yr++sRa*hKBZ$If3xMcoTUNK?C^9jEwB<;3w3KVaa9y#3X|srkYGu2vyt zZ@WU<*szMxC}+*}TI%dsZjJvq4k9!JeMpN>`%*L#l*B%wpQ8OL+`AVo8LNO0=ySIc zGRD+P1l@}S_P(y&>GA5=th(P%atf|J+}=tjuefQHjfg0se72&4pRR@2h$N>OozuAf zzTEr4GKyT&hZswI=?7ejpH=E%?<6vxc@{ur;|IykvwN9ENwGX3xa{sO1Lb+sW4R(- zW$Fl*#ZUl*!83PX_ji8!^k3fltWQ5SEzOSgyz#TPZ6Xd!)85DbHLZK&m>*o>->%Rs zLR&M{nL%j+nOjRXUykxeF6fh(q2$k43C9hS|2|vy6U~~LG<_{Kpqxk=8I{v{`6Stt z4+@;Z9Ogebl}1-4MbfsVS>_W`jFM~K$n`0Lsmz-`dF2UKQtzY_ng7}{(pQ^E{||n? zSn{}tn2fX4vcIFpY10E$n>5+opMY4hYA9aL93nJar@Ei#IC}=8_8tcu7W&cjGQ3!G z5%Q$OH!&Jg#DJF=Ts}=i8saQ`jvtJGP_=pt;&gnAab_;C=Fzj9367l08~*_tcIxjA zMkt;3N-R6_wYwhpC5D)YJpAw*eG~KZV;BB-XRfDkTEvGZwy1?nVb0^WxnX}H9iK9J z7a0qDa+)BOTX^ezkB}-zr(MWV@aY8XsTHi%O7D4bPvKKOX#aT)+v`4I8=nI~d;J-< z9}(GJ^-s1RRoWhZWc!aE+|b`4zR1AJGzdGd6wE=c|B{@Or51W zLlMYl{I(zZ-dhftP%RYfoF3zKLozSW0{C*j@*??PL^_-zIQ@SQ7lqRvNVYv2n(^>e$QP!6bh>kY(dLk4upfjA*WlB~QXmMF`q5#Q zbkNzS8xi_^5P7R{`tgNt*a>^^Q_^@*n0xMxPYo{m)&roPU$&v)C5VabkEJi2!8~vl z#Fp!(5eE?{Du<>`1o9FaH#KM}pIvaRVl$e$=;4lJYJu}LTFO<-ZeXQl2I^9;e11mB z*}p+(sEyDup3fz(cgfbJ4D|~%r4N{1N*<50JsHck-*aqBSF@e1oGy>ix5PcA98+of z(ld(Vn>z{;1bl4{x`RbO!}?jDU}%T<39}FIWUzsM z{CY;1(h?9-a8g-lPvZ4eJR+Wz#rVp?sT(`$iThvtsH6BC`me$BWgm6K#~c^`ZKKfq zw}Pk@cOCDP)8v`g zI-8HT-*i&mDNl|6sHN-Jjkg|amUqaGk8eGCBA8$F)*45S2DQ}S{{(ijf#uHr~CmP$j+d7ZSo8|Fm>f1X`b|3HTXg}W7 zd8|d=C|d`=ex&L6iO%-!me$s`rk0kY@&TK@p>~2;y&*L+p9de>NcI<2IXsA7euk|#`@v?sWp@zm&-6y-6+uK{(4u9*z$;rRnw70GMh#V*DKIv>b zcC59nqpho{rK`EKM;@DzUu{0rbXbm+wa<5)=<4e1?C5B3?`Y{f{D3@ev(C&Osyoss zN6QY=ef5xo_;#^~i;c!cx93>CEdsF9$=5{xKTU%pi=fUptj=?GU zgPQ#fEpnth;C!^D>v(tPVGhvN*4omr_9c*m8P7aqX?>4nkjF#48?4+foxw-DBc+}q3baekU`|1zx zuRV+qZJ+!_L)X_jfQCX9bq+cb;m3(8oHC zG`f3hYC77`yub12!9$I;O$QGj*pKLLe*IK?cWb9xZVuUmmBXl0M;aOqAFe%gsOF&R zATq}FvTfogb*(oZZ*Np+QLGo6#U>YNoktq#>%=;#_TVA$(0t9o8?MJKn&Hz^hZ~M{ zb#!+uPDbhZ=n?UVtG-rMJ5_V=(4m9-_g%Xif$#JGIoff&<7msj(?W=P$WVKzrsmLr zz1Lp7y9yEUocV{1UHHybcWV<2a8cx=M@UR+(AFHd;rgqqtHi26dEou-!^dxeAovV! zN7KUm7Kk~)gW|#212^ow`YMtmAM%$co@{`no@i}vg*;d$v1zXHC^^K2iTcBb>*}DB z1N--0bJgxDRh3jxj(GgB@y-J$PIPr1X=-ZkYH4nBYjJ55g#{eDE32z2E6dBvkOx@a z?rQ1kJlS!)_3)9#Bh80fnj4QEQ5_K*q$BmUVy$>+7L&M6k!q%TPeo}(bp^5lljB#1 zI@&Os=Jq3X_dfo^x}yz8z$YG=!xCWfszYMU{PovCQL3t$@{00`s z?WGB;&FZ|+&{BW6uC}fY6Pv2pzxUcbRTY($(6_2`rhIo%X?0p2f}(b~ySBUiSX*l| zeM)W3$M%sav+J{KTfwTTbzo8VU9+dEs+vMNDk~}~iz@QA)%_I#OgsBZeM@I+M@M64 z)6t`?ZAWVEH^S9P_Z_Uq+$L*iW>@dVh*g!)3wg^cOSYH)ng^*K*Be!xEo~jG?NDk{ zBR<^ST=VUZKR^G-zM30gNr!6oUw=)tN3|1*tW;Hs6=L~(Uhci4$d^vjHJj<0KkA}u zex^E~uG#p_?2-z)=C@mN%SsC9njfpoE6yn|rE4CzExWLwthy+duK6!Dx#cCrC51&5 zbj`M36%>|~7F8A%SLD()UpSRjQdE*#Sy)<{S4`LZ%D!FYB_(C$S%sy!`9*ZiKPV_I zE-x-EDK03^%SE4X&7ZYoWtWu~6_n%`6cmCi7p{5w57l|a)w#um#bWVHVSZs@Nq!D} zv+@4y+>*Tf{L-SLoPzwKlA`>ALilFWM~&M`D+)_WO7lyLi+APb7v^W@7tlBVx-6@z zxUjUOw4k(XXIXVgPG(+y#xDBip_>cy%L~hj3(HFL3bI>YH5vYRU~6tcCVlhU2eOLM zQE6dues2D*iuYYp^RC|&?a0oOW8@LnkMpuhN{Wk%iXc{ALB`1m^VICTwJ2*>7M=6O zW4Wcp3cgS*9M3P#O+8^5v)QJ#C0Ti?bk5ITTToh1N$!Gz{QT_Vl8hTZni;jvznr}z zBbUy3?4FW>{KE1)H$h%rQAT!t`mO(*H2>|$&g>mobk1j+in7ZJ3Mz`(MP6QRZhl^N zQO5SGj(1mWOV7{Q4CmbUTxE7)X<2!GPLZ1=2Xo(*k+UmfOKL`L);4HCEl=vdn^#g* z1dSFI=jVxe=G>f|?CiqK{H%=BtX(--sacs@v*Dbb@3-v2bZq6t`Nc)K1soBCV)n$Y z!kzi~nY%K!Wo=E{nTo2N$@N(IuA;)~#rb_fnzd_J)~<~7^t9AeJoL}~cjV@kV+|GB zd@*0l6LVecb|!0A=B{0t8R@F@`Hb`(Tj`zuwl6=oytuHWcu_5i2-#U;mMT+~;gLR{ zx_$Eod~EQn^Cu+*<;8hiyG8AAQdt;36J2IxW@MzNZQs0cLo!^m_R^`Gy!_mfisD=_ zD5_i{7BlT22Yvd^Z5vaP#pFPF?&WJU3X7o$OMXsaQSQRz7W)<<#%x;Zj*S~ql2yt6 z^7zCLuzISR*Ik{jHv*vbXre|hSGVpHS zws`|)GLw*)keD1FN5A|*<4!0FwppB2`m1r5^UsyLva+*waYE^7Sg)Ckw4K{GLR}uo zGn*6QlM<5>;g|JKm8X`K7Z>Cf6qjb-ZkwK(k$#e{tcaL7m5~Njq^EA(l)OB7K0aZ? zhIla^e%W+uI=o9>K~W);SMY|zHsz2%DoBTEa86>zR9fn`%^Q;wlaeqsRnknthS-En zYJQ+n`~J0AWqD>J@aSnYE48`e59J5`RRh__?YMz zO-wR9v+n+QO;lWLN=#gG6g~6%HDNJPF_Fo!@$q3%^vqA^gv3V0BqoJL$A*Qe>6uT) z#6(1eM@7a*M~A7SBI%jm-5(wfQcu|^SOqEs02-1TugF8cx2=?&kPRzwm2v}Dh!T!?B&Yf=!Ce$ z^vRr7ZZb#qcoc6a7|d9aZH}KUyKV{A3@Lj!NHh> zXa%nkHRBP{k)id5DcLo5aZ_k`Fg^35J0s)bw#CM}`v{GWizxe0M(*)^Qb?;k(!`LrprruKSV#L&KIt zr;Z{cBO=1oVPcpAFX~XWnrj2U{8?AL21KVuYj)Q;X{P@UT#IxESsX z3+1QJ46kNHM<&L_B_}RU+1)cJCe)#!Ft5;XaD=S$hj;Fq`Szx;xL8emd`#q`2owVp zpbJrlg@J8Wy?$Mwzdzjb;77HQVUZE>8{;A(Bmb@YaFB~?M?`oSRO06g`T%+4>FiMG zJ0>nVJW><QY4jnDijQB zF#tJSe(=#}{&8J+oF+0QUK6g-sH0OhEzCE3VH!|UC|LY`{Cs`kH zsK}%nPSiwcpoR!&Xf8}0f&~-R;Da850{vC~Gd{jPzW!@g(@Vcn85kKC6BiQ}soDOZ zNuK@PW;KLVCIvQ?dL~>QvVQH_0DlkvnYF7|`G6H(dg8w}252G^VxuC$V-t=~Vsp*R zeJ2e1=a7ytEG5=)eW0(um;cPF)&BmgR;@w3$U4{>9uuPpi;j;A3y<9Jg6smL>Aj=? zNE*g{;E{4hI}-^}X3Rem8}est6??9wDe#6`!2$HYZwLgOBnt+NyIyJ}y^ zs*0EcZ^&BkhykAd4lID*s-S>XIZx0}_dOIA9TNfXk*J0Pi4G6HRxgjwj|6~a&8K16~Gf-4YNBJ!k#ipIO+TD)& z+csPwZAJNQ3rcUBP;SHL2~gk*M9puloFuH16NL40ybvtoPUmUt%@{!~M+;%HMu?Cj zg-98nJC26C2@|4awGboYqVaCHw_qV&UN0ob>jc+_a)P{8K>i}6$o|4c*-zLcqqf~7 zZ<5ytTjkZlHd(30Z9|=PC(GT87hXad9(1(>*Quvf0kYk8;g#S1>cx?6UR_t03}YZAnxjFEch-zBry6&OSh`qqs!RX)^THZXKN!4bdzrJM$y@~iXT$0l;j{|@z+ z+q=6_s!(<6ySolIwKcWvYiMg~Y;I_7XgJVZUzgSF;^|R;|KZkSUpvtyqD!$u*M8%{ z+uxs+tq;{5X=-RbjI(+}P3_TrseeFW`}2=(K63K-iH>IUx==;=WZE_>&wYP?{b8Os zYib(m>aQ(3I1W2M`$SvkiIb;}HFqdA8BC?&yQ5ai_{8kF{k1r$*Vi1rzNxnEhHDF- zA$9$+6DLocKxu&$9uT+IJ#5pO$EKzYH#8hYmFnQ3hJ7`M_EqP1fcng>$2yPmgxyU! z+E|6TbKGK^b2#6=s{YWC!!`R49;`cf;6Qa%4nMy$+d(~ac8DEw?vjnztgUZ)cHS{5 z>rOTtsI5csqNe7+{u{2{Q(5pDi@e96jyYwHSjqvVdeL@6>u)|f_u`4#12uJ^K8Wu3 z??bU5_m^Ne)4fo+QeWx zvEVRSTaS5C@Iwc0fH3oi_Ux`ISCu=VW>>fxTDn zE~~DDsO2TQOUueiE6Vb>-hKhMFRrF~v3>yMyMy~#>$@IBn5#iiRaU&axO`7ZS;4M@ z@AFcu?@)a`h#()!EsC0o%d5)vT#M>R`JUX8Uryum6!W!*Q3FyPFdSq7sSp@A-rNoQ_U@^!5G%%^y^> zVuh+)ESoB?tg0-_O|O0q&USJi3}>N2RiS99B(J)(xNJw!-F9Si`U3~|Uw0MiNDu(! zwenI-qPjTqG5$UJt>OED}yTMgadR1A~p3;{6r+xgzw^`P%7VNsoaT3BrWLk7U{pfeXAd68%gNy}vA8%lx40-XHS^lX z5Zdgoo-EBRDlaLmtST!Ki^RgI0u&wcv$nL4q4YcC`k$7<+^Xu*3e>?$U}JejC55O2 zWanhWH#m{B+Fn0goQwKUK{>F+`Ko+dQ7(%NI2Yz*MW0}F>-l%xO-u%>zYpOFXJ2NX| zS9WH`uGGv;iP?IFs~;2;pvYHN~_}bjC4hmtWbHmBXS>_O|@A^zB=s9tX9Q zS&W%qDk>~N)ki5_?YMK?JUTihf0UJ*gBn_9X7;wU^c^WtwV*yzURGRz5?!IYM1-R-<-Ix^pG@Wr-iC8kH)ba|%0w)VVVy;sVKYb9UvWrKV+MZ%@n2O5d59x()O@QE*8Ly9wlH$`=ZKd2ZsI?7VCU zo|dxFmH>AX*#GD8DOp&6%oNSfSzbaFg z4pGuTy@O&WB_}3pk~YO{j@Zp#&6q7zieV^HiMzy17t3clRAiv?ojbN+c`(c5#DvWo zHHnGg{{`c5737EuQe=vmb1XfkrHZLof09EhsKz)FlanLjlM^%O5jF>5A zxY8lKxD(d0W9!D`WY%t2@`;b$oSdSGRDLU?kXzARn-ep}**_GyZOf*V#Q0>Y24$&~ zgqY;8=x@Al;Ir?i6>Lwr<2U6BAHUN=!&dOp4!-99;M;H*2oDS}CSOgjCl~ ztc|p3Ly~8b4V5^|GCn?K{f6(D8Q542W2IEwsY_#hYC~LdN^;7E6jW|drAmnpkLdoC zku4ia7+|?E6~(sgTQ_e=+PonlaYJIV)7!;36UL9v-OGWulvH1*?R}v6;|^uBU>uQ1t>WoB^rOAvAEdK!rwEpp}!Pz zB_?c%PsZHhLR0S2F|s9z8^okZFvZ}co){I=`Vk`=7TuPVfI&7!$4A9#3ZG|WgN9KO zrNyD%wfRSkY#0rCOpJ?$d~vbKVb{LS$cFdviHXs%n}0dZ$d0kIw#bz48yMMYKff%p^}qV?H%>J+c6{UU4;k4E&)v0W zOI&!!+Q4-oYIVwWw+%3|Ip4Z9J6s*6(S)lbB7=i9QR@TO=RbhRru(0RaqC0kQYohRA`M(32eW-CQcn6uStxLiA{v<#3m-kMQb8aD_C{a468A(m&T(`6dt`X zIyN>s0`(%)jlwZd#Jb?HmGum4AJ&D(s$xB2j3`&eYBXWNQQ_ecu}Q(|*wDyO&u=lX z{jp3F6)VQ7V#H`M$`qlFj#?WNA0CVnKxBv}LLISI{SxDv_3rHnp>awvP8F?RuZi50 zx@lXaCR&X`O-yi1a73`iyV$@}+k5qKQCOQ8QRz{qi3qvox&D!lZ;FYE2#$%2R7Yuo z)$L#F* zjQmDyWN1W`MvWR_s5;nh1;2xI24xSFR^y-xl-Z(Ci}Sx4H~3>#>CuQtlmH{bLqfu# z-L-*n{Lsy8yjxx|YEJ1oQjFB9xBbK^&&+?eRTCT*j>QN?r79#Sz-OiMnf+Mk!%Z6{ zMh&=YeIeQRzV_BncZUUsqCyfH0*gVN#NWs38yMkCq+8k1Q41B+$e?g6byUpy(C|og za7f5H^dGW5(9g%C8RQoh6d0v(%Y$N1=-TzDw5ru8$OVU~*JFU-wWz^Jp8R5}Hg2)o zg~DTaj5-LFJJia;RADys971wGU%zGblqT6d-!Hn4iH!;kULO{PDqC1+(0X++ihisK zDde+ZP$IM_YL;r1li`tJA?gTJrNbe;7&0Hu5|ql%8L*6BHgT;-QPoXMu1GOrBs?-S z4CSk zFu%8AieD^iDy2A!HclxbD)lEZ+{H4h(>;bo z9U6p!;F>jlsO_x{Shi-3>VDXdDioKxziV45W@>Y*yNIduIxKR=%}D0lh#`+9i$ ztX{6U0Qy;9MIMwIQQ``7sY5+OP3Sqm-LpSBU%A%TZ-w{$Jh4SUB};YCab>ossIkhr z#%GPM=ahvHbAxQNrts#g%GoY9BNl6 z*JW)0XjFa@ikG7QG?sLo7<7W)Mw`{ZsIdhgaHtL~)B;!fU|q|?-&@_s2eeK8ny8C>--hTY-{~}*Q{9SztVU0nj`#CpCN1>Dm)|v2$&#> zjJ~T@tnl++5x6qvB}BI2$WTho%|g6ifPZixmc-Y0RqT%#*tF}_>JZdKS(Q}Kem?8H zSNM8Hw+}O}1&4(K=i^Ti{Q?90VJ3bnJS+apxE7`k@B`f6$A1M->jDE-`>afR0&z`T zYw>5n5=I`f#y@a%ppRGR*ZL6+ELep=>=kIm$JZMM;ZyrA1Dl`dZ}Y?I_^(^J%4_AS zoxfpV^T#If@x>ynS+#n(`Yr}GF#7lfVD8@DzIA69*!-1MUA@YGMbV!a*zg|HT)8Uf zK|KST|9aomt385l8e(7zSRJ&&tKuaFHg6ws(=Qm)hd`R}HF|#* zKc7ad!H+yNK^OSpoVzh?{19585H1G@8q_+YQRH-4A;>CJM7{B?{S`uz^57c|UZ{#L z|G)jX*;b&7l_;!wqq|i?9LlFLsHR5Cz8K9PCD}lfX4eU!{MO1kAp~{U^^k5I%Cl=B zYyc{@esUBFy%;?THQ!j&f8#No1k{6*P!~=?jd&wIaJUIIWAuO{Dn7~ihU}V^Gv#^X z$nY?4^|T|yBRYdY-j1&;>vVdZsGHXtj3(U6*=NTL`jKJ1#b`8{Rpw3doPKz4SZ5TC zQ|2)f4o()c1s82o&XIw^5xv=DG>hh0i`8TSMzkc#;-F(#tJ50{qd<+SMqOr$rzK9F z85`~+*r+m2S>SWy(4+YBrfIqgI%YT{k$S zH|UIJgNbs9mT9xaY8|y$MlH)MVe*6vLJw<=MuW*>T%eG^DvpX47fC|#NxqR0oldJa zm@Sajyu$1vngW=YnSO~m3cdzt5P zIAkv_i)ooDP@Wna85|g-CP62fW}qv~zzRaqW$zxvRF@k!L2N}>nPydisZ^S?@qe(Ow ztQK|)(fAi#Sj`mIWb`yTF|Ntegiiq>VHxSW*ss;=bifd88dD^)kXm5_zrtoT!490} z(J?4>xojJ}&@(tZJfwqoz^hF9aq@9OFfk4X&2hRE(qXoa;~nv`r+miPdtsojZ_qs` z2!I$jV|k2bEHv~2(gkJ%m}fC0nQ4UhVE22@^Pl(k_6-h?XmvXFXoP`|nZRx^j#^Cy zMa7Hkz~g^K6xjGOdAjFZZ*SiK)Gx@X)r=3S8|I6(Kti1;qWS>mu$Y! z)6+jVg!Rtee3A$&3~w$uR9@vU%)$Pj7$!z|hEu zmZT6{VHQpOMy#m9ZNcjTH#f~Fxv4ZSd1CO=#oqqD!I7a6Y&R1YS;E428YQrb#tHOi zzKmBi&YG=CkD_sfJlcPu2MQk=92(J^Oa{GP;lzGYlmj#8;1E{X>2N4|MQ8(Z0B!&I z-oCzpfx#iI(WKXr5ky7*?+5hn%_*Ix3yP2&V_l zLp`JL20FdPpcnO`!LIla{DHb~I5GB|>j&eYUN<~~xxjN{GEj`lIBh{ge{$aIg#mnS9vG?7 zK$nBlS>MHT7yA0K#fJ2HY%^?B&QH;`!36mbGd%QjbWR4HO6Mm}&i0+Z)Qg=lpwo?D zYpL|2$)>E0PH%>YpcnOax-&=#nYBods2_w1tnjH=BFxgH1&7W&0$HZDSUoLP0uv3C`}@!K4E79y zes~yRhK2&4Vb*Ce0H;l=DbOQ?VL0R)0~gNHJ?IC9xJ#8;>#&kqvjO^Eq&Iy*KRkQ> zTz}ty9$N$>XbsR3LWezHpT9wv5 zf}R&UM}9bb@j{j>qb8B)%ViDFfSmj)}qrIF#e(# zG;7kXmkno0+5>MgtV3D>L$v5epaLxFG?fImO5;-!lo=){hT)+hCMd&nAy5<&lo>77 zO&pnL`oIKb#)@q)gmu88sEkNZ5V(i5%HFoZ?imM^1Z5ce4D#vWK4xKt+|{B93Ce(D z2nvU70flIb21}COE&HRWIMla($_YJctlVO}0t2 zLLZi31VT^}B(Qvim)65j!3KW9@dqa2L}YGK}&B0~^I=HLeuFJ>53TmpPN#=D$b z)QS2TnvI(sU;R4I6k$Ldm>k0PH*&Xg`l4QC7=YTqZNSlhNfQ2C93b%;hi#1i)O?X{k=e9v4rg{y`PX;a^1oRFJGsyt6sFScXo_Yza zqJ9GXF@ad*#cnoQlpaMrQUyehe(*9%8Q_Ps3MZ^tQ4eAU2ZywfNJMZOMUsoRI_4(~ z-93E_-te3wWK`Tb>>Q-ndpfvHH!rQ#I7)qCX3S4!&VSz1hd7Kqic^Zh>Y-B^$FNdH zvle`Mb_O@c7=ieLd4du7iSGPGde2@&E_eqHsnYdh6n1Phj~Ee87C7Nj>Ep4?{3)EV zzVjC@^nwo#cUboYAL1Kg>t%chkQ5zfLGU3z>HYjt4-*aD$Pir>!hn+c(s~eDO(Te9 z2)ZJp4W@@bP#5!)OP}{$ggt3Ta7Z(Pjgx_5RQhQ|Oq2FXHs}o#XMSQmcj4ltOT9xl zHEF?y_>ay_qRxRSGWGxnh>Kt(CPwJTi7>41Po}7n*`v-LW14vMYm06EqA%~5+VJK&T-tYze@XWc-`+ECzNE$H$;w}vx z@x!2pAwe9wBF;khNKdrqE?w-?!PYncQ-k3VgC1*%Z8@wOwhy7_#m<%VB-q1T@M`$mYSKu$gpW-7>P4iND+fpONJ%!0^`DhDe1`^(-V4crYAU^kB&`DFg=+=tQ`}_=Eo;&lT1(M?4vm4 zSSDqPulFFebzBO=>SG_AU%Qcn?|i; zV^cs)sis_Zho=MSiQS~dxq58eCZd}Oo84iz*&N_?f|2-dD-0degsFbL2Tz`o3m6a^WXX%anhM0C4A;W6c9Ung5!x)GxVoY24| zk$CK0_W4P>G6uw5Hi-;nk{gSf#1zEI8R!ZVmJGi8HXZNs6jw$F@w8Mvu^y%zlW9S!Jb6|+edF+ckHc&6}AVV1$95GtQ zZ5YU9n-nJv(++lvx$-Z%n8xCAC{LRc<4$@`;x`wNp%{k;b=L8*31EoE6!?mTv|&%4XXb$!OwUqJz%xS`9qL2a zHDS;(LW!8QV|i@$Y1pSSU#ODI6JVajkR%gr_$>|8Po&;U4DGCfjj~7E#Dr}I#*6=_ zrojQW$whYH@jn-4(x$R4lV{*U=~q$jnHyJl#R+M0(i_i(Dh3!pNj|r81jBaLX9DFu*HqL!9g_%N^ zWJG#m?i*5aT+{f3)y=1x=*K8jnSE-^ZeQY@7>5e+2byMjGBPkSf~?;(2EGZ7=jL;e zZ~8Jm+XS7>#28nI>B&$JTs{)`iLp_-sj&&{XD%<)GX-ZbHtv{!0~9ChiVMLXsEg^z zU?1Fy-ZVaD8O2=SxG@JSs^u*4@OJ^7zvzW)hHm-E;5TQy;4TAtk ze@n0v>jV8b$0sJ4o|rB$D`#z7XSKmcVMWF;k#SfdD4+-%T!sfi0+$3x8}gHxAsmm8 z7@=nCoEWzvri_k{O=9+w&y;Y7#7?f5vN1~w`us3?145&|M( zc#O}{IZceI#*m=Q4fi7J$1r1KqgK(X8W-UdNk2Ajhm4>X$L(}y5EDXMnV^g^+eX4? zRapAR5Je`TH;6W7AGbOXtT;g|23f45Oi&CL25=TM8mtCX$6*{W8Ys{1%4#1)&x@TSLDBXMXsy&LPQcKD6*GVmZIEk;80vvoN`lfmVnsg3GvXMG z10mHhW}U$Ji(=5ONs9!fhooB8-Nq)yp%{x}d=x6cqE1su_@XDR4+#oBq>mqwzz6SD za+>>xG4LZJ(fs|KErRr+6}tyN;!^~RN#=c50IrL0-?a|6M^dyE14WTXE#ue5qme7% zH_PwF?Odo3i*En`62~tA+>I{*;76jZ$X@UZU?0jUxUfQ38+kkSAu=EQ_C~b46*pTL zW;^c3@M~clBNad1l7Sy}0ckp4qVa+s&S^p#1-upb<@lli$?-EjMfkD+I`|MBcndqw z<#y%E1^Bs|B7BQXxrW<}yRt19db9E^1pJiQhq&V7ml6Vm4ayf3_(g@a1Sdl3B;4pF zDqmyZ*BQPp&%0)gIt=2RMH(p%&$Gl0*?uoX7|94T>>zKFr(|M{VAevH0R?@vu5!q0sK6i`6oooM0x( zC6>pihgBT)#1jms!TcV_F_m&x3z##yH4m;Hb*#HR>hX zT^t7oDcq4@hLgNyvtY|H;T;WtxLy{OY;|c-r^h)7CX8`7lxS92W*~^tS-_~1O|Q`j z98egRP>Db|R_@yn+n_Qi83V2xok!OsbmCBoT8)I#l7|IqC8?K1g5Jb2+~0Xve3nSZ zl?oT#grWqcx6x%7oFZDy7B9>4(K%poJGf#LB|Hc==O)EvAT$k9d=~;}v&Gko+dr5)Zz~^(X!mUKnVC{gmZkut3;9ahGri# ztleT>Wu9S?l$Vk!vqS0C-S_eZDqg*!ylOHp$Fw}mHid*npbQDgp(FDevt;7MF)N|o zkaiizRYY>XD3*?u_Q5`4Arn`rbG*zP<@Q1C)GZZy6)h5IRpxo{S;iw8P^7$$`d3dO}MvL9UKEmC5KDZ86fH2y_V=SgC0i_fnWXI9Hhc*f%z) zXq;1oLNOJcL4FpOag}WHf+PVZbVV9~lo3$~Q!ZPT5;W(9>t-_f8F^97iY$aznLSL3 zR})!iMY)wlW(@=!xZ|LYf{R-0 z_zPF>Sj- zO;3gqeLxH+WOT8DtTLfI&igUk2=LO8_h@dhOh#W2Qq)=Y3$zGY&d`GQAY~!jjUdAs zg^W|;6!B+XnV**d=0RFq+p?&N-cg~2B25Ng!xTs3#UANBjLQuZD2n49)xz68RGi(t z_@JhXDS*_A1$g{L0)g`uhi4fblV;_TzSyt^sSYp3K&eJUS z3-`>T>2e4LAnz3?ak&Y8RG%Or{Vm!m{R~E6(Get07D81hRO21x0NzVar2;pKG=!ye zR&$m4Da=qK?=+O|MZG7=_M)B_4+i}TyK#DFS=xxsEK*8_A}G8qh0jD&9_#5O`D$MLn)Y z{*4WzfKAldpiNGTFQDgb{5(l@o?w#{kvw$#RKq1H-0_R1IX4$R4M0iyS}$7(pALNP!wb61YfG#Inde=Dyxz&Yk<7o!!~#=lh#o ziK(sS%scP%JKx{?JkRerzQ1j?BYOVCQMfnD3#B_~f1S$rL z#8ySD7@J$uqF2Z5`~rVFIgEP4Oq{&{PpxlOEnVYBh_fT6od6hjC`Kiu*WUps<6`Pm6!oW$f=1E03oBUAoZI$ z*=W^m9x}Z`7okSGqS4!xBa-`uZd(okTtcvffV{9q@Jqby#$V~{;)m1?W9O0BH4`yW zC#Ds+MK3Xk&TPzl0 zMyk)0=k{(Z{%ZI3mhq?>dAo>Q-v`NpWV<@6P%gTo5mTZxOQriix<5lD=pxiD5vH39 z6R91LzEIk&qfejdrY3RPPe<`#pq1T}b9ml#fNiyr_GD;3t*lt6frX%3hIKqjw3y0G^99B7uCWlO&1Xls2%)x4-t}GL~_d;^&|)l zt5mXI?=~gpN=s5V;K~SK(V4sCT*bWY9~y(%qH> zw@ZPdak3*Ok%30?n&2zAy!4~OAq2E|x6z(pBHo4n6%NG< z51ZPn_#8C=sNfq1D9}>@dEPX#S=57F;xbdy zG1SsKXmL@3s>BE-y6oZ{U1J(*l#?1Ffo{50EF6STT}2`qhH62p5faIT#~T()b>@Z& z957@Wk|Gl(GLH*n9${C|;$)|Bqb>yUQepB3^F6gDC#>iLh1q;_BCXXvtgwW*MsiZi zVjcqlZRV&Ct&V>goYV-_$hWuO#8r84xUm@QYb2)U0qw{m^?Z~s9tTA`Sg zZ84h~mOh}D#;ATh7bj^ihE{yn9x{T@>qWnQI zBPTDCRgT!*BC;z{!(5gy4Ye*eGp+bN4V2IU{Hp<puHqqX@U8kZTvxX42P^Y$$_VfhpiBHq^Os*>{x1>D(;I%c8vbqh8mEG|fvb3Qf z`vCQdYMEM=PypR{i>@05Ol7Hn2dNTPHo)yI6OUmPQJz(b$PFz8Z5q^ttvu?DDL^5y z6mHJ)M)7dB<)SySV)Sc<2?*lymCM^r$8m;JlgxZtGB&rSMX#RQ*#-V~WIfyT)3F3y zv+{kjZt0w7dr*5LrkD*9V?%EodUQI=%vDYAH7PWcikrdMG)&k+)?e^96~N1n0pId8 zgOq4`Q1Z?5h$o3Bj50Z_Zb+HLv|%IxLCjJnWqz`1+nCmI5?U>k$&N8WOuUAh-NELd z-Ocf z4&P3oYjuMBp-f`Zmw6qxp&YRLdRBi(wf)WhbRXz#lp*-LoER#OTLic6LT{&!UwG6X zK!Y}^Jtq7Wmlobe`M*2eK=;6hM7`-AR!MdZuwHE7^pOAe1_QOyJKVGTGd(GjMkT;l zS?e&wHXRst`-W)HTkZw=GhKV^?!ex0_)&{|P-?2z>-V9<8BAr%y={G+=`}ZU1Y+TT z#gi(xD^6@YKK(%7qPZizEh-jSuVkEq_>#M}yyf_3p{MhsUVmVbozBP>e|+XG6^bZ2`*{oe1*^{lqtvAe?wDKkDHr6F#2x(B;9U8VoC-2l&m5CD7c zmjDlrq1e5p?o4lP?>5hCQXtUXGOlmqvtR56d;1_+=$gs`-2r1Y9)GK+Cp=4~`#-ur zLnR}o%*Nwyo0|*r{M%4WL%mx^pFY!1GiwvNEmg^yn%$!gP~szMp4A@pzRrYpp0Niy z^SIV0_vDc`D?^XM#%azKuJlg#5M)+&wzEsf@AnW6bs=xNC3k#hvBP@rb{0AVf)n^% zkpOxQdqrktrxJMUS7Hb}t z=b@I$1gL`U*$hFg-a~5Zo~FCC+%f}zLYz{hxDMqsmAiGf;*Y4Qx7ok}ii=dS36zI) zrd2QweS!lN4kHgKK2)f11syw- zHG0J~$n(y3iXXrWDt|M?re=wedkZvFfZB|sQd4xWxe2GFw&pAyGu}%JPc`0iY7paA z??!yL8v|NEE*bZ_Joa`|^I-bH{h5X)73XIaw>l1TxzQ)%GliK(eAK$n{5OIGh**s+ z4Yo1YFprK3aiEG|I4LN~0N#8i&(v!-?M8rdn^dfFvk;mJ>;~eZi3@hXTyRr$&{0}L zSeg#cH1r(lDFdY#Q)K8(F zk?B&Pu3)9zNl}@KA_X8px>3YDwMKoeu6JR?uXcm7)H7tRp_r+caBS+~d{~4L$^AT6 zCqzW8DYY6CYMDNXt8b?S(8$< zZEE0As)SV!a2u#GUQVk$oRlKYMvDPvv7Yw_9|!+?=#ZR0L$ttI!TS9O zSU=$W-Ury#;P+huzYp*K_xJyJJoQq2T&k0_()hRE{*~{6!1t|R6Lgt@3#$Nh_Kh!p z&xJrfl+k}G^W7o-IuQmGwh{jzxH>;gp~*%%@-OUy!nsc1;+2Y-+uc&133S# z^=|X)Z*!6X&|#pM|NQ6PxMkg{-vqkpwbyRne)aZ^moI((&vaB0DG;c#jrebT=^wrQ zz3+VM*WUfcd+!-^FzcJ{J0N%oa9xOIfJwdn>Z`AV2=&VKn=f7d^uO2HzFFf?-55mw z>6gFr?eF~RyTAPQE=1cqz}#D3Edam;WIde)hw9avZvd=y1SU0&mE#QYK=;zg^#*e9{w3F4PQO?&nBYgga>TOvZ{YF05h{_|h@5?FP>u>m~y`J11+3C7qP z%*5@R*ZFq)^7TL1Ary!$$xJm{&NasW>&-8I^Xp%K`^~rB1{MFcS8l&?{nqQR-gxc0 zb$$58i_iVGLx{;-HHF==Gy1bHy!Gx^-}%N@zjXWN?a#gO#`V{3-FofSo40=cv(LTv z(>|d*bJfg7vydHj|K#RZzxkE7zWl|{LvrIfRbGD;psr_c{;6`<$=02zy0v<-k?(E( z@Qt^=2C(1t>%jjr39mD%&%F3M0%@76`t=GKLd|~vcW>VK#+SeFDnMfZfPL=vOD{eD zy$HG8^VK{qe^PC$oB!_m7k-hM1TgHz?OQLt^b0~F!xKMK$%~K5khj_TH`jmhI&*dl z7`w~2|3Da(OVviH)%<5K+yFT5#*OE%{Q5c}GfS0Lw$Z8n>QirAfBom4{ig89mMZN! z;dsfnpL^xXjUV=j(p{>MpPDPxW4B-Utqcj}OO<{%U-xs3JO7(kaETa(a;}_B?U2>J zRN369E4hoPo2Bw5T&h&?_&!&eD-W@0Ih$M@DsL5uSSHd%-%4(tZwD|FZxsufZNBf< zcdR5XFZKc+X53nZyjL_=vIDb8b;y*T!)$CWvpX4c*(D2s91^@vDZ+f_Hpp9%b8#c@ zWVxQ-=wVpJz?Dnxn9Dx)PAo>AOC-)*sb40Bg~00+p+Q_3KbfCa0>19aI_IWG<_gPV zc9ij8H}WoRZj`;Emt^BRja8AYzn@aR*EH75A=aW)iU^=-d5#KOIX_k4%2-ba*rW2< zM3&GaIf`_d^PCjSOLkQ(p=`h1)F#)mLdg7uYbi;9QCHThV%~MU7<=!oF_N*_Pt>@= zW2SoCOa>rR3<20o^2s!@N_=PQo=(*HZsmu5!3}4LWZJ}d;-xZ#HVt!bE*=$_wwK8y zihS8}+*p?2@om+1b!=sxACKY#u(vs^_WM(RIj)ZjwW9C1?_T}vYoGb-i!Z#qi-Bc5 zpL+Hx=vSAYyTaeIpS&n|AweyjJ$v%ZnNz3EoqFV#{u-9L<6#S_Zq$Bq^_Aye{ka!j z{_OJ;ynLXd(wDCQK@R-WbI*d)M0v22&R;ll>cWMSPo6#b@Rc8mrCMa_5F5Gj&ZQey zUi!>)FN4fJ0r#@5)xpmPt_%343ul2J0&&PXEm)z)&pdhRku$%kGQ`P`-;y0`e|jO&NjT(`9HsK z_2rkZy>RWt=M93_l`EgpxSzgY%AE(p3-q8fr_P^$;@FEnf}@g$3R`Iu>w~|z^)#4s zz{u~R)q^5`=@aLfB%#lPqI4S6wR4|%?4gVQrT|N|>exy0HKIAYfBx#TuU_I{2J33g z%+Hli2@&nwlK|-pJ*s#1hh;QdG5lgCoVjB=KN`}uufU0hUbny`Y+eTQOhZk zuf@&6h|#*ip>p@iT>kLm#zVv258nP zV6mw3%;|HFKk>k){_98_RnatV0W7>iz5E}Zy!71Dm!3X<0i=3igPl8j^0Cvu>4>A2 z{Sw@#h;H?pPo01M(#4Y(&zuB!>LRF2$KU)rIO-fpOl1@CRw@4TPdxR>^E@xQbK(5) z6QBNXVyR^uX~SxXrGm$$z$HMg>dV*vykN?-#^UOPhq7X5oymR02SVI4EWYVyUGq5>{NX zRDE-VmURGrge5j@SSsJQFts{jqH;}Rq8_;gb7ed#yF3&xjJfo8(qgH_F6m6HK2mIw zVyVThGU7<-$!2`#zN9GFm z27c+}wzWxy4SgHaLN?WA-B*aBAd@6#_O%!&ohB-fvD;u6F18Q!|_mX+>RJl9` zvS)g6Upy6isV;p2ZpBmKnbZWg%OnQksWQ-V+w+{bc&ZY4@_5M|@l@D07HwR~qqt$J z#@?R=A3#hM`!AN}L<&ZVjT_7Bb_y!EYIf-6oRDBjHnW~fCaSoqpYfv+aaBsk3*xG2 zC+dr!3|Dn_xoW3FK1=@d%csvD7bZTaR1@5#)KLJG?iYj={{aC7^7i4w zM}TPGe_&yL{@~*5=YK3kwa_YWw735D?BkD|KJl@W;CBmQOOWR2`yT}LN8mpX1E~Y* z5r8@O9X)tpd3hh8OfzSG$LOieZl%)M4uA2X(>rE^2lSyk3Ie&;t`lw*24wBk%XHg#Ga$T z<;?Cobl~vp{#$=3uG()8+Wldt{TEL^c=9BGX(x6YKK8&vAA9WB{fAAtWk7un-FImJ zzNPyP&VTZU(o?Z$^t!`(XZycjedzf8pkCbrvh>i=LxL3oFR3N`o~60PBMbA7{fo5p zR1AVNi6MID&o4dr*z$2rk!K*B9(_=a9Xbpgk>K)|!IWB@KJ?Dtz*DWx>Hr&VceuSX z`ooj+#~yxS0_gPc!}l*ATv`T>6}14kd@w1Y#3?>lmsC$qe)xjM3M?&|-{!BXdY4(5Y# z>jy`V9DC>ipq8LmI!J}UG0o2hPX3o|anvpiyZu3%Qyj|w3RJa+9(fRaDUDwMTnn?y zzZMfm?fL!gpxfW>SN{n!_Q=tFO9%IZcy#2x`Gtj7ek_jK%h4wiy*k6%_a0onPX`Db zWSK3`&0qXIanv66b`Gz;;21qt5y2)@_mc*`xh63 z2fp%Caa3$so2}%Hz>$Te{nJqjt$&X*Wt0an#P{u+<`vOdJ)P?Pd?NS#D_V413LX zr2r>g>hP5SUNKedeOt|1cEtDn`rc|(@?xWy9Iavf42%}6nC#Kr!ca+9Zr%~9bUN{UV zD65kz41IO>)H~)1yF-=G0x{Lrbdv+(0PK++YhN&v~pvh*SY!p=)dzwU? zc!sAM!JyKzwd5aah1%6N`UZ#~vRc{{8_H3LXOK>XE7AKv_Ksl+`i8a6Jm()d>Kvp5Tar zQ&UG(SRZH1A8*zBQ>Ved(m@7i!N8)fGkoZ5gL9m5pmPq+1B(S5);T_&1k#K0=lQq* zP!_d6$;U;`BBk5~K0b~`L{^bcfSsi?6Q1JiginH@^|T9;GZ_NotDMcCGa8>{9|;v`ik#IG6`@wvh3%4;ddxFl7A7aJrFlYoh2QcLmH}{n~0xvmuq{$4-s&89oV_dvEm6>$gNCh6LzH(dRbpDV^H4^N`DK+IpC>@Id#*=nQ@>NQDcCTY(J!eJ4 zaW@H|1ISB;$SryFo-T8fBzO6c=_LNnb~qGMr&5knaE+U|seJDbX|m9?{LmVxBurX5 z=g5~#A_$rbF6e^E3c-5}Z>+d?qaMM9sWeyvG=^_kfC>EqzFW-&%+gD&Bmn~gF2g0O z9PJJY#<)jtVq9AfrIN7(`Q4d3z2v^U$7G36gOz=JoG${J158gklVs;eFK{e-jKWb;y`cxdqIyt5jM#=srO zcvNkQs$rf2yt^5W=w2f3Qj{sT+yNQT3JKt5wY0JpO|h!TxZ@`Nb1fx;>R{(ZXDyTx zqCtXGX|g-WsM4B{qFha`CKBN|*imHH=;1(MduK^HiA6K%6j~yRv*Urj0Ctk=!QZRf z(To#M3r>PZ#=7*>lk?+pZGAl+VdB_pXVaNH8G~fLpC4CvaPd?m>Ski8L^|iIYjE>w z>P{pTi#Wcp9nf-p-~750U5&@1NjD8L4~tHZ?83OU8Rn9Nen(mTzU_Z}TraGJqtUd( z+M;XZ$ivSYnf0}Bif0?6N)pfE)TFhce zKBx==K=9mvOS}s85U`@4i=IAsV|eheq}XaA#(Tt*5bzKWPOO9XQjm8nYT9tcl5$J} zlbO!gnIdahWbQp=Dk7HTuZNN`8U=EUOr{+17#~up#=P%{CB@g1kqG!3u!;=m47^)= z9rMHyAQDUB`SY+rKgbkVrHo4Lxvm02%#wp8h1R3O)!?D$GQ9tmM?~>GG@e+Jx4xQ4 zr$9MK`e`wVEnWtfmzi8E%iR)N5KD6IMBH?gh3aP1nCtU8TV9`+l!5Rrz>?y3nNYY2 z6U7u63dXoc083h1PXN0`pHn2F>pl$(!Qh|<%Y!8qqS1I*&|z`5UFMYp--cI4w_!=y zNHoNllTi;)ExoWnQt7SXE-c9pMMA*Z#8SX#`TVMv8l;$Q9-3HEW+e(g13t>3YEx7V z^AyCALhz(y3SL1`rrgR5d^$zGs#sF-?rJ2-+w8<$Zt@0dDUE>kfF)&Cc~YS`Ym@#k z|41xxSh1wUN<5B0KsYi~9u5Sy14|l2Bg{7S`yPux%!dc4U43;sl7_noWQKLbx^&bN zv81)NXebeld&!uWgqgwf9kC=HTr|lmEzXeiNyv3FVoAvl!pUej<0g2jY?ocv{K`8R zi$xNdWSpl1Tndj2mNW<@DCPU4^TN7pS1f5Y#M%SMB~I7UnT%LcdTlk7NJS`5*HdY+ zq{M0{o(Sm|X#nCt>mMW%>k;?}V#EB9yBkhN*20l=47d%kB*a2;bv=}p>=sLkuf@Y$ z0y&rxOG+4``eSKwp zRV=9xTUmn>!IEQ#UIwgGEd@cy;>Jb%I6%gR!Ht=M^`b z(u8sPRYy20ZM>WK+)%QKu!kiTGrn+k2muF$R6a#KaU?d|)t)}^3Xug zRAaMN7E5yC1liMLs))prcri)X&gV8ZT4j2#BvR$Yff-6L7fIc) zq*Rh5eAG%zsT)ejR9f*pG)iY}72`zBp%e<2N_m!B{D#gg)&6lt$0 zo8MV_-LSE@xhX+;b;by0yYvU zmgJ2wIJ-6?n;({xi^j8=6p@!@`osLQyOi5WEXj>HUdqjFFy*Ay@Jfq6V&BvdQnNyE6(L~1pBKcoQL z6?jq!H7!#RiVLffVmr7-YKqN+#jCko3I}pAO5m%6jCsk0kytH`I#%dUwCOj#FiY8_R*qq@>$*_1* zarGXaq)hcEI2rGxNs8Agc@HZm{Wx3M6KrLVlR5vGIZN+RWw;+RN9!GDcdKLePKfd$ zbL!p`>~c?n>8!)}PU+{P=4?LgbSK+eyzV--&!p&|SEGm6(Jq?<{m9?{0d}y9^2{%? zjXh|#v6C$DD&I`|I_(VEEZVt17UV`fX_%Fr3-Qts25yq0*b1;h0ii;> zPb9`*l5kBdxk7kVI!IaeoUjuUfo&1-o8Y^)3EbuFRuD65IUd{GD{R7|+03fh0Kv-& zLkBF%?CK2Cmzv<6p$19zOcGBUN3`+-zfx{w{v}H+NO7tG4T$jcnd~-OV=1IWm9sH| zS$e5Rc5IKW9I?9s^qLf>pxW(1#F}S!ogp0|T#vmVI(>D>w z&;+s?Goj>a&d!2r$xKobuq~QJFG?mHo<-GkS36;8(V7jB*g{Ry;FKk%Qs~9nd8@Sa zUrgiiG;lRh&7G8CJOrk+xs+L6C1Fp(4TA!el34ahaqv^qsYFI-d<0$3WOvl(Hsn&i zO-y%oTEKu<2eN@|led<~Lm;)2j6`*+-<)qs3{Z(_Kr^w0n(p~QU%mt;*5@y8djm1L zR9N_mbAFv|0_r5&Gyb8kzBl6qfX(0wPhNdw^B;=}GH=HtP`W{9wE;jo7rE!p`Zids zng^6knnQ8q9S4bKLe6Z_1?Ep*Aei?}xOiJrM+L-y#{`Zy=WlCLiC5R!RIjC}+n(uj zPMLZvoMG0+t=KTi5w9aJEFf}ocS~(*gN-*CA@D(MOZYMyD*~xmdo|UUWk78if zpz)93b|y|*xwTk%`dA1+8cQazg9m-vtFe>7M58?y%iUxG*x7*J;E5VcGVyUg>F~}2 zfj$|Z*`Vo7;!mUV{As_zjIdc^D7atZ;WVs*Sas!dT5|A^_sJt&9g9%v%?6m6NubrcM$Me=Bp>0Ng|qtEdgmd<9C5~U5l7HCl^eWd|)@n+)5l3rI!_K#`4uIB;TlwM1;wEofO@x?~Q3nlw8H+DV{}Z$mo-T8_!J@5*q9indKYl_ zbezW;)geY!ioLGphD};9(_@nZOITmYB7D2-d3jc5>ql?W19sifyAc_>dhrDBpl;Dm zD{ZF+Np`sF$dOayQViU~$vrJzj1sX#XeEsvwt?p2YUhWAjGa)DP1N1h9NJcyw>cww zRGR%?JcqKG0cBSOG5CTQVbZ8iR-_S40nd$1GMRKR*eKMc9!VrHbZP6B zN=ELk<#8S~qOmAeGt3igpjRR#v?pu5vlfX*Lg8>|jR}2fY=ydOk%VkQku~krI)5L& z8&0svy0czGE$XH3tWo&RN*rZppk3ZNL%~zJ9k~-u=H?^{0}*Lct1 zHpULZ;?mNy{eTC$A@)pCm$?eTHh3ploZJ=z!1~IrJr$3S$EGw}00{-%4pJ5yjTSqW zMDxm%EJI-}X)yuucb;uDI@l{Lo@2xrMFTMKO|!ZuWzi%hKg5Go1Mi;0eESmN{2*qB z72EtvZcdQmNJVd(w*$$stp#quwP97faV*kg*p~>0rD9>t<|f4{sCK&$!OGYn$g*U) zs0ii!8V@8CV?#gn6 zSkhPwi7n_!4USje$Rsg57@A6pA<$|xDG{!@qab#tNojK_X`_;uNoZX~d7qJ8lj3^U zuter1!d>QLM}2NX4jDaAMbjRt2ONb#G~uH>9zu}PiEup`t{%TW6bPcq1lUHQ_olPQb6}3r}8s1YI}EJBbK);Mom23uA&s z8*;~;MFy}YA><_&Um>49>`H_ia%M{$*v!11>AZ_*xBWJQQ3b3@co!nv-PWYqzJVX-y#h)5dVBi@aPn6@6O*BnZJE(A`hbHQ zjv5?U2@OalPqV`S;J&=cuwW|_a9FyU@>J4FZpR}jiSR6ss0-(p$YN<|TAUFK+jf{* z*;$rjbo0-X23?6-)E@s&GsVDr{ zgPUg((Fo_uLFOZE0!{-076Y79aK6+RfVEpmD@`B6lHgeivsNtT4VkAWrE4j$xHU2f z_TT6Z>0~$~3C?2?u`N2r%0HszNBj+1<+KyW>tQ6g{3v2DL0&Ev2G1JN3pavH%5fwa zzMGT;XYB;sqGp4cp)mL$+dD@hIkXMEIma$n|QAo$5sm_#NylSQ*JcWYQ7g z%xVU;tBE9%JTl|q!%BDbGDLBp+zI6+!qco)2{bz$KrDNiV5WnNM@&W5R*5=9gyWY| zRdz~(5@E#*rR>xwor&A=ZA@|!;d-)|!?|Ong6Sq}CaGO~JQ7)7Ga_7*BZ9efi|4?4 zipLX?wU8?jo=%y2sr0^d56&D*L|3CmgkwFT5UVdp>s=7glL)tPBqaN=1^!@1QIo!XsgvfZRyNlZhx!lkKQVUutK z&P97*a~?$<_G0u7;upi!Fq^rS`C1S*QDkNx?BK& zy#SDK#Nn$X;2zARv19gV7EpcOdhNnZcV%hC0A^-G=0m%NQBSw9D-x0+iVpgPe6~(SeLhqzFL6dAMi%p4eMYr#$&uz$Mht-@` zfb1iCNO>cHY7-Db9*<68@RO;kM0mxN7+{U0fr)`uCR3o&lr;hx41aaI415ya^h8x6 zoJy!%*mVNpqY1^I1}0y4^6Deu22QFh5iT%2J$x*wq#_vdGFW-GzCnoLo92j1@RJ3J za6`^)$!aSzvw5cTpw+QN+nPFxl{fr!St7gwD5AnaJ6fCSwNBpB!CqPr!TOF;Z-^N! z#!3?57|f}FC#PqinwUosj(8}+dbh!G3k01b@+88mK+aj3|BAAKA*L^Sc}&g@NjLH$ z^fB(2t3cCBgo9I8!*-1eK$xD4SjK#oN}$4IX;nkaVKzlKD-jNchZ$c(#CntX)9A{e z=iseit~3-5Q*aZWM7SDP#+Fiyy55?hnun$;ErAj*}3EoJ1aPV=NTB{zc<@IjzsD-o(svwlazeEV zyKpf200@aiNR&t3jVI*!8J1YALZksE1^l5Z2zDJEqJ+wVVAYxMTb#$Gs_deUDllMh zI!g*r7Hq5AZZanct|u#mGd;cOa;wUDPmBZ|EFP8P7)ZR``@G350mK>55lpaoL^yH9 z<}S{J8DM&PSemkVA}&U3#L031S4~EG5Do2(E->L7_J{K0N{r{h^3n_m$tt)35DCr~ zCl=0YE6za>2K0?ZA^bwl6U=2oiIU_vM-WVIaYq9K=mJQE)KGHV^XVD}Tmx6YC!s)T zwtL8NW=-HMigIEmHA4|L$jEU#)TE3E*M(;##VVSvN{;7*dyI8iCksKlw40_N|NJQ6i7J|MrJGi{CM=< zl>7_tPwB&+x;hR_T_Kq6Iq+~Vk7vxEBI<;MtEjqX$imx;ta}RZTSeD>9OPR?*j)hf zmMFUuCd!VFLj>9p0k;g`Ezxg>!Mr`n36u8|b4S?QQQQ^}5_$*01IBalQ3M|GamP4v z@-bW$$ES!Z{SjwMa`NPRQz!6JoZO4Yd!Kl`N$}ky;tt019wBy!%vA#9el&$)V(R+`r^fgEO?`i%Grs3Ib+0xC{UQRB5-?0C)bjghxxOSaJ*KXXn^_ytN+qZAsqW?|z|BEhMzH;Rn-L<Fw%U7;G7P&`v z?QeZ62Di7I_mN+*qAv{GxfreAm3xTeyqKiE_r{8G8&=qEOlAL2vU$%Fm!oXt8Q=7T znfC!b?Z0RbxqS5+-37ou+4<)qAB$Z1Ti@-T)myA6gBbGb_g+o|_p2LIFRbs2GjeSj zaxKqUn!?m?uq)UI0HyLFY;FkHvcfSdY zUs4cP10B9)>KCSLAY$L3zj>>Mg_!@i5q73Q&6P!ATUmFLG9s5|(w-kDl90iQf-s2d z1k~s)P>$mwUO(t*LBgbMw7=G<3${nl>e-=U_qbTs^34ye*x^;BR|s#l+p@?$k!UkX zld<@1PT)vWc)Ufl@`$O+!{x;19x4vsT?|{}c-e$r{f`T!$}7L7pXJh>lZiqiEUy?? z*1BDgsjPYErkZ9+-54KLmarw6yP@-{*!gWU_}cpH^{E=mS^(g-t^Q~TC#qXeGfG^_aog^&}2=O>C;PKlW?l{`?7);f{F#e<4Kho z0wWGozds}}o-W%gd(xJ8n0ffz{=EZ93F32Hk2olZTI5%vc1<;XFg`i3Qi)#tQ=)lL zCplJ&WiZC8E*fDzc?C3hZKO2r&)6s^3z4D)wmS?ecJuGKwyU*b-nQQqSvazqiA$yH z53qIZxstuE;$Y}7`6Le~m55o7oQmLr>QaRI+D*w4 zpj8&#n7?*^0RU_Pgk^z1%b?RvP+9&VZ-5Q_2US@vrFTWShxa|NNKL@a!PuoSak$Q^ zx9T8jLm-snUj4@ChG?+q5ziQwETi^x)?Q2zQ5Qgj!q)$h0;+FoEn2ax=rHN#Z+)Ki zbrnuY`$YNnKPA=+p@$6vPtvF;o+cY#F`1Tl?EtoY$&MoIkg)6eU;&PJ9XB=Rx*YSt zEy;Bf>XPp8;U9ZtM#D)NC7_C#&t1 z>tq6k$G57w5~!lR)70-CddV!t^pbSCOONV;IlcUA$&4_Tavs(akJM6Y4E*BPUhNTn zc9YE8%#B*Q6|5km3b;oG-n*uw9z`k6M?YeR4fd(0-=bs`JN)s(uHr0pT(*Y!D|s5l zJGBte-0+siUu>{Zli@fd>Dukq&O{?A9Kf`$eJ=Pj6?9?b6zn(Kv}>|bGOZjmIM9}` zH5FadRKJ9S1^{oHih__OwSU6aQzy)R-16WkQE-4>&&T)mJwblwD^?{*Xo%j0TerK< z?iv4hQkk)&JUc!yR{etXKHs{a{-9v#m3BT@Fugix0{++|uGW8W+p>qJanQI$Oztw6dN|vg`R{pLw;NVGRwbE}mgxf>z}nlj%wlf90%J9}ko0^6 z^Le;|O!BHuo7yycgy}sAM^x4suo;X zwejp%r>;OBM4~p>o3uB=5|c}yV!2{W9J$>A_JSi`Cj=%1X?jLf-HotYRhq2(O)nOQbVSc2^MC@vf(0Q(k$ciX>b%fcS@xmAUsbzK zOV)v8I_doNj86k5qb0g3gjM+0*a@AC!V!AD%%>id3d?)zdCR`!iV{&hom7@(i|vpL zE^UJcV*;IBcD8wDZ7%6C2-H!#{wF<%TLVl_V&><}B=wW81JPH>vFRxGe?*T>PkzkV z$5Hcn+*1=*9}0vQ*~LW0)YkY1Js2?>nV{wv;&r4wj9H=fD<-g5&ku}q4YGKMcFkg^G#V6Z(^lU4io!V;a)lFOe!{usZGj6q9R%pGvLDR~r zEICLPQPud%j;<;)mo0qoy0+(@v9G1QEV0xo1p@0QNy?v36usNL&G z8;}x9;wH;e&H*eHSaK9#OX5yqzgZ<|V^uF4*J(m7y0Syx-GT|wU%)}`i{P!to=CXl zYZ-WzbSkjeC_CL|gbS9&ejl-Vm7(8X*>d$oIiIUBN}q9{w4;dXWVVH1pCtvxq4Vw5 zAL9QfRPomJZr!}sX3P`fXK`BY0K8LA9Z$* zt46vGMn61O$Qd~7GEU5rF2v-uQC&gh=Gu58|>Ng@H(I zX55K_aE+JC$&Urw)d%y@%bt0e9tjgdj45F|Tuf}Nw&wuB!Eu$t3psA7u3)E~HO>HD zt`cf_We_3Gy=fs8ZM4hezZKS9uH0N;xu5?}1pNPAE*JVbsGG?GUSzO{7h9rahm?+& zhZwRXT9O9{)DsjjA^VA#$%)qLC|fRW7g_tg5UJ{A|h265U; z%&z0Q)9-<^>Tt7XiPVfz2hGZx0y9slqSyqO`DM50HO{Q-8Q0YGv;11Ze|HX` zhVe+h?;_yrlUhrBLmS>L%lb7lelVH{KjLefJwoOr5kpUWVZl1mVxaW?s(1_Gt#d%a zgequ`d%%n+BfA+e$63yN4ru%vk3G$s4x!$wuG}qPcaJ1h;}6_J;WFl5XmYr~EHmBb zVdobTHt5}HOwgS~2^A)QMr~P;sHE@QE*i^RRRd(CeEUR95-ApXm^1ul=dl-9^S0^_ z8O6D-($ZdfM}hQ()GVzQ=xkNB+jq71w ztuK0_{`jBbr|XzoqI67v3GFFxr%ND>f|Pbv7aN1(dFhVu@Qfutv7F(Els1UgO1;4202@U6^y}G)zPjrC*PiKBGw?$$UjFmv0U5XrC?h$;MUEt083@}tvIIG(Z6CR6eM)p?~TN62){2Zkl%l> zETc_uTd9~uKu!A(*3Btu7EZVQ5Wx_&Iav9@!8o-`kg$X$CQKotl|7i!9A}kaqM*$Z zUG&4(llK>W`%HS0>y6wC{?d^6;L|-|I$S(tv&kwZ5u@Ag08~$#4xg7+DC&yy(R@8G zw8ADOX({7p(-(a>QFHj-NA?_$ggfJZ=ma5XZ@4e8Mc6Yf~3-P-aV#F_2JW0}tsYmIo#t@#|Vp z#}ZU0Y`I|{ZqzR{Y1kSatM|VL?g;24Ekn|o*C36s%?HV6kt#5r z#{Ry=JR`_PdIhmv4ySIMYZdKJYSP8{ZA4F>xc3?^mCBH_7@(WU=YW)?`l>U~WNmPY z`2&T@nu?eNR7yqD9Bg?N`zp3iGgxcdA207>D6@`X-APQ_2^8zFzZVl%ui+T%+LoNG zf1qRb$+(nhv&+4MoM*o5XAr^12{WQ)Fb$i|vdUDmuSgvG-@-R3RN0re@mI7h%u zItvd}qFl8*>CPoOyWYr<6rq_vJ<$DQz4>CwQUT61*+F z^*yHoX$ko@De4l42&(TlHFNVblqd!5*Wi`qTT@fyN5ob|r}k&wcg-0q);M(x1XYGS zO_FZO1Al1EHlJ7>Zy7Lhajf5}AzqdCx!>CDTQR!gKDTcZ7PzId3LX-|HN>iW+FvBy zbtZ6Cw@L30iLXytH$Mtu?_~=*Ef_~IRM$#yP2zgOuDs_3jU6GEb_QOnM(sR|shH9W z<$-+aJ+rsmhn&-lAP-lZ^Jw{n9{Fy7mkC(^M z3304YNHYiICsN15a=-(~I>ci(ONor4g!w;w`CLojwen$pfK8|*Y8JVKpe31#i;Gv3 zl`pFnR-k33dWWHhn@6&AZiOS460#yR#9Ywa9-xHCl2|pVnvRyCG6eAmD+7|yn_Ae% zLH)Llq)Vh_HV653OmMF9ST|0Tg`ZeDdR5s7agQjz%M#L4<$*dAR1`nG4ksyCzdFpz zS&;~(i%8s8$Bt3d3(H3>)XGgDItk8emgHh#C*CM-TTCkBYRgTlx;|1hWgd_-L6+$8 z&oLP=mL_588JvsX-rt3o*QzncXEtgsj+X4E-vK?*G#$0MRXmto;$kxEj}KsMqwLdH zO_F&(IA=BYVjRh28lQE(GP|uS`x!T1M^Ui0Ha<-YELLnU6!~3tG)~R|1_lc-iJ0 zv7;ncD4VFta*5zjhfR)Dmi&s_3LLjHqCA#a@f7tiDNUj&7K0!9JHKu zoRXq0<%uLcbis{+VJ_V~&|tzC4*DRY)Ia+nxx%UMfmyE!*-$p#f^k~JQq?1MyFw7D zDWB>;*pWytbfm|r-_Pfbe{Q<>Ikp)WK4WWdG1hY0lXabA^=>wlPBw_*vJ(9%lu-q54~@ndrs z+~A54EtKvR5lr6Fl~u_}mr&_syO}~IV%is0W{f8`w+tRd>L)SvjNV^4NveXs80JUc$pb*mAH!$uKwutnF+*hv+d`f=PKVvv$iUCsL-;3~M?Ae59lG?7A!> zLp|%rUePc=5n~?vN`y67Xx2yWH7oU--_H$`CBjt+F%zJd#iJyLjN9Hs=2Gg!;GRWQ zP}o&p5_}*iFbKXUkq(XbQu}l)1KWd`-&O0c8OeU45h+Cd1ewwEFUq! zqAdb%Yz-5^XJbn$@&p4^pe&vEsvdeGa9?YFgcFm zu&_9R&WZ#U*~53!;F88o$#V0a*TsuKTLk-nf9ar+b2eXTU(b68H+1Z9Z=JM>rcz7F z_RaiG;ct2$I4qW=SFhZ8v8BbjUVA#W<%e|pF)4v+A`4#*pQ}s<^H*546Y(gMK40!y z1v#H~1xK>A#0P*fP1U$rCUg2)=C`Y%+K5mt9uF&rM8s#i$CU}al5D@pnEkcY-D!D&aOFQ zK&z44P_<6S11PKivg3)hSEyu(nRuoFrGXcJ64*s0-!<+nsV9Vy+F$gfv8~*|Y7gi( z8B3T?wi^jly9a8j#NVE{`##ZW$5%Mc1Nq9MHb}#%0|pMrg>zVC;#x@_aIYJ@spP*k zczyv7OHC*{&^TwI0dWViNO$^p+VkAG5=#k$SwXC+5Kru!ADj5Lv~8jjD<)uUl@^%} z0)guWc+>)mBNaWzeSDa!^GR54^u8C?eDN$Uuu3v+yf%&FDn` zi28m%Ss^eK+Nzf?oH-i!Mf#j-wC$E2c zQ~DW4r1Vips&AZT94fYzLd6axqvre)jelxDJ=oWt=38SImsJeg5jLto9NF3iTZo3_ zm*QNDyPUKr$K5PjP+sZaIBWkLjs*{2^UPSZ{~=6d>_|ue8H;pW4U1B`I)1O`Yld~D zS=!45aD)>)q0(NrMZzL;)3J!PaA4gN?u;4ck!0@c2L_<6$O#QTm*<`U|GIf8<0$D+ zExwi@GxWNLNybqy5eHFNm7n1A&6!mZcW6jrB~>`YOxvS_1PvtW1XcHW6fTw@T|_CX z0LRjdSR~W>CvME>x!B!1>jg)o18Qmxdd40e5P2q#% zw7|kf4`V$vqOY=F(J?xhL$J|UiF4ItQl~Uxc{^>SUNW74sCOUIooH3P)gKvj+jsdE z6I6|g@tpgD_e!+vIy3r7aOM*l2wkD;e5+6%j+QmPGGRe@? zNAziOY->1JLPCMDjfL-$){?*^VCkf>D|ik%J-Jmh{#2U}N} z9b26d@VDYEo9Lbx3mTc<$Kz8P?L7BD>vZY3*hXP1&PG26(bw&)h!x8ni-jVrp|cA= zIih4yZYb5Ps>H0KL@V33Yh}>_nAIzNEU%L<46#bV`rK!zaOM;4wxJN%Tc5N_c=J8% z+twhnluwsxcEMPhDplHuK~86GQzsZ3wvrMXm5!-|(SB>cy(I#KA&J~Pyw(i9I_dV% zrY~pRTXJ-iSkV!@XCZ_90s#DNS#ek)WTVK;|L1nG_3I^4`V!V#CvYz(C&H<|46omj zWic81V!Z;YpuPh3SLi1}qU7e2-aSP|F;v8~LhLZBZoUL3VNv7q@`*k&)24<%x$sSs zfmI+-K0GDkF(RLzL9BzJAby->QCsjM(Ku!G#OI~vK)|K|va=P;m*>k&0+n>@4)JaH zNa5|vR#5clgO0q~rG4CD)JAIve9HkEY`8@v4~>24O;#N>h~k(cmtyP(O$LotKZ&n> zP+qIhUY@V&7YHfw*O?`wBGxLJ2)edxK-dJ^*$ZGWPG}~=aGO3*hk0bC%0BZ>r@Wzp zg2!Bp)_t%i*R6*HjXAQy{+meO*;-ssOp=?xZGZlm8JCU$W6sx`uaBZl7Ek;()O*0> zii76|`P}+yw_HM*aySu%ZNmfE5@=0YAi{aCqqD34G~Bi&7daqWPY;QS?bze7Y!zLS z_H|W)#+GlzCq<_5L{CgSM~7PK(_ zEb1o(%#d4#2F>#7TFH<+)?AxG#j{IqU_CIQ{D~knomb(`Yt@HiR(By zOQ=vb&&5j4KWxJ31>*g67@#99=wFx!jDn*46ShYJ=Kwkhm>eOHrM1NoZIjYQxEt9& z@~9KVUe@k(JGUo*+j*mg7FJOBVAE!dZF%{g;R#zKSh7T#vRagaV`ijuREGK*KIk+* zxx$N)ym&_3i>jJaoogE0=^(x*u(xclzgC)CevbNqlsx*HA-=Ct_ob<&xl>3@-aSix z$e15~5A(shosFT7DhvHx!;V+g#b%D-Fq@0syD{o=U%OtFqCLril#u7h8W;HLnh=k( zcErjdwu>9#!5Hk_Waer(x`OR|jo^u9M`u<>;10i*lG2=3N(;(s9g@6YT)#51K?s&7DM~5BZo>t#oDjMV7 z`_#2o#Wc4BJeI^6YnW&CCH9{2Mp@{^E9*(_3s8}fvyV+dKHsEK;D9X^c60?Gc@=+I$;a=UZ)g{Z! zS&;j@OLq%UXSxuj#AY2KEu~r8Ei^6Hw7u$JWX#Qg(x-P0FK*4|Wn^UJCvtdHrW=;H zn^itK7*mb=@$>sUi-{2?EYjtrq+h>-h~1nT-=^QK4@6fVCR7AT9#k4gE=(5{{)9xa zqb7L5@4PoxvxAVDzo^;Q~nwmusC%#8< zp{C{(IHf{@p3zS}JBy0yN`xKSP~LKL{@UB|&P)R>jWH#?<$q#*Mj<)aJVB!UkJ5l* zd4c5cb*HY7FpVlt$!$3Nc3vH*az*{uTSVlOCv(_{AK{rFUJTOb%x%kbW4`$Q+X4Nr z^CgsmYWGCpA-!eR-13-WOiQv`aw)4?8EC*EOHL8g;c1aFQDc{@TAS7(s#(T2}`|?{0*u-PX{^3I5E7 zmy;CRKBnd8!?_Af^g)ep76I}3{UyXCO%;B|=$DHxHxnbgdN~Atwb_=Bb+!M9m3lyJ zv3lNDm~>Bs&5h3&9}T4sdDc|g$W$Sf07bjXVISmFeL9V()-HZUoB1Y-m*;@L@kz8A zA^doj?#VjUl(%1Tq+oNHEOBG{G&@=+YT>RYEFg}fPlx{GUgN$uQy_;B@oqX?A?O@n zgoz9I{4eSl2_+RIpk#1|=K zWyfEQSpM0wGpcyvsy7a6CuXhbMP>{sz(Vv!pS{pP{bGs{7q`RwDSkCX!$LL2`$eBB zrN$5{G^6{RJ)>iUWl+MdAT!QbcSwAi1a7s(|5*~iitU(qXi_|c_sL1c8SOp;1wGch zq+GRN_>y>A1tp#~zAu$XLzvFz09+-~Do9>1Az|lUy~3D%@uYd{0e|pfuT*=2#ykJ& z-mI>`%r?k(4t6CSEq2Q*%BZ@+Wwsx@Ve$LY!R^iU1!?T{?a^^*Q}f{~Rd;L9TXl5{ zIcZIi8H<1I;)B(e*5{**7GX+U2WzwCcK9VSY8{kNm0;X#-t1FSvARCrdhd{YeD_C> z^HEUA8@QXqHn}wK1d>{r8Y~66EsuzlswCW--Uay_9mr+Vg#@`fjo-wmG*cTTPx@C> zkLVY>?%m8m@E@E5o^y3-HU;IUm1iCT4IQ#%0&tcMurgun6gdDvzOGQ&HBkjHw$aNWt z6%OT-hh0dMx7)TZ&YYaSG_Xc^Gdo!9510YUGg|J*-7i`|d;3eWUleM-U{(+b(A^z- zV^Hw*CF8|N+@+e2QC-!s)L<8rT>L&?y0yLj*~Im=6?vu@1}^h=JA(Jj>2KQ2;TIdGV&SpWM}f+;N;0>C$xxcVuDGMLoy68H$epyNYPR$Ve(T|w zhp`Qh(Vjhm8J-}>2p+@3?Le(nJrAK6xbxeF5nakri1t<@PzG7@Xt3Muz*XJ;6=F=( zZcR`mydh4YovWYRHWtRTN3qV*w8^#EcD3m(!4qE85*;t8cKh3XQ!6;+%6qK(x+}zW zCF93l&3>PZE%2mOLS7}#iPqy3cw98Qy#`yD+7TTgf1*v_RnDc4Gza(Xr~{=rI+`7_ z{h2kbA;C6F>l|R8MKcWKi^`)rk}4yk9WybECHz&ar}-0-v=by5dO={*MsdmQpELDL|R5h|Ws&@q#* z^(ko2>}amfd+-srn7RClJCNQHZDf%b!re4sFFS;;x35MqbaTo_`G;1XGH`t9*_N2k zp;h??V^3v|>NTq@&tf3MDY-uimnYA@dmW`Hs_5~?@ok5u56Tz#8&mm&iyU9F@MbAN z`UB4an>C7FRlG7GH41zV7KekUiY|HApI`uhfh^I@lE&>v#?~{s4_3vbuO(-6X(ge8 zdH8BL8=H6M88e9#9$;8a~JhGfX(D-;5U0ioJ?q`0_M3xfpRZjx^>zlkTNIm`oiS4`9?s2Cmb%CXk_&0$x14+Y z`fHPD9efiut~u=^Sf4{F#MW%VdkAC5^VG$C@{{5ii^ja=;Di1JEo-2uvqNfX%poB~ z2BV)-JwwvNs^%{d!w5bCAkVSj3rb}}?#j-J{AJ7T?QhPj7tGirYn;k9+=>=xVa#FM?r;k) z!&sf5Gp&C_>AT19C{G^l^^+k^;PM2OQtZ*WBun?^zZj!kTzFsmaq;+JSGv~r1IL+) zp*{ML>^g{5k{g!Mfa0|V`LcT$btv(?O4gCY(ps0iIR3(WZ)+66yR_42UR5cTJWBfH ztx89k732NVS0 zlMCtvwm);O#Qt#L)G%5#A;(HJ6M0o9g&XViY{G)@h?ZviFrq=yUQD4hgC4T+Q zwn3En5RbmTp+6r}Sl67m%2|zT28YnjZmQtD8S$YS**<>La{#^x@-pi?J3sT^r8O|c z(Y+sZKi^x3yT#eC5^ru@QXQ>&`bB|Jn1m}+n zC^cQBuW55*)m?NB_{vhTVitUOy|wPpAYoB))o^Gfv0?!<`QoR$3*;|x7(7m~&w}A! z@j^N+UijlMe}8^*SMxxs`Wpa%2&67e>MlT|5Yk_#qV%18fxTnn(xd}eNd(qxUIfbD z>w}F9&nPYZA*HZ~o3a!wbE%czYE9#v!VD?8vqD|r+q}sJ^^}5_^%rn7k9MX{Fhofb9g#Rni>}ckB!Wi$uV?l=c438~$ek_$A8|1MHh&I4H0*=jeF_5Nrt@~!Ga&tY9WU=W{{q}L@+gn$X7end%M~czV#B}Z3cYju?$?F9zl?53w?({^J6=N1NiWR;KCD?3mzE*P?GQ#6 zm_a|TA&t>)JGvlhY7I_5iRo{SI9iD4?qxB9XLk~Y39=9_nK=L5uM0VG)<+j|pmHf^agJHPDme&cm9R65z{lJ2 z`^U_uhO3aP`+oqgVFp{}gvHS)X=v_R20Xo8Y2JSQtC29c!qg z_vWDNn%fC!IWAn>C*am=)WxjXcMIVW%42_(`b#%n9%4{~!ynC6r*uU>uyL+m4U%dGmdY0z>9`VY-OUR%$VpsmO% ziypab?9!Q|V#y{`@@<>K*C1A?O;o$3DOez?nKuD0JL7P#mk;MU_?0e}2-a4!Ej1C8 zY6a^_hbM(8fxRPAeg;^qQ5JXh0>_+10+nNd7}odaq|<4$GAvKaX>flFsul?H+Fpl<+Ya0>Hk$2J3KHW_b6 zc6iKhyak%m3{+7QUk=dAhrFcplR}ZF*cI@72^mjy%TI1=gn6#gUcSy?&MgOF_qGEo zy?$>0Kxopy+vUW5ctOzDvht>EfLbXQ7qZR#Y&_$g0{^Or58IZs)G=<-O~(J}UVU&o zTSMzyE5_X?dv42NYtK^rICbZ%1w`COL|W?;yIG@@WASK-U8hu8jnw97+X%yqQvxiv zsjQxJH?_ybF{9oR3iX^Vm7Mrtk@{n<%Uao@6lWv9Nm@YJ=$XnBY0F)_)2Svzb+qSD z46~zrD)*V?V~|fTvgU4XNJdRn-bT6FoN23f%FVgbb)Pal`8MJm5I(b_0H&v){vqUL z*GA=Dm1}B{+#^jvy|n+ZQcuxgvMubu08wo}3>m8@{mR#uxv9#lz)QTYZ^{LVc3xLe z^3JK5bJGv#y{k5Y0>X1<{(5fgP-OUva)pKPatCOk<}19HN$2nXbH}Za5p7UkRCf`r zuK?hq4re~Pzu~5)@9fVY|dxqL_*qz}e$!t^^uFy27Id{D3=o&&6DSRn}YYhoHegO3uH zyE$f&rNFW9H_nh`v&a#Qa&$vL3@1Jo&d@C9E^fp7| z*U80}kb}{401ZPN!AV(+T$q!88nkZ$=#5;EX~uhPr%&D4HSUtEz=Yi`?$Sr#i@JFa}uePD2oy-WZfq4&p-gD^LD6?&Hwg zMti-V>`H0-2&{`y1qlqj`?f%aunEpRd2rxsl>9^Hzqp6cM~w0InD zb)cJGQ@2~<^Fk4ZKQI*qZ;u`v&6zBPi7F`*7oh5XNt7&$F}0O%ZDnL(qmgDF-fAPr zW%W3+_724@%|AhL>>)04Cg$Ou8YT5ICG$V5=u%^kb>fu#(TL)J$TNP!#fsyf-IYJV zU1_$^;klt1L(e_xY9t|b$m8Mo!|x7SPwS6P+% zXr)?!K|}rISVov!cTVMh&&JZF6^b1oXhp37>E-+ptIb}S08+uAKC{2_LJN20koUR1c>lB#61PP&>K{^3PW~AMG>OafC@j5Y+6qeBGDmL zd)rc#rd0Y|-;R4P-&%+>TLXRmA?~tXAaJYJC$?6#!&|^dj3Rh~wuk~Ra=cDB8DTN2 zOxT((QE(a*_1l+9UhiM{weB&ht!TUz1G0MMz&2U}@p2wtsu(FY7hz}yDR7u3&SeII zLWvnkQ?AckyMpRiDpk}trKR|QH_{eqYgmL|ubQaf%9>~$w!IUHULJb(ZpMr&Pc_HY zN{TK#G0lu|7bvKa~ybm`lWjWK(#v8enqqby$ z>}H6BpwRPY4N)Cl3Sy7k6FY-XeDRk!4qeu>13!xF)`an0QeIi`_O&35tGEp6DF}2B zRrfa9p?7Z_9T@)-V3Q2Fd{8)9%^57=a2!0Mc@Jm`7^wQQ)TVCesC12|#KYeGu66#B z;J`;kf#m>S($c8&;=i1;C(SwQSbm@seIR!ilvh=%M@@fjhs0+rX0FEfj9Ko=@d#c2 zs58`X8oIn|5FDAQ_XIe9TfR6zcL&p9FlpYaNmt7oqBPn6iI$tFI$L%%k@>x@Jld!0 z#w;G&@kRh`FN*CZb*jH+VYyA`AQ~)EG%;kMI=GjDsPdOys-K|cigb6T*Eo+t;evA5 z?w}~%7nH886Ah}K^OctvVgH-7mV)63Jguj|qF*v1PT4)|CftXmIgYexFPvPY#B-UTVHYJWL(YaMo z{ESk5le68KnUZc83P1QH*t~Y17}o0%sDo33Iq22ATTm;YHN0G*b$zet6yJU%v)HdD z6iL0B-m1fq^qCm83Er_d^V&KrjG89%KU@+7YX@uVvsZP=#s2yHD$Y2ut$&6vkMk`wX#`^XJoFZ_;$lhn4@IJ zfY|dj_~mU2-1_iEyx#qb5x6_*r#MI|0We*RhTOkcYJp#|_!5{Ue4|&A*7c}UjK4V3 zXkFhG<@P#(fK>c8F}t}gZbpeQ`S#y=p}&dl3X-VLXGq)=))U)u7oCDyt5$AyJUdNq{teV&_OG!w>rr1)HEng8t2&_e}mwz_< zZ1B;FVjAUBCV~*LuTXtVW+E(WDO;CN-~T;g?y$8K`FYH|Wm)U0EVg?l7gI0@z1LuR zC$L@>$r1L6us425u)sWyF(yBFmTyodM;0x%Uz=tyUb`k{dFA{wxxysfxVI9$GohjCpn#6wtZq7t!b4=~Ju6sgsaVRQ~81kWFkb zfYERmPL;m%=|KU{bgAIloaBN1_R{7GOmkx5k(MtSmW%{{YiYg(8z$2#X5@l*oJ_+A z4uRhrd%E;lBWX>o1V(0OzGurV7{6swN{wCTr_$S4{y&G{-b&YbGu6(W+vu)$gS|rF8b@O-yxLc|B(2XZwwOh5v$?Il9 zY_k9qR8y+c5oyPioMDA1VxrZ7tqrvj*&KeYm3(ROP^D-GgT$e1Y2^|2xYV@gvk?h_ zu9g#5Ylw)ZIrbb=gm6#2_JrrnrsYtt=*9Vyk^IrDlO#gKlPK_XZ&YP$&uqQCAd+SF z;LgIhb=J}cft{`aJyON@o?XLsx%;}F{%_s4tGtLp*IikOVJI-ik8O+nVbmxphgo$T zaui4u5J_BiJv-K)M&s5lB(F-WSFW~OKpML1VtehJH9y%pe?s^Iy*K5isti(aPOJ># zr}d1vU8`tZhB&p@o-+r@I*2lM=7XuI@ZN7>UaDW{R#}qC^PMeoXbiaCNIVo<+RLvM zfD$m65kc3VC}PaNYe$B0A$>uNw7~GR@uv4A)%XVo|KvFU8@H>64`jRZ#~ooCTbmqz zH<1Ojhp~J1iqRzl%Ce66!m2}7-@Xor6iLh+$B#7Mleb`q``!tvk>`;WluJt8*IEX< zV3Qc(Y-Lpo&hXxpT90{c^DhHKPASo*xJ($bHP+>NtH8Y z591pS!D!Ddt=rp*=v#7<#t%x3&9wYt6Ht#ls(MQ#e-1gYzHAspwvV(R;FPh~b4w!m z@kfTT3U8gVB%S(XbJkz9b8wd{IGJM-)%a%vs%pB;iZOXc?*~1@dt1UnBVfHuWsNp) z2?b=^xEkzpyA0)+b;SJBc3mP-+N7heR2d{27Q{cMy)%YnkVw$mzcWFOwB$0OX_!7E zYxo>m-#a^hiJ{nV>Yso!_utjCpnn0*JcWkefRnD@=cUP-LFKlNGnAK<@9v|pS98;7 zrn)l2WO&emEvS0%!DQO1R83Q2kI#dK+clb(igyUEp7p$&%1#CTF0X3PBnvIa(J>)Z zi}{b)V!UfkB4oTIdUj6R!b*>d2{pxzU3IO|(#r~AuKl?VJ#scA%fsW;Bl1y?d?%YuQ1LGJ zqNMKOq)3AL*9gMHTRK29kqE2;Bc<3_bT&OPO>03msQyozNW*x+C6=@uyWq@2L$@yZ zASle^Mrl%GP149ng8WBBX^HplQNQ#vz2a!g#tNtR)04fdy%vUkk6gq=*1;cw&hk4c zWNcvzC#`u$qKMWqj=gC(=llk6*?JkG9?bt`(~LNwz@H)SM>UsbfY(C$r1&uI>chl) z{Dm`Qq$@RR?<_>fsa0{1!#qUrrRqmNe_AuAMIEM3cqs5kf2ux9>HEh4ES9hHoSg0* zR$$H$(Ne!!=j(v&bzK{;DW|==dy>^HqLPx&DF{S30;>NsJOWGqM-|cFKhYxq@VxyI zk4+A;aA7R|x3RpnnZ<5dmvXQVffXt5RMu9V555vB3Ulw}AnA=Ojzi|fILwYUAb_Wl zIHQXRmtrw^%jUD6HzG4TVCyfRBK+4>E~Vm04c2eBkPBp>v5&j3Bl`ryA6OK8ZFIjQ zAykUDQ)Lyx@wp#**O&a?{Ttv9NXhtx!c}~_d83PPP2YZH;-0}6ma&W0yJ8;Ru7I+g zMp@K=iiioAT~<((-~ZLlbw)Lrb!+B2iZiwW1eAIQ5TuErgbw3KM?efUp^QXoh)4o~ zgr+keLTDL!5r{((2n3`g5J+GKDMADcp@ycRx6nZX=#Ag3yS{JknmcRVpLeY}zut4+ z_3m@dKKqn+@8>y3qODITCF~DwL?I1!V#oK)=NI_=PJJh4NK`Bdm1^p{Rn|~#1-dnOs5!PVj8lcck)u?qQ8M2Fvh^vt*u{%k z$LhZ1fx9lsW$oU=R1b&Vj1Ytj+@+*QmSG_A?Cv}dv!@lZzGk_*enO^bExImBcf%m8 zM)MWJbNP7NmCCOGhmcTk*KVbpw_lpc?!HIo8Z!D`O}b=-9$4Dyx5C_t#gf%a-we$> zzi|1v@R`KyY_6>88+Xs7L7ni)!;R7aw>R;GF;#GR{hd9VK4`RXQ4k2Ff=pw$AUKV} z6SMhQYuv>;@gIH^f&3;+4c%r#p>PyYEW;}MY-FCV>u{_d8KaR(=^{9%%t{Sye&TZ# z9&MG=i5h)sL@pL2FwSbJ}Q%=rN6 zRPAf9T(c1e!Ah`(*;+tKMp@s4KpU&CO%lCNj1ns6dAbp{sRm`u5i~!8@6_~^!aoo8 z*9jQ}B}aiChf@16rEsYy?~DIY;W9NH-{3L-%jy_>z~UCC+3pnF3azthvDB7&O6cfZ zmd;@9yTdc+(Q9Q)qtW93UfeVaZ9*3>7_0S%l}6voI1+jC{`mz7cX7#`yu@Xz!h~Xb z!T1c@IRl>FUu7NF4(BrmG)|YUhvy9k%*=V4T3cIWFZb@QbF1w9r{c^cfONPLdy2EQ zUa>&^U>-=epX5L?63*o>)T41ubg7v67uQyaU#4y?#F}71QTS zYB>Y*>JhEc>nGaz2CMzLs2;AX;60Q>r>xyG=N1Oj)0{=5}k+hfLygxvGhD;w1gSn5DM+iNy#Rnn1^@(M0Y4P$(if|?6 z9xWjVEBOzkqZnbA2xLASQkPhjmJS#lrhXmVSlf2-t*%N&z>N;~>Bl;ilNDgjSi>ot zDCSAKWw%HJMddlW$6Ndk~-px`YPKroZd`vG-O5b0gWjtst4XSbyfimQECYX zx0H$+-}Z;kTMDT;H28)2O$3T2Zvs4G9N3yGF zsNS;(9+h%Kj+7iJ;xD}6aHJYg5YByIPBZ>q*_fkmcRqy4G>9(AU~q*jmL$BqLY~T> zch6%y*oL@XLqsV)2=zD!r}bJvJKcgyp{BmUl%B%rz^vTN4mgr$cRYR$e~H#d?acyD z>`#h5nJ(0kL6`hCD@C#LeYHflr&T$XX_^fI%M%>LyT)gd4H@v^s7v>&<rE6Tcy z*1fZA__RBWT3OM09z<>~F+l0q9KTb?y*l8@ z>R#ZAlAkRhbycd;omZa0AW;b)0C!u_JZ^&=o(_vLf+X)Qr=K`+C3M?@2aVk2BV*ER zw`Vf)q%&O(Rdx&FvV$MRXH%5p(4*NB`b~2VHrTO|A}xJvpgzVcS@~=7MpQNu9%Lji ze(v4UqpE2?#CCU{p3$oLgCL)7mb>+r0~9h$Qbv`Em64HUfC3u*i;b?+JylvzkctWK zq*SptH*aIa81Zwn!>-;z0dlDFwo6$hF~_}Oy)mEomeoI6oxr-yLsEOA6jK}NZr$iY z8Y#m22}KvYihA8y=|0!-nl*1~Gr+bx6>a!b&z3p~Nuf~au2udng&VZOuRh;@#&@$( zoqK=25m4+?@$8)s8^(KDpPIZW^YXJ`!!xk?Wrir}@>(ae!gYkXVEQoTkvG*p#h|bd(IYW1C-r#YDR7$gkpXbE zKASdkHM3hQ)%r`7!&+ql9ce)npr?Lmv%rL*rtqE&g3UgE6 zdegNM>98-nFW;2Hz8W5*1O%GY{4(dTOJ8lj?^DAu&FWs|)H6#t?j#>`8uNXQ_++ldC<<*OE(e(O^rTRQ_ z5CR&sb@h65Vj7w_UBMphD95wa8*S(kFRsNJKU!bR2OHx=?=g!#g59i=n%rzkLMP<1 zaTpnlY-}%7QeW+tg65ASy6Cdu<8foNklRV*JbhwRx#!O?r|K?t4w<1>kWy$x1Few~ z*`c$`sZz%$XKCFW{X6~%%@|+YP&(JwYI#Z=8^>R|_0ntW?lCg zfi_3uQ5iCmzjf>@JR~kq4@=jsHazA*s(u|6OS>RlT;E*u{`J@v5HtmB$+8$DbbUmY zGd&=Oia<`B*~IyII@o;g&TWvOrRKH~J4|^ow?ECfdM{!WOJ8+V%xk3y2((K$dnr}V zvCtL8J|_lCaeg;IKLm#X4$;`!QWy7A9nZCDtYx{!4r8`DzrO%A5G;UD zIri+kgUt^LwLR#cC*(U5PIHx~H>6M5RNWAA`C}{oI>*nWBJYdW3|n%Q&<45A`C4CA ztm_Z9DVjw+Gv!4GYKqjB`y)OudC`H#wYzQEMoRn~WBg~-fKA%U%il%+2OzY#S{%!8 zS-${NRU-+v+o8IuhidB(h5-cw*-wPlDg=hMmn~{A!$)E8MY^tUrtB&&oI0eOxSv$; zTFRp(=eWOv?TdZTKBlXVjxDR_jP#qDsjW&8ac>DI_|0RtRLRsXdv{RiN2BSH6lA-B zFDG!8H$NbFl|Stq1cSTOg?r6v+|Y>`0vM&qFDfLUo+;mlcdBcNvl?brL66q9oD!$w zPYb_}D@rA^w|#!|>r4xs#!`f6&b8-#LPR~3`OoNYl<=H_t)`Q#eJ$e#25<<|^`C!Q z-~U#qckqet!rRg*E;Oyc}VT?etZRn0-yM7>u0*~M^&eBrN@*l9**lk zVc@fWRK6$F;dhbC^L1mBBI+9&Ih2$EQK(Q^OLK0xt3)90!n_<7kBN%b=uR1JaH_4M zyBNG%P#w@DajYFJQg9OU)^4Wt2c6Kb31$feDWOj0(V&d-hsa1sUu)bjH@Y9#yrO?q`9Gfc+qVcu)=#w;^>`-)N5bzGes->SGLa@xv6bP>a<~L4D>7al`F9gn z!$;2_?(tA`C5)Sd&jQaVt;pu++CmUrNBTczk`E`a>uZ2|N^+`fR@iX=JB+g9J)q&e z3MzZrC*NfOng=a`D^n&-;hbd+h6fUMCUfu%hx5e1-@){bnQNYy~Uz+Evd!%^9tYjXU(3i%cFq zDAn`cq-JyO(Fg&z zx<`n6)urN?NIkpR#$m}%e8?Y43wAzZqy8jG>&MG9$L{Ztq%ULJ7iths0>R7ItjkI# zR=ve51+}6o#M&S&9pl=fCOje$JVCx#*2|t4U@ve1n)4^p{;KG-)U-UM%~jaw8!hDm zrt7hY8HcHmK&ubYuo~DCVVXL?h6U_Bh6qaX+qg(n)mGg2md^{TjuD1Y<16XM1M{VU zSE_8*cL%^geVG3X1mOdKfChsni(EZb6~$N#DRsQN!cvrVk#1PTnTl?DA`ftmn7-{B zY?AvDeY0!Su$(}2bkZM34Gu83AX2(JW_k2Q$p|FQC_Epz5MQ0go#$D1$mbrYKD@sq zA*#MhTu2{M38PGIwkon6ET1>w0uf!64(xz~H3!0tPR)f^ZlrB`y?=A-gxrHN9q!55 z+f{ld3*bIOFvG{m~||T%y?JBV!!{;AGD Date: Sat, 7 Jun 2025 09:17:36 +0200 Subject: [PATCH 160/491] Remove the old artwork (#6874) --- artwork/README.rst | 20 -------------------- artwork/qlassik.zip | Bin 120204 -> 0 bytes artwork/scrapy-blog-logo.xcf | Bin 52428 -> 0 bytes artwork/scrapy-logo.jpg | Bin 23398 -> 0 bytes 4 files changed, 20 deletions(-) delete mode 100644 artwork/README.rst delete mode 100644 artwork/qlassik.zip delete mode 100644 artwork/scrapy-blog-logo.xcf delete mode 100644 artwork/scrapy-logo.jpg diff --git a/artwork/README.rst b/artwork/README.rst deleted file mode 100644 index c1880ef6c..000000000 --- a/artwork/README.rst +++ /dev/null @@ -1,20 +0,0 @@ -============== -Scrapy artwork -============== - -This folder contains the Scrapy artwork resources such as logos and fonts. - -scrapy-logo.jpg ---------------- - -The main Scrapy logo, in JPEG format. - -qlassik.zip ------------ - -The font used for the Scrapy logo. Homepage: https://www.dafont.com/qlassik.font - -scrapy-blog.logo.xcf --------------------- - -The logo used in the Scrapy blog, in Gimp format. diff --git a/artwork/qlassik.zip b/artwork/qlassik.zip deleted file mode 100644 index 2885c06ef4bab2fd9027bf748bd5ad2a69eb857f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 120204 zcmV(+K;6GkO9KQH000080H&bGI*MgI(|wQt0Pr0F01f~E08wmVb8~5HUsOUabaZCy zy?LA@MVU7q5qaO0m06W_SJvHKUENh(-Cfo9HQjUH1H%l*a1Q4TH;5M^iU%U#feN0i zB7(Z2;_HYgq9U&Ax*m8dBI1h2x+-|X)cZtaRZq`=``gd&^ZvO*=Tli(RT=R-@f_dh ziHHPYgir_wkI=g9+jf2LW50Mk!l@?^!s@nN_2!3ub^U%W*@qDl zpF~J_?979wUp)QD+@BEAzl@N)^~@_T_mK^nF7>3W^jL=`^&)t9eS^qWhid*3`9lqZ>7cPje;ywlMm*9Qz+=G{2m3?3R9(YgN zLl^A7^g`iD;VOjfJ#ejj;G#26-}jWi7Gd)dXztpBr(bn3w-f&nK06nF?_YTO!Tn#o z|K{H!9KHsAFI{}mWtR{C$Nm<=9`GaVz4+4o7au}s0Qz$Sq7tC8z-V#J)B2OATGM|- z3il-C=1V*Gd?0xJMXU3f4&v_P-UM$IL{RV#en)=W*^M&XkKyxW?ml*>RsZljy#n;* zNb3Cp{;BXgkPAEK@V)qHU=or0$@TF5hTs+d7&Xz~WJ{2G_qqm=k8DT&#@_pjUwA3I z!{14_4{yTAM>xWo`7n71UVLcgXWg&=EI9D}CdHx2_?{{Q4Wuz88`~~^Jo}9 zg&OFSNWn!^fY%bA70DoKkZHK*ooJlQ zqda~HK6@4P`zy#GeW;CpjSP-MQT%(P;jh8*E$HLf=mNYJ#o;xDKMB&Ne7z2?KZM5d z#mL7G!*L@V5@`J$NCIBl=qqRxeXa8ud;@Bse?e_>C#vHw!trU=Rs(+-zF&j)C&K#~ zXzyV(3Kn6LQ`tQ%{A#4)^FS6&bP0M84FI1T@SmZ-_ahCx1=`;Y#~0yq~t2dZ*B$n+AFhI_a1*HMYw3;q5*jM<|wPIOFefHvssQ)mlW z1Ub7%eZGT^6@EF4)2ai;sf7N$^E)`6fa4F{G4i`du6ItyiH_0ggN;#doL=`l$Lry^ z0)BhL(FgD`guc@GBOJei$`vo4`iO%x(g?Pf(dNnOuT$OlLtJ@V-fs$iN?k@jnUlgxVO@b$Y-# z`6lpAZF~@H4GwBwqBrKdYld4 ze+K%)gOB+*yr(wzMKr?fj@s*iZu<{|%&DzC%>2d#IT6}E8P4B}A|Uf|kjXgc_XNmt zg0*)kdK;dEdzC;|4}$(spNr`~q@%yc9V8>U^DOvq1*nQtpc>ME>Oh}C2GW6=$N*{~ z6KDuooxdX+g@A^U4b(wlpe}Mce?t-E0*#^wP!C0c#(@46#gPXzfnq?DC=N7*5}m(* zZ6<+cPzq=krGfe=)A=*>H48M4e4quC16l<7C$PCZ&@w6jt)L>%K2!pW=tpIsRa5~w zfciR5qZ;Z5I*6)3htL4fI;wT3H>Y$M4FYYVA)q6u-uVM)Q3L208V1@zO`vTw()m3a zN25R|&=}B3)B-vM^mk|)wSmr{aiFtk0_Yr??EDv+M^iu-&@|9RGy`-En(h1+EulG} z%V-|xTC@Ok9nk+o>(L_64QLI}jc5tzCbZo74cd&>0^Nev0o{t$1Kox;bbgJtqm4j! zpiMw`qRl{e0sR#^4s8K?JlYC$H`>yMjrMhZj?O`+0zDU< z2J}31I?(fhK7lSkX8=8b&IEc8odxtlw7>H+bP+lm=*8$9pqHR?fnJKv>pYGwL+1m% z99;nP3UmPIl|X-rUVsh)y$W3j^o8gmpf5rfcOFA8MwbA63Az;MOVMROuSS=5eu7?x zt^j%sx)SKi(F;0{qH7s_1-c68E71#qUWZ=P`7wGGdNI&fqn7}E4SFfi*P^RCKSI}| zmjQhpx(4VC=;c6PkFM?f5Z%b=P3RRsZ$_^K`UZ4e=LhJG=v6@9gkBBw&FD2iZ$Yo^ zd>>-n^+4Z>UI+AT=mwy-0{tGk4ZR-d+tH0c-+^ud`c8Cn=ey`#=nX*Mjot|KJ?Kq9 zZ%1$LdUHZ|C38L+BGgzl8n|(1+0{fqoh2R{?Dt0{Rv7DWLy~?gRQ&^y$vO zqJKm81N|EM3{YqX=-1H$ov)y8pw9vQCVCL)x6tQ1|AHQ2^xNnQK)-{&2=u$?q0X1l z_t2Msejhyy^atq6K!1q-rSmZQ5k!rL(T~x;0(}&H73fdUzjeNZ9%J;UfP=n-9*0Qt zB|wQ^?>vN_K;HoRbM#H1|AD>*^cU!n&KJ>>=-WVliM|8$SLnMye~rG^`2zY4qyLG% z5A?U_2SEP|{jl?S^gHw;pua~y2Koo|DA1?SPdX2xr_p0T{|K@8LG&l|IM6?%pLITm z{=(>A(Gx)bhJFt8@8~}|51?nzFMvLao&PG=wWO?dGPH;@abjn<$d78tKhq9;IoIoS2w^%H^DcLf=_ONFCGUUJPE#c z8gyj_e90Vm+6C~iYrwNEgGXHlo^%6v&`sbuw}8jo2A*;Uc*tGg8IK2#cmjCB2f+`1 z1@sGny}u4@`$n+TTfy$$2exzvSk+x%O?QL+{4-e3hrwPx2G;UXu#bDe7Cr%1aR@Bn zGob(XgNA(%{OGqq_s#*`x(0mAtH9@63|{O9;HzE)I`$*btw+JXLPiE!fIzz*1AY1# zXwXZ*UtI{k_zBRXpTZ~9=UxE*5`q7@0ldSD!ISL=zjrG5(^-hm*Mg2c0KVX4_{Qnr zlg|WS{9W+D=YrQg2>$CP@MJfGM|vZ8qBnyFdMo&xZ-TFRC-{~3fQP&U{Om75H!lP4 zdL`)PufenO9L-O!0vQS*Sp?%&0ol$%e@}znUkyIz4v@h|f#XjDAJ3u=9>$A!7d{$p4S&*K@;{Wz=gN68AI>N8 znfyThwEX_kSN_uZJLzG3)=V@&;@FB zpT!E6nSHziBGP*y)?5O*9BB5hLCb#wKK9SxZT}9Q@)_`%f5B<=Cy?GB@DzFqGS;WT zzf#MhR&_43uk*l?zY8|}pD+^tYZL!#6aQ-yN81E6hiaP+A!YlXk79h|-b1)^-66D; z{uD%f?$mvQhY+s${_=TC_u$ju4XMF}Y7Wl0n!olQuC#XNp2A-LdjI+jXI<~F_0K*1 ztb6zpdxa17U%$8R--CATIS=S@dvf>8?@g|p?%%t267Iv(`|#{O*YAZpoZr2}`RopG zgJqo_jvGl+5kj3*Qn~dJ3;EJulYn zvjlw>t8gL*ciFY)p84cGXz%sc2k#2Gd#<_u`sDS%NB8|9^tod$V07$-`R)af1id3y zT0VrY+0H(?rjSe03x!-E2mRc;1Z^v|O}q9iLr-&i2NA2qQeQ*>G3~^+l0HB$rvjcj zgx1y%p&f7};3(A_!|l;_d2F=R9+}1M!Wf*+lJ-ceJc~!75m%H}pQf?ocqfLcUy?1*4!}fcyDaP`PPA64vmXz~e;@Yv~-X@uv{1WpbRxO=3ae zCwZBNFYr8G3cjEVr-BDGRM|5HS=a*~@8c!mUoAo6i6>^JGLFP!MN>$qoQ?2OwCPzT zKgvnoa77Dv<5$$4PYlF^Ki%$II{b(7 zLOmJohqtGODr;^K6;1F?x$;%lUfO5NqAoi-55D}`7dK;{of)5)T^O6j{LJ{|?EKgi zvxP+GucV*c3f%97eBuzQfCTDrEY$A;-+wI^`ugAu3DYR;O#^p zm^VO{nxN_Bv88IGDVvVQ3*lU9sGO+Jm9y=^vNdpgEjLzly|7r&Oubx9Z$18mVtIPF zFD%n}5R7v8ujp3teaNjWkO&9T5a6(>UV!1#5s$c%$w{KbY`aY@pe<&@qG+3#Xdzdz zLb5H`VJu>fkbA8tA?c{hyM3js8sY>oyOblm?9#SoI?Fgp{s7q=)jtQ?k!d?rQXtDA zZUzUY2Jc=UDG#TGSSUQ%?k}}w;~|%u-hzu8u060j70W0~SHAxOy#L)-ZVq$l$nw|V z-i6LG`uobg#U2l%w9_LQE{*Bf!N|_y8T`SGvqH?WC&%H|GjZD`6WegPtWzvIO_!d%{6{N{}UwVz=;iQhxs2@%VI2w#D`Qi3)ldVdqwaTzQli(46DVX<8C zqR>#K+-gg$s3(b_C%};eQUl=~CYs}5ON&G=S9A&E>lBroA&MbSUE~FOt!(IvLddp- zGi6@AUg1SUBH#vv*Qv64Ig|WhDYYLAsu|U7lc*GS4a0K9x9*G<8SF9T`7E=kCf4^+M z{xmBj;d=`Hu5iVYILur2wfWp@85}}N=Vrhp2O&R~*jPd&p<0sf>Btz>p6DBi5lw~k zoaUy9j5$pLJQ-<5YSn@+tNthi2+dclXd<|00eu@I+=SepMk)jPZFTig4(?CmON&O9 zD5%Jf?CL-@ENY_UR8K2t8nJ)Q2?FTgGOr0lNcFLMmO6g|1a*K_7XV)L?llUXlc)ig zOH+9VW3Q0*nvSPm!J&3;4Bd5j6(3~@r_ z5@C*DjFp5T%Cab&^w^c0sw!9tDVD8-Lpl$BtoqU$-x!R4r-N^UKF)xZ9YQ*bPlr%H z95wLJY1EH&=Aj)XsIghD7pB;_aDW8BWsd}71e(Hh99%FBQzqFU6A=J}0&X>fYty9= z#zH<}<|>VJZbP}SXhMt)w+d^nXz+x{+UV|?++fuy=TrFSq7k-vD-qUW{o}r9h@}lh zx34i-#=Ku9gu|SwSy5~0oSo~6wVJG%l$ToP0emy?vIQ(8koPk1fB+*eGaeYq0E5gk zV*w=?^(e-eC6)4M03-r$#TXbO^gZPYKAOR^T%a$Kh?V(u<%K;XeGN?&K-I!6zcf{G zLE_@p3E7ch95_*k50@b9WTJ)XC=m$g=S z5#Tu1jjv7Myak^`fb;txBM#1}6}HA`Af=27RNtCWfKm~_nbc?F2#lndAHcx?7o658 zRY;K1EbidU%y>Oy=tR!+4G#8)%%Z7Az@xe)oj?q9L`5(ZTPxTCCf;CwUiUXvK_{)E zgF$2UjOom-+qBfOj1b{Th*G-*IZrQ#u4$7L~!Rr+^ z_JP$Aj)>9HSV;*hpr>bDj3Y`cstH)r;sevg_5{QX&=P;^mbEdexMM2gE8Tet?Cx!l z!BM`bUzi5Is%RQ%!8lE0=I}u6V1T4@U~d6L>TyFAl&Ud9iCkB;(?sac$J?OW$$mjt zJXrLyGAGWDMk^V+FT=}%nbK4(meGX#aMtu18=A-M|D}}+A9qd7uf|o)vBP|RIK88l z7)zhE&#g?9#>YVzBIkw^qK3JAb1v6BekOlq;Ik{8uj4n7Q&FCI*ib!)OMt@KU57wB z2t$yg-4~?1IE2@DvkP-xYR8m6XcJkqYjG@t;%k+m;aWk_GYLC8bIaRq8pm*%7qzS@ zkdSYd_g{S3c_$<>M==yf=N0kW$&IK6659+>EU@=996rb~3^t!;nm^DzTV?$eKZ4NO zD>(&-cqDMf(8*}Uje1dU7I%G^(-n6)0H}C(t{88u9c-;1bZjd-?&Xb8C_Xq}mJO-Y zD%xSwD8~Fx+hH)$`dBHPi--0)rL^t1v1Z@I9B!l*221meWOjHqn;mxh3ueZ0hGON} zff(lE{bQNTP}t7ck&k%3?Z&;y;^2T5$sZmy;^l;2bYiL4Vs&&PkbSE2S$qq~ewoRB z7;hOFFlBmV|)2+#V zU(`iC2I0d{5;mw$u9>&v!y6hS%T>L4GJvR=NSpBwUZ#vC+q335knnFdTvok;g4o(&37yT2i{$7cMn@Q*+{b zGB)`1Y$Ad$_QxXN_+VheZZw=u+mQWE_yu1#aU!WZy66^$Q}JrKaQGL=WFnu+j0b%l z=$t`5PR>Rbp_j2fOVDQ*`YccgDFuj&LNJacZ`33!EElOTTn5P>_%m>St+JGdZ-beR z2K-T12>-bek^V`b%PYRUoBfS2AylIDK69Aqek6lQ-5^>-JM;K8m+TmTC?m8_zIyX( zZp)1tKh4qHn4Kq@PG7jUg)dr6yU$tb5?xSKN}F z**n=cn{o!Z@nm!clW5zG%O>wvPr7NpY{@6S=A`-?Z#m)eT>Kk`$ z3jhMKgmR78qUcB*SO{+35AdC4)c(4i)rbz!-^4r@nVfFff*@#u zYzn4eSNh7KNXEdtk~fs_-|x`EYRHj{RK)1tI?~*mmn5E3ZF%m!ciz^DfN(fzZi|9-#^v)JHDOVii&6yt!2Jn>T;U{E+Z(1RL0v__YACM z22+JPg8G@K?*&a@qg!m6W(c8*rLk7C-K6#uTmh4s!eS&cl^I@Ji)6!6NRNf}q!-Fh z?HDT0miLH7ZFm_=^TV1iNo6w>cCuI)tth@gl=BpU0!6}meeLYnlm+iZRf^>5{>09) z1TZ*9xb<5_RcKepA~39s)&!Za0`K7C^PS)0&yYJHhoF{3u{ol75%t@&J|fb<6m;3- zg8B(0YJrodagJt8quEA2!pR~pm`WreTM%l6!t7A$v^~W&HegfNobR7GeYv!||1@%k zX4#S*m$AsJJXN>+(z)wj`Qoq!F3{Fpf6LYTK7UHh5A1vz#^FC<9Ac=C^^RtZVdm$! z6*05?(H?DK8bS+s$qr0*z2aA@&aMOV@&0DCF5*L8GBQyd=-fRuVvTHM#H!XL(THh1{Y)9mT{Rg6G;lcFY`#4e2 zl{c*ezktgrkPk;RN5enK&1Q&>iwS|lUyY?K;preg(|(-?{mL-uDBZF1@*t1$b6a<* zcRV|30Rl}qf8`WSOL^BkKm^rP-U^;4!O6tUjm4m$QjF*D_dRH6=NpzufG2&>lOK@x zp#@e`D%5+~0O-mnHDl6b6`4Q=RZwbfY$Xd`(WIUi1tp?M0$z`aq!mhe4dj58lo_2a z$+kSUXRgpH=1-qnrB{`qlh4VFEEbakvf|nX3E)K?$W-|pQw;*#D%40T@Sb$@|KJ9L`iI!D@ zAOV5VX0pWB%l2^fT0s`Gebjl&g_6ksR+WABPAYQ{J`IcyhvS+|iYK)2hWduI%qz*n zcfV2aB(8YE*x`E{YvB^~YohZkx&`_b)OXC4u>>iO(eT82!D^8-=D?qcmlWELSSg7d zeJyj&#GNNkTr}9lX2j>bz&kN<4A+^uZ02 z16G)Zo~W)H0*4rnf5YE^J}g3f2>hwdB6rXiItGJvTC=|onSgOz=3}S3XCRBAqeg=w z&?ZIJQCiGowStV`v2h!~Gs?6+JQWo5eB{8Gpo*#^$0c47oZ8S(Bvv1&htqE4fZq#zL8sT@yc6T}XgKO(cG^?(OSA(1v7)mXMg9n@M?#8#c@hDFV+C zo~y^AqpeXdk`umIosP*|V52au_*F;26<%nZ)D}V zNvDut$a<42vyh`^~rJS%&NltDxSxi5q;L5F-^ zj@KiE=gn#uunp+CtZ+i50=Xmn1XJHh|KzX}mh6Hp*lJ8Os!Ns@|gh_Kze6Znr>HydsY@^p9Qc8tveMr@WBH}W-~z3wgKGtkOI=Y8EZN-N&iKQVaoGu} ziWn|u+;FBpN^y-UMqM9d`XbSiMG&($ysh2ZP>*s%wk=+?3X=oHX4W8w@4qN+N?dLK zbbxj8om0r)K^84QoDvGWvcSxU)(lyBhQ?GDR#!U9?2tN-Ag>H$RtbVE`D84o(Qzy# z{q42d4nJK>n-e=mVz0b<{?wUFvbk;mi@F<9Y&9{SO%8inT*Hl$68U}=?3xY^qa@~#V&tAyjSHEG+ znTt8f6ZQAs0sJ9y1j(c-A!DnH-Ans=#r$`%l<&YJPS&%lDnM zV+JoO^V2gk1a~m*)lGg6KK~BLm#9`}>p?7cSfTBC=9JldoVdD}_1tOc?@Z52mLQ6{ z*}rN!+03TS*wM? z`QQ)+JMUAj@f<90MB)TZwySB}EDW6s}N5YaI z$cmQUe&VSE=iTTk3gP5zZt*R^&Tx+9}Aq~C6b10=?Z{Z(~KINDDq~>gO9<3IQ>yNiFFMt${6X! zZRd=g+-U3_J#go;q(W{4;6n&ondRE-8(y*Bku*X?kT?9-xL`6qH8-E}rWfW@QmHXk zF9@;Ol>nHxATm&cLwBE zXhDpPBAC(>0IB+2)ltoiZI{BKiW|=9f+|TVQ&xrY8BJ%f)@N}-s6Q^MvK~?GT*xlC z21N*#28kS>IIgj{8*l)Y2rr~3{ldmvdQ;h-ib*`CxNY5s_mp?-*p%T!3eRja(&v$5uO3w-o->WHH*de}cC;d+2S7(hQLc-td-%3i59V-2=^FO{Idw~W0WPBau(W_H zlxf)}kV_rYNp#2{@I|tz$Wa@HRO=t?Qv?Z+p&Cv{WysW>)M5hLnWh+xCu5F4-T)zi zKuQ*2kyD2D%IFW8hyuM{Yw?sE&d@ zQ#?Xz3VuDn<$bKqR6=5RG=no%;iV=<=Q~&$CQr2A4Qzo1NjQ`~pl_I1f$q@!dX3?WCSf;nCh zy?E_~uY2=af_ZE>W{lbee4@#~DJODk-gMJ-ZBZnYw+lK~;Eh}c`mA-wB^Ts5Ue_N` z;fs+JiAI>`fS5>87dV1Zmg>%SOSeByjKt)STeggd2CiDQ%dw0@cuAZ&Kidt$Jc;K^ zT!z=hq)A~v5rpiTFTHmDl%?T_pJW&8h%SP@JsTNUxEG(GH5axr0U&Vo(>C(m6R~eV?$M@SDUQv?25sf7N(~!2MI_vNB{yC zlR(AGII=c#L1s1ouj4f-VF9lgYm1M({eeQG$9vx8dYt3>#aCW2V$fn7V;%Y^b>4;V z>HHA3m zDP|coLrozmu!ZiV#3XfD6bM8_IBAP4kIt7XT`$*N%|&YkUKcCUn!nQ;ox$Zin}er? zO)r4a6PSc{Hg(k)uhJ^Iu46IjhV$UOWxMFc1`Cnm=%gnbl4FMa#6&c#N#cV>YkY@j~n1*PI^lW92Lf8$=57gYnK&_)hW#DJYtZdu>mw^^0(185 z)pHrr8d?#N`?CTUnJVU|Yw=LB;AGck`WMGlKO7k>dgH_HfNkn28}hWFu&rv@{CyUP zXQ-Ovi|tx}votAF{RI!h;&g8E;u6reXi zy-FcIMS@6b%IT~k)Mui_p;&w{M0l(#WGj2EIc4Mf)Re{(1M|C2Y%P^d zO_nV|i%P2QjPBdC{j6jvqUdHIJ6gNE17tUZmRN4ZR>!PT7Lh57xT5enC&#hwm9Zh(*V(tsiMDRBX$jfyN6lexAaYft+h- zo=G}drxiU~#iMfdR>FI^JNX;z zNQa1KO|&fOMrgS`2qj{^kitFg3P++yd5jgnTPxFm5RRD%> z(Z?Hv`d~qmGBerPYnHHm;VDJ8|*mI@K)>VS{>eT?am87wbyTF<`$dyG1RO1ER~gs=x;xMs-(O^bOENf9MZN|!1jP9xzMPbJMkI<6I>vSDU*cWtA(FzD%! zy$cX41xvL(Ox%Ip?TO~({Mc|S@K0^fwbz3mXt9;TYB%2JnHNP}FY3)6TV)PBQyQn( zBHg-jLtB@eX5RwmrPr=|SEAIdwBAC*rS`e>oyhUB})+1xtl zXXi7WY4mnzs|=a9gO1E{(Ta8dQEhhT2D;E?q#2}s&+W$z0>DGV$WDm55Y;mMt#U&_LNyU8(%q_?VeGzGmA&*XxH&`gJy>FqHGI5xtpGL3)Q{VhoZ#B z9n0m6D5hrfao>@1wL#r#O^@5MuGx+iu|+qLG6dVPc>-yWU6}ByStn%MgDz*Zroo9_ zm>6_ZVO`CZ@AaH;Djyn{tHpVBdfi&^k-VT9V$9E3g5!pHqH=H00nYwSP3J37+e@bc zoqig95Og}tP(hsK5%(bhbR{03f}=qZEt0I7=ecPmhA9qsJUS#az8J z6nzC!>>5!axKd%O&FJzdc%AGeqroW`*dU!apo2z#oW}TCkgw5C(&f}Z2<%L?<1Us2 zF;DY)E?fv@a=aqs`;BnP&SU|o<*IRoDA_8dH1T$3!*o;SiQBnE&~n|TM4X8*4Z8+x^@}H&xHKBH@9w3TokPG z3nGTFvTnJ|mM!CVrAOCRPdO_+x_;oZhks}#%Tc$O47CP{TTF$1JGXDnzx-NSUH;SP zx&g--SUY;gPG?|=ty$_c@dxqGKwGBSiVBM7gDiq#dJd(SZC1NyV4JhCAND8-)09!kQZx+%v0B0-;oExGOWTwTWmb8nr_2h6l+4jihM`@4`bnW_K*d&BKqI;V!Oh;9FtLefs1O zbfiiuuvNhVN;01C%9+|d11XPDJz!d}8Ui!kfb$3hZg4jYbFw8D`1Mw55aVz=o1Ys_ z*UH{xL2;A~%Yw9>V|R77fFDBx3>9nC!{?7E=vez01*N%q*CUqmj}2!yh-z$-h%T@6K{*ZH zF3<%=6<$(!K`=x?!8iJcA2b{p@=nmqsk=bHirJN~K=_E3GifcMS2uqO&x1}*vmCQm z)5&&g=Rpa9Cd?~~4!t7_uIYBSW)BJB00{vM0=eEVTjF3p2hc8RcmU?iuruZP$;8l1 zzP?tLWnK$~DiJ{ul3CYtYj&)59DW?|MM_bVG5`uC?HMuC>UR<&eqpo_jyZZtl_dqf z;>Yc*ZRX6#z{uo6z(W-5s!Dd#b+)Ki)1hneF~a??M6NmlI7Dbx{q4qape|HzT^y@FqcnmLyG4 z)J$VK^JhWOHId^KSvb-1NMSBpW)+;mvY(rbSI2l!3i>|Uc^bb3KLN3l`Y>9{ru)fS zpv?u;k6M8b15PCB7NV`Ty{ZgP3xz$zYO=+4N7UygK~z-;bBSStm%0;+$M~~a3;dZX zi%Oa2?GXOF-CzD>z8X_VB!p-Gk!K^ZftkK5hy+SEhIZ_bizQSvkt!S<+SxU?Sy-!UR<(SIxMx3+z z(wr4Cuu$j|6;84>*)p}W)W+P}TxGp4Kn@|RQR}1!FOekzdF=FcCn!3AP@Qz*@1t6&SZ1-bS{C$)Y*ElCVjbnq`tWmj^lxsBoe!8B*s?G_I1zb&Kkzg zP;VgwtJFt(#a*sV!|m+zP2;oKMXwo&_G+cwy9C8!PWq)F%3{iuAvD>JtJ)Dk=&PuL zWm!=fSR#h0gK~wUQBf3mIihKfpvO(sq>CLP61z@_B6MC$rqjmCtspL&p2q8zEu}*t zS_~GGMtpwlhQi!tO;eTV^xXP`>G0+;WpsK}Ap%x0Eno=YP-%Cgu{Bb7p2gcmi%xnr|ZGqpr4fJG|A;(-j$TxfbCa7x#V=xMG*63<* zCf&a78Q3#LQ>zQ=RBc$U#ZfFdg^w00RkA>m8JxxVHaTlMDLJf$ie|XsL$agAG$p&X zWabMwofC3qPH_w~Z)!E$fzYFZ6_BJxjhXHa?n`w!l~H! z#F(pas+^6)r#+|XSt(1Ecuq9Lf&Gn%t0r`2<7#IDUkFInBas|Y0#Uvq)x|L)l|EB?qyev zkFBeQT|-s`y4XVya=8EuV4FVG{}sEB{SL;9-7dOW>#&uMkgV-Sc;t3{Y_mZ6%3-6x;ik5w`r z_~lDFS@a-hf+qH23f=qXQPgbHWgrgy1UDyS6Oo~6GbiXKwqADqNNRav2Nhe#4#fBVB)G7&@kC`FY6C*?Acy_4m(F>L2np(33Ue-yt=z!be4)9)M+swkAiL_|vZ0g9E{PUgf;O~-KfD4LC z%&yI(SWTa+9KDUYkY5KGm5TMTwc9o?1^b~50^wlumJJz8ApJy=Rl2|Vx@#}%3(JD3 zIy(+tbL~r-@nEcb^ID56hSB&%t9!k9zc!l}YIk!xhixlcJ$dCkXQrzi6)k*Zx=Sht zOV>XQpzYY%tXCX8@x_Vw0FlCbyR%;x()krl+KG#VBgf8b;l$AsVWKN($ud#SSeXsG zPjiYQUH-%75}rX~+thQ>)K*J^MhN>EZLuS_E~A=bO#I z*_G)GxEGaWiFs$*?@B$$DQ4;c zW?@Upd8xOuoU4?db5&vyKd08}Rav$)$%-apHsN#SjHFqLlGgyHFNgEK8nAD*1?LRx5MVG_Pz7`&V< z_2*-pP@in+oM~#2>4Zy& zlSsuZ+G(X&9kA8k+w*H@O{_tpA?Nz45FofjaiHSMfFpHGH`~R+w9P(1K~FumO}g`y zIuyP=Bm@tD;JoNl+F)}~g%nW>#nTA~+KfS)nq_iI0oqiawC9(hO;Hcgg}wc(&C)<$ zPL@qg3fi=Lb?n!ZhtVE3a>`M$CfFNxq*|9Z@qb^phQ9z>w}Ls+ZV^1cx(ME$#Q3^y zna{2D`|@)u*b1bAK?ytE2!~7|D1i%6&8#^=3H(k-mya$A{s!ZJRL_Q7S|HtmHZeqo z?}Dnh1ktQp6emm;Pvbj57N@Yh@&wBN=>MVnh5C z$kFvL63z`rIR1ZF-#V8L$ky6HRaes#Weh_^Dy%SIT<2~<#D|_BMtM|M&(8tkN@d@2_sW)VK zK3zl@4G>+UGEh%KLKgOcZ7lZn_m!QjXY>WXw>ob|H=!3m-sG(ErnDj)W_FCb&yxsN z@Y0w*OZU2!2gIx+01h{!37r>tNQ?|O0&Y>*X?le7Bi?YWsL(b1s=w?L-le|V?%aY0 zpe+xLu=$bRYUO0tlISWcn$1-i=dE7;#u}$Bx7xuf`RD7<5dIfKg{YxhZkYB-(?dp? z={ z9Xhpg?DIRYAiL_32YVF#qaN%*z=!JipCA?$ARnb8Pvt+?<%!M(l#cbLbWu3N%Dt=H zijT$rqzyEx3W4AN(Iyrq#E!+IrUu!+llJxTPr zJ=}fd-N*(_X`nUeRCEElirFU3>zAlxyZdHWo*l93xjg?5AL~Qz4j0A>>?uBLG9yKI zgYTycel}BhcH-Cv{%k+yfj>X_ZyxyLuRQSQqbm>mx&5dI{EYOT<0Je(dkhd;H?k2@ z>z_lO(Pq*dsdt}+#fpJ5N0xvE*+sXmN>?Fp-TJF@Bn>*{zum?m1am4{fK-IMOjbvZY*#p1Y%2C*pM<1|7p>$lr+eYV;bf&d0)aY zQkAmBf<|~%*2CFq%_5xE%^l7s6-XMt-QC^1a$l++EFI{g+V9F?3!>apI7rolDppXj7{YgXCM1ofE@Nw`6iJv8AWWM0 z37RTlVtKw3kD0Pc&KGi;q{w)1IA%&pB&^D+N-RHNCrw!lWPC1}MGun?9TSIF?f5>o znFb)@W?~MxvXw|A{3O%QHfiG~xt;kQXO-s(yv|W~SnY9|DY%)Fj=Aq#IU@&0M(| z0|+aPF)#O}6UCSTX``faW(?wv!Yke911*pou8@BP?ds{5)77se=$8TC>FzxWDzeR$ zvf4ZPiZPI5i{avJ7Q+C})F4@vYm7sdTuj700k z9E)(ckPFrW5ya4!Ik1PwQeVE~t_h5B!hS`78Nn@)pS4f?Yk$`jHAW z%{%ETgY|$6nyk*Wv3tg51}Gf%R;q|WO|8JP_wM`PK`QXa9&&*PXmxQUc;(>A2oU^j zdnoBI$3dGzZmT@9DNdJ1i{!3|?}a2z4QZwdT{LOHu$(2wdf~3og3g%s$=K1QF+x0^!Ea8LOtJ zufkgz+sZiY%}|+esM>jql*zpi=ca(;O1CaVy0)8OYjg=pFp11{&p^M#m0gEJxcm5# zU_!Xf>Zr6D*PHb*vExQPhb_+%AjhB&45G6p=LnK2Ng9#TW@7cVzG_<=pZGlOxAv8y6lpt6WJgoX_S6pC-Q} z{{*OTE5n6eR~KABxp~HUseAVPx-hnSevtAH`NA<9drPaHF2&ZV5ziX*{Pk(NX;AMg zw037e8PX?>FYTL3jy3&__4e^8F>e0`G9@h=yW-PCQViuyhtH$is;C%Z0~5L2)IfYa zO?kRnAgDsUS{XXw>g@;LP))>(r8?1YA=%rJEUi6jE|Xb2Z{wD8<`Vw$`5Wl?r#nxP z9r$q=`x61>Q2U;$2lE+TH-5|j?^C^sI^}p}b>I;p1eMC}3Z}7On>Q`QcAXEngkVNE zSYRwPsn(S9Y{xq2NxK^l+TAH^BIqeAyO6N6wNzmuOxL@mOmjHhw?3NKG1Zt)49ym- zBp8&mkf=M3Dao9@XL48ETT?Byi?rY)_aA=CXpZCA{H9QD|1>$BFkD$7**`}<Cf{n_z7&Y)cFDaApR4oqA~DU zS=3vvaUW!w0;@05(Ghy9NFb6PF-e9mAuA6?X?6PUdh9DOK*3{eD$H81fY7UQT9j~! zP$V)twqtF-Waug}B1DKAkx1V2maFASB~;#2)_r$i`aHLk3WdUbeU*@rcjZV}BVj|T z#?re+U%D_=P+VIjj;Yq%^!CQ$NohY9dUDL4ymBgR9)I{rXJG9}yPTezor-%F<2BX! zJ$W1cDH;dgd@>qFJ^c%+YeQ`PR~l6q&xw_@f$kYyE1Lhuy4iaSLU(z>YE4{alJHxm ziK`TG^YbfWZEMooG&HnsIKCWTDBDquTneTH3){PA>RUKzwX?SdHE&K$> zbD~2PZZwFCh0fFXQSwp9m?&bQcDtq?>`&NN51!{S6zosyO}R76q(M0nb-kWau!2r^ zi79nn)2k-Mx=%&oo->`_V!Nf-1o*YY((Fn&k{<0>%z|&4f+m*B0^N&8LSPz>0%YGS(nvEXM4tr@h?H7;f2ihqL9WgqjTH z`sp3Tndy;<6W_bTifbYd>9ZS7Po$?`I7b%<$bw`@BkLlT0S2C^G?M9}rn%9$1(07$ znz6}palGtCjhL;Ars^aET#&}^#fQjS(N1&`+R?S#-Y&7NaBQ#B3fUoaVm+u8(8{4+ zKT@xzJGIZP2fMUs9b`XfYa6nh?o;!aJS$@@`cRu==5j~dO?s6sB=EW$_R6D`GJ6O~ zxB52F-81a62?IBn{SLOw)5!w5R(OovY>F@^^C+hZ7^`wf;3ZMe1zw?Bn-oVDI8F|U zbbl{TxRA@Um2ra3OQtLnTd-wL6ENTeQRcP1STJ0!Ey(O!JmK%+=+2Wn8W+=RY(mhs ztc<0lcrK-R^(Zah2}XV~-zf7$Rzq1^jT=Txu?lLq;&RqtM9!Oh)=mw_6p6!Mdn=3{ zk0tFhSQ8|k=X71N}}zXM3NHzP*tSdnfWD+e%Hdfq2NHLzmh7b~Hkjnb^o z@e}UK$uH>hny&&R9?K^1*5DP<|65)TpNuac^#5@dW6$U9;c{iA`gRN~XIq^oyZZn$ zbQ3o#UIi$i03XvqVG|>*k(S3`d?7*|N^bAAZM#m`w*9!qOlxFmx`oFU_sWJeV*|<+ zL2x|g6rK;sA`dTyO^cR<@H3VP($--tiGKV^C+yjA+6jA?=hrOFFD+y+ZaZ;F=}!`# zZKhB7D!-1y_tQ7yMhMSwa<{hl3y>#$0$OiR2&xBrM7yB5kPAe|9zg}7%IDedwxFH( z1Nc7pJ@qS8>qA{l?_+r|?wWrW=m)io-ZS`u?HiR|{TE`=!`bO9uZEn=OeWDxh#^7J zWLxkT{8T$7NSv&M!G*Q5rX)M8qAtU0?`^mAS2yxnx3Tq1K$+V!+XV~Nx ze19RQ>ar#K^Po3HbUR6tPeO0YV8gvR1vX!`QmY9b{MPP0KCkQcOK$He(iMOt6pwWC zQ^rWwJu+ldEU!6rV*KQVLTT}o@rhH`6sOKTvAS*~ofuwQtF0eP;{>EDoG3!dAsX`k zrS8gO>?*JO{l2@eZ-4jA+qZe!EN|8~i)ZoJ#ybU!V~P!!V1o%zT0$t4s!EfBL{ha$ znv?_;rB0i?xC;O{$Psmy@xzebLdWM7)H)zJC5dF~9%(#@e}qqxl2p z_FZ@=y?E2?^o>jD)WVIkzbS=90y+b4wZ5pU<5`6l4c^oQb?>p-QqgY`J<&Gb#|nRz zVm^I{-k%?>d!s!a4Y+7e>}kn)x(2i-Nc5=9;YmS}6hYBfhlYPjiYgcYMe!%DbovN8tbCAf+21^Y&5M+}M0N zMKU%FX~dPE zrMXvz78_r0E9$-f(%|B1p0yKHFV9h}Y_)OZyq$Ek@<>xFk2JON$hB!@A7$QaP*Goh z72+YZ_N87mWrF=OPBHl6&ejTsFxREh8X5qe?}uJh*;D;P_H{1c;=k`3%nvgRJ%}mB zoZgl#1if!@OZ7oe^x6W1K2i1Z-oZdmus!s-&c5H}>pbvvobokZ9~K?3t0V7|$BkZ2 zyC=7}+RI1t4$p^x=LY09cFWtG9QciTJKR25KH}kAdvb{`x89*Jo*v;65OsuFJn4jc zJJ;$b3HWW_T)s!$qKezxqI#h}PVv7`A1L!`(+3I)q_?Jis4noLi}n+g~FRZ0UG}u^eO3}Cy_cxPb9>ys?$oZ(o07= zQ_f%Lp&b0?+d%(+^49_VRgkhf$C7nUQFlqt(HSQvnAlb$XyxF^xQ$|V+_p!akH}A;b=M?wo@Y`&hOOcZ2k=Be1V?#MJi`<-jkl^JW9hsz8mGACB1Sq;xn5^TjL?% zyP<`0=Ln19))7_|7R9AHqgS%|v0tE{K>Tr>d$OAchY98N5 z@NsRxdaL~fAiIXYhd$mbF5wAo0#5RGQH%-JRYAO)O-@M{guaDVgYk(DkbNh;4M!&= zzglNE+@$parFEUsYS*3Fv2L_>GaQ}hexjEF>BPbJx>^r`;UR)wB|KCev_ir~jz-zt zIVpSk&`xNHiigweMD>F^U1d|&zX9=T)j^FX)?=pbl}%E7AHECW=%`1!sEFVe|Dgl% z_=jLbMl_lW`edCL5zsAggIDkDgaPT&e1gjMIx1O$HPPtjJonc9PABDPpGw_9@)x?Q zC}uc={DWMUyMsnyq$qm?A4Zr%nyfWh#8(Bt!Ex>&cUqL2SyWYuT1s{FQ|DNx&cuK@#nkc zA&kd!z~hM~k26E?*tMG2C%=1p8OQ-S=WD%l62`wcd*cc=jN$-Y-cG9btJ z&DQ*c@%IqKYu7g|?);X;UBlbMHK3OtjAOJ!t+0kcy(C_7LpWJfOZ_p8e?TKUNf1m+ zYesgWp%_(5?g5%HMS|F%}jKEK!2}Esvt=^ zQ%cKK%0CzY{qH4gC}7A{y+t8c2Sx?t9exji*Ix%KMdq&wmd3vN7M9@xi)RO6vBNnY z0O9aSkFz{5>LGfzC4`{;x>wM5`Hz6_Iz!RmTYc9?eX9mbFp(J+`7O><55Fd}8}Rf% zFp2YT_i||883IP}^)3$W(SacrnXRK3dH79=k@p}yE4S8nBIL2$=L12=<6GN_cMCEP zx3?wlB8nb<%)7&ROP+5~c|J<}6qziUTz`F=OfPZwe!$Z0+(x}ya-Qhkj=h^Qf3#a? zx8TVAB9;3&ILV|#@C-=&o|e;m<^z!a(roLQY|mB?9DA;H5;8)HX2J z3uYl=Q~uM7r2M1L~m<3pSOLG=d-aj;Ird~aktLsmlwxTbTn=)k5!jvQc-f_XeV~_ zkQQot0?A@%Y^gD6XR?t9**xc-({E6@ew=Fl-5}9lYmw~TAE0D=r}euf;(Pln1MH&E zmG0F&(w9E&V6`5h`2@M=SgsF-*2;IfpX?vT^;=%Ill@`4UM+SWE#VnPD%e4Sk$S2* z54BOhW|Y*najKzVQ+@3L&yYx}Xt6_Zvr`Udsr53P==zR2=?F2n4XR_;D3va8%h2*k zb`MmBmR0w0GqCwgW*4ce6eFT`xH)?{Mt66$#|rPp$1*`+vT)cALrw*3JyImso3Vw(d|p5Vi^N(?m4xXupKlKiR;~PE6(q-7l9BP7F@58BYW?oR zA74m?WL`}A)R=7Q==@ks)TOXVfOpW5gCatU-hlw-CO5Be{|IsjI=cu-pLWQ50;A1B zY#+&(_Q-O)rZf=hl&|2;{{~v#qO?p?jb^m4&1rc~*vwpVoRMcWrzk2T2fr(~XH+xyhq_n9Z% z*EGnW8XK4^wV%{aWyXMER7;Bnf1gKWdRg&@rwZSkyrl$LZ77dUW|PS15MsjFqS&Dj z3(az;X(U7@EsoW9B8akWdiCty zPkt=Be#h$cfw9nuyXFoOQV-#rr~13JeuL6_xG7n;hpPKY>8;(}#XR4iN_Uq0^}Tus zFvE1kf6J|aZ6g{%kSxi}Q>M4CybU9pO^BwgNJt}k zea{wPt0Li{0b6kd>$0r!64CCEuBd`SH2#1K<2p7P7o^0HFs^j&fXM11cZ561ou&Tq zc%4nm(v1{D%fuVjX_ruJBVKy48_2ewYtx=XS&vNOepEdDtHDrd-s zoiC4$3Rv`OvJU5`RFDkOFJrPVRX}GarlS^UEos=4{K**nC7i28RkGL3P|WC9C9;?z zUE%_KLp1`?gkQqL4T;w<&K34& zHnO2YKndw;&}U#|X{2Yv3`R9MsQB~2%u1rV5>O2+>b#_q-7SQ6F%${2*pA!$U-V=A z4A|*88fB9njV9t0%TnRk1PTNkfeM3Xs1U5W9H9_4ku-Y!<_GqURA-`7)AjK9k@5;s zt)if)*>Y_pyFQz_?m!}!j{Fu<{O5f|{LJLd5ADD3*{N{cN1%#V3c0^aCv*hFUHLP6 z{iCNp`hYI+FX!{Wz&yw5-2a7Mz~2P>aT`Us2-A#=AliqWiNQtZ34CgUx(pw~D2_M{ zGwz!t-N0-i6Wc~bp=nr&tM0<4)5E~CqWSzBEr*d0L67)FS+=4|0M{Csc>HkfbR(87 zW+Syy&M0QXwv{xKX$3?Kp(w-|F)_Kns01~iuQo9j&?CAY`EjgJ4b0YbeD~`>Mzv4rmtI*S^*MvYpAvnZKf|?e%Qx zyR}rF;+Rl+x%a@Ars>-yv*gh_*|Z^Z(E!8qU{8cjXK(Q;@*?ds5CWA8av7+ZbN1j| zPpDb{%iO-WA|ys%ivkVlx7cm&Dd($WnQffu_(= zH#>?R6Q%tP@B%i*c5&Qp?3c){Uj6F3lBOPA9ht2pcvCU(sG5-HDuH|=h;SgD4OFI- z9DEzAuZX4^)fXp2aoQn_?JXwKvI z_znud3!#vtX~$OLGu1e6sK!YxuG|JcKY$VW`Q>T&`6i-2Ux{f;li^YL`8w4zwE6z# zPw5OBLpdrq5sd_Twn5*+yz>NFMv-|KFttV2V%YT2IYiCjwK9RKZmgry-24h!aK3PF zo|?mNXs@V?^QS`ZU^)2D2%*1#-=FrMKDaf;Hg{^@gjxeLsrbNrmgqwuxTkvO?6w)W zB0i18<2S_;C2X5=@^h!WAV+Q*I|H}G!c54dG&(6aH z`a^UQzrdAfHIGyl6LmHRLhdy~a5`PgJkiNh#4^MCVx5-2>PwQHNGISZ#dJj%{h?66 z}Pi&$h;ucDgOc1=?X2PgCbEZ}~}z!^Di?d}AGLy9n6wflW`ihQ18m zED;SDsz_ZIk5h}0%2?DSt_~wLTFYTs^VbvL*@>gGqSF&+!Do?1EZGl!t3ckL4u1t) z`sipmZ-XO+&%yW0;YdiN&JFXB?(^VM4K=V*O{`>sdA|~rwP?^piUQ#R5_3U4qy!3~ z>_%pPA&0TXgYPA-jWm{Gxt0Ql>B|T0sW`aMO5L)iYw1;Rtfm~HvV^P*y@l?F7>Z1a zk>(D5*#fS zf{~1+!CM!VZ-aAN*jF@gCKf4_a-{TJrh%fk* zM|&S1YVAEeg&XK5h`|T@>{DLYg%9lfY{b2aUqQdmpM^|sM+DK@YW*Yc7Has%9ebh? z_Zj>;dX_&iJZ|KV`YVv}Mz1;;;y#9JD2y*V*qqw7NNH!9`h0udi7$I5l9;<4=g}yB zhqJ)Tx%e^$eIK2gMsD|N_P|eibZBt@&-MZ4HouKt#BXu4+&xql#KN7SlO1Vc+kzk3 z%wBRQ8rTYEXY1?24E*^5F>O4%sgz3OyA7hXeA6s8$jn5PSUnKuP1%{XJu_woGXAMp zcs5~1$1PP5!h4eC1{M;zlSgMGmL8RL-ikboqh{vVz6oAPW_i$dg%`Dvd?;9}77g|R zmDO}r*DN_&87Vi=<-*45!c^5)u8N9So7TZSE23BqDQcmBWg(u(?K^&Yd2~5w8b}E@ z?u@0(%<}%58i8PZLPr8`j97`K`AMHIo6Gtt6$mVAQ+fo$am{M*8Prd0uA!&VWyl*z ztVi}t5FBK(MR5;}p`u)g`tG3fMC#EkwWd-z;>5^hGB{Umt2=Gm5rsb8tTBahq{dW> z3#KLTff0ceURbFK>7mq)MN#+ftSRYVH3JCTg})F(+80}kOFMQ&QN3$yzK6b#zs{X( zR+b3vqyic<-@GNIVH!CRYj`0_LpCRjr7wJr_%kQ|09QuhDJKGv96v`7OPGY0G?IBk zfLu$eWN*mog>se51mb||kIrNpPaH$}vIy^Sg@qx!Al0Wo}46B`E^W(G+8`K(~`6u_TAx z%&5%5vw5QYL^LGPTSsqMIaA^ZB*0e$lA%fhmW^Vy;wPPb#UO-A5| zvV=BUn~iOZF~%egcFblBcAVHOAtVqUVT(fy0Ybp+;6OqWKc3@&V+@ZiJmOe0@0_~T zJtJ#D2EXsidoLsP?V8(n`c~CFbIu#!GMP{Pd0E5x<1?&veRA*H}RR@?gDi`{5!XhlFI;WF=0;MDI|dy52SuMr>Jcen~x< zZA>Mbmjy&6^wW) zSQ;(%jTgL=4PT$>c7Mhw2D06aqBk9}yjov6S+zWR&Yom6nIfC0LH`CQvX5DB5kqvh zIZYl3lZlKWRKptqnDAi867o1Er(_x%_~7-Mb~|D;mVK_R&{t_&p+`IrnmDT;AC#47z|Kfg#tuXs?fzy^?lbXlSq8=Wffx0N zX7!ln_qZd17;<}TuI69C)O`FNS*LVypZ#Qz^|_N{#8KBeot|W6Q&8<^opx+m3R{n@ zc2HJvvS(LLc1wcSXZQ4#ZN*e&(*!0A{gcb#SXMWbn?c3Y77Yda4cb_zZV$^N`b*(jVAqq3N9Q@lsB*c3`4@QYx!fB%n9Kezc*gqHJnXb=BoNuUOku zLDG1{t0S>|N|uMru+Q}62hwzG?wh?65iGs!je`;*sb)k|G~m2e%w=kH?lak_*V%JJ z=bxDGrR#)e`0l%Uo3ny2OmuwIynM}2N@S`;6L(}W`RJ%J2pqYMY*9V?fJFoEqFSUk zAMUeQK)UD)1(cM{Dxye~E z%Dz&dk>0s)SI+3^E$OTXy(a2duSYQ?W(USHH(H^m$j5gG#`o7`2OFRBY=E-PJF>G% z^mIn2EE?R|#eS5)JI_7V7ZvE?Lspc6$dSJ zyMSH--A{%7?nGufs;i1FsFnas!>fwAsFnJXg|IHrd=T?N$kt!Tme~hWFGZ3OKdet_T>9X?N`uoocYbczeF&gsS5> z@Ik0e*=6nz7LxWIufpy7cB&LS##s87oHbw(fWpZs33Y8?V33UP-Y2zB+;?*N%<889K3*fXM!>Ub(PrpkMBG#$@xE-BevGCjn%)BGd zGH|HC{d_RnCYQ4~%MD+t6>+bf@F{Z7<#jC|9v(S7 zAgJ=(zXQsoub2HcHB{kV3LMfCNqO(2p5G3}|t3Kwkv+WN1WT@jj8mjqSx

za(*i1v0_81P`mbzcx`NMq-0S_s zqh7C57iE?%7v2PG80Am97HB)uGRxyS$|Y#N1IHLy-(q zcq0IFA)dtL3fv?YrYxte&_(wNkHPBkaL(bYwryF=9@q|E;ua?Qo!S~dRSi?tB)d1C zFWcO#C$U1hh+ z^jCV~-r`h6^J(Q~!Z;F$SfT-VP;^JSV!=W#=aWn&kWQtY=XQxO0R4Xg=BSrxg*Z!r zs$&(v98rG9T(2rFiVAR$(~deoJ(FEK#w204oX}<9Ae9m7TM0IBPp+g|7<% zODm6Bm@`js7rVP$vM!$E9C7T|r|BW#KETars|a)Kfb(otk)|{fX4IACEugM5b8@x( zB_}IayNE_uObK=1(DWcnytS4m-g@klREybEadK}Re(W(dmQ1A{|AJr$?_hZdCU{TVzW8(HFHPF*Oo>iwgtvZ!Oo{Xlf9*7gFFn8E z|C{;XC>fbQP2MWpPZAxZ{VTJOy*N$R`EzppUy~>7zxKi`Y#N{>JO5L*XRg`Fu+oK` z5eEfwRdvAeLg2vSt#aYcwz@L?UU^viV?p)3RSf6*!jXy>?x|U|7;cQ~O*tFmynuWa zwu4So=mXNS>3ES=MzgMSZ-tYc{c;S)0bSiMJle@{AFN)?4vWRxDi+(|&#{@JV|D*e zx2J>Dfe+nIe+2k_4dX&yF2llACsw^xbt=y~!g9)q6|1xVfuIs#k(Bt1CMa%xWQip5 z$fY>vYH=$|Xiy8qa{)ON_C>mfGX5?tq{)6sQJE7XUXj(jE?TjWU)4eZ-RCmBAD3;- z2+4}y%Jd{cTJOk`q!{k1M01U-&Hf0TkDB1g27b}IB_k-N{XXjL?&heDYjX%?^?_Q*7Yr5(2&;Zl3{pGHRC*iSRW~y;R z2xSUh+s?Zv^>hbxLjnCOmGETZKKkhqP`@?F8yAXaDc_X#p0a|9tOhG9>z^N9SC2|k z029+FFt=)<)(F4XvR&1zWji+QI1%Yvgj>k6j`lL_*s^&@f$C_c!FFtpqoOLRqt`MX z+enG+tpJW@`*?QJ#3%x@n)JlAtf`pMp5)+YPeySmu82-mw;n^-maXUP=)me)W4i38 zG~MMJ6?V(Ok^)KF@WQ&ZhSyXFs%u+`jmrl|(T*?3zoFhR4xRjRbr0023GIz!MFn?+BX$2NrYu&mGhiA!sy z2fgJl#Y$WK*79grva}DZ{7bi>DK{L`48ir6@qyQT@9wbB6+8Ve^oN#BU8fHG$&3|7 zj3tt>`Tw8~3vUMvqDf|$j-_(cl4TljmKZ#i!)d|%qKRXu&E$Az91j{+r>3O^M_Ywu zDQ*qchtNC7&%gs1qe?J1mKvDp3#NUl*A?-(;$d%YY-6Q3S=_F4nFG^QT{2*1RJCY% zef~6+n2 zIxPc+VKK))qkjl^vILkV$azJYS%|$P?ToEGC>Bxd;FT8Ol?qm|3GTAkb?Y8Tyx2C|sW%YHpn9wSpeY zZ6nj7QrW4K{MMx?CS@u!IQVMS*Mxtm??h+M8`fIv1(-mU3|&cxT9M#Z!h6&lUpyv_|D( zQI@o{KTycV4cX-lx-EA|)_*!AgZy3RI(--+) zy6o?(ROsRGvem1nD!MF*f+SWWq2^#S9L&m}EssYuQ47F;Gno< z%|tSyzQIgktQ0_Gy?#XVYXBBSMUD;>!kRA?%8!QxkQE{bsATU@rhhc&4n^q^*ZQSH zTSk%|T??8zIwnjDF@vnjl^(QM%cfs5;<@gK6%V)q9+(v)Y|5^%ZAJ_IfxfO@NAG{0 zJ`c3Gj%B_@SzqCFs4+Q*R5rjfWUb*XmMP3JsWA8%VhP}o5wtVFO>(;|)8QQkhg(KR zog|P+W+p;V1lLj7idl)O?eQpqpe<2dE4r?@(!#Z)T}SS1kgm5b>KE)k~lK2N_6bheE7Op)iI{@zB=sGnlYJnk~LNR0&8?2hvm=&Z=GF2st| z^D?^?I0ffnEx`-$FUDBDdtPplxFAz<#%#_1QrHS(?(6Km?B=qRws#(~4J1r$6*;TT zT29_Xj;pEeDK2Z|%eW`R2wO(bh$cNK_wxn13DLb;ENDNXAG_vVr#pGI(;^hMR zFQREJ980)?Eh=sW1c>ZXHIEjp20^E>%05vNR1YW)qFgGOSYcE|^_wjP&FA;2cHWk4 zBVxMCA;D{yewodrDWdbya_b>Wlw2OK@M_tGc369KK<1*&FplJ+aBWCt)R4=Tg1E&Z8s4O@HO;nhgu9eDhMZo+*sFw|I+% zynD%wRm-nS5~ybr_r3f54lh1U|B=y)pL3o(UmF(Cj9f&a-P<++OrEhK8cw~^4(-sp zG<%?YyR0ecZX6qsiz5BV&@$Z)&4|FEKqAihqNXNvT{=itGiuS}v2TApA6CT$vSIOW zKM|6`yIFqYcJ^y#*}gz#@$2cM9(#8=blK^ zw@=eI3lH;b{K#EU->xxT8zUYYi>w6Bs_GPg1a&q^F?yY4JZgnNpcGc5vVeS)0 zqn0Z6?HzZpLXPjfsWQ%AZ1dv9`((I!ZJnn`6m4F7K`7zf7hU#=*(y)d&w#whF?rFp zRbo^H)(NHdd90J6dfRwGy^eL_wnf%PyPTwTX1Wm8FIXwgHX<@T{-^YW@D{R~-M@#+ zYNrpi-(dgn%d}Hic+0Q9D_HCAY?Q`4U8jX7V4gE9LkrSDtlEj-P#u#(ZlayB6Yv}x zDrWqb;SsPP=axlfSlhTG3&>S?;`i?bzE;rheXwW!(HZVny6^3$?>l^w(&I;k%#j=E zjfd%~n{HB)Syhw#lg6z#PF+5pk-|wColBN}`-v4BwBl^Jd$vp6HaWFfyT5m1LESzD z&js^cqYZmCSxOweVyf!k3FPUR^8hmvPE?(|2VSC*ejYv0ZVmDlaC~@R3y%9rJ9B;K z3_MuiTaKCcOzb;FS@t9z;76YZ5Y`hg{DzgX?Sju+3iz@v*-+Jlr5SQ@cirFL*XF{~+2kd~%vc1A6~dBU`Tp(2 z%^NRCi3-A4+X{C33oDYzHN|i>q^pu>xm$19aQtCs?TpSJ7KVg}c%9s~CmHKjC@AL~ zw;LNVJi2@qL6$?npnSqZ3}!owJZOi60pNR777|7DaV536*R>1O0N^$|IBZ3&yelue zBq?BrN0S29mZ6_SG*;MJsqC!&KyU*blN1KW=5BX!z40{tUE%jYYs9E2)ESFaM^7Zs zcbmt0un4~i0s0dP+9_smuX9<$i}E5xxv_ zI!fZiiT_yaA;OAZgU;Nt&@Sk7wmG;ktx2H)OJ(t|i>6rkt2T%|{by~2`?>ZUf37pb zImED-@D?6NXLcFZO)u+O^c(Pqg_+*U=|ME1?~|9sk=+Rwo#Mt_?R}#2awLUd!!D*z|8R6`xNx)46**p}EbS zfqq8TI)0STfZVAvYb+1sDTq@}+0R7aQFA-vmB#duztn z^R=@(Mj`V4BQnZMm{)ryg68E>`=IxLZ7ZMIR{&vRj+x~Wi+X$H% z9+Z(Y9PXx>>utoG{4Ic3CM! zi~IVAi1EAU{c@$Eei?I2J39H;{PT1J{UPA{ zCf`ri#`sQrC3<6UKP8T>O6_qvC9oj2Iz!nxB_ArG%a5t-+&Po?x07dp#u5bP?Kgrx z*FgW8L|@zhQU=QGi?1x?N&sM_yX2TDyHq!5b%lfy77O_SD);BLkSs;1rTAi9kqwn@ zic0zyB~UxD2461ST9Fn+4>PkB(wdZt3v+igh7Gxt5I%+p75*#lt&^(?_fNh+*V5-m z{*2h!9AD{CC(dV#89Xsqv6pFdK{nbF<{%wpDEkt3#Qjygo2AlsQYz`OlqzbHDB$yW z4)CWQFbzYP`$DqMRzyuytPs6d4=9Q%iVm-wn17hQkNz7-m0lM88bPEJR2vGj>Ew~Z z7Q!PMn8*bKB=TxIK!Uz+fTFGWJGy38QdvFq4jQrTm6lQU*@B_mvgQF(R771~HEpV@ zU_M|cFMIf?EqFcAxx48m!=-Oob5q^$s_mRI$AK3hZ6IZYsxM0{+^-$=_G?DCG1-WI z?j-#_xs6=IpD*t2zpUz1IoMfsa`GR8*R54YZ^t+z^tRQij?RYqSqXR|VOm+U)4I0P zI>lSS3&!X|A`<3_NITnaduhj`kfdRUd@9ba!9{Xm!tqovMZE3X9TU=&$rSDhZw+DG zf4J>}Lh=*n-+4{T1=M6DvN^9O6_1%#SQS;p6)?k5)vCs2AWT6?B{2Y8f@;WeMH5L+YH1WyciH7JJ+>%B>rq9KT?(dqi{=#pK5@&69IHoQ#=lQpif$OX zRqCw-z2*LDFxCUKtf_EW$xhE^WUq3=NM6t+N%QHCDqcA=o7Vld3(&U;JV5P<1*`pK zZ}*Oz4oqBDTtEfcRe4R*Y;9Ar;>RdlSqu5z0KaQmepObar0pADMxQ1dIyQg=kAs?Q z7xuH9&1{Sy6~2pikbms#-Djte6Eo#`icX59ir{RE##wKG)QOHI!@VB2A!H)iUP&}P zR&T6OjD!FNskpgSg?@SMNFW`T#H8Q$f>I5nC#bUM$;CaO==y_tBBXg$fF*xwSh8gZ zg4gHi@@ET6&%XMXbK=l0ia1hwNZU`fFUM2!-;}v z^9arhJ^)dMv<8biEjUtMg^3aef!2n+CUyI424tz3jOsG_kI)F@)lx+pH~MN}!3@CC zrDh^%ff^>fKT**^?_hD6etIKGhM68qR{r0-y?jLtS9@K6&N$;?K;OST>7%k0;8aF z`Nx05$=P^*2l>l2Yn#vI9(R1>*Ml6~$g=FA{OjU85d9B`b}>h1o7NBs&%(^QfgYV~ zrro?%8<2C{_kAX>LmjDAItTL_tdH?rI}S5k#5Q3cWPcmRT-y&AD5=O0AA*{m%Kgw)ld|;o2+SqAUQsD`e|5y6ATZwGlrnt z=&w4vRUOpCShW!3euf3mkXHg2sp+U@nAH;}p%JbRUaAL?Urdn?%>;n<1a4;E&3&0XsGnk5iu{ zYKMm8u8<4JO$ijOeUq>XHJ1wC4&Q35d~ElX)HHC;sT#O^oh~X;+`<}ylBi-}R3Pfz zm}rKf6MzSy!^7=5Pf8-7t8aw}NuIWb-1bH2}=dL7i@?!s#J_HyNeb*-UEV+M` zubMt|AusfKJ_C?z>@{kYfMkt z4)lcQKCOt7j;J=Xx|^zV_gO(UC$;%H{bTyiq@Rql=+Z>Bot*^rYmp>uCv&OOsttrK zCu)zImsknN#kCOH8!?zqW9ZC4T{&rk5=#|X40w?UEf4cW#_g`jYJ8v=9||X{?lpFI zF4U;SDqgeuuO(S@4;1M)BV8WZ(Au8GNORNZRj1M+kJosQEBIqyW#i<=>8LgLl;z~0 z%J4lSj_nC8oaO3EtFmJ#;f3jd@d?9Qj;l&!4Hr;iDJ0sg>=U za5O%Jj_-D~IZ)a*0JmYm?I=}4?quL;%-i(Lxlkc%dBRm6T@lFZ%JlNd*S%jH4ab|F_6td-40d20}(wKLe@&g5Kakuwn|mYH?LDw8E=#?K)!a&QJho4!L! z+x~>+GrV1vua=R0VKZXt>6wC+%V%ApoVBvL-)-eAGj8cVT{XRC#si=&xipa_$c6 zD2igypQL~aB(*ER&;m8Ht9$4QQ8bXb$Hud-dw?Er#WbCXg1P4yKPFA4okXjSjiHY}CzU-PnP=rG1Qr_xKq9tH0wAxMD|<|+54E#9acNJNQoSpv zilA}!#k@ma;7wXA1CpHTs>hcz71;=;Caw* zXo)yz{S3=XHb~gH%~X~HE_?#Q4OEu+!#E|;e>Oxm!A~fP#D%S3PJyKDJPQ!CbN?Rb z37vU@zQT$*G$YK?pOBSoJZ)<}?sdrXwZm$tD=(${LyJcbvLv9gM0-VnYHs;OBas8` zk;qyi!N>{_qUH;^WTYuJkmXLl)y(Q19LS4K(1WhHf|Iba7pTl-g(tcV8E#Qh^ zh>i!FvGjnAYHPplf#Fd!a)L0S{@&15-L$$wqnR#uFm{zE0WvagrCnZ+#|=PO)Pi2R z7o?5^;!%+`!GYkWuS;I58O8GYp2_}H(48CZ8?PG1Q<~#rq<;ZpDzHi#S-##!_{eb0 zPqP>=Hy@ebk(q7J8;;C=vBhkWV-dK3>|EK=O_|gM^+g4N11s){WWU{I1!JyIGnT5` zAo)@0=<(rlv-5^08FVM|p0ufks10a2DVvy(t}DjEhT!Kwz*nB_ndndHmItT6PSADX zv?U0BRn?-=jhZp{St~ozJ6_ez{~^g;up~7vy|EMp0=E@gQhe?myLZk65}HZ4FLE=V zk0_rFXC{8LS$GMoF5<|y> zs+962rnrpFG_NdlmAYqkwknlCNL7r1$;?n+Qu&dbK0OI4Fb%%@w6QGYXRGZ#@v`Cie^BM1dFAaV%>DZ z;bg`ap}MW>R!Glgx@O8vjk*AjD>{8d0Mz66=tjWhiduqb`iy8JyQEn18K&Ee>!$8< zr;~wfh`Kz6<_@{am1d?Ga*1NlPX!hmPhX(2@Ga1Mt4zYloG&1Mz=#Yd-UN5@i}@y~ zUW{Im5m;jQ>{B}hVA&FAgL>AZDVj_{m$~~K=LAgvVswu#9dQ9S7lM`+tXJ~txlR(S zFZsX7+l6~O`}Vb}<6YxLHc&Oh-525ZrukNLi6Ehsw9$t?AO&=rUcJ3>|I3A=H=Zn8 zhM5Rtc3rzO8T9U9oAy(~Wbw~EAY$$myuyrdLU=@YUhEUMiMNRlil3DvX`A#3xhcO# z@hPX252;P{LG^PQ(Tdt_`k?+nqiCEkK5k0psQHljtSjca!u2h8!2Mlok9EKGtY?eo zYu=joI`60Ln*C$n?Y{5&*Zbe)|5_jy7!4c=JRe*eygx)j`$GR1elYxtNMGdks1jWk zeLR+nT^9Rnyeobv{*^>u;th#slYfv(rQVeKWjc^vlm2-6N13ZL_hg>SZp+@AeLS}z z_nqAH`A~i^e<=TWSH0`Ot|tq36~0?^6$gtq79T5qr=*nzOZ!UiEj`}7rTbewTY8=^ z=gRl@HhaI-_h8>Q`d9Q{-T(1Qt@8Qmk(yL{WMFjQnfhh*&o!1b-q%btk2Js1%C&A9 zY!2Qv_}tKvp?3^R!}pIEBX^BFKYDER*|FW@WcPk527gl3VhorJE~{>+L~E50;6Jbmx<_hz=uJhSrZl|Ndwf7N$apIW1=dGL}Im)y6uvi8ok zpI>LJTeI$K>tpLbHEYZc&+eOj%j|bHNE@0P?%LS3@xD!&O?Pf~Z64qJrp-@o*|z18 zEiY`{zV*><&eH?|ww*Kfan`{UbR*iqhb(~hrPw(7DEUG~)FgO`8e z3hRm$SG?g$a^+oD{&;8K&PR66T{V8yhpxWu>R;}-@TnWSZ+yoq zrB}Y;mCxKXeA7cW&D~tS`Tm=qy(L2&PDW^sVC{rY0lwb?vTthsZa6Mw$7SreoE=xp zKL^KY*1m)t*Rta}c3jVnv+THm9XHP31;P)AP5yao5Q~0+w?(p+ew()?k`VH|EtB=a6mKgeCfv>2D%mIeA#ZCWC%SoChjx{> zP0}rH!MNX2pTS2TmM2kXgR#IJ{kda&M+y zs}1#KZn)uw3S5ZS!NnU7?K*z^z`@MAy?YLvJiKY|wI>hlI>!IB^OvKUGcOy(uNpb? z+_t^PjvqLBBr{MMs11y5*qo`=1}g9x+kf=LHAjz}f)8qy#xVaQ=WovZhRlKEnO&I^ z$9C=6dwAEegPEiI&K^VMtTShwnAtQvoym0XIeKK*3Fy!byN>Pc*?;21b)(hlnL}q6 zo_QVlBa$H}$bPaH7V2&|9wB?+%r0^eUKxOc2gqS~ogl~Hm;o}}MUL~|t|uqq`V6U) z8vGj~ebByv{j2bAY=58g%^S%f_SeVZZx6z?>)_lTXrF{TY=ZW+(84Q@z38rI-*XiH z`a<6}+UeEE1+Uu%?PKhdqtMR`sE!qQ)!;QoHh{vFVc&sgYy9kfxbg(KhFyP(|3wX6 z4P5$X+-dQBF4Tt%ydP&Xo?&C$1)uDN^EeN9?NPGtqMk$LBCh$>SIoeZn1+8D_}2|R z!Ex?_E1Vg+f%R%H>4AAT0spRpnXbaW3qAFPZ@kd=5rJ_W0z9!!&hbN+z)OIuMBpGQ za0{J$40wYJ_<%(`#0%Wa2WUS4s5t~EFap|P43J;~uvQ9godfN#3$(@}aLI1CU77R( zx%NXBtL(wn;Q=;*-wpy94+GJR!lTCQZj)q+EFsItM`7xwVJIufD!|Wc$R%VgJjC^6 z7RY2H3~)2Zx2-@u&$sObqMg;a`ytQIWi! ze3tw*`3QL*`D5}q@(4VNyUE{?zbF4l?jip~o+ST+{295Iypep6`~mqkd6@h+^4sJM zfEw>2|C9Wf{0{j7`5Tzt_X8<@fDz2+$>ZdUKr&w=Uxx+tP4cJYY4Qa5 zTX-zauazK$ckIZZR9oN z)#N$yDEU6wN&b~QOP(VCOwxe3C9;%U4G(bf&o{`O=p+Zbu3 zq#+un5gMg28m9@0s!Ez>XqM(^o_5g!Ez%P0raiO_N?af9r|5#MQBYoKgEnc44$>hy zOh@P_D1PHC3&0dzLYLBIbU9r?r|Aq`NmtR;bPc_PuBGefdOAxt(2aBx-AuOt-E9R4 zCdri}Cl4L6uG@R;z|lR|>^*W~@3Fmmr0J8#j>;?c9y+m0IlSwdV@HpuCyyMcH3w?W zdu@Qf5AydR=e@DgnRak-?pCd+sVbhIe%a0^HAsg ztn+@?d4KDCJayjhI`4O#_q)#fUFZF-^M2QPzw5l;b>8nf?{}TgUxVMj!SCB@|IY8< z;P-Fv`#1Rg8~pwae*Xr)e}mt@!SCPT_iym~H~9S<{QgaTza}4lllP;^@89J2Z}R&$ z`Td)1I^g$j^7}XW{hR#$O@99-zkie8zs0}5)nd#OUIy^NK0*)eI&lQJCss=&^YiQz zdT{SCJm*}8yRMxW2@PJ46Q_Q$1uH$k@H?V+#O{dP5w|01N6e0h9q~G%b;Rn3)Dfp6 zN=J;22p#b`qI1ONh|FUQnIkSoRF0S&5jo;Ur{;%!9Rh_w-EBhJQrP$;6w{{~P?0|XQR000O8#GuGJY~r*@vUvaikmdjY z5&!@IQEXvzb7^ZrZ){{=R6;IvbY|?lcbFqrl{b9vt(>c*3W}vHb*qEArIysWC-mg; z%y`E3csw|aJ>!HUIDie<#zY%1X2F0LmSisrOBe%Y2QR^7Ff1(DBx5jOSYUU-nC#Ja z?k%ai#|EC~ecpe*Ki;0Hqmopry7$E2IX57T5VC>r2yNN3d*6fad-UZ9{p1A*VRiSu zM)Mtiy8I$Ie+gW5&S}dhoqoyuGe3%ua1@S*&%5-{6+el6Z!1FDD-n{+^RBrnk6gGC zxd|ZT{3|ZFR6Ov8E<&+4!#$pP!6nZ=fBrq)+u{2RLZ8(yJaXvp|4iL{9KO@x`p$*$ zfp`;l9lT$L_vH&Oz3N%7dvErG@cw5Aai4qok!M^c{7iTjLf^OHv+5<6pLd8n@a14_Xs^whUT4p z#WRjvaSWjmXb){Qk0>uN+VBbW+n+t#ntvQA+|OulKC}1quSc(sUe>#}$8#Uypub2# z1Vw-FJM!PX(@@Ag3g2(%KEm#__U||Bli#4bkTm!O{HgFe;0rtF@KN$%B)}&hCbz)* zmqoAm`>2WjBwK}1FZy&tB;@QujcgX*l^xN=C^p`xs4tvb&(Bt$wc+4a4 zN{=s~3a+3e{xI_JXV5tL462e{Ahmx-3A_Qd;Yj08pmF>d8b%+7JCsoxUNgW^6Rr>N zOVJ#>X5gs6p~Eo*hmVhoxHH%V-i`2j^dbzP}C*2g&%o@VhUf3G{W) zfd_l{;yX|SeGPbW548JB(63JekG>A)Uxn*CsD=;2J+}ZKzJkW^tB^`ALMq%t!ncF; zo{2QvL>B^oi_p*Y1Vg(&13GmFJo7a8{ilF89=v+Mr|%*Ke+`bGfet-@%Ak=obRY2N zG-wmO_LHbUZbwb-QF!)Nu+H_sx83Lveg!%WA47Za2)Ynof*QcX^Kl&Q#7EG2ZW0CL z^GG2FQJFJ=$Je6(?%l=T0X}{dFDwCJNGwADMXfxRW--n3*YX_ATz7=@B z<^VYjqi^?q2FLf{_&Jl&Jj$~JWYvF7Q8|$z)MW?AioP-#4dnEizjC}5jvIhqcR>5r z93Y<|^!45!;P??7zh$!d0`%$02bIrPK+bCqDjRyd0L6fZAxZZB1Ts(Jn|qI;U-TZs z4}e@hi{|lndynH6f=aQ!jC}zzJjXwmmq`V zr~pS1pNCFm=fGF;7toI~@EU9fjud$X=-Z9^S=^xV3ALJgAk=%O%^hg1!A{D5HG@v@r zzaRtYKuu%-wU7zaMpp07$U!#H7;=EdQ4FYy;=MnC?YcmHiN2Z0_&PwV{@9YJRTJs+I~ z^a4gNL}&MYf-XYm0KFI;0{V1xF3?NRdA%Q_OVMGVm!Ts-FGuGCy#ig(djwsHE(H1v zMxTi;0(up?7_9SZ^mL%tpi6*Wi!KHFEOc4#hv?bpa-i3tD}Y{)t_1oV^o-ua=(&tO z4?Pp;4d|-g4q=~xc43O zYDQm!UIO$E^irU&MYr_6jb4Xt1^RmQGN5;&mjnGr^orht=nd#Lpl?L41o}_tcA#%U zuj+jZy_wOspnm}RR`hD1Z$q!?eG?-89YF6!uLb(g=ygEfj$YsU2KpCtC(w7Ge+2qY z^ah~+3iRvfUFeNK-;MqW=zGwcfW8;Kx%V}66ukxLJ?O1K--q4?^!@0r-dE8F(A_}q zMgI)+gN*(gdVB9H=otDJpdUi-0D2#KC(wUK|JwUc^kMWapdUf+2KrI-9-tpX@9ljV z-H(m}{W!V@=qDKcBzj-(OXySR{XjpBJ^=JH=w6_oMIY?_2l^cPH=v(K$ACV7J_Pg& z=)T?;(HGIb1N{$1zl1&v^vmcYy)Phm8qlwxj{*HEx*zD*(8qfZfRFzK&~Knm0{tfX z6wq&>Pxn5L9%S^}=rcgSgFXxNyXbSh&!O+3&jbBFdI0D{=nFu9fWFxKEP5FI2hbm) zF9H1#`ZCZ*(0}$mgMQ5DPtaF@{uF%`=%eUsy-%Z`K}7j9`Z@Xr(Emc;1o{i~t=^~5 zF98vJ3jGRV+^5igqwfIyAE2K^|BJp0^w;QnK!1b25A-qgQ127yx9A5zA4d-Z{T=!t z(BGpU^*)aNfF1$*NAzQ$e`55{=qJ7V(O=L{fj)sA1$rF)toJe0Lq7)!NH9A1>-s1$ z1Me`!w}22HU})}Pcu(L{9`K?Fi1IA(|0?)>9sIose%=QE9s|GbfZRj zvR(jRumqlRJ$T5C;2Af8N8AFQa2t5Q9pL#+0*|*FJl)CQ;r4-NI~6?Ie(+=;2S4@z z=m!FOy%lWp6<}$v2fMioY~gOOig$oDyc6{QJ)r&X27P}YX!|{&-yZ;7z8AFm7-;ZE zKz~0B8uB3c!Eb=>TmZUpBlw6HfzP-CywG>R*IWlW@;%UvhrmBl&W=SAQjUjlyR#o$$L2XFEZ;6+{oe&cnZ*>{3(_$TlN&jnxk4A8It0zY{* zc*b9Xeq9Ur>eXE+ku^20+ieYi2cuw(1<$*u+{!Ds3 zrZ-f|^B}pipdZ(RkGdQ9{vP1&ebB!ju&1oYd+<5<3V6zM;VGXX55iO8`9waG&$Fk@ zta*yF@|45uDY(}IzmC}P12`U{%G*1J*8KJE@9pf}%+7A?)q(D&=TGDY-#@(h;f05X z9!@-*`N7#g_}D{mr-#1y&_^D+_o3?^diHli>1)Ed%qGr-0JYIeF12} zC;GSg16URP`wKg0jwxap>)1w5`osK_KY_oq`j!&u`ao~b=5Gf)BEu3KDQizJ6znq!^P|laDyk{B4Kn#{wP=7bNb1rAH8uoeRSROfpnpe-*oi; zJ*OYNe>q(^Z~(3nS6T|M&%G$oZ?gn#7VB^#M|atG`q6diqv*gbw?ywsg`+p#a!dLa z=tuwkF?9cvKEUY7AFS(t0GyzA zWV4xan4f3QPvXb1LB0vtr3=R~WW)1gaE!t+4u=3o0ooHcxzbb>n$(6SHG>dmvB^on zU;Xp~;&`seL2C%sD~=|r3Km3}d-u^_7@C0NARL##aXIDV6mYUV)#^s(!7t`X#Vws?(A}9I&5K zK8^zwvv9Xy@E`BWhufEkK*lE2YFC#OUQ&g`QgL`PQc+pW2QaTym=MNJT6Zd`KwLNzs(1qN{% z<*yx@*~!4KS2B`24A(xBPpE2E=fUViIiKh3lo=0gUgESkf_Pr-JwomvZ-Ts_1gQNa zh#izaP2f)%_*0?U)eH$RsZ<;%_{6tyINFOsx5G>*Fa@b4mMX3v@eVkG*PdS!t9Glo zsj|CuYrb9ejvtLxrYpIrio10iymz02P1`iuoA>NmP8f<}Y9`Sp*6rN2BgD3WN7Csa zmrka_HQtT?`Ih^TM01ti@!> zP-A;9z2TK zPHYDB1!Uiv?Q2bmnOY=pYEI*X>Izi|U<*~0NMBk_YOE4d5zh??y?hb3=W&f|cDn2r zp6laxiem{$iJMA1X%jt_kW8^Wq;h85Rb)*wR3#R#?w@FEok+O(Aum@-jxJVOiH?8;Rd!<|#dKqR zHEfSpt@^3;LZ{*;U2({A%u+3}v}JoyleOj33X&ure`7jdJ++jWs|?M6EhX@8!1sR} zxMBgPxW2cIQE%IKr(Om;3MYvY*hVL^t6IAu6@5jF#WB%sS9Ri=FBib4VgkB(j}s6f z=W`N?4^{GNjFaVsjiAl4DbW!ijM}}eI3<e@XRI9z2io6F3fzJThB&^IESmnGv&I zY<{L*?JlK)BsaeUSGL~p^iwkFys~`FKVN*_o37p!6T)EA1MJ@E-WG7}zXrL}dy5Sk zP2k>O{xpnnOw7=MMQ#DVW!r)ny0N*LdS!YknMjh%WZA>j7kp)1I<0KD^34|@ z1@CLVC;{i_McqzFEG$-Q_i1s5GtI-2-j=l;Adm7LTwRWn%RHsa^cIp*PX6@Z5NJbB4x+Q(z85$jl$)?Ji_V76+crbDO zA3;Rp2w$yWUK6m$hjotiE7SWOpyaDq1qL7}dj^f{tpOOyEY|{zQ^TcHXE6-3m3+hx zv-df?i+l|9h3bk)<5U!(>3^=&iP83S0=z5JPP-$X6xRhk9ws_Z@9>(NERkbp9y@F6 zE4L=cG+rS1UVPaHb%Su2`MIF?8+9I@h%;!g#<_RFv|7zY@MPlzJXbJP z_oj36J4c*zFC5?2OkZjw3U0iRG>sr1A8zBJi(i;`V$Ho5O~2)>r5)GoUVnN!2&T{3 zK7Vr4dph)KuJ;7Kf*b_>FR(sEh%F9B0**HDy@C>`9eG)k^O#$ueiIr4|4m#hfu{8V z4EzVRpe70D9Eb)C45F!|%VvVtK<5SGw_1~N9J+!gf*Ucb)ld>-ESO4iW@=QI9kOfq zIk(=n04@@OZ7MEs7~`a5NE%$vku$z}HLof%R>Ao?iV}AWj!1o(_V8PPf9oLQI)+qe zdku~{9AnUjG^!(&^&yIlT%V!Riv0X0M8s~V*@|#6sMkQj7Qu%>U!~T1Sra*>+?>yk zud6y33#F7kr(%R}7VwIW;d8WddSSV` z@!(W_O2e8Y2Q%4XryK`C2wU26!T`h~3CTvq&sOu(z9cJ>z-wo1Y%G*b9lyw_#LZoc z-BV{mOI3Z%Nt=#eYlk3EnX5x@H6jX{E32tOEl3T=Z9!E7MMw=NYh#{2!fZkB{QSHu`WR;b_6p0S+v|w{gaS<^CD9;7*%?z8ZiZ=w#D}^ah+?7K6WHnNQ$* zz$LLb(ubENFt7GJ>9j#@eTuUm$J1AoER)E&+Su5zZB!jSE)f~}9;;{|DPHtE5!68U zv|(Ss*sT_`T4yVKbD0AElQ%;%zPM%Qa>q0*4$yeePMke<79cdRcv-enBDvgh0b=3{ zHCI)2P191d^Rr1VJ(rGr%n-=?4zR~a-dk9Jhr0GPXD6~J5qqojO+kW6(RUusNrXA`0>$Xe3U5?a`e6Yz7qCm-?PT-q znm=Ao&-zYL;}k^|IFY3Hp0+!|V_7CK4qR-0O9|6#LkudN3gibd!4z?GPd5ydv?r*B zX$JM#V!I6>8gk0S-27b1&S_NVX9bzCe*3-0zJeROzxc8f6*_@qI2@iBl(e}YIv%z@cBk&S4WTpe>6DfjO|-HQ=l1q zCZ3Tr%oSTp#n!0{rE8%t`QBIY8_0R!b0ns{6O%#2c(J%vG6k02Kmtg6QU{dLowq8( zBc5NfQ#+>$lQCYA5@P`d6XCIvuh!jc!?enIk0gc;o_pRzb8i*XtVAn05@&)d=5V<0 ztRoW}o;dOppP-=2*O=IwhkF)2Z0X*+~DapMg>4QOped z>{e?luz3zs&N2aj=1F}Gt44sCbNxP&j?#-WN|n^)w#M|1aW`hAr$>FqPBk_R@e*%V zXDYF{sg*s)m!IPllC~31)Jv_Is^zFzCqLB4=bE^k-7p3zL?+){%7sm@R0)%ABT-u% zOJFWl?}XXOn3=P*nDA>oQBH(KCy@*mtIan4S3k5gSM%G&`p^*5F|YSAkjpt}GmEe^ zvtOWb-|xq+sKL+$&>Wg)c6~SM+Nd_$3kKict4Dt4{L&bPeGBYwH?tEc#eYv%7u8`k!@s1;J>&DZ7)@0et>I6sm5yLY01At|`6*LoPxM2m& zI8O#P=~{911Yfh5%VsON<3H>}9l~%Qy=MXJkKWVjXPtio`PBxR;j;;q#} zwR)^vs#Y2A{NC^J82JaVj~Uj_8bdsAKc)m=0DT|;fs|Hb1{xyK&JGX+w^~4Zv;htu z3u(j2i-IA0n(SmjsVjjQ$hm4gUh5RB`lYm*7F2RceCfil;>dt*wxYe#X90V2L<=<_#y^^2TC&V`0RzrizJrK8`b~ zV9KSaK#&E@d6_V8dNL>oXKWd%P8mAoCH%>PkyLku4B{qE%UfIjBQB) z)8rtW3uY!rdPOs|&6FXVvIKfJp^IMYl%;1~@}|4@T)sI+G=Qk&u9!hvMEsc7@kq4| z;f~5f?EQt<3AmXLwU%*U=nUo!z-n?QiSX`Wm)i;MmQX{;`3zGIbEvdPcSR(Y}M(NZE9Oc_ny+1>~{d!bF zZM2=`cVgd)^9_n*^GJy>`us}QS5P{1K8ohh(S8|lo_4uZ2gK8%;2ro>q9!^3x>$Bp zZ$8`F+VBc-K^HAk&m`^A{Jx3m=IX^tN*)D|70;A?IU$P%Kmgv15hsQDv6AS^e+{jz8m;C@3{yJbKrrN;wJL41!y3GuB9r*3m#t6L zTmEEo!W%oOE+}}x&Q0V~In9%krEs>9CAVXaknw$2ZoJ@?6K={9M1gF~W{l(C=$_mO z1rZwUV=FHQqbBPrK8KMG+ z-=b0aO=isv_v8e}jnCA_c4q;u5(Q+dTYm)eS0sNdv+EcqX}132-2~V30O~ow()c_1 zbs;pFCp>o3w!-oF{&da}2JO2N+85PDjJ|9G#vp3C5Qe%1*0w<-vXGozXi_i5YgkS= z7;%QFWfNC@n25Tg-zr1UlZa86N)QPY3D+|`hrgK0JHlt5#;Np~NzkVslDp7G7X6Cw zOc#!T>J%#wERtr64VN&M)TwE-S4sx~9h4ARt$f9Y1HNg~((wXxQR4N=a9IWG&YAu~ zO?Kp|Gd5JF%cb+yG40b7QCBe}aFzOL=VTfS#h|39uC3!sAjc7CaLK>&RT7NZ%676zxw7c=!nDTm!|SKPl)-r4iMbXhbhm z+s;asMvBy~oqpMm*0Y#p00!|Kh||zre1l?*m#z^NUiS(Ks`To}h%9_lQ->z#J7uJn z$UrN87fTuf&y^0g@uu-@Syj}tsV_fJ4ivt8aO(Jn8k@432yL6_J%RoS+E!%pbynpI zA(G}9@WR^CgoMkbSOH#q%KBx6#Xh+{WQgpQQ_X8f54FO%?9+xCQd}*TpHtjaEUwR; zo2_alJdEml?|*wAMn7bJk@7-ka*cSxbe(4KE4_y?QLSmFXr2$z@eNyAG)Ow}@};}?u=ogK9tkqAOkGaad~(|^JbLK}9V zR=i6=vMl&2J~BG$di9~Q?G`lXooR&yO-<*^kV42< z8Xm@~XW;MD*QF%c$fUj$8;VUHZh-64B+1}3Rl=#c`PsBTyM9Ahp3^jm@ccx=pXxL{ zFD~4Zs5)bs5ar;LpqtNFk;kAOMk^2{>J_OFRuY627Zk*Qx;2`8d4sIrOYGH(v>`9;JfNQd>aMJj=$ClyBlxL@C!DetVp{jkdul0A9U8H!8IT+9-AF`>lNbSJ)a%{V8+4Fm}jY4Vvj zzGPE=L!=MGV57eWeb~*aVzgGf42RcPt<`dn1)BXjk=L)5)6q{_;RByZn#^ggC@hGs zfmdV%*Y!g^b+`SWaM2iI^dIkz6)l2Q#fWDU6jL}-sJUYaBV`V0;Aa#X3kWTCiKdrJ zP`FJ**Y`!ow;O5M01cP$-qPlL@s!f9iyi>Xv5V)9Zy_-vRCd*I&Cqyqy#w2Kw0Lksg<(2dA}&cisaqZ z@;L;>SL|KCX=hOD=GO0i^@c-Bx#z!d)42=T7vtsUKkvp{Hl4dJeE!$R_usJR?H`&r zaKoN=zjxIq+)3U7-qBivTO0IAwEn$XqzCUDJ!YjyPoYxZ=>cP#SWgD+^wi#&u+W-M zseW9y6uX#?+3B*U$7QT5VVDLsUSseIPdO1ZcTab=Hv*0*aYsP9;6y!z|jmGLl7eLTYc+PN_LEhs&|1^;=4_3CgDd-1aLG&uXj=LhenK zb&lg@qm~MH*GKmh!m+%KXLh%Om)uA>mzvx>0(pa@W7mk)ebJIq%|Hu|KWllXTWJ?I z?b^3d$^f)hZPg(}Q#Fg{tIeh(%^tXV!@J*0xxD+@Q<|ER2wX@TAp0YNNM3N`Et}6< z%Ho?}e$x#sPp7fz`=Ixm&_3|V(fIEu<9<{R?lxAvW15YEL41S=Lwj2*RtLpi#(s(u zqx6FM!6u!?F^8M}U{*||=iExF?*Vi0HznJ!Jkttdrl6*Cr5F)Ok(cCB(WP!i)TwF` z8PKGsOo*0v@Sq>&)$F9_<&^*Oi_Ia!mxr{tt3Y~$Z7-#JQ&TMue@BliDmQxJ96l*M z1|W+LRxvMzg)`2{%`}Q~{a~#+t>CTl{Os&3#=pgpF1mmPp8>L7W*T2?MEE353#0vd z{t2@lR>o{jC=0Jnqv*Q;THgBR(z5I*s-^;Hrqdius~*FgmDTg%`R7h;GA&z6F6DnYb8+d}Tn>p@BsB&AX@;yKG1azJyUi|4!TSx3Y?l!L=~Wilhrkq zAcp^bjeiGiX);@<_=jSEVX9eRDi9Rg@u<9FVhRBJ=^7hGSp(B(x{ecLV;Zk27&v3N zA$UZ?%s+j8ZqH;&0G$E2Gpg_qKUy0MnQL7)kF8h^0v3mBhq`av^Ze;*CO^BYk!)rE zePU-$lk8&l`2U36#KK8T52W57SUfm~gUH`qZvfJk$rYCi(A`CqF6ef2+g zLhP5;yrJ+d@J|>iW7!p!jD(_5IjVxa>;=P6eRDvT1M(C|Ho&!^cK0mRhXLUw6r= zzM=r{oZcVEAIa;`+t@5MntKevG17=;sl5=67dN7LB~RwpDzlcbH1M0WY}O2($|nYQ z`Vt8I0ZNQ2H7nP!3)-t#3HO0LgK4zV^As83OCXX;mM$rr?CAg&DE~xJ|2tiY@v+O3GK%=}CyYLlr6deaPYe!rHi}B@82D0aJm6dfWu%mlJ9%r%RS9 zg$7_K&Mxd95lq1y3A`bbmt*5uS(UXwaROVmWyzLQN9Mq(5UjXHEkQVv+Se#djg(@% z0IZYJoh*DN$CTtypahyT7R${9su&^Zg*$g@ajm*pzC-gL{eaxvzH zGI=GP+6s7Hwp2jyseSd*Mo;t2S0;v>aRG9TMDk~n#<_FF^h`Flm<7e%UYl4SEzbtO z@dWhgPKdoDYid)q6-06$N}y3z$D}pi(LqI%ik22FXy_Gaot&n`{YeJ^M8c@Uq}W#( zR%7TaU`UQiwCqr2xSHsio^|mAHr$16yQ?EtcG6XqXf&6Bd|39fLz#ouR|p-a9@@KW zQ~JR|njC~4NwO^3w3Mk6i3?IA&wA-?8))>=9NEf{=+>4DA_S*Qq>Zn*_4yrH=A!>$@F9RxXU<#-~WT&nqqfhKPxZ94>aptv(M0;9=EzDGzZYrtwj(%yas%weFs) zuUp!gwZapaM>rm5Hk4jDG~k{eagWXqdlUY*-cKOk(E9dG`{RLhDInui{7z0il*wtC zeANF0rgPK3hjNV9bk17j6X7 zViMW`=E8uJx!NvYRqTZgbbcZ&ZxP5HoZjR3YWx$l9i6u_Gl*7f>DUaN85E-OpgKZj zOQ$W((x z5^>S3(Hi6HAz2V)$pIIJ`K=p9Gcy?_k?F1=YbZjZ-*9unVoy*h0ktd!Nkb z9hv8~gnaxfN?>h(R%mYMmomhm<7X__n(RG>-$Oo$3TT4O#Gw2NS=BG>^QrsfTHi`G z8AUvnT?~c`r3LDjmC33gc&$QtaXe*b%JKZxTz%O;sHDvJaKUSi+EvRj(+>V)e%x_& zqfmUOZU?PB)2)*iMoix}&1qhUjqjY>vrgBj9`$~MkAO@knBI*v`Z)p}SLx6Bmgrz) zRODvExGOSSsfgnvvaHLtqGt?I811_Gx*tqJ5{gaDn#|UB%oN7LaCojXvqe3E6p|-iU zK<@|b)x3d@Dbp^PWY7bt)9y^Q@$G$`k#j*jrwW?vRK41UiF75G*;);92C%tV1w?g$ z3OK43#?4g8b&D#P_qkLnI1QYpjftz#F;E2nV6wA+{l@c4xlj}?9x(p+y;nG0oRZ}D z$nxa)V%0Wv4C)uyQQscJ*8$(h(1w+H0V!6Yrfg=ezBNES{iz}Y_sIkx)X717Fw>y0v&53iR!j6lMdV4Mz(GjAsJR0ez2tkhC^oD^q7!IbuC&F1jL)k zZZ>Q)Bg?j_ahwznvrk*xc4R4g(gpxc#iVQlCdGKi3_g`#zsyR0f~^2pRflTSAIT8Q zPbHs5r?Q-rRuky_dRph++lXesP=QGzAh<{(61W4}cnIBH`c<)%Sy`z7d3O}yXcE{4 zadMF_?|RXkCA^0^E5N*lCh&4xvC=-IQ6tWf0?1tjamd6h`CyWYT~Ta`fa`I>O8~JA z0J(f+Y9U=1lF3m)5e3L@AglvqmxEepf#Cp6K(fDJH)Y9m<`yRPL+71U6X@hK0LAj0 znX{i=SSS+Hicv&IIJa!$*7}NNMD~t*JiZ%bdL?=;v-dOl_I^3o`xy~BiwXzKq;jiE zDOL@>%4T>&C<>ygIGv9jMN1m>qughzJHWsL#i-J^rAX=F33Xb3tQE!R9`F5rJPaWAz|6(N#nX3?#+a(aY`A=?i9)TqNtk`YULWaP*Kes*ruaWyTI`>1RP zg5-^FZQVY+DJ_8WPNe?FjLW0Bgnj%QR#F#4j_J)v?+Nr4av$>hqoA}}qYXw!7`Sam zra)U=Tb&qmyU3ZG>J?JZJY}MI*5Y7w8GcaD9QwhhemCqJXDXFfvnl4<2wmQvo#}Q@|L1OV(3;1XIG~+ z-2_-`wC6e|z9=>7raeg$P6Yc0<3oYz74GotXPd9n#$R+*{_ zb}FkJ530trVodXWI$MtB`&4_k(MavG z$-xxDMT{GmQ}ju#6AfwxbR2DVkiWCjQpgR=F={vcb=SeSxYr4rHf^GRMv3b4oOE-uidkW;SK1ff1*VNcz^`9#nQ(&i%(QDZ3js`Y&{H|zj@e}y9_{XTj z@OPy~-=<6DHvr>eesy>>4mhq6~SiTLmEDBmqkcjCBR%qI}SYx)J zLFhKTLQ)Iyt9*w${zdRGkT>9=!!!R9HCKIi)8ay=HSZ)~2mD?YS=&M)k>@%!*k!R}_6Cr&j6V+!}78l6iUjdQa& zzoMAk4jnrP)~wKofB>B~*ImWHPKaMC=Hd>mQUDINWz19AhgLRLSwER+4yC74lUcIf zE{E=T-5;{F;>URa3e771kDzSHkOLTUTmw$E_syPlO+K)kI}BI-i!*l8{7D-;$8uW=@9V>J(j!mXY z>sz_;YA|0>;>z}&f^;D<)4Dg5bFEa_b*kDsKs1GY%}LElbWdNm_31@7D`1?n%1KF9 z!KwR=t25&Q^hFqEwKb&d7)9iFM`49g3!?hGSP^ydnfi$Ky2ORLtb~>G&s* z?1h>d(vlD1I3?xkz9HJO=BuerzT7TDJlAr%g6SMgk`0`!39wSn>h5@JegnfBCfHSz zoDMN*usWR1X{dy3Y2aWegW$W(NGj1QRVeJU8g@x z#0F~U%(_T@qHc_HwY6Gyry#0|leB_iou|uF{)$%{pULZxI*DS!6-DBC_!G`>^#l2# zKqipaOgo;ctoS$0B!;3{;NM7MbaD7gnR-mHR6^Idsl7kp=i-OZGFx*+D-qR3v{s4E zHJSlyC8&;OqL|p%{+0hLl~S5S_RFM2sU;3dhpoXX{}sjjcS$A^PiGW>nW%!SYjQde zpmVY%yCW$j$<`-}#Ln{0a0c_1WniJ+fRvFF>3D%;exaP!GG=a5-YJ@H%@KIWW59j_ z2dAb|8*baB(A>k+=AM6sGCnyh*rp8V+c%u5o0(<({Ta}OKJf9tEX3o&K!=$7N5Xx0UbRAGKp#|J63$* zV65>zWJ6RLiN-<(n63Z6&r((p8l-+T>IC~^ilq>Ij2BI14p_zW1Pxyii^iH7lwBq6S>WLyqm&IjrCu@XyhQ8k2|%5J1n`8KrUFW4a5@V zxLU7=4}<;BvoRC8{)o<5SdFc81yYgq{XRf0#Q_%}xX{{fe-%zNS^~j_f&gh{aBYP{ z5RK_N9jZv}SS|f|I`P>WYP8dCGvQYAJ`dU-cP1JW7gTwbPEnHMYVFk0nUMB*xsWqV z{Cyr$44E!&lwxLf`>AJ+V_N&x5^|o#Bu6Mz2Oz>v7}`~y8rNny&`&mGeCNS zcuG-fKywC%&bN9%v;d+|JfHYa04Ut|M4CVoRP60QZvw0~*r$b#IBJ7Y2QFZ>|TeX*pPRT>8P(3RX(Q>B}AL`(iVQ(d|vN)%f= zEj$gaPS@!)2wjsL@SJ3N%Fb3jznRihtw_rO1iWie@W(4bIM$)4j>H16bTC_TT2K!J zPZu(}#6eDpPu)E}ThJ|I3fctunUjy%CgCm-6O$*+E$*Mm$+9T}y0YmUfJg_^y>H=% z$Q__v6fZb^-7ECRU#KE+YvXIsADJ%|H&R#a5xui%ceT7R5OlB)QL~BGTHZ5MTuj@# zTm$U@+X~LS_C@O!udK?tz>Bu#?!WrQ8)u$AmO5c9bT?!Z(enqZM7FRo!!0Oe`ajh_ zV`GL(cy(65>X_$=FyO=iQ~1tjZ`X{R!nxA z(BtTNY_A|Hlfh0wLgR1k4s`zC*azru@ArdHv?tN;@dEiC%76!=vpwi+-bQ0}L^>Mn zXL`P76pCYuNKLx*#HZ8f{_X%3uC0tvVY-V(Ea6#>1MWj0cd~CdckS2}oiD}>#;$S! z2l#QQoY!1CyD>-jn2-PNm{tnu5SzaOk$DHJgUOAkvOzU!U4zvN+4B2nhYWFYuvG?E ztG4^zvj5TQH%!Ue895&}vdK7z+*VShqRffKat0uet`vucZTSQGcrqBKbN`iWr7Vi2 za#pcDT@2Epq4|m={N8A!;=|>PjCIT8w25vLz;tt_qXVkcvDuuP4QvBDOT}=;Aev?J zT5Ebjkte!S8pEAhp<0VsSwnI9b2O5OM~~qL!DHv@| zKb>-Uu~H2c%hRP)p=`^x#0!r@6N>9iqFWX(=PJYHv>>f&2hinrCxHpQ)WRk5O> z3J$VZU84gPt@(6*+yTQk4S@}LaH0^C6}rA<*s;<8nQ(1a)yXPPt@u6yclxWt7uX78 z6dKdPi{|>OMRbvt=P9C-MM0G%70@>Opg$r)Fyq&6@q2+^hggl_AmtHVr~H3BlJit% zQC2^tSV^zjFB^Ayu0 zR${h|2b2&;X#`yr> z3<*#w@ijd|hH_fKkL20!%jkae&v@}kd#J2gbzENALj|LiO;j-0@lo^*{IvdaxdG~- zbCUf&svVh_0<5>>sXkI4atjHg9@&7{y92!r-GD-71A|$=G@iyIzYy(nMEyah zB$ilVd=azN>$Q?p3>49gnO+(gZ3&zTS%C|=0OmP-x|t+=A@Cjx$|3 ziXn3EwYbHeyU1o+4xT*Hk6N9lKAkNbkG5H&Y7|fn>XukRr4=pA?Za4>9F?JQvgpQ5 zk1i9FR8IF}LsZ@%=r`E9NUI=VV~N1>0R`!-=`qObqjvjIyCHqqxWJP6nB%)?T7RKy zOUgme94*tG8|3_QRxnsh{0+p!Z-X3{*-jIKSV(6y(V37G1=9NAbfZ6esTmc#*X|%S z;A?e?4Nk1tU@EeSLDEG)bc7;HmjgPmuW^8(GOA$D>rn^d_FMO8&5vK(&6N840EUkgaV?|^tYjP|U= zOFGg^vk+GGH(8^Z1LwO@4JF#Yj{`?8t@)O)4y`zPi2HBX>IpsY9G-Jr3uKpw(<}lj zg?z$t~cr`iSBBbZmrf@fO({=%^3DZxyN18Bcfm6jhLxBUZC&8`@oebi98Tm`H!m>brs$TN*~#rVH%K}$ znj%H?6Y^kxKR2p}N$Oqt<@9J#C!6BNO6l(x%O0e!SW(ipkS`Y{q8N}+l;uKP4_!i+ zYsGW*F$-{&X(%){93ml(lW+9*qC-sAX*C73r*Ff3WdAM2i|GJM%zPoM+9A=4#Vq-T zYSW=L_%N4M`J@R6zcV(Rf%~3D(%>sU4Zc6y_a0s&x<9SkSV&%}Sg^#sxH7-H6cs2s z-Og8(Fqc)Ngr=x6~d5$FgW z1u0gLk0%qRtdfg`0$pm3$0ie|1bk9uRV7wF<)lqHTCa*OB#Y>?q7YaCKzuHb^&6|;$dTsb2bO{deTVm2GS zz`f5Wvv`CY2V9YSDzfOik7{>~M>xMb=9g0D%xG<9GV2BDbiz-QJBR0-bSYk+ZjT0G z-t!6t+TWeMQS`Ikk0I}pP;}Rz;}7CIEz@-AFwZzCE#kARf|Vah50A$+-KdxCWM~p0 zOH0T@*;F}UaKwkuZYEsS0>_<-__>}GaE*Kcv}>SW@xFegLBDAIP>%E~D(yE{ivS_p z{EmXwcPZI?MiD{5G`l1lDFf2rNWD~5(n(>UU-1d^1nnYx)P?~4fJ@=V8eFDEk+D>W zW~mjxAauLzKCzEiovKUTiMihxVk#?XQYstyC8hUA@_zDKh_X*(mA=8M{lQLbi_n`> z00AztO2--fvuK6llNR)^ls!reO>|GF_t8pCzhFFAX79U{DRlN34dyLofcUGNNbI2? zw>wEnF<=5wT1MRb+1JrQ7QV_wUKU zCyFhL3a;iv!X=uC#AK?Ge|=+1r8wQ4DLNxGUYC0hlTq>kh?C3cWL75{)Z4|rubGCc zvW$CM`)Bc!Xaew!#!LSi?ew?&AJPGW4mn z6JzTxu5Seqi@Yo=F-v1EIG@d}J8yYpM+Ra5-Vr#}cx6jio+wNY`Ntnrwax{dLYr070~xShF59kTV<8AG^#q$ir46jjb@&>I|SYI z4*1hvJK{}Q#Ht~!z6wP9+RbOA++22i|MT`- z@~V0|k<26j&Ela@=!7puJeO28QT3Q8bq!l`edcL3}3Lhr)-)+TkJbCy5_eZidjM|G^t*(ZAw6m zaFN!%D1$rQsVd?0jFp{9;@M_l(WTop<{Yyb*0uzxJu_3w>526f)1m9dI>A^xh|3zU z6h_z3n3lcmPhb|S|)$|UB$ReaCQIqJ*5fP zmLr@U_P&kZhyR2|!9KSj6hyNnPq0YlwwXl+kst=eh8oi~YL9-+9T+Vuu>Ybw*_fwZ|}fYzz>*7W)0hJB5wuHnnygFRKkB-aOKS{Li>%^ zI|MG*aPV40%WvKx1%(#bT2~z-H4_UYOR>yCgwWY!Ol`uco=!wh*RDKxBDPYwHX2*U zS2z_Uv3~PuGZQn$%AqV?e>4k*u`I==Z5%fv82>9QOXcaT9#plYf`y3_^G<5$@#jCi z8i^mxWT@B-2SoFW!Rqnx%cp~tV|CIc1TG(5!!fv^Q`t#wy$k*g{yy_b>cKSX*aVNq z>J1(zGEhqnU2wZUsB-aCcWfLV`|RXX-k^tA0>zMOcztJ_IFWMZ$E}!_W{JScU+N6k~HKd@Yq` z3n5F_VZ2J}nLMi}#CSSBaIbe$xWO5+Kqs0wB=Mw(Q3 zs6@orT(K})vf_Hg5_w)9qITZ~uYhmEFEaNtpJG;;dsTqu!= zPMIhzI(42a=6y<&;sTX^*H@XNgk8dSA z*f43>CdC32jk9!82XX)@JTIUaGDXUD0RfIrSpls9F_c6#q$uRPRqkupPVnG2oYlNl zA-!V=f3qCY1j5PB0+GydN;Wnw8ia%(ir2WzY9>+(3LG}Xq9KY{F0iYjuzFgDtUh8{ zMH8PRD*-N4(L^5%cdCdp^l_Qt>mz~k@_@ozfH zobfYR5k!SRR0XLsz(JD@O+Gl^97^gg)B|3GL&rOj`C{@PEiRtkO z{jPi%e^Yomc6npt{KbtsFZ`-#;2#O7Vg%n@D0h(!8@2FAKefQZoqihc71j9sZne{PLA2YwHHi&>zyi*!ws?m&_W zJM7g|MjE#k8A-(Iumq{a&Npm1fy$<(0^X$!Y$U%=I`fY1=s5a;ykI4r!;TVGCRpCU zQ94jeWaP|1!qlT$GQneMRl@jNQpwq*9#I7J+4QkgAZ3_YLkvif%#n-|Q>IuR%XL^Z zhfFhN>Tx}p!f5h_9LdaOQj2L>7X$R}Wxfpa@EdqXL)d2~@`mazPkG4C9Jw^6GnU=d zDjF^{a{Up~PDmTtBEF~OTWD{HrMPx=?#Sikp~C8wBXd_)3umw0MMiHfZ&X&t62Rsm z5?M(Aii9*FT4Yf^tqv&&VY4gcV#iKIN|Q;aiokC#Z5|)WuWTM$+*~Wi zJ9hQ4_JCUZ_0u>J?#(Q z66+QY-E_?aaUw_{>k^rG-3b$H3)AENAb(nnISBMRA=Q>uGb523Ei8$NR0`i9@BR92;z-Urlhah@?`1y7 zj5VFgMw8@k_>#ZjOa8{bPo`D!*|u4#eV)ZG@xN_682}*UM6K8Tr@f%QOZLAv8H>fz z@ud5S-k~nOPhfm6GLKT3zrDWATzUzkcss@Ovj6mu_q0D`ORW3D#m0D8qcfA_)ectH zpA^awt85aE9bJT~#AnMDowTdS3U~b=q#M2oZC#0qydcX$fEDDBlhLpbz-gKSZr0lH zp*H0DC5-zW%ry`BhFlYt|8p2+m*s!Mf4by7?HBo$vPz3_Z?2;35BQ%4lI3W$m^2nf z!^NbTZ^c@(&5&hTW_T~4v+;1z37E-(Rh!l0h4+5aD(wGmC7fU=m13IY{BCMTK2EL3 zMU3==O|nj`$VK0ZT=cET#eE~b>;CMWvEL{1yG)AEUj<+<2eHe_{CW2(0?FsqJbv7B zeimlAz88jEwo@*CaO&J>yX^gde7f%~cFFcbm)jQmZzIeJ>V?nOn;C8gVz%Aof(<}* zn`rM{^}~P*Uhd>QZSU_syuW+s{_few{ydKebn%D=*z&v>Pg3^-r8MH~1j0?ujuG>n zoqjNY)rK#lr`mfmZvCBLahmeWuhOm&!k@B_6L0T&41mEMZM=$p_6z=br`OOp=a1sO zou=o!wD(;#3c!ttBWeWmHuVAq*xOxVC2G}7o-{_0CVqOw0~HCfiu&5)m#`^N;v$Ew zB@zwRBsaJsvbv#q?^>vAy~q3z>u%1gyOTFa^vMj&w(9C}*`*2|_Nan~tL-(i7lSbT zZYOPoTR+6>eH-gyj#;PcUGIOr{pem}3)PD1vK=dZYFMv5v75cM3+va`Kj8hn%dX)4 zjd*il{562iER+l}tqc z@-wSf=9ACe)!qt$)#so6^yf}Ia4h}&Up8~&Ry(@_{#qZTYh9viB{idTueJJJGz~Jn zCZp47-OtIHTv|a9MU?|(B{ZY~(me;HcJPLf%%){6E@@Rcl9MT=b5O)) z#!-2MV>HSyjPG#5FI@K6f62T6sXf>`c9L)1NU7~@-HCm;&CT42k@)e}5Z=f71`{V4 zc&5S5L+l{o!F40-2`|E?W`pf=1Yn(XB?bxiqAuY37NtvNoscm~f6)Pd_^(_j8j}$x zi=rVYAt$Rz0;_7u&8zKkXXyK>-hnwBj}dHFDYB&Fv|9Jx!qXYqncf!Q#r81Y(Wg>( zK)zRV6m+?H*+=kx9;F^1=`p>d?!H6%GXA($A3YGJ-Wh-Vb)1$3)5NdJPd7?h)WOQ{ zna(#V#*#^TlRNs(n;c-b$-Bx`6czTx`&;9~@i_S{hQlF^T=QA?nwQBnukfUIcLwA6 zHhTp}>#h$&Di%w1mx*_=eFVfMT~#E25HAO?ssT-RuFuV$8I4h5UcOiqSe8=` z%k2%~tS(M`VhIE-N8>mMN|z3Q`P_4}`9x;+>_}|V`PSH(w20Jf>gIoDren)z;TO8a zOzs}<$9vrH_c*?XJ$5sk{qFZ@FGD)u&e>{L150feejCg#&^4Dq1s%2iMz29op)p}F!RiL%Jy@rzRmi2 z%Jm2v#(UQgV^7vsk~gND`>s0EA0=&E1GB+8q9fu6QP2X_a!!;2ng~@fSPcWaD7WW{ zjp)D9gC`bDum-+X6irLVO{kfcAV^5G6(z*EIY^AlX^g`OtnGxQC0}===XDRArG2B* zUI7NhCM7=IMiutTo(o=Vby}dyrHlQ;73M?qtuv(Ra0!20tv5635>7goRoAUOEO}3( z`()Lf>gx*KGxP2@TdvUU+FH{UN}}PtQ}liehso~!iNj=ke=3bI*}p$om;V&LMfp#k zqVnc4pMI0sth?-|n=Y2Ol>M|#5B|XtqAG1V`fmwID!kEQOu)zbpD5PP(Ei;`jPuj= zoout4REi&epO9xe&meeWmim?LIP#WYue2ESo$`N!^6e4%C-3keo!Sh>-j)5D_vEr&jChN+Tg1OwG8hc=SZ_q}dLCf%S%o zc%k8BjX^4IIX$HT=S(l|M5Mlu2bs!EJ67@5b15+AFiMHYS= z|G!kS_)t`{odCeN%coF0lU0GGC{6~)yHGZQ;AJtY;CR8zWX3RHW?|t_ej*z{63M3p zfhnT|MqHFM!Hz~Hd=9ncllVH84O0;D8KIkhjYRzGO_u{^GiD{{DYosKlr}LHsQi+H)wOP^8M1`8n6p-{m5|T$hc#Z zxt&Pj)(H4ebnx z*^$~iN>X>HX+MBGw~!kl*sP(IM!y#yQf(;AjWc9 zR9QabEPe3Q;zcJF5?EHz1EQGJ0~)xyRF?7Hr2wKc1FOe4u+rYEfP^QQ`Fo6o&5Za} zn~vPxZLG%U2~CXRb9{%}#bzDYz5&}Qykg+i|KfGMMb|Y&HJz+$vhK1-7x79X4WDxN zD!YV1EsW%Lk2t3D!n zzKhrOGR3sE^Q`G47Rfp~BwsR38IO01@@KMRr1%`qh84Ul{^2$r@%N>WHJba~=!m^R zRFJc?;Py;X1$+mm7P>su@axPpbBXqrxGdGKMxOQ7HW*uVskg5*2%+`)K0Tp9d9VMm z*X%u+$NLF*9iJEB#h&%&9dh*^%-C(T6ujPzEc^3ql(sX0Hh80>+P19uDAind%_Q$h z7#|UnbNeg1?x3>Aq^r4ApAnX?4BViFC-)cHPP$s^WIO zS=qa+|MV_>-+2}@{(KgdpT-!bI~DFOx-mQx+NJs6_51EWv_74>es%cNbn50?Ml=(w zj_UZoRyL{^b{m5KzVV6F;+3P%K9Me1$A`73Mm{w~h6Fm9BT)N0?ZH zgm5|ITq(My3RSCdAJRgq4^)9Dy zSOv&IIQEf1eIlqxycE!blA=H}n>1~m){K7&PQo{tHD;4)2g!qz_plTwUqYdg?LH9x z@tH0h5(9IS2W+34e6lP5#6FqJ=en_1kat$S3tqu8J;|J=5}mAjwY9$3zey3T!57Z9 zF_V?c)W)3Sn})xVMAK3yECVIz@fca9t?sv+9hEJ z+;?kwb;sGey%14P&I>vykvs<34RzbXuD+suaf_o;=epbI-Qi=d(*}6HRd+v9xpFKxC$=R-M0@BuJN-i zNbV8Dw)%2V4CL{4inMQ5@-OV_pMy+rYYRMr@8EgnUivm*HxJUxR_?NtA2%B;^nRq{$(EnrC~C=R6{3!T|mbulQa z5NYK794h2ko+Pv&joW{=^?%@}@Yk3Wj*=T}di2e_ZVkp$T9cN-T*yKms}Xh_t3V*{ zav26;`+IkP_*lF$VULZEg%6#s&H*hH=OsB;9*(Bw>P{d%nn|P-TYL)0kqZg<|D9b6 zcpTMrzW2^^cOJVlJF~Oew|2D;t+WrlW!aV=vWy?{Tb2#BF|sYNjV*yJ1K~{|#1J5a z0x2o4Q1T*y5DE#UltP-&gaQFVfDn?V2@OfgM_o!&`hCIbxp!t|8ym1IX|L|i&d%J& zx##@n%(>^>R#|uMz*#retY|D`-^kgETK?lhP4~6HBZ|qsqodhZIRKD0WP$gMQ5NMo`WoEOxRmN{ay{Eb+?mclh39R7Pjr5h&r+>GsoZJ^vcSPyauSiXH^S*$pqx`kAZs=>89{^G=J$(cn1(hG zWYT@t#yLB3*?d!DdbrkoYBn6zd5#Ok6-g82pe(}5)3l)N^U0D%LJdMR7EtY|4wFqp z8YfCn4h>c3HU-8u^;U?X+O|C(;pzXH{s5l~^WV<=4Cw6B_~|rmToFb$ih`)|sC-Da^E#e?`7(uXSW2B?rRbC8i?&(lmi z%y%eFLQ{aF;mdeoL1(xn3wS1-FIlxNN#GSZs1JO0k*x5-@;MFd%~4L)WWP$9|!_C zif(VaDdT>W(?a_e^E_VA6>iNqM2t7J*zI#b?gmT+4_vbd8nJ9nw5=E=LEWc~$O%7f zPSn7>6-x!1TIGyC;I&5gHK@JiP(EoB;6t=6)BipFpKR~u2%9(RskIp9O@wh<%;v4v ztHFw9F~g*Onw21cb$6!1-tuO|IJ1|NF6un(3ruT|R|m+^<2{*kw}e))UBP&s;1_Bu zLp|qr6JO*dj7z7Nppu)P3l7iA42I~Q&cMQ!u8oVzKE?47PSAkY`fLempr09YOQT)h z&QQ?GWigZ|Yq2me1P~?F!r7{$JFvXdE!6Hlv^1=^`#YWP4f8OE!$nahNi{?jSysn8 z=a%go z7~4;zNGzdg(6ofEgX(OBd@^tdRnp_KXbO5vARyWr8X`VH@yiDJhTK%^jzkJRH6DXHbg`c%MVx*~)%yQiz73VxA4Qhr~^o-8wvLL!_AH>#Z^&vx~e-}0nh2Cpx|_tW7&X$09ppJ zsW1(cM6m%OXNe&zD7egx6KQ-Nl}26ybfd~&#zc?c z-{Nync_uowQ+4;UvFA^e4mb)4-8IfESUv5`xSe6*oW--x&JWmrInk6i3v;40n{X%> zw88@C*IN`ZoNjikTrO)CEm_x9K@C3&sIhl2r;@ZoqlL#y*+58Qn-*( zsr)`beu=N=PC3b#tp|?dIm(4TyoYpzbmKMT?vuRs$Z>Q6vghb)WAqd9EWUx;iXum& zqYrNb`2F$S`{BzU>93E`=gH6T?cCb`e=OeLpV;#QPW`0k0T@k@61K=|jHA@^2fHZn zX#M_c$13dPwd1Pb1Z^Q1oC98<1CFRigJl^SKhno%%kjK&$Y+jqpkbIWI{i5QA^8R9 zrI#@YLOq~|f!pjwSv1Tf$R%Elj;0q-!Q(Mv)@3(cI?LS9Qr{6%-zsz1CE81`-qY?d z9~=-u4psA+YX=2_E}x=*Fg?tx-uu(>^TblkWFX$t8UtpN&Tzaa0c!|4F%8=t2`y6A z0oswsylA#ch^_=1%~XGTIvh}fEoTzFh%BMpObH))0upu{`P5RQA*eS?Olz7Cc(5> z3`!PhLaZy3w98Bm)A)ZLRDri0Kzc5bxe!J4AT^^@%XR~LhDE$3H7Ch~4k+f5r^J;^ zvDF4iM#4l1L^>LK4h-SO0!_%MXEeYlQb3jnFSa!-E;qI}g?XQ(gcZ@z{SjYNo(E`M zJhec4coipcW`_k5%`f$~0D%&o_s4ve?&AeP*XNo8Wq>iERtrR#-7LbW>MiD2UnToswz>3%Uk@^{=kbg za+;`Sa!oo7UD<4lAggJPCycN2Lz`7jmM;TIPS#N{;1@tEDq2~e=955K1CXYJY=_~- zg)Q5+S2*6u2?CZQ@uaqJywxt4jc|iPR1m*TLwP`7Gt(F2SIHW*i|yE>+$di?9AR>l z`GC+(@Ge7vc4zXmQ`Eb!o2CF}j$(AfL48=_%*g6m6&WeeEtc8(0XN{VJ4iq?o|5a~ zpKb*}3V~tZ{g-4HX9OPsm?|olm>e)1o+wejB5~9xc?%#Kj^@@O!39>@U~hR0@3A6k zY~*P2$(-W{whq;OG!>QA807fD3h~*=8EFLmb$vQ3Gi~o^)8gNX1DBTiKiv@ z+5h|$tM+fZ?B>YQ)39dk8hRDVxAdf@hN=U@@Y#-+^7+-t@j~w0=4`lZ2hPgpseQPN zHj?M?t*8Yw%Vlg_8@zEfP=hzFd9UV1<+0g#F$6=&cP5y&>l4Pny43H*F816kh5EG9 zFN8@fx1bisT(aDotPO$s5|E;U<)(p-RHl1DvSUM7OKG9@7yWU?o||p$Qk{$)O=Hwl zEaXBlP(zg#e8A3kr#X@Hrz^=orA3uP5{@VB;_Q66KNsw*2C_a~eM-)m>1-_ON%LMxLn8?S>4ZL!8Do+xi-U; z@P+vt2EGMaLb92MmCX?r6A9WT;}>Dv-S{GIR%6#Y$kUmRHA3`wx{?#-j&329!@uFh zNXSU=(0wy3GlPVd8b}JVuQBQr0Cz~{9Up2iv@{UJe=jHeMnH3Da%?SFw@x^14nR~# zjaWt!Kr`kJy>fIfcpxsCpN{t91=NK47^0wjfkut)X8m~#uw6G2Ty)pz=`x4y!gOsG zvK}*}T`O=O9{fTk-vw(eBn1?{;6xjFL6I&JgEZ(qFSs4vloA5L;%G)PBoYck&q^kM zUK!sg$NWJ~kolxxMMOjO>Cz^erOYSFS-+vHHn-@|j|+J^Ilu|MZgE}?$;9;gWG@i_ zC()dlIZsB=_^GsqHe1(1Jr=f?XT``G)O5xCkPZtC3LVh-s&qC?1`yI+hyB zwx@CXp*KqXjsP;%j^e8b5k<}i&=iOe8wpkMlS9uYoDhI9mD+Qjo37wt@|L$Vl}1!% z@zCjd_k0TK?M`w`nPEFHK$`bsmJ_X=g@mQTUzBqxBXW+crr3stP1b;Ju)-qTU{2Na!fk7ymwt1nWz4Pd;3y!;WfubRA z;{sf20nE^>4782Tx<+`*P%V*@L?hYQ70DI?Ce@AgMqL1PT)-C)PWD?dRpI?M49YqX z(mbcwn5&fHMw8Jk^9rX2h@@gr-g#asm6Q31ABZU)vS4gDDJVQH=>7uX!cjX&WEu&M z^?xZ~m(PIw4bSMfK6k$W>kTOE(veXR%S&eUFI>`h*~3Z4PR1MTB>v;-HOtnmU2%JV zwK})2t&Q>rv=={zuLQXgK!WxHrT75tSVClqYX;0}FL(eahi>$a*2 z1pla^pu4=Lj4+u1eWk?qQMuQlGWsz@AGrCMd^Wuri`_ii4La}PF3Ed@m@Rl)7+hZ= zsVo@Arsg@Hy`09x=r5fq8Zxo%mxWpVZmf)XojfiTnlC!-{%_N`7hA35Ab*ixw&HKUM`y&_RO9R_KcwgizI->`eRQ}8@j{S zWLITzAQ5;PLo1hBIji4F$+IRpcmr@J7pvkwtiFFWwcuTfg-d-toN2h_&}+9yq#-!< z=gZw}b_v&p07WO<--H&VJUNaS-6Ev0(P{K*wYCllLZMk85S_5;gvBh*7ex95OcKK# zjWyc9ouQN^3xzE;UoqO>J6`=4s4Hpue!NG&4Ezg)XTH{{OE<#;R= z$q!f3e0j7M$Qj|EwX7*=`hBJ0lBT(|wVSNM1LP(cE2{6*bIcqWy)D$|$=mD^53Zf> zrsqXy{^Ot#RGnjRCsDWVC-y(KZQHhO+qONiZQJ%Fnb@|IiEW>p_ujhao=@GqSM~m| zYIjxdy`J^_CN*DDHz$J~K&=ewwF(DSbs|jRZWbQx-MgvB4C#|-?M=mlh z#kGfT&Ra|gXn2!aN6tkthk{A5;Au!Sg~jfWA(4m4i9Swnz^vKLq}o zkp!-Ha#IN|3iD3xmY~t4dpu=`;x&~Fh}q{5DA zs6&p899EUR%Rt1YXZr8_naM(-YA(a3yS!GASXlYmAk*1wasf7usoAfvuBc0P|L^dn zbdbig#fQhUxFia3Dg|)jsw)!l?>>8$V|>1OllO$HgqacP(qA-8L1U89M4uUPSAVf@ zvrNc2|2{gIJGD7o-+F$*evy3veZdhD>Ck-WZIXRyt<(7Gqa9%%KKo+s;O*4(27igC zq>RHamUqhfO89DHFJH~+tsK1ZzFh-fgVIC#QNNfQx82rutaXokD4&>}t9dtk8nk14 zP2AkewKdIiB1npp5zDiwVA;ip71j-L86Py19Z(-YAbA;y??3%5ICQwXQej|C@uKXh zZm76jn{1pc%1CQcd+NMsk|ma1Ve84-$mpVfmisyjQaz!mrps1-QJbUE+n`~ixAbdk zZgjCc?VaDCy`-hqrnBp)J9=F9u=YQ%8}}ZEpk`Oat3s=SUv_JqvhT{j+;Z#r40;O2 zb$7qIigfFl_Z6OTn1SiJ_s+{*;ye%JYq;+2JvLu4!_RK=m|i*l@Vpw`B!72*>f^g^ z_MDm4XZa2VBZfH&m50UASnNCb?1whE$?H`MN{?Dd(=r@XDxs(tOZMmxQidw4hbK7t=_AM4^!#KpvF zu{Ye5uHWUDjZ%%4}1}89T^+qRDYV{_ksfQMY+MF-AoYdMD!K z9)}(A{A_sl)6lMkxBtA4s8fc1C3haR##vTREUnRzoHC)&k)6Z*cO11^XKY-xT4!pU zwOWV!Q<-9=;f;ZMrO7EI1zV#7QR)qO=o8ZWyl?`SU9IwFx&~Q538tG1V$>@_{dg~% zS5NnsiFcqnx}baKM3poH^s}ccCgw?+-WJ58dv_hjDF4BdCkkPyI{K`X(%l&mbP`N=XUH@OyN4^L>8Q8fRydQ+ zY85rb%%hhpYD)V;Pbc^Yybm#Ni*t3Ot&NYk*Tco!L!qv>yR7V$6nykmu>bKDzI5A) z6>rVy{S1E}F!}ub?jftU*z@IvtDD!}-&|blk?Hk5bC>z|{mJLIdGXxE5jy9?goK7M z-!8J@1>2NXt&NMN^Z6DnZ7{ticR3l`JPdw#zH(JAub#SD`^CN!D8r*3hy-8Dby~E_u&aI?edg@*rmA{r2!y zp>MAjZTq?ne!S1))&XLtZO{GgGhxo?g-{%gAT^;g(?QmA^5pz3BgWS&5IY~`)Q`Ax znQ>g^ptX!+v2p?%^W|__E8sg8!cBsyb-~1}Pqq7>d2qZL0cbYn$9c!TMc{RS&hfZT z=}$cT-e>6cV?gVp{%fLCc;J^%$ZkP! zz8hYRZ>nlnT$V%v2O4CK6fj6$*i~v^EHf0AHx8+Ml1fn7iP(tF>M94=k$xJei~ZU4#nXIE4_d}K}NgEW`*+=ebGLD zeIoDze>`WMSh~PZ4nCf+J~%&scLKfK)Xnx4_6f2V(&t0WZJnq-m_FdTz;yx`j#^Im zX9XWPAB-R589JXq;Q0QOABmq}{t);yA8@?zK0!~8z#fRI^YaSri`eHTPRtqT>+=1@ z{loo6PVC{=;fngu|Lx+4y&exlXG1hK;2-`in-x7Vav|&l{lxA>(T%MYNGoWYGd`i6 z1^x>7`t29&7o}8==uF@~9q`J1cTDQS`bP8y`^NId*NKEHkSESw&?mrNq&=%&ura4T zhkGLDLO+^UXMp!fi#AI?TQ}F2zfu32l*TyEZ+i8xHUf8JWVmdYY{cy*_??_QMAX>3 zRBWiHXoH-5xFsUzePjkJXS2_J=H!dgc1iJi8UNAlJK=Z)FLeyjCWRXJw!8th3 z%@mu@+a?jud1mki)9~BWgLiNmR1>_X1w*cYBGd>aQh|c6;3UKdIZ}p#uAn@`2q{v6 z0AfYOZhQP?iNE$NN%`Qj| z1+pvC3=`RWQg;(RGBTdcJ}%alWNc&jSAPY>l9N+lMWDh&(`MsPVwn#@_nJXRFM}ex zH&sC~L{o|NZP;|yN~5GVP9-I?Yhb|m#n}Y^)OnUqIHow$=j8zMYk^Y6(jQOea^1+0 zut`lZWKYT@PunR%y=M)xi8uQ?4Syk^uMl|5!>mkDp-dA<1GJb}OIV9;r-fc>_7KR~ z>@?r9yjC;gDGEmcTUyTiK>w3&#F9a-wqNqmvp;Dz|4BLe&s9xS zNs~ZQRzY4#MNm$KKu%so^gpSW@3$IIK2vVgkm_5jBV~o1( zx!D%Gz3wV_0$I=DotnOwHY=KB7{q75FnH|X@-h7E-N79{YW?e`UYidzF$M0spN`fy z&@y6efmEsqk)QyF%VKzPG~?fd_&IUM>?t|2vF@B*t}zeat2{Q~?X;r~*_|C=ao?_yTn z(NEi~^Ul6hlRS*^nh(J-RSN^D6Yn|HfH z(Y@r&;vmku#s)~FCLBSrX`aN??k0bK+Z#aL-97Q@eh2mGyn?4nk@Xv*dQ++Ru{1$n%6*Xf2qSnu+ zrNRqwS_IM38zPv)*otmVglbwP4?xSU8 zW_k7096g_1TnM52z2&TU;m@Yc!m&7@YbiRE@#@*&Y&M<2Yks?o+t6e_lL57`Py9+s z`RZv}Z}c?z0tVj+c>b7iG-8f2PBM~?OjKHMN{VbWzF+tzUt#_hA6vh)Uj94Gmy(&6 zrUHDLMvht{%d4cMRBy3d{hhs`F!*f~?jo#9BN$6yIVwYf$#);GqLCP4GKdmYEyk3f zlP(oc^b#G~C@&O=e+ZqRw91F3HE83(A1o8@@jAyVVyj zwM3{JMQIgq3;P?cmEz4_<(qs;wEp>quD7D1kFRi?iMg(1?Nc z2GC&greMDZ(t+ii313tSTTkOXVZ<)=FzNR%db7^_#^!H8K!dNbx|rojG?2xj!OuQ7 z*zh6w{I;qw|I%;PgjsQd>o(r|L@8$Gh$2}v^7uLuW?6`@a7ZP+p$o8Ph1vej+Wgu< z{BFAiuR|Q9Jez~qHX3CLzhjIXjXwqww*6Xq`hLeM&?wb_<{gnZAx-{~PXU8p`m;2< zvfeQ~$Qb9Be1pF+z%cF$L+A1vDE@<11WlND+aWU0L?g? z0KXSb6XfJuGZUD_LR*JWHv)PB?t<1!u_pq0LhppG(W-cVfFl7B5g`ys2#b*YW<)bF z?(DzLc)VhCPJsJ}u>}5nT(*E5QCa-LDCZ%p6B=injz}#*TKvSQ_b9}u-=XfItp#ph zFm?=B10fkj3(}#mNYqJuTzp)De7t;u$RV!8{vlPoumoBoikCL?CX}9tRy@ckwL<0CEU?lZUjl&DNa0xH zZ^SAh`o0*?Z}eM2zxJ6$0(-(13Euet7DBIbf6>vlQFjG=>4GhvxDO~R@drqs-$_Rb zK{G_b?^2ubC}_fw5%KuAhYhH;V~QN#_YT8(1pLH(6#ppHsN5(LQ9>i3MSpBuUAfPa z5LXI|3RC9d#1`h7ijKwXW4FXDEh&){{ED`cv&4WHV!YR|3H0Y{4UMq643>! zp!R#^4*Iy`yn~e)rSA(rT@Va2kh!lqPqXf`c^hI1S^BzT@v+1F>tBj^_~p*v>-_Eg z^6|Cu<74x_mN=99X7gbl_x&|{t^a`%`*~`er}z3WS=K4vg0DE*2%EZ+>B*B%>yyKB+RPN<#2`sl@ZSBOUL* zc3sa3*jVVZG2Q{`p$o+N{3!|4<=gqgyuM&qJ##CpED+Y|3;9k3uUmz$lf$khqG=^z zuO}f}6@CYj>4h(=au>(A&W>Ynw3QO(*n~8)3azyYLb(f9od2*Llz$WJVbFLI-9aKTQW2Gw|F#r4Y&dyryw1z|<}^J#Ft+3Wcv+2RY>B)6 ze%BxYc?#1*4&A;(JRuwruET)o*;P5YMicUH#eA#bodt(@E(dO}6>A?y(pg~0ww#G) z?S(vg;z}M8SK!+r@(*?XWhtcui*TtTs?=2P*heG$7wAL2*h7Z(+;C>$1({T&&TiNwlm<+s?RSXIa- zW*fDR>`Fn!OT}JEQpq%qGbGwk7^DcNc&R8}Bqy2^)rs|}PmIERFXil{W~;BPmF#bE zb>Yb{6;26MYHpzQYhj=`C@8zR`y;V=V7@qevio7OdlJc3PRsFDK~PLlOG#2|4nZ$I zBSSqgRWUiaiIkm|7%EIBgHn>SYQae=(do|B(i78isHCGqI8ez_@85-|vl)Egnsb=o z0E34a!5*0GUs5?q$UF_ z8Xy>T(BOzcgB4Z{KA3gT!1gO)o`<7uelxpxK9RO>`r^a2^nsr#Soat(<8sBI?i}u! z%{{rfX*hfgxaV+BAL$t6na2I+c*PL+e?kYFhIs!kB#q23L+_V9`vK8m=yf~zupfRp z1iui4UyA;palH?Qo|pd}V7E^n=-zvs!Y1RkPv7n~^tul_ox(0e1l}shaD$6R}BG)3BQa!GgQeLTAEv59Tno~JZ$zA!Zs$cb0`c`(9HdiDbhZz%> z%jMO7f?C8LyUpfxd=gv~7d?%s%g|}n?AUDF?CfIUa(+@&G!xy+@HT#uW+!H5EfTDJ z&Rn3^cL+V(U%OCn@K;lPPSC4Mi3K~LGw*#04s?&by)x&1qROCUcPTrtMIm}?Aj>nk zO5&e#L^H<9Gx7qz$O=w3?o zHz!_Gx!^jjT@k&*2W@9G)D!1Q7?2XR#eR#a_UR=p&jb$lN1Akx96u-?U$E+{U&IPy z^l#nmY4`PZQWPsNWQB=))o5DZKCexkrf*YJH;dXy45%-=P(QW^3#MKx@p9K5{ykOG zG=;mjj!lxt9!K6&ayBq#iRiLJDp_O15S5KvrS%=Lf~0vP#|7uwgS30K0;1+#%7|9C z?U&y5wDzd3`DZ-y((kXe=XH1Z{$C|GpW)67QmU94&m^UZAbqA~NtLMnv4`^gDl_pZ z8M&;*6WrVEG8#3TllEJ|J3kVqIQ+-Z;!ZRipCX>Y6)oXpsV2K%w+%V3jna?PM)0%} z{5&uydfoCW>wG0f75&aOD}KT_t#3jNI5)qSxfMOFfXIJ@L4LW<3y(HAHDOTeK5TIT zk$`9k;-&LZXs9LHovd6D?P|8F7{Lq!yOc*;w)CyCt*?zD={<1W%ot( z1pzSM-g*QfUYmeN<6(IuVWDweln zbP}@#pv0q_@p`tuN;$anJ*z9nZqkVq1#q4@CCOTc82w_&$J#ZjE~awR&`T@3LMDZ! zKi|LjMv@=-D9|xt1hb zGqTip0DE9!k#5*1{^9e|S?*n^{~ z)0Ce5y8>R+jEl9^VE^5lkN0IeWWk-j+xMuM<(tvPXB~VNs7<+s5T3 zm+yy>E&ajDE7cZ-c|+z@bFGGU;RG}5VIGCGaF1G565#{tkr&KXqMNs?HBcp6pk#t> zCRbNQZ!%LmZ{IHl4%UFkvCRzq* z@%4ad9a9|rZWN_k6DPZLT&9$Cx)V2beaP0hYL=sih(|Y|Upmjlm7VP= zjJtazyG+5n&*L93U|Z?QnUE|#LiCOWJhh^}te|_a|6rNHI~zJQgliw4QTwRe(`hpn zla1N>h_-OrJn@3U`;~S?cmGJPe@*6{R9?VJY$!nCV1#qRa05WZs)=xS;{In9{@3<5 zw|)=GZcllha~E8CH#J$$>>gFA>QX)vldf|Yp+6<<3N^K8&xXWSb&8K)CjXFrXKXO0 z6hq@-Z8cR528vNdi%49Cb#p{iPcodViPsC~JUr52;0ZBulorC2>hmS3YTqK-)9Bss*&d*EE|lN(T9fH*O-2;)hxK z^10Z;6^`#!p;gg}3W3KU=UF6;pk>jasY#~V`H<}Oll6RQ=AQWf##(>y_5*>P6s0r_p6cK zPz(v=6ot0?vYuONcF-F0Kd84RDacT&tD8Zm7R>=Q`WySxb2kDAhzU6odu6d$bxI?>_4e2oVA$9hYQ;j>INyJgvL)C zrO_3-*_gwIS5zkqCD5~MFuWe6F9x+>cszU!rSqDY)G5`hdGQ^UWF1d$cV~Ke!swLP zN=DGns)Nd!8+*Ik3k&-Bc`DlJSxUo~4AbAlr22~5bc7^mEt{>dQh%jd0TNU=$Bdh~O8X}EY~T!Pf2{i|=RF1zS45v5bUK!2ZQxbH zn&}q%IW=;F6d@<>JI@C83w3;DpKco&&RVrdsGBF0aE0B`Hn$cP11oErq5a^uCi1!i zgL%gTS~}Wkzt>ZvW?XFGOv4x6F3Kx24K$X==PRH10=Wn^umDJvp(%hG0KE%}!_Xpt zJ1J904hb%HpKq31KIs?W(|0(~SubfpY)ijX zXR6Zd~3~E1b4-Ru+zzz zEyd(a9mV(}XMpZ_Y9xC*Improiyqe{!WdOOt5Rvkz0dyqq=K9~ z-j>#~(wz3Z9-|;BnRG|8!n~^Q09i*iU;m)4!OmV?k(KT-*n;%m3Ti4@AlQKIIiJn# z9eF6mPuevo7?=g6B5a4zvD*$2`-!! zsXRgA4hcb0p1Cm0gjAgJp-(0J-c_0POqH=w>v3a*G4|D@czoQH4 zk!)6c7TwMsPXArJ!obWg0MZ_PWh!*UFlkB>pl-5SMglt?^L7KyHD~vuvn3F=5 zDLY7-%Vb5ngq6y1A;e0kN+qrM=w~Zw8Z{I$E=Yo-s*_wB(o{scX7lG?ir)Zvh{~J} z!{b<04#6w?ElO0}D`YMmyvUn!btlLw=-V&c9yT)_LsTWwSf?&4Zz^t?rXFHP1eB^R zb8_pu^_4lOHZ4G6K?(!uxL%Y)LC%2HH z1Pv&S*#bz2hxNdciUi`x5{N;`VqOuoWO@een(`KR zc8%xvkdQ7eg-7Wv^8B0UUjn19vDZj8j2S$k(;?KM*L|yl^@G^qoX9Se=Y!%^NoF$I z%x0tcH%V>s%Ul-=rQzZM$#uz6$yCWn62nCIM1Lg@rSsx#$uctC?1%Ycb+R7b(?Q8R z`8T)uxFmF0Jx0$1*V|-Hp}%XOFq{EV-ohikKooH=git4Uwxt)^McnW@HUFY8v%r;N3{DNRQ4%=?j!nVFVp z*U28X=ci5mebigxedGN(4;ns7K2ko~udnVP&z+eaM*m}RZ!W111J?C z(;&Y7cmuf;$rJhZcn8|Ol#ni{I?_#PkLQf7nG4_2s|%?^{##K02NiD`QMUZ2ams);(DoqG&x5@&Nm-rlA5uIo$V$VWdDro+Yp&4kNXL$z zDWa=G@CmT8!Goc;Ar9|Pac_q*VIxwiUPEIh;hAX1$Co(wJvzU78@<8%@al&!jAc72 zBS(x3DYnY}+(gNS5+=%%RE(}69l(TWmr0pNRxsm%0o7{$E1Ow;C@okH;(b7OC#(-J z7wh`l-FjkJcj`dt{J*>5GcRxCGZj6KIj^d+oCX%9>X9jKsImn@sb>B|%Zh&5!0Iqr zIevkWYx?Hnh0<4IbCnX?tw?hSr`S?uIMYj0-J!>nN7WY9laj^G0T&-}r?u42a3R^( zWJ&w!+JTzHDf5znx1_i8Y{^(W|B|IO$#;MLJ9yVLJKqrfrCf=tqQyTBbV_OxLH2hH z9Tw=|bXJnD=RQaj{l^B`{{*xup9mWGKKEnJam(l*=g6P7wr}QPPM&v3V=P=8$UXuL z-*d?;z;-4;e|zKiIW|5T$%nTT=0JyZZ|QZjQg&MohvnREtheTE$z4jUH6CI5jBlat z7I4=!cRQtMeIZ)g5o39MDCBebyIsGj&#m-Bj%f3?Z|v>ss%XoYj~)Z|$f47fa>tLD zY|sFz6TB-B%hpQuv9)P&26wOwh}kOvE$l#_b1TU`$~!9mMA$FFH%@6fFppF2pcuq* z{To88pxK|jPE4=u!I}H*bl|w@K+svbgAW`$&(S>R8$FXX5U6Y#it6-1i49C)=qLos zc+e&#Y)ZymJ87st$3D^yDets&bisHM%w#ckiMN(g6GxG8=J#nT>?Z1%VgphTWu0NxJ?tw{ujHdbXEn{z1V+^oe1$n9=SOI8m6&R@bIOOn)Z*^e}X36Z+l`{O1XsL+4j z0Ug`2{#a#VcM>N`4*`mC8DbSA{yCxtGE^6|zaoIRcwpxsNqpSHiCz9Dgj)cJsecjH zBRVG`QBhax*H9Ltd~Wo2!csuW@l_9SQEXP3+j!Jba0LCuYN8KJbhFB}6_skRM5uT!?T zzUhVEZ<-O0$|+zt>Jl*SIQBhG;xCuFIxOm3RT0pwf6AA5+hp;`)emVBsCZ2Gq!xdi zlb(nRf74Tq=;lig;{S7ZgqXv92R&ujdjG;j_#E%jNhFSt4+D-MNI1YsBj^B4yI=Qk z3+7pCO=Pg6NgcsFF8UJ7wJz{(cYfkS_UdZ99+mQW3u|dgWa=((8t)%eW^&PH1Fs7b zAD|lFy_(x5{zd5XSpY8+0JD4J8SKOGJIl}GYH_}kszRz7$+-kcykxicU)O$@gZQuC zE(7o#C%|4Y-7;R@oWIF_`Mt~k`er(~*W}Zx7i^PA9vrks)ipJ29#Vra9?B^oQ|H}t zyzP$SGrizz1opSunXa*h?*=BzcENSWEBml#?_2b}o43AZ^|2H#3D7|Pvr*kq3u`nH zv>)>L&~NK`kKqYAj>;`H;rO`c&55Pvt$Tv{_6xW)5S~83@(NtOsS7G)kWsTEs4JvZ zg@M#-lWr9(wvo-9(L7>D7dZs^;r>p=vzW#%H4oEeekU+1vl7cz)>GEr*;=?Ati~SB zHE)1l%Ez`coGA+o`Sx!noV*!CV^-vegQv7+8(R%WU41tKEj@z?`2h_5G= zJ-4}2VR8yi)QiPUz8%loyVLHP_qgKAnS#4>DeFhV=JvJK)fj(Q|01)AFJ=2p4)-f9 z`mHM4m1WgJT+#kJ5}&Q@$t{JkZ8H`JFfeRg+LKN2acy1e^Ib*#$qIL@+O0xH zfrA~Vp{FjY8NHJlL0VK7s7D?x^%P_lv|2y`r5 zSs`Mr>vHt73Nmrx6#>VZr+U(mr?W`rKuGJRU#WC8sX>*C^Er$JOj z(#vPw6%g+#J@prfeLVbFGLKlR%0hPpd!|E22GD^CXf>(bHWW*b*h;6>M+;DIYDF8U zA}74eECy*!+NFSQ_d=+@N9`?KWR?vzMHNl6Ji{PiwE|hx=;^A|#qFwJ#LdP@#&P=vfMvOpgO}MeaRum8IQ{Bwprs<~M-Cfq}M^ zcDL)G_z0H(od;_{_u3)sBz^+bK~Kz-vGg}Rd#0w!C>j&|-g8&748h+}s7KcuN6x;b&$PN)prhn;7zb)_W0%+glCStBu6L4H zAnf3ozCW7?rbTW81BeB5_%2~w<{r086=uVMlN!^yCRr3;t32uA*f?nbT0N23Q0`T> zIYGlVNFvNUt!uPCB`TP+tg8yk_=i?dc{_yQCukM@+UNrId3W6~Vh0HxM~r+Xa7y0N zt9LX^;v65qbl-xf2`kmErMKl?jA-yi5T({0cYaHJG)iR`$7dN@QoJzQ0+q)JXOAQp zj5nKHF(&0m4LTy&H?=$P~t8B_KV@FBdwFauXn#Qf@J^ZNAOOtxv zU>Y!P+h6+-Z{Alw)5z&;x4M3*NZI}_24nTinF%aY`vR$X{=ORI-#55O#b!Dyg$_*+ zG9|4euB`OzjbyuL+aAZ}3~OsD$H%h-V;`OH8nC@(r>y2%)}Tjr#KvTXq2g5Fhy{sM1EY4_#woZH_#v;nEn12lr8 z@kQ~9%MCAH<}X)TqI}kB`EN{}8+?M**MHu3s#{GLPJ?TNd3&H!D{wVP%|cU5aA^z- zb8t}GGB=QUw-9!4RXAnYIS+ANhD9E&AH#ORwY9Xl+M5KXZ}y@2E4ohOv4JV%q#ifB zaGIN5gb5fvBbS-JwO8{{^UpP^FiA(72;=I8<`BV}>&J?~@l-N81*i4+_~b_ao3NN8 z`Q%!VI|n+rJj;6X?$}Kz#HaR~@s3e15Iube7FKsaYns}vfuy&|^^+NvAB)mD=BUaR zO9&)Jcacyj!ng?muKnk15L{NZw^jNLy=>GgCsdp zW_9~dTmFaZ2sXph7DeDlZ;)atKD?)pl7>c^@urLAI+K zP@YUgAbNv?0RnQB2b%wa)_1}&k(XbbTyiBUlCotd`&amht2Xd@Z~fJ5y2e=zlC(Vj zdQ?`%#YXba5s6yv=uA;%jaOGJ{V_}jFo5=m^B?54HJ=%$o$yjRGUa>$38v+TzW9FC7GMv|(OziHqzWP`uIs1sL? zQ@z8MjgH>ibZp;hIiPoG4F|FvypQs|YC~Ep)Q4u{toz@eypCE*XO&y<@nKER*!k1@ z=#Ibtg3J1g0=j*Vj;6^ZVGSu{Bwm77?Ew?6AL)};B zJvs5IA-NR4DUe`wSDrGvTWwI%;T#T%^w4iA4T5X1>yDN+T6TvT=C~m~p5A?A{9RPd zIy{lBd7nm^?;PKskAz;dYS6~i8MUV6Dig=;DRp+M%(Xw#elc?!z|4|ArWONJW7u#! zp;)P#FLblTQx>v#REjUj7*6sX=^Sj)G56ekJ*0nTrM4v-3*ChDeCDJw_>)EXrHxu62!m0yudyu2kDQTsf5r;t8?W_s1WZWh8CO*Yn#+r|g}nqU``&J8W66=nJ>|YgmR*ua z6j^vLaGLEnbi=r5z8!>UU*11?4mzEYeiYL5c>mASLZvKK=@f0Lha*%9jzXTnB?cg> zidj9Y{7@u9dZ<2rwGM}qm=-T;N{F;1Yr?5Go>)9Ao&uLOUo?ykY+&ig z*D0XVgz^eRQvOZKV7(&tTT)Cg`7)RMo!%N+5U4;@@e-6q<_d85tEQ};irW$%-sG;ip@sL_p1+vbZCFEJ!?WeTGzEo zBY-=q%+M+x6%&2_S1^(Rkz01VR$5<3DtEHi*kQ8*yHFy%gtKq#y}H9q^&P!w-Hp5pja}jYO-e3V|qy`Q9|+xWg2(x@w37Tu<*ns z`lmzQi@;ww1oeU7X0gGysZG2y7a5LS1fFJf7tYUiFVYISm`PtGWqQJ?gouQ2mY!NP zdEzAphdzIN!vO8XK)yf4s)vUK-teY%epqQ~KCMUaW$)J7S{0Aua36MG1ghHS<)m@~ zpB96}%*mnSuXhr%lB#a3&1oRV0Z4zheP7wO{e;@AIE(FLI|_J_S)hq9*0zU(^&5OE|m-|KiG6{S%U`hqbhKw(N&*eS=`;{ z@lDf!=pL|Z>o8&Sf?geg$7%c_Ktf$X6WLjd1LPb~Z^~ueVLSaoWeuoe&qsr~p6t&s z$mvOo#~l~l^W1sBxd+9jpl3mxi-P70-wF5jq`xj*oRD~wa=4+#QkGwfGtE1r$C%FQ zx^$Hp-4}IF(QJXetmdKd`!(1Ql03oMflXdwS1TwXSZ zS=pZh_%RYHE~M%gz_8`nB#D-M^L^VIx-DTc8W&$80C4!vvh;2|nSZyev$jy5t+Y$? zg5?%UU2rJmf=qPz3+X;i|GE|@-5ygU@*XWAw{e58F4OE9oYt%npM>~vN;`0kqJGxo zuCiCzs_CrqU-MX7{tHf%D|O6AJW@zTdR>1{V@f@p>R8(D6yujxvg*=!7JPRtV=hxB zTecd%Wq+IB6;#IqUwDzt2K)Z#;_Kaf^9qdvV!Soh~B#68g-i$Y9HY&jIPAQC)@ojg{;QT~ zL)2v!0kK&Baiy~txy_%$k;8){bF?7YPYp}9=h7QnI={|7k#&vrlawejCIyXX5aLg*q;KaJgd2KOUF?S017an?4 zlrwm3xK(`67g$v$E*a&{GyVkBSdDXN1$*W2{A{=gr3goGU%!mJ9ppvjDL2!4C-}0SILxciG@;HGQU+`z3vR)Uw44^g*^iBzzm1Gx zha0|pQ!I2rqEa5jb^*J$XPQ8bnYZwsbluXEE@DMhP=hgYLl1>NI!i3GIsY8wl2J>O z6oWC+4E{1c1xK`&Zy?p(g?HU<&c9_Pa?;hypco3OIW-M~G$?{wHlDOhUp8nqc^WL6 zEA!8g8QRVs7yFG2_v_u9uvnpT&+3duXPS{ai6a;()nO1m3Hh-)33a zWEdq=PB1$DieO;t!zUz}Tdt^SxwBqZ-mNl(J^gosjRI67mGsR;-)Kl#CKG0EJgWjk ztZJu5WzDz!Zi8Jph`~WjKYHcvD96iaD7WRz_Z$>heAP&6D4wEgn?=$k6GQ4pwJA@h{ISt8y<4`K@e-U z7*55!+3Xp49Xv%Yf%+#0!~L(tk;|QbHQ~7+Koi8~sfoK~fntBGeC!J3P6Yk+u#{CS z>gdX*VD&R%;H!UI+9IA~S+X*SGwsaRDT-_FDP}`yzuWGLCeL zR73zebz25JRrt`AhtXGIqh)7Qw5invo?ps0+q80z+8{3M0H&XObRsOAynN)k{+*9+ zvJBv0(niIyavo6%;zL%+++Ha?;yS0uyMEh;WKB;TiNB+51Zhf+%!!IiZc5n@-oQFJE~U`^?CrAC+6D#$D>C-L0nBZGIfws^Dz}NI zJZd3>iNaIRliOMDO!V;cKS~&U2t%LG!}D=Yde!!%(fGdFdOH7H1S+ve!I+C|EYth9 zO+?o$m?y&CIkw0fJNUEXCP-O$E(4Mzrs&UY^ZQ;(v*#fvAA4t`$ze6Wd^dpMDVvE3 z5hyLy<-)r8A>K8tNcYBTB%}-HRq)I==4fk zY@WyvLrpn4X@acBK(X>OVGQ1yTvgbPl2kbO`QyZkwpv7SeaxywSfOvpJ}vNm+pzZ! z+*#Tg<(CZQ#)|8(UVR(a${B?`T3aV-X8o2h;BPn=(9|N!@oBd?5c09L@HWR%P1Vh{ zWtZ-(?@hhGiy^uf({}%v-KzhG}%jX(ezDA)QZu<4CAFVD-P+jXuoPX=!-KTu0U^EIwA~n@go<;A^&pE)XLwU2qY{sjtX1mpj z=6ed7%BW|AMbD=4X=%mwYCFycnAVm?(Ho$okR#lP^Z#%Fga<10q|hfGqud|0_Fsh| zxz{6j2)~c#eDoQnE$A|swtt=Q6`k3R?#Zjsdk^>a?*So*Twd7=ecw2dDP=@b1uqa6Ld@G|4kXOkeQ^EiLudX-fAxcAcz@?Oo`dYdC)@*+%^ru-MZcvS{8wkUlV>H` z9pkUADR&(~IhO~<*t{S*M8fNGz=ahAhbZEr($ZJnWeTEozM60we!RRfS)c^7*4yPOzjFQP~LMuZs zU?fhRJ9YNd+0%zlA3k*m4Fuh_HMGa$^0<6ADG1*O=84d@Bgd5DrzQ}Toyy~yBz;y8R zkADLT!OE#$%TLSGsF%kXF!1;%9|9DNdixn@agQ%rC`Zexd>8fztay&Vi{4Vm(nb>49G15H)y9X{0l$l8v z8avA0UyJlkZNEqEl&A|QvJ{>4` z4)R<|n$0Y2HdY(S!uJzfO{koBKmG|!y$sv)w>WG$*fo&1oow9oEH)Y2H!uax9div_ zg>%Q`@dJ4?urI*t1Nj<{==Qk-sM8~PBqIhnLi;in->N1U#$vY9Rn<8rdhnmv+&(hv z=Y&b5ngyu?2iB405~^FVH~lbewK9YM-tBPk4$3YvPAlayxlLXU?Ues87gIlY>nn%B zxEYuTDh4vQlk#)2Y6%vLu>;BkG8Ri5h?xd~yd@HP0>XrWnAxyWj*s+jBG<)|agksg zHfug9Z@!-mL$kFL$;;ri1QJ3wcx@UPkIl!XVfDbDKu*JofPWgf2Iijx{@K_{au-OC z!)B0RR>YJ*La2@5wf!b)x~C7&j@KR-q^s2t`b`U9T)?wD%i_?_IBk^2 z>^1v2bk?>fN&WLol|5MfO6Bc+q+f8*9rF&9#ZDv0mmkwj!MZvSp|z=5z$0nL3ckM5Iv%wayNQge62M- z!Cv<MLAbzc=dtW{EVtrHZb^KJtnaY!txzXHc3v)iMP6(jtJ0NplhGyt%AU*tf zP~M-R)Y{4p@bqo@jhJkVa^L@xX6W-E4Wz#)n+Dw*Esp_*VszNhr$b6z6&VEwse-;p zZ>}=y&3X%Z>d?{?$%N5jvoN%_$Xj}exln#0e}a+OknX^(Cab9guXI|SECo%J3%DEn zje&+h*xwLn@Yhy^ib9#Fx2)7zMC5F-Y=41HI#{@6CH`FVw%%mw(DBQ~w~6al+b$lX zk6mi{?Z-G6oeW}Lc#HD5&a^x5hM-u}i3d23O`_~hD{r!+vT*--&6S!9&0SP?Yin&k z(b;J!ZlXPT&U6^~j44{a?%=B9;4w-aFOsX0#)B6geB9$nd8cw`PNmsh){;EobjwIN zXCDScrCPR%59J7=Q5`4xU%4t&!1@ z3mUtZ@t9#i$FOEAsyE1u@<9s;I^@IUm}~*7OdiVb0amclf%|ODOpsEWKKQ-HY4KRS zgML=>deBf1G=hT;+~bh~b$GzywRkAG6CAd>;fBvou=0$}W~U~-nZt5LNByc79BW3@VXE4_x+4}fUNJX8e!6m`tZ2Hk=NsbEmD*4uiI+v+@r9&gwFFXz+(tw`HQKc<`52wmsM^ux{|D zR?vRuLi^?F4X3w3`_)1FwFry=1EIkmj7cazWJONNEqN)?DfvYAN$!-n(|}$p?H;sW zZL%(zG8LOkm@>}9=>@Y;P?z12T(a>Nl?xa&MUP``9+&tF@#)fFGvB0Az>=wJ3t48-Qx|~cRH@B-Zo6hV_ zYdIjIhqjl3ErLAgavv=C-qXVL4>isH(YeYiXm%vkhQJQdajw2~GTZ4MiY zCngu~PCxAN7f`k$CMRfZu=X-tJnC{foJ8MoPpE_Lt?ln|-9Vw=b$Rd(V`q6Awfl~o z4Mq!qLV*%Irnzvs^~IxYR#O+{tEp?(*PsD?ejtTN5i*$qeRfr9u5AY@kc<^C4CFT! zQavee9=McpCHZ24ewAIz=$J~*?qr?9nNawc>li8qg&lVsxcg`ZHmur@6AqH0rw z`iExu;IEQ&W$BffBDz1X;SaR11ry|HBjpBVwyo4!M!A@JGmIE`X)t`O_tu16G+3yt zw=~;3dGsLJ9Bf;Fsly-_e5e8AL;rfCx>PsP-pDpL;xH5!Je2GGuB+F4wNjX?*A zNm4+P3D>mix{A82ZusB2!Q1}D|9H20@O}Hg?|(FBs=E8u4JX`l zPSrW5J%~5T3~RibTPjPd%S>y%%kg@#($_|rSqJB2<8VzF;644957l5a&}|7Tpo)WAeI1ln^f4kiIMBB%K?99Qi|4`HZxT~z%*pJS zbN~(M>lLGUqFMwzBT~FqU?^!G)T^d0@{{Crql7Qv>l0&n0&*lohNnb1$yb`EIwZb! zh)6S0Z-f#V^cmUN>O~Wgy(KS)yI6+x64C<6pmrlii=kacS4$(o07m2x?&M5ccrT}v z^wSL()WGCwX&_kO&``(*Kv>mDInyEszg($5c%b{Fco`q%$9E7JtvbjD1i!C4l_#RT z#Ygz$@ZvI+^mCPzd&?qZ_TTjJ((mX)zJ1}T5dMT(Qr5h%s)cg3nY^u}wNvo5)5sTd zi0Bn`hgsa)+w1KkM@NJGV>EJMY{<3y4C<5+vlt#yz4s4gh02lD>G=;RCAX^0D~EQ$ zM%6+|$1+MusggrVIr>}GVrix_rt0g7cE#v7ey@afi$SVU>EQ9q?o6J(rov%vqK%z3 z6{aTq-G&thmyu82vqjR}^~IWRP95moIZ4OkecKZ${KfuLCFjVGKivIUkVbBGnE3qZ z)Av!5S$aj^++?(uk}I}O?LWGI&s!%B)fR52%?+)!9Tpr~8}$XHeS2%@RDaiCB8K<$ zbcaXDlVfFNB~{yArghR{sDu^z>8RJ&Pfy3iE;ww|tyGa8d zgj0GsM9G1aiJO&Gw?$IVLz?^gxow(<&u7*4S8oF%;^~!@wzT*hoU4~}g4{G4mUDs* zWpjcC#K?AGmq)aVE`GE78OK@=4!N1&tK|_FWp_Qi5)d|Deb7evodFM0_gqj21>p66 z%nJoWp%{^5qV6CCB=x1vIEAPsxRHH{41Wjiwt8CagbqHN`QZ&@yfVNQ!tKTucN1CB zkvKrZi+@(H)@->_yG(QJ%B`wvxmLbX1AHBpO7E%{46Ogd-u)BXMKtr<;eyO^eINgI ztAUNsAw~`-@cTvN{WTeaULQ3bdQhU7TmK+0y|~WsJr9#mN$8~YaDI;-y^|CB{y)ic zeW0(^@NYp1(|r|a&<6AG*;MNcI%&}dnZM|fd20)}aeqJ%5%s%}_nAE3MVbl1tbetqv3EQ5l8K=s2-wD8AiK@d5ev z4pLW~k1kYq0tf5LqU}(|rvF>%u_u1Dexrm**#3e~E*?AaPfA+B>3=smHbDuhU4zvN z=hfC$Zml2vlGaU}%G{`LZ}&9SAT#YN5T$HDS_7zIVK*;^aUmka;{>c#R|S+hSOoX; zdf)rXrTv?}fWIW*+Vfw2L-DFVy!W`YT3WdIk*AHcf0+^%srtpph&VZ&sD?`_zv|Q* zN57CWgDaoNMw>l09c`vNvha-jx=8EIswH^kCT7ZR`_X59Iwe^Ec)1RlSk;J`ruET%3}Yt5#h}C|NVtXhK|w zW%l0`^aq11{XE>Z2pSi`$~FGHng(N?on9_2e?eNgR$7V|TgHnHlJI#%=Z^3e(MiGE zuaqj0oyK@csaw_3-rF0bA(>gE6hon1X+r18+zT7^fruDO5CiUhYcEyyPSLr7Go`+~ z#%7zblP+De^`&hu;f{8X(L@;irbs)rclo{-4zHYYD_PcJcBPgs}Imkrg`mhJ2fpc=U~T5$5{>i{yk5%DAMY80Gfz;qkTqiS;L`OQOH0=fOFwHm z=A;q=GT?}iS9A!}ny!}kFoA}w`l0L|!$NrVqMj{X4@$_YS9k%rTQ9*~s6(3)Dpnh$ zuj zBJM3})|8f4HfeTD%8C2Ya3q}|f=rZ^^Y$wpRpbl#L-2YR4e#NV1H$M|hL)aIMK15# z|ArseM`Q@_OL^F>s71qiulPl!bu8holk?eWS{7G*`%&l!ts&6e5*7I&z? z3yE+PIUdldX995$CF5Oy576)#HSe_WCbFoeY4z&9#(oE_`w)8IRlRuh;9hc9j~wUI zE;x;Z^n{vQ#msANt2fs9+pIWXje?W129wS~G7{y*IK4k|{sa&25#XjgIGb;Je6KHY7k4&o+qld)my>4T z(gxw+0!WhZXYX8w#Gea3c{obD102l{qJx_UxA;vWA*EE(UT>2Pea6}oX9q%t+{>oYsm zJ8DKMknvH}tdwq1OLIDXn%O&S^1G=Y}b@6%Y}?U_q@=vAThkEjay8jrmh z=bQMELgGq~daGe$so7XZow@Rh3WN1z=6~t0oK?$N(9o;SUa+Nir3Uo^Bbi`Ep?^vv z-LHP`+eJ{Z^iSEKBB!O3vy*1diHvl&;PE?<&I{TCu47Bzn{#sUy>lLvnzvTqE}n~c zDJGa=(quxl)0f_z7$1J~@aHGa436{-M1#0=Ky4B%NMo)sBvgeOIjEs4le17WU!$9^ zIiH!eMDw@?Nl{m3gXrOvehX!HG_e~;u`IX@jgYJ(|^TU05 z^DBk~AB%Jv;C|RE+0+}rtLuqHUTu zc-nxhk%4;*&l#T4tS~J8>koI|rP0BIKUBcq>EYblzLqw|C?D|qUof~q8()n}FWM#5 zGt%RP^e|M+hifcUGeF$cDS86+nh+J{diS7BA_QlC3Lhn|{BK@=>4WodPMF%ruU$vH z4Aye%DQRwG$;m$unSFwO{<7z#M;sJ)1iW$TC^&wUfDhZ>Up1f5J*t6t=JacNKG6~` z2;rHZx-=HfYLm-nbT$xI`lKFcRdeUSg|oz2a9TL>vgQeH8JACGu2)6-cri_HsWw>P z1sKq`HqR3`aQH8|H{fjgKh$r4L%WDbBr2RG{Z^p^iR%t^n~?74Nu;RyiEhgPjHa>+E&dDFvPU-{8fuG(E8DJ>Ay}_L zoiEl;*CBux)#V0Sx7+}4h3!8q)UU`l*St#Cg_Nxz8G z(if_7CQ&;;!lNpQ=)C7vi6&RR)Vs5Q_Ba^3o!Hh@HD;wc_DnerkZ=v0i(YHs%hFtU zzqE$d!Qo5+8bx~O!JDNm4tgE0{FMLV;)5EQspJASZ*cGanmBar65;`B zh*$C{nexdv$0g}7=SQY*lnrn`2IobChu{Jl{y9FM+R)0y?6fPwWW&-6)h-XmdFWU2 z3+C77cUGCUlry;3i|7Yd3b7OgvtjAsG)R$M19CYKIV|076q?9YB_)qNzP)7JO6zXc z=GAF9sn?0ODn=U8lyob6RPXB%LgcPtbE%K!voTI3rbaW8 zHiwgQk)2!%aC7)B$)dNnh`fh}E7Stonxf>PiH=tSG}546H?g>^vVhXf(>$-y^=4*$ zsP6^v@5t|UCv6eR7U}SJa5$%u?l&u?8S0zu^V?QY?zYa378~wuK<>IUSAEDPyApoi z?0{bk1OYgaLYAef9Uybl<#YSIlponS0c6gojzpDkt{RksPHKr&FWsODg@aKsj`zJC z|Lm{C=jy{@WjyMzOa~*smD!YO*UNX=s|vSm*tTglFxGdpPG=)L`PrSFd!-qZ#zcoYraEeQ?+p}yOVJH+@g!} zx%@7lXJYf==l8BFEh<}5{gSDoy~g2Sm^?;p^D`j^X@@wA%n#L=m>7@rcki2cbLUhr z7>R{LcomTD2b7h!A_XCMm44NpEtv-9e5pcLT2h+~Bg*(EXX;%_4 zeKN_W$e}@hcz~XaZSU_M>lqL3_3rW4J{rUjAPnWfS1CWw$TWDmG^mQ3b%7t1snXK>CZA`4fy|I4#_ z#$`v6WapI*2Vfnvhh;rVj`3QZ1gqxM#=x+32MO^MTnOg_;kSI6PQ#g?i);hZ_q-;A zSVD+|Y0;)%55w@NozB+@-+w#)AgaugGnOX?)T zXcwU6=d!*q9}*(71zr>gaj*Z4;Cr~x1xAJlfIQ|9#mfnh-bihzUU1={`(;2SisM_t z^@1}mq>99OAxg)^V89p0Idx<>zB9f%J~=zwADxWuz*)et1mGJ1)yOySZIoc*TYRr5 zfmi`3-XgznT@5^*Y1F%|Y^M<*MQJ2V(oA3`y8*l&P(y6o6{WoL4pL#_Y%&cR`HIbi zWKpl@O12TUsCr*qe#8=q3yCxU@R!|0>MgY4jd$~Wl5Pdk!@MS_+LPv^(P^3>@*X0P z$%4o`N$^pnO;y&)S)BB>pmYn&i|a#CKHiNw`*SzZ1@jP}7=gq9Y8_b56;n=i5ba`- zULcUkM93eaeWEg;hVQGl^TWdg-v@eoh$!NkUKLxyxez&6&Z~?XJRIQ>n4pt#0_h;2 zUPeY`5rw9|D+N*!TK8l&kK|7HUS_@?JPu40V$B19+QQc%L&vQ~8>iiB;YvxjiU}y) zVK$mV^ZUyQIST3UR57tPAW$N)GwMofZT+#bqotZLfgs`iux%17~6_m#^<$v)|R zeQc5%+i{@lu;ZX(PyN6a{37>fE=moA2K-0J5YM;1+Q8lAAO`mA89o?396n$gtH8&$ zb+3JjkbYYJ)T<>_d2N}y+P@`OHDK<>FV5^S51B_C_`!~M_Q1!8uNw{*I_#|1PE}hg z8=4yAh;KW*D(E@8o%4{=L{!Be;E%jcz)#dZxt2?+RpV516`-`R2yEVr81EOxYqahG z@LNEoKsRzeW3d>&hm6`(ToJcvji_lO_*$`O9pUmgSuf>vyZDVkXAc*f&Gn&IKdyTa zemJa8aXs4!$f!^AhxbpjSg9Q#ed%&}o7>S)FWL!@vx99RO{xx+!xap1v;g3GG$G9|9wzGV0_qIR0tWO@I42`W7MVPk`zYdjcV3Y15eq zFJm3)ak<-W(A!VQ*Q0JfZa28H-LAE0E+K>L$B(a7*J;(bvfZwo?zT7EZA$D#-O7vE zh15G*Ofk3VB~fD{YJ?JD9p&^o+;+;zZwxql5gGbeL)xgj zYOu8`zLl(PblOey$LeIjKRQ4}A63_KHOLx4yD!TkedY16F5K&6I;;f47z9_4Q?ii= zh>>6nBq+3(d=02p(c-tsrKpsazDegka>ve4&j`J6yWZR``t3AqRmW3(!EW+!y6y=H zFRL!4b(4!VFjl3Hs-s-jFmX}Meart<`L#L?VqA#wb6mHngt&;94&l*(A^$LW>FreyC+KEr zE7}t{JKA(uhG_S9gdCJR81jZm0DlGgsfRn3uX>a$DT@zQ(e{wrrbNu_a1b@Ah9Ns; z+dk$VBax?}Mz34mtAA_bYxHXyRtzsA3kt2xTabHz!GC7F`ut6;rImE`R@2MgCAfIW<3@lW65Ub!(If>`e>Nu)Mm2nu`L2 znloy^9ZM5jPy|6zd^WXiWGh}g_FBt1^1_jU!xQxQfz*Z12v~Oefk&t->t-(BbQazS z4F}J@ec{MU5+M=AFD|2X&ud^cH0d8O+y@W*P==1uz0jC>RL^9?Vf}tVj;up!PZ^5K z=`Es~6O}1jdDqM2=B<{-Cc3rNR#Qn-Me6%&saSimv&$TI=7F#sP!UJi5p>`@ry8$~ z80(3$=9aRW+HlH9Il?ZR(rIaA`QM~jj&B31kD!FSx%XrOu*mR_B89o+o0Oqo7;Bi1Gt zXHsGU_P%0lR+O__`OaGhH`+-CLSFHS;ygH0v~0O<2>X*`fJF z^E=I7HInv0?Go)OZHd;e?bZ%zcW4i2FK9p1eyY7Z3!9ZY>(*Hh&w6s!>RE-eDrPxn zfmz{M!?X6yIyCFuS-+X})vUkHg1IwuiQKul&*ZMl-Im*&+n(#pWpabLvD{SdQ0|W0 z1Gy)1&*XlX`)TeMxnJl0E%$q!LN`lygO1Qqy1R69bPww0Exp;G)pw?%OpHtqi@q2w z1T`Yy1fDP-GoG#cZ85xYC0xB3{=(~biB17`+W~8%T~6fGA=}YuVd%}$ZIVJduts{^ zYh!Gj1^0H1FbT3R;17#*@?h_|{m&Z$E zg^2=OTBy?MBLS~HLL0lIEfeIi$s_v$w4X5zzHDf6**jWEhcg&<&|SvPo;q?vZAE?? z&4|4%Y0@wH10o&njVF>KuG2$C4#5tE&j)GgMOgIpYJ=q2tPx5^Ts6`E@| zjjQCF#@a98#Hrg55~OF}p7+gP-@F7@!_~_^eX!oJa-Ou7hL6C1>EWW^Nz%=fHOY(0zOVjhgdzl>F|*g zl%P7ZXM^-}i7F^qmtS-kQndC?L&r)_Cll^G-+p$3L3`7!2AyzAe?G)Lvo{<1V*?8` zi!@sOtp+VpRMq;DYPq&&17BT0NJ}3yXj|VD^~XFr*<&0IZ}{^&@ZL+Cc2sxM)}Y02 z_N4-mU^s{m^bGF0NPNETE$Qc_#SBwNSvUvhVeyE;%>afWZdHFJ*GLTyfh76Xp|-_g zns))WleleN!Sfw8qTr-5Lo@lnh!7)>?`~R?^|0M;Vs&Fhqm^Q!wO@|_EP9{xhy0nYOK z@g8uXhX|Mx#mNov;#isJ7@arvaC!}%Ov`t!V?x01i}zpXf2;p&_wntQd*8-+pdt7M zP%o77o54D9D{m5ve4DR8EEX&9Hn9WQ-)7MovQozfgjkUFo_GJ=e%S`Ex5Mil@CLSz ziF)IBQx(IQXpfh5F|N8rRrgmuU~jZ-v)4IVoJ}s1tIOTv9CQviMx6ulgKj2*SA16j z@s@*ltBGUHw2O1ezK5IP>~2D*Pn=s&Fi!J`DlE}%Hgz&o-SFDu8L^ETE@T%%(HUaKQ~+c0#hgqJej+g;by`kXo+M zF1y7b{R|e~VX(pr8Ka(MS*EkTWlQab*`Tc)P^Ky+Xet6!M_p%~wI1h3`H2w%KBMkt zVxAbqiHx8Ocep?q^0ifbA-{1gAt9p&nfrmHBS{Lqz!#x?f*^M9=-b`5XZG%QlwkYo zNHg?C2crEr@P&OLimTx&xpG{3LTwUEK?h}x#W-X`BI9L0ID8UW?FU0dAS8%BLI8eF zq&S~fPzJ4~$PRPb8Lxwu7f}k|dO$OtF-G;JzgzqFhvA&_|2p%}9T&xj5E8<;udYe# zByDcS%g|m&M{RqRxq5cP`leS}wzTePIn;c_GT1rXK4R~7%PEnSbzW~P?aEdp-~0Hl zrJo+VbGd{sm2~Up1d$gR<Nhdjb?PzVR;)Oi2CfvMr6!zX(83=|H&(qGV5+*8)Q z1$V{cY@8gL0O1JD1Hg*_ZK$;Fk1&zBMc+M<9_t$0b$;S>-xMB*M8zl=RoY|+D0xFWN-`sb6^8LxTaBUB6_Ju87 za$aj+Bp&n!a959~f1J?jcMlE@rbqEae`sioc)j`qN&D#Y)m0soN8Kr!gBFSr1Wq7D zIX#o^8J$WVB2V{Jl&xG|@N5I^wtG9hopRVZ@5cEk@;u3pPc^(4qIoxP^Dcb8u;#hD z$l|g{vWeypk{E(@uy!laDRjhblr=GKJwZb9``^MlFRefEXo}YH)CBKJYi;i``ZWfh zTpF}c^OWXF^R)*WI=!S{|KyWfqvaFDba&HG(`a3nsmq)+0XQm5zScw%SVyPJaTs%NA-rCg~_!ldVL zFJdS!@AVLZE$9?joIUyp3w7j2qrA{ZCxv)V5Sgza%0_U`UgIqxpL&27R@482FTz={ z{shGFP}-MDhPZ&=s*#SV8jOO_PTN?<#(EhG+sU-!x|1Ki_wBKZfnYG|_Zz~3KY%=# zfSIT4oxFu~x&TtwOYd7GDW02yBR-nslCC(lzR~M)(tHJ9yoS(z5A|}@@~70F@E7>k zskNMqx7u-sg|)O2TBPm%OW)Y!vbE5I;3S|X*n}rK?S3^?fd(70T$QYvH}P#Xgmg*W zCRqGdN=`0RTKXrQdy(xQ=^mvg2KOF^3gl<(Lw?3UOLd}*td-tF{(#T#pka$Tk?aff zkcZOsk4x&URVB0*z_ULre8|w&OjNcsmwPGB?c&^IU41azO8e~&(aFjgUv`VTwVh~; zoG_gu|M=12Z>HKOiYDor!Gi6LJ$O^MGnppFhI{vQr5I!crrk*{!Qi22us2S4V~qo= zd^j)H9wN2+^T)@&8KixTY3OCcy5`3E0&-L1@QG$R$S|Utw3r&uCNg;c@tO-9uDe$w zJq+)@!?05Gf1h<|U7AkLYX9+hPrtMQVLD&45fpMIl>A1OuX%!9mQQ3ZAzQ~~W7=3N zzUwu6U44sXjfr+3Vmb*&xGlj_$3mYCTqJf^jTS|yHh(i{<8bZ3sc+$&k$3Sx!XJqd z0vF(Xl+Y~K=C>jj+eT~e`zT}6GYn!i)Y0r|@9fI$>JHKBcT9V~d*gE1x$IpSuAdv( zgSObIKB5QShP)tN6&9jFF^=;p5wT>9h)Shub34zQX|0^E*W(W*{Kt?@4*;bFd<$)J z@ov)V^=vd~)k3FW5j$~a{5X4#JgE(L#l8?u;vu9hl6Er>udGpP&8_9_ z6#~AouKN#kL@TY)`vXBa4g3bU5!PmA?*E95I)YZe!RZoNlJ$73F8W^SA&I$i=lZ$4 zWzsAnD#+VSpp!Zi7o%Z%jN5mT&<4n4S5G2LH}BTg7{Wi6CZ-5@w_KW7Et& znPyHaC;R$r(+SCi)=kZBO$;fpNOVw}hy_zZ7Z*n+N>JMtGNx>lD;jsl$U056TRaX3 ztvAxgg~{&T1lsOowW8&FsY||nJqJl`hsM#{l1&pvfDY_1~&*OFMRDH2RGi zt^4>P?`iVHN3qvW({G<2{1Y?~@Y#HLlhmMXF*xOn5N#jP<=8yFfYeGKWK(-T{e#<3 zZ`icx4q7|JcSk}ov=Mjl2}*lpK!|AdJJa2}_FB6tO6-jl99_-@O?ydQWtVyT_U_)X zVeRv`5YnbUmO@NBoKJ3CZ{50u)_R;>G!uz1L2Wo|qIe_O;>aO)kPY&)1AM^m58=V6 zZ#YT-t*Va=+x^rw&cr#9E8E7`H4)ltx#T4^Omn57hTpb<&t?N z(tU*1Z;rP{oI`w%5aJ^~ArkD>`i|qg5BPk4O5 zM~hZirDa4`o6%VeS9Gd2uU7pj|1N);TBj|+oxvCrCAE9;kjo+3NiAn-0WG99v~TAD zl2@JR+VT|5Y5PE5xkg)VHg47OfY%};PO$i_l;~1w)&0Gr-SW_^nHco+yR`I`>s3at zoV}o>Y7E*sP*YAwm(@>b1wqi-J2^ACtw_`&HbRI346mgV0WFsOy*v5eSJR+4g8c%) zqd7;g`*N0G1oj$si(&!xB!b%%Z(t88{((J!9l)+r+>YIbj&~?FV|l1OrLbbZLdV+> z+=t*s1P>rED2B1S(D4?zU%7quTGR*XkIMNBo0EgQWAxqQii_B-IoD%Ppne`xv|*3s z;MmPMH(`%q?_vuTKf9Vyyort`Y++6uyFF(P_CU@PsLv5>LCzxVR>f)bod?bPJeHet zH#T4KXDoNd4D2SwcbHc34uW5!Ic~+O6w9#L==cl80Zg4;7eJq%#ug~*v6~fV5QqrW z*bRyyOoP@sA3Kjdg1vL~sDj5H#m-^#b4IYaic<()%Z~Mc;w)-^A9}w5y?+vocLsYz z!C*hhDa`h9y~2z7*@kfO5Y~Xbj@^saxm*F!e2-vSEQ-cohu{7h5pnGlb)((A*azx~#$~(Y)6w4r2L=d$9^dHCjK5 zZB^WetyGj@i)TEDkvYe)>vL9Pznn1>;j{rWpuXoT-ox(5*@x!+9irLq5uIe3@MsMA z_z&!boM#Z8e}-xPCy;5Sup>JC8z4H}fxUb6ZwS6d@b_ypy7?O1^5eNoCz(e74p|!g zNGI?=fPkO{UCaM>KyiNyDV*9N4)s|f=r(?h|d2GGHqlqV}xQUb|>OJ z4AJ~%h3V=Q?9$aMh`#rucF!m-Ui~kH8=E7yQT!8|jn>bgvDcyFyJ(F*G{0s91&V(n zKKTv8-x<{QOYBk9_HG3CC`#q;&^f|u&J`p-x1;$W*^wc~g^r7_exR6z;10w)ixK=C z;k^rcB0{`81(K0FvvM#zk_LvlR7l3R^e>!ycQFhpxX7$ztR5ct{===6r=@^z_w#Wa(Lj z<|5Cx5#i}Fnj4b4oS&lkRHL{e*4`=00me=ZQOkANf}vH5i56 zhbgLyn`!K>MC6jf!7@@Lb(FfCVm&&eKf1n09Yb1)rNg#8SA`l^5}M(_-R zUm;j>^)J{`bXj9e5v)e=($!z1k*3f{ zQ)r|qG}06rX$p-rg;tW2{e@oa{{T=+0|XQR000O8G%v_HX%^Im1Y7_B9BKdn5&!@I zQEXvzb7^ZrZ){{=R6;IqbY`Tx2bkQ{xj(FR*%1PUmH}G=X)(rhS1@2>1I9Ks#=YR~ zt=`^ecgjv5X;eniXnHT(d$HHOcVg2+O+rsk$R#%k(e9OV`OYlPz5MTezVG=z&o|nm z^PZwp-|~BpziUnmRcZa>R%ve@}aVlg5qyytB-)P0T33Jfs9=HWFw$| z4Z_T_YOOvpd>?4LWEH6Y)9M0!9rV`?peP3hx7?b7>JkWFlK|;D2%2K6t8LVNe+-%r z#?t}H^Prq;)#G0VAqw>|eAGZ;bYD*4fA#+q1l^!>pj$xd z2YM2uf?m8qPljeft8UO!epF{F^bk~agU*F+hWs~Z6|?|K-JoIU4(Pob^i4maZ-(xI zzP>@<0%^e5exPstQU2{e>g$e)@l1wtrrZQl6* zp7NtQQ=xS^U*Di}p}Qx|y+Ny>{gYPSpkZjnq=6gsO+TV_6|?m~ z<&9MZjg94%dCO|6iZ_*%HdPfg+<5T6o?Ddn<4TJrn%P#;&{$qulQ(bnyt(rhZ`hnS zckaB|AS^Dc)fUy(G=qYf`Md4T8Vg6nDMaX{-PuGu$Q{8xKFw7e1U zV)p4q@N6Zhy9}xYImOT>P*w`s%9b|#j~@Sbe~ZAA|8FfV`q5~%fw~P7HEO{a@}PNO z6!SpIJZLcx< zkO$(%iIvQo7=HmMDFOM}IcDqDLi_)()_3;*MV*yk4Of6G4|*8%k{y2mDF0#Rx``f3 zphv*`wBV`(t275(|6eo;uNw0XGesUO+J_RlRPvJ&zq4qD{pSz zqP*pK1NYCkf7XoL8F$V=XW%m)n~^`GFJN~Pe;)z~z zzW@IFub`(vfWH3%gwL}(32f_+e&3(_{?+fD6O`@yN5Or4mi{O4uktTfe!1k!hrgWm z<-K3*`{L~9pry~>{QUIihd(!dZuljxdVlJFTeZC)(Etb36O#F&<^Mt^aivO+68?Gy$^bF z0SlH7@P-rm1M~^>DUgW|pq~SIY=(XXWU~h1AR1yI9%3OkWPk+73yF{ilAs{u12!}O zU4TMR3<^ULC=Nv-3XD7hC4ogvgFUfXYli`U907B-K&QbDo`g<8XP|GPe*-&fg%Z$lz(wbPeCwb;0%>o8{sg4>8z7r{ zz-KU6vwdLA%At>-kAd#|Gg!CDKTQ1l!&xl!zkfM^Wucs$oLhcmXR_%B#%_nk=I4wp zm^8Lv^4L$Nyr4+`vGG44N zq_J@#qqL!xaVkSXC4kYD3{z-r<05d?HIe-PAzD3Y(2=EJF48T)>g#d3t1e8NVK$M}qIXO%& z;61=hKo+379Kb_h!*e`2fC+NEle6mB4FUE8{FW1(4B2u3a{>;|iB5t3Iw`xa02;jn zZOPd*37NEc()W|`$y+CjQ|_3ua>}MD)+zLq^HZUzHB*0^YgXN{Y^LDB;9oI zrZ;c;5?Ot7{mtfEZn@>QTejY^~6>rX#9`;!Ylxvbf!iQRqQ-Sh9>boaKqyYKG5XYM^u-a}1uPW#KW@22mX z{u}fr^bC3)eI0ul>$~^Xdo}mQ?oHme07voP;eW)xC31<)#2dXM8u~KOaCJ*!{q-XRe$1?#!D(mz3`QVsfB-d`kAMtXI4Jb@$Bd#^&-t8Y|)HGk1U$IXyKw~7C*B%`CRpqyrrt8 z3zqgSm7gb{KmWpgFGOGX+lvJ+URYMKEVk_8OZZFXm%d)UZaKC5=<;ut|9!>ND_&T! zZpF40dlg=t{p)x`09Zh$zuv1E9S-(;2tF-z3;oisee%uxRk*{ICX@K8e6oopglD}9 zF3k{1_)5MWKKevIaxxr8lvn;%h{zA{B}k3aVX8sYRbR@xHPn7`KUE3~0Zzo|c#H{R zy@3mdKT^>rJ zq=rmms=mXRv?k~bEUFIox1R0lpqy!(@%lM0=9S!%p9sbMtr64_sZ6hxg0gDb`U+PS z!PL_=Y(2a{pTA)vS{<$&D#jD~c3o$)ga|Ar2&_a09UeMj^N}GNe6l!RUxsck)otCo zFWy#(n+?|eCY1D%0Vf`}95kP%?aHKS*LtNsLhKSd#ZLJ1%=s%Hqi6I7^M`S@Y#l@7 zXPt=hYxy0Ga_^Tc0T_gF5h+Cs|ou3$q>Qi%4&JLTs=1FNEhM|U6{*($QR_T znvXMS#4z{?u6%7pWYyThu@^Oh&CNQ{eYS1+^KfgF5iC*ox!*h{PrZho9cVq&)$M5N z#bfEvfFJGJn|%4?!$##-&nT-Jsn6!c(|Vov7`1!S9@ur^+wLX;kh~H1ZR>+wr<)L#dPabm2MWLFFY> z-5{?U+oO4WRrH{?ljx%|j}d8V?Uu-MX6i*GI{ zS)kmH$~lZiUOf8d`7^_>DNicf3%Bjqw*R91BCdW`{^{7)8mDBD>|%vue%;FrR;%4c z*?6nh?6V|2KT946vdSD*d4N@Jb}~+uV&OWcmNGkdhXZX;o&AjGFB783doW*+7eYkT z%Lj%~pWdhU;x>^K9U?C;6l9wqPlDB-jWuex+<5!x*5lae&f2Z3H!fK=pQtIXEGpY; zYN9QSo(E5O>@a=pEBDvfiD6M_CsIPJ-*W<{v~JE!aM~KS7TaCq<_n3DQysmXy`BB7 z9jUItlbvV1$6?;j2Lk9 z?owF=qtAkqlEip1pGOcqM5gCp^e}cNUQxbo&AwIj1Zk(uj0t9Vpd4|Y&5E>;Nb*7K zy|dcQK7wc5G|lh~gDw_cTQdhMDU2km36~LYCTbE*F^e0q#0ISwF`1Baeknin*{*Yo zQp9jaTSu%7s7h9zq)`x6PaiKtiEc@X0@0W(s{B5;;3vBM;gmlEdpQr|#W{iVuo7&p z<=94o|Ec?p*U>+!`uUMQlp}ct6C4(?1#6;f>_+37O^QY_R4x$>{^qnEb{+qm`4F>k z4l63jE={wFE9DB=Qg~MZFO(8JZ=e3{xp!f%*3H*iII|g5k3nNynvfv*c(3R3H0BlM zDc4IFW2Uuqv-Rbc$DOahv38DaBYN1w^spViU>c}gfzDmGdSOX%Fjz-TRn~^MCrKUq*Wn-tP|j(4@-C`Mjv(+m}b5OPItK!6fJequb2G&Rp;Dz-j;C zYy7vKAwR5cm8}}(1@WWjWf&~r)o*0%Hzi*^8X)*w-bJ}dtVu_cCPMEZbq#2(N=kM6 zVwfaxoChfCVMkZRa`f?i+qbRFzbyZxPwesZ;K2Awyf^r>%-Qad<5xdQ{>lY=!a_WP zc9Uh^V(it0jZdv2_HL?~M=B7^YTbGj)qJ3ma5RziCX^80IY~UdJ>8`gQ^x@`Qmy35{_b*#DvociCT2I7`ZTfC~ zX=6=;K@00@Efq!RGbd-ukH~kt_w&EJ-f(F5L0q{_p)}j}jl6Yc^ziXsqN}CN7;k|) znnL;_bj{w%J@}Hb=QS5cI}aThuG_M=x~!lE*G6?wV3wNF_GAyL{?Mawx!j_OXc3wN z2Ha>`l`hI$Ebk&4O99;x+ybODmRW^#;9!j{~fWvq2^t zM!VyKeWUmyMe0XRh2n^u89TpE(=0ZU^`r?Ybrd>w0^2)l@h^ZzuKV_H@?!b!cULM4 zl;^f=FQr|gNE3XVKiG=$0e2)2XfP5i z!_hQKt2mxvQNb=+f>t;Z7Nj5%OlQ0qY=nV@vGS zTDYuEU$PAqW}VYrW8q`Nu~3>YM>Av(_N#w;TQFvVDzj_pD42HrmgRpE>^|Pl*0MC|iY!zX5SlopA z-LZQa&Dkz78YZ{{+_9^u{7col{8j!WzNVS8S_r3+F*KseUZp}VQVlV2dmP_X&sZ!3 zU&8NPhbli;t>D&jyYV;D?r@L@C&XAgDswWTI^OH%(}WUH=|!Wr1y?JfcNon%En?hO zw;Bn9^v0f$#%UEqGeJ8j8)pzrQGF{3Gr=eu$NUku5F|Q0X@5KfUp?giP`()*dqOqD z#T+qweI4BxM5_3Lt*G*q%BL(@>nZ18we4q&=6t6R4G_mvtz1VxDz8w5NXgz&duq#x zgezEy)n=YUG*;~Ip^B%(_I=p^Z3J8)H z4cG!rJZ{V-NuIj4lyTDzBRaf$YmH@0a!luXsfHb?l z7{5)!6w-U>0;96lpuFN#?uKcR@#9P)D-oHHkqun-i2u*dD;McngK|7ZR$gbw#X zei)AMK#8@hyhbV!+Zid0l!%T%gReHAO>azX441&EHeLj%Q}lbKST7=lJz*)5Je@k5 zI2j$uyq>(k+uc^)3R`)b73C}3B|;rc@)n-Nc@^&jMRuW4XyBW8gJ9%Mpqibx<2=bb zov6`c37YVq_6YF+L0_f5u>ZOTvB_$?e5*rNv+Zoy6@zQ`GjucIAc2qU;`wPDcL3p9 z@{KzTJFI0b1(q_hg=!-8w4Jdt4wuDcc3H``X*@SoVEh0r(P1V`N9ZsWW5QGd~OYE(@P)ri?`f*3?{ zE5f#;OuG#CeDGIJE`{Za#qyoX3T5dN%FT*FS#d?llONOJ!oDL9{K9tBbs=&&dNFb# zFv<)vU6zC;lyQq5zv!RVlF+u61>oxs%Bbe)wO;`5Yr};venX8f&7T&6HabE{&d%-Xl15o zl!l|Z?vDjSJFZye&os2d>^5UXGLs9`DWAa3jTA3kT31qIBh-r#H959u>=&8{qP&?~ zQfEOdDD7qiC(a}jTnv+Mx;~U^@RHw0uc5q zW6-!pV`3PFra`Yxj;c{|>y+GvnT|qoKUL`*NR8flZ+&EIY-4m&`nmMe(QTqtvI`W_ zPrU0Nu{Wf&7!$mai}Clvm*dX`3Sz6{YvZp(wh4AHN;@pLL_2U$_-Q=NvpfyEhidx- zJR^hx(E!g2!$|V*QF-~WJj>VX>+^LZDt^!NF?eI2lr<|AIzr}Ee>f)(RQ2TuZr zipI=Nx64JiNWf{pBu;a+=;k&)v|pJqovNejT#a_aB(&5 zY9wr6TU=x3XX1j$nZ<@7M$=SCE4tm|32&}Wn`8k+Lv~vM05)>l= zBHoud*nTL|o*W3Z`D6aLFPy9jHO4EvR>|tM@vOk`Y`P>>nka=Gi8vF(x`*A7C?UDs zoCjAwkEowR)OoUD{D7wTMKA-x<=_l9lvOZRS*_f*mD~Cns?1dFaqr?waUsZgFy`<_ zu8**^SMr}mjZ(9y=WV>r?c`f{tJ}i6B20!$!`ylIJMt3r8~$7VU7SDU{<`;{!av3_RQ=O41>e;hkWRHNFaer33&x@$L8~jXrlfm^|yId79{YxJSw9dU!<1 z8&w`Wpya`?$Ap-l;3c=mgQn;V*@@T4ldI%O)$-(;iFO41`;V>$u7m)B${ZD`(`iToVgV0Oe|yA&byK`-o%21*=QPmOiS9 zMBM2S^!n0Vi`lK;kE)eJa^BczC&Gj{fe|O!RA!_RzF^!?; zg#99R=xm_pFmd+4iI3#_fKhny^~d&ASX*jwd8z6clLUr*M-sU%sI-F7Z@~WmJe63w z+n>R%4DHWT=B~|uO>gi>Rzm&gdPHp>d+^ViB!9eh#C>1{mA_COW3vrxPa^lLd`6w8 z8r!j5SGb=j+*7x5KB~O%@#ep|@NTZt(;DtUf|B3sl_KU=V~5bow?=|dSmHbQC|*U1 z$K2@5^O|P2iL>EAw{slM^HW#+PeC z_PXFcY}um~3T7iHq=hCqlGRa^%e5EjY3oM3qR96mDW5kao`*SWr5*7MrAD!E)Q9*z z9*=-{VZU2)`Ed2Vv0LOfHT`6JO9#${y#PtKhf|$VI!yZsdo#xY44K=^)pAWhI(eXI z;E=W|T!zhjag8!{ZD#k8I)bKo&PAKd`WCYpcG_JeiSpGQb*D_QyVri4#=XFE!`ir3 zu9X?2gGPTd%($Hl$`@{bb+rLz3^kMvv)BbcOVG)oq5iI9f*Ndyyjn2pso)N{B6!?( z8tWhONS%ayd(Usb{T0ew$sbx9b_iyGcv|=tp@lCNr&WlCKmhgdUXKS)r>C;vFqg#s z@#fWk5X#E$G{-th*LjIoCByE$*v`g-*IdNU$nL>|nA&uS(Oe9>qe&tV7vgDDKC8;K z_N4}~-b~8|kr3=|ivd-fDudVp;PKC5Vgw-YVfTsisQjQx#%eA+P^Wc~2HZp%oh`Nz zM0JRXI%D{bN>@EtxF)`)5WRk@Y7w^zKpFKb$b$w%3CqM-t!8fp(-1@|_|k2t;!)Lc z)m#(KY;jc9Sz$+W6J=ysp2h6Bl#MYtP*E4Nw$U&X2y=1F8*uY};*ivz$@C|Bf^UeLI)Tb>sSdIcDuT~6AS*93ny>xtVX$T@iHw?~=CCznlcx+LykLoWj%P*rE%5Zt1>Z~gckp0do zvNnj+P37zOnsQY6r>c}wZsW@Fzsvur8SLmvq_DWpq<<~HV0#^*Rub~#f7XlzRA;z0 z%8Ok0sdPbI%7QyXF)E4yao`cPTrozZH1c0m3^O&8?n?GzMfW!6>5H zQQQowyu$9lPv@q>Zr)3T0W1%r>J^B(Y3$j@0XUN9D4R4w1T+A&jLMekpa!wQgZ|!O z;@shpPvkpLdG6w?4>uZUQitnEFdPdQjwNWo+%C71#hV=_lCr}-okv@RUN2B>cUOo; zoQX!+FxJr~Nq*v8*{^AA2*=EXJg7>fyAwUw#s0zvl?CgI3JA4h?BTInH3L$cFO8>n z?g(witd=ZT+({KSSJ%PY_g1|07^*z~>-E1M912I<@tvyG+-7bcej)d$=<{}=56Eea z&QxpDVq5bM%adgE(xnSToJ(>k04`meg9du{S-ElSUX3WaSs%gqgIov;xZQ$}h^9T= zF_eiJf;FOEY@&^%o@{pLEpU`_c$+br4j|5>h!?Chr!1bjlzW-mhQAr(!EOeC4+NI_ z9aSpT>g&SPwMe%JRd~ zk4Ss`5fQlWcjX@7`}xCoT(zFA*E_XbEmOtSQVhVTtc5nwCOhnOa_$;5Mkc5(;KDmp z&!*;o+kYSsYr}nB$t(C^-tP_t&;Wo4UR=EdQ7@C%%lkCW5&NiPgzO?a$yQf_jseFe zM#gL*cx{8DnIt^sL1l8VK|&5Fw{$LUeGWEDZMF<1NN!#v*j!!^+&mWZBt7Bq;NQCc z)b({}C^!@x>_L33o>&+a-5$!1d#YM^8)l|h5~y-5S?{c)^i(6&WX-qlvKPAaEqAn{pBieE?yAYG!~9BFh{F)}7|n zmRBvi=p^tP2VH5p)zxacYQADS2@HME7Sv1YO$-e?RGizzQn;(JO1X<_wCCFksRpu> z3tBI`+FV^!m;I3au;U=`1*3L9yspO4;vl@1*tXVZI_CD|C0BbU#ylbY9ag0ARu^PsiD@>g4NM3vz?$E zq=T}93*0Tu=2~kL%mHwLTGLd}+~EX+AVVgE6T(Jaw+XRk>@iaq-k+>; zpUr3WI=l|aE;(VUHN{4-4!%_gd0_P-ME$_2Wq`An$a}|IHLtX|O-ABB>r`(f+|dM3 zkR9?pa;K(vzrC6vxMsG7t#r*Nm7BSBaJ{7S8S%G|cv}U6{fv@v*LUm)H^6nk`~ILg zoX#9PY)TjIvexbAh!QTKJAtX?3uE7EDBv7nl+(>oL^W+A*=87+G0u*2cDJ3g!)tW& zHzv!>F;m%vOVK z+TA6q|2}x-L&3{&Ko@c$CP>4ql(q4`fFK2cAr)JFs54#_EA&mR0g=J!jddC1m!)lGEy8G0Nevkwo?OU5j@;`=Ii?a+9*B zVx3@1+QK9cAiRf$)gQ|P8q&^I3$^{?5ou6_m6dyt->bUCHlbS_5QPC|w1O$7%NfBh zhD0gU)p@e}WZ#8>tFghgNb!^TPZmFEYtwH;I(8>YLbW}}a;FXWDRy$YTKV~33&BoJ zmrst3YPcvHp(CsS`~X2X6h6^<5DwX`k{L6ZNQ;%IuddldZwLH!45?xfWj5c5uoJkH zC6W4?n*YM3k)}=>=R7{Qg!OhLGhRZi?UY%KQ?vooc6j|qd2RdQexYk9hz=}}?`&xW z2$17M58T@SsSJ0c{<^_x5x3|H)|bB4P^K-n8DV)jqoG7gz>0^BF_Fi_>O7|M54?5*5evD@5~Z9w+O7LAVy zxI#D?v`3ubeOK3gZRmw<^e$ZkTKOjhi|KHVqg^Zm*A_3%t3hi{>ii7u55?Nzt%onY z{&~6|R=dYE019!|W_Q!}*VxyvR(FC=H)Zw@;N;QUk}3al)&;@e@!r-UB*4mAU39%c9C>Z#GZ2XfC9)v8W&?Nkb90uHGnto2uh3PH<{F^HXM z%C@{0>2A*gY(byJ)CaL}jAfz(U(46+L)DS78FHznKasp75fY!`qWJZDa;q8*+c<)8 zYx(RO*3Mx$F!9-0e^8EVHni5h{B^qh7*!hTUwu)u3&xI%M&{l08Nbi^aMMa<)qhLHY=W_qk0bw4!Yy3ve0r%_`U903w%i>#&MIM%V(|uIAzd5)mW5V5pkE7L;y$}Xi7r^*Fju*$3auN_hekM;AW4^ZA*$a)x&z$`=#oT`m@W{igwBSN6Qi zt=GMp5YUmq?0J{lemL)PxC?QLE}$3@`BL`0OFE4h2S35(Gk1TS85tN%RV`l9R90E8 z2YLUz$%-fseI@5=_V2FPyBb|GvgvPU-bxH-@B@aVUcmL$4P-M`(U3ZPu&uYPpAbEQ zNA$#6W0_zp9LRW5DKyGPnF!wPY=|^r#wK7&2@>oYi36Bnv7)stPkJ=`Up!rWa^r@=JzMcpyrw(d+0&n@ zudUTLRMj@M^#T#x{fC-0h=Ry3j2&Kz$bTHq$SzI9?@sbV@IRO4*4NSICc@(23}xu} zi@BX0LLx(`Wyjcl`E`waUv6(Oe$7iL>vFk4c7W|?``v?FKm1MZP>&#Hh@{__;7`NK z>D(r+26%*7B6{bt%7Y=rtW)W?_*UfgLIPgvM>`2unDY73IrtpEJQCa zBoDTqKP#(X`H@_4t!KSwEl_eaPlC(AlQ7%=I{g-QdW7dYiB4ZA4iGR`vCdtHRn@tL zW`g;c{)qkrRvwd|(1g1qeZf|7uCPU%2P;!^GpEu=d#(ab{fV4guL;B=$#5F38#!yb zgvqNvmG71tex?7c;B_L=*Lp;X!cnfKO^exRnsPauHq!1e!QCZ0!<(_?o0|*riS?T* z9#y8H^W#q+TZ=6-Xjt6SMv87C4cRj`IU8{_6B_P+Noci z`fbPY&XaA2;Mjiu&W)&Y%A}DuU;X@xp>Dgj4Sz|wdzFH`q}&6W4p4_LqiXr}u~GRA zjf~~?h66wI5z4FbmcQo?q}@`Ii2M98;T){cV7<+xgSJxB8dW3vx5ZauYj&DS^NE)? zmprOGfGXwhDR;_i-@cSObP(66ct#|726hW1@4}hUx0&~_V_iHSBRajIM0Q1s*1HR^ zhB{u*5!W9C{K2c_&2N4B>8EeJs_axYuX|zPvui$-SK(^;$+57!U(+KFN5Vl^$_V{E z=%3hKKbwJV-)k(ZB8K#3iOtw^tE-zE3C6_%?l<~dLM?cdk4jNsz(O5;kv?oF(^4)G zyqz~&QLReUldK+lc*%>iA3am}E<=2DKGQu&l=Yo9pT$OpecmL|>hEj?<|WK#s5oq{ z+h^W~tu7e)2e2{&quc>3o#Od8q5M{vsi|tooVYg9dfZ3wUdipnqPd0Qc260u)z#Z- zF*zWAqd5^RT}E(r#tGm&=d`#jSapdYmJ@&Q4-XswGTkZPH&&%7tYVt&1Ygfr?a8{$ z>$nZu(TjZ4?F+(zs2FcU<=?9gb1^cCt2xnikhE{n&IHYJ%5 z6HSCu;pDXD?gP$~*oRZJ`e2)h7`=Mt7biY|y*g1_iz;^)?Ae7sAurd+w;w)zx-~)S zqqx=JY_S?jx0Y`y+Xhn!wyhH|ly2;ExkS?~jzmJC?56eouxSrq8((WGs34A3>`Je} zw(c=hRS>)PHm-aURbGC7g?#7b^PQvpc=7PLhKtz25ucO*8`0IqC%Gt-aV6lE>U?uP zwz_cWyX;1cxJR*c+MV5q|5P5;>?k@aqw*c2$4{Rd-lyEDp!s|E5b}WhSIzNQcKsa_ z>(AQlZWC5hEO;u2-}{FK4r1#5F+{#cQ{*kHFT!dY!hSPhwlM%AJ1s6V*8=M!N1ETj zzWw0*zn#_|EFU0jA$?pQqea?7izY;uY)QAG0|V_tZ8)17^-WF2;~fW3?)}|wJP?F= zyN$PEMTHK11+k;LVi&b4dxn8>n}CkyM4kl_<)kSJuJ_lZoA3g@s1Q|Pm5$RJ&`l~6 zYvV|qIe?&S)ZW|KCWfN87>&{AF~AKPdF$-uK%_5Bp(0;O9jw5a8GV zG308kFF<2+Q<6`}o1m<~`0DojRTLoVgyT$r7ufbNGW)ei?C*aoJC zt*7_UyP0CxD(}?zg@7dZVZqObLa4GT*JWdz9KpH3MZqQ&6Y~P0J*uqEB?YJG#7BJY zK!|8l0Vh6`eGjibgmin`&vhX3Bl4@7l8FBBsZhtD8A%9M-F`y- zT*p{Squg?JgFH#jdGkGRN7h|Ya+I7^uP9S-`N4nJ(5@`%q5+)4>@J1@eyOr$YJbS> z4-(gazwhR8Z*IUEG)ko~2An%y!UEpxJ0YFE-d=;PDIWd$tD}dmy*09Tagn3i z*hF|#&Ot+uEegj&VW2LC8(a4ZIPbK%?bwm^8WU}&7@SwJ6wT2XdoEW1yM~2*b~6Fy z*yHIq%Xc8^_V;C6!&^mrzy=#)Q7VOv96ES-WOTG(xa43yoJjg(QNri-1wFt3xdKjq z4`N6gQu>rJ*^)FQTiX(;Olxafdq=vJ3R>f)kO}cRogN2fwcDIdf?>9AGn>ptlWBVK z;nI=fBe1T;VYU(uD{W)#X!mNV!Y7=lSOzP$sYVM7{@?d*wSdL8>xR zg1xY+P?^ZU{FdOQ0?JP>em zKEemQBQMMG(-_IiNtnkgie5tWN(n!zeh4`fhg0*f7^s4jhTH&*wjiRiFO;Z z{djX6e);;rrx8$Z%UI8xY`w|jyEQ>iTXzs;0+g5Z!G{s2gCT6R-9kH#ATCs-y)GZ! zYU?(3nfeU9#sC8^z5WPj13WB_4`?==EV_8@_>s%!3QoPcX5a2j_=W3+r;&lN0G zds}C;v7yndZD=xPI&si#*I0Tc@-njYpWD_Ti)SHS<99E}#^=Yow`AkR<9p_0*BU zuOUar$G3p!_3<`MX~r--)YdxKW9%p{X>O>(m1plRZ8Z!HxBif~zX{}JkHPFjem-8M z3H`7dKFaU(nUJCV&s;WRR_8?TjNRmNW_$OzfUadS_I_i#so&6Jj8L%p8RX%y%`=g2 zWR3jEa-{I^f`5@ddD{-;#yJN?Bdg|sg3rfwnxL5I@q-D{66J-5_dj>pf{~O1EWX2L z0lfkANKvT3co>N}S^x2#b~~)>xxQd|_RPfn*|v6lHxpTm=#FRt8OiS@`sxs^H9XA{ zvBa7aCdB8o33kk&ZHywjTTQU~6M5j$d}QbNLko}}NNN1c4rCsp{A9dh4#;|IyjJ5A zya8_r_V`4fi1Kc?8)w1tQxZH}x)_X(qMQzzFj=bXM0QwS`eeh$yTPz3{<&~IqGn|4 z7@@JX>0d}(Y^>ZWS3H>|D&&u!Le?SvZAe&-Eks^Hs^!>YAkxV*HPwhR=kBaKQjZiJ z2mXCrzIUuvv#)GJw)*{Y4D^ZYkn3k6PayKFF>)b@|1m~B2BN)V>IpyQc&G}bM#jje zK=f1jKQvy~RDCjL?8PpPoH^20wp5j_ z1s;IdQQwM1M4!(?1X96pQ1oyT?RUcJTBJuGSc&MaEJYqc^zsJ_L9|i+;4u&>+vL(G zLHwBf!Bfa=q{v?AC_I2r98Fb{)l?(Q@oru~sdi^0^5>J9?yhrZhK41??~^1!g1wB_ z<;CMRL%azytX{aMieOzF&7q`7c_`d(_d8-t_C;um{LquglNXmE@~v+#MS#oa9ABc5 zCeDBa`K&3I5MZxG-UExGn-@`ml^7q+32uQCSPva!eS4XG&N>^sxv?^T3tAB`P88!j z%QHOd*ZH+RtzZ|-9LY7Bs-6T`isBqD)ZuYRPQXiHGRj?W`5OgLi3Qpm<~Wu^sU}Cg zvl%uN==T(%`%15f<1nZ-!F1JkFp+0bO>f~njQFIG!N(d8+(RP zKN!3ZAQ-SV-n1{{Z;uXy2fTf-7!tz~^rW_bUm8E2P7j<$hYg2J!#F2$U@m0BnWEBc zfQ|D3ArT!p5r)-Q=i3^h!*Q5!1UsKe^CG^0kz z>2={=fO8W5R4^4vg`)m=AQlLjLWWQS>@n#C16Hxtxp6a*zo%~fOX#a@8~gL|eZyxO zu3?wYb)Fa^22ZuW^Gj5|sYsr(^?lqY4Y#_{wx|&4LBovS>A{6u(r4q$6s)vdU!yq@ zK9=dmyIb3$16X@IrA-s8Fx6M%Dcz4Mw=})9zVMYn`Ht)fKKs~YOIZ zZ&v+w)vs3t$sikSMWn6}>Q*+$_?A&T=#x$IN;ev&+gu@Bt$Xh)jpPV8!%T<`c!GX7 z5|$J4E;kwoh~YRIb_VPLeBRr$-=6=r332-Df{n9rPKt8Cq{UThL)~(zGJBvAcTkE& z0nPFju~D+aYPbCU*l#r?<>pA@RccdV!>kYdpZnhTiG^%sC9Ujlp2oFsCIfI#FM8kd zUi69uAfMUkDst^~!Lu@#cYT2!JDZGk5Ppkf;w`M++0tTxjk?_}n=s}6M=#4u8VECI zWUPZoPz>>KuPV-kgcxq+T3E**5(0Svep=|AuFIo`;VT#3PQHiz>O{FNNN7oe-GV!E z?HncAB$m%!z;kY>m$aDrDS7w!^BTJYplredW=WFTihYmPZ*m%)Eez>1r|l^U_C@`H z>_56?jwTNU(*CqRCQ@#9u&Hf#|LT|?cKH2N2uo$!;~|0!8@rADg=+?7aRMt znyRJPSd4S*0fee$cf1L_j7V4v1c)yp=~x19ZV+AkN5{Yi6Vr1e;Dm(x0khvMlzYY2`SDx&#V zAzw<>nRtF$SxZqBwJG}r8z3j1VcZO6<#Z0S0oK(Ui*^A=_V>?y_A=hCYt?0RQB%+) zw!nhx7*bH$w5NP`AY6$vCfaC5EuuBx!c$~F)z86^+yg_tSU16OL$N~x(xvH~iudsz zKbmmGoe4bqy;mfT#nMC45G;iFXb_d5#c#d%Q(TwO#~a&P;w>?u7gnY!8qH|9Kh}Xu zxld9wpdMTMHjr}4Q0~T{5Np+Fp@k; zvma^5-OJev5amr(i@wTSBfy3A$u9}M7zrY!eDNlfS00(F#8j28CbAj#kqJk;4Q9RB zuW;?bSkg<#PpD#anr-Fa_O!>BCVz@wAkvZ>+0(2eapT;!j?cJEV3^GDx#<`45R2b>MY+s^UaJ?@hktI|8~p#&piM0<$2z| zm8x6!+o+?7T)^qHr6cJKPlu6M^;B3WQaVqYEgHb8jg|*u^?0ErS;x1?^ zD=Ob+T;-V$*9+x>k#K=dY@;0%828GB>ZG^F+vbJ+HDW~_MrER9sU4- zehKSy54pRE1HJ>|K_9H!E7y#Dr?;_LPR8kWVajynUaj5jraYvx+1yaP`iAEAegd%M`^EahqwQyeZ{b7Q1fNV{ zV`{C)i-O?o>B#qjAxp$0hv*=bS2fB+MfHaAr1HScIX{rY!d_{Q1nf#66LOALfhrcS zZ6YY6$z#H;CPAzxdCF(;wfexs8l_=RN4uvR9~h9j2T5JGd|m!R{}&dSrupb^Sp}Nb zjE&S>-B@9&0(Od~hH3;xnq01VL&?f|M3W9tWy#-bW-IEfZcT4jxGhSayqP5%SVY;b z(a9rY`^FyBmu+{N>d0nmLrt>*zF4(r|2+KF$y=42$NppapDrBk*uRHlwZULG777O= z`OAAwmtVqvbA9;B02$KyfN_f5o_>A`0VeQO`o@N4M=8F%Yd08ShASMZ$1fDjDp-kM)(7ZmTZeKFu3ePd?7H0OK^|_^&nVn*j^jlgwKAmJiY0Y*U}B_v$Y%f zx8`EXDmM~@>DD&HCAtHM+@k%S|Kk}v7__2}h=+K>1T&S2@s)}Wsl5n*5Ev9*Lq zcejE@wQ);TG+p%F{iJ5v14ija%HwetWgeun@FsriW=wg^jf7y@+=fsh6#zc&g3Dk~ z6|dxj8<<`^$c$f&YR z4@M#evpZTS3+~ahdL6-jY;Q-T*GKSTkoV!jR*i*oFfJnfSn|}EMgOca)MK0kH(2SS<8>c8mP)xW#Fa>lC-HPsc3eYXf**Qv1w^hOg^Q-;ZZPzp4@I4G{) zJHY!B>6O}09FwcG#Zs0|UM-*G^}ZWfbGtU)k{`q5rRRQk`^(ESl!eNRo2Sh*65<io8bdkEN z@-N0(_2z*co!i|Wmf`?dGwo(SQeMBV%=%n;Ce+l~)F)h&@_XfVJ#qo~g|BJ)qOnnd zD(**Hty{*IWx zqpGReN-k1nEl^gjQD(rKE&DdUhsz7bcIjFEu=4nO%GATk6O&k;;|L|JX|W}fe)5p~ z(5U7Qy1tVQq(svKnD|V!9-8cV z=0xRXFfA(ua;~D*pJ?xu$Dbr@0c*(ShLzk6#lsgusw(cJc7eZtVlIR z`!Gmq{eb#ub+y{4PN@&7Z;pF# z-12d2$9**JvvD`Zzcqev{I&5nvs77+WzEa_vu3j9Wz7fL4O&5aMfKbjxd(Dj z=U&del6x!nPq{zlY4YyPBk~^0o0|7}-rISr^0wra=QZZp^0>TkUMjCMuQzXB-r>A6 zc^~F|k#{ri%e-&$zRUY--j5U26XA(D6YrV$z(jl^d2H+%z1QYzj+xl_ z#CQ7|3(56hlyh+o++pKb2f4_lj4xGkuxCCfywcy}jdT-!Usw`-aBH;N+=UPDov~mP z_VZoo?f;Z8%BA?xE{xyL4e!SeX}_+0?=@h)w$MN`-(nV3HyU8Z&ARE#R6`N+3gxsO z3=}u0v9%w#dIBiCnkCn*dnl9GesJq_yp)24JZC=>b%CUBm^02Ij1S1E_ z;#S($>LKfLUbf$CtD z)F3qk8f~3-G@Reo$7jZ9gNYu|qxTXuM(=e<)M(M86DC9_QGSV@AToL{K@cru7z7Ey z=tlI23^52s4PJTg@80*f?t0I&*7NLjp7YszpYzu_Yn}Do4hJO-KiIWRu#x7M@RjBJ zOfQVc!a*tl^FLWQgHI1DIPxeS`tjGxVZki?ok4Xf`}B406{xD3NZzZBglyw6`r$y> z@pAOib^d$9st&=|m%8IOf~u)}=h}zgGx()l7INrM)iLUIO!rQ(Q$8+mzy(kEjtuuC z75ydJ_eSPoX@?EV-&3i)b~LL|+8{dL#-h^sV^&Q?aJ)PpuZtS;_3dGU3*tCU`=<{* z!z0W$?h@%yuqErU8jeK9PzGr_KLk!^Y9dJ!)a^P#oaA=&vzD)F0>E_5E(oJG3?%B^ z#-P`=5IO0y;DA*)<}Hea+E238mEw>NH+)r1PvcO(DE5-_3!~`D^Iu5j5ig!Go$Y@2 z5_{rJd#<1JFUK$Lk-pv|x)izKTLoU+Bft9w`(+M7Gbz?}bpI6OBJ6bYBrCD$~ajG2e-!?P8&6N5S5hAes!ki(KF?;Cz zu#{V^Tipbr?@|rJX3pP`J$!|j?1q2d`9<`qONo((B|aXGZ(yN>-UY=TRo)(A31IZg!%y``u7uM8EUW!LdIN}tcICuNrt) z@n={67o-e$^FAEt@4}1>OxBvA>kpGfx-{EYoX#(h5>baFt@!UBz$=Q z-YKjSj#Rv<5@ehIm0zQtdXJ~nA>V250Ptg@5n6m%!B2A4>D7^k$yi6 z`abGb@tO7Fc~PWqA9ktb#kQK;CD3qt3i{4x8tQZ}vx0%?p4RX7xa>yFj^Pds0q%?s zv*|z9e*DUrq1#DLs~UaM8iih~n)CXd@SEJJD7k@I@70)d<$zTg?5E78)=+p4zYBJ_ zxrJB0y1+hiokeP6ad?ZRd(Cc+$UBEyt>#SyendL2nL@(#cA+V<{3Q!Zo{&oN(y?Ns zOSWMI`)})biIbVR(c{Eu3^S>h+)J}}16CAd1Xq+H3Zsx(-#Te^9Uv68YMGD{)B`NI zpCMr!Hso6ZkQ=qNysHZeoBdwyAawTzfx2D#dbx4}&iVTNMfvQ=R}hIZYot-F^qqy) z&5wr>83iPr-19B(y69YF#lk(DZ`@9j5OLd-M6^BY?3~~~!oTF{{<+`FRz~!QlWwTx zzJ6XJ+iQ9ehFms^($FZo!s!_yq^49&8uJ0{$Evx<8gr((`NKjtspA%Zqqv2e&0jv43(f5bG|^*XoH4#vW}@C{lP+PWxpd z&$AxiD^|1!aj=N2<}%N(i4DOo55`QPemV-}ryTy=G=eAxY_Z{~E|{b8t#GSfs7i+n zTPK|%WecCWzAb1nE&9k?M=>GDT&K^(mo5$`ch=(edgA2DQEgkUu7Wp>)1xp$fst?K zOELrr;)0>Bb&c*Vx#F(|%XeJO^3{q*Ic@MZtbKI>Vb9*gZ$6yj#u>if8n8Tow1l1> zywLreFY3~R@xaxkTJ3AK;%IP{yiQa$%VRy7t~4z!XDQ}3vaN(nn%d;7=Zs%gN_v8N zCWAg6NQx~*5u`%qZ}b-RXt)@RV|;bH74fbi#hve*UKLGD`uZFl0HiKv9Y&vZD!(d8 zEECA% z`J9${8O{C@-wpc1YXlrD_iZXEKd zXe3ixIFuA?>*wiS*Vt;lS`E20@`CS1xG zlb)sL7b;`cVe8gdR_w&sPPws|Aq1I0e@F;akm&nBqc-44;A@@0VIi$8VNhB&W^9kfYPvZ#l9X z$D(<<>w*ev_Lakbon0zM+1T@m=Y$0cB`ChbX}K~8cBNU&yzV-=XIBAylvnnpVE%z< zxT6`(AhYxeC4;7I0`+}`XwUY;WJ5FKx9ij#>rOQC^!wThk|*Z~3CjmzegV`v$;nn_ zRaGzetOonqf1O$j&T2MVcMU^XA9eK4t$Is1Po+!6U)xn#l0O}w>0RtWxdb*z`X;r; z4m;WaeD&`(fTw4$EpqG?Rq`~4v9Be#8y))xdRDCJVb4RHGEs?|*}>SyEGYqd#2CUN z($Bj6g>8G`rpI@7!(Zpl?Y{@HCC^+~(*Gi%U6hsGOo5fuzmIJ8psTB7jJP?P_9`;y zJScjNSA~D2h-`@D6PPVDR8p1&2{bN#Bi4!cfJ$ zr&~p1cP>5&@cr;ey_Hv4F#_L(c81Fw)&#NKw^c02>!LI@sr_tRlt|+tlU-{kJBPMf zIoH8Y$s}hn@X;C$HPHovsU+j&+1sWf7+8ds-t%fQeIywHe4v&T>*qR04V*%+ejBQ)cn$x1IYNq|>8t9pvRZ>K!brx8H zM0;jx8?xwM_VJeasem%JjkWgLR+PG}U^Em7FT+{06v=~g>nF43gm%k@f(K)3lpe3< z>h2Yz$sac^;Pun;b+3x#&=?79kT$ALbSF25_wgCNMiJ=+;EIq21pZC<*U%MLBo%y4 zt;Of({P>;8&Sqzu0NAN4Ew9OuZv{{J-7x5u!c$o;pJ9AXdgc4xr={)QpW1tq$*o#> z7ZJy1O{{|Mo60G{i<6og&V=48L51y!I?hZ6vXcCP#R=eGP;)!A`(sGYe>b<%P zc0#O$XYcd#ro|kKe{JyrawtlJ3o*FXUC6o+khB zb+c3JmFFC~zDQ+Ej>E&=jtVlvWHqQEDKUd9WpWr{61EXW`QNeI6ut#s6Kc{Q{>cCE zzKA;ebWj88DU&s8b8E+0E2pDkqZC^jwTGGsWF52t`O99VGWstbE|eYK&LQ+KYfMya z+nRgMEFX7^nb)^1MJcoIri&JG>3P+o%g$!O1q)cv#i5_^MirIbP9d;Hn#mRF?`r=7J4-DYdTJ09|dJ{7c# zD%}%s2W6S!Jp}&xZ-xt=7;FFM<#y7~HeaPM7AS<_7TyiCvfK3B9D>Nz@hlB(y<3!- zwJ^2wseDu8xy8(hmkFR!s|mb%V9&%sg8{?S3PyTE@nlO%voxjnmaN6{x-7jU zxwXA9iT0nzRW0z~$!zIk$C1xtA}ekJNY`QA#O9X8=oIZi>I_3Ybi6!qNf>KKGRM@# zVhbreWj*bW*k2{xM;^XG#DX@zU(;H|E2fc$HZJqAxHhB-ye-R_J!Phx zvheRSMk})2(o`d6WD86r5Ixfmb6}3^&a?YC zhetdjt6B~AMw?_o)1RPkfi@nfD+=AIfA96ynflV;Yt%f~SV3>3D11_XuTo`RkY|OH zUotP2bjqgrgGN0?P(BUQv{*;wydtTV>9s6z{ey~NRS5XJ zQbjjFai9@AxWDWha=Wd+n6j^E&9as|F+-4PF2Yt8wC;sLeZ8;zpuZMQUoO}mqu}^C zIv|5oBYC4vhzz8kbA$erwa}>$+zmnqPLkk+Q^pBoyhT!>$fDoC zHAbs1pi0oZC!A9y)X;HK^RhJPT9j&^>f4wDwdl_PEWi}7ZHZ4{2MLlC8vhubrx*l$ zUwC@~MHWwMF#F?)r;NxfjX?+|Tu3HcsC~pWX}xpm11}uOLGJUmeaoqVg^|F z1Zmtuf~`rx&E3nd7;!U8vOlTB% z?h$ks$1HyVUW+0>&GZx&2xMRGHOF~mf^TlJQG*6C)xR>5h!%;O*rtMN>IhnmQ6kU;n0Z&x`mNe)J4KO8Xl;;oy(D6IwNx{4$%7gm3Yf38H zFx4>Sj0z<2aSK?;j2z=RCD3F>;4!PN2HbKgq@6-xxTa*?r`xJ9M4O2iFo6U&_D})+ zjS&YlIh3CA18}4gy%$t<>mmx!ABrWF9^Y6hdcTQXRsben7NlimJ||MVlmpyWf-bb} zE1~^>7isBZ2wV|mbj^0Tl{I`*5Yu#Jk9%lqU%oKKMX7?(kYffC!kHZ4#4x6A;Rew$ z^D@T`tSyPp>1BG>s5$44C>?8AkG8@vz%_D0DeRKUMOp9rhkaw5aqvY|J9sS=h^%W5 z`9aD+j9^~7b57K*6TzWLq+Cm~!9bWuyVisPI&imnD>%NTZ&$C~Dp zwS%lgoz+pG(_aG1GB_9@+Zxc`u{)AbeFo&yFb0Mwmz}a4P&|T!TAxc~)aH}A>;nsx zAemKAjUB-2PDsji&)`;!Fesa9mk*r_i;7sY(-qp>_COK22f_mgNDInUqBn)(Q?4`XxJ5G>mSK3pMPyv77dDy zTmH=z9SmS35&>F4>M0#g@{ECHG^!q86J}tBHsF8FCe6GPF~~s_(8K@;q6g4l>4TF2 z$_qa!Ja<)^h)4qfcG-Xc=vD8(8t32NJKm0gfgWCe^urMdw|}x9uUPNZCcVZG03dDs zUy9@Z$5vf={G0u^{`i0KNU=`}S~R?I+CTun0|Wq2{2dVp_!B!30t6u3U7cKgUBnQN z5&sNKQ`W1{3NTK>Tn&401z-OU_{TV<00Jzu?%w0nGcqy0Yo-P_0j&OG@Csv;SK;{htZ^e_HE*sDHID!T(|ZJ#)W5D@RQ7=Re8S N6Lgg{VDiuFe*lPvsYCz( diff --git a/artwork/scrapy-blog-logo.xcf b/artwork/scrapy-blog-logo.xcf deleted file mode 100644 index 320102604f4511d26094cd4c964ae767151d880c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 52428 zcmeFZ2XtK3)jxXgNS1BcR<&hGqcZBvy{K8eSDR{MTb3-@>dg%T1Y$@?LQ4bkg^&c2 z5C|b3ACM3rgc=ATKnTUgOd?wch&HdjGec;cCw8a?U>c z?6dbi`^-#pYv(b|%}1IvO|2b`f*^Qk1VKzgga7>Th$8+3;#pDfLqXt!e{1o0;!)!X zT4Xwd1&B^YL)}u+@iA~K%x;wg0 zXcCo|jVGEL>NQ(7ZBhPxNn`5*DhaF&U&6YG*L-z@tT&%LR)3_mt68&kaV!9D`7+=F zY~zhxt*11tT}N6PPm;g8sp(|nsU-o){huH=625@oqUNe3sr*hA-Z&1?A=M+@Umo}w6|3S^+(#9PjuhdrI?r(Kr~Ak&x(aM$yP43@dDA^G;FWBtiAfO_L|Gu zzG$7CmByBo(en%@U-E%70Mc8o>%a^kLMyD6P`KD*BehLo@6|kcq;H5z|)53Ry_AoAE*8P4}V}Kr!Rm0|M~pC zYfUd-8@B(ywmxg#6i)x2{CEurr!RYuE-=~nncG(i@8Pw2p-pEWztG->SEeWUC;k`y zFrw1uufF!kiTY!=XbL*IoAFjq@els}Z^npkqW{BqzAQ#CO+tDk?!&7)Jv!BR^Qny; z^|v&h07yxXb|n&~iM~wqXe1X+-CY1)-BEw?WNW*oy76f1jh!17UKjBjPqyCLh=k^{ zjD~Cq*qwtMYrVO#gXv34YZp@w%`HkYQ;#?G9j)~z7v6W&cQxOLyiF6`*tPM-lkC5t z{$%6Eqpc?zkDO}l?$UIeIsxFnCvM1wc5K<6v2EK@TC~()&ee9~$y2ROw`e*Vn@+LI z?wdDub{}m-+Ie#$GeH8r)_U|*i)P1`)WtN<4Q^>{ZEiWG*}8S-m2f2~TbvbDxbV;W zg1fO~4hX^Cv&bTTj^`;nFIN*anUZWWEk9Y4toQ-zEO<1YRv92-EAXzvsoX*4cS%LDx4Zj?Q0*5$3&H z-*w5$WDh|KcM|%d3i3L|b|WV1DYzcRi|bw?Qjo&_DtiMWh1&@_E`$rxHm$r$UM9Fs zD2h3ZbBMHM3`$;xNxeasbU+9JhY4iT)EF{YDO69mbHA`&kUZZ-dw%l4J>f|0q>LGb zqaG(2*ViG5({-O8@?q%)iEvjZ(mY8s#pzB|BiD5`LLApO;B;J1C^{%a>MXUpurRJ4 zuo*8=35_6-8VQviQIYKzguV@S4vq8)pm<-$6HkTARcM(9o&`Zg{%ItTE50@Gm)u7?TxEn)8vmJ65^<6^Uy zz*pHS$CiOzVIk&uKyv+wz&ZlIN$xPh9w+Q(!uArDNZ5UZJxW*}VOt10N!a^@We}E3 z*j0c@^Q$RN6@ifi#yS?>|}q2&_~%C=Dhz*{9Gg_u2RIA&B>UFqU9x-!LeL%4Tuo^WfKKnub8>*} z`VlngT=}pIE@R%Al=L|_hs499)1%EKua#XrRCPQyrSygkyGeO31^iVGqTymL7fJLR z64l5)vP;?Qw-NCO5!VrM2#7f$xad0<>6lW8tfzSMrU}eUy1;WRyv<5^uJ@(KfAkn8 zGoN=lded`)*WnlEVVqBT3$=7XhL+gvz^<>LkhV~xJ?IobgExj{jKbKOI;*d+k@yFu^;2S(vd z010$M;4@SYLiK>GbwdK6gkOWk>FjxjqXsR4Vf1if!{5(e1*hX$?*_yD#4q3Bs-=ENqGHJ8b0=7P$m_kgW{DqLyF}uvPjWd*w|*l;_6Yd$f8P{FzPM zcJm9pcF8{c*9Z5mfv<$M8uz0gXTp6WupsBmlUuMYusa?`Sz>PYKc@e-1<92&@=1wy z==!-I`d8w}D(LDXSm~V3Pw28EDoZ^Z;1J>Sr`N-UO6$4tq^~~*+m=SR!b0Kn?shxe zaEuuS`vhc!kSF8_S$a>$N% z0Hw1<+@oUK1Q?|;A8wgiQanhCtu*=hTQHO$Z6}567WT1fp3CtvDPln@>2@(<6>#aL z0g+vNOoD%sKv^3q10e|f`NTd)?0I5?Ih2BpBli2m{)*V)%dlA2cS(1K*q;(>xDtCi z2X?LD=6?7~(1q+e^Q*Ayh;8^%EKF2!wO?Pt>Mt7@*b5ZvAuRB`+D}l1)?G>P9tj=? z0lbfWg0hWVNuY$5r$K{FJh1JFSg;L37OW1#0#tUo? zu^X1Kf4dUfj-DlH)e`!xE73|~@F`70S%BBC#12!epD$tmav2sTxs3xiE@A(CCHA`< z*cHBne(^Fik|$3c`+k?f<8dnUg)7i-Zs_|MLd@waLhswcbjm%i^^VOtuMu`472-BW zD&U7u`nDbZfEx($)n}nU>R#LokB4H_ny++topp)iTA{%v`QJMh= z;;#G2{dagSB)_1nA?!WE?j|geu-|d>de0G9f$Yrn3Ax9YG36^@)9FK~l7$x#c$$w$PyM6-Wr%zDu-@*@^2h0*?~7 zgWTH)Q&=LHi478~A&i z8wtU=(@02}ut9M3q2>HsNW=?G0FWPoMq1ZQiYJ7Pf~%14ul_CHUx)Y7vT@S>&?tK# za;?JBTw(6wR$(UR{uj=h>@U#TJsu~qO=LFIz%^LM*3E2fgc+H|u>6KQAA9+uk+CUh zc4F}FPu)_qoJXB*Qe?sc=Dq&ZJ0trcN&S6=$V9A#n0(sCdL)X{r^*?Lxw5N-z+fC@ zrKflmfus?@ayKvpX-5!GI{OH$a05eS>62A7=4%P8bOY70>sN#^J9kyNVP3N1ldG~f zMQOHFfAb8FW4VmNd#<2>H&`x6d7mOn!uABZ`rpww`z`w&(Q!2Qs((i#;;P=J3%T)2 z(b!)JjJK{oekmFf|Akm3Zv02zz_tR*^cy<*GISb8X0Gz3SZ`tjzZAQg*^T#?qA)5m zTS*eX3az-cuM~MJC!{$4FOzsbu|N1KW&DiT-Cv4@CcPL8CCiGhz+=~e`@3Al{}>s| z6l7Mo{k{~>RaQcB4Xx-u*kL37e`fE$qBUNLw!fBETD50yL)YoY%pj5ewEW*`)+0Il zCJ*vgaITY`(z6>FIxg>TojgDLhx*lw6qkd8WvBHnCbCyRu@#03|6O1;mhR^NoTRh= z>m4m(%iPI3fBxPScf);Os>bCw2ww9d)6m8*U4AS>dN1rsnu(npHQB+Dk!n2NssKe(Z@ci8vlufqO-*tW029wheO zufnDi`?IgY?jZKRzY42V51#odtWs}y{Hw6nvge1t3foES?O%ocF|oB@g?*RUv@gXX zZCW!=P0#R}_&+s`b*dxxJ72L*2y4Ke{8B7z_$pwTv3+3z%dU(^ULOe)TaW+t1Ea$^ zY54Hzn|FC2iJJDzzw=M$#^z>VxX!@Sy*M~I4L^*;S=AYSp9PzHh2UI%hh6r?@&Ccq zIOaRO?wZC(*u2T-C=V_DT+hOH(g>#!qmVk(+Oc~x?xW1~r- zEYHkM{^eJ{dxxn(ybVEM#`cE??|K-gH=O_Uf!Jh{>P-yn9B*EU$w*$GVmPPn2yf%1 znBOpJJk6chM$e;*;fA`IQ-7D&?b5+Hv|U+%N^2|(xXx{C2k|T;o%QARJV1)K$afrF z!2AHai|Dxx-1bU6Y$G?Cl6|I;eT5Zxcv#H9Uuix-HE*H>qbl}vox&eVcXTgFhloGM z?v;f88N!~S|53`lTe$vD68qom-BYPV?xol2Az=s69}xW*`_5qBPJf~wBl-!VjYR*J zXyw@dG4lbXo~WD#rigu>ScT(JV)NO}l*rsosVICzv{DAViRegbC6rhPnQg=xIov+3 zua^>Uw-T)YW4JON^czS3Py`ugtAyx}*lOhTSu;Xv1MDDSdx?FDtv?W}RQIBYJxkaj zVp$rLoNCTb(Nh@F%E9ai(OcNzgTyLlmk?r=v%*neCGn!dy`7YsN%;kL4S7xiCh23C zd?pdCjLkw3ayg#sbpSf8M?INJMw37ho%V`}LEwY1@VU|7+}RmGWHtw%BvLVzM^QPF zMDA`>-l5GEq*+IU^!fv~lqBeh>Lf}zIE=&Sq-76dgGhpMzW+W^N%da}(~ST~7Li8r9C6oUy(G^kVDVG;bwB*#yYK&1$fgV_+=(7zSW?NQ zY>?&8fkyEn(#lzK-bAycak$>5yenTJ{x&YZ5}ZFGT1g?AxaQuk11)`~Li_ZxJDTzE zRXDA>Mwl~v`pecZNaP`pyZ(BO=o6K)vtZAm_FKMl7jz17&pNPqowz>aJq&K+@G{>Z z2(!p(XYs)VpN=uqiKLkTr2UfmBFmuP=fZ~Zd@*}lWnD{~vJi(knf{fEqv)lrc2i@g z*gC_OavDL-adu%CZj5pX)&LYpSbq}#WWa}li znv{V@*<`s3ew=;?MGV39I)M+c)lHz32rW8SeutMWNOF(`oPKt+td8!YfVD{k<5 z^EFsmSf`~NyzqRkr$RBDhwL1EWHVgK+#i3^jWYGs!s*p-Hsj$dYlJmi;G1Z?O1)l! z+e4;M6bgfp{QHPZq~X4fmZS^E5cA7NjC*MiD+t3m7EbOuy4`^dT<;nbQl0OVaVk>G zM|8A@Xc6MaT(+s(Rw>dj=D7<2{IC||54G!2K*CtfTQ2vd4T!7 z7X|iy3g!T_2>BP{i_>{ph3zL@H;8^wlV5qqB#P|Po7+)$onKb_8gDt42~NM`AEO_q z?KfGdQ969{kN@iL=X8$wQOs#9{O}L|e8%i>Iw!U7KYeQfjCtjgSEv_zj^#ZZGO@$H z!&VzBrtwR#R<@obENKarjg~YWf+ceXbbX)dQjY5ELt3faW4%cwI=q8ZuDkPXr##_7 z$F#-Gj}f1Q;LdYbFT-|m`t1EJ*C>$`+ZQRM4ZInfcYN}h1ygqg5SdEk*XfQP?muFxJ9?;RTbES(n(Qmz4drA82VVf=(gwIg*}%Q7dsq zdW;{?lWOJ(y@zUgtHykGXW1jHl}g(!KLsD^od32`nBnB5m*1l2dv`NdY^M5^iHU#RPuGEwkt{tdl>Cv? za?bs!g0-3jZHa>2r~KQGzxei_etC2LYF1np5GX4A@3oZ6j)z&eSjPoYT7gR~zonLP zd81&Ji=T&406{rf@s4Qquyq??Q`?^%mF)em+_qLY?b&yQ3BekuCd63$d0yDeyloW| z@6B*-$~Eq+udqdMy$QQSAz-WE>SU`HEh$*J)&_o?;QAd`;RRli$4*nc8h6zNHb1q7 z&F@i`rzzh**}4iaDVRkiX?!{HN;T~l#Q%Z#G~)edDbz;3%|=CUjD%|74StcV*J*^Mnq2Q&hjd5PU5*+LU*0c|m?f<74B5Ji{u~ zL(DZs0@*yr++sRa*hKBZ$If3xMcoTUNK?C^9jEwB<;3w3KVaa9y#3X|srkYGu2vyt zZ@WU<*szMxC}+*}TI%dsZjJvq4k9!JeMpN>`%*L#l*B%wpQ8OL+`AVo8LNO0=ySIc zGRD+P1l@}S_P(y&>GA5=th(P%atf|J+}=tjuefQHjfg0se72&4pRR@2h$N>OozuAf zzTEr4GKyT&hZswI=?7ejpH=E%?<6vxc@{ur;|IykvwN9ENwGX3xa{sO1Lb+sW4R(- zW$Fl*#ZUl*!83PX_ji8!^k3fltWQ5SEzOSgyz#TPZ6Xd!)85DbHLZK&m>*o>->%Rs zLR&M{nL%j+nOjRXUykxeF6fh(q2$k43C9hS|2|vy6U~~LG<_{Kpqxk=8I{v{`6Stt z4+@;Z9Ogebl}1-4MbfsVS>_W`jFM~K$n`0Lsmz-`dF2UKQtzY_ng7}{(pQ^E{||n? zSn{}tn2fX4vcIFpY10E$n>5+opMY4hYA9aL93nJar@Ei#IC}=8_8tcu7W&cjGQ3!G z5%Q$OH!&Jg#DJF=Ts}=i8saQ`jvtJGP_=pt;&gnAab_;C=Fzj9367l08~*_tcIxjA zMkt;3N-R6_wYwhpC5D)YJpAw*eG~KZV;BB-XRfDkTEvGZwy1?nVb0^WxnX}H9iK9J z7a0qDa+)BOTX^ezkB}-zr(MWV@aY8XsTHi%O7D4bPvKKOX#aT)+v`4I8=nI~d;J-< z9}(GJ^-s1RRoWhZWc!aE+|b`4zR1AJGzdGd6wE=c|B{@Or51W zLlMYl{I(zZ-dhftP%RYfoF3zKLozSW0{C*j@*??PL^_-zIQ@SQ7lqRvNVYv2n(^>e$QP!6bh>kY(dLk4upfjA*WlB~QXmMF`q5#Q zbkNzS8xi_^5P7R{`tgNt*a>^^Q_^@*n0xMxPYo{m)&roPU$&v)C5VabkEJi2!8~vl z#Fp!(5eE?{Du<>`1o9FaH#KM}pIvaRVl$e$=;4lJYJu}LTFO<-ZeXQl2I^9;e11mB z*}p+(sEyDup3fz(cgfbJ4D|~%r4N{1N*<50JsHck-*aqBSF@e1oGy>ix5PcA98+of z(ld(Vn>z{;1bl4{x`RbO!}?jDU}%T<39}FIWUzsM z{CY;1(h?9-a8g-lPvZ4eJR+Wz#rVp?sT(`$iThvtsH6BC`me$BWgm6K#~c^`ZKKfq zw}Pk@cOCDP)8v`g zI-8HT-*i&mDNl|6sHN-Jjkg|amUqaGk8eGCBA8$F)*45S2DQ}S{{(ijf#uHr~CmP$j+d7ZSo8|Fm>f1X`b|3HTXg}W7 zd8|d=C|d`=ex&L6iO%-!me$s`rk0kY@&TK@p>~2;y&*L+p9de>NcI<2IXsA7euk|#`@v?sWp@zm&-6y-6+uK{(4u9*z$;rRnw70GMh#V*DKIv>b zcC59nqpho{rK`EKM;@DzUu{0rbXbm+wa<5)=<4e1?C5B3?`Y{f{D3@ev(C&Osyoss zN6QY=ef5xo_;#^~i;c!cx93>CEdsF9$=5{xKTU%pi=fUptj=?GU zgPQ#fEpnth;C!^D>v(tPVGhvN*4omr_9c*m8P7aqX?>4nkjF#48?4+foxw-DBc+}q3baekU`|1zx zuRV+qZJ+!_L)X_jfQCX9bq+cb;m3(8oHC zG`f3hYC77`yub12!9$I;O$QGj*pKLLe*IK?cWb9xZVuUmmBXl0M;aOqAFe%gsOF&R zATq}FvTfogb*(oZZ*Np+QLGo6#U>YNoktq#>%=;#_TVA$(0t9o8?MJKn&Hz^hZ~M{ zb#!+uPDbhZ=n?UVtG-rMJ5_V=(4m9-_g%Xif$#JGIoff&<7msj(?W=P$WVKzrsmLr zz1Lp7y9yEUocV{1UHHybcWV<2a8cx=M@UR+(AFHd;rgqqtHi26dEou-!^dxeAovV! zN7KUm7Kk~)gW|#212^ow`YMtmAM%$co@{`no@i}vg*;d$v1zXHC^^K2iTcBb>*}DB z1N--0bJgxDRh3jxj(GgB@y-J$PIPr1X=-ZkYH4nBYjJ55g#{eDE32z2E6dBvkOx@a z?rQ1kJlS!)_3)9#Bh80fnj4QEQ5_K*q$BmUVy$>+7L&M6k!q%TPeo}(bp^5lljB#1 zI@&Os=Jq3X_dfo^x}yz8z$YG=!xCWfszYMU{PovCQL3t$@{00`s z?WGB;&FZ|+&{BW6uC}fY6Pv2pzxUcbRTY($(6_2`rhIo%X?0p2f}(b~ySBUiSX*l| zeM)W3$M%sav+J{KTfwTTbzo8VU9+dEs+vMNDk~}~iz@QA)%_I#OgsBZeM@I+M@M64 z)6t`?ZAWVEH^S9P_Z_Uq+$L*iW>@dVh*g!)3wg^cOSYH)ng^*K*Be!xEo~jG?NDk{ zBR<^ST=VUZKR^G-zM30gNr!6oUw=)tN3|1*tW;Hs6=L~(Uhci4$d^vjHJj<0KkA}u zex^E~uG#p_?2-z)=C@mN%SsC9njfpoE6yn|rE4CzExWLwthy+duK6!Dx#cCrC51&5 zbj`M36%>|~7F8A%SLD()UpSRjQdE*#Sy)<{S4`LZ%D!FYB_(C$S%sy!`9*ZiKPV_I zE-x-EDK03^%SE4X&7ZYoWtWu~6_n%`6cmCi7p{5w57l|a)w#um#bWVHVSZs@Nq!D} zv+@4y+>*Tf{L-SLoPzwKlA`>ALilFWM~&M`D+)_WO7lyLi+APb7v^W@7tlBVx-6@z zxUjUOw4k(XXIXVgPG(+y#xDBip_>cy%L~hj3(HFL3bI>YH5vYRU~6tcCVlhU2eOLM zQE6dues2D*iuYYp^RC|&?a0oOW8@LnkMpuhN{Wk%iXc{ALB`1m^VICTwJ2*>7M=6O zW4Wcp3cgS*9M3P#O+8^5v)QJ#C0Ti?bk5ITTToh1N$!Gz{QT_Vl8hTZni;jvznr}z zBbUy3?4FW>{KE1)H$h%rQAT!t`mO(*H2>|$&g>mobk1j+in7ZJ3Mz`(MP6QRZhl^N zQO5SGj(1mWOV7{Q4CmbUTxE7)X<2!GPLZ1=2Xo(*k+UmfOKL`L);4HCEl=vdn^#g* z1dSFI=jVxe=G>f|?CiqK{H%=BtX(--sacs@v*Dbb@3-v2bZq6t`Nc)K1soBCV)n$Y z!kzi~nY%K!Wo=E{nTo2N$@N(IuA;)~#rb_fnzd_J)~<~7^t9AeJoL}~cjV@kV+|GB zd@*0l6LVecb|!0A=B{0t8R@F@`Hb`(Tj`zuwl6=oytuHWcu_5i2-#U;mMT+~;gLR{ zx_$Eod~EQn^Cu+*<;8hiyG8AAQdt;36J2IxW@MzNZQs0cLo!^m_R^`Gy!_mfisD=_ zD5_i{7BlT22Yvd^Z5vaP#pFPF?&WJU3X7o$OMXsaQSQRz7W)<<#%x;Zj*S~ql2yt6 z^7zCLuzISR*Ik{jHv*vbXre|hSGVpHS zws`|)GLw*)keD1FN5A|*<4!0FwppB2`m1r5^UsyLva+*waYE^7Sg)Ckw4K{GLR}uo zGn*6QlM<5>;g|JKm8X`K7Z>Cf6qjb-ZkwK(k$#e{tcaL7m5~Njq^EA(l)OB7K0aZ? zhIla^e%W+uI=o9>K~W);SMY|zHsz2%DoBTEa86>zR9fn`%^Q;wlaeqsRnknthS-En zYJQ+n`~J0AWqD>J@aSnYE48`e59J5`RRh__?YMz zO-wR9v+n+QO;lWLN=#gG6g~6%HDNJPF_Fo!@$q3%^vqA^gv3V0BqoJL$A*Qe>6uT) z#6(1eM@7a*M~A7SBI%jm-5(wfQcu|^SOqEs02-1TugF8cx2=?&kPRzwm2v}Dh!T!?B&Yf=!Ce$ z^vRr7ZZb#qcoc6a7|d9aZH}KUyKV{A3@Lj!NHh> zXa%nkHRBP{k)id5DcLo5aZ_k`Fg^35J0s)bw#CM}`v{GWizxe0M(*)^Qb?;k(!`LrprruKSV#L&KIt zr;Z{cBO=1oVPcpAFX~XWnrj2U{8?AL21KVuYj)Q;X{P@UT#IxESsX z3+1QJ46kNHM<&L_B_}RU+1)cJCe)#!Ft5;XaD=S$hj;Fq`Szx;xL8emd`#q`2owVp zpbJrlg@J8Wy?$Mwzdzjb;77HQVUZE>8{;A(Bmb@YaFB~?M?`oSRO06g`T%+4>FiMG zJ0>nVJW><QY4jnDijQB zF#tJSe(=#}{&8J+oF+0QUK6g-sH0OhEzCE3VH!|UC|LY`{Cs`kH zsK}%nPSiwcpoR!&Xf8}0f&~-R;Da850{vC~Gd{jPzW!@g(@Vcn85kKC6BiQ}soDOZ zNuK@PW;KLVCIvQ?dL~>QvVQH_0DlkvnYF7|`G6H(dg8w}252G^VxuC$V-t=~Vsp*R zeJ2e1=a7ytEG5=)eW0(um;cPF)&BmgR;@w3$U4{>9uuPpi;j;A3y<9Jg6smL>Aj=? zNE*g{;E{4hI}-^}X3Rem8}est6??9wDe#6`!2$HYZwLgOBnt+NyIyJ}y^ zs*0EcZ^&BkhykAd4lID*s-S>XIZx0}_dOIA9TNfXk*J0Pi4G6HRxgjwj|6~a&8K16~Gf-4YNBJ!k#ipIO+TD)& z+csPwZAJNQ3rcUBP;SHL2~gk*M9puloFuH16NL40ybvtoPUmUt%@{!~M+;%HMu?Cj zg-98nJC26C2@|4awGboYqVaCHw_qV&UN0ob>jc+_a)P{8K>i}6$o|4c*-zLcqqf~7 zZ<5ytTjkZlHd(30Z9|=PC(GT87hXad9(1(>*Quvf0kYk8;g#S1>cx?6UR_t03}YZAnxjFEch-zBry6&OSh`qqs!RX)^THZXKN!4bdzrJM$y@~iXT$0l;j{|@z+ z+q=6_s!(<6ySolIwKcWvYiMg~Y;I_7XgJVZUzgSF;^|R;|KZkSUpvtyqD!$u*M8%{ z+uxs+tq;{5X=-RbjI(+}P3_TrseeFW`}2=(K63K-iH>IUx==;=WZE_>&wYP?{b8Os zYib(m>aQ(3I1W2M`$SvkiIb;}HFqdA8BC?&yQ5ai_{8kF{k1r$*Vi1rzNxnEhHDF- zA$9$+6DLocKxu&$9uT+IJ#5pO$EKzYH#8hYmFnQ3hJ7`M_EqP1fcng>$2yPmgxyU! z+E|6TbKGK^b2#6=s{YWC!!`R49;`cf;6Qa%4nMy$+d(~ac8DEw?vjnztgUZ)cHS{5 z>rOTtsI5csqNe7+{u{2{Q(5pDi@e96jyYwHSjqvVdeL@6>u)|f_u`4#12uJ^K8Wu3 z??bU5_m^Ne)4fo+QeWx zvEVRSTaS5C@Iwc0fH3oi_Ux`ISCu=VW>>fxTDn zE~~DDsO2TQOUueiE6Vb>-hKhMFRrF~v3>yMyMy~#>$@IBn5#iiRaU&axO`7ZS;4M@ z@AFcu?@)a`h#()!EsC0o%d5)vT#M>R`JUX8Uryum6!W!*Q3FyPFdSq7sSp@A-rNoQ_U@^!5G%%^y^> zVuh+)ESoB?tg0-_O|O0q&USJi3}>N2RiS99B(J)(xNJw!-F9Si`U3~|Uw0MiNDu(! zwenI-qPjTqG5$UJt>OED}yTMgadR1A~p3;{6r+xgzw^`P%7VNsoaT3BrWLk7U{pfeXAd68%gNy}vA8%lx40-XHS^lX z5Zdgoo-EBRDlaLmtST!Ki^RgI0u&wcv$nL4q4YcC`k$7<+^Xu*3e>?$U}JejC55O2 zWanhWH#m{B+Fn0goQwKUK{>F+`Ko+dQ7(%NI2Yz*MW0}F>-l%xO-u%>zYpOFXJ2NX| zS9WH`uGGv;iP?IFs~;2;pvYHN~_}bjC4hmtWbHmBXS>_O|@A^zB=s9tX9Q zS&W%qDk>~N)ki5_?YMK?JUTihf0UJ*gBn_9X7;wU^c^WtwV*yzURGRz5?!IYM1-R-<-Ix^pG@Wr-iC8kH)ba|%0w)VVVy;sVKYb9UvWrKV+MZ%@n2O5d59x()O@QE*8Ly9wlH$`=ZKd2ZsI?7VCU zo|dxFmH>AX*#GD8DOp&6%oNSfSzbaFg z4pGuTy@O&WB_}3pk~YO{j@Zp#&6q7zieV^HiMzy17t3clRAiv?ojbN+c`(c5#DvWo zHHnGg{{`c5737EuQe=vmb1XfkrHZLof09EhsKz)FlanLjlM^%O5jF>5A zxY8lKxD(d0W9!D`WY%t2@`;b$oSdSGRDLU?kXzARn-ep}**_GyZOf*V#Q0>Y24$&~ zgqY;8=x@Al;Ir?i6>Lwr<2U6BAHUN=!&dOp4!-99;M;H*2oDS}CSOgjCl~ ztc|p3Ly~8b4V5^|GCn?K{f6(D8Q542W2IEwsY_#hYC~LdN^;7E6jW|drAmnpkLdoC zku4ia7+|?E6~(sgTQ_e=+PonlaYJIV)7!;36UL9v-OGWulvH1*?R}v6;|^uBU>uQ1t>WoB^rOAvAEdK!rwEpp}!Pz zB_?c%PsZHhLR0S2F|s9z8^okZFvZ}co){I=`Vk`=7TuPVfI&7!$4A9#3ZG|WgN9KO zrNyD%wfRSkY#0rCOpJ?$d~vbKVb{LS$cFdviHXs%n}0dZ$d0kIw#bz48yMMYKff%p^}qV?H%>J+c6{UU4;k4E&)v0W zOI&!!+Q4-oYIVwWw+%3|Ip4Z9J6s*6(S)lbB7=i9QR@TO=RbhRru(0RaqC0kQYohRA`M(32eW-CQcn6uStxLiA{v<#3m-kMQb8aD_C{a468A(m&T(`6dt`X zIyN>s0`(%)jlwZd#Jb?HmGum4AJ&D(s$xB2j3`&eYBXWNQQ_ecu}Q(|*wDyO&u=lX z{jp3F6)VQ7V#H`M$`qlFj#?WNA0CVnKxBv}LLISI{SxDv_3rHnp>awvP8F?RuZi50 zx@lXaCR&X`O-yi1a73`iyV$@}+k5qKQCOQ8QRz{qi3qvox&D!lZ;FYE2#$%2R7Yuo z)$L#F* zjQmDyWN1W`MvWR_s5;nh1;2xI24xSFR^y-xl-Z(Ci}Sx4H~3>#>CuQtlmH{bLqfu# z-L-*n{Lsy8yjxx|YEJ1oQjFB9xBbK^&&+?eRTCT*j>QN?r79#Sz-OiMnf+Mk!%Z6{ zMh&=YeIeQRzV_BncZUUsqCyfH0*gVN#NWs38yMkCq+8k1Q41B+$e?g6byUpy(C|og za7f5H^dGW5(9g%C8RQoh6d0v(%Y$N1=-TzDw5ru8$OVU~*JFU-wWz^Jp8R5}Hg2)o zg~DTaj5-LFJJia;RADys971wGU%zGblqT6d-!Hn4iH!;kULO{PDqC1+(0X++ihisK zDde+ZP$IM_YL;r1li`tJA?gTJrNbe;7&0Hu5|ql%8L*6BHgT;-QPoXMu1GOrBs?-S z4CSk zFu%8AieD^iDy2A!HclxbD)lEZ+{H4h(>;bo z9U6p!;F>jlsO_x{Shi-3>VDXdDioKxziV45W@>Y*yNIduIxKR=%}D0lh#`+9i$ ztX{6U0Qy;9MIMwIQQ``7sY5+OP3Sqm-LpSBU%A%TZ-w{$Jh4SUB};YCab>ossIkhr z#%GPM=ahvHbAxQNrts#g%GoY9BNl6 z*JW)0XjFa@ikG7QG?sLo7<7W)Mw`{ZsIdhgaHtL~)B;!fU|q|?-&@_s2eeK8ny8C>--hTY-{~}*Q{9SztVU0nj`#CpCN1>Dm)|v2$&#> zjJ~T@tnl++5x6qvB}BI2$WTho%|g6ifPZixmc-Y0RqT%#*tF}_>JZdKS(Q}Kem?8H zSNM8Hw+}O}1&4(K=i^Ti{Q?90VJ3bnJS+apxE7`k@B`f6$A1M->jDE-`>afR0&z`T zYw>5n5=I`f#y@a%ppRGR*ZL6+ELep=>=kIm$JZMM;ZyrA1Dl`dZ}Y?I_^(^J%4_AS zoxfpV^T#If@x>ynS+#n(`Yr}GF#7lfVD8@DzIA69*!-1MUA@YGMbV!a*zg|HT)8Uf zK|KST|9aomt385l8e(7zSRJ&&tKuaFHg6ws(=Qm)hd`R}HF|#* zKc7ad!H+yNK^OSpoVzh?{19585H1G@8q_+YQRH-4A;>CJM7{B?{S`uz^57c|UZ{#L z|G)jX*;b&7l_;!wqq|i?9LlFLsHR5Cz8K9PCD}lfX4eU!{MO1kAp~{U^^k5I%Cl=B zYyc{@esUBFy%;?THQ!j&f8#No1k{6*P!~=?jd&wIaJUIIWAuO{Dn7~ihU}V^Gv#^X z$nY?4^|T|yBRYdY-j1&;>vVdZsGHXtj3(U6*=NTL`jKJ1#b`8{Rpw3doPKz4SZ5TC zQ|2)f4o()c1s82o&XIw^5xv=DG>hh0i`8TSMzkc#;-F(#tJ50{qd<+SMqOr$rzK9F z85`~+*r+m2S>SWy(4+YBrfIqgI%YT{k$S zH|UIJgNbs9mT9xaY8|y$MlH)MVe*6vLJw<=MuW*>T%eG^DvpX47fC|#NxqR0oldJa zm@Sajyu$1vngW=YnSO~m3cdzt5P zIAkv_i)ooDP@Wna85|g-CP62fW}qv~zzRaqW$zxvRF@k!L2N}>nPydisZ^S?@qe(Ow ztQK|)(fAi#Sj`mIWb`yTF|Ntegiiq>VHxSW*ss;=bifd88dD^)kXm5_zrtoT!490} z(J?4>xojJ}&@(tZJfwqoz^hF9aq@9OFfk4X&2hRE(qXoa;~nv`r+miPdtsojZ_qs` z2!I$jV|k2bEHv~2(gkJ%m}fC0nQ4UhVE22@^Pl(k_6-h?XmvXFXoP`|nZRx^j#^Cy zMa7Hkz~g^K6xjGOdAjFZZ*SiK)Gx@X)r=3S8|I6(Kti1;qWS>mu$Y! z)6+jVg!Rtee3A$&3~w$uR9@vU%)$Pj7$!z|hEu zmZT6{VHQpOMy#m9ZNcjTH#f~Fxv4ZSd1CO=#oqqD!I7a6Y&R1YS;E428YQrb#tHOi zzKmBi&YG=CkD_sfJlcPu2MQk=92(J^Oa{GP;lzGYlmj#8;1E{X>2N4|MQ8(Z0B!&I z-oCzpfx#iI(WKXr5ky7*?+5hn%_*Ix3yP2&V_l zLp`JL20FdPpcnO`!LIla{DHb~I5GB|>j&eYUN<~~xxjN{GEj`lIBh{ge{$aIg#mnS9vG?7 zK$nBlS>MHT7yA0K#fJ2HY%^?B&QH;`!36mbGd%QjbWR4HO6Mm}&i0+Z)Qg=lpwo?D zYpL|2$)>E0PH%>YpcnOax-&=#nYBods2_w1tnjH=BFxgH1&7W&0$HZDSUoLP0uv3C`}@!K4E79y zes~yRhK2&4Vb*Ce0H;l=DbOQ?VL0R)0~gNHJ?IC9xJ#8;>#&kqvjO^Eq&Iy*KRkQ> zTz}ty9$N$>XbsR3LWezHpT9wv5 zf}R&UM}9bb@j{j>qb8B)%ViDFfSmj)}qrIF#e(# zG;7kXmkno0+5>MgtV3D>L$v5epaLxFG?fImO5;-!lo=){hT)+hCMd&nAy5<&lo>77 zO&pnL`oIKb#)@q)gmu88sEkNZ5V(i5%HFoZ?imM^1Z5ce4D#vWK4xKt+|{B93Ce(D z2nvU70flIb21}COE&HRWIMla($_YJctlVO}0t2 zLLZi31VT^}B(Qvim)65j!3KW9@dqa2L}YGK}&B0~^I=HLeuFJ>53TmpPN#=D$b z)QS2TnvI(sU;R4I6k$Ldm>k0PH*&Xg`l4QC7=YTqZNSlhNfQ2C93b%;hi#1i)O?X{k=e9v4rg{y`PX;a^1oRFJGsyt6sFScXo_Yza zqJ9GXF@ad*#cnoQlpaMrQUyehe(*9%8Q_Ps3MZ^tQ4eAU2ZywfNJMZOMUsoRI_4(~ z-93E_-te3wWK`Tb>>Q-ndpfvHH!rQ#I7)qCX3S4!&VSz1hd7Kqic^Zh>Y-B^$FNdH zvle`Mb_O@c7=ieLd4du7iSGPGde2@&E_eqHsnYdh6n1Phj~Ee87C7Nj>Ep4?{3)EV zzVjC@^nwo#cUboYAL1Kg>t%chkQ5zfLGU3z>HYjt4-*aD$Pir>!hn+c(s~eDO(Te9 z2)ZJp4W@@bP#5!)OP}{$ggt3Ta7Z(Pjgx_5RQhQ|Oq2FXHs}o#XMSQmcj4ltOT9xl zHEF?y_>ay_qRxRSGWGxnh>Kt(CPwJTi7>41Po}7n*`v-LW14vMYm06EqA%~5+VJK&T-tYze@XWc-`+ECzNE$H$;w}vx z@x!2pAwe9wBF;khNKdrqE?w-?!PYncQ-k3VgC1*%Z8@wOwhy7_#m<%VB-q1T@M`$mYSKu$gpW-7>P4iND+fpONJ%!0^`DhDe1`^(-V4crYAU^kB&`DFg=+=tQ`}_=Eo;&lT1(M?4vm4 zSSDqPulFFebzBO=>SG_AU%Qcn?|i; zV^cs)sis_Zho=MSiQS~dxq58eCZd}Oo84iz*&N_?f|2-dD-0degsFbL2Tz`o3m6a^WXX%anhM0C4A;W6c9Ung5!x)GxVoY24| zk$CK0_W4P>G6uw5Hi-;nk{gSf#1zEI8R!ZVmJGi8HXZNs6jw$F@w8Mvu^y%zlW9S!Jb6|+edF+ckHc&6}AVV1$95GtQ zZ5YU9n-nJv(++lvx$-Z%n8xCAC{LRc<4$@`;x`wNp%{k;b=L8*31EoE6!?mTv|&%4XXb$!OwUqJz%xS`9qL2a zHDS;(LW!8QV|i@$Y1pSSU#ODI6JVajkR%gr_$>|8Po&;U4DGCfjj~7E#Dr}I#*6=_ zrojQW$whYH@jn-4(x$R4lV{*U=~q$jnHyJl#R+M0(i_i(Dh3!pNj|r81jBaLX9DFu*HqL!9g_%N^ zWJG#m?i*5aT+{f3)y=1x=*K8jnSE-^ZeQY@7>5e+2byMjGBPkSf~?;(2EGZ7=jL;e zZ~8Jm+XS7>#28nI>B&$JTs{)`iLp_-sj&&{XD%<)GX-ZbHtv{!0~9ChiVMLXsEg^z zU?1Fy-ZVaD8O2=SxG@JSs^u*4@OJ^7zvzW)hHm-E;5TQy;4TAtk ze@n0v>jV8b$0sJ4o|rB$D`#z7XSKmcVMWF;k#SfdD4+-%T!sfi0+$3x8}gHxAsmm8 z7@=nCoEWzvri_k{O=9+w&y;Y7#7?f5vN1~w`us3?145&|M( zc#O}{IZceI#*m=Q4fi7J$1r1KqgK(X8W-UdNk2Ajhm4>X$L(}y5EDXMnV^g^+eX4? zRapAR5Je`TH;6W7AGbOXtT;g|23f45Oi&CL25=TM8mtCX$6*{W8Ys{1%4#1)&x@TSLDBXMXsy&LPQcKD6*GVmZIEk;80vvoN`lfmVnsg3GvXMG z10mHhW}U$Ji(=5ONs9!fhooB8-Nq)yp%{x}d=x6cqE1su_@XDR4+#oBq>mqwzz6SD za+>>xG4LZJ(fs|KErRr+6}tyN;!^~RN#=c50IrL0-?a|6M^dyE14WTXE#ue5qme7% zH_PwF?Odo3i*En`62~tA+>I{*;76jZ$X@UZU?0jUxUfQ38+kkSAu=EQ_C~b46*pTL zW;^c3@M~clBNad1l7Sy}0ckp4qVa+s&S^p#1-upb<@lli$?-EjMfkD+I`|MBcndqw z<#y%E1^Bs|B7BQXxrW<}yRt19db9E^1pJiQhq&V7ml6Vm4ayf3_(g@a1Sdl3B;4pF zDqmyZ*BQPp&%0)gIt=2RMH(p%&$Gl0*?uoX7|94T>>zKFr(|M{VAevH0R?@vu5!q0sK6i`6oooM0x( zC6>pihgBT)#1jms!TcV_F_m&x3z##yH4m;Hb*#HR>hX zT^t7oDcq4@hLgNyvtY|H;T;WtxLy{OY;|c-r^h)7CX8`7lxS92W*~^tS-_~1O|Q`j z98egRP>Db|R_@yn+n_Qi83V2xok!OsbmCBoT8)I#l7|IqC8?K1g5Jb2+~0Xve3nSZ zl?oT#grWqcx6x%7oFZDy7B9>4(K%poJGf#LB|Hc==O)EvAT$k9d=~;}v&Gko+dr5)Zz~^(X!mUKnVC{gmZkut3;9ahGri# ztleT>Wu9S?l$Vk!vqS0C-S_eZDqg*!ylOHp$Fw}mHid*npbQDgp(FDevt;7MF)N|o zkaiizRYY>XD3*?u_Q5`4Arn`rbG*zP<@Q1C)GZZy6)h5IRpxo{S;iw8P^7$$`d3dO}MvL9UKEmC5KDZ86fH2y_V=SgC0i_fnWXI9Hhc*f%z) zXq;1oLNOJcL4FpOag}WHf+PVZbVV9~lo3$~Q!ZPT5;W(9>t-_f8F^97iY$aznLSL3 zR})!iMY)wlW(@=!xZ|LYf{R-0 z_zPF>Sj- zO;3gqeLxH+WOT8DtTLfI&igUk2=LO8_h@dhOh#W2Qq)=Y3$zGY&d`GQAY~!jjUdAs zg^W|;6!B+XnV**d=0RFq+p?&N-cg~2B25Ng!xTs3#UANBjLQuZD2n49)xz68RGi(t z_@JhXDS*_A1$g{L0)g`uhi4fblV;_TzSyt^sSYp3K&eJUS z3-`>T>2e4LAnz3?ak&Y8RG%Or{Vm!m{R~E6(Get07D81hRO21x0NzVar2;pKG=!ye zR&$m4Da=qK?=+O|MZG7=_M)B_4+i}TyK#DFS=xxsEK*8_A}G8qh0jD&9_#5O`D$MLn)Y z{*4WzfKAldpiNGTFQDgb{5(l@o?w#{kvw$#RKq1H-0_R1IX4$R4M0iyS}$7(pALNP!wb61YfG#Inde=Dyxz&Yk<7o!!~#=lh#o ziK(sS%scP%JKx{?JkRerzQ1j?BYOVCQMfnD3#B_~f1S$rL z#8ySD7@J$uqF2Z5`~rVFIgEP4Oq{&{PpxlOEnVYBh_fT6od6hjC`Kiu*WUps<6`Pm6!oW$f=1E03oBUAoZI$ z*=W^m9x}Z`7okSGqS4!xBa-`uZd(okTtcvffV{9q@Jqby#$V~{;)m1?W9O0BH4`yW zC#Ds+MK3Xk&TPzl0 zMyk)0=k{(Z{%ZI3mhq?>dAo>Q-v`NpWV<@6P%gTo5mTZxOQriix<5lD=pxiD5vH39 z6R91LzEIk&qfejdrY3RPPe<`#pq1T}b9ml#fNiyr_GD;3t*lt6frX%3hIKqjw3y0G^99B7uCWlO&1Xls2%)x4-t}GL~_d;^&|)l zt5mXI?=~gpN=s5V;K~SK(V4sCT*bWY9~y(%qH> zw@ZPdak3*Ok%30?n&2zAy!4~OAq2E|x6z(pBHo4n6%NG< z51ZPn_#8C=sNfq1D9}>@dEPX#S=57F;xbdy zG1SsKXmL@3s>BE-y6oZ{U1J(*l#?1Ffo{50EF6STT}2`qhH62p5faIT#~T()b>@Z& z957@Wk|Gl(GLH*n9${C|;$)|Bqb>yUQepB3^F6gDC#>iLh1q;_BCXXvtgwW*MsiZi zVjcqlZRV&Ct&V>goYV-_$hWuO#8r84xUm@QYb2)U0qw{m^?Z~s9tTA`Sg zZ84h~mOh}D#;ATh7bj^ihE{yn9x{T@>qWnQI zBPTDCRgT!*BC;z{!(5gy4Ye*eGp+bN4V2IU{Hp<puHqqX@U8kZTvxX42P^Y$$_VfhpiBHq^Os*>{x1>D(;I%c8vbqh8mEG|fvb3Qf z`vCQdYMEM=PypR{i>@05Ol7Hn2dNTPHo)yI6OUmPQJz(b$PFz8Z5q^ttvu?DDL^5y z6mHJ)M)7dB<)SySV)Sc<2?*lymCM^r$8m;JlgxZtGB&rSMX#RQ*#-V~WIfyT)3F3y zv+{kjZt0w7dr*5LrkD*9V?%EodUQI=%vDYAH7PWcikrdMG)&k+)?e^96~N1n0pId8 zgOq4`Q1Z?5h$o3Bj50Z_Zb+HLv|%IxLCjJnWqz`1+nCmI5?U>k$&N8WOuUAh-NELd z-Ocf z4&P3oYjuMBp-f`Zmw6qxp&YRLdRBi(wf)WhbRXz#lp*-LoER#OTLic6LT{&!UwG6X zK!Y}^Jtq7Wmlobe`M*2eK=;6hM7`-AR!MdZuwHE7^pOAe1_QOyJKVGTGd(GjMkT;l zS?e&wHXRst`-W)HTkZw=GhKV^?!ex0_)&{|P-?2z>-V9<8BAr%y={G+=`}ZU1Y+TT z#gi(xD^6@YKK(%7qPZizEh-jSuVkEq_>#M}yyf_3p{MhsUVmVbozBP>e|+XG6^bZ2`*{oe1*^{lqtvAe?wDKkDHr6F#2x(B;9U8VoC-2l&m5CD7c zmjDlrq1e5p?o4lP?>5hCQXtUXGOlmqvtR56d;1_+=$gs`-2r1Y9)GK+Cp=4~`#-ur zLnR}o%*Nwyo0|*r{M%4WL%mx^pFY!1GiwvNEmg^yn%$!gP~szMp4A@pzRrYpp0Niy z^SIV0_vDc`D?^XM#%azKuJlg#5M)+&wzEsf@AnW6bs=xNC3k#hvBP@rb{0AVf)n^% zkpOxQdqrktrxJMUS7Hb}t z=b@I$1gL`U*$hFg-a~5Zo~FCC+%f}zLYz{hxDMqsmAiGf;*Y4Qx7ok}ii=dS36zI) zrd2QweS!lN4kHgKK2)f11syw- zHG0J~$n(y3iXXrWDt|M?re=wedkZvFfZB|sQd4xWxe2GFw&pAyGu}%JPc`0iY7paA z??!yL8v|NEE*bZ_Joa`|^I-bH{h5X)73XIaw>l1TxzQ)%GliK(eAK$n{5OIGh**s+ z4Yo1YFprK3aiEG|I4LN~0N#8i&(v!-?M8rdn^dfFvk;mJ>;~eZi3@hXTyRr$&{0}L zSeg#cH1r(lDFdY#Q)K8(F zk?B&Pu3)9zNl}@KA_X8px>3YDwMKoeu6JR?uXcm7)H7tRp_r+caBS+~d{~4L$^AT6 zCqzW8DYY6CYMDNXt8b?S(8$< zZEE0As)SV!a2u#GUQVk$oRlKYMvDPvv7Yw_9|!+?=#ZR0L$ttI!TS9O zSU=$W-Ury#;P+huzYp*K_xJyJJoQq2T&k0_()hRE{*~{6!1t|R6Lgt@3#$Nh_Kh!p z&xJrfl+k}G^W7o-IuQmGwh{jzxH>;gp~*%%@-OUy!nsc1;+2Y-+uc&133S# z^=|X)Z*!6X&|#pM|NQ6PxMkg{-vqkpwbyRne)aZ^moI((&vaB0DG;c#jrebT=^wrQ zz3+VM*WUfcd+!-^FzcJ{J0N%oa9xOIfJwdn>Z`AV2=&VKn=f7d^uO2HzFFf?-55mw z>6gFr?eF~RyTAPQE=1cqz}#D3Edam;WIde)hw9avZvd=y1SU0&mE#QYK=;zg^#*e9{w3F4PQO?&nBYgga>TOvZ{YF05h{_|h@5?FP>u>m~y`J11+3C7qP z%*5@R*ZFq)^7TL1Ary!$$xJm{&NasW>&-8I^Xp%K`^~rB1{MFcS8l&?{nqQR-gxc0 zb$$58i_iVGLx{;-HHF==Gy1bHy!Gx^-}%N@zjXWN?a#gO#`V{3-FofSo40=cv(LTv z(>|d*bJfg7vydHj|K#RZzxkE7zWl|{LvrIfRbGD;psr_c{;6`<$=02zy0v<-k?(E( z@Qt^=2C(1t>%jjr39mD%&%F3M0%@76`t=GKLd|~vcW>VK#+SeFDnMfZfPL=vOD{eD zy$HG8^VK{qe^PC$oB!_m7k-hM1TgHz?OQLt^b0~F!xKMK$%~K5khj_TH`jmhI&*dl z7`w~2|3Da(OVviH)%<5K+yFT5#*OE%{Q5c}GfS0Lw$Z8n>QirAfBom4{ig89mMZN! z;dsfnpL^xXjUV=j(p{>MpPDPxW4B-Utqcj}OO<{%U-xs3JO7(kaETa(a;}_B?U2>J zRN369E4hoPo2Bw5T&h&?_&!&eD-W@0Ih$M@DsL5uSSHd%-%4(tZwD|FZxsufZNBf< zcdR5XFZKc+X53nZyjL_=vIDb8b;y*T!)$CWvpX4c*(D2s91^@vDZ+f_Hpp9%b8#c@ zWVxQ-=wVpJz?Dnxn9Dx)PAo>AOC-)*sb40Bg~00+p+Q_3KbfCa0>19aI_IWG<_gPV zc9ij8H}WoRZj`;Emt^BRja8AYzn@aR*EH75A=aW)iU^=-d5#KOIX_k4%2-ba*rW2< zM3&GaIf`_d^PCjSOLkQ(p=`h1)F#)mLdg7uYbi;9QCHThV%~MU7<=!oF_N*_Pt>@= zW2SoCOa>rR3<20o^2s!@N_=PQo=(*HZsmu5!3}4LWZJ}d;-xZ#HVt!bE*=$_wwK8y zihS8}+*p?2@om+1b!=sxACKY#u(vs^_WM(RIj)ZjwW9C1?_T}vYoGb-i!Z#qi-Bc5 zpL+Hx=vSAYyTaeIpS&n|AweyjJ$v%ZnNz3EoqFV#{u-9L<6#S_Zq$Bq^_Aye{ka!j z{_OJ;ynLXd(wDCQK@R-WbI*d)M0v22&R;ll>cWMSPo6#b@Rc8mrCMa_5F5Gj&ZQey zUi!>)FN4fJ0r#@5)xpmPt_%343ul2J0&&PXEm)z)&pdhRku$%kGQ`P`-;y0`e|jO&NjT(`9HsK z_2rkZy>RWt=M93_l`EgpxSzgY%AE(p3-q8fr_P^$;@FEnf}@g$3R`Iu>w~|z^)#4s zz{u~R)q^5`=@aLfB%#lPqI4S6wR4|%?4gVQrT|N|>exy0HKIAYfBx#TuU_I{2J33g z%+Hli2@&nwlK|-pJ*s#1hh;QdG5lgCoVjB=KN`}uufU0hUbny`Y+eTQOhZk zuf@&6h|#*ip>p@iT>kLm#zVv258nP zV6mw3%;|HFKk>k){_98_RnatV0W7>iz5E}Zy!71Dm!3X<0i=3igPl8j^0Cvu>4>A2 z{Sw@#h;H?pPo01M(#4Y(&zuB!>LRF2$KU)rIO-fpOl1@CRw@4TPdxR>^E@xQbK(5) z6QBNXVyR^uX~SxXrGm$$z$HMg>dV*vykN?-#^UOPhq7X5oymR02SVI4EWYVyUGq5>{NX zRDE-VmURGrge5j@SSsJQFts{jqH;}Rq8_;gb7ed#yF3&xjJfo8(qgH_F6m6HK2mIw zVyVThGU7<-$!2`#zN9GFm z27c+}wzWxy4SgHaLN?WA-B*aBAd@6#_O%!&ohB-fvD;u6F18Q!|_mX+>RJl9` zvS)g6Upy6isV;p2ZpBmKnbZWg%OnQksWQ-V+w+{bc&ZY4@_5M|@l@D07HwR~qqt$J z#@?R=A3#hM`!AN}L<&ZVjT_7Bb_y!EYIf-6oRDBjHnW~fCaSoqpYfv+aaBsk3*xG2 zC+dr!3|Dn_xoW3FK1=@d%csvD7bZTaR1@5#)KLJG?iYj={{aC7^7i4w zM}TPGe_&yL{@~*5=YK3kwa_YWw735D?BkD|KJl@W;CBmQOOWR2`yT}LN8mpX1E~Y* z5r8@O9X)tpd3hh8OfzSG$LOieZl%)M4uA2X(>rE^2lSyk3Ie&;t`lw*24wBk%XHg#Ga$T z<;?Cobl~vp{#$=3uG()8+Wldt{TEL^c=9BGX(x6YKK8&vAA9WB{fAAtWk7un-FImJ zzNPyP&VTZU(o?Z$^t!`(XZycjedzf8pkCbrvh>i=LxL3oFR3N`o~60PBMbA7{fo5p zR1AVNi6MID&o4dr*z$2rk!K*B9(_=a9Xbpgk>K)|!IWB@KJ?Dtz*DWx>Hr&VceuSX z`ooj+#~yxS0_gPc!}l*ATv`T>6}14kd@w1Y#3?>lmsC$qe)xjM3M?&|-{!BXdY4(5Y# z>jy`V9DC>ipq8LmI!J}UG0o2hPX3o|anvpiyZu3%Qyj|w3RJa+9(fRaDUDwMTnn?y zzZMfm?fL!gpxfW>SN{n!_Q=tFO9%IZcy#2x`Gtj7ek_jK%h4wiy*k6%_a0onPX`Db zWSK3`&0qXIanv66b`Gz;;21qt5y2)@_mc*`xh63 z2fp%Caa3$so2}%Hz>$Te{nJqjt$&X*Wt0an#P{u+<`vOdJ)P?Pd?NS#D_V413LX zr2r>g>hP5SUNKedeOt|1cEtDn`rc|(@?xWy9Iavf42%}6nC#Kr!ca+9Zr%~9bUN{UV zD65kz41IO>)H~)1yF-=G0x{Lrbdv+(0PK++YhN&v~pvh*SY!p=)dzwU? zc!sAM!JyKzwd5aah1%6N`UZ#~vRc{{8_H3LXOK>XE7AKv_Ksl+`i8a6Jm()d>Kvp5Tar zQ&UG(SRZH1A8*zBQ>Ved(m@7i!N8)fGkoZ5gL9m5pmPq+1B(S5);T_&1k#K0=lQq* zP!_d6$;U;`BBk5~K0b~`L{^bcfSsi?6Q1JiginH@^|T9;GZ_NotDMcCGa8>{9|;v`ik#IG6`@wvh3%4;ddxFl7A7aJrFlYoh2QcLmH}{n~0xvmuq{$4-s&89oV_dvEm6>$gNCh6LzH(dRbpDV^H4^N`DK+IpC>@Id#*=nQ@>NQDcCTY(J!eJ4 zaW@H|1ISB;$SryFo-T8fBzO6c=_LNnb~qGMr&5knaE+U|seJDbX|m9?{LmVxBurX5 z=g5~#A_$rbF6e^E3c-5}Z>+d?qaMM9sWeyvG=^_kfC>EqzFW-&%+gD&Bmn~gF2g0O z9PJJY#<)jtVq9AfrIN7(`Q4d3z2v^U$7G36gOz=JoG${J158gklVs;eFK{e-jKWb;y`cxdqIyt5jM#=srO zcvNkQs$rf2yt^5W=w2f3Qj{sT+yNQT3JKt5wY0JpO|h!TxZ@`Nb1fx;>R{(ZXDyTx zqCtXGX|g-WsM4B{qFha`CKBN|*imHH=;1(MduK^HiA6K%6j~yRv*Urj0Ctk=!QZRf z(To#M3r>PZ#=7*>lk?+pZGAl+VdB_pXVaNH8G~fLpC4CvaPd?m>Ski8L^|iIYjE>w z>P{pTi#Wcp9nf-p-~750U5&@1NjD8L4~tHZ?83OU8Rn9Nen(mTzU_Z}TraGJqtUd( z+M;XZ$ivSYnf0}Bif0?6N)pfE)TFhce zKBx==K=9mvOS}s85U`@4i=IAsV|eheq}XaA#(Tt*5bzKWPOO9XQjm8nYT9tcl5$J} zlbO!gnIdahWbQp=Dk7HTuZNN`8U=EUOr{+17#~up#=P%{CB@g1kqG!3u!;=m47^)= z9rMHyAQDUB`SY+rKgbkVrHo4Lxvm02%#wp8h1R3O)!?D$GQ9tmM?~>GG@e+Jx4xQ4 zr$9MK`e`wVEnWtfmzi8E%iR)N5KD6IMBH?gh3aP1nCtU8TV9`+l!5Rrz>?y3nNYY2 z6U7u63dXoc083h1PXN0`pHn2F>pl$(!Qh|<%Y!8qqS1I*&|z`5UFMYp--cI4w_!=y zNHoNllTi;)ExoWnQt7SXE-c9pMMA*Z#8SX#`TVMv8l;$Q9-3HEW+e(g13t>3YEx7V z^AyCALhz(y3SL1`rrgR5d^$zGs#sF-?rJ2-+w8<$Zt@0dDUE>kfF)&Cc~YS`Ym@#k z|41xxSh1wUN<5B0KsYi~9u5Sy14|l2Bg{7S`yPux%!dc4U43;sl7_noWQKLbx^&bN zv81)NXebeld&!uWgqgwf9kC=HTr|lmEzXeiNyv3FVoAvl!pUej<0g2jY?ocv{K`8R zi$xNdWSpl1Tndj2mNW<@DCPU4^TN7pS1f5Y#M%SMB~I7UnT%LcdTlk7NJS`5*HdY+ zq{M0{o(Sm|X#nCt>mMW%>k;?}V#EB9yBkhN*20l=47d%kB*a2;bv=}p>=sLkuf@Y$ z0y&rxOG+4``eSKwp zRV=9xTUmn>!IEQ#UIwgGEd@cy;>Jb%I6%gR!Ht=M^`b z(u8sPRYy20ZM>WK+)%QKu!kiTGrn+k2muF$R6a#KaU?d|)t)}^3Xug zRAaMN7E5yC1liMLs))prcri)X&gV8ZT4j2#BvR$Yff-6L7fIc) zq*Rh5eAG%zsT)ejR9f*pG)iY}72`zBp%e<2N_m!B{D#gg)&6lt$0 zo8MV_-LSE@xhX+;b;by0yYvU zmgJ2wIJ-6?n;({xi^j8=6p@!@`osLQyOi5WEXj>HUdqjFFy*Ay@Jfq6V&BvdQnNyE6(L~1pBKcoQL z6?jq!H7!#RiVLffVmr7-YKqN+#jCko3I}pAO5m%6jCsk0kytH`I#%dUwCOj#FiY8_R*qq@>$*_1* zarGXaq)hcEI2rGxNs8Agc@HZm{Wx3M6KrLVlR5vGIZN+RWw;+RN9!GDcdKLePKfd$ zbL!p`>~c?n>8!)}PU+{P=4?LgbSK+eyzV--&!p&|SEGm6(Jq?<{m9?{0d}y9^2{%? zjXh|#v6C$DD&I`|I_(VEEZVt17UV`fX_%Fr3-Qts25yq0*b1;h0ii;> zPb9`*l5kBdxk7kVI!IaeoUjuUfo&1-o8Y^)3EbuFRuD65IUd{GD{R7|+03fh0Kv-& zLkBF%?CK2Cmzv<6p$19zOcGBUN3`+-zfx{w{v}H+NO7tG4T$jcnd~-OV=1IWm9sH| zS$e5Rc5IKW9I?9s^qLf>pxW(1#F}S!ogp0|T#vmVI(>D>w z&;+s?Goj>a&d!2r$xKobuq~QJFG?mHo<-GkS36;8(V7jB*g{Ry;FKk%Qs~9nd8@Sa zUrgiiG;lRh&7G8CJOrk+xs+L6C1Fp(4TA!el34ahaqv^qsYFI-d<0$3WOvl(Hsn&i zO-y%oTEKu<2eN@|led<~Lm;)2j6`*+-<)qs3{Z(_Kr^w0n(p~QU%mt;*5@y8djm1L zR9N_mbAFv|0_r5&Gyb8kzBl6qfX(0wPhNdw^B;=}GH=HtP`W{9wE;jo7rE!p`Zids zng^6knnQ8q9S4bKLe6Z_1?Ep*Aei?}xOiJrM+L-y#{`Zy=WlCLiC5R!RIjC}+n(uj zPMLZvoMG0+t=KTi5w9aJEFf}ocS~(*gN-*CA@D(MOZYMyD*~xmdo|UUWk78if zpz)93b|y|*xwTk%`dA1+8cQazg9m-vtFe>7M58?y%iUxG*x7*J;E5VcGVyUg>F~}2 zfj$|Z*`Vo7;!mUV{As_zjIdc^D7atZ;WVs*Sas!dT5|A^_sJt&9g9%v%?6m6NubrcM$Me=Bp>0Ng|qtEdgmd<9C5~U5l7HCl^eWd|)@n+)5l3rI!_K#`4uIB;TlwM1;wEofO@x?~Q3nlw8H+DV{}Z$mo-T8_!J@5*q9indKYl_ zbezW;)geY!ioLGphD};9(_@nZOITmYB7D2-d3jc5>ql?W19sifyAc_>dhrDBpl;Dm zD{ZF+Np`sF$dOayQViU~$vrJzj1sX#XeEsvwt?p2YUhWAjGa)DP1N1h9NJcyw>cww zRGR%?JcqKG0cBSOG5CTQVbZ8iR-_S40nd$1GMRKR*eKMc9!VrHbZP6B zN=ELk<#8S~qOmAeGt3igpjRR#v?pu5vlfX*Lg8>|jR}2fY=ydOk%VkQku~krI)5L& z8&0svy0czGE$XH3tWo&RN*rZppk3ZNL%~zJ9k~-u=H?^{0}*Lct1 zHpULZ;?mNy{eTC$A@)pCm$?eTHh3ploZJ=z!1~IrJr$3S$EGw}00{-%4pJ5yjTSqW zMDxm%EJI-}X)yuucb;uDI@l{Lo@2xrMFTMKO|!ZuWzi%hKg5Go1Mi;0eESmN{2*qB z72EtvZcdQmNJVd(w*$$stp#quwP97faV*kg*p~>0rD9>t<|f4{sCK&$!OGYn$g*U) zs0ii!8V@8CV?#gn6 zSkhPwi7n_!4USje$Rsg57@A6pA<$|xDG{!@qab#tNojK_X`_;uNoZX~d7qJ8lj3^U zuter1!d>QLM}2NX4jDaAMbjRt2ONb#G~uH>9zu}PiEup`t{%TW6bPcq1lUHQ_olPQb6}3r}8s1YI}EJBbK);Mom23uA&s z8*;~;MFy}YA><_&Um>49>`H_ia%M{$*v!11>AZ_*xBWJQQ3b3@co!nv-PWYqzJVX-y#h)5dVBi@aPn6@6O*BnZJE(A`hbHQ zjv5?U2@OalPqV`S;J&=cuwW|_a9FyU@>J4FZpR}jiSR6ss0-(p$YN<|TAUFK+jf{* z*;$rjbo0-X23?6-)E@s&GsVDr{ zgPUg((Fo_uLFOZE0!{-076Y79aK6+RfVEpmD@`B6lHgeivsNtT4VkAWrE4j$xHU2f z_TT6Z>0~$~3C?2?u`N2r%0HszNBj+1<+KyW>tQ6g{3v2DL0&Ev2G1JN3pavH%5fwa zzMGT;XYB;sqGp4cp)mL$+dD@hIkXMEIma$n|QAo$5sm_#NylSQ*JcWYQ7g z%xVU;tBE9%JTl|q!%BDbGDLBp+zI6+!qco)2{bz$KrDNiV5WnNM@&W5R*5=9gyWY| zRdz~(5@E#*rR>xwor&A=ZA@|!;d-)|!?|Ong6Sq}CaGO~JQ7)7Ga_7*BZ9efi|4?4 zipLX?wU8?jo=%y2sr0^d56&D*L|3CmgkwFT5UVdp>s=7glL)tPBqaN=1^!@1QIo!XsgvfZRyNlZhx!lkKQVUutK z&P97*a~?$<_G0u7;upi!Fq^rS`C1S*QDkNx?BK& zy#SDK#Nn$X;2zARv19gV7EpcOdhNnZcV%hC0A^-G=0m%NQBSw9D-x0+iVpgPe6~(SeLhqzFL6dAMi%p4eMYr#$&uz$Mht-@` zfb1iCNO>cHY7-Db9*<68@RO;kM0mxN7+{U0fr)`uCR3o&lr;hx41aaI415ya^h8x6 zoJy!%*mVNpqY1^I1}0y4^6Deu22QFh5iT%2J$x*wq#_vdGFW-GzCnoLo92j1@RJ3J za6`^)$!aSzvw5cTpw+QN+nPFxl{fr!St7gwD5AnaJ6fCSwNBpB!CqPr!TOF;Z-^N! z#!3?57|f}FC#PqinwUosj(8}+dbh!G3k01b@+88mK+aj3|BAAKA*L^Sc}&g@NjLH$ z^fB(2t3cCBgo9I8!*-1eK$xD4SjK#oN}$4IX;nkaVKzlKD-jNchZ$c(#CntX)9A{e z=iseit~3-5Q*aZWM7SDP#+Fiyy55?hnun$;ErAj*}3EoJ1aPV=NTB{zc<@IjzsD-o(svwlazeEV zyKpf200@aiNR&t3jVI*!8J1YALZksE1^l5Z2zDJEqJ+wVVAYxMTb#$Gs_deUDllMh zI!g*r7Hq5AZZanct|u#mGd;cOa;wUDPmBZ|EFP8P7)ZR``@G350mK>55lpaoL^yH9 z<}S{J8DM&PSemkVA}&U3#L031S4~EG5Do2(E->L7_J{K0N{r{h^3n_m$tt)35DCr~ zCl=0YE6za>2K0?ZA^bwl6U=2oiIU_vM-WVIaYq9K=mJQE)KGHV^XVD}Tmx6YC!s)T zwtL8NW=-HMigIEmHA4|L$jEU#)TE3E*M(;##VVSvN{;7*dyI8iCksKlw40_N|NJQ6i7J|MrJGi{CM=< zl>7_tPwB&+x;hR_T_Kq6Iq+~Vk7vxEBI<;MtEjqX$imx;ta}RZTSeD>9OPR?*j)hf zmMFUuCd!VFLj>9p0k;g`Ezxg>!Mr`n36u8|b4S?QQQQ^}5_$*01IBalQ3M|GamP4v z@-bW$$ES!Z{SjwMa`NPRQz!6JoZO4Yd!Kl`N$}ky;tt019wBy!%vA#9el&$)V(R+`r^fgEO?`i%Grs3Ib+0xC{UQRB5-?0C)bjghxxOSaJ*KXXn^_ytN+qZAsqW?|z|BEhMzH;Rn-L<Fw%U7;G7P&`v z?QeZ62Di7I_mN+*qAv{GxfreAm3xTeyqKiE_r{8G8&=qEOlAL2vU$%Fm!oXt8Q=7T znfC!b?Z0RbxqS5+-37ou+4<)qAB$Z1Ti@-T)myA6gBbGb_g+o|_p2LIFRbs2GjeSj zaxKqUn!?m?uq)UI0HyLFY;FkHvcfSdY zUs4cP10B9)>KCSLAY$L3zj>>Mg_!@i5q73Q&6P!ATUmFLG9s5|(w-kDl90iQf-s2d z1k~s)P>$mwUO(t*LBgbMw7=G<3${nl>e-=U_qbTs^34ye*x^;BR|s#l+p@?$k!UkX zld<@1PT)vWc)Ufl@`$O+!{x;19x4vsT?|{}c-e$r{f`T!$}7L7pXJh>lZiqiEUy?? z*1BDgsjPYErkZ9+-54KLmarw6yP@-{*!gWU_}cpH^{E=mS^(g-t^Q~TC#qXeGfG^_aog^&}2=O>C;PKlW?l{`?7);f{F#e<4Kho z0wWGozds}}o-W%gd(xJ8n0ffz{=EZ93F32Hk2olZTI5%vc1<;XFg`i3Qi)#tQ=)lL zCplJ&WiZC8E*fDzc?C3hZKO2r&)6s^3z4D)wmS?ecJuGKwyU*b-nQQqSvazqiA$yH z53qIZxstuE;$Y}7`6Le~m55o7oQmLr>QaRI+D*w4 zpj8&#n7?*^0RU_Pgk^z1%b?RvP+9&VZ-5Q_2US@vrFTWShxa|NNKL@a!PuoSak$Q^ zx9T8jLm-snUj4@ChG?+q5ziQwETi^x)?Q2zQ5Qgj!q)$h0;+FoEn2ax=rHN#Z+)Ki zbrnuY`$YNnKPA=+p@$6vPtvF;o+cY#F`1Tl?EtoY$&MoIkg)6eU;&PJ9XB=Rx*YSt zEy;Bf>XPp8;U9ZtM#D)NC7_C#&t1 z>tq6k$G57w5~!lR)70-CddV!t^pbSCOONV;IlcUA$&4_Tavs(akJM6Y4E*BPUhNTn zc9YE8%#B*Q6|5km3b;oG-n*uw9z`k6M?YeR4fd(0-=bs`JN)s(uHr0pT(*Y!D|s5l zJGBte-0+siUu>{Zli@fd>Dukq&O{?A9Kf`$eJ=Pj6?9?b6zn(Kv}>|bGOZjmIM9}` zH5FadRKJ9S1^{oHih__OwSU6aQzy)R-16WkQE-4>&&T)mJwblwD^?{*Xo%j0TerK< z?iv4hQkk)&JUc!yR{etXKHs{a{-9v#m3BT@Fugix0{++|uGW8W+p>qJanQI$Oztw6dN|vg`R{pLw;NVGRwbE}mgxf>z}nlj%wlf90%J9}ko0^6 z^Le;|O!BHuo7yycgy}sAM^x4suo;X zwejp%r>;OBM4~p>o3uB=5|c}yV!2{W9J$>A_JSi`Cj=%1X?jLf-HotYRhq2(O)nOQbVSc2^MC@vf(0Q(k$ciX>b%fcS@xmAUsbzK zOV)v8I_doNj86k5qb0g3gjM+0*a@AC!V!AD%%>id3d?)zdCR`!iV{&hom7@(i|vpL zE^UJcV*;IBcD8wDZ7%6C2-H!#{wF<%TLVl_V&><}B=wW81JPH>vFRxGe?*T>PkzkV z$5Hcn+*1=*9}0vQ*~LW0)YkY1Js2?>nV{wv;&r4wj9H=fD<-g5&ku}q4YGKMcFkg^G#V6Z(^lU4io!V;a)lFOe!{usZGj6q9R%pGvLDR~r zEICLPQPud%j;<;)mo0qoy0+(@v9G1QEV0xo1p@0QNy?v36usNL&G z8;}x9;wH;e&H*eHSaK9#OX5yqzgZ<|V^uF4*J(m7y0Syx-GT|wU%)}`i{P!to=CXl zYZ-WzbSkjeC_CL|gbS9&ejl-Vm7(8X*>d$oIiIUBN}q9{w4;dXWVVH1pCtvxq4Vw5 zAL9QfRPomJZr!}sX3P`fXK`BY0K8LA9Z$* zt46vGMn61O$Qd~7GEU5rF2v-uQC&gh=Gu58|>Ng@H(I zX55K_aE+JC$&Urw)d%y@%bt0e9tjgdj45F|Tuf}Nw&wuB!Eu$t3psA7u3)E~HO>HD zt`cf_We_3Gy=fs8ZM4hezZKS9uH0N;xu5?}1pNPAE*JVbsGG?GUSzO{7h9rahm?+& zhZwRXT9O9{)DsjjA^VA#$%)qLC|fRW7g_tg5UJ{A|h265U; z%&z0Q)9-<^>Tt7XiPVfz2hGZx0y9slqSyqO`DM50HO{Q-8Q0YGv;11Ze|HX` zhVe+h?;_yrlUhrBLmS>L%lb7lelVH{KjLefJwoOr5kpUWVZl1mVxaW?s(1_Gt#d%a zgequ`d%%n+BfA+e$63yN4ru%vk3G$s4x!$wuG}qPcaJ1h;}6_J;WFl5XmYr~EHmBb zVdobTHt5}HOwgS~2^A)QMr~P;sHE@QE*i^RRRd(CeEUR95-ApXm^1ul=dl-9^S0^_ z8O6D-($ZdfM}hQ()GVzQ=xkNB+jq71w ztuK0_{`jBbr|XzoqI67v3GFFxr%ND>f|Pbv7aN1(dFhVu@Qfutv7F(Els1UgO1;4202@U6^y}G)zPjrC*PiKBGw?$$UjFmv0U5XrC?h$;MUEt083@}tvIIG(Z6CR6eM)p?~TN62){2Zkl%l> zETc_uTd9~uKu!A(*3Btu7EZVQ5Wx_&Iav9@!8o-`kg$X$CQKotl|7i!9A}kaqM*$Z zUG&4(llK>W`%HS0>y6wC{?d^6;L|-|I$S(tv&kwZ5u@Ag08~$#4xg7+DC&yy(R@8G zw8ADOX({7p(-(a>QFHj-NA?_$ggfJZ=ma5XZ@4e8Mc6Yf~3-P-aV#F_2JW0}tsYmIo#t@#|Vp z#}ZU0Y`I|{ZqzR{Y1kSatM|VL?g;24Ekn|o*C36s%?HV6kt#5r z#{Ry=JR`_PdIhmv4ySIMYZdKJYSP8{ZA4F>xc3?^mCBH_7@(WU=YW)?`l>U~WNmPY z`2&T@nu?eNR7yqD9Bg?N`zp3iGgxcdA207>D6@`X-APQ_2^8zFzZVl%ui+T%+LoNG zf1qRb$+(nhv&+4MoM*o5XAr^12{WQ)Fb$i|vdUDmuSgvG-@-R3RN0re@mI7h%u zItvd}qFl8*>CPoOyWYr<6rq_vJ<$DQz4>CwQUT61*+F z^*yHoX$ko@De4l42&(TlHFNVblqd!5*Wi`qTT@fyN5ob|r}k&wcg-0q);M(x1XYGS zO_FZO1Al1EHlJ7>Zy7Lhajf5}AzqdCx!>CDTQR!gKDTcZ7PzId3LX-|HN>iW+FvBy zbtZ6Cw@L30iLXytH$Mtu?_~=*Ef_~IRM$#yP2zgOuDs_3jU6GEb_QOnM(sR|shH9W z<$-+aJ+rsmhn&-lAP-lZ^Jw{n9{Fy7mkC(^M z3304YNHYiICsN15a=-(~I>ci(ONor4g!w;w`CLojwen$pfK8|*Y8JVKpe31#i;Gv3 zl`pFnR-k33dWWHhn@6&AZiOS460#yR#9Ywa9-xHCl2|pVnvRyCG6eAmD+7|yn_Ae% zLH)Llq)Vh_HV653OmMF9ST|0Tg`ZeDdR5s7agQjz%M#L4<$*dAR1`nG4ksyCzdFpz zS&;~(i%8s8$Bt3d3(H3>)XGgDItk8emgHh#C*CM-TTCkBYRgTlx;|1hWgd_-L6+$8 z&oLP=mL_588JvsX-rt3o*QzncXEtgsj+X4E-vK?*G#$0MRXmto;$kxEj}KsMqwLdH zO_F&(IA=BYVjRh28lQE(GP|uS`x!T1M^Ui0Ha<-YELLnU6!~3tG)~R|1_lc-iJ0 zv7;ncD4VFta*5zjhfR)Dmi&s_3LLjHqCA#a@f7tiDNUj&7K0!9JHKu zoRXq0<%uLcbis{+VJ_V~&|tzC4*DRY)Ia+nxx%UMfmyE!*-$p#f^k~JQq?1MyFw7D zDWB>;*pWytbfm|r-_Pfbe{Q<>Ikp)WK4WWdG1hY0lXabA^=>wlPBw_*vJ(9%lu-q54~@ndrs z+~A54EtKvR5lr6Fl~u_}mr&_syO}~IV%is0W{f8`w+tRd>L)SvjNV^4NveXs80JUc$pb*mAH!$uKwutnF+*hv+d`f=PKVvv$iUCsL-;3~M?Ae59lG?7A!> zLp|%rUePc=5n~?vN`y67Xx2yWH7oU--_H$`CBjt+F%zJd#iJyLjN9Hs=2Gg!;GRWQ zP}o&p5_}*iFbKXUkq(XbQu}l)1KWd`-&O0c8OeU45h+Cd1ewwEFUq! zqAdb%Yz-5^XJbn$@&p4^pe&vEsvdeGa9?YFgcFm zu&_9R&WZ#U*~53!;F88o$#V0a*TsuKTLk-nf9ar+b2eXTU(b68H+1Z9Z=JM>rcz7F z_RaiG;ct2$I4qW=SFhZ8v8BbjUVA#W<%e|pF)4v+A`4#*pQ}s<^H*546Y(gMK40!y z1v#H~1xK>A#0P*fP1U$rCUg2)=C`Y%+K5mt9uF&rM8s#i$CU}al5D@pnEkcY-D!D&aOFQ zK&z44P_<6S11PKivg3)hSEyu(nRuoFrGXcJ64*s0-!<+nsV9Vy+F$gfv8~*|Y7gi( z8B3T?wi^jly9a8j#NVE{`##ZW$5%Mc1Nq9MHb}#%0|pMrg>zVC;#x@_aIYJ@spP*k zczyv7OHC*{&^TwI0dWViNO$^p+VkAG5=#k$SwXC+5Kru!ADj5Lv~8jjD<)uUl@^%} z0)guWc+>)mBNaWzeSDa!^GR54^u8C?eDN$Uuu3v+yf%&FDn` zi28m%Ss^eK+Nzf?oH-i!Mf#j-wC$E2c zQ~DW4r1Vips&AZT94fYzLd6axqvre)jelxDJ=oWt=38SImsJeg5jLto9NF3iTZo3_ zm*QNDyPUKr$K5PjP+sZaIBWkLjs*{2^UPSZ{~=6d>_|ue8H;pW4U1B`I)1O`Yld~D zS=!45aD)>)q0(NrMZzL;)3J!PaA4gN?u;4ck!0@c2L_<6$O#QTm*<`U|GIf8<0$D+ zExwi@GxWNLNybqy5eHFNm7n1A&6!mZcW6jrB~>`YOxvS_1PvtW1XcHW6fTw@T|_CX z0LRjdSR~W>CvME>x!B!1>jg)o18Qmxdd40e5P2q#% zw7|kf4`V$vqOY=F(J?xhL$J|UiF4ItQl~Uxc{^>SUNW74sCOUIooH3P)gKvj+jsdE z6I6|g@tpgD_e!+vIy3r7aOM*l2wkD;e5+6%j+QmPGGRe@? zNAziOY->1JLPCMDjfL-$){?*^VCkf>D|ik%J-Jmh{#2U}N} z9b26d@VDYEo9Lbx3mTc<$Kz8P?L7BD>vZY3*hXP1&PG26(bw&)h!x8ni-jVrp|cA= zIih4yZYb5Ps>H0KL@V33Yh}>_nAIzNEU%L<46#bV`rK!zaOM;4wxJN%Tc5N_c=J8% z+twhnluwsxcEMPhDplHuK~86GQzsZ3wvrMXm5!-|(SB>cy(I#KA&J~Pyw(i9I_dV% zrY~pRTXJ-iSkV!@XCZ_90s#DNS#ek)WTVK;|L1nG_3I^4`V!V#CvYz(C&H<|46omj zWic81V!Z;YpuPh3SLi1}qU7e2-aSP|F;v8~LhLZBZoUL3VNv7q@`*k&)24<%x$sSs zfmI+-K0GDkF(RLzL9BzJAby->QCsjM(Ku!G#OI~vK)|K|va=P;m*>k&0+n>@4)JaH zNa5|vR#5clgO0q~rG4CD)JAIve9HkEY`8@v4~>24O;#N>h~k(cmtyP(O$LotKZ&n> zP+qIhUY@V&7YHfw*O?`wBGxLJ2)edxK-dJ^*$ZGWPG}~=aGO3*hk0bC%0BZ>r@Wzp zg2!Bp)_t%i*R6*HjXAQy{+meO*;-ssOp=?xZGZlm8JCU$W6sx`uaBZl7Ek;()O*0> zii76|`P}+yw_HM*aySu%ZNmfE5@=0YAi{aCqqD34G~Bi&7daqWPY;QS?bze7Y!zLS z_H|W)#+GlzCq<_5L{CgSM~7PK(_ zEb1o(%#d4#2F>#7TFH<+)?AxG#j{IqU_CIQ{D~knomb(`Yt@HiR(By zOQ=vb&&5j4KWxJ31>*g67@#99=wFx!jDn*46ShYJ=Kwkhm>eOHrM1NoZIjYQxEt9& z@~9KVUe@k(JGUo*+j*mg7FJOBVAE!dZF%{g;R#zKSh7T#vRagaV`ijuREGK*KIk+* zxx$N)ym&_3i>jJaoogE0=^(x*u(xclzgC)CevbNqlsx*HA-=Ct_ob<&xl>3@-aSix z$e15~5A(shosFT7DhvHx!;V+g#b%D-Fq@0syD{o=U%OtFqCLril#u7h8W;HLnh=k( zcErjdwu>9#!5Hk_Waer(x`OR|jo^u9M`u<>;10i*lG2=3N(;(s9g@6YT)#51K?s&7DM~5BZo>t#oDjMV7 z`_#2o#Wc4BJeI^6YnW&CCH9{2Mp@{^E9*(_3s8}fvyV+dKHsEK;D9X^c60?Gc@=+I$;a=UZ)g{Z! zS&;j@OLq%UXSxuj#AY2KEu~r8Ei^6Hw7u$JWX#Qg(x-P0FK*4|Wn^UJCvtdHrW=;H zn^itK7*mb=@$>sUi-{2?EYjtrq+h>-h~1nT-=^QK4@6fVCR7AT9#k4gE=(5{{)9xa zqb7L5@4PoxvxAVDzo^;Q~nwmusC%#8< zp{C{(IHf{@p3zS}JBy0yN`xKSP~LKL{@UB|&P)R>jWH#?<$q#*Mj<)aJVB!UkJ5l* zd4c5cb*HY7FpVlt$!$3Nc3vH*az*{uTSVlOCv(_{AK{rFUJTOb%x%kbW4`$Q+X4Nr z^CgsmYWGCpA-!eR-13-WOiQv`aw)4?8EC*EOHL8g;c1aFQDc{@TAS7(s#(T2}`|?{0*u-PX{^3I5E7 zmy;CRKBnd8!?_Af^g)ep76I}3{UyXCO%;B|=$DHxHxnbgdN~Atwb_=Bb+!M9m3lyJ zv3lNDm~>Bs&5h3&9}T4sdDc|g$W$Sf07bjXVISmFeL9V()-HZUoB1Y-m*;@L@kz8A zA^doj?#VjUl(%1Tq+oNHEOBG{G&@=+YT>RYEFg}fPlx{GUgN$uQy_;B@oqX?A?O@n zgoz9I{4eSl2_+RIpk#1|=K zWyfEQSpM0wGpcyvsy7a6CuXhbMP>{sz(Vv!pS{pP{bGs{7q`RwDSkCX!$LL2`$eBB zrN$5{G^6{RJ)>iUWl+MdAT!QbcSwAi1a7s(|5*~iitU(qXi_|c_sL1c8SOp;1wGch zq+GRN_>y>A1tp#~zAu$XLzvFz09+-~Do9>1Az|lUy~3D%@uYd{0e|pfuT*=2#ykJ& z-mI>`%r?k(4t6CSEq2Q*%BZ@+Wwsx@Ve$LY!R^iU1!?T{?a^^*Q}f{~Rd;L9TXl5{ zIcZIi8H<1I;)B(e*5{**7GX+U2WzwCcK9VSY8{kNm0;X#-t1FSvARCrdhd{YeD_C> z^HEUA8@QXqHn}wK1d>{r8Y~66EsuzlswCW--Uay_9mr+Vg#@`fjo-wmG*cTTPx@C> zkLVY>?%m8m@E@E5o^y3-HU;IUm1iCT4IQ#%0&tcMurgun6gdDvzOGQ&HBkjHw$aNWt z6%OT-hh0dMx7)TZ&YYaSG_Xc^Gdo!9510YUGg|J*-7i`|d;3eWUleM-U{(+b(A^z- zV^Hw*CF8|N+@+e2QC-!s)L<8rT>L&?y0yLj*~Im=6?vu@1}^h=JA(Jj>2KQ2;TIdGV&SpWM}f+;N;0>C$xxcVuDGMLoy68H$epyNYPR$Ve(T|w zhp`Qh(Vjhm8J-}>2p+@3?Le(nJrAK6xbxeF5nakri1t<@PzG7@Xt3Muz*XJ;6=F=( zZcR`mydh4YovWYRHWtRTN3qV*w8^#EcD3m(!4qE85*;t8cKh3XQ!6;+%6qK(x+}zW zCF93l&3>PZE%2mOLS7}#iPqy3cw98Qy#`yD+7TTgf1*v_RnDc4Gza(Xr~{=rI+`7_ z{h2kbA;C6F>l|R8MKcWKi^`)rk}4yk9WybECHz&ar}-0-v=by5dO={*MsdmQpELDL|R5h|Ws&@q#* z^(ko2>}amfd+-srn7RClJCNQHZDf%b!re4sFFS;;x35MqbaTo_`G;1XGH`t9*_N2k zp;h??V^3v|>NTq@&tf3MDY-uimnYA@dmW`Hs_5~?@ok5u56Tz#8&mm&iyU9F@MbAN z`UB4an>C7FRlG7GH41zV7KekUiY|HApI`uhfh^I@lE&>v#?~{s4_3vbuO(-6X(ge8 zdH8BL8=H6M88e9#9$;8a~JhGfX(D-;5U0ioJ?q`0_M3xfpRZjx^>zlkTNIm`oiS4`9?s2Cmb%CXk_&0$x14+Y z`fHPD9efiut~u=^Sf4{F#MW%VdkAC5^VG$C@{{5ii^ja=;Di1JEo-2uvqNfX%poB~ z2BV)-JwwvNs^%{d!w5bCAkVSj3rb}}?#j-J{AJ7T?QhPj7tGirYn;k9+=>=xVa#FM?r;k) z!&sf5Gp&C_>AT19C{G^l^^+k^;PM2OQtZ*WBun?^zZj!kTzFsmaq;+JSGv~r1IL+) zp*{ML>^g{5k{g!Mfa0|V`LcT$btv(?O4gCY(ps0iIR3(WZ)+66yR_42UR5cTJWBfH ztx89k732NVS0 zlMCtvwm);O#Qt#L)G%5#A;(HJ6M0o9g&XViY{G)@h?ZviFrq=yUQD4hgC4T+Q zwn3En5RbmTp+6r}Sl67m%2|zT28YnjZmQtD8S$YS**<>La{#^x@-pi?J3sT^r8O|c z(Y+sZKi^x3yT#eC5^ru@QXQ>&`bB|Jn1m}+n zC^cQBuW55*)m?NB_{vhTVitUOy|wPpAYoB))o^Gfv0?!<`QoR$3*;|x7(7m~&w}A! z@j^N+UijlMe}8^*SMxxs`Wpa%2&67e>MlT|5Yk_#qV%18fxTnn(xd}eNd(qxUIfbD z>w}F9&nPYZA*HZ~o3a!wbE%czYE9#v!VD?8vqD|r+q}sJ^^}5_^%rn7k9MX{Fhofb9g#Rni>}ckB!Wi$uV?l=c438~$ek_$A8|1MHh&I4H0*=jeF_5Nrt@~!Ga&tY9WU=W{{q}L@+gn$X7end%M~czV#B}Z3cYju?$?F9zl?53w?({^J6=N1NiWR;KCD?3mzE*P?GQ#6 zm_a|TA&t>)JGvlhY7I_5iRo{SI9iD4?qxB9XLk~Y39=9_nK=L5uM0VG)<+j|pmHf^agJHPDme&cm9R65z{lJ2 z`^U_uhO3aP`+oqgVFp{}gvHS)X=v_R20Xo8Y2JSQtC29c!qg z_vWDNn%fC!IWAn>C*am=)WxjXcMIVW%42_(`b#%n9%4{~!ynC6r*uU>uyL+m4U%dGmdY0z>9`VY-OUR%$VpsmO% ziypab?9!Q|V#y{`@@<>K*C1A?O;o$3DOez?nKuD0JL7P#mk;MU_?0e}2-a4!Ej1C8 zY6a^_hbM(8fxRPAeg;^qQ5JXh0>_+10+nNd7}odaq|<4$GAvKaX>flFsul?H+Fpl<+Ya0>Hk$2J3KHW_b6 zc6iKhyak%m3{+7QUk=dAhrFcplR}ZF*cI@72^mjy%TI1=gn6#gUcSy?&MgOF_qGEo zy?$>0Kxopy+vUW5ctOzDvht>EfLbXQ7qZR#Y&_$g0{^Or58IZs)G=<-O~(J}UVU&o zTSMzyE5_X?dv42NYtK^rICbZ%1w`COL|W?;yIG@@WASK-U8hu8jnw97+X%yqQvxiv zsjQxJH?_ybF{9oR3iX^Vm7Mrtk@{n<%Uao@6lWv9Nm@YJ=$XnBY0F)_)2Svzb+qSD z46~zrD)*V?V~|fTvgU4XNJdRn-bT6FoN23f%FVgbb)Pal`8MJm5I(b_0H&v){vqUL z*GA=Dm1}B{+#^jvy|n+ZQcuxgvMubu08wo}3>m8@{mR#uxv9#lz)QTYZ^{LVc3xLe z^3JK5bJGv#y{k5Y0>X1<{(5fgP-OUva)pKPatCOk<}19HN$2nXbH}Za5p7UkRCf`r zuK?hq4re~Pzu~5)@9fVY|dxqL_*qz}e$!t^^uFy27Id{D3=o&&6DSRn}YYhoHegO3uH zyE$f&rNFW9H_nh`v&a#Qa&$vL3@1Jo&d@C9E^fp7| z*U80}kb}{401ZPN!AV(+T$q!88nkZ$=#5;EX~uhPr%&D4HSUtEz=Yi`?$Sr#i@JFa}uePD2oy-WZfq4&p-gD^LD6?&Hwg zMti-V>`H0-2&{`y1qlqj`?f%aunEpRd2rxsl>9^Hzqp6cM~w0InD zb)cJGQ@2~<^Fk4ZKQI*qZ;u`v&6zBPi7F`*7oh5XNt7&$F}0O%ZDnL(qmgDF-fAPr zW%W3+_724@%|AhL>>)04Cg$Ou8YT5ICG$V5=u%^kb>fu#(TL)J$TNP!#fsyf-IYJV zU1_$^;klt1L(e_xY9t|b$m8Mo!|x7SPwS6P+% zXr)?!K|}rISVov!cTVMh&&JZF6^b1oXhp37>E-+ptIb}S08+uAKC{2_LJN20koUR1c>lB#61PP&>K{^3PW~AMG>OafC@j5Y+6qeBGDmL zd)rc#rd0Y|-;R4P-&%+>TLXRmA?~tXAaJYJC$?6#!&|^dj3Rh~wuk~Ra=cDB8DTN2 zOxT((QE(a*_1l+9UhiM{weB&ht!TUz1G0MMz&2U}@p2wtsu(FY7hz}yDR7u3&SeII zLWvnkQ?AckyMpRiDpk}trKR|QH_{eqYgmL|ubQaf%9>~$w!IUHULJb(ZpMr&Pc_HY zN{TK#G0lu|7bvKa~ybm`lWjWK(#v8enqqby$ z>}H6BpwRPY4N)Cl3Sy7k6FY-XeDRk!4qeu>13!xF)`an0QeIi`_O&35tGEp6DF}2B zRrfa9p?7Z_9T@)-V3Q2Fd{8)9%^57=a2!0Mc@Jm`7^wQQ)TVCesC12|#KYeGu66#B z;J`;kf#m>S($c8&;=i1;C(SwQSbm@seIR!ilvh=%M@@fjhs0+rX0FEfj9Ko=@d#c2 zs58`X8oIn|5FDAQ_XIe9TfR6zcL&p9FlpYaNmt7oqBPn6iI$tFI$L%%k@>x@Jld!0 z#w;G&@kRh`FN*CZb*jH+VYyA`AQ~)EG%;kMI=GjDsPdOys-K|cigb6T*Eo+t;evA5 z?w}~%7nH886Ah}K^OctvVgH-7mV)63Jguj|qF*v1PT4)|CftXmIgYexFPvPY#B-UTVHYJWL(YaMo z{ESk5le68KnUZc83P1QH*t~Y17}o0%sDo33Iq22ATTm;YHN0G*b$zet6yJU%v)HdD z6iL0B-m1fq^qCm83Er_d^V&KrjG89%KU@+7YX@uVvsZP=#s2yHD$Y2ut$&6vkMk`wX#`^XJoFZ_;$lhn4@IJ zfY|dj_~mU2-1_iEyx#qb5x6_*r#MI|0We*RhTOkcYJp#|_!5{Ue4|&A*7c}UjK4V3 zXkFhG<@P#(fK>c8F}t}gZbpeQ`S#y=p}&dl3X-VLXGq)=))U)u7oCDyt5$AyJUdNq{teV&_OG!w>rr1)HEng8t2&_e}mwz_< zZ1B;FVjAUBCV~*LuTXtVW+E(WDO;CN-~T;g?y$8K`FYH|Wm)U0EVg?l7gI0@z1LuR zC$L@>$r1L6us425u)sWyF(yBFmTyodM;0x%Uz=tyUb`k{dFA{wxxysfxVI9$GohjCpn#6wtZq7t!b4=~Ju6sgsaVRQ~81kWFkb zfYERmPL;m%=|KU{bgAIloaBN1_R{7GOmkx5k(MtSmW%{{YiYg(8z$2#X5@l*oJ_+A z4uRhrd%E;lBWX>o1V(0OzGurV7{6swN{wCTr_$S4{y&G{-b&YbGu6(W+vu)$gS|rF8b@O-yxLc|B(2XZwwOh5v$?Il9 zY_k9qR8y+c5oyPioMDA1VxrZ7tqrvj*&KeYm3(ROP^D-GgT$e1Y2^|2xYV@gvk?h_ zu9g#5Ylw)ZIrbb=gm6#2_JrrnrsYtt=*9Vyk^IrDlO#gKlPK_XZ&YP$&uqQCAd+SF z;LgIhb=J}cft{`aJyON@o?XLsx%;}F{%_s4tGtLp*IikOVJI-ik8O+nVbmxphgo$T zaui4u5J_BiJv-K)M&s5lB(F-WSFW~OKpML1VtehJH9y%pe?s^Iy*K5isti(aPOJ># zr}d1vU8`tZhB&p@o-+r@I*2lM=7XuI@ZN7>UaDW{R#}qC^PMeoXbiaCNIVo<+RLvM zfD$m65kc3VC}PaNYe$B0A$>uNw7~GR@uv4A)%XVo|KvFU8@H>64`jRZ#~ooCTbmqz zH<1Ojhp~J1iqRzl%Ce66!m2}7-@Xor6iLh+$B#7Mleb`q``!tvk>`;WluJt8*IEX< zV3Qc(Y-Lpo&hXxpT90{c^DhHKPASo*xJ($bHP+>NtH8Y z591pS!D!Ddt=rp*=v#7<#t%x3&9wYt6Ht#ls(MQ#e-1gYzHAspwvV(R;FPh~b4w!m z@kfTT3U8gVB%S(XbJkz9b8wd{IGJM-)%a%vs%pB;iZOXc?*~1@dt1UnBVfHuWsNp) z2?b=^xEkzpyA0)+b;SJBc3mP-+N7heR2d{27Q{cMy)%YnkVw$mzcWFOwB$0OX_!7E zYxo>m-#a^hiJ{nV>Yso!_utjCpnn0*JcWkefRnD@=cUP-LFKlNGnAK<@9v|pS98;7 zrn)l2WO&emEvS0%!DQO1R83Q2kI#dK+clb(igyUEp7p$&%1#CTF0X3PBnvIa(J>)Z zi}{b)V!UfkB4oTIdUj6R!b*>d2{pxzU3IO|(#r~AuKl?VJ#scA%fsW;Bl1y?d?%YuQ1LGJ zqNMKOq)3AL*9gMHTRK29kqE2;Bc<3_bT&OPO>03msQyozNW*x+C6=@uyWq@2L$@yZ zASle^Mrl%GP149ng8WBBX^HplQNQ#vz2a!g#tNtR)04fdy%vUkk6gq=*1;cw&hk4c zWNcvzC#`u$qKMWqj=gC(=llk6*?JkG9?bt`(~LNwz@H)SM>UsbfY(C$r1&uI>chl) z{Dm`Qq$@RR?<_>fsa0{1!#qUrrRqmNe_AuAMIEM3cqs5kf2ux9>HEh4ES9hHoSg0* zR$$H$(Ne!!=j(v&bzK{;DW|==dy>^HqLPx&DF{S30;>NsJOWGqM-|cFKhYxq@VxyI zk4+A;aA7R|x3RpnnZ<5dmvXQVffXt5RMu9V555vB3Ulw}AnA=Ojzi|fILwYUAb_Wl zIHQXRmtrw^%jUD6HzG4TVCyfRBK+4>E~Vm04c2eBkPBp>v5&j3Bl`ryA6OK8ZFIjQ zAykUDQ)Lyx@wp#**O&a?{Ttv9NXhtx!c}~_d83PPP2YZH;-0}6ma&W0yJ8;Ru7I+g zMp@K=iiioAT~<((-~ZLlbw)Lrb!+B2iZiwW1eAIQ5TuErgbw3KM?efUp^QXoh)4o~ zgr+keLTDL!5r{((2n3`g5J+GKDMADcp@ycRx6nZX=#Ag3yS{JknmcRVpLeY}zut4+ z_3m@dKKqn+@8>y3qODITCF~DwL?I1!V#oK)=NI_=PJJh4NK`Bdm1^p{Rn|~#1-dnOs5!PVj8lcck)u?qQ8M2Fvh^vt*u{%k z$LhZ1fx9lsW$oU=R1b&Vj1Ytj+@+*QmSG_A?Cv}dv!@lZzGk_*enO^bExImBcf%m8 zM)MWJbNP7NmCCOGhmcTk*KVbpw_lpc?!HIo8Z!D`O}b=-9$4Dyx5C_t#gf%a-we$> zzi|1v@R`KyY_6>88+Xs7L7ni)!;R7aw>R;GF;#GR{hd9VK4`RXQ4k2Ff=pw$AUKV} z6SMhQYuv>;@gIH^f&3;+4c%r#p>PyYEW;}MY-FCV>u{_d8KaR(=^{9%%t{Sye&TZ# z9&MG=i5h)sL@pL2FwSbJ}Q%=rN6 zRPAf9T(c1e!Ah`(*;+tKMp@s4KpU&CO%lCNj1ns6dAbp{sRm`u5i~!8@6_~^!aoo8 z*9jQ}B}aiChf@16rEsYy?~DIY;W9NH-{3L-%jy_>z~UCC+3pnF3azthvDB7&O6cfZ zmd;@9yTdc+(Q9Q)qtW93UfeVaZ9*3>7_0S%l}6voI1+jC{`mz7cX7#`yu@Xz!h~Xb z!T1c@IRl>FUu7NF4(BrmG)|YUhvy9k%*=V4T3cIWFZb@QbF1w9r{c^cfONPLdy2EQ zUa>&^U>-=epX5L?63*o>)T41ubg7v67uQyaU#4y?#F}71QTS zYB>Y*>JhEc>nGaz2CMzLs2;AX;60Q>r>xyG=N1Oj)0{=5}k+hfLygxvGhD;w1gSn5DM+iNy#Rnn1^@(M0Y4P$(if|?6 z9xWjVEBOzkqZnbA2xLASQkPhjmJS#lrhXmVSlf2-t*%N&z>N;~>Bl;ilNDgjSi>ot zDCSAKWw%HJMddlW$6Ndk~-px`YPKroZd`vG-O5b0gWjtst4XSbyfimQECYX zx0H$+-}Z;kTMDT;H28)2O$3T2Zvs4G9N3yGF zsNS;(9+h%Kj+7iJ;xD}6aHJYg5YByIPBZ>q*_fkmcRqy4G>9(AU~q*jmL$BqLY~T> zch6%y*oL@XLqsV)2=zD!r}bJvJKcgyp{BmUl%B%rz^vTN4mgr$cRYR$e~H#d?acyD z>`#h5nJ(0kL6`hCD@C#LeYHflr&T$XX_^fI%M%>LyT)gd4H@v^s7v>&<rE6Tcy z*1fZA__RBWT3OM09z<>~F+l0q9KTb?y*l8@ z>R#ZAlAkRhbycd;omZa0AW;b)0C!u_JZ^&=o(_vLf+X)Qr=K`+C3M?@2aVk2BV*ER zw`Vf)q%&O(Rdx&FvV$MRXH%5p(4*NB`b~2VHrTO|A}xJvpgzVcS@~=7MpQNu9%Lji ze(v4UqpE2?#CCU{p3$oLgCL)7mb>+r0~9h$Qbv`Em64HUfC3u*i;b?+JylvzkctWK zq*SptH*aIa81Zwn!>-;z0dlDFwo6$hF~_}Oy)mEomeoI6oxr-yLsEOA6jK}NZr$iY z8Y#m22}KvYihA8y=|0!-nl*1~Gr+bx6>a!b&z3p~Nuf~au2udng&VZOuRh;@#&@$( zoqK=25m4+?@$8)s8^(KDpPIZW^YXJ`!!xk?Wrir}@>(ae!gYkXVEQoTkvG*p#h|bd(IYW1C-r#YDR7$gkpXbE zKASdkHM3hQ)%r`7!&+ql9ce)npr?Lmv%rL*rtqE&g3UgE6 zdegNM>98-nFW;2Hz8W5*1O%GY{4(dTOJ8lj?^DAu&FWs|)H6#t?j#>`8uNXQ_++ldC<<*OE(e(O^rTRQ_ z5CR&sb@h65Vj7w_UBMphD95wa8*S(kFRsNJKU!bR2OHx=?=g!#g59i=n%rzkLMP<1 zaTpnlY-}%7QeW+tg65ASy6Cdu<8foNklRV*JbhwRx#!O?r|K?t4w<1>kWy$x1Few~ z*`c$`sZz%$XKCFW{X6~%%@|+YP&(JwYI#Z=8^>R|_0ntW?lCg zfi_3uQ5iCmzjf>@JR~kq4@=jsHazA*s(u|6OS>RlT;E*u{`J@v5HtmB$+8$DbbUmY zGd&=Oia<`B*~IyII@o;g&TWvOrRKH~J4|^ow?ECfdM{!WOJ8+V%xk3y2((K$dnr}V zvCtL8J|_lCaeg;IKLm#X4$;`!QWy7A9nZCDtYx{!4r8`DzrO%A5G;UD zIri+kgUt^LwLR#cC*(U5PIHx~H>6M5RNWAA`C}{oI>*nWBJYdW3|n%Q&<45A`C4CA ztm_Z9DVjw+Gv!4GYKqjB`y)OudC`H#wYzQEMoRn~WBg~-fKA%U%il%+2OzY#S{%!8 zS-${NRU-+v+o8IuhidB(h5-cw*-wPlDg=hMmn~{A!$)E8MY^tUrtB&&oI0eOxSv$; zTFRp(=eWOv?TdZTKBlXVjxDR_jP#qDsjW&8ac>DI_|0RtRLRsXdv{RiN2BSH6lA-B zFDG!8H$NbFl|Stq1cSTOg?r6v+|Y>`0vM&qFDfLUo+;mlcdBcNvl?brL66q9oD!$w zPYb_}D@rA^w|#!|>r4xs#!`f6&b8-#LPR~3`OoNYl<=H_t)`Q#eJ$e#25<<|^`C!Q z-~U#qckqet!rRg*E;Oyc}VT?etZRn0-yM7>u0*~M^&eBrN@*l9**lk zVc@fWRK6$F;dhbC^L1mBBI+9&Ih2$EQK(Q^OLK0xt3)90!n_<7kBN%b=uR1JaH_4M zyBNG%P#w@DajYFJQg9OU)^4Wt2c6Kb31$feDWOj0(V&d-hsa1sUu)bjH@Y9#yrO?q`9Gfc+qVcu)=#w;^>`-)N5bzGes->SGLa@xv6bP>a<~L4D>7al`F9gn z!$;2_?(tA`C5)Sd&jQaVt;pu++CmUrNBTczk`E`a>uZ2|N^+`fR@iX=JB+g9J)q&e z3MzZrC*NfOng=a`D^n&-;hbd+h6fUMCUfu%hx5e1-@){bnQNYy~Uz+Evd!%^9tYjXU(3i%cFq zDAn`cq-JyO(Fg&z zx<`n6)urN?NIkpR#$m}%e8?Y43wAzZqy8jG>&MG9$L{Ztq%ULJ7iths0>R7ItjkI# zR=ve51+}6o#M&S&9pl=fCOje$JVCx#*2|t4U@ve1n)4^p{;KG-)U-UM%~jaw8!hDm zrt7hY8HcHmK&ubYuo~DCVVXL?h6U_Bh6qaX+qg(n)mGg2md^{TjuD1Y<16XM1M{VU zSE_8*cL%^geVG3X1mOdKfChsni(EZb6~$N#DRsQN!cvrVk#1PTnTl?DA`ftmn7-{B zY?AvDeY0!Su$(}2bkZM34Gu83AX2(JW_k2Q$p|FQC_Epz5MQ0go#$D1$mbrYKD@sq zA*#MhTu2{M38PGIwkon6ET1>w0uf!64(xz~H3!0tPR)f^ZlrB`y?=A-gxrHN9q!55 z+f{ld3*bIOFvG{m~||T%y?JBV!!{;AGD Date: Mon, 9 Jun 2025 12:53:42 +0500 Subject: [PATCH 161/491] Pin older Twisted in tests, update type hints. (#6882) --- scrapy/core/downloader/handlers/http11.py | 19 +++++++++++-------- scrapy/utils/test.py | 4 ++-- tests/test_core_downloader.py | 14 +++++++++++--- tox.ini | 1 + 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 74a6e54ee..54fef48b6 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,7 +8,7 @@ import re from contextlib import suppress from io import BytesIO from time import time -from typing import TYPE_CHECKING, Any, TypedDict, TypeVar +from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast from urllib.parse import urldefrag, urlparse from twisted.internet import ssl @@ -27,7 +27,7 @@ from twisted.web.client import ( from twisted.web.client import Response as TxResponse from twisted.web.http import PotentialDataLoss, _DataLoss from twisted.web.http_headers import Headers as TxHeaders -from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS, IResponse from zope.interface import implementer from scrapy import Request, Spider, signals @@ -286,11 +286,11 @@ class TunnelingAgent(Agent): key: Any, endpoint: TCP4ClientEndpoint, method: bytes, - parsedURI: bytes, + parsedURI: URI, headers: TxHeaders | None, bodyProducer: IBodyProducer | None, requestPath: bytes, - ) -> Deferred[TxResponse]: + ) -> Deferred[IResponse]: # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy @@ -329,14 +329,14 @@ class ScrapyProxyAgent(Agent): uri: bytes, headers: TxHeaders | None = None, bodyProducer: IBodyProducer | None = None, - ) -> Deferred[TxResponse]: + ) -> Deferred[IResponse]: """ Issue a new request via the configured proxy. """ # Cache *all* connections under the same key, since we are only # connecting to a single destination, the proxy: return self._requestWithEndpoint( - key=("http-proxy", self._proxyURI.host, self._proxyURI.port), + key=(b"http-proxy", self._proxyURI.host, self._proxyURI.port), endpoint=self._getEndpoint(self._proxyURI), method=method, parsedURI=URI.fromBytes(uri), @@ -426,8 +426,11 @@ class ScrapyAgent: headers.removeHeader(b"Proxy-Authorization") bodyproducer = _RequestBodyProducer(request.body) if request.body else None start_time = time() - d: Deferred[TxResponse] = agent.request( - method, to_bytes(url, encoding="ascii"), headers, bodyproducer + d: Deferred[IResponse] = agent.request( + method, + to_bytes(url, encoding="ascii"), + headers, + cast(IBodyProducer, bodyproducer), ) # set download latency d.addCallback(self._cb_latency, request, start_time) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 2da526cd8..94b1a1fc7 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -10,7 +10,7 @@ import warnings from importlib import import_module from pathlib import Path from posixpath import split -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar, cast from unittest import TestCase, mock from twisted.trial.unittest import SkipTest @@ -211,4 +211,4 @@ def get_web_client_agent_req(url: str) -> Deferred[TxResponse]: from twisted.web.client import Agent # imports twisted.internet.reactor agent = Agent(reactor) - return agent.request(b"GET", url.encode("utf-8")) + return cast("Deferred[TxResponse]", agent.request(b"GET", url.encode("utf-8"))) diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index ef77f7843..464320602 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -4,7 +4,7 @@ import shutil import warnings from pathlib import Path from tempfile import mkdtemp -from typing import Any +from typing import Any, cast import OpenSSL.SSL import pytest @@ -14,6 +14,7 @@ from twisted.trial import unittest from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse +from twisted.web.iweb import IBodyProducer from scrapy.core.downloader import Slot from scrapy.core.downloader.contextfactory import ( @@ -76,8 +77,15 @@ class TestContextFactoryBase(unittest.TestCase): agent = Agent(reactor, contextFactory=client_context_factory) body_producer = _RequestBodyProducer(body.encode()) if body else None - response: TxResponse = await maybe_deferred_to_future( - agent.request(b"GET", url.encode(), bodyProducer=body_producer) + response: TxResponse = cast( + TxResponse, + await maybe_deferred_to_future( + agent.request( + b"GET", + url.encode(), + bodyProducer=cast(IBodyProducer, body_producer), + ) + ), ) with warnings.catch_warnings(): # https://github.com/twisted/twisted/issues/8227 diff --git a/tox.ini b/tox.ini index 92cfc3794..5680d98d1 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,7 @@ deps = sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures pywin32; sys_platform == "win32" + Twisted < 25.5.0 # https://github.com/twisted/twisted/issues/12467 [testenv] deps = From 24a827c72e9f5b35ddcd12ccce2ce7c6611d2845 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 9 Jun 2025 12:53:42 +0500 Subject: [PATCH 162/491] Pin older Twisted in tests, update type hints. (#6882) --- scrapy/core/downloader/handlers/http11.py | 19 +++++++++++-------- scrapy/utils/test.py | 4 ++-- tests/test_core_downloader.py | 14 +++++++++++--- tox.ini | 1 + 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 74a6e54ee..54fef48b6 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,7 +8,7 @@ import re from contextlib import suppress from io import BytesIO from time import time -from typing import TYPE_CHECKING, Any, TypedDict, TypeVar +from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast from urllib.parse import urldefrag, urlparse from twisted.internet import ssl @@ -27,7 +27,7 @@ from twisted.web.client import ( from twisted.web.client import Response as TxResponse from twisted.web.http import PotentialDataLoss, _DataLoss from twisted.web.http_headers import Headers as TxHeaders -from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS, IResponse from zope.interface import implementer from scrapy import Request, Spider, signals @@ -286,11 +286,11 @@ class TunnelingAgent(Agent): key: Any, endpoint: TCP4ClientEndpoint, method: bytes, - parsedURI: bytes, + parsedURI: URI, headers: TxHeaders | None, bodyProducer: IBodyProducer | None, requestPath: bytes, - ) -> Deferred[TxResponse]: + ) -> Deferred[IResponse]: # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy @@ -329,14 +329,14 @@ class ScrapyProxyAgent(Agent): uri: bytes, headers: TxHeaders | None = None, bodyProducer: IBodyProducer | None = None, - ) -> Deferred[TxResponse]: + ) -> Deferred[IResponse]: """ Issue a new request via the configured proxy. """ # Cache *all* connections under the same key, since we are only # connecting to a single destination, the proxy: return self._requestWithEndpoint( - key=("http-proxy", self._proxyURI.host, self._proxyURI.port), + key=(b"http-proxy", self._proxyURI.host, self._proxyURI.port), endpoint=self._getEndpoint(self._proxyURI), method=method, parsedURI=URI.fromBytes(uri), @@ -426,8 +426,11 @@ class ScrapyAgent: headers.removeHeader(b"Proxy-Authorization") bodyproducer = _RequestBodyProducer(request.body) if request.body else None start_time = time() - d: Deferred[TxResponse] = agent.request( - method, to_bytes(url, encoding="ascii"), headers, bodyproducer + d: Deferred[IResponse] = agent.request( + method, + to_bytes(url, encoding="ascii"), + headers, + cast(IBodyProducer, bodyproducer), ) # set download latency d.addCallback(self._cb_latency, request, start_time) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 4a732bd72..3780ad23e 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -10,7 +10,7 @@ import warnings from importlib import import_module from pathlib import Path from posixpath import split -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar, cast from unittest import TestCase, mock from twisted.trial.unittest import SkipTest @@ -216,4 +216,4 @@ def get_web_client_agent_req(url: str) -> Deferred[TxResponse]: from twisted.web.client import Agent # imports twisted.internet.reactor agent = Agent(reactor) - return agent.request(b"GET", url.encode("utf-8")) + return cast("Deferred[TxResponse]", agent.request(b"GET", url.encode("utf-8"))) diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 1bffd69ed..668a2cd1b 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -4,7 +4,7 @@ import shutil import warnings from pathlib import Path from tempfile import mkdtemp -from typing import Any +from typing import Any, cast import OpenSSL.SSL import pytest @@ -15,6 +15,7 @@ from twisted.trial import unittest from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse +from twisted.web.iweb import IBodyProducer from scrapy.core.downloader import Slot from scrapy.core.downloader.contextfactory import ( @@ -73,8 +74,15 @@ class TestContextFactoryBase(unittest.TestCase): ) -> bytes: agent = Agent(reactor, contextFactory=client_context_factory) body_producer = _RequestBodyProducer(body.encode()) if body else None - response: TxResponse = await maybe_deferred_to_future( - agent.request(b"GET", url.encode(), bodyProducer=body_producer) + response: TxResponse = cast( + TxResponse, + await maybe_deferred_to_future( + agent.request( + b"GET", + url.encode(), + bodyProducer=cast(IBodyProducer, body_producer), + ) + ), ) with warnings.catch_warnings(): # https://github.com/twisted/twisted/issues/8227 diff --git a/tox.ini b/tox.ini index 92cfc3794..5680d98d1 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,7 @@ deps = sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures pywin32; sys_platform == "win32" + Twisted < 25.5.0 # https://github.com/twisted/twisted/issues/12467 [testenv] deps = From 6b5a4a64173fc051063f01c05925519e45dbbfdd Mon Sep 17 00:00:00 2001 From: nakanoh Date: Mon, 9 Jun 2025 19:07:01 +0900 Subject: [PATCH 163/491] Minor improvement in cmdline.py (#6875) --- scrapy/cmdline.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 3d448532b..2b0204071 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -32,8 +32,9 @@ class ScrapyArgumentParser(argparse.ArgumentParser): def _parse_optional( self, arg_string: str ) -> tuple[argparse.Action | None, str, str | None] | None: - # if starts with -: it means that is a parameter not a argument - if arg_string[:2] == "-:": + # Support something like ‘-o -:json’, where ‘-:json’ is a value for + # ‘-o’, not another parameter. + if arg_string.startswith("-:"): return None return super()._parse_optional(arg_string) From 7400868ad5b80f6c91fd69dea476aa8f59ce9081 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 9 Jun 2025 15:19:02 +0500 Subject: [PATCH 164/491] Release notes for 2.13.2. (#6868) * Release notes for 2.13.2. * Update release notes. --- docs/news.rst | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 8b1d51674..8e7d80e26 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,52 @@ Release notes ============= +.. _release-2.13.2: + +Scrapy 2.13.2 (unreleased) +-------------------------- + +- Fixed a bug introduced in Scrapy 2.13.0 that caused results of request + errbacks to be ignored when the errback was called because of a downloader + error. + (:issue:`6861`, :issue:`6863`) + +- Added a note about the behavior change of + :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to its docs and + to the "Backward-incompatible changes" section of :ref:`the Scrapy 2.13.0 + release notes `. + (:issue:`6866`) + +- Improved the message in the exception raised by + :func:`scrapy.utils.test.get_reactor_settings` when there is no reactor + installed. + (:issue:`6866`) + +- Updated the :class:`scrapy.crawler.CrawlerRunner` examples in + :ref:`topics-practices` to install the reactor explicitly, to fix + reactor-related errors with Scrapy 2.13.0 and later. + (:issue:`6865`) + +- Fixed ``scrapy fetch`` not working with scrapy-poet_. + (:issue:`6872`) + +- Fixed an exception produced by :class:`scrapy.core.engine.ExecutionEngine` + when it's closed before being fully initialized. + (:issue:`6857`, :issue:`6867`) + +- Improved the README, updated the Scrapy logo in it. + (:issue:`6831`, :issue:`6833`, :issue:`6839`) + +- Restricted the Twisted version used in tests to below 25.5.0, as some tests + fail with 25.5.0. + (:issue:`6878`, :issue:`6882`) + +- Updated type hints for Twisted 25.5.0 changes. + (:issue:`6882`) + +- Removed the old artwork. + (:issue:`6874`) + .. _release-2.13.1: Scrapy 2.13.1 (2025-05-28) From c6740604a405c51e92bedce8617b9151cd9766c4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 9 Jun 2025 15:21:06 +0500 Subject: [PATCH 165/491] =?UTF-8?q?Bump=20version:=202.13.1=20=E2=86=92=20?= =?UTF-8?q?2.13.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/news.rst | 2 +- pyproject.toml | 2 +- scrapy/VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 8e7d80e26..b2fe78bc6 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.13.2: -Scrapy 2.13.2 (unreleased) +Scrapy 2.13.2 (2025-06-09) -------------------------- - Fixed a bug introduced in Scrapy 2.13.0 that caused results of request diff --git a/pyproject.toml b/pyproject.toml index 68c1e07bb..d6aebf514 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,7 +115,7 @@ module = "twisted" implicit_reexport = true [tool.bumpversion] -current_version = "2.13.1" +current_version = "2.13.2" commit = true tag = true tag_name = "{new_version}" diff --git a/scrapy/VERSION b/scrapy/VERSION index 94f15e9cc..0e83a9a9c 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.13.1 +2.13.2 From ac956f8595354fde80bf64b3eaf95a22b3433f98 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 10 Jun 2025 13:02:27 +0500 Subject: [PATCH 166/491] Replace most of the @inlineCallbacks test helpers. (#6883) --- tests/test_crawl.py | 101 ++-- tests/test_feedexport.py | 619 +++++++++++--------- tests/test_spidermiddleware.py | 219 +++---- tests/test_spidermiddleware_output_chain.py | 75 +-- 4 files changed, 565 insertions(+), 449 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 4c1f6216b..42f9899f9 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -4,7 +4,7 @@ import json import logging from ipaddress import IPv4Address from socket import gethostbyname -from typing import Any +from typing import TYPE_CHECKING, Any from urllib.parse import urlparse import pytest @@ -14,11 +14,12 @@ from twisted.internet.ssl import Certificate from twisted.python.failure import Failure from twisted.trial.unittest import TestCase -from scrapy import signals +from scrapy import Spider, signals from scrapy.crawler import CrawlerRunner from scrapy.exceptions import CloseSpider, StopDownload from scrapy.http import Request from scrapy.http.response import Response +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler, get_reactor_settings from tests import NON_EXISTING_RESOLVABLE @@ -55,8 +56,13 @@ from tests.spiders import ( StartItemSpider, ) +if TYPE_CHECKING: + from scrapy.statscollectors import StatsCollector + class TestCrawl(TestCase): + mockserver: MockServer + @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -72,16 +78,17 @@ class TestCrawl(TestCase): yield crawler.crawl(mockserver=self.mockserver) assert len(crawler.spider.urls_visited) == 11 # 10 + start_url - @inlineCallbacks - def test_fixed_delay(self): - yield self._test_delay(total=3, delay=0.2) + @deferred_f_from_coro_f + async def test_fixed_delay(self): + await self._test_delay(total=3, delay=0.2) - @inlineCallbacks - def test_randomized_delay(self): - yield self._test_delay(total=3, delay=0.1, randomize=True) + @deferred_f_from_coro_f + async def test_randomized_delay(self): + await self._test_delay(total=3, delay=0.1, randomize=True) - @inlineCallbacks - def _test_delay(self, total, delay, randomize=False): + async def _test_delay( + self, total: int, delay: float, randomize: bool = False + ) -> None: crawl_kwargs = { "maxlatency": delay * 2, "mockserver": self.mockserver, @@ -91,7 +98,9 @@ class TestCrawl(TestCase): settings = {"DOWNLOAD_DELAY": delay, "RANDOMIZE_DOWNLOAD_DELAY": randomize} crawler = get_crawler(FollowAllSpider, settings) - yield crawler.crawl(**crawl_kwargs) + await maybe_deferred_to_future(crawler.crawl(**crawl_kwargs)) + assert crawler.spider + assert isinstance(crawler.spider, FollowAllSpider) times = crawler.spider.times total_time = times[-1] - times[0] average = total_time / (len(times) - 1) @@ -103,7 +112,9 @@ class TestCrawl(TestCase): # code above to have any meaning. settings["DOWNLOAD_DELAY"] = 0 crawler = get_crawler(FollowAllSpider, settings) - yield crawler.crawl(**crawl_kwargs) + await maybe_deferred_to_future(crawler.crawl(**crawl_kwargs)) + assert crawler.spider + assert isinstance(crawler.spider, FollowAllSpider) times = crawler.spider.times total_time = times[-1] - times[0] average = total_time / (len(times) - 1) @@ -428,8 +439,9 @@ class TestCrawlSpider(TestCase): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - @inlineCallbacks - def _run_spider(self, spider_cls): + async def _run_spider( + self, spider_cls: type[Spider] + ) -> tuple[LogCapture, list[Any], StatsCollector]: items = [] def _on_item_scraped(item): @@ -438,9 +450,12 @@ class TestCrawlSpider(TestCase): crawler = get_crawler(spider_cls) crawler.signals.connect(_on_item_scraped, signals.item_scraped) with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver + await maybe_deferred_to_future( + crawler.crawl( + self.mockserver.url("/status?n=200"), mockserver=self.mockserver + ) ) + assert crawler.stats return log, items, crawler.stats @inlineCallbacks @@ -521,9 +536,9 @@ class TestCrawlSpider(TestCase): assert "Got response 200" in str(log) @pytest.mark.only_asyncio - @inlineCallbacks - def test_async_def_asyncio_parse_items_list(self): - log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider) + @deferred_f_from_coro_f + async def test_async_def_asyncio_parse_items_list(self): + log, items, _ = await self._run_spider(AsyncDefAsyncioReturnSpider) assert "Got response 200" in str(log) assert {"id": 1} in items assert {"id": 2} in items @@ -546,17 +561,17 @@ class TestCrawlSpider(TestCase): assert {"foo": 42} in items @pytest.mark.only_asyncio - @inlineCallbacks - def test_async_def_asyncgen_parse(self): - log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider) + @deferred_f_from_coro_f + async def test_async_def_asyncgen_parse(self): + log, _, stats = await self._run_spider(AsyncDefAsyncioGenSpider) assert "Got response 200" in str(log) itemcount = stats.get_value("item_scraped_count") assert itemcount == 1 @pytest.mark.only_asyncio - @inlineCallbacks - def test_async_def_asyncgen_parse_loop(self): - log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider) + @deferred_f_from_coro_f + async def test_async_def_asyncgen_parse_loop(self): + log, items, stats = await self._run_spider(AsyncDefAsyncioGenLoopSpider) assert "Got response 200" in str(log) itemcount = stats.get_value("item_scraped_count") assert itemcount == 10 @@ -564,9 +579,9 @@ class TestCrawlSpider(TestCase): assert {"foo": i} in items @pytest.mark.only_asyncio - @inlineCallbacks - def test_async_def_asyncgen_parse_exc(self): - log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider) + @deferred_f_from_coro_f + async def test_async_def_asyncgen_parse_exc(self): + log, items, stats = await self._run_spider(AsyncDefAsyncioGenExcSpider) log = str(log) assert "Spider error processing" in log assert "ValueError" in log @@ -576,9 +591,9 @@ class TestCrawlSpider(TestCase): assert {"foo": i} in items @pytest.mark.only_asyncio - @inlineCallbacks - def test_async_def_asyncgen_parse_complex(self): - _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider) + @deferred_f_from_coro_f + async def test_async_def_asyncgen_parse_complex(self): + _, items, stats = await self._run_spider(AsyncDefAsyncioGenComplexSpider) itemcount = stats.get_value("item_scraped_count") assert itemcount == 156 # some random items @@ -588,27 +603,27 @@ class TestCrawlSpider(TestCase): assert {"index2": i} in items @pytest.mark.only_asyncio - @inlineCallbacks - def test_async_def_asyncio_parse_reqs_list(self): - log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider) + @deferred_f_from_coro_f + async def test_async_def_asyncio_parse_reqs_list(self): + log, *_ = await self._run_spider(AsyncDefAsyncioReqsReturnSpider) for req_id in range(3): assert f"Got response 200, req_id {req_id}" in str(log) @pytest.mark.only_not_asyncio - @inlineCallbacks - def test_async_def_deferred_direct(self): - _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider) + @deferred_f_from_coro_f + async def test_async_def_deferred_direct(self): + _, items, _ = await self._run_spider(AsyncDefDeferredDirectSpider) assert items == [{"code": 200}] @pytest.mark.only_asyncio - @inlineCallbacks - def test_async_def_deferred_wrapped(self): - log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider) + @deferred_f_from_coro_f + async def test_async_def_deferred_wrapped(self): + log, items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider) assert items == [{"code": 200}] - @inlineCallbacks - def test_async_def_deferred_maybe_wrapped(self): - _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider) + @deferred_f_from_coro_f + async def test_async_def_deferred_maybe_wrapped(self): + _, items, _ = await self._run_spider(AsyncDefDeferredMaybeWrappedSpider) assert items == [{"code": 200}] @inlineCallbacks diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index cdf03ca76..262c0b434 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -33,7 +33,7 @@ from zope.interface import implementer from zope.interface.verify import verifyObject import scrapy -from scrapy import signals +from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.extensions.feedexport import ( @@ -48,12 +48,14 @@ from scrapy.extensions.feedexport import ( StdoutFeedStorage, ) from scrapy.settings import Settings +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler from tests.mockserver import MockFTPServer, MockServer from tests.spiders import ItemSpider if TYPE_CHECKING: + from collections.abc import Iterable from os import PathLike @@ -89,24 +91,25 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]: return (client_mock, bucket_mock, blob_mock) +# TODO: replace self.mktemp() and drop the unittest.TestCase base class TestFileFeedStorage(unittest.TestCase): def test_store_file_uri(self): path = Path(self.mktemp()).resolve() uri = path_to_file_uri(str(path)) - return self._assert_stores(FileFeedStorage(uri), path) + self._assert_stores(FileFeedStorage(uri), path) def test_store_file_uri_makedirs(self): path = Path(self.mktemp()).resolve() / "more" / "paths" / "file.txt" uri = path_to_file_uri(str(path)) - return self._assert_stores(FileFeedStorage(uri), path) + self._assert_stores(FileFeedStorage(uri), path) def test_store_direct_path(self): path = Path(self.mktemp()).resolve() - return self._assert_stores(FileFeedStorage(str(path)), path) + self._assert_stores(FileFeedStorage(str(path)), path) def test_store_direct_path_relative(self): path = Path(self.mktemp()) - return self._assert_stores(FileFeedStorage(str(path)), path) + self._assert_stores(FileFeedStorage(str(path)), path) def test_interface(self): path = self.mktemp() @@ -124,20 +127,21 @@ class TestFileFeedStorage(unittest.TestCase): def test_append(self): path = self._store() - return self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent") + self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent") def test_overwrite(self): path = self._store({"overwrite": True}) - return self._assert_stores( + self._assert_stores( FileFeedStorage(str(path), feed_options={"overwrite": True}), path ) - @inlineCallbacks - def _assert_stores(self, storage, path: Path, expected_content=b"content"): + def _assert_stores( + self, storage: FileFeedStorage, path: Path, expected_content: bytes = b"content" + ) -> None: spider = scrapy.Spider("default") file = storage.open(spider) file.write(b"content") - yield storage.store(file) + storage.store(file) assert path.exists() try: assert path.read_bytes() == expected_content @@ -153,7 +157,7 @@ class TestFTPFeedStorage(unittest.TestCase): crawler = get_crawler(settings_dict=settings) return TestSpider.from_crawler(crawler) - def _store(self, uri, content, feed_options=None, settings=None): + async def _store(self, uri, content, feed_options=None, settings=None): crawler = get_crawler(settings_dict=settings or {}) storage = FTPFeedStorage.from_crawler( crawler, @@ -164,7 +168,7 @@ class TestFTPFeedStorage(unittest.TestCase): spider = self.get_test_spider() file = storage.open(spider) file.write(content) - return storage.store(file) + await maybe_deferred_to_future(storage.store(file)) def _assert_stored(self, path: Path, content): assert path.exists() @@ -173,44 +177,44 @@ class TestFTPFeedStorage(unittest.TestCase): finally: path.unlink() - @inlineCallbacks - def test_append(self): + @deferred_f_from_coro_f + async def test_append(self): with MockFTPServer() as ftp_server: filename = "file" url = ftp_server.url(filename) feed_options = {"overwrite": False} - yield self._store(url, b"foo", feed_options=feed_options) - yield self._store(url, b"bar", feed_options=feed_options) + await self._store(url, b"foo", feed_options=feed_options) + await self._store(url, b"bar", feed_options=feed_options) self._assert_stored(ftp_server.path / filename, b"foobar") - @inlineCallbacks - def test_overwrite(self): + @deferred_f_from_coro_f + async def test_overwrite(self): with MockFTPServer() as ftp_server: filename = "file" url = ftp_server.url(filename) - yield self._store(url, b"foo") - yield self._store(url, b"bar") + await self._store(url, b"foo") + await self._store(url, b"bar") self._assert_stored(ftp_server.path / filename, b"bar") - @inlineCallbacks - def test_append_active_mode(self): + @deferred_f_from_coro_f + async def test_append_active_mode(self): with MockFTPServer() as ftp_server: settings = {"FEED_STORAGE_FTP_ACTIVE": True} filename = "file" url = ftp_server.url(filename) feed_options = {"overwrite": False} - yield self._store(url, b"foo", feed_options=feed_options, settings=settings) - yield self._store(url, b"bar", feed_options=feed_options, settings=settings) + await self._store(url, b"foo", feed_options=feed_options, settings=settings) + await self._store(url, b"bar", feed_options=feed_options, settings=settings) self._assert_stored(ftp_server.path / filename, b"foobar") - @inlineCallbacks - def test_overwrite_active_mode(self): + @deferred_f_from_coro_f + async def test_overwrite_active_mode(self): with MockFTPServer() as ftp_server: settings = {"FEED_STORAGE_FTP_ACTIVE": True} filename = "file" url = ftp_server.url(filename) - yield self._store(url, b"foo", settings=settings) - yield self._store(url, b"bar", settings=settings) + await self._store(url, b"foo", settings=settings) + await self._store(url, b"bar", settings=settings) self._assert_stored(ftp_server.path / filename, b"bar") def test_uri_auth_quote(self): @@ -291,8 +295,8 @@ class TestS3FeedStorage(unittest.TestCase): assert storage.access_key == "uri_key" assert storage.secret_key == "uri_secret" - @inlineCallbacks - def test_store(self): + @deferred_f_from_coro_f + async def test_store(self): settings = { "AWS_ACCESS_KEY_ID": "access_key", "AWS_SECRET_ACCESS_KEY": "secret_key", @@ -306,7 +310,7 @@ class TestS3FeedStorage(unittest.TestCase): file = mock.MagicMock() storage.s3_client = mock.MagicMock() - yield storage.store(file) + await maybe_deferred_to_future(storage.store(file)) assert storage.s3_client.upload_fileobj.call_args == mock.call( Bucket=bucket, Key=key, Fileobj=file ) @@ -432,8 +436,8 @@ class TestS3FeedStorage(unittest.TestCase): assert storage.region_name == region_name assert storage.s3_client._client_config.region_name == region_name - @inlineCallbacks - def test_store_without_acl(self): + @deferred_f_from_coro_f + async def test_store_without_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", @@ -444,7 +448,7 @@ class TestS3FeedStorage(unittest.TestCase): assert storage.acl is None storage.s3_client = mock.MagicMock() - yield storage.store(BytesIO(b"test file")) + await maybe_deferred_to_future(storage.store(BytesIO(b"test file"))) acl = ( storage.s3_client.upload_fileobj.call_args[1] .get("ExtraArgs", {}) @@ -452,8 +456,8 @@ class TestS3FeedStorage(unittest.TestCase): ) assert acl is None - @inlineCallbacks - def test_store_with_acl(self): + @deferred_f_from_coro_f + async def test_store_with_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" ) @@ -462,7 +466,7 @@ class TestS3FeedStorage(unittest.TestCase): assert storage.acl == "custom-acl" storage.s3_client = mock.MagicMock() - yield storage.store(BytesIO(b"test file")) + await maybe_deferred_to_future(storage.store(BytesIO(b"test file"))) acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] assert acl == "custom-acl" @@ -516,8 +520,8 @@ class TestGCSFeedStorage(unittest.TestCase): storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") assert storage.acl is None - @inlineCallbacks - def test_store(self): + @deferred_f_from_coro_f + async def test_store(self): try: from google.cloud.storage import Client # noqa: F401 except ImportError: @@ -532,7 +536,7 @@ class TestGCSFeedStorage(unittest.TestCase): f = mock.Mock() storage = GCSFeedStorage(uri, project_id, acl) - yield storage.store(f) + await maybe_deferred_to_future(storage.store(f)) f.seek.assert_called_once_with(0) m.assert_called_once_with(project=project_id) @@ -556,14 +560,13 @@ class TestGCSFeedStorage(unittest.TestCase): assert "GCS does not support appending to files" in str(log) -class TestStdoutFeedStorage(unittest.TestCase): - @inlineCallbacks +class TestStdoutFeedStorage: def test_store(self): out = BytesIO() storage = StdoutFeedStorage("stdout:", _stdout=out) file = storage.open(scrapy.Spider("default")) file.write(b"content") - yield storage.store(file) + storage.store(file) assert out.getvalue() == b"content" def test_overwrite_default(self): @@ -641,6 +644,8 @@ class LogOnStoreFileStorage: class TestFeedExportBase(ABC, unittest.TestCase): + mockserver: MockServer + class MyItem(scrapy.Item): foo = scrapy.Field() egg = scrapy.Field() @@ -670,8 +675,9 @@ class TestFeedExportBase(ABC, unittest.TestCase): def tearDown(self): shutil.rmtree(self.temp_dir, ignore_errors=True) - @inlineCallbacks - def exported_data(self, items, settings): + async def exported_data( + self, items: Iterable[Any], settings: dict[str, Any] + ) -> dict[str, Any]: """ Return exported data which a spider yielding ``items`` would return. """ @@ -682,11 +688,9 @@ class TestFeedExportBase(ABC, unittest.TestCase): def parse(self, response): yield from items - data = yield self.run_and_export(TestSpider, settings) - return data + return await self.run_and_export(TestSpider, settings) - @inlineCallbacks - def exported_no_data(self, settings): + async def exported_no_data(self, settings: dict[str, Any]) -> dict[str, Any]: """ Return exported data which a spider yielding no ``items`` would return. """ @@ -697,20 +701,75 @@ class TestFeedExportBase(ABC, unittest.TestCase): def parse(self, response): pass - data = yield self.run_and_export(TestSpider, settings) - return data + return await self.run_and_export(TestSpider, settings) - @inlineCallbacks - def assertExported(self, items, header, rows, settings=None): - yield self.assertExportedCsv(items, header, rows, settings) - yield self.assertExportedJsonLines(items, rows, settings) - yield self.assertExportedXml(items, rows, settings) - yield self.assertExportedPickle(items, rows, settings) - yield self.assertExportedMarshal(items, rows, settings) - yield self.assertExportedMultiple(items, rows, settings) + async def assertExported( + self, + items: Iterable[Any], + header: Iterable[str], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + await self.assertExportedCsv(items, header, rows, settings) + await self.assertExportedJsonLines(items, rows, settings) + await self.assertExportedXml(items, rows, settings) + await self.assertExportedPickle(items, rows, settings) + await self.assertExportedMarshal(items, rows, settings) + await self.assertExportedMultiple(items, rows, settings) + + async def assertExportedCsv( + self, + items: Iterable[Any], + header: Iterable[str], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedJsonLines( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedXml( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedMultiple( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedPickle( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedMarshal( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass @abstractmethod - def run_and_export(self, spider_cls, settings): + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, Any]: pass def _load_until_eof(self, data, load_func): @@ -771,8 +830,9 @@ class ExceptionJsonItemExporter(JsonItemExporter): class TestFeedExport(TestFeedExportBase): - @inlineCallbacks - def run_and_export(self, spider_cls, settings): + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, Any]: """Run spider with specified settings; return exported data.""" FEEDS = settings.get("FEEDS") or {} @@ -781,11 +841,11 @@ class TestFeedExport(TestFeedExportBase): for file_path, feed_options in FEEDS.items() } - content = {} + content: dict[str, Any] = {} try: spider_cls.start_urls = [self.mockserver.url("/")] crawler = get_crawler(spider_cls, settings) - yield crawler.crawl() + await maybe_deferred_to_future(crawler.crawl()) for file_path, feed_options in FEEDS.items(): content[feed_options["format"]] = ( @@ -801,8 +861,13 @@ class TestFeedExport(TestFeedExportBase): return content - @inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None): + async def assertExportedCsv( + self, + items: Iterable[Any], + header: Iterable[str], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -811,13 +876,17 @@ class TestFeedExport(TestFeedExportBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) reader = csv.DictReader(to_unicode(data["csv"]).splitlines()) assert reader.fieldnames == list(header) assert rows == list(reader) - @inlineCallbacks - def assertExportedJsonLines(self, items, rows, settings=None): + async def assertExportedJsonLines( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -826,13 +895,17 @@ class TestFeedExport(TestFeedExportBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) parsed = [json.loads(to_unicode(line)) for line in data["jl"].splitlines()] rows = [{k: v for k, v in row.items() if v} for row in rows] assert rows == parsed - @inlineCallbacks - def assertExportedXml(self, items, rows, settings=None): + async def assertExportedXml( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -841,14 +914,18 @@ class TestFeedExport(TestFeedExportBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) rows = [{k: v for k, v in row.items() if v} for row in rows] root = lxml.etree.fromstring(data["xml"]) got_rows = [{e.tag: e.text for e in it} for it in root.findall("item")] assert rows == got_rows - @inlineCallbacks - def assertExportedMultiple(self, items, rows, settings=None): + async def assertExportedMultiple( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -858,7 +935,7 @@ class TestFeedExport(TestFeedExportBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) rows = [{k: v for k, v in row.items() if v} for row in rows] # XML root = lxml.etree.fromstring(data["xml"]) @@ -868,8 +945,12 @@ class TestFeedExport(TestFeedExportBase): json_rows = json.loads(to_unicode(data["json"])) assert rows == json_rows - @inlineCallbacks - def assertExportedPickle(self, items, rows, settings=None): + async def assertExportedPickle( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -878,15 +959,19 @@ class TestFeedExport(TestFeedExportBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) expected = [{k: v for k, v in row.items() if v} for row in rows] import pickle result = self._load_until_eof(data["pickle"], load_func=pickle.load) assert result == expected - @inlineCallbacks - def assertExportedMarshal(self, items, rows, settings=None): + async def assertExportedMarshal( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -895,7 +980,7 @@ class TestFeedExport(TestFeedExportBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) expected = [{k: v for k, v in row.items() if v} for row in rows] import marshal @@ -956,8 +1041,8 @@ class TestFeedExport(TestFeedExportBase): crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage") == 1 ) - @inlineCallbacks - def test_export_items(self): + @deferred_f_from_coro_f + async def test_export_items(self): # feed exporters use field names from Item items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -968,10 +1053,10 @@ class TestFeedExport(TestFeedExportBase): {"egg": "spam2", "foo": "bar2", "baz": "quux2"}, ] header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows) + await self.assertExported(items, header, rows) - @inlineCallbacks - def test_export_no_items_not_store_empty(self): + @deferred_f_from_coro_f + async def test_export_no_items_not_store_empty(self): for fmt in ("json", "jsonlines", "xml", "csv"): settings = { "FEEDS": { @@ -979,11 +1064,11 @@ class TestFeedExport(TestFeedExportBase): }, "FEED_STORE_EMPTY": False, } - data = yield self.exported_no_data(settings) + data = await self.exported_no_data(settings) assert data[fmt] is None - @inlineCallbacks - def test_start_finish_exporting_items(self): + @deferred_f_from_coro_f + async def test_start_finish_exporting_items(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), ] @@ -998,12 +1083,12 @@ class TestFeedExport(TestFeedExportBase): InstrumentedFeedSlot.subscribe__listener(listener) with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): - _ = yield self.exported_data(items, settings) + await self.exported_data(items, settings) assert not listener.start_without_finish assert not listener.finish_without_start - @inlineCallbacks - def test_start_finish_exporting_no_items(self): + @deferred_f_from_coro_f + async def test_start_finish_exporting_no_items(self): items = [] settings = { "FEEDS": { @@ -1016,12 +1101,12 @@ class TestFeedExport(TestFeedExportBase): InstrumentedFeedSlot.subscribe__listener(listener) with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): - _ = yield self.exported_data(items, settings) + await self.exported_data(items, settings) assert not listener.start_without_finish assert not listener.finish_without_start - @inlineCallbacks - def test_start_finish_exporting_items_exception(self): + @deferred_f_from_coro_f + async def test_start_finish_exporting_items_exception(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), ] @@ -1037,12 +1122,12 @@ class TestFeedExport(TestFeedExportBase): InstrumentedFeedSlot.subscribe__listener(listener) with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): - _ = yield self.exported_data(items, settings) + await self.exported_data(items, settings) assert not listener.start_without_finish assert not listener.finish_without_start - @inlineCallbacks - def test_start_finish_exporting_no_items_exception(self): + @deferred_f_from_coro_f + async def test_start_finish_exporting_no_items_exception(self): items = [] settings = { "FEEDS": { @@ -1056,12 +1141,12 @@ class TestFeedExport(TestFeedExportBase): InstrumentedFeedSlot.subscribe__listener(listener) with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): - _ = yield self.exported_data(items, settings) + await self.exported_data(items, settings) assert not listener.start_without_finish assert not listener.finish_without_start - @inlineCallbacks - def test_export_no_items_store_empty(self): + @deferred_f_from_coro_f + async def test_export_no_items_store_empty(self): formats = ( ("json", b"[]"), ("jsonlines", b""), @@ -1077,11 +1162,11 @@ class TestFeedExport(TestFeedExportBase): "FEED_STORE_EMPTY": True, "FEED_EXPORT_INDENT": None, } - data = yield self.exported_no_data(settings) + data = await self.exported_no_data(settings) assert expctd == data[fmt] - @inlineCallbacks - def test_export_no_items_multiple_feeds(self): + @deferred_f_from_coro_f + async def test_export_no_items_multiple_feeds(self): """Make sure that `storage.store` is called for every feed.""" settings = { "FEEDS": { @@ -1094,12 +1179,12 @@ class TestFeedExport(TestFeedExportBase): } with LogCapture() as log: - yield self.exported_no_data(settings) + await self.exported_no_data(settings) assert str(log).count("Storage.store is called") == 0 - @inlineCallbacks - def test_export_multiple_item_classes(self): + @deferred_f_from_coro_f + async def test_export_multiple_item_classes(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), self.MyItem2({"hello": "world2", "foo": "bar2"}), @@ -1117,53 +1202,53 @@ class TestFeedExport(TestFeedExportBase): {"egg": "spam4", "foo": "", "baz": ""}, ] rows_jl = [dict(row) for row in items] - yield self.assertExportedCsv(items, header, rows_csv) - yield self.assertExportedJsonLines(items, rows_jl) + await self.assertExportedCsv(items, header, rows_csv) + await self.assertExportedJsonLines(items, rows_jl) - @inlineCallbacks - def test_export_items_empty_field_list(self): + @deferred_f_from_coro_f + async def test_export_items_empty_field_list(self): # FEED_EXPORT_FIELDS==[] means the same as default None items = [{"foo": "bar"}] header = ["foo"] rows = [{"foo": "bar"}] settings = {"FEED_EXPORT_FIELDS": []} - yield self.assertExportedCsv(items, header, rows) - yield self.assertExportedJsonLines(items, rows, settings) + await self.assertExportedCsv(items, header, rows) + await self.assertExportedJsonLines(items, rows, settings) - @inlineCallbacks - def test_export_items_field_list(self): + @deferred_f_from_coro_f + async def test_export_items_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] rows = [{"foo": "bar", "baz": ""}] settings = {"FEED_EXPORT_FIELDS": header} - yield self.assertExported(items, header, rows, settings=settings) + await self.assertExported(items, header, rows, settings=settings) - @inlineCallbacks - def test_export_items_comma_separated_field_list(self): + @deferred_f_from_coro_f + async def test_export_items_comma_separated_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] rows = [{"foo": "bar", "baz": ""}] settings = {"FEED_EXPORT_FIELDS": ",".join(header)} - yield self.assertExported(items, header, rows, settings=settings) + await self.assertExported(items, header, rows, settings=settings) - @inlineCallbacks - def test_export_items_json_field_list(self): + @deferred_f_from_coro_f + async def test_export_items_json_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] rows = [{"foo": "bar", "baz": ""}] settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} - yield self.assertExported(items, header, rows, settings=settings) + await self.assertExported(items, header, rows, settings=settings) - @inlineCallbacks - def test_export_items_field_names(self): + @deferred_f_from_coro_f + async def test_export_items_field_names(self): items = [{"foo": "bar"}] header = {"foo": "Foo"} rows = [{"Foo": "bar"}] settings = {"FEED_EXPORT_FIELDS": header} - yield self.assertExported(items, list(header.values()), rows, settings=settings) + await self.assertExported(items, list(header.values()), rows, settings=settings) - @inlineCallbacks - def test_export_items_dict_field_names(self): + @deferred_f_from_coro_f + async def test_export_items_dict_field_names(self): items = [{"foo": "bar"}] header = { "baz": "Baz", @@ -1171,18 +1256,18 @@ class TestFeedExport(TestFeedExportBase): } rows = [{"Baz": "", "Foo": "bar"}] settings = {"FEED_EXPORT_FIELDS": header} - yield self.assertExported(items, ["Baz", "Foo"], rows, settings=settings) + await self.assertExported(items, ["Baz", "Foo"], rows, settings=settings) - @inlineCallbacks - def test_export_items_json_field_names(self): + @deferred_f_from_coro_f + async def test_export_items_json_field_names(self): items = [{"foo": "bar"}] header = {"foo": "Foo"} rows = [{"Foo": "bar"}] settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} - yield self.assertExported(items, list(header.values()), rows, settings=settings) + await self.assertExported(items, list(header.values()), rows, settings=settings) - @inlineCallbacks - def test_export_based_on_item_classes(self): + @deferred_f_from_coro_f + async def test_export_based_on_item_classes(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), self.MyItem2({"hello": "world2", "foo": "bar2"}), @@ -1223,12 +1308,12 @@ class TestFeedExport(TestFeedExportBase): }, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for fmt, expected in formats.items(): assert data[fmt] == expected - @inlineCallbacks - def test_export_based_on_custom_filters(self): + @deferred_f_from_coro_f + async def test_export_based_on_custom_filters(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), self.MyItem2({"hello": "world2", "foo": "bar2"}), @@ -1282,12 +1367,12 @@ class TestFeedExport(TestFeedExportBase): }, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for fmt, expected in formats.items(): assert data[fmt] == expected - @inlineCallbacks - def test_export_dicts(self): + @deferred_f_from_coro_f + async def test_export_dicts(self): # When dicts are used, only keys from the first row are used as # a header for CSV, and all fields are used for JSON Lines. items = [ @@ -1296,11 +1381,11 @@ class TestFeedExport(TestFeedExportBase): ] rows_csv = [{"egg": "spam", "foo": "bar"}, {"egg": "spam", "foo": "bar"}] rows_jl = items - yield self.assertExportedCsv(items, ["foo", "egg"], rows_csv) - yield self.assertExportedJsonLines(items, rows_jl) + await self.assertExportedCsv(items, ["foo", "egg"], rows_csv) + await self.assertExportedJsonLines(items, rows_jl) - @inlineCallbacks - def test_export_tuple(self): + @deferred_f_from_coro_f + async def test_export_tuple(self): items = [ {"foo": "bar1", "egg": "spam1"}, {"foo": "bar2", "egg": "spam2", "baz": "quux"}, @@ -1308,10 +1393,10 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": ("foo", "baz")} rows = [{"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}] - yield self.assertExported(items, ["foo", "baz"], rows, settings=settings) + await self.assertExported(items, ["foo", "baz"], rows, settings=settings) - @inlineCallbacks - def test_export_feed_export_fields(self): + @deferred_f_from_coro_f + async def test_export_feed_export_fields(self): # FEED_EXPORT_FIELDS option allows to order export fields # and to select a subset of fields to export, both for Items and dicts. @@ -1327,17 +1412,17 @@ class TestFeedExport(TestFeedExportBase): {"egg": "spam1", "foo": "bar1", "baz": ""}, {"egg": "spam2", "foo": "bar2", "baz": "quux2"}, ] - yield self.assertExported( + await self.assertExported( items, ["foo", "baz", "egg"], rows, settings=settings ) # export a subset of columns settings = {"FEED_EXPORT_FIELDS": "egg,baz"} rows = [{"egg": "spam1", "baz": ""}, {"egg": "spam2", "baz": "quux2"}] - yield self.assertExported(items, ["egg", "baz"], rows, settings=settings) + await self.assertExported(items, ["egg", "baz"], rows, settings=settings) - @inlineCallbacks - def test_export_encoding(self): + @deferred_f_from_coro_f + async def test_export_encoding(self): items = [{"foo": "Test\xd6"}] formats = { @@ -1357,7 +1442,7 @@ class TestFeedExport(TestFeedExportBase): }, "FEED_EXPORT_INDENT": None, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) assert data[fmt] == expected formats = { @@ -1378,11 +1463,11 @@ class TestFeedExport(TestFeedExportBase): "FEED_EXPORT_INDENT": None, "FEED_EXPORT_ENCODING": "latin-1", } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) assert data[fmt] == expected - @inlineCallbacks - def test_export_multiple_configs(self): + @deferred_f_from_coro_f + async def test_export_multiple_configs(self): items = [{"foo": "FOO", "bar": "BAR"}] formats = { @@ -1417,12 +1502,12 @@ class TestFeedExport(TestFeedExportBase): }, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for fmt, expected in formats.items(): assert data[fmt] == expected - @inlineCallbacks - def test_export_indentation(self): + @deferred_f_from_coro_f + async def test_export_indentation(self): items = [ {"foo": ["bar"]}, {"key": "value"}, @@ -1574,11 +1659,11 @@ class TestFeedExport(TestFeedExportBase): }, }, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) assert data[row["format"]] == row["expected"] - @inlineCallbacks - def test_init_exporters_storages_with_crawler(self): + @deferred_f_from_coro_f + async def test_init_exporters_storages_with_crawler(self): settings = { "FEED_EXPORTERS": {"csv": FromCrawlerCsvItemExporter}, "FEED_STORAGES": {"file": FromCrawlerFileFeedStorage}, @@ -1586,21 +1671,21 @@ class TestFeedExport(TestFeedExportBase): self._random_temp_filename(): {"format": "csv"}, }, } - yield self.exported_data(items=[], settings=settings) + await self.exported_data(items=[], settings=settings) assert FromCrawlerCsvItemExporter.init_with_crawler assert FromCrawlerFileFeedStorage.init_with_crawler - @inlineCallbacks - def test_str_uri(self): + @deferred_f_from_coro_f + async def test_str_uri(self): settings = { "FEED_STORE_EMPTY": True, "FEEDS": {str(self._random_temp_filename()): {"format": "csv"}}, } - data = yield self.exported_no_data(settings) + data = await self.exported_no_data(settings) assert data["csv"] == b"" - @inlineCallbacks - def test_multiple_feeds_success_logs_blocking_feed_storage(self): + @deferred_f_from_coro_f + async def test_multiple_feeds_success_logs_blocking_feed_storage(self): settings = { "FEEDS": { self._random_temp_filename(): {"format": "json"}, @@ -1614,14 +1699,14 @@ class TestFeedExport(TestFeedExportBase): {"foo": "bar2", "baz": "quux"}, ] with LogCapture() as log: - yield self.exported_data(items, settings) + await self.exported_data(items, settings) print(log) for fmt in ["json", "xml", "csv"]: assert f"Stored {fmt} feed (2 items)" in str(log) - @inlineCallbacks - def test_multiple_feeds_failing_logs_blocking_feed_storage(self): + @deferred_f_from_coro_f + async def test_multiple_feeds_failing_logs_blocking_feed_storage(self): settings = { "FEEDS": { self._random_temp_filename(): {"format": "json"}, @@ -1635,14 +1720,14 @@ class TestFeedExport(TestFeedExportBase): {"foo": "bar2", "baz": "quux"}, ] with LogCapture() as log: - yield self.exported_data(items, settings) + await self.exported_data(items, settings) print(log) for fmt in ["json", "xml", "csv"]: assert f"Error storing {fmt} feed (2 items)" in str(log) - @inlineCallbacks - def test_extend_kwargs(self): + @deferred_f_from_coro_f + async def test_extend_kwargs(self): items = [{"foo": "FOO", "bar": "BAR"}] expected_with_title_csv = b"foo,bar\r\nFOO,BAR\r\n" @@ -1675,11 +1760,11 @@ class TestFeedExport(TestFeedExportBase): "FEED_EXPORT_INDENT": None, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) assert data[feed_options["format"]] == row["expected"] - @inlineCallbacks - def test_storage_file_no_postprocessing(self): + @deferred_f_from_coro_f + async def test_storage_file_no_postprocessing(self): @implementer(IFeedStorage) class Storage: def __init__(self, uri, *, feed_options=None): @@ -1697,11 +1782,11 @@ class TestFeedExport(TestFeedExportBase): "FEEDS": {self._random_temp_filename(): {"format": "jsonlines"}}, "FEED_STORAGES": {"file": Storage}, } - yield self.exported_no_data(settings) + await self.exported_no_data(settings) assert Storage.open_file is Storage.store_file - @inlineCallbacks - def test_storage_file_postprocessing(self): + @deferred_f_from_coro_f + async def test_storage_file_postprocessing(self): @implementer(IFeedStorage) class Storage: def __init__(self, uri, *, feed_options=None): @@ -1727,7 +1812,7 @@ class TestFeedExport(TestFeedExportBase): }, "FEED_STORAGES": {"file": Storage}, } - yield self.exported_no_data(settings) + await self.exported_no_data(settings) assert Storage.open_file is Storage.store_file assert not Storage.file_was_closed @@ -1753,8 +1838,9 @@ class TestFeedPostProcessedExports(TestFeedExportBase): def _named_tempfile(self, name) -> str: return str(Path(self.temp_dir, name)) - @inlineCallbacks - def run_and_export(self, spider_cls, settings): + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, bytes | None]: """Run spider with specified settings; return exported data with filename.""" FEEDS = settings.get("FEEDS") or {} @@ -1763,11 +1849,11 @@ class TestFeedPostProcessedExports(TestFeedExportBase): for file_path, feed_options in FEEDS.items() } - content = {} + content: dict[str, bytes | None] = {} try: spider_cls.start_urls = [self.mockserver.url("/")] crawler = get_crawler(spider_cls, settings) - yield crawler.crawl() + await maybe_deferred_to_future(crawler.crawl()) for file_path in FEEDS: content[str(file_path)] = ( @@ -1797,8 +1883,8 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data_stream.seek(0) return data_stream.read() - @inlineCallbacks - def test_gzip_plugin(self): + @deferred_f_from_coro_f + async def test_gzip_plugin(self): filename = self._named_tempfile("gzip_file") settings = { @@ -1810,14 +1896,14 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) try: gzip.decompress(data[filename]) except OSError: pytest.fail("Received invalid gzip data.") - @inlineCallbacks - def test_gzip_plugin_compresslevel(self): + @deferred_f_from_coro_f + async def test_gzip_plugin_compresslevel(self): filename_to_compressed = { self._named_tempfile("compresslevel_0"): self.get_gzip_compressed( self.expected, compresslevel=0 @@ -1846,15 +1932,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, compressed in filename_to_compressed.items(): result = gzip.decompress(data[filename]) assert compressed == data[filename] assert result == self.expected - @inlineCallbacks - def test_gzip_plugin_mtime(self): + @deferred_f_from_coro_f + async def test_gzip_plugin_mtime(self): filename_to_compressed = { self._named_tempfile("mtime_123"): self.get_gzip_compressed( self.expected, mtime=123 @@ -1881,15 +1967,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, compressed in filename_to_compressed.items(): result = gzip.decompress(data[filename]) assert compressed == data[filename] assert result == self.expected - @inlineCallbacks - def test_gzip_plugin_filename(self): + @deferred_f_from_coro_f + async def test_gzip_plugin_filename(self): filename_to_compressed = { self._named_tempfile("filename_FILE1"): self.get_gzip_compressed( self.expected, filename="FILE1" @@ -1916,15 +2002,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, compressed in filename_to_compressed.items(): result = gzip.decompress(data[filename]) assert compressed == data[filename] assert result == self.expected - @inlineCallbacks - def test_lzma_plugin(self): + @deferred_f_from_coro_f + async def test_lzma_plugin(self): filename = self._named_tempfile("lzma_file") settings = { @@ -1936,14 +2022,14 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) try: lzma.decompress(data[filename]) except lzma.LZMAError: pytest.fail("Received invalid lzma data.") - @inlineCallbacks - def test_lzma_plugin_format(self): + @deferred_f_from_coro_f + async def test_lzma_plugin_format(self): filename_to_compressed = { self._named_tempfile("format_FORMAT_XZ"): lzma.compress( self.expected, format=lzma.FORMAT_XZ @@ -1968,15 +2054,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, compressed in filename_to_compressed.items(): result = lzma.decompress(data[filename]) assert compressed == data[filename] assert result == self.expected - @inlineCallbacks - def test_lzma_plugin_check(self): + @deferred_f_from_coro_f + async def test_lzma_plugin_check(self): filename_to_compressed = { self._named_tempfile("check_CHECK_NONE"): lzma.compress( self.expected, check=lzma.CHECK_NONE @@ -2001,15 +2087,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, compressed in filename_to_compressed.items(): result = lzma.decompress(data[filename]) assert compressed == data[filename] assert result == self.expected - @inlineCallbacks - def test_lzma_plugin_preset(self): + @deferred_f_from_coro_f + async def test_lzma_plugin_preset(self): filename_to_compressed = { self._named_tempfile("preset_PRESET_0"): lzma.compress( self.expected, preset=0 @@ -2034,15 +2120,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, compressed in filename_to_compressed.items(): result = lzma.decompress(data[filename]) assert compressed == data[filename] assert result == self.expected - @inlineCallbacks - def test_lzma_plugin_filters(self): + @deferred_f_from_coro_f + async def test_lzma_plugin_filters(self): if "PyPy" in sys.version: # https://foss.heptapod.net/pypy/pypy/-/issues/3527 pytest.skip("lzma filters doesn't work in PyPy") @@ -2061,13 +2147,13 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) assert compressed == data[filename] result = lzma.decompress(data[filename]) assert result == self.expected - @inlineCallbacks - def test_bz2_plugin(self): + @deferred_f_from_coro_f + async def test_bz2_plugin(self): filename = self._named_tempfile("bz2_file") settings = { @@ -2079,14 +2165,14 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) try: bz2.decompress(data[filename]) except OSError: pytest.fail("Received invalid bz2 data.") - @inlineCallbacks - def test_bz2_plugin_compresslevel(self): + @deferred_f_from_coro_f + async def test_bz2_plugin_compresslevel(self): filename_to_compressed = { self._named_tempfile("compresslevel_1"): bz2.compress( self.expected, compresslevel=1 @@ -2111,15 +2197,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, compressed in filename_to_compressed.items(): result = bz2.decompress(data[filename]) assert compressed == data[filename] assert result == self.expected - @inlineCallbacks - def test_custom_plugin(self): + @deferred_f_from_coro_f + async def test_custom_plugin(self): filename = self._named_tempfile("csv_file") settings = { @@ -2131,11 +2217,11 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) assert data[filename] == self.expected - @inlineCallbacks - def test_custom_plugin_with_parameter(self): + @deferred_f_from_coro_f + async def test_custom_plugin_with_parameter(self): expected = b"foo\r\n\nbar\r\n\n" filename = self._named_tempfile("newline") @@ -2149,11 +2235,11 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) assert data[filename] == expected - @inlineCallbacks - def test_custom_plugin_with_compression(self): + @deferred_f_from_coro_f + async def test_custom_plugin_with_compression(self): expected = b"foo\r\n\nbar\r\n\n" filename_to_decompressor = { @@ -2191,14 +2277,14 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, decompressor in filename_to_decompressor.items(): result = decompressor(data[filename]) assert result == expected - @inlineCallbacks - def test_exports_compatibility_with_postproc(self): + @deferred_f_from_coro_f + async def test_exports_compatibility_with_postproc(self): import marshal import pickle @@ -2240,7 +2326,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, result in data.items(): if "pickle" in filename: @@ -2255,18 +2341,19 @@ class TestFeedPostProcessedExports(TestFeedExportBase): class TestBatchDeliveries(TestFeedExportBase): _file_mark = "_%(batch_time)s_#%(batch_id)02d_" - @inlineCallbacks - def run_and_export(self, spider_cls, settings): + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, list[bytes]]: """Run spider with specified settings; return exported data.""" FEEDS = settings.get("FEEDS") or {} settings["FEEDS"] = { build_url(file_path): feed for file_path, feed in FEEDS.items() } - content = defaultdict(list) + content: defaultdict[str, list[bytes]] = defaultdict(list) spider_cls.start_urls = [self.mockserver.url("/")] crawler = get_crawler(spider_cls, settings) - yield crawler.crawl() + await maybe_deferred_to_future(crawler.crawl()) for path, feed in FEEDS.items(): dir_name = Path(path).parent @@ -2277,8 +2364,7 @@ class TestBatchDeliveries(TestFeedExportBase): content[feed["format"]].append(file.read_bytes()) return content - @inlineCallbacks - def assertExportedJsonLines(self, items, rows, settings=None): + async def assertExportedJsonLines(self, items, rows, settings=None): settings = settings or {} settings.update( { @@ -2291,7 +2377,7 @@ class TestBatchDeliveries(TestFeedExportBase): ) batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") rows = [{k: v for k, v in row.items() if v} for row in rows] - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for batch in data["jl"]: got_batch = [ json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines() @@ -2299,8 +2385,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None): + async def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update( { @@ -2312,15 +2397,14 @@ class TestBatchDeliveries(TestFeedExportBase): } ) batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for batch in data["csv"]: got_batch = csv.DictReader(to_unicode(batch).splitlines()) assert list(header) == got_batch.fieldnames expected_batch, rows = rows[:batch_size], rows[batch_size:] assert list(got_batch) == expected_batch - @inlineCallbacks - def assertExportedXml(self, items, rows, settings=None): + async def assertExportedXml(self, items, rows, settings=None): settings = settings or {} settings.update( { @@ -2333,15 +2417,14 @@ class TestBatchDeliveries(TestFeedExportBase): ) batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") rows = [{k: v for k, v in row.items() if v} for row in rows] - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for batch in data["xml"]: root = lxml.etree.fromstring(batch) got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @inlineCallbacks - def assertExportedMultiple(self, items, rows, settings=None): + async def assertExportedMultiple(self, items, rows, settings=None): settings = settings or {} settings.update( { @@ -2357,7 +2440,7 @@ class TestBatchDeliveries(TestFeedExportBase): ) batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") rows = [{k: v for k, v in row.items() if v} for row in rows] - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) # XML xml_rows = rows.copy() for batch in data["xml"]: @@ -2372,8 +2455,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] assert got_batch == expected_batch - @inlineCallbacks - def assertExportedPickle(self, items, rows, settings=None): + async def assertExportedPickle(self, items, rows, settings=None): settings = settings or {} settings.update( { @@ -2386,7 +2468,7 @@ class TestBatchDeliveries(TestFeedExportBase): ) batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") rows = [{k: v for k, v in row.items() if v} for row in rows] - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) import pickle for batch in data["pickle"]: @@ -2394,8 +2476,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @inlineCallbacks - def assertExportedMarshal(self, items, rows, settings=None): + async def assertExportedMarshal(self, items, rows, settings=None): settings = settings or {} settings.update( { @@ -2408,7 +2489,7 @@ class TestBatchDeliveries(TestFeedExportBase): ) batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") rows = [{k: v for k, v in row.items() if v} for row in rows] - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) import marshal for batch in data["marshal"]: @@ -2416,8 +2497,8 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @inlineCallbacks - def test_export_items(self): + @deferred_f_from_coro_f + async def test_export_items(self): """Test partial deliveries in all supported formats""" items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -2431,7 +2512,7 @@ class TestBatchDeliveries(TestFeedExportBase): ] settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2} header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows, settings=settings) + await self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): """If path is without %(batch_time)s and %(batch_id) an exception must be raised""" @@ -2445,8 +2526,8 @@ class TestBatchDeliveries(TestFeedExportBase): with pytest.raises(NotConfigured): FeedExporter(crawler) - @inlineCallbacks - def test_export_no_items_not_store_empty(self): + @deferred_f_from_coro_f + async def test_export_no_items_not_store_empty(self): for fmt in ("json", "jsonlines", "xml", "csv"): settings = { "FEEDS": { @@ -2457,12 +2538,12 @@ class TestBatchDeliveries(TestFeedExportBase): "FEED_EXPORT_BATCH_ITEM_COUNT": 1, "FEED_STORE_EMPTY": False, } - data = yield self.exported_no_data(settings) + data = await self.exported_no_data(settings) data = dict(data) assert len(data[fmt]) == 0 - @inlineCallbacks - def test_export_no_items_store_empty(self): + @deferred_f_from_coro_f + async def test_export_no_items_store_empty(self): formats = ( ("json", b"[]"), ("jsonlines", b""), @@ -2481,12 +2562,12 @@ class TestBatchDeliveries(TestFeedExportBase): "FEED_EXPORT_INDENT": None, "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } - data = yield self.exported_no_data(settings) + data = await self.exported_no_data(settings) data = dict(data) assert data[fmt][0] == expctd - @inlineCallbacks - def test_export_multiple_configs(self): + @deferred_f_from_coro_f + async def test_export_multiple_configs(self): items = [ {"foo": "FOO", "bar": "BAR"}, {"foo": "FOO1", "bar": "BAR1"}, @@ -2536,13 +2617,13 @@ class TestBatchDeliveries(TestFeedExportBase): }, "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for fmt, expected in formats.items(): for expected_batch, got_batch in zip(expected, data[fmt]): assert got_batch == expected_batch - @inlineCallbacks - def test_batch_item_count_feeds_setting(self): + @deferred_f_from_coro_f + async def test_batch_item_count_feeds_setting(self): items = [{"foo": "FOO"}, {"foo": "FOO1"}] formats = { "json": [ @@ -2560,13 +2641,13 @@ class TestBatchDeliveries(TestFeedExportBase): }, }, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for fmt, expected in formats.items(): for expected_batch, got_batch in zip(expected, data[fmt]): assert got_batch == expected_batch - @inlineCallbacks - def test_batch_path_differ(self): + @deferred_f_from_coro_f + async def test_batch_path_differ(self): """ Test that the name of all batch files differ from each other. So %(batch_id)d replaced with the current id. @@ -2584,7 +2665,7 @@ class TestBatchDeliveries(TestFeedExportBase): }, "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) assert len(items) == len(data["json"]) @inlineCallbacks diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 6ebaa19ce..28ffbe767 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -8,7 +8,6 @@ from unittest import mock import pytest from testfixtures import LogCapture from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks from twisted.trial.unittest import TestCase from scrapy.core.spidermw import SpiderMiddlewareManager @@ -18,7 +17,6 @@ from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.defer import ( deferred_f_from_coro_f, - deferred_from_coro, maybe_deferred_to_future, ) from scrapy.utils.test import get_crawler @@ -130,25 +128,22 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): yield {"foo": 2} yield {"foo": 3} - @inlineCallbacks - def _get_middleware_result(self, *mw_classes, start_index: int | None = None): + async def _get_middleware_result(self, *mw_classes, start_index: int | None = None): setting = self._construct_mw_setting(*mw_classes, start_index=start_index) self.crawler = get_crawler( Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} ) self.spider = self.crawler._create_spider("foo") self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - result = yield self.mwman.scrape_response( + return await self.mwman.scrape_response_async( self._scrape_func, self.response, self.request, self.spider ) - return result - @inlineCallbacks - def _test_simple_base( + async def _test_simple_base( self, *mw_classes, downgrade: bool = False, start_index: int | None = None ): with LogCapture() as log: - result = yield self._get_middleware_result( + result = await self._get_middleware_result( *mw_classes, start_index=start_index ) assert isinstance(result, Iterable) @@ -160,16 +155,15 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): ProcessSpiderOutputSimpleMiddleware in mw_classes ) - @inlineCallbacks - def _test_asyncgen_base( + async def _test_asyncgen_base( self, *mw_classes, downgrade: bool = False, start_index: int | None = None ): with LogCapture() as log: - result = yield self._get_middleware_result( + result = await self._get_middleware_result( *mw_classes, start_index=start_index ) assert isinstance(result, AsyncIterator) - result_list = yield deferred_from_coro(collect_asyncgen(result)) + result_list = await collect_asyncgen(result) assert len(result_list) == self.RESULT_COUNT assert isinstance(result_list[0], self.ITEM_TYPE) assert ("downgraded to a non-async" in str(log)) == downgrade @@ -222,41 +216,50 @@ class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware): MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware - def test_simple(self): + @deferred_f_from_coro_f + async def test_simple(self): """Simple mw""" - return self._test_simple_base(self.MW_SIMPLE) + await self._test_simple_base(self.MW_SIMPLE) - def test_asyncgen(self): + @deferred_f_from_coro_f + async def test_asyncgen(self): """Asyncgen mw; upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN) + await self._test_asyncgen_base(self.MW_ASYNCGEN) - def test_simple_asyncgen(self): + @deferred_f_from_coro_f + async def test_simple_asyncgen(self): """Simple mw -> asyncgen mw; upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE) + await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE) - def test_asyncgen_simple(self): + @deferred_f_from_coro_f + async def test_asyncgen_simple(self): """Asyncgen mw -> simple mw; upgrade then downgrade""" - return self._test_simple_base(self.MW_SIMPLE, self.MW_ASYNCGEN, downgrade=True) + await self._test_simple_base(self.MW_SIMPLE, self.MW_ASYNCGEN, downgrade=True) - def test_universal(self): + @deferred_f_from_coro_f + async def test_universal(self): """Universal mw""" - return self._test_simple_base(self.MW_UNIVERSAL) + await self._test_simple_base(self.MW_UNIVERSAL) - def test_universal_simple(self): + @deferred_f_from_coro_f + async def test_universal_simple(self): """Universal mw -> simple mw""" - return self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL) + await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL) - def test_simple_universal(self): + @deferred_f_from_coro_f + async def test_simple_universal(self): """Simple mw -> universal mw""" - return self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE) + await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE) - def test_universal_asyncgen(self): + @deferred_f_from_coro_f + async def test_universal_asyncgen(self): """Universal mw -> asyncgen mw; upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL) + await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL) - def test_asyncgen_universal(self): + @deferred_f_from_coro_f + async def test_asyncgen_universal(self): """Asyncgen mw -> universal mw; upgrade""" - return self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN) + await self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN) class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple): @@ -266,27 +269,30 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple): for item in super()._scrape_func(): yield item - def test_simple(self): + @deferred_f_from_coro_f + async def test_simple(self): """Simple mw; downgrade""" - return self._test_simple_base(self.MW_SIMPLE, downgrade=True) + await self._test_simple_base(self.MW_SIMPLE, downgrade=True) - def test_simple_asyncgen(self): + @deferred_f_from_coro_f + async def test_simple_asyncgen(self): """Simple mw -> asyncgen mw; downgrade then upgrade""" - return self._test_asyncgen_base( - self.MW_ASYNCGEN, self.MW_SIMPLE, downgrade=True - ) + await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE, downgrade=True) - def test_universal(self): + @deferred_f_from_coro_f + async def test_universal(self): """Universal mw""" - return self._test_asyncgen_base(self.MW_UNIVERSAL) + await self._test_asyncgen_base(self.MW_UNIVERSAL) - def test_universal_simple(self): + @deferred_f_from_coro_f + async def test_universal_simple(self): """Universal mw -> simple mw; downgrade""" - return self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL, downgrade=True) + await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL, downgrade=True) - def test_simple_universal(self): + @deferred_f_from_coro_f + async def test_simple_universal(self): """Simple mw -> universal mw; downgrade""" - return self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE, downgrade=True) + await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE, downgrade=True) class ProcessSpiderOutputNonIterableMiddleware: @@ -300,25 +306,21 @@ class ProcessSpiderOutputCoroutineMiddleware: class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): - @inlineCallbacks - def test_non_iterable(self): + @deferred_f_from_coro_f + async def test_non_iterable(self): with pytest.raises( _InvalidOutput, match=r"\.process_spider_output must return an iterable, got ", ): - yield self._get_middleware_result( - ProcessSpiderOutputNonIterableMiddleware, - ) + await self._get_middleware_result(ProcessSpiderOutputNonIterableMiddleware) - @inlineCallbacks - def test_coroutine(self): + @deferred_f_from_coro_f + async def test_coroutine(self): with pytest.raises( _InvalidOutput, match=r"\.process_spider_output must be an asynchronous generator", ): - yield self._get_middleware_result( - ProcessSpiderOutputCoroutineMiddleware, - ) + await self._get_middleware_result(ProcessSpiderOutputCoroutineMiddleware) class ProcessStartSimpleMiddleware: @@ -445,39 +447,44 @@ class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware): MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware - @inlineCallbacks - def _get_middleware_result(self, *mw_classes, start_index: int | None = None): + async def _get_middleware_result(self, *mw_classes, start_index: int | None = None): setting = self._construct_mw_setting(*mw_classes, start_index=start_index) self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES": setting}) self.spider = self.crawler._create_spider("foo") self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - result = yield self.mwman.scrape_response( + return await self.mwman.scrape_response_async( self._scrape_func, self.response, self.request, self.spider ) - return result - def test_just_builtin(self): - return self._test_simple_base() + @deferred_f_from_coro_f + async def test_just_builtin(self): + await self._test_simple_base() - def test_builtin_simple(self): - return self._test_simple_base(self.MW_SIMPLE, start_index=1000) + @deferred_f_from_coro_f + async def test_builtin_simple(self): + await self._test_simple_base(self.MW_SIMPLE, start_index=1000) - def test_builtin_async(self): + @deferred_f_from_coro_f + async def test_builtin_async(self): """Upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) + await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) - def test_builtin_universal(self): - return self._test_simple_base(self.MW_UNIVERSAL, start_index=1000) + @deferred_f_from_coro_f + async def test_builtin_universal(self): + await self._test_simple_base(self.MW_UNIVERSAL, start_index=1000) - def test_simple_builtin(self): - return self._test_simple_base(self.MW_SIMPLE) + @deferred_f_from_coro_f + async def test_simple_builtin(self): + await self._test_simple_base(self.MW_SIMPLE) - def test_async_builtin(self): + @deferred_f_from_coro_f + async def test_async_builtin(self): """Upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN) + await self._test_asyncgen_base(self.MW_ASYNCGEN) - def test_universal_builtin(self): - return self._test_simple_base(self.MW_UNIVERSAL) + @deferred_f_from_coro_f + async def test_universal_builtin(self): + await self._test_simple_base(self.MW_UNIVERSAL) class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple): @@ -485,28 +492,35 @@ class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple): for item in super()._scrape_func(): yield item - def test_just_builtin(self): - return self._test_asyncgen_base() + @deferred_f_from_coro_f + async def test_just_builtin(self): + await self._test_asyncgen_base() - def test_builtin_simple(self): + @deferred_f_from_coro_f + async def test_builtin_simple(self): """Downgrade""" - return self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000) + await self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000) - def test_builtin_async(self): - return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) + @deferred_f_from_coro_f + async def test_builtin_async(self): + await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) - def test_builtin_universal(self): - return self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000) + @deferred_f_from_coro_f + async def test_builtin_universal(self): + await self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000) - def test_simple_builtin(self): + @deferred_f_from_coro_f + async def test_simple_builtin(self): """Downgrade""" - return self._test_simple_base(self.MW_SIMPLE, downgrade=True) + await self._test_simple_base(self.MW_SIMPLE, downgrade=True) - def test_async_builtin(self): - return self._test_asyncgen_base(self.MW_ASYNCGEN) + @deferred_f_from_coro_f + async def test_async_builtin(self): + await self._test_asyncgen_base(self.MW_ASYNCGEN) - def test_universal_builtin(self): - return self._test_asyncgen_base(self.MW_UNIVERSAL) + @deferred_f_from_coro_f + async def test_universal_builtin(self): + await self._test_asyncgen_base(self.MW_UNIVERSAL) class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): @@ -520,33 +534,38 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): def _scrape_func(self, *args, **kwargs): 1 / 0 - @inlineCallbacks - def _test_asyncgen_nodowngrade(self, *mw_classes): + async def _test_asyncgen_nodowngrade(self, *mw_classes): with pytest.raises( _InvalidOutput, match="Async iterable returned from .+ cannot be downgraded" ): - yield self._get_middleware_result(*mw_classes) + await self._get_middleware_result(*mw_classes) - def test_exc_simple(self): + @deferred_f_from_coro_f + async def test_exc_simple(self): """Simple exc mw""" - return self._test_simple_base(self.MW_EXC_SIMPLE) + await self._test_simple_base(self.MW_EXC_SIMPLE) - def test_exc_async(self): + @deferred_f_from_coro_f + async def test_exc_async(self): """Async exc mw""" - return self._test_asyncgen_base(self.MW_EXC_ASYNCGEN) + await self._test_asyncgen_base(self.MW_EXC_ASYNCGEN) - def test_exc_simple_simple(self): + @deferred_f_from_coro_f + async def test_exc_simple_simple(self): """Simple exc mw -> simple output mw""" - return self._test_simple_base(self.MW_SIMPLE, self.MW_EXC_SIMPLE) + await self._test_simple_base(self.MW_SIMPLE, self.MW_EXC_SIMPLE) - def test_exc_async_async(self): + @deferred_f_from_coro_f + async def test_exc_async_async(self): """Async exc mw -> async output mw""" - return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_ASYNCGEN) + await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_ASYNCGEN) - def test_exc_simple_async(self): + @deferred_f_from_coro_f + async def test_exc_simple_async(self): """Simple exc mw -> async output mw; upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE) + await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE) - def test_exc_async_simple(self): + @deferred_f_from_coro_f + async def test_exc_async_simple(self): """Async exc mw -> simple output mw; cannot work as downgrading is not supported""" - return self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN) + await self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 62ec1a624..60464d696 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -1,8 +1,8 @@ from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks from twisted.trial.unittest import TestCase from scrapy import Request, Spider +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler from tests.mockserver import MockServer @@ -299,6 +299,8 @@ class NotGeneratorOutputChainSpider(Spider): # ================================================================================ class TestSpiderMiddleware(TestCase): + mockserver: MockServer + @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -308,53 +310,52 @@ class TestSpiderMiddleware(TestCase): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - @inlineCallbacks - def crawl_log(self, spider): + async def crawl_log(self, spider: type[Spider]) -> LogCapture: crawler = get_crawler(spider) with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + await maybe_deferred_to_future(crawler.crawl(mockserver=self.mockserver)) return log - @inlineCallbacks - def test_recovery(self): + @deferred_f_from_coro_f + async def test_recovery(self): """ (0) Recover from an exception in a spider callback. The final item count should be 3 (one yielded from the callback method before the exception is raised, one directly from the recovery middleware and one from the spider when processing the request that was enqueued from the recovery middleware) """ - log = yield self.crawl_log(RecoverySpider) + log = await self.crawl_log(RecoverySpider) assert "Middleware: TabError exception caught" in str(log) assert str(log).count("Middleware: TabError exception caught") == 1 assert "'item_scraped_count': 3" in str(log) - @inlineCallbacks - def test_recovery_asyncgen(self): + @deferred_f_from_coro_f + async def test_recovery_asyncgen(self): """ Same as test_recovery but with an async callback. """ - log = yield self.crawl_log(RecoveryAsyncGenSpider) + log = await self.crawl_log(RecoveryAsyncGenSpider) assert "Middleware: TabError exception caught" in str(log) assert str(log).count("Middleware: TabError exception caught") == 1 assert "'item_scraped_count': 3" in str(log) - @inlineCallbacks - def test_process_spider_input_without_errback(self): + @deferred_f_from_coro_f + async def test_process_spider_input_without_errback(self): """ (1.1) An exception from the process_spider_input chain should be caught by the process_spider_exception chain from the start if the Request has no errback """ - log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithoutErrback) + log1 = await self.crawl_log(ProcessSpiderInputSpiderWithoutErrback) assert "Middleware: will raise IndexError" in str(log1) assert "Middleware: IndexError exception caught" in str(log1) - @inlineCallbacks - def test_process_spider_input_with_errback(self): + @deferred_f_from_coro_f + async def test_process_spider_input_with_errback(self): """ (1.2) An exception from the process_spider_input chain should not be caught by the process_spider_exception chain if the Request has an errback """ - log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithErrback) + log1 = await self.crawl_log(ProcessSpiderInputSpiderWithErrback) assert "Middleware: IndexError exception caught" not in str(log1) assert "Middleware: will raise IndexError" in str(log1) assert "Got a Failure on the Request errback" in str(log1) @@ -362,60 +363,60 @@ class TestSpiderMiddleware(TestCase): assert "{'from': 'callback'}" not in str(log1) assert "'item_scraped_count': 1" in str(log1) - @inlineCallbacks - def test_generator_callback(self): + @deferred_f_from_coro_f + async def test_generator_callback(self): """ (2) An exception from a spider callback (returning a generator) should be caught by the process_spider_exception chain. Items yielded before the exception is raised should be processed normally. """ - log2 = yield self.crawl_log(GeneratorCallbackSpider) + log2 = await self.crawl_log(GeneratorCallbackSpider) assert "Middleware: ImportError exception caught" in str(log2) assert "'item_scraped_count': 2" in str(log2) - @inlineCallbacks - def test_async_generator_callback(self): + @deferred_f_from_coro_f + async def test_async_generator_callback(self): """ Same as test_generator_callback but with an async callback. """ - log2 = yield self.crawl_log(AsyncGeneratorCallbackSpider) + log2 = await self.crawl_log(AsyncGeneratorCallbackSpider) assert "Middleware: ImportError exception caught" in str(log2) assert "'item_scraped_count': 2" in str(log2) - @inlineCallbacks - def test_generator_callback_right_after_callback(self): + @deferred_f_from_coro_f + async def test_generator_callback_right_after_callback(self): """ (2.1) Special case of (2): Exceptions should be caught even if the middleware is placed right after the spider """ - log21 = yield self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider) + log21 = await self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider) assert "Middleware: ImportError exception caught" in str(log21) assert "'item_scraped_count': 2" in str(log21) - @inlineCallbacks - def test_not_a_generator_callback(self): + @deferred_f_from_coro_f + async def test_not_a_generator_callback(self): """ (3) An exception from a spider callback (returning a list) should be caught by the process_spider_exception chain. No items should be processed. """ - log3 = yield self.crawl_log(NotGeneratorCallbackSpider) + log3 = await self.crawl_log(NotGeneratorCallbackSpider) assert "Middleware: ZeroDivisionError exception caught" in str(log3) assert "item_scraped_count" not in str(log3) - @inlineCallbacks - def test_not_a_generator_callback_right_after_callback(self): + @deferred_f_from_coro_f + async def test_not_a_generator_callback_right_after_callback(self): """ (3.1) Special case of (3): Exceptions should be caught even if the middleware is placed right after the spider """ - log31 = yield self.crawl_log( + log31 = await self.crawl_log( NotGeneratorCallbackSpiderMiddlewareRightAfterSpider ) assert "Middleware: ZeroDivisionError exception caught" in str(log31) assert "item_scraped_count" not in str(log31) - @inlineCallbacks - def test_generator_output_chain(self): + @deferred_f_from_coro_f + async def test_generator_output_chain(self): """ (4) An exception from a middleware's process_spider_output method should be sent to the process_spider_exception method from the next middleware in the chain. @@ -424,7 +425,7 @@ class TestSpiderMiddleware(TestCase): The final item count should be 2 (one from the spider callback and one from the process_spider_exception chain) """ - log4 = yield self.crawl_log(GeneratorOutputChainSpider) + log4 = await self.crawl_log(GeneratorOutputChainSpider) assert "'item_scraped_count': 2" in str(log4) assert ( "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught" @@ -461,8 +462,8 @@ class TestSpiderMiddleware(TestCase): assert str(item_recovered) in str(log4) assert "parse-second-item" not in str(log4) - @inlineCallbacks - def test_not_a_generator_output_chain(self): + @deferred_f_from_coro_f + async def test_not_a_generator_output_chain(self): """ (5) An exception from a middleware's process_spider_output method should be sent to the process_spider_exception method from the next middleware in the chain. @@ -471,7 +472,7 @@ class TestSpiderMiddleware(TestCase): The final item count should be 1 (from the process_spider_exception chain, the items from the spider callback are lost) """ - log5 = yield self.crawl_log(NotGeneratorOutputChainSpider) + log5 = await self.crawl_log(NotGeneratorOutputChainSpider) assert "'item_scraped_count': 1" in str(log5) assert ( "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught" From b4d11b8b2565b5686c9f395ca1a8dd085609aafc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 11 Jun 2025 04:28:09 +0500 Subject: [PATCH 167/491] Further reduce deps on unittest. (#6884) --- .../test_downloader_handler_twisted_http11.py | 10 +- .../test_downloader_handler_twisted_http2.py | 14 +- tests/test_downloader_handlers.py | 7 +- tests/test_downloadermiddleware_robotstxt.py | 4 +- tests/test_downloaderslotssettings.py | 2 +- tests/test_engine_loop.py | 4 +- tests/test_extension_telnet.py | 2 +- tests/test_feedexport.py | 43 +- tests/test_http2_client_protocol.py | 480 ++++++++---------- tests/test_pipeline_crawl.py | 15 +- tests/test_pipeline_files.py | 16 +- tests/test_pipeline_images.py | 17 +- tests/test_pipeline_media.py | 2 +- tests/test_scheduler_base.py | 6 +- tests/test_signals.py | 4 +- tests/test_spider_start.py | 2 +- tests/test_spidermiddleware_process_start.py | 4 +- tests/test_utils_defer.py | 6 +- tests/test_utils_signal.py | 16 +- tests/test_webclient.py | 89 ++-- 20 files changed, 355 insertions(+), 388 deletions(-) diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py index 70f55e787..7b26ce03f 100644 --- a/tests/test_downloader_handler_twisted_http11.py +++ b/tests/test_downloader_handler_twisted_http11.py @@ -39,23 +39,21 @@ class TestSimpleHttps(HTTP11DownloadHandlerMixin, TestSimpleHttpsBase): pass -class Https11WrongHostnameTestCase( - HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase -): +class TestHttps11WrongHostname(HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase): pass -class Https11InvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): +class TestHttps11InvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): pass -class Https11InvalidDNSPattern( +class TestHttps11InvalidDNSPattern( HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase ): pass -class Https11CustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase): +class TestHttps11CustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase): pass diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index e058cedae..3e685bb28 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -163,23 +163,25 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): assert json.loads(response.text)["headers"][header] == [value1, value2] -class Https2WrongHostnameTestCase(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): +class TestHttps2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): pass -class Https2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): +class TestHttps2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): pass -class Https2InvalidDNSPattern(H2DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase): +class TestHttps2InvalidDNSPattern( + H2DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase +): pass -class Https2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase): +class TestHttps2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase): pass -class Http2MockServerTestCase(TestHttpMockServerBase): +class TestHttp2MockServer(TestHttpMockServerBase): """HTTP 2.0 test case with MockServer""" @property @@ -193,7 +195,7 @@ class Http2MockServerTestCase(TestHttpMockServerBase): is_secure = True -class Https2ProxyTestCase(H2DownloadHandlerMixin, TestHttpProxyBase): +class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): # only used for HTTPS tests keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 2c8e96040..518dc6b24 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -12,6 +12,7 @@ from unittest import mock import pytest from twisted.cred import checkers, credentials, portal +from twisted.internet.defer import inlineCallbacks from twisted.protocols.ftp import FTPFactory, FTPRealm from twisted.trial import unittest from w3lib.url import path_to_file_uri @@ -340,9 +341,10 @@ class TestFTPBase(unittest.TestCase): self.portNum = self.port.getHost().port crawler = get_crawler() self.download_handler = build_from_crawler(FTPDownloadHandler, crawler) - self.addCleanup(self.port.stopListening) + @inlineCallbacks def tearDown(self): + yield self.port.stopListening() shutil.rmtree(self.directory) def _add_test_callbacks(self, deferred, callback=None, errback=None): @@ -478,9 +480,10 @@ class TestAnonymousFTP(TestFTPBase): self.portNum = self.port.getHost().port crawler = get_crawler() self.download_handler = build_from_crawler(FTPDownloadHandler, crawler) - self.addCleanup(self.port.stopListening) + @inlineCallbacks def tearDown(self): + yield self.port.stopListening() shutil.rmtree(self.directory) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 146b0057e..dd5d47cab 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -247,10 +247,8 @@ Disallow: /some/randome/page.html assert request.callback == NO_CALLBACK +@pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed") class TestRobotsTxtMiddlewareWithRerp(TestRobotsTxtMiddleware): - if not rerp_available(): - skip = "Rerp parser is not installed" - def setUp(self): super().setUp() self.crawler.settings.set( diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 9b7c09448..ddac95edf 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -49,7 +49,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): self.times[slot].append(time.time()) -class CrawlTestCase(TestCase): +class TestCrawl(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index c7dbc82d4..bfb8eeced 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -27,7 +27,7 @@ async def sleep(seconds: float = 0.001) -> None: await maybe_deferred_to_future(deferred) -class MainTestCase(TestCase): +class TestMain(TestCase): @deferred_f_from_coro_f async def test_sleep(self): """Neither asynchronous sleeps on Spider.start() nor the equivalent on @@ -119,7 +119,7 @@ class MainTestCase(TestCase): assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" -class RequestSendOrderTestCase(TestCase): +class TestRequestSendOrder(TestCase): seconds = 0.1 # increase if flaky @classmethod diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 2ac4d7830..f9e54cb28 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -8,7 +8,7 @@ from scrapy.extensions.telnet import TelnetConsole from scrapy.utils.test import get_crawler -class TelnetExtensionTest(unittest.TestCase): +class TestTelnetExtension(unittest.TestCase): def _get_console_and_portal(self, settings=None): crawler = get_crawler(settings_dict=settings) console = TelnetConsole(crawler) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 262c0b434..01797fd20 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -91,52 +91,53 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]: return (client_mock, bucket_mock, blob_mock) -# TODO: replace self.mktemp() and drop the unittest.TestCase base -class TestFileFeedStorage(unittest.TestCase): - def test_store_file_uri(self): - path = Path(self.mktemp()).resolve() +class TestFileFeedStorage: + def test_store_file_uri(self, tmp_path): + path = tmp_path / "file.txt" uri = path_to_file_uri(str(path)) self._assert_stores(FileFeedStorage(uri), path) - def test_store_file_uri_makedirs(self): - path = Path(self.mktemp()).resolve() / "more" / "paths" / "file.txt" + def test_store_file_uri_makedirs(self, tmp_path): + path = tmp_path / "more" / "paths" / "file.txt" uri = path_to_file_uri(str(path)) self._assert_stores(FileFeedStorage(uri), path) - def test_store_direct_path(self): - path = Path(self.mktemp()).resolve() + def test_store_direct_path(self, tmp_path): + path = tmp_path / "file.txt" self._assert_stores(FileFeedStorage(str(path)), path) - def test_store_direct_path_relative(self): - path = Path(self.mktemp()) + def test_store_direct_path_relative(self, tmp_path): + path = (tmp_path / "foo" / "bar").relative_to(Path.cwd()) self._assert_stores(FileFeedStorage(str(path)), path) - def test_interface(self): - path = self.mktemp() - st = FileFeedStorage(path) + def test_interface(self, tmp_path): + path = tmp_path / "file.txt" + st = FileFeedStorage(str(path)) verifyObject(IFeedStorage, st) - def _store(self, feed_options=None) -> Path: - path = Path(self.mktemp()).resolve() + @staticmethod + def _store(path: Path, feed_options: dict[str, Any] | None = None) -> None: storage = FileFeedStorage(str(path), feed_options=feed_options) spider = scrapy.Spider("default") file = storage.open(spider) file.write(b"content") storage.store(file) - return path - def test_append(self): - path = self._store() + def test_append(self, tmp_path): + path = tmp_path / "file.txt" + self._store(path) self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent") - def test_overwrite(self): - path = self._store({"overwrite": True}) + def test_overwrite(self, tmp_path): + path = tmp_path / "file.txt" + self._store(path, {"overwrite": True}) self._assert_stores( FileFeedStorage(str(path), feed_options={"overwrite": True}), path ) + @staticmethod def _assert_stores( - self, storage: FileFeedStorage, path: Path, expected_content: bytes = b"content" + storage: FileFeedStorage, path: Path, expected_content: bytes = b"content" ) -> None: spider = scrapy.Spider("default") file = storage.open(spider) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index ef1806cc0..80edd50d6 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -8,7 +8,7 @@ import string from ipaddress import IPv4Address from pathlib import Path from tempfile import mkdtemp -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Callable from unittest import mock from urllib.parse import urlencode @@ -32,17 +32,22 @@ from twisted.web.static import File from scrapy.http import JsonRequest, Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + deferred_from_coro, + maybe_deferred_to_future, +) from tests.mockserver import LeafResource, Status, ssl_context_factory if TYPE_CHECKING: - from twisted.python.failure import Failure + from collections.abc import Coroutine -def generate_random_string(size): +def generate_random_string(size: int) -> str: return "".join(random.choices(string.ascii_uppercase + string.digits, k=size)) -def make_html_body(val): +def make_html_body(val: str) -> bytes: response = f"""

Hello from HTTP2

{val}

@@ -92,7 +97,7 @@ class GetDataHtmlLarge(LeafResource): class PostDataJsonMixin: @staticmethod - def make_response(request: TxRequest, extra_data: str): + def make_response(request: TxRequest, extra_data: str) -> bytes: assert request.content is not None response = { "request-headers": {}, @@ -179,7 +184,6 @@ def get_client_certificate( pem = key_file.read_text(encoding="utf-8") + certificate_file.read_text( encoding="utf-8" ) - return PrivateCertificate.loadPEM(pem) @@ -238,6 +242,7 @@ class TestHttps2ClientProtocol(TestCase): uri = URI.fromBytes(bytes(self.get_url("/"), "utf-8")) self.conn_closed_deferred = Deferred() + from scrapy.core.http2.protocol import H2ClientFactory h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred) @@ -255,7 +260,7 @@ class TestHttps2ClientProtocol(TestCase): shutil.rmtree(self.temp_directory) self.conn_closed_deferred = None - def get_url(self, path): + def get_url(self, path: str) -> str: """ :param path: Should have / at the starting compulsorily if not empty :return: Complete url @@ -264,143 +269,146 @@ class TestHttps2ClientProtocol(TestCase): assert path[0] == "/" or path[0] == "&" return f"{self.scheme}://{self.hostname}:{self.port_number}{path}" - def make_request(self, request: Request) -> Deferred: + async def make_request(self, request: Request) -> Response: + return await maybe_deferred_to_future(self.make_request_dfd(request)) + + def make_request_dfd(self, request: Request) -> Deferred[Response]: return self.client.request(request, DummySpider()) @staticmethod - def _check_repeat(get_deferred, count): + async def _check_repeat( + get_coro: Callable[[], Coroutine[Any, Any, None]], count: int + ) -> None: d_list = [] for _ in range(count): - d = get_deferred() + d = deferred_from_coro(get_coro()) d_list.append(d) - return DeferredList(d_list, fireOnOneErrback=True) + await maybe_deferred_to_future(DeferredList(d_list, fireOnOneErrback=True)) - def _check_GET(self, request: Request, expected_body, expected_status): - def check_response(response: Response): - assert response.status == expected_status - assert response.body == expected_body - assert response.request == request + async def _check_GET( + self, request: Request, expected_body: bytes, expected_status: int + ) -> None: + response = await self.make_request(request) + assert response.status == expected_status + assert response.body == expected_body + assert response.request == request - content_length_header = response.headers.get("Content-Length") - assert content_length_header is not None - content_length = int(content_length_header) - assert len(response.body) == content_length + content_length_header = response.headers.get("Content-Length") + assert content_length_header is not None + content_length = int(content_length_header) + assert len(response.body) == content_length - d = self.make_request(request) - d.addCallback(check_response) - d.addErrback(self.fail) - return d - - def test_GET_small_body(self): + @deferred_f_from_coro_f + async def test_GET_small_body(self): request = Request(self.get_url("/get-data-html-small")) - return self._check_GET(request, Data.HTML_SMALL, 200) + await self._check_GET(request, Data.HTML_SMALL, 200) - def test_GET_large_body(self): + @deferred_f_from_coro_f + async def test_GET_large_body(self): request = Request(self.get_url("/get-data-html-large")) - return self._check_GET(request, Data.HTML_LARGE, 200) + await self._check_GET(request, Data.HTML_LARGE, 200) - def _check_GET_x10(self, *args, **kwargs): - def get_deferred(): - return self._check_GET(*args, **kwargs) + async def _check_GET_x10( + self, request: Request, expected_body: bytes, expected_status: int + ) -> None: + async def get_coro() -> None: + await self._check_GET(request, expected_body, expected_status) - return self._check_repeat(get_deferred, 10) + await self._check_repeat(get_coro, 10) - def test_GET_small_body_x10(self): - return self._check_GET_x10( + @deferred_f_from_coro_f + async def test_GET_small_body_x10(self): + await self._check_GET_x10( Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200 ) - def test_GET_large_body_x10(self): - return self._check_GET_x10( + @deferred_f_from_coro_f + async def test_GET_large_body_x10(self): + await self._check_GET_x10( Request(self.get_url("/get-data-html-large")), Data.HTML_LARGE, 200 ) - def _check_POST_json( + async def _check_POST_json( self, request: Request, - expected_request_body, - expected_extra_data, + expected_request_body: dict[str, str], + expected_extra_data: str, expected_status: int, - ): - d = self.make_request(request) + ) -> None: + response = await self.make_request(request) - def assert_response(response: Response): - assert response.status == expected_status - assert response.request == request + assert response.status == expected_status + assert response.request == request - content_length_header = response.headers.get("Content-Length") - assert content_length_header is not None - content_length = int(content_length_header) - assert len(response.body) == content_length + content_length_header = response.headers.get("Content-Length") + assert content_length_header is not None + content_length = int(content_length_header) + assert len(response.body) == content_length - # Parse the body - content_encoding_header = response.headers[b"Content-Encoding"] - assert content_encoding_header is not None - content_encoding = str(content_encoding_header, "utf-8") - body = json.loads(str(response.body, content_encoding)) - assert "request-body" in body - assert "extra-data" in body - assert "request-headers" in body + # Parse the body + content_encoding_header = response.headers[b"Content-Encoding"] + assert content_encoding_header is not None + content_encoding = str(content_encoding_header, "utf-8") + body = json.loads(str(response.body, content_encoding)) + assert "request-body" in body + assert "extra-data" in body + assert "request-headers" in body - request_body = body["request-body"] - assert request_body == expected_request_body + request_body = body["request-body"] + assert request_body == expected_request_body - extra_data = body["extra-data"] - assert extra_data == expected_extra_data + extra_data = body["extra-data"] + assert extra_data == expected_extra_data - # Check if headers were sent successfully - request_headers = body["request-headers"] - for k, v in request.headers.items(): - k_str = str(k, "utf-8") - assert k_str in request_headers - assert request_headers[k_str] == str(v[0], "utf-8") + # Check if headers were sent successfully + request_headers = body["request-headers"] + for k, v in request.headers.items(): + k_str = str(k, "utf-8") + assert k_str in request_headers + assert request_headers[k_str] == str(v[0], "utf-8") - d.addCallback(assert_response) - d.addErrback(self.fail) - return d - - def test_POST_small_json(self): + @deferred_f_from_coro_f + async def test_POST_small_json(self): request = JsonRequest( url=self.get_url("/post-data-json-small"), method="POST", data=Data.JSON_SMALL, ) - return self._check_POST_json(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200) + await self._check_POST_json(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200) - def test_POST_large_json(self): + @deferred_f_from_coro_f + async def test_POST_large_json(self): request = JsonRequest( url=self.get_url("/post-data-json-large"), method="POST", data=Data.JSON_LARGE, ) - return self._check_POST_json(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200) + await self._check_POST_json(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200) - def _check_POST_json_x10(self, *args, **kwargs): - def get_deferred(): - return self._check_POST_json(*args, **kwargs) + async def _check_POST_json_x10(self, *args, **kwargs): + async def get_coro() -> None: + await self._check_POST_json(*args, **kwargs) - return self._check_repeat(get_deferred, 10) + await self._check_repeat(get_coro, 10) - def test_POST_small_json_x10(self): + @deferred_f_from_coro_f + async def test_POST_small_json_x10(self): request = JsonRequest( url=self.get_url("/post-data-json-small"), method="POST", data=Data.JSON_SMALL, ) - return self._check_POST_json_x10( - request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200 - ) + await self._check_POST_json_x10(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200) - def test_POST_large_json_x10(self): + @deferred_f_from_coro_f + async def test_POST_large_json_x10(self): request = JsonRequest( url=self.get_url("/post-data-json-large"), method="POST", data=Data.JSON_LARGE, ) - return self._check_POST_json_x10( - request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200 - ) + await self._check_POST_json_x10(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200) @inlineCallbacks def test_invalid_negotiated_protocol(self): @@ -409,77 +417,59 @@ class TestHttps2ClientProtocol(TestCase): ): request = Request(url=self.get_url("/status?n=200")) with pytest.raises(ResponseFailed): - yield self.make_request(request) + yield self.make_request_dfd(request) + @inlineCallbacks def test_cancel_request(self): request = Request(url=self.get_url("/get-data-html-large")) - - def assert_response(response: Response): - assert response.status == 499 - assert response.request == request - - d = self.make_request(request) - d.addCallback(assert_response) - d.addErrback(self.fail) + d = self.make_request_dfd(request) d.cancel() + response = yield d + assert response.status == 499 + assert response.request == request - return d - - def test_download_maxsize_exceeded(self): + @deferred_f_from_coro_f + async def test_download_maxsize_exceeded(self): request = Request( url=self.get_url("/get-data-html-large"), meta={"download_maxsize": 1000} ) + with pytest.raises(CancelledError) as exc_info: + await self.make_request(request) + error_pattern = re.compile( + rf"Cancelling download of {request.url}: received response " + rf"size \(\d*\) larger than download max size \(1000\)" + ) + assert len(re.findall(error_pattern, str(exc_info.value))) == 1 - def assert_cancelled_error(failure): - assert isinstance(failure.value, CancelledError) - error_pattern = re.compile( - rf"Cancelling download of {request.url}: received response " - rf"size \(\d*\) larger than download max size \(1000\)" - ) - assert len(re.findall(error_pattern, str(failure.value))) == 1 - - d = self.make_request(request) - d.addCallback(self.fail) - d.addErrback(assert_cancelled_error) - return d - + @inlineCallbacks def test_received_dataloss_response(self): """In case when value of Header Content-Length != len(Received Data) ProtocolError is raised""" + from h2.exceptions import InvalidBodyLengthError + request = Request(url=self.get_url("/dataloss")) + with pytest.raises(ResponseFailed) as exc_info: + yield self.make_request_dfd(request) + assert len(exc_info.value.reasons) > 0 + assert any( + isinstance(error, InvalidBodyLengthError) + for error in exc_info.value.reasons + ) - def assert_failure(failure: Failure): - assert len(failure.value.reasons) > 0 - from h2.exceptions import InvalidBodyLengthError - - assert any( - isinstance(error, InvalidBodyLengthError) - for error in failure.value.reasons - ) - - d = self.make_request(request) - d.addCallback(self.fail) - d.addErrback(assert_failure) - return d - - def test_missing_content_length_header(self): + @deferred_f_from_coro_f + async def test_missing_content_length_header(self): request = Request(url=self.get_url("/no-content-length-header")) + response = await self.make_request(request) + assert response.status == 200 + assert response.body == Data.NO_CONTENT_LENGTH + assert response.request == request + assert "Content-Length" not in response.headers - def assert_content_length(response: Response): - assert response.status == 200 - assert response.body == Data.NO_CONTENT_LENGTH - assert response.request == request - assert "Content-Length" not in response.headers - - d = self.make_request(request) - d.addCallback(assert_content_length) - d.addErrback(self.fail) - return d - - @inlineCallbacks - def _check_log_warnsize(self, request, warn_pattern, expected_body): + async def _check_log_warnsize( + self, request: Request, warn_pattern: re.Pattern[str], expected_body: bytes + ) -> None: with self.assertLogs("scrapy.core.http2.stream", level="WARNING") as cm: - response = yield self.make_request(request) + response = await self.make_request(request) assert response.status == 200 assert response.request == request assert response.body == expected_body @@ -487,8 +477,8 @@ class TestHttps2ClientProtocol(TestCase): # Check the warning is raised only once for this request assert sum(len(re.findall(warn_pattern, log)) for log in cm.output) == 1 - @inlineCallbacks - def test_log_expected_warnsize(self): + @deferred_f_from_coro_f + async def test_log_expected_warnsize(self): request = Request( url=self.get_url("/get-data-html-large"), meta={"download_warnsize": 1000} ) @@ -497,10 +487,10 @@ class TestHttps2ClientProtocol(TestCase): rf"download warn size \(1000\) in request {request}" ) - yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE) + await self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE) - @inlineCallbacks - def test_log_received_warnsize(self): + @deferred_f_from_coro_f + async def test_log_received_warnsize(self): request = Request( url=self.get_url("/no-content-length-header"), meta={"download_warnsize": 10}, @@ -510,20 +500,22 @@ class TestHttps2ClientProtocol(TestCase): rf"warn size \(10\) in request {request}" ) - yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) + await self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) - def test_max_concurrent_streams(self): + @deferred_f_from_coro_f + async def test_max_concurrent_streams(self): """Send 500 requests at one to check if we can handle very large number of request. """ - def get_deferred(): - return self._check_GET( + async def get_coro() -> None: + await self._check_GET( Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200 ) - return self._check_repeat(get_deferred, 500) + await self._check_repeat(get_coro, 500) + @inlineCallbacks def test_inactive_stream(self): """Here we send 110 requests considering the MAX_CONCURRENT_STREAMS by default is 100. After sending the first 100 requests we close the @@ -532,6 +524,7 @@ class TestHttps2ClientProtocol(TestCase): def assert_inactive_stream(failure): assert failure.check(ResponseFailed) is not None + from scrapy.core.http2.stream import InactiveStreamClosed assert any( @@ -540,14 +533,14 @@ class TestHttps2ClientProtocol(TestCase): # Send 100 request (we do not check the result) for _ in range(100): - d = self.make_request(Request(self.get_url("/get-data-html-small"))) + d = self.make_request_dfd(Request(self.get_url("/get-data-html-small"))) d.addBoth(lambda _: None) d_list.append(d) # Now send 10 extra request and save the response deferred in a list for _ in range(10): - d = self.make_request(Request(self.get_url("/get-data-html-small"))) - d.addCallback(self.fail) + d = self.make_request_dfd(Request(self.get_url("/get-data-html-small"))) + d.addCallback(lambda _: pytest.fail("This request should have failed")) d.addErrback(assert_inactive_stream) d_list.append(d) @@ -555,13 +548,15 @@ class TestHttps2ClientProtocol(TestCase): # with InactiveStreamClosed self.client.transport.loseConnection() - return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) + yield DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) - def test_invalid_request_type(self): + @deferred_f_from_coro_f + async def test_invalid_request_type(self): with pytest.raises(TypeError): - self.make_request("https://InvalidDataTypePassed.com") + await self.make_request("https://InvalidDataTypePassed.com") - def test_query_parameters(self): + @deferred_f_from_coro_f + async def test_query_parameters(self): params = { "a": generate_random_string(20), "b": generate_random_string(20), @@ -569,133 +564,96 @@ class TestHttps2ClientProtocol(TestCase): "d": generate_random_string(20), } request = Request(self.get_url(f"/query-params?{urlencode(params)}")) + response = await self.make_request(request) + content_encoding_header = response.headers[b"Content-Encoding"] + assert content_encoding_header is not None + content_encoding = str(content_encoding_header, "utf-8") + data = json.loads(str(response.body, content_encoding)) + assert data == params - def assert_query_params(response: Response): - content_encoding_header = response.headers[b"Content-Encoding"] - assert content_encoding_header is not None - content_encoding = str(content_encoding_header, "utf-8") - data = json.loads(str(response.body, content_encoding)) - assert data == params - - d = self.make_request(request) - d.addCallback(assert_query_params) - d.addErrback(self.fail) - - return d - - def test_status_codes(self): - def assert_response_status(response: Response, expected_status: int): - assert response.status == expected_status - - d_list = [] + @deferred_f_from_coro_f + async def test_status_codes(self): for status in [200, 404]: request = Request(self.get_url(f"/status?n={status}")) - d = self.make_request(request) - d.addCallback(assert_response_status, status) - d.addErrback(self.fail) - d_list.append(d) + response = await self.make_request(request) + assert response.status == status - return DeferredList(d_list, fireOnOneErrback=True) - - def test_response_has_correct_certificate_ip_address(self): + @deferred_f_from_coro_f + async def test_response_has_correct_certificate_ip_address(self): request = Request(self.get_url("/status?n=200")) + response = await self.make_request(request) + assert response.request == request + assert isinstance(response.certificate, Certificate) + assert response.certificate.original is not None + assert response.certificate.getIssuer() == self.client_certificate.getIssuer() + assert response.certificate.getPublicKey().matches( + self.client_certificate.getPublicKey() + ) + assert isinstance(response.ip_address, IPv4Address) + assert str(response.ip_address) == "127.0.0.1" - def assert_metadata(response: Response): - assert response.request == request - assert isinstance(response.certificate, Certificate) - assert response.certificate.original is not None - assert ( - response.certificate.getIssuer() == self.client_certificate.getIssuer() - ) - assert response.certificate.getPublicKey().matches( - self.client_certificate.getPublicKey() - ) + async def _check_invalid_netloc(self, url: str) -> None: + from scrapy.core.http2.stream import InvalidHostname - assert isinstance(response.ip_address, IPv4Address) - assert str(response.ip_address) == "127.0.0.1" - - d = self.make_request(request) - d.addCallback(assert_metadata) - d.addErrback(self.fail) - - return d - - def _check_invalid_netloc(self, url): request = Request(url) + with pytest.raises(InvalidHostname) as exc_info: + await self.make_request(request) + error_msg = str(exc_info.value) + assert "localhost" in error_msg + assert "127.0.0.1" in error_msg + assert str(request) in error_msg - def assert_invalid_hostname(failure: Failure): - from scrapy.core.http2.stream import InvalidHostname + @deferred_f_from_coro_f + async def test_invalid_hostname(self): + await self._check_invalid_netloc("https://notlocalhost.notlocalhostdomain") - assert failure.check(InvalidHostname) is not None - error_msg = str(failure.value) - assert "localhost" in error_msg - assert "127.0.0.1" in error_msg - assert str(request) in error_msg - - d = self.make_request(request) - d.addCallback(self.fail) - d.addErrback(assert_invalid_hostname) - return d - - def test_invalid_hostname(self): - return self._check_invalid_netloc("https://notlocalhost.notlocalhostdomain") - - def test_invalid_host_port(self): + @deferred_f_from_coro_f + async def test_invalid_host_port(self): port = self.port_number + 1 - return self._check_invalid_netloc(f"https://127.0.0.1:{port}") + await self._check_invalid_netloc(f"https://127.0.0.1:{port}") - def test_connection_stays_with_invalid_requests(self): - d_list = [ - self.test_invalid_hostname(), - self.test_invalid_host_port(), - self.test_GET_small_body(), - self.test_POST_small_json(), - ] - - return DeferredList(d_list, fireOnOneErrback=True) + @deferred_f_from_coro_f + async def test_connection_stays_with_invalid_requests(self): + await maybe_deferred_to_future(self.test_invalid_hostname()) + await maybe_deferred_to_future(self.test_invalid_host_port()) + await maybe_deferred_to_future(self.test_GET_small_body()) + await maybe_deferred_to_future(self.test_POST_small_json()) + @inlineCallbacks def test_connection_timeout(self): request = Request(self.get_url("/timeout")) - d = self.make_request(request) # Update the timer to 1s to test connection timeout self.client.setTimeout(1) - def assert_timeout_error(failure: Failure): - for err in failure.value.reasons: - from scrapy.core.http2.protocol import H2ClientProtocol + with pytest.raises(ResponseFailed) as exc_info: + yield self.make_request_dfd(request) - if isinstance(err, TimeoutError): - assert ( - f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s" - in str(err) - ) - break - else: - pytest.fail("No TimeoutError raised.") + for err in exc_info.value.reasons: + from scrapy.core.http2.protocol import H2ClientProtocol - d.addCallback(self.fail) - d.addErrback(assert_timeout_error) - return d + if isinstance(err, TimeoutError): + assert ( + f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s" + in str(err) + ) + break + else: + pytest.fail("No TimeoutError raised.") - def test_request_headers_received(self): + @deferred_f_from_coro_f + async def test_request_headers_received(self): request = Request( self.get_url("/request-headers"), headers={"header-1": "header value 1", "header-2": "header value 2"}, ) - d = self.make_request(request) + response = await self.make_request(request) + assert response.status == 200 + assert response.request == request - def assert_request_headers(response: Response): - assert response.status == 200 - assert response.request == request - - response_headers = json.loads(str(response.body, "utf-8")) - assert isinstance(response_headers, dict) - for k, v in request.headers.items(): - k, v = str(k, "utf-8"), str(v[0], "utf-8") - assert k in response_headers - assert v == response_headers[k] - - d.addErrback(self.fail) - d.addCallback(assert_request_headers) - return d + response_headers = json.loads(str(response.body, "utf-8")) + assert isinstance(response_headers, dict) + for k, v in request.headers.items(): + k, v = str(k, "utf-8"), str(v[0], "utf-8") + assert k in response_headers + assert v == response_headers[k] diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 00e534c4b..cf827e481 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -5,6 +5,7 @@ from pathlib import Path from tempfile import mkdtemp from typing import TYPE_CHECKING, Any +import pytest from testfixtures import LogCapture from twisted.internet.defer import inlineCallbacks from twisted.trial.unittest import TestCase @@ -218,18 +219,20 @@ class TestFileDownloadCrawl(TestCase): assert "ZeroDivisionError" in str(log) -skip_pillow: str | None +pillow_available: bool try: from PIL import Image # noqa: F401 except ImportError: - skip_pillow = "Missing Python Imaging Library, install https://pypi.org/pypi/Pillow" + pillow_available = False else: - skip_pillow = None + pillow_available = True -class ImageDownloadCrawlTestCase(TestFileDownloadCrawl): - skip = skip_pillow - +@pytest.mark.skipif( + not pillow_available, + reason="Missing Python Imaging Library, install https://pypi.org/pypi/Pillow", +) +class TestImageDownloadCrawl(TestFileDownloadCrawl): pipeline_class = "scrapy.pipelines.images.ImagesPipeline" store_setting_key = "IMAGES_STORE" media_key = "images" diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 492409d02..b4eae108f 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -3,6 +3,7 @@ import os import random import time import warnings +from abc import ABC, abstractmethod from datetime import datetime from io import BytesIO from pathlib import Path @@ -265,7 +266,12 @@ class TestFilesPipeline(unittest.TestCase): assert file_path(request, item=item) == "full/path-to-store-file" -class FilesPipelineTestCaseFieldsMixin: +class TestFilesPipelineFieldsMixin(ABC): + @property + @abstractmethod + def item_class(self) -> Any: + raise NotImplementedError + def test_item_fields_default(self, tmp_path): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", file_urls=[url]) @@ -302,7 +308,7 @@ class FilesPipelineTestCaseFieldsMixin: assert isinstance(item, self.item_class) -class TestFilesPipelineFieldsDict(FilesPipelineTestCaseFieldsMixin): +class TestFilesPipelineFieldsDict(TestFilesPipelineFieldsMixin): item_class = dict @@ -316,7 +322,7 @@ class FilesPipelineTestItem(Item): custom_files = Field() -class TestFilesPipelineFieldsItem(FilesPipelineTestCaseFieldsMixin): +class TestFilesPipelineFieldsItem(TestFilesPipelineFieldsMixin): item_class = FilesPipelineTestItem @@ -331,7 +337,7 @@ class FilesPipelineTestDataClass: custom_files: list = dataclasses.field(default_factory=list) -class TestFilesPipelineFieldsDataClass(FilesPipelineTestCaseFieldsMixin): +class TestFilesPipelineFieldsDataClass(TestFilesPipelineFieldsMixin): item_class = FilesPipelineTestDataClass @@ -346,7 +352,7 @@ class FilesPipelineTestAttrsItem: custom_files: list[dict[str, str]] = attr.ib(default=list) -class TestFilesPipelineFieldsAttrsItem(FilesPipelineTestCaseFieldsMixin): +class TestFilesPipelineFieldsAttrsItem(TestFilesPipelineFieldsMixin): item_class = FilesPipelineTestAttrsItem diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index f2ee18bd9..74b4495ad 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -3,8 +3,10 @@ from __future__ import annotations import dataclasses import io import random +from abc import ABC, abstractmethod from shutil import rmtree from tempfile import mkdtemp +from typing import Any import attr import pytest @@ -208,7 +210,12 @@ class TestImagesPipeline: assert converted.getcolors() == [(10000, (205, 230, 255))] -class ImagesPipelineTestCaseFieldsMixin: +class TestImagesPipelineFieldsMixin(ABC): + @property + @abstractmethod + def item_class(self) -> Any: + raise NotImplementedError + def test_item_fields_default(self): url = "http://www.example.com/images/1.jpg" item = self.item_class(name="item1", image_urls=[url]) @@ -245,7 +252,7 @@ class ImagesPipelineTestCaseFieldsMixin: assert isinstance(item, self.item_class) -class TestImagesPipelineFieldsDict(ImagesPipelineTestCaseFieldsMixin): +class TestImagesPipelineFieldsDict(TestImagesPipelineFieldsMixin): item_class = dict @@ -259,7 +266,7 @@ class ImagesPipelineTestItem(Item): custom_images = Field() -class TestImagesPipelineFieldsItem(ImagesPipelineTestCaseFieldsMixin): +class TestImagesPipelineFieldsItem(TestImagesPipelineFieldsMixin): item_class = ImagesPipelineTestItem @@ -274,7 +281,7 @@ class ImagesPipelineTestDataClass: custom_images: list = dataclasses.field(default_factory=list) -class TestImagesPipelineFieldsDataClass(ImagesPipelineTestCaseFieldsMixin): +class TestImagesPipelineFieldsDataClass(TestImagesPipelineFieldsMixin): item_class = ImagesPipelineTestDataClass @@ -289,7 +296,7 @@ class ImagesPipelineTestAttrsItem: custom_images: list[dict[str, str]] = attr.ib(default=list) -class TestImagesPipelineFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin): +class TestImagesPipelineFieldsAttrsItem(TestImagesPipelineFieldsMixin): item_class = ImagesPipelineTestAttrsItem diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 2d0db6e25..40149f184 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -345,7 +345,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): @inlineCallbacks def test_use_media_to_download_result(self): - req = Request("http://url", meta={"result": "ITSME", "response": self.fail}) + req = Request("http://url", meta={"result": "ITSME"}) item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) assert new_item["results"] == [(True, "ITSME")] diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 5c2772c30..26482fc8d 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -115,7 +115,7 @@ class TestMinimalScheduler(InterfaceCheckMixin): assert not self.scheduler.has_pending_requests() -class SimpleSchedulerTest(TestCase, InterfaceCheckMixin): +class TestSimpleScheduler(TestCase, InterfaceCheckMixin): def setUp(self): self.scheduler = SimpleScheduler() @@ -145,7 +145,7 @@ class SimpleSchedulerTest(TestCase, InterfaceCheckMixin): assert close_result == "close" -class MinimalSchedulerCrawlTest(TestCase): +class TestMinimalSchedulerCrawl(TestCase): scheduler_cls = MinimalScheduler @inlineCallbacks @@ -162,5 +162,5 @@ class MinimalSchedulerCrawlTest(TestCase): assert f"'item_scraped_count': {len(PATHS)}" in str(log) -class SimpleSchedulerCrawlTest(MinimalSchedulerCrawlTest): +class TestSimpleSchedulerCrawl(TestMinimalSchedulerCrawl): scheduler_cls = SimpleScheduler diff --git a/tests/test_signals.py b/tests/test_signals.py index 5a536896e..b20a949e8 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -21,7 +21,7 @@ class ItemSpider(Spider): return {"index": response.meta["index"]} -class MainTestCase(TestCase): +class TestMain(TestCase): @deferred_f_from_coro_f async def test_scheduler_empty(self): crawler = get_crawler() @@ -35,7 +35,7 @@ class MainTestCase(TestCase): assert len(calls) >= 1 -class MockServerTestCase(TestCase): +class TestMockServer(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index 1815aad76..3c7fc65d5 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -18,7 +18,7 @@ ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} -class MainTestCase(TestCase): +class TestMain(TestCase): async def _test_spider(self, spider, expected_items=None): actual_items = [] expected_items = [] if expected_items is None else expected_items diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 725833a49..e1c8b5fec 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -47,7 +47,7 @@ class UniversalSpiderMiddleware: raise NotImplementedError -# Spiders and spider middlewares for MainTestCase._test_wrap +# Spiders and spider middlewares for TestMain._test_wrap class ModernWrapSpider(Spider): @@ -106,7 +106,7 @@ class DeprecatedWrapSpiderMiddleware: yield ITEM_C -class MainTestCase(TestCase): +class TestMain(TestCase): async def _test(self, spider_middlewares, spider_cls, expected_items): actual_items = [] diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 372213319..1cfaf70fa 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -92,10 +92,10 @@ class TestDeferUtils(unittest.TestCase): x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2") assert x == ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"] + @inlineCallbacks def test_process_parallel_failure(self): - d = process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2") - self.failUnlessFailure(d, TypeError) - return d + with pytest.raises(TypeError): + yield process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2") class TestIterErrback: diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 97dade26e..79bac8bc5 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -59,12 +59,12 @@ class TestSendCatchLog(unittest.TestCase): return "OK" -class SendCatchLogDeferredTest(TestSendCatchLog): +class TestSendCatchLogDeferred(TestSendCatchLog): def _get_result(self, signal, *a, **kw): return send_catch_log_deferred(signal, *a, **kw) -class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): +class TestSendCatchLogDeferred2(TestSendCatchLogDeferred): def ok_handler(self, arg, handlers_called): from twisted.internet import reactor @@ -76,7 +76,7 @@ class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): @pytest.mark.usefixtures("reactor_pytest") -class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): +class TestSendCatchLogDeferredAsyncDef(TestSendCatchLogDeferred): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) assert arg == "test" @@ -85,7 +85,7 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): @pytest.mark.only_asyncio -class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): +class TestSendCatchLogDeferredAsyncio(TestSendCatchLogDeferred): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) assert arg == "test" @@ -93,12 +93,12 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): return await get_from_asyncio_queue("OK") -class SendCatchLogAsyncTest(TestSendCatchLog): +class TestSendCatchLogAsync(TestSendCatchLog): def _get_result(self, signal, *a, **kw): return deferred_from_coro(send_catch_log_async(signal, *a, **kw)) -class SendCatchLogAsyncTest2(SendCatchLogAsyncTest): +class TestSendCatchLogAsync2(TestSendCatchLogAsync): def ok_handler(self, arg, handlers_called): from twisted.internet import reactor @@ -110,7 +110,7 @@ class SendCatchLogAsyncTest2(SendCatchLogAsyncTest): @pytest.mark.usefixtures("reactor_pytest") -class SendCatchLogAsyncAsyncDefTest(SendCatchLogAsyncTest): +class TestSendCatchLogAsyncAsyncDef(TestSendCatchLogAsync): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) assert arg == "test" @@ -119,7 +119,7 @@ class SendCatchLogAsyncAsyncDefTest(SendCatchLogAsyncTest): @pytest.mark.only_asyncio -class SendCatchLogAsyncAsyncioTest(SendCatchLogAsyncTest): +class TestSendCatchLogAsyncAsyncio(TestSendCatchLogAsync): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) assert arg == "test" diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 8b32e40bb..569f4f639 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -234,35 +234,31 @@ class TestWebClient(unittest.TestCase): def getURL(self, path): return f"http://127.0.0.1:{self.portno}/{path}" + @inlineCallbacks def testPayload(self): s = "0123456789" * 10 - return getPage(self.getURL("payload"), body=s).addCallback( - self.assertEqual, to_bytes(s) - ) + body = yield getPage(self.getURL("payload"), body=s) + assert body == to_bytes(s) + @inlineCallbacks def testHostHeader(self): # if we pass Host header explicitly, it should be used, otherwise # it should extract from url - return defer.gatherResults( - [ - getPage(self.getURL("host")).addCallback( - self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}") - ), - getPage( - self.getURL("host"), headers={"Host": "www.example.com"} - ).addCallback(self.assertEqual, to_bytes("www.example.com")), - ] - ) + body = yield getPage(self.getURL("host")) + assert body == to_bytes(f"127.0.0.1:{self.portno}") + body = yield getPage(self.getURL("host"), headers={"Host": "www.example.com"}) + assert body == to_bytes("www.example.com") + @inlineCallbacks def test_getPage(self): """ L{client.getPage} returns a L{Deferred} which is called back with the body of the response if the default method B{GET} is used. """ - d = getPage(self.getURL("file")) - d.addCallback(self.assertEqual, b"0123456789") - return d + body = yield getPage(self.getURL("file")) + assert body == b"0123456789" + @inlineCallbacks def test_getPageHead(self): """ L{client.getPage} returns a L{Deferred} which is called back with @@ -273,22 +269,20 @@ class TestWebClient(unittest.TestCase): def _getPage(method): return getPage(self.getURL("file"), method=method) - return defer.gatherResults( - [ - _getPage("head").addCallback(self.assertEqual, b""), - _getPage("HEAD").addCallback(self.assertEqual, b""), - ] - ) + body = yield _getPage("head") + assert body == b"" + body = yield _getPage("HEAD") + assert body == b"" + @inlineCallbacks def test_timeoutNotTriggering(self): """ When a non-zero timeout is passed to L{getPage} and the page is retrieved before the timeout period elapses, the L{Deferred} is called back with the contents of the page. """ - d = getPage(self.getURL("host"), timeout=100) - d.addCallback(self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}")) - return d + body = yield getPage(self.getURL("host"), timeout=100) + assert body == to_bytes(f"127.0.0.1:{self.portno}") @inlineCallbacks def test_timeoutTriggering(self): @@ -307,12 +301,12 @@ class TestWebClient(unittest.TestCase): if connected: connected[0].transport.loseConnection() + @inlineCallbacks def testNotFound(self): - return getPage(self.getURL("notsuchfile")).addCallback(self._cbNoSuchFile) - - def _cbNoSuchFile(self, pageData): - assert b"404 - No Such Resource" in pageData + body = yield getPage(self.getURL("notsuchfile")) + assert b"404 - No Such Resource" in body + @inlineCallbacks def testFactoryInfo(self): from twisted.internet import reactor @@ -320,63 +314,60 @@ class TestWebClient(unittest.TestCase): parsed = urlparse(url) factory = client.ScrapyHTTPClientFactory(Request(url)) reactor.connectTCP(parsed.hostname, parsed.port, factory) - return factory.deferred.addCallback(self._cbFactoryInfo, factory) - - def _cbFactoryInfo(self, ignoredResult, factory): + yield factory.deferred assert factory.status == b"200" assert factory.version.startswith(b"HTTP/") assert factory.message == b"OK" assert factory.response_headers[b"content-length"] == b"10" + @inlineCallbacks def testRedirect(self): - return getPage(self.getURL("redirect")).addCallback(self._cbRedirect) - - def _cbRedirect(self, pageData): + body = yield getPage(self.getURL("redirect")) assert ( - pageData + body == b'\n\n \n \n' b' \n \n ' b'click here\n \n\n' ) + @inlineCallbacks def test_encoding(self): """Test that non-standart body encoding matches Content-Encoding header""" - body = b"\xd0\x81\xd1\x8e\xd0\xaf" - dfd = getPage( - self.getURL("encoding"), body=body, response_transform=lambda r: r + original_body = b"\xd0\x81\xd1\x8e\xd0\xaf" + response = yield getPage( + self.getURL("encoding"), body=original_body, response_transform=lambda r: r ) - return dfd.addCallback(self._check_Encoding, body) - - def _check_Encoding(self, response, original_body): content_encoding = to_unicode(response.headers[b"Content-Encoding"]) assert content_encoding == EncodingResource.out_encoding assert response.body.decode(content_encoding) == to_unicode(original_body) @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class WebClientSSLTestCase(TestContextFactoryBase): +class TestWebClientSSL(TestContextFactoryBase): + @inlineCallbacks def testPayload(self): s = "0123456789" * 10 - return getPage(self.getURL("payload"), body=s).addCallback( - self.assertEqual, to_bytes(s) - ) + body = yield getPage(self.getURL("payload"), body=s) + assert body == to_bytes(s) -class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): +class TestWebClientCustomCiphersSSL(TestWebClientSSL): # we try to use a cipher that is not enabled by default in OpenSSL custom_ciphers = "CAMELLIA256-SHA" context_factory = ssl_context_factory(cipher_string=custom_ciphers) + @inlineCallbacks def testPayload(self): s = "0123456789" * 10 crawler = get_crawler( settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers} ) client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler) - return getPage( + body = yield getPage( self.getURL("payload"), body=s, contextFactory=client_context_factory - ).addCallback(self.assertEqual, to_bytes(s)) + ) + assert body == to_bytes(s) @inlineCallbacks def testPayloadDisabledCipher(self): From 92c18d15b4dbed7c98dc3d5ac329c90abb23950f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 13 Jun 2025 22:18:27 +0500 Subject: [PATCH 168/491] Remove ProcessTest and SiteTest. (#6885) * Remove ProcessTest and SiteTest. * Restore the support for Windows line endings in TestParseCommand. * Add a test for running a scrapy command in a project subdir. * Remove pywin32 from test deps. --- tests/mockserver.py | 43 +++- tests/test_command_check.py | 6 +- tests/test_command_fetch.py | 46 ++-- tests/test_command_parse.py | 379 ++++++++++++++--------------- tests/test_command_runspider.py | 17 +- tests/test_command_shell.py | 153 ++++++------ tests/test_command_startproject.py | 4 +- tests/test_command_version.py | 26 +- tests/test_commands.py | 49 ++-- tests/utils/testproc.py | 67 ----- tests/utils/testsite.py | 47 ---- tox.ini | 1 - 12 files changed, 360 insertions(+), 478 deletions(-) delete mode 100644 tests/utils/testproc.py delete mode 100644 tests/utils/testsite.py diff --git a/tests/mockserver.py b/tests/mockserver.py index e0ac127f2..841a2cfe4 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -18,11 +18,12 @@ from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory from twisted.web import resource, server -from twisted.web.server import NOT_DONE_YET, GzipEncoderFactory, Site -from twisted.web.static import File -from twisted.web.util import redirectTo +from twisted.web.server import NOT_DONE_YET, Site +from twisted.web.static import Data, File +from twisted.web.util import Redirect, redirectTo from scrapy.utils.python import to_bytes, to_unicode +from tests import tests_datadir if TYPE_CHECKING: from twisted.internet.protocol import ServerFactory @@ -245,6 +246,14 @@ class ArbitraryLengthPayloadResource(LeafResource): return request.content.read() +class NoMetaRefreshRedirect(Redirect): + def render(self, request: server.Request) -> bytes: + content = Redirect.render(self, request) + return content.replace( + b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"' + ) + + class Root(resource.Resource): def __init__(self): resource.Resource.__init__(self) @@ -256,18 +265,26 @@ class Root(resource.Resource): self.putChild(b"raw", Raw()) self.putChild(b"echo", Echo()) self.putChild(b"payload", PayloadResource()) - self.putChild( - b"xpayload", - resource.EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]), - ) self.putChild(b"alpayload", ArbitraryLengthPayloadResource()) - try: - from tests import tests_datadir - - self.putChild(b"files", File(str(Path(tests_datadir, "test_site/files/")))) - except Exception: - pass + self.putChild(b"files", File(str(Path(tests_datadir, "test_site/files/")))) self.putChild(b"redirect-to", RedirectTo()) + self.putChild(b"text", Data(b"Works", "text/plain")) + self.putChild( + b"html", + Data( + b"

Works

World

", + "text/html", + ), + ) + self.putChild( + b"enc-gb18030", + Data(b"

gb18030 encoding

", "text/html; charset=gb18030"), + ) + self.putChild(b"redirect", Redirect(b"/redirected")) + self.putChild( + b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected") + ) + self.putChild(b"redirected", Data(b"Redirected here", "text/plain")) def getChild(self, name, request): return self diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 975f31dfe..97bd9d726 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -7,10 +7,8 @@ from tests.test_commands import TestCommandBase class TestCheckCommand(TestCommandBase): - command = "check" - - def setUp(self): - super().setUp() + def setup_method(self): + super().setup_method() self.spider_name = "check_spider" self.spider = (self.proj_mod_path / "spiders" / "checkspider.py").resolve() diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 89f664336..c83594361 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -1,35 +1,35 @@ -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest - -from tests.utils.testproc import ProcessTest -from tests.utils.testsite import SiteTest +from tests.mockserver import MockServer +from tests.test_commands import TestProjectBase -class TestFetchCommand(ProcessTest, SiteTest, unittest.TestCase): - command = "fetch" +class TestFetchCommand(TestProjectBase): + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) - @inlineCallbacks def test_output(self): - _, out, _ = yield self.execute([self.url("/text")]) - assert out.strip() == b"Works" + _, out, _ = self.proc("fetch", self.mockserver.url("/text")) + assert out.strip() == "Works" - @inlineCallbacks def test_redirect_default(self): - _, out, _ = yield self.execute([self.url("/redirect")]) - assert out.strip() == b"Redirected here" + _, out, _ = self.proc("fetch", self.mockserver.url("/redirect")) + assert out.strip() == "Redirected here" - @inlineCallbacks def test_redirect_disabled(self): - _, out, err = yield self.execute( - ["--no-redirect", self.url("/redirect-no-meta-refresh")] + _, _, err = self.proc( + "fetch", "--no-redirect", self.mockserver.url("/redirect-no-meta-refresh") ) err = err.strip() - assert b"downloader/response_status_count/302" in err, err - assert b"downloader/response_status_count/200" not in err, err + assert "downloader/response_status_count/302" in err, err + assert "downloader/response_status_count/200" not in err, err - @inlineCallbacks def test_headers(self): - _, out, _ = yield self.execute([self.url("/text"), "--headers"]) - out = out.replace(b"\r", b"") # required on win32 - assert b"Server: TwistedWeb" in out, out - assert b"Content-Type: text/plain" in out + _, out, _ = self.proc("fetch", self.mockserver.url("/text"), "--headers") + out = out.replace("\r", "") # required on win32 + assert "Server: TwistedWeb" in out, out + assert "Content-Type: text/plain" in out diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 6681aba17..5c3120c21 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,29 +1,25 @@ import argparse -import os import re from pathlib import Path -from twisted.internet.defer import inlineCallbacks - from scrapy.commands import parse from scrapy.settings import Settings -from scrapy.utils.python import to_unicode +from tests.mockserver import MockServer from tests.test_commands import TestCommandBase -from tests.utils.testproc import ProcessTest -from tests.utils.testsite import SiteTest -def _textmode(bstr: bytes) -> str: - """Normalize input the same as writing to a file - and reading from it in text mode""" - return to_unicode(bstr).replace(os.linesep, "\n") +class TestParseCommand(TestCommandBase): + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) -class TestParseCommand(ProcessTest, SiteTest, TestCommandBase): - command = "parse" - - def setUp(self): - super().setUp() + def setup_method(self): + super().setup_method() self.spider_name = "parse_spider" (self.proj_mod_path / "spiders" / "myspider.py").write_text( f""" @@ -171,260 +167,253 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} """ ) - @inlineCallbacks def test_spider_arguments(self): - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "-a", - "test_arg=1", - "-c", - "parse", - "--verbose", - self.url("/html"), - ] + _, _, stderr = self.proc( + "parse", + "--spider", + self.spider_name, + "-a", + "test_arg=1", + "-c", + "parse", + "--verbose", + self.mockserver.url("/html"), ) - assert "DEBUG: It Works!" in _textmode(stderr) + assert "DEBUG: It Works!" in stderr - @inlineCallbacks def test_request_with_meta(self): raw_json_string = '{"foo" : "baz"}' - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "--meta", - raw_json_string, - "-c", - "parse_request_with_meta", - "--verbose", - self.url("/html"), - ] + _, _, stderr = self.proc( + "parse", + "--spider", + self.spider_name, + "--meta", + raw_json_string, + "-c", + "parse_request_with_meta", + "--verbose", + self.mockserver.url("/html"), ) - assert "DEBUG: It Works!" in _textmode(stderr) + assert "DEBUG: It Works!" in stderr - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "-m", - raw_json_string, - "-c", - "parse_request_with_meta", - "--verbose", - self.url("/html"), - ] + _, _, stderr = self.proc( + "parse", + "--spider", + self.spider_name, + "-m", + raw_json_string, + "-c", + "parse_request_with_meta", + "--verbose", + self.mockserver.url("/html"), ) - assert "DEBUG: It Works!" in _textmode(stderr) + assert "DEBUG: It Works!" in stderr - @inlineCallbacks def test_request_with_cb_kwargs(self): raw_json_string = '{"foo" : "bar", "key": "value"}' - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "--cbkwargs", - raw_json_string, - "-c", - "parse_request_with_cb_kwargs", - "--verbose", - self.url("/html"), - ] + _, _, stderr = self.proc( + "parse", + "--spider", + self.spider_name, + "--cbkwargs", + raw_json_string, + "-c", + "parse_request_with_cb_kwargs", + "--verbose", + self.mockserver.url("/html"), ) - log = _textmode(stderr) - assert "DEBUG: It Works!" in log + assert "DEBUG: It Works!" in stderr assert ( - "DEBUG: request.callback signature: (response, foo=None, key=None)" in log + "DEBUG: request.callback signature: (response, foo=None, key=None)" + in stderr ) - @inlineCallbacks def test_request_without_meta(self): - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "-c", - "parse_request_without_meta", - "--nolinks", - self.url("/html"), - ] + _, _, stderr = self.proc( + "parse", + "--spider", + self.spider_name, + "-c", + "parse_request_without_meta", + "--nolinks", + self.mockserver.url("/html"), ) - assert "DEBUG: It Works!" in _textmode(stderr) + assert "DEBUG: It Works!" in stderr - @inlineCallbacks def test_pipelines(self): - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "--pipelines", - "-c", - "parse", - "--verbose", - self.url("/html"), - ] + _, _, stderr = self.proc( + "parse", + "--spider", + self.spider_name, + "--pipelines", + "-c", + "parse", + "--verbose", + self.mockserver.url("/html"), ) - assert "INFO: It Works!" in _textmode(stderr) + assert "INFO: It Works!" in stderr - @inlineCallbacks def test_async_def_asyncio_parse_items_list(self): - status, out, stderr = yield self.execute( - [ - "--spider", - "asyncdef_asyncio_return", - "-c", - "parse", - self.url("/html"), - ] + _, out, stderr = self.proc( + "parse", + "--spider", + "asyncdef_asyncio_return", + "-c", + "parse", + self.mockserver.url("/html"), ) - assert "INFO: Got response 200" in _textmode(stderr) - assert "{'id': 1}" in _textmode(out) - assert "{'id': 2}" in _textmode(out) + assert "INFO: Got response 200" in stderr + assert "{'id': 1}" in out + assert "{'id': 2}" in out - @inlineCallbacks def test_async_def_asyncio_parse_items_single_element(self): - status, out, stderr = yield self.execute( - [ - "--spider", - "asyncdef_asyncio_return_single_element", - "-c", - "parse", - self.url("/html"), - ] + _, out, stderr = self.proc( + "parse", + "--spider", + "asyncdef_asyncio_return_single_element", + "-c", + "parse", + self.mockserver.url("/html"), ) - assert "INFO: Got response 200" in _textmode(stderr) - assert "{'foo': 42}" in _textmode(out) + assert "INFO: Got response 200" in stderr + assert "{'foo': 42}" in out - @inlineCallbacks def test_async_def_asyncgen_parse_loop(self): - status, out, stderr = yield self.execute( - [ - "--spider", - "asyncdef_asyncio_gen_loop", - "-c", - "parse", - self.url("/html"), - ] + _, out, stderr = self.proc( + "parse", + "--spider", + "asyncdef_asyncio_gen_loop", + "-c", + "parse", + self.mockserver.url("/html"), ) - assert "INFO: Got response 200" in _textmode(stderr) + assert "INFO: Got response 200" in stderr for i in range(10): - assert f"{{'foo': {i}}}" in _textmode(out) + assert f"{{'foo': {i}}}" in out - @inlineCallbacks def test_async_def_asyncgen_parse_exc(self): - status, out, stderr = yield self.execute( - [ - "--spider", - "asyncdef_asyncio_gen_exc", - "-c", - "parse", - self.url("/html"), - ] + _, out, stderr = self.proc( + "parse", + "--spider", + "asyncdef_asyncio_gen_exc", + "-c", + "parse", + self.mockserver.url("/html"), ) - assert "ValueError" in _textmode(stderr) + assert "ValueError" in stderr for i in range(7): - assert f"{{'foo': {i}}}" in _textmode(out) + assert f"{{'foo': {i}}}" in out - @inlineCallbacks def test_async_def_asyncio_parse(self): - _, _, stderr = yield self.execute( - [ - "--spider", - "asyncdef_asyncio", - "-c", - "parse", - self.url("/html"), - ] + _, _, stderr = self.proc( + "parse", + "--spider", + "asyncdef_asyncio", + "-c", + "parse", + self.mockserver.url("/html"), ) - assert "DEBUG: Got response 200" in _textmode(stderr) + assert "DEBUG: Got response 200" in stderr - @inlineCallbacks def test_parse_items(self): - status, out, stderr = yield self.execute( - ["--spider", self.spider_name, "-c", "parse", self.url("/html")] + _, out, _ = self.proc( + "parse", + "--spider", + self.spider_name, + "-c", + "parse", + self.mockserver.url("/html"), ) - assert "[{}, {'foo': 'bar'}]" in _textmode(out) + assert "[{}, {'foo': 'bar'}]" in out - @inlineCallbacks def test_parse_items_no_callback_passed(self): - status, out, stderr = yield self.execute( - ["--spider", self.spider_name, self.url("/html")] + _, out, _ = self.proc( + "parse", "--spider", self.spider_name, self.mockserver.url("/html") ) - assert "[{}, {'foo': 'bar'}]" in _textmode(out) + assert "[{}, {'foo': 'bar'}]" in out - @inlineCallbacks def test_wrong_callback_passed(self): - status, out, stderr = yield self.execute( - ["--spider", self.spider_name, "-c", "dummy", self.url("/html")] + _, out, stderr = self.proc( + "parse", + "--spider", + self.spider_name, + "-c", + "dummy", + self.mockserver.url("/html"), ) - assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) - assert "Cannot find callback" in _textmode(stderr) + assert re.search(r"# Scraped Items -+\r?\n\[\]", out) + assert "Cannot find callback" in stderr - @inlineCallbacks def test_crawlspider_matching_rule_callback_set(self): """If a rule matches the URL, use it's defined callback.""" - status, out, stderr = yield self.execute( - ["--spider", "goodcrawl" + self.spider_name, "-r", self.url("/html")] + _, out, _ = self.proc( + "parse", + "--spider", + "goodcrawl" + self.spider_name, + "-r", + self.mockserver.url("/html"), ) - assert "[{}, {'foo': 'bar'}]" in _textmode(out) + assert "[{}, {'foo': 'bar'}]" in out - @inlineCallbacks def test_crawlspider_matching_rule_default_callback(self): """If a rule match but it has no callback set, use the 'parse' callback.""" - status, out, stderr = yield self.execute( - ["--spider", "goodcrawl" + self.spider_name, "-r", self.url("/text")] + _, out, _ = self.proc( + "parse", + "--spider", + "goodcrawl" + self.spider_name, + "-r", + self.mockserver.url("/text"), ) - assert "[{}, {'nomatch': 'default'}]" in _textmode(out) + assert "[{}, {'nomatch': 'default'}]" in out - @inlineCallbacks def test_spider_with_no_rules_attribute(self): """Using -r with a spider with no rule should not produce items.""" - status, out, stderr = yield self.execute( - ["--spider", self.spider_name, "-r", self.url("/html")] + _, out, stderr = self.proc( + "parse", "--spider", self.spider_name, "-r", self.mockserver.url("/html") ) - assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) - assert "No CrawlSpider rules found" in _textmode(stderr) + assert re.search(r"# Scraped Items -+\r?\n\[\]", out) + assert "No CrawlSpider rules found" in stderr - @inlineCallbacks def test_crawlspider_missing_callback(self): - status, out, stderr = yield self.execute( - ["--spider", "badcrawl" + self.spider_name, "-r", self.url("/html")] + _, out, _ = self.proc( + "parse", + "--spider", + "badcrawl" + self.spider_name, + "-r", + self.mockserver.url("/html"), ) - assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) + assert re.search(r"# Scraped Items -+\r?\n\[\]", out) - @inlineCallbacks def test_crawlspider_no_matching_rule(self): """The requested URL has no matching rule, so no items should be scraped""" - status, out, stderr = yield self.execute( - ["--spider", "badcrawl" + self.spider_name, "-r", self.url("/enc-gb18030")] + _, out, stderr = self.proc( + "parse", + "--spider", + "badcrawl" + self.spider_name, + "-r", + self.mockserver.url("/enc-gb18030"), ) - assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) - assert "Cannot find a rule that matches" in _textmode(stderr) + assert re.search(r"# Scraped Items -+\r?\n\[\]", out) + assert "Cannot find a rule that matches" in stderr - @inlineCallbacks def test_crawlspider_not_exists_with_not_matched_url(self): - status, out, stderr = yield self.execute([self.url("/invalid_url")]) - assert status == 0 + assert self.call("parse", self.mockserver.url("/invalid_url")) == 0 - @inlineCallbacks def test_output_flag(self): """Checks if a file was created successfully having correct format containing correct data in it. """ file_name = "data.json" file_path = Path(self.proj_path, file_name) - yield self.execute( - [ - "--spider", - self.spider_name, - "-c", - "parse", - "-o", - file_name, - self.url("/html"), - ] + self.proc( + "parse", + "--spider", + self.spider_name, + "-c", + "parse", + "-o", + file_name, + self.mockserver.url("/html"), ) assert file_path.exists() diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index 7f8d9fb61..c1a6d9b18 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -7,7 +7,6 @@ from contextlib import contextmanager from pathlib import Path from tempfile import TemporaryDirectory, mkdtemp from typing import TYPE_CHECKING -from unittest import skipIf import pytest @@ -58,7 +57,7 @@ class BadSpider(scrapy.Spider): return self.proc("runspider", fname, *args) def get_log(self, code, name=None, args=()): - p, stdout, stderr = self.runspider(code, name, args=args) + _, _, stderr = self.runspider(code, name, args=args) return stderr def test_runspider(self): @@ -288,7 +287,7 @@ class MySpider(scrapy.Spider): log = self.get_log(spider_code, args=args) assert "[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}" in log - @skipIf(platform.system() == "Windows", reason="Linux only") + @pytest.mark.skipif(platform.system() == "Windows", reason="Linux only") def test_absolute_path_linux(self): spider_code = """ import scrapy @@ -317,7 +316,7 @@ class MySpider(scrapy.Spider): in log ) - @skipIf(platform.system() != "Windows", reason="Windows only") + @pytest.mark.skipif(platform.system() != "Windows", reason="Windows only") def test_absolute_path_windows(self): spider_code = """ import scrapy @@ -370,18 +369,16 @@ class MySpider(scrapy.Spider): assert "The value of FOO is 42" in log +@pytest.mark.skipif( + platform.system() != "Windows", reason="Windows required for .pyw files" +) class TestWindowsRunSpiderCommand(TestRunSpiderCommand): spider_filename = "myspider.pyw" - def setUp(self): - if platform.system() != "Windows": - pytest.skip("Windows required for .pyw files") - return super().setUp() - def test_start_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") assert "start" in log assert "badspider.pyw" in log def test_runspider_unable_to_load(self): - pytest.skip("Already Tested in 'RunSpiderCommandTest' ") + pytest.skip("Already Tested in 'RunSpiderCommandTest'") diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index d9f17d76b..76c1eb663 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -5,140 +5,137 @@ from pathlib import Path import pytest from pexpect.popen_spawn import PopenSpawn -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest from scrapy.utils.reactor import _asyncio_reactor_path from tests import NON_EXISTING_RESOLVABLE, tests_datadir from tests.mockserver import MockServer -from tests.utils.testproc import ProcessTest -from tests.utils.testsite import SiteTest +from tests.test_commands import TestProjectBase -class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): - command = "shell" +class TestShellCommand(TestProjectBase): + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) - @inlineCallbacks def test_empty(self): - _, out, _ = yield self.execute(["-c", "item"]) - assert b"{}" in out + _, out, _ = self.proc("shell", "-c", "item") + assert "{}" in out - @inlineCallbacks def test_response_body(self): - _, out, _ = yield self.execute([self.url("/text"), "-c", "response.body"]) - assert b"Works" in out + _, out, _ = self.proc( + "shell", self.mockserver.url("/text"), "-c", "response.body" + ) + assert "Works" in out - @inlineCallbacks def test_response_type_text(self): - _, out, _ = yield self.execute([self.url("/text"), "-c", "type(response)"]) - assert b"TextResponse" in out + _, out, _ = self.proc( + "shell", self.mockserver.url("/text"), "-c", "type(response)" + ) + assert "TextResponse" in out - @inlineCallbacks def test_response_type_html(self): - _, out, _ = yield self.execute([self.url("/html"), "-c", "type(response)"]) - assert b"HtmlResponse" in out + _, out, _ = self.proc( + "shell", self.mockserver.url("/html"), "-c", "type(response)" + ) + assert "HtmlResponse" in out - @inlineCallbacks def test_response_selector_html(self): xpath = "response.xpath(\"//p[@class='one']/text()\").get()" - _, out, _ = yield self.execute([self.url("/html"), "-c", xpath]) - assert out.strip() == b"Works" + _, out, _ = self.proc("shell", self.mockserver.url("/html"), "-c", xpath) + assert out.strip() == "Works" - @inlineCallbacks def test_response_encoding_gb18030(self): - _, out, _ = yield self.execute( - [self.url("/enc-gb18030"), "-c", "response.encoding"] + _, out, _ = self.proc( + "shell", self.mockserver.url("/enc-gb18030"), "-c", "response.encoding" ) - assert out.strip() == b"gb18030" + assert out.strip() == "gb18030" - @inlineCallbacks def test_redirect(self): - _, out, _ = yield self.execute([self.url("/redirect"), "-c", "response.url"]) - assert out.strip().endswith(b"/redirected") + _, out, _ = self.proc( + "shell", self.mockserver.url("/redirect"), "-c", "response.url" + ) + assert out.strip().endswith("/redirected") - @inlineCallbacks def test_redirect_follow_302(self): - _, out, _ = yield self.execute( - [self.url("/redirect-no-meta-refresh"), "-c", "response.status"] + _, out, _ = self.proc( + "shell", + self.mockserver.url("/redirect-no-meta-refresh"), + "-c", + "response.status", ) - assert out.strip().endswith(b"200") + assert out.strip().endswith("200") - @inlineCallbacks def test_redirect_not_follow_302(self): - _, out, _ = yield self.execute( - [ - "--no-redirect", - self.url("/redirect-no-meta-refresh"), - "-c", - "response.status", - ] + _, out, _ = self.proc( + "shell", + "--no-redirect", + self.mockserver.url("/redirect-no-meta-refresh"), + "-c", + "response.status", ) - assert out.strip().endswith(b"302") + assert out.strip().endswith("302") - @inlineCallbacks def test_fetch_redirect_follow_302(self): """Test that calling ``fetch(url)`` follows HTTP redirects by default.""" - url = self.url("/redirect-no-meta-refresh") + url = self.mockserver.url("/redirect-no-meta-refresh") code = f"fetch('{url}')" - errcode, out, errout = yield self.execute(["-c", code]) - assert errcode == 0, out - assert b"Redirecting (302)" in errout - assert b"Crawled (200)" in errout + p, out, errout = self.proc("shell", "-c", code) + assert p.returncode == 0, out + assert "Redirecting (302)" in errout + assert "Crawled (200)" in errout - @inlineCallbacks def test_fetch_redirect_not_follow_302(self): """Test that calling ``fetch(url, redirect=False)`` disables automatic redirects.""" - url = self.url("/redirect-no-meta-refresh") + url = self.mockserver.url("/redirect-no-meta-refresh") code = f"fetch('{url}', redirect=False)" - errcode, out, errout = yield self.execute(["-c", code]) - assert errcode == 0, out - assert b"Crawled (302)" in errout + p, out, errout = self.proc("shell", "-c", code) + assert p.returncode == 0, out + assert "Crawled (302)" in errout - @inlineCallbacks def test_request_replace(self): - url = self.url("/text") + url = self.mockserver.url("/text") code = f"fetch('{url}') or fetch(response.request.replace(method='POST'))" - errcode, out, _ = yield self.execute(["-c", code]) - assert errcode == 0, out + p, out, _ = self.proc("shell", "-c", code) + assert p.returncode == 0, out - @inlineCallbacks def test_scrapy_import(self): - url = self.url("/text") + url = self.mockserver.url("/text") code = f"fetch(scrapy.Request('{url}'))" - errcode, out, _ = yield self.execute(["-c", code]) - assert errcode == 0, out + p, out, _ = self.proc("shell", "-c", code) + assert p.returncode == 0, out - @inlineCallbacks def test_local_file(self): filepath = Path(tests_datadir, "test_site", "index.html") - _, out, _ = yield self.execute([str(filepath), "-c", "item"]) - assert b"{}" in out + _, out, _ = self.proc("shell", str(filepath), "-c", "item") + assert "{}" in out - @inlineCallbacks def test_local_nofile(self): filepath = "file:///tests/sample_data/test_site/nothinghere.html" - errcode, out, err = yield self.execute( - [filepath, "-c", "item"], check_code=False - ) - assert errcode == 1, out or err - assert b"No such file or directory" in err + p, out, err = self.proc("shell", filepath, "-c", "item") + assert p.returncode == 1, out or err + assert "No such file or directory" in err - @inlineCallbacks def test_dns_failures(self): if NON_EXISTING_RESOLVABLE: pytest.skip("Non-existing hosts are resolvable") url = "www.somedomainthatdoesntexi.st" - errcode, out, err = yield self.execute([url, "-c", "item"], check_code=False) - assert errcode == 1, out or err - assert b"DNS lookup failed" in err + p, out, err = self.proc("shell", url, "-c", "item") + assert p.returncode == 1, out or err + assert "DNS lookup failed" in err - @inlineCallbacks def test_shell_fetch_async(self): - url = self.url("/html") + url = self.mockserver.url("/html") code = f"fetch('{url}')" - args = ["-c", code, "--set", f"TWISTED_REACTOR={_asyncio_reactor_path}"] - _, _, err = yield self.execute(args, check_code=True) - assert b"RuntimeError: There is no current event loop in thread" not in err + p, _, err = self.proc( + "shell", "-c", code, "--set", f"TWISTED_REACTOR={_asyncio_reactor_path}" + ) + assert p.returncode == 0, err + assert "RuntimeError: There is no current event loop in thread" not in err class TestInteractiveShell: diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py index 08bf9b0fd..988ad50b9 100644 --- a/tests/test_command_startproject.py +++ b/tests/test_command_startproject.py @@ -108,8 +108,8 @@ def get_permissions_dict( class TestStartprojectTemplates(TestProjectBase): maxDiff = None - def setUp(self): - super().setUp() + def setup_method(self): + super().setup_method() self.tmpl = str(Path(self.temp_path, "templates")) self.tmpl_proj = str(Path(self.tmpl, "project")) diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 87dfb16df..de58203fc 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -1,29 +1,15 @@ -import sys - -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest - import scrapy -from tests.utils.testproc import ProcessTest +from tests.test_commands import TestProjectBase -class TestVersionCommand(ProcessTest, unittest.TestCase): - command = "version" - - @inlineCallbacks +class TestVersionCommand(TestProjectBase): def test_output(self): - encoding = sys.stdout.encoding or "utf-8" - _, out, _ = yield self.execute([]) - assert out.strip().decode(encoding) == f"Scrapy {scrapy.__version__}" + _, out, _ = self.proc("version") + assert out.strip() == f"Scrapy {scrapy.__version__}" - @inlineCallbacks def test_verbose_output(self): - encoding = sys.stdout.encoding or "utf-8" - _, out, _ = yield self.execute(["-v"]) - headers = [ - line.partition(":")[0].strip() - for line in out.strip().decode(encoding).splitlines() - ] + _, out, _ = self.proc("version", "-v") + headers = [line.partition(":")[0].strip() for line in out.strip().splitlines()] assert headers == [ "Scrapy", "lxml", diff --git a/tests/test_commands.py b/tests/test_commands.py index 8ca5d51e5..851c92db4 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -10,11 +10,9 @@ from pathlib import Path from shutil import rmtree from tempfile import TemporaryFile, mkdtemp from threading import Timer -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from unittest import mock -from twisted.trial import unittest - import scrapy from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view @@ -61,28 +59,30 @@ class TestCommandSettings: ) -class TestProjectBase(unittest.TestCase): +class TestProjectBase: project_name = "testproject" - def setUp(self): + def setup_method(self): self.temp_path = mkdtemp() self.cwd = self.temp_path self.proj_path = Path(self.temp_path, self.project_name) self.proj_mod_path = self.proj_path / self.project_name self.env = get_testenv() - def tearDown(self): + def teardown_method(self): rmtree(self.temp_path) - def call(self, *new_args, **kwargs): + def call(self, *args: str, **popen_kwargs: Any) -> int: with TemporaryFile() as out: - args = (sys.executable, "-m", "scrapy.cmdline", *new_args) + args = (sys.executable, "-m", "scrapy.cmdline", *args) return subprocess.call( - args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **kwargs + args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **popen_kwargs ) - def proc(self, *new_args, **popen_kwargs): - args = (sys.executable, "-m", "scrapy.cmdline", *new_args) + def proc( + self, *args: str, **popen_kwargs: Any + ) -> tuple[subprocess.Popen[bytes], str, str]: + args = (sys.executable, "-m", "scrapy.cmdline", *args) p = subprocess.Popen( args, cwd=popen_kwargs.pop("cwd", self.cwd), @@ -118,10 +118,10 @@ class TestProjectBase(unittest.TestCase): class TestCommandBase(TestProjectBase): - def setUp(self): - super().setUp() + def setup_method(self): + super().setup_method() self.call("startproject", self.project_name) - self.cwd = Path(self.temp_path, self.project_name) + self.cwd = self.proj_path self.env["SCRAPY_SETTINGS_MODULE"] = f"{self.project_name}.settings" @@ -136,8 +136,8 @@ class TestCommandCrawlerProcess(TestCommandBase): "Type of self.crawler_process: " ) - def setUp(self): - super().setUp() + def setup_method(self): + super().setup_method() (self.cwd / self.project_name / "commands").mkdir(exist_ok=True) (self.cwd / self.project_name / "commands" / "__init__.py").touch() (self.cwd / self.project_name / "commands" / f"{self.name}.py").write_text(""" @@ -363,6 +363,19 @@ Unknown command: abc assert out.getvalue().strip() == message.strip() +class TestProjectSubdir(TestProjectBase): + """Test that commands work in a subdirectory of the project.""" + + def setup_method(self): + super().setup_method() + self.call("startproject", self.project_name) + self.cwd = self.proj_path / "subdir" + self.cwd.mkdir(exist_ok=True) + + def test_list(self): + assert self.call("list") == 0 + + class TestBenchCommand(TestCommandBase): def test_run(self): _, _, log = self.proc( @@ -389,8 +402,8 @@ class TestViewCommand(TestCommandBase): class TestHelpMessage(TestCommandBase): - def setUp(self): - super().setUp() + def setup_method(self): + super().setup_method() self.commands = [ "parse", "startproject", diff --git a/tests/utils/testproc.py b/tests/utils/testproc.py deleted file mode 100644 index 85d7c940f..000000000 --- a/tests/utils/testproc.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import annotations - -import os -import sys -from typing import TYPE_CHECKING, cast - -from twisted.internet.defer import Deferred -from twisted.internet.error import ProcessTerminated -from twisted.internet.protocol import ProcessProtocol - -if TYPE_CHECKING: - from collections.abc import Iterable - - from twisted.python.failure import Failure - - -class ProcessTest: - command: str | None = None - prefix = [sys.executable, "-m", "scrapy.cmdline"] - cwd = os.getcwd() # trial chdirs to temp dir # noqa: PTH109 - - def execute( - self, - args: Iterable[str], - check_code: bool = True, - settings: str | None = None, - ) -> Deferred[TestProcessProtocol]: - from twisted.internet import reactor - - env = os.environ.copy() - if settings is not None: - env["SCRAPY_SETTINGS_MODULE"] = settings - assert self.command - cmd = [*self.prefix, self.command, *args] - pp = TestProcessProtocol() - pp.deferred.addCallback(self._process_finished, cmd, check_code) - reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) - return pp.deferred - - def _process_finished( - self, pp: TestProcessProtocol, cmd: list[str], check_code: bool - ) -> tuple[int, bytes, bytes]: - if pp.exitcode and check_code: - msg = f"process {cmd} exit with code {pp.exitcode}" - msg += f"\n>>> stdout <<<\n{pp.out.decode()}" - msg += "\n" - msg += f"\n>>> stderr <<<\n{pp.err.decode()}" - raise RuntimeError(msg) - return cast(int, pp.exitcode), pp.out, pp.err - - -class TestProcessProtocol(ProcessProtocol): - def __init__(self) -> None: - self.deferred: Deferred[TestProcessProtocol] = Deferred() - self.out: bytes = b"" - self.err: bytes = b"" - self.exitcode: int | None = None - - def outReceived(self, data: bytes) -> None: - self.out += data - - def errReceived(self, data: bytes) -> None: - self.err += data - - def processEnded(self, status: Failure) -> None: - self.exitcode = cast(ProcessTerminated, status.value).exitCode - self.deferred.callback(self) diff --git a/tests/utils/testsite.py b/tests/utils/testsite.py deleted file mode 100644 index 473738773..000000000 --- a/tests/utils/testsite.py +++ /dev/null @@ -1,47 +0,0 @@ -from urllib.parse import urljoin - -from twisted.web import resource, server, static, util - - -class SiteTest: - def setUp(self): - from twisted.internet import reactor - - super().setUp() - self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") - self.baseurl = f"http://localhost:{self.site.getHost().port}/" - - def tearDown(self): - super().tearDown() - self.site.stopListening() - - def url(self, path: str) -> str: - return urljoin(self.baseurl, path) - - -class NoMetaRefreshRedirect(util.Redirect): - def render(self, request: server.Request) -> bytes: - content = util.Redirect.render(self, request) - return content.replace( - b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"' - ) - - -def test_site(): - r = resource.Resource() - r.putChild(b"text", static.Data(b"Works", "text/plain")) - r.putChild( - b"html", - static.Data( - b"

Works

World

", - "text/html", - ), - ) - r.putChild( - b"enc-gb18030", - static.Data(b"

gb18030 encoding

", "text/html; charset=gb18030"), - ) - r.putChild(b"redirect", util.Redirect(b"/redirected")) - r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")) - r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) - return server.Site(r) diff --git a/tox.ini b/tox.ini index 5680d98d1..f28467ec1 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,6 @@ deps = pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures - pywin32; sys_platform == "win32" Twisted < 25.5.0 # https://github.com/twisted/twisted/issues/12467 [testenv] From daa1a7d0b6549f901a002bf4c8bb7c4aed23e068 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 14 Jun 2025 14:01:20 +0500 Subject: [PATCH 169/491] Remove the chdir fixture, re-enable fancy pytest asserts (#6888) * Remove the chdir fixture. * Re-enable fancy pytest asserts. * Remove doc files from pytest ignores. * Restore docs/_ext in test collection ignores. * Skip a doctest that fails on Windows. * Fix tests that were writing to the current dir. --- conftest.py | 8 +--- docs/news.rst | 4 ++ docs/topics/dynamic-content.rst | 9 +++++ docs/topics/items.rst | 4 ++ docs/topics/leaks.rst | 7 ++++ docs/topics/loaders.rst | 13 ++++++ docs/topics/selectors.rst | 3 +- docs/topics/shell.rst | 8 ++++ docs/topics/stats.rst | 4 ++ docs/topics/telnetconsole.rst | 4 ++ pyproject.toml | 16 -------- tests/test_feedexport.py | 32 ++++++++------- tests/test_pipeline_files.py | 11 +++--- tests/test_squeues_request.py | 70 ++++++++++++++------------------- 14 files changed, 110 insertions(+), 83 deletions(-) diff --git a/conftest.py b/conftest.py index ed7d14166..f952127b9 100644 --- a/conftest.py +++ b/conftest.py @@ -12,6 +12,8 @@ def _py_files(folder): collect_ignore = [ + # may need extra deps + "docs/_ext", # not a test, but looks like a test "scrapy/utils/testproc.py", "scrapy/utils/testsite.py", @@ -46,12 +48,6 @@ if not H2_ENABLED: ) -@pytest.fixture -def chdir(tmpdir): - """Change to pytest-provided temporary directory""" - tmpdir.chdir() - - def pytest_addoption(parser): parser.addoption( "--reactor", diff --git a/docs/news.rst b/docs/news.rst index 36d229760..05ad611ef 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -4454,6 +4454,8 @@ Highlights: Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. skip: start + * Python 3.4 is no longer supported, and some of the minimum requirements of Scrapy have also changed: @@ -4494,6 +4496,8 @@ Backward-incompatible changes (:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`, :issue:`3998`, :issue:`4036`) +.. skip: end + See also :ref:`1.8-deprecation-removals` below. diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 65270433f..6c57a88f1 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -111,6 +111,8 @@ you may use `curl2scrapy `_. Handling different response formats =================================== +.. skip: start + Once you have a response with the desired data, how you extract the desired data from it depends on the type of response: @@ -157,11 +159,15 @@ data from it depends on the type of response: Otherwise, you might need to convert the SVG code into a raster image, and :ref:`handle that raster image `. +.. skip: end + .. _topics-parsing-javascript: Parsing JavaScript code ======================= +.. skip: start + If the desired data is hardcoded in JavaScript, you first need to get the JavaScript code: @@ -220,6 +226,8 @@ data from it: >>> selector.css('var[name="data"]').get() 'value' +.. skip: end + .. _topics-headless-browsing: Using a headless browser @@ -242,6 +250,7 @@ it is possible to integrate ``asyncio``-based libraries which handle headless br One such library is `playwright-python`_ (an official Python port of `playwright`_). The following is a simple snippet to illustrate its usage within a Scrapy spider: +.. skip: next .. code-block:: python import scrapy diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 0365c95b3..3588d033e 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -214,6 +214,8 @@ the :attr:`~scrapy.Item.fields` attribute. Working with Item objects ------------------------- +.. skip: start + Here are some examples of common tasks performed with items, using the ``Product`` item :ref:`declared above `. You will notice the API is very similar to the :class:`dict` API. @@ -375,6 +377,8 @@ appending more values, or changing existing values, like this: That adds (or replaces) the ``serializer`` metadata key for the ``name`` field, keeping all the previously existing metadata values. +.. skip: end + .. _supporting-item-types: diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index cd8914644..bbe1f3dd4 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -60,6 +60,8 @@ in control. Debugging memory leaks with ``trackref`` ======================================== +.. skip: start + :mod:`trackref` is a module provided by Scrapy to debug the most common cases of memory leaks. It basically tracks the references to all live Request, Response, Item, Spider and Selector objects. @@ -203,6 +205,8 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module. ``None`` if none is found. Use :func:`print_live_refs` first to get a list of all tracked live objects per class name. +.. skip: end + .. _topics-leaks-muppy: Debugging memory leaks with muppy @@ -226,6 +230,7 @@ If you use ``pip``, you can install muppy with the following command:: Here's an example to view all Python objects available in the heap using muppy: +.. skip: start .. code-block:: pycon >>> from pympler import muppy @@ -253,6 +258,8 @@ the heap using muppy: `, using the :ref:`Product item ` declared in the :ref:`Items chapter `: +.. skip: next .. code-block:: python from scrapy.loader import ItemLoader @@ -130,6 +131,7 @@ assigned to the item. Let's see an example to illustrate how the input and output processors are called for a particular field (the same applies for any other field): +.. skip: next .. code-block:: python l = ItemLoader(Product(), some_selector) @@ -250,6 +252,7 @@ metadata. Here is an example: ) +.. skip: start .. code-block:: pycon >>> from scrapy.loader import ItemLoader @@ -259,6 +262,8 @@ metadata. Here is an example: >>> il.load_item() {'name': 'Welcome to my website', 'price': '1000'} +.. skip: end + The precedence order, for both input and output processors, is as follows: 1. Item Loader field-specific attributes: ``field_in`` and ``field_out`` (most @@ -294,6 +299,8 @@ the Item Loader that it's able to receive an Item Loader context, so the Item Loader passes the currently active context when calling it, and the processor function (``parse_length`` in this case) can thus use them. +.. skip: start + There are several ways to modify Item Loader context values: 1. By modifying the currently active Item Loader context @@ -320,6 +327,8 @@ There are several ways to modify Item Loader context values: class ProductLoader(ItemLoader): length_out = MapCompose(parse_length, unit="cm") +.. skip: end + ItemLoader objects ================== @@ -350,6 +359,7 @@ that you wish to extract. Example: +.. skip: next .. code-block:: python loader = ItemLoader(item=Item()) @@ -364,6 +374,7 @@ the footer selector. Example: +.. skip: next .. code-block:: python loader = ItemLoader(item=Item()) @@ -401,6 +412,7 @@ those dashes in the final product names. Here's how you can remove those dashes by reusing and extending the default Product Item Loader (``ProductLoader``): +.. skip: next .. code-block:: python from itemloaders.processors import MapCompose @@ -418,6 +430,7 @@ Another case where extending Item Loaders can be very helpful is when you have multiple source formats, for example XML and HTML. In the XML version you may want to remove ``CDATA`` occurrences. Here's an example of how to do it: +.. skip: next .. code-block:: python from itemloaders.processors import MapCompose diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index dbef07b73..40a85201a 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -308,6 +308,7 @@ Examples: * ``*::text`` selects all descendant text nodes of the current selector context: +..skip: next .. code-block:: pycon >>> response.css("#images *::text").getall() @@ -878,7 +879,7 @@ Example selecting links in list item with a "class" attribute ending with a digi >>> sel = Selector(text=doc, type="html") >>> sel.xpath("//li//@href").getall() ['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html'] - >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall() + >>> sel.xpath(r'//li[re:test(@class, "item-\d$")]//@href').getall() ['link1.html', 'link2.html', 'link4.html', 'link5.html'] .. warning:: C library ``libxslt`` doesn't natively support EXSLT regular diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 4898843e4..85a08cebd 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -142,6 +142,8 @@ Those objects are: Example of shell session ======================== +.. skip: start + Here's an example of a typical shell session where we start by scraping the https://scrapy.org page, and then proceed to scrape the https://old.reddit.com/ page. Finally, we modify the (Reddit) request method to POST and re-fetch it @@ -232,6 +234,8 @@ After that, we can start playing with the objects: 'X-Ua-Compatible': ['IE=edge'], 'X-Xss-Protection': ['1; mode=block']} +.. skip: end + .. _topics-shell-inspect-response: @@ -268,6 +272,8 @@ Here's an example of how you would call it from your spider: # Rest of parsing code. +.. skip: start + When you run the spider, you will get something similar to this:: 2014-01-23 17:48:31-0400 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) @@ -301,6 +307,8 @@ crawling:: 2014-01-23 17:50:03-0400 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) ... +.. skip: end + Note that you can't use the ``fetch`` shortcut here since the Scrapy engine is blocked by the shell. However, after you leave the shell, the spider will continue crawling where it stopped, as shown above. diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 9572a3785..e34999b58 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -42,6 +42,8 @@ attribute. Here is an example of an extension that access stats: def from_crawler(cls, crawler): return cls(crawler.stats) +.. skip: start + Set stat value: .. code-block:: python @@ -80,6 +82,8 @@ Get all stats: >>> stats.get_stats() {'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} +.. skip: end + Available Stats Collectors ========================== diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 3e9bbe56e..ae9cb634c 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -97,6 +97,8 @@ convenience: Telnet console usage examples ============================= +.. skip: start + Here are some example tasks you can do with the telnet console: View engine status @@ -146,6 +148,8 @@ To stop:: >>> engine.stop() Connection closed by foreign host. +.. skip: end + Telnet Console signals ====================== diff --git a/pyproject.toml b/pyproject.toml index 0742991db..bc809a7b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -224,23 +224,7 @@ disable = [ [tool.pytest.ini_options] xfail_strict = true -usefixtures = "chdir" python_files = ["test_*.py", "test_*/__init__.py"] -addopts = [ - "--assert=plain", - "--ignore=docs/_ext", - "--ignore=docs/conf.py", - "--ignore=docs/news.rst", - "--ignore=docs/topics/dynamic-content.rst", - "--ignore=docs/topics/items.rst", - "--ignore=docs/topics/leaks.rst", - "--ignore=docs/topics/loaders.rst", - "--ignore=docs/topics/selectors.rst", - "--ignore=docs/topics/shell.rst", - "--ignore=docs/topics/stats.rst", - "--ignore=docs/topics/telnetconsole.rst", - "--ignore=docs/utils", -] markers = [ "only_asyncio: marks tests as only enabled when --reactor=asyncio is passed", "only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed", diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 01797fd20..f8f3eb22a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -5,6 +5,7 @@ import csv import gzip import json import lzma +import os import random import shutil import string @@ -107,8 +108,13 @@ class TestFileFeedStorage: self._assert_stores(FileFeedStorage(str(path)), path) def test_store_direct_path_relative(self, tmp_path): - path = (tmp_path / "foo" / "bar").relative_to(Path.cwd()) - self._assert_stores(FileFeedStorage(str(path)), path) + old_cwd = Path.cwd() + try: + os.chdir(tmp_path) + path = Path("foo", "bar") + self._assert_stores(FileFeedStorage(str(path)), path) + finally: + os.chdir(old_cwd) def test_interface(self, tmp_path): path = tmp_path / "file.txt" @@ -236,24 +242,22 @@ class TestBlockingFeedStorage: def test_default_temp_dir(self): b = BlockingFeedStorage() - tmp = b.open(self.get_test_spider()) - tmp_path = Path(tmp.name).parent - assert str(tmp_path) == tempfile.gettempdir() + storage_file = b.open(self.get_test_spider()) + storage_dir = Path(storage_file.name).parent + assert str(storage_dir) == tempfile.gettempdir() - def test_temp_file(self): + def test_temp_file(self, tmp_path): b = BlockingFeedStorage() - tests_path = Path(__file__).resolve().parent - spider = self.get_test_spider({"FEED_TEMPDIR": str(tests_path)}) - tmp = b.open(spider) - tmp_path = Path(tmp.name).parent - assert tmp_path == tests_path + spider = self.get_test_spider({"FEED_TEMPDIR": str(tmp_path)}) + storage_file = b.open(spider) + storage_dir = Path(storage_file.name).parent + assert storage_dir == tmp_path - def test_invalid_folder(self): + def test_invalid_folder(self, tmp_path): b = BlockingFeedStorage() - tests_path = Path(__file__).resolve().parent - invalid_path = tests_path / "invalid_path" + invalid_path = tmp_path / "invalid_path" spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)}) with pytest.raises(OSError, match="Not a Directory:"): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index b4eae108f..808fde23d 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -524,21 +524,20 @@ class TestFilesPipelineCustomSettings: expected_value = settings.get(settings_attr) assert getattr(pipeline_cls, pipe_inst_attr) == expected_value - def test_file_pipeline_using_pathlike_objects(self): + def test_file_pipeline_using_pathlike_objects(self, tmp_path): class CustomFilesPipelineWithPathLikeDir(FilesPipeline): def file_path(self, request, response=None, info=None, *, item=None): return Path("subdir") / Path(request.url).name pipeline = CustomFilesPipelineWithPathLikeDir.from_crawler( - get_crawler(None, {"FILES_STORE": Path("./Temp")}) + get_crawler(None, {"FILES_STORE": tmp_path}) ) request = Request("http://example.com/image01.jpg") assert pipeline.file_path(request) == Path("subdir/image01.jpg") - def test_files_store_constructor_with_pathlike_object(self): - path = Path("./FileDir") - fs_store = FSFilesStore(path) - assert fs_store.basedir == str(path) + def test_files_store_constructor_with_pathlike_object(self, tmp_path): + fs_store = FSFilesStore(tmp_path) + assert fs_store.basedir == str(tmp_path) @pytest.mark.requires_botocore diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 68bd6df68..8353ad73c 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -2,8 +2,7 @@ Queues that handle requests """ -import shutil -import tempfile +from pathlib import Path import pytest import queuelib @@ -23,30 +22,17 @@ from scrapy.utils.test import get_crawler class TestBaseQueue: def setup_method(self): - self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-") - self.qpath = self.tempfilename() - self.qdir = tempfile.mkdtemp() self.crawler = get_crawler(Spider) - def teardown_method(self): - shutil.rmtree(self.tmpdir) - - def tempfilename(self): - with tempfile.NamedTemporaryFile(dir=self.tmpdir) as nf: - return nf.name - - def mkdtemp(self): - return tempfile.mkdtemp(dir=self.tmpdir) - class RequestQueueTestMixin: - def queue(self): + def queue(self, base_path: Path): raise NotImplementedError - def test_one_element_with_peek(self): + def test_one_element_with_peek(self, tmp_path): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): pytest.skip("The queuelib queues do not define peek") - q = self.queue() + q = self.queue(tmp_path) assert len(q) == 0 assert q.peek() is None assert q.pop() is None @@ -60,10 +46,10 @@ class RequestQueueTestMixin: assert q.pop() is None q.close() - def test_one_element_without_peek(self): + def test_one_element_without_peek(self, tmp_path): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): pytest.skip("The queuelib queues define peek") - q = self.queue() + q = self.queue(tmp_path) assert len(q) == 0 assert q.pop() is None req = Request("http://www.example.com") @@ -81,10 +67,10 @@ class RequestQueueTestMixin: class FifoQueueMixin(RequestQueueTestMixin): - def test_fifo_with_peek(self): + def test_fifo_with_peek(self, tmp_path): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): pytest.skip("The queuelib queues do not define peek") - q = self.queue() + q = self.queue(tmp_path) assert len(q) == 0 assert q.peek() is None assert q.pop() is None @@ -108,10 +94,10 @@ class FifoQueueMixin(RequestQueueTestMixin): assert q.pop() is None q.close() - def test_fifo_without_peek(self): + def test_fifo_without_peek(self, tmp_path): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - pytest.skip("The queuelib queues do not define peek") - q = self.queue() + pytest.skip("The queuelib queues define peek") + q = self.queue(tmp_path) assert len(q) == 0 assert q.pop() is None req1 = Request("http://www.example.com/1") @@ -137,10 +123,10 @@ class FifoQueueMixin(RequestQueueTestMixin): class LifoQueueMixin(RequestQueueTestMixin): - def test_lifo_with_peek(self): + def test_lifo_with_peek(self, tmp_path): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): pytest.skip("The queuelib queues do not define peek") - q = self.queue() + q = self.queue(tmp_path) assert len(q) == 0 assert q.peek() is None assert q.pop() is None @@ -164,10 +150,10 @@ class LifoQueueMixin(RequestQueueTestMixin): assert q.pop() is None q.close() - def test_lifo_without_peek(self): + def test_lifo_without_peek(self, tmp_path): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - pytest.skip("The queuelib queues do not define peek") - q = self.queue() + pytest.skip("The queuelib queues define peek") + q = self.queue(tmp_path) assert len(q) == 0 assert q.pop() is None req1 = Request("http://www.example.com/1") @@ -193,34 +179,38 @@ class LifoQueueMixin(RequestQueueTestMixin): class TestPickleFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue): - def queue(self): - return PickleFifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/fifo") + def queue(self, base_path): + return PickleFifoDiskQueue.from_crawler( + crawler=self.crawler, key=str(base_path / "pickle" / "fifo") + ) class TestPickleLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue): - def queue(self): - return PickleLifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/lifo") + def queue(self, base_path): + return PickleLifoDiskQueue.from_crawler( + crawler=self.crawler, key=str(base_path / "pickle" / "lifo") + ) class TestMarshalFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue): - def queue(self): + def queue(self, base_path): return MarshalFifoDiskQueue.from_crawler( - crawler=self.crawler, key="marshal/fifo" + crawler=self.crawler, key=str(base_path / "marshal" / "fifo") ) class TestMarshalLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue): - def queue(self): + def queue(self, base_path): return MarshalLifoDiskQueue.from_crawler( - crawler=self.crawler, key="marshal/lifo" + crawler=self.crawler, key=str(base_path / "marshal" / "lifo") ) class TestFifoMemoryQueueRequest(FifoQueueMixin, TestBaseQueue): - def queue(self): + def queue(self, base_path): return FifoMemoryQueue.from_crawler(crawler=self.crawler) class TestLifoMemoryQueueRequest(LifoQueueMixin, TestBaseQueue): - def queue(self): + def queue(self, base_path): return LifoMemoryQueue.from_crawler(crawler=self.crawler) From 85aeda365db01939f70d0888593e7808380c8514 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 16 Jun 2025 09:28:06 +0200 Subject: [PATCH 170/491] Clean up setting getter defaults (#6892) --- scrapy/core/downloader/__init__.py | 2 +- scrapy/downloadermiddlewares/ajaxcrawl.py | 2 +- scrapy/downloadermiddlewares/robotstxt.py | 6 ++---- scrapy/extensions/feedexport.py | 2 +- scrapy/extensions/periodic_log.py | 2 +- scrapy/settings/default_settings.py | 4 ++++ scrapy/spiders/crawl.py | 4 +--- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 9293d7b78..4b5654826 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -124,7 +124,7 @@ class Downloader: ) self._slot_gc_loop.start(60) self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict( - "DOWNLOAD_SLOTS", {} + "DOWNLOAD_SLOTS" ) @inlineCallbacks diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index e7a8962a1..a23deaa45 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -43,7 +43,7 @@ class AjaxCrawlMiddleware: # middleware parses first 4k. 4k turns out to be insufficient # for this middleware, and parsing 100k could be slow. # We use something in between (32K) by default. - self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE", 32768) + self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE") @classmethod def from_crawler(cls, crawler: Crawler) -> Self: diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index aba455bdd..fbd737970 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -38,10 +38,8 @@ class RobotsTxtMiddleware: def __init__(self, crawler: Crawler): if not crawler.settings.getbool("ROBOTSTXT_OBEY"): raise NotConfigured - self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy") - self._robotstxt_useragent: str | None = crawler.settings.get( - "ROBOTSTXT_USER_AGENT", None - ) + self._default_useragent: str = crawler.settings["USER_AGENT"] + self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"] self.crawler: Crawler = crawler self._parsers: dict[str, RobotParser | Deferred[RobotParser | None] | None] = {} self._parserimpl: RobotParser = load_object( diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c39a9c92e..d9e9ea775 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -479,7 +479,7 @@ class FeedExporter: uri = self.settings["FEED_URI"] # handle pathlib.Path objects uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri() - feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")} + feed_options = {"format": self.settings["FEED_FORMAT"]} self.feeds[uri] = feed_complete_default_values_from_settings( feed_options, self.settings ) diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 9158482fa..98210990a 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -78,7 +78,7 @@ class PeriodicLog: ) ext_timing_enabled: bool = crawler.settings.getbool( - "PERIODIC_LOG_TIMING_ENABLED", False + "PERIODIC_LOG_TIMING_ENABLED" ) if not (ext_stats or ext_delta or ext_timing_enabled): raise NotConfigured diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 7cd470f11..b6f47f1c3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -20,6 +20,7 @@ from pathlib import Path ADDONS = {} AJAXCRAWL_ENABLED = False +AJAXCRAWL_MAXSIZE = 32768 ASYNCIO_EVENT_LOOP = None @@ -49,6 +50,8 @@ CONCURRENT_REQUESTS_PER_IP = 0 COOKIES_ENABLED = True COOKIES_DEBUG = False +CRAWLSPIDER_FOLLOW_LINKS = True + DEFAULT_DROPITEM_LOG_LEVEL = "WARNING" DEFAULT_ITEM_CLASS = "scrapy.item.Item" @@ -158,6 +161,7 @@ FEED_EXPORTERS_BASE = { "marshal": "scrapy.exporters.MarshalItemExporter", "pickle": "scrapy.exporters.PickleItemExporter", } +FEED_FORMAT = "jsonlines" FEED_STORE_EMPTY = True FEED_STORAGES = {} FEED_STORAGES_BASE = { diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index f44f70e40..98e7b23c0 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -213,7 +213,5 @@ class CrawlSpider(Spider): @classmethod def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = super().from_crawler(crawler, *args, **kwargs) - spider._follow_links = crawler.settings.getbool( - "CRAWLSPIDER_FOLLOW_LINKS", True - ) + spider._follow_links = crawler.settings.getbool("CRAWLSPIDER_FOLLOW_LINKS") return spider From 91b186cf1868038569ef05a334650eae2a74b5b1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 20:42:11 +0500 Subject: [PATCH 171/491] Use new pytest for new Twisted. (#6893) --- tox.ini | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index f28467ec1..85935b011 100644 --- a/tox.ini +++ b/tox.ini @@ -14,16 +14,16 @@ deps = pexpect >= 4.8.0 pyftpdlib >= 2.0.1 pygments - pytest != 8.2.* # https://github.com/pytest-dev/pytest/issues/12275 + pytest pytest-cov >= 4.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures - Twisted < 25.5.0 # https://github.com/twisted/twisted/issues/12467 [testenv] deps = {[test-requirements]deps} + pytest >= 8.4.1 # https://github.com/pytest-dev/pytest/pull/13502 # mitmproxy does not support PyPy mitmproxy; implementation_name != "pypy" @@ -96,6 +96,8 @@ commands = [pinned] basepython = python3.9 deps = + # pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0 + pytest==8.4.0 Protego==0.1.15 Twisted==21.7.0 cryptography==37.0.0 @@ -195,6 +197,7 @@ basepython = pypy3.10 deps = PyPyDispatcher==2.1.0 {[test-requirements]deps} + pytest==8.4.0 Protego==0.1.15 Twisted==21.7.0 cryptography==41.0.5 From d1575220efc605a4e617aee4d4d9948de56fb529 Mon Sep 17 00:00:00 2001 From: Thalison Fernandes Date: Mon, 23 Jun 2025 12:49:57 -0300 Subject: [PATCH 172/491] Add .venv folder to .gitignore (#6901) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0a3f0ac1c..e02c2241d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ docs/build *egg-info .tox venv +.venv build dist .idea From 712e965dbd2d58bcfdcab13e289d1e1d1cd0abae Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 20:56:48 +0500 Subject: [PATCH 173/491] Replace Black with Ruff in contributing.rst. (#6903) --- docs/contributing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 0172887d6..3976d34c2 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -251,10 +251,10 @@ Coding style Please follow these coding conventions when writing code for inclusion in Scrapy: -* We use `black `_ for code formatting. +* We use `Ruff `_ for code formatting. There is a hook in the pre-commit config that will automatically format your code before every commit. You can also - run black manually with ``tox -e pre-commit``. + run Ruff manually with ``tox -e pre-commit``. * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. From 0d86fb69dcfbc51383e2e6fb926b9f166fee4395 Mon Sep 17 00:00:00 2001 From: Thalison Fernandes Date: Mon, 23 Jun 2025 13:56:29 -0300 Subject: [PATCH 174/491] Fix FileFeedStorage handling of Windows paths without file:// scheme (#6897) --- scrapy/extensions/feedexport.py | 2 +- tests/test_feedexport.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index d9e9ea775..f7bf50a5c 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -185,7 +185,7 @@ class StdoutFeedStorage: @implementer(IFeedStorage) class FileFeedStorage: def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None): - self.path: str = file_uri_to_path(uri) + self.path: str = file_uri_to_path(uri) if uri.startswith("file://") else uri feed_options = feed_options or {} self.write_mode: OpenBinaryMode = ( "wb" if feed_options.get("overwrite", False) else "ab" diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f8f3eb22a..7073d5a35 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -155,6 +155,11 @@ class TestFileFeedStorage: finally: path.unlink() + def test_preserves_windows_path_without_file_scheme(self): + path = r"C:\Users\user\Desktop\test.txt" + storage = FileFeedStorage(path) + assert storage.path == path + class TestFTPFeedStorage(unittest.TestCase): def get_test_spider(self, settings=None): From 9d324ebd1303613adde27dd28a2233803f1bcf14 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 21:58:54 +0500 Subject: [PATCH 175/491] Add .vscode to .gitignore. (#6907) --- .gitignore | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index e02c2241d..4100bcd97 100644 --- a/.gitignore +++ b/.gitignore @@ -5,15 +5,16 @@ _trial_temp* dropin.cache docs/build *egg-info -.tox -venv -.venv -build -dist -.idea +.tox/ +venv/ +.venv/ +build/ +dist/ +.idea/ +.vscode/ htmlcov/ -.coverage .pytest_cache/ +.coverage .coverage.* coverage.* *.junit.xml @@ -27,4 +28,4 @@ test-output.* Thumbs.db # OSX miscellaneous -.DS_Store \ No newline at end of file +.DS_Store From 9149b6e7fc47743cdbcb5d3471a9650a2163b09b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 20:42:11 +0500 Subject: [PATCH 176/491] Use new pytest for new Twisted. (#6893) --- tox.ini | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 5680d98d1..d3dc7a3db 100644 --- a/tox.ini +++ b/tox.ini @@ -14,17 +14,17 @@ deps = pexpect >= 4.8.0 pyftpdlib >= 2.0.1 pygments - pytest != 8.2.* # https://github.com/pytest-dev/pytest/issues/12275 + pytest pytest-cov >= 4.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures pywin32; sys_platform == "win32" - Twisted < 25.5.0 # https://github.com/twisted/twisted/issues/12467 [testenv] deps = {[test-requirements]deps} + pytest >= 8.4.1 # https://github.com/pytest-dev/pytest/pull/13502 # mitmproxy does not support PyPy mitmproxy; implementation_name != "pypy" @@ -97,6 +97,8 @@ commands = [pinned] basepython = python3.9 deps = + # pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0 + pytest==8.4.0 Protego==0.1.15 Twisted==21.7.0 cryptography==37.0.0 @@ -196,6 +198,7 @@ basepython = pypy3.10 deps = PyPyDispatcher==2.1.0 {[test-requirements]deps} + pytest==8.4.0 Protego==0.1.15 Twisted==21.7.0 cryptography==41.0.5 From 020bfa7e5fd98c53db17e0c72db88d43d6fbf1f6 Mon Sep 17 00:00:00 2001 From: Thalison Fernandes Date: Mon, 23 Jun 2025 12:49:57 -0300 Subject: [PATCH 177/491] Add .venv folder to .gitignore (#6901) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0a3f0ac1c..e02c2241d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ docs/build *egg-info .tox venv +.venv build dist .idea From 843ad1afb1383a263b8c8da9004a899a71ea864d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 20:56:48 +0500 Subject: [PATCH 178/491] Replace Black with Ruff in contributing.rst. (#6903) --- docs/contributing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 0172887d6..3976d34c2 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -251,10 +251,10 @@ Coding style Please follow these coding conventions when writing code for inclusion in Scrapy: -* We use `black `_ for code formatting. +* We use `Ruff `_ for code formatting. There is a hook in the pre-commit config that will automatically format your code before every commit. You can also - run black manually with ``tox -e pre-commit``. + run Ruff manually with ``tox -e pre-commit``. * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. From 5e20b46e35c3affc5df7b5f63778e4fb924b28a8 Mon Sep 17 00:00:00 2001 From: Thalison Fernandes Date: Mon, 23 Jun 2025 13:56:29 -0300 Subject: [PATCH 179/491] Fix FileFeedStorage handling of Windows paths without file:// scheme (#6897) --- scrapy/extensions/feedexport.py | 2 +- tests/test_feedexport.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 8bcd4e40d..4ac546765 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -185,7 +185,7 @@ class StdoutFeedStorage: @implementer(IFeedStorage) class FileFeedStorage: def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None): - self.path: str = file_uri_to_path(uri) + self.path: str = file_uri_to_path(uri) if uri.startswith("file://") else uri feed_options = feed_options or {} self.write_mode: OpenBinaryMode = ( "wb" if feed_options.get("overwrite", False) else "ab" diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 44cd10ec3..7c6425f19 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -143,6 +143,11 @@ class TestFileFeedStorage(unittest.TestCase): finally: path.unlink() + def test_preserves_windows_path_without_file_scheme(self): + path = r"C:\Users\user\Desktop\test.txt" + storage = FileFeedStorage(path) + assert storage.path == path + class TestFTPFeedStorage(unittest.TestCase): def get_test_spider(self, settings=None): From b53faacfcd64f6717645bf4cd33159cef87cd5d9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 21:58:54 +0500 Subject: [PATCH 180/491] Add .vscode to .gitignore. (#6907) --- .gitignore | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index e02c2241d..4100bcd97 100644 --- a/.gitignore +++ b/.gitignore @@ -5,15 +5,16 @@ _trial_temp* dropin.cache docs/build *egg-info -.tox -venv -.venv -build -dist -.idea +.tox/ +venv/ +.venv/ +build/ +dist/ +.idea/ +.vscode/ htmlcov/ -.coverage .pytest_cache/ +.coverage .coverage.* coverage.* *.junit.xml @@ -27,4 +28,4 @@ test-output.* Thumbs.db # OSX miscellaneous -.DS_Store \ No newline at end of file +.DS_Store From 0d75355b41a84896d4ab1c19ed1b88f65206fb9f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 22:36:54 +0500 Subject: [PATCH 181/491] Handle exceptions in _start_request_processing(), cancel it on engine stop (#6900) --- scrapy/core/engine.py | 48 ++++++++++++++++++++++++++++++------------ scrapy/core/scraper.py | 1 - tests/test_engine.py | 23 +++++++++++++++++++- 3 files changed, 56 insertions(+), 16 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index fe635dc82..d6c171247 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -7,12 +7,13 @@ For more information see docs/topics/architecture.rst from __future__ import annotations +import asyncio import logging from time import time from traceback import format_exc from typing import TYPE_CHECKING, Any, cast -from twisted.internet.defer import Deferred, inlineCallbacks, succeed +from twisted.internet.defer import CancelledError, Deferred, inlineCallbacks, succeed from twisted.python.failure import Failure from scrapy import signals @@ -108,6 +109,8 @@ class ExecutionEngine: ) self.start_time: float | None = None self._start: AsyncIterator[Any] | None = None + self._closewait: Deferred[None] | None = None + self._start_request_processing_dfd: Deferred[None] | None = None downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) try: self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( @@ -139,9 +142,9 @@ class ExecutionEngine: self.start_time = time() await self.signals.send_catch_log_async(signal=signals.engine_started) self.running = True - self._closewait: Deferred[None] = Deferred() + self._closewait = Deferred() if _start_request_processing: - self._start_request_processing() + self._start_request_processing_dfd = self._start_request_processing() await maybe_deferred_to_future(self._closewait) def stop(self) -> Deferred[None]: @@ -150,12 +153,16 @@ class ExecutionEngine: @deferred_f_from_coro_f async def _finish_stopping_engine(_: Any) -> None: await self.signals.send_catch_log_async(signal=signals.engine_stopped) - self._closewait.callback(None) + if self._closewait: + self._closewait.callback(None) if not self.running: raise RuntimeError("Engine not running") self.running = False + if self._start_request_processing_dfd is not None: + self._start_request_processing_dfd.cancel() + self._start_request_processing_dfd = None dfd = ( self.close_spider(self.spider, reason="shutdown") if self.spider is not None @@ -217,17 +224,30 @@ class ExecutionEngine: # Starts the processing of scheduled requests, as well as a periodic # call to that processing method for scenarios where the scheduler # reports having pending requests but returns none. - assert self._slot is not None # typing - self._slot.nextcall.schedule() - self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) + try: + assert self._slot is not None # typing + self._slot.nextcall.schedule() + self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) - while self._start and self.spider: - await self._process_start_next() - if not self.needs_backout(): - # Give room for the outcome of self._process_start_next() to be - # processed before continuing with the next iteration. - self._slot.nextcall.schedule() - await self._slot.nextcall.wait() + while self._start and self.spider: + await self._process_start_next() + if not self.needs_backout(): + # Give room for the outcome of self._process_start_next() to be + # processed before continuing with the next iteration. + self._slot.nextcall.schedule() + await self._slot.nextcall.wait() + except (asyncio.exceptions.CancelledError, CancelledError): + # self.stop() has cancelled us, nothing to do + return + except Exception: + # an error happened, log it and stop the engine + self._start_request_processing_dfd = None + logger.error( + "Error while processing requests from start()", + exc_info=True, + extra={"spider": self.spider}, + ) + await maybe_deferred_to_future(self.stop()) def _start_scheduled_requests(self) -> None: if self._slot is None or self._slot.closing is not None or self.paused: diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 1f0d57c63..dc3a287b4 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -140,7 +140,6 @@ class Scraper: def _check_if_closing(self) -> None: assert self.slot is not None # typing - assert self.crawler.spider if self.slot.closing and self.slot.is_idle(): assert self.crawler.spider self.slot.closing.callback(self.crawler.spider) diff --git a/tests/test_engine.py b/tests/test_engine.py index e181a36cf..d9d25c240 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -25,6 +25,7 @@ import attr import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher +from testfixtures import LogCapture from twisted.internet import defer from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest @@ -451,12 +452,32 @@ class TestEngine(TestEngineBase): @inlineCallbacks def test_start_already_running_exception(self): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) - yield e.open_spider(MySpider(), []) + yield e.open_spider(MySpider()) e.start() with pytest.raises(RuntimeError, match="Engine already running"): yield e.start() yield e.stop() + @inlineCallbacks + def test_start_request_processing_exception(self): + class BadRequestFingerprinter: + def fingerprint(self, request): + raise ValueError # to make Scheduler.enqueue_request() fail + + class SimpleSpider(Spider): + name = "simple" + + async def start(self): + yield Request("data:,") + + crawler = get_crawler( + SimpleSpider, {"REQUEST_FINGERPRINTER_CLASS": BadRequestFingerprinter} + ) + with LogCapture() as log: + yield crawler.crawl() + assert "Error while processing requests from start()" in str(log) + assert "Spider closed (shutdown)" in str(log) + def test_short_timeout(self): args = ( sys.executable, From 7fbd56bc9baa33a4eb874f4337d740f5f25cfad3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 22:36:54 +0500 Subject: [PATCH 182/491] Handle exceptions in _start_request_processing(), cancel it on engine stop (#6900) --- scrapy/core/engine.py | 48 ++++++++++++++++++++++++++++++------------ scrapy/core/scraper.py | 1 - tests/test_engine.py | 23 +++++++++++++++++++- 3 files changed, 56 insertions(+), 16 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 0df9ad2b2..6c6d24d5c 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -7,12 +7,13 @@ For more information see docs/topics/architecture.rst from __future__ import annotations +import asyncio import logging from time import time from traceback import format_exc from typing import TYPE_CHECKING, Any, TypeVar, cast -from twisted.internet.defer import Deferred, inlineCallbacks, succeed +from twisted.internet.defer import CancelledError, Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall from twisted.python.failure import Failure @@ -102,6 +103,8 @@ class ExecutionEngine: ) self.start_time: float | None = None self._start: AsyncIterator[Any] | None = None + self._closewait: Deferred[None] | None = None + self._start_request_processing_dfd: Deferred[None] | None = None downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) try: self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( @@ -133,9 +136,9 @@ class ExecutionEngine: self.signals.send_catch_log_deferred(signal=signals.engine_started) ) self.running = True - self._closewait: Deferred[None] = Deferred() + self._closewait = Deferred() if _start_request_processing: - self._start_request_processing() + self._start_request_processing_dfd = self._start_request_processing() await maybe_deferred_to_future(self._closewait) def stop(self) -> Deferred[None]: @@ -146,12 +149,16 @@ class ExecutionEngine: await maybe_deferred_to_future( self.signals.send_catch_log_deferred(signal=signals.engine_stopped) ) - self._closewait.callback(None) + if self._closewait: + self._closewait.callback(None) if not self.running: raise RuntimeError("Engine not running") self.running = False + if self._start_request_processing_dfd is not None: + self._start_request_processing_dfd.cancel() + self._start_request_processing_dfd = None dfd = ( self.close_spider(self.spider, reason="shutdown") if self.spider is not None @@ -213,17 +220,30 @@ class ExecutionEngine: # Starts the processing of scheduled requests, as well as a periodic # call to that processing method for scenarios where the scheduler # reports having pending requests but returns none. - assert self._slot is not None # typing - self._slot.nextcall.schedule() - self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) + try: + assert self._slot is not None # typing + self._slot.nextcall.schedule() + self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) - while self._start and self.spider: - await self._process_start_next() - if not self.needs_backout(): - # Give room for the outcome of self._process_start_next() to be - # processed before continuing with the next iteration. - self._slot.nextcall.schedule() - await self._slot.nextcall.wait() + while self._start and self.spider: + await self._process_start_next() + if not self.needs_backout(): + # Give room for the outcome of self._process_start_next() to be + # processed before continuing with the next iteration. + self._slot.nextcall.schedule() + await self._slot.nextcall.wait() + except (asyncio.exceptions.CancelledError, CancelledError): + # self.stop() has cancelled us, nothing to do + return + except Exception: + # an error happened, log it and stop the engine + self._start_request_processing_dfd = None + logger.error( + "Error while processing requests from start()", + exc_info=True, + extra={"spider": self.spider}, + ) + await maybe_deferred_to_future(self.stop()) def _start_scheduled_requests(self) -> None: if self._slot is None or self._slot.closing is not None or self.paused: diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index ac720e03f..a2fe281e0 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -139,7 +139,6 @@ class Scraper: def _check_if_closing(self) -> None: assert self.slot is not None # typing - assert self.crawler.spider if self.slot.closing and self.slot.is_idle(): assert self.crawler.spider self.slot.closing.callback(self.crawler.spider) diff --git a/tests/test_engine.py b/tests/test_engine.py index b2e436425..a3c5243d5 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -25,6 +25,7 @@ import attr import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher +from testfixtures import LogCapture from twisted.internet import defer, reactor from twisted.trial import unittest from twisted.web import server, static, util @@ -448,7 +449,7 @@ class TestEngine(TestEngineBase): @defer.inlineCallbacks def test_start_already_running_exception(self): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) - yield e.open_spider(MySpider(), []) + yield e.open_spider(MySpider()) e.start() def cb(exc: BaseException) -> None: @@ -459,6 +460,26 @@ class TestEngine(TestEngineBase): finally: yield e.stop() + @defer.inlineCallbacks + def test_start_request_processing_exception(self): + class BadRequestFingerprinter: + def fingerprint(self, request): + raise ValueError # to make Scheduler.enqueue_request() fail + + class SimpleSpider(Spider): + name = "simple" + + async def start(self): + yield Request("data:,") + + crawler = get_crawler( + SimpleSpider, {"REQUEST_FINGERPRINTER_CLASS": BadRequestFingerprinter} + ) + with LogCapture() as log: + yield crawler.crawl() + assert "Error while processing requests from start()" in str(log) + assert "Spider closed (shutdown)" in str(log) + def test_short_timeout(self): args = ( sys.executable, From d70f8a3f14252715fbc9b9541364c5b5d142335f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 23:39:24 +0500 Subject: [PATCH 183/491] Refactoring of test_utils_*. (#6905) --- tests/test_command_startproject.py | 2 - tests/test_request_cb_kwargs.py | 2 - tests/test_toplevel.py | 42 ++- tests/test_urlparse_monkeypatches.py | 10 - tests/test_utils_conf.py | 11 +- tests/test_utils_console.py | 35 +- tests/test_utils_curl.py | 6 +- tests/test_utils_datatypes.py | 12 +- tests/test_utils_deprecate.py | 5 +- tests/test_utils_display.py | 146 +++---- tests/test_utils_gz.py | 78 ++-- tests/test_utils_httpobj.py | 27 +- tests/test_utils_iterators.py | 80 ++-- tests/test_utils_log.py | 185 +++++---- tests/test_utils_project.py | 33 +- tests/test_utils_python.py | 283 +++++++------- tests/test_utils_request.py | 100 ++--- tests/test_utils_response.py | 370 +++++++++--------- tests/test_utils_serialize.py | 30 +- tests/test_utils_sitemap.py | 421 ++++++++++----------- tests/test_utils_spider.py | 26 +- tests/test_utils_template.py | 27 +- tests/test_utils_trackref.py | 95 ++--- tests/test_utils_url.py | 543 +++++++++++---------------- 24 files changed, 1251 insertions(+), 1318 deletions(-) delete mode 100644 tests/test_urlparse_monkeypatches.py diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py index 988ad50b9..1edef0b4a 100644 --- a/tests/test_command_startproject.py +++ b/tests/test_command_startproject.py @@ -106,8 +106,6 @@ def get_permissions_dict( class TestStartprojectTemplates(TestProjectBase): - maxDiff = None - def setup_method(self): super().setup_method() self.tmpl = str(Path(self.temp_path, "templates")) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 9d2e5f997..1714bd4db 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -150,8 +150,6 @@ class KeywordArgumentsSpider(MockServerSpider): class TestCallbackKeywordArguments(TestCase): - maxDiff = None - @classmethod def setUpClass(cls): cls.mockserver = MockServer() diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py index a4f31096e..66a6f5318 100644 --- a/tests/test_toplevel.py +++ b/tests/test_toplevel.py @@ -1,31 +1,35 @@ import scrapy -class TestToplevel: - def test_version(self): - assert isinstance(scrapy.__version__, str) +def test_version(): + assert isinstance(scrapy.__version__, str) - def test_version_info(self): - assert isinstance(scrapy.version_info, tuple) - def test_request_shortcut(self): - from scrapy.http import FormRequest, Request +def test_version_info(): + assert isinstance(scrapy.version_info, tuple) - assert scrapy.Request is Request - assert scrapy.FormRequest is FormRequest - def test_spider_shortcut(self): - from scrapy.spiders import Spider +def test_request_shortcut(): + from scrapy.http import FormRequest, Request - assert scrapy.Spider is Spider + assert scrapy.Request is Request + assert scrapy.FormRequest is FormRequest - def test_selector_shortcut(self): - from scrapy.selector import Selector - assert scrapy.Selector is Selector +def test_spider_shortcut(): + from scrapy.spiders import Spider - def test_item_shortcut(self): - from scrapy.item import Field, Item + assert scrapy.Spider is Spider - assert scrapy.Item is Item - assert scrapy.Field is Field + +def test_selector_shortcut(): + from scrapy.selector import Selector + + assert scrapy.Selector is Selector + + +def test_item_shortcut(): + from scrapy.item import Field, Item + + assert scrapy.Item is Item + assert scrapy.Field is Field diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py deleted file mode 100644 index 0e1e89e81..000000000 --- a/tests/test_urlparse_monkeypatches.py +++ /dev/null @@ -1,10 +0,0 @@ -from urllib.parse import urlparse - - -class TestUrlparse: - def test_s3_url(self): - p = urlparse("s3://bucket/key/name?param=value") - assert p.scheme == "s3" - assert p.hostname == "bucket" - assert p.path == "/key/name" - assert p.query == "param=value" diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 26f158380..ed7dda18d 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -47,12 +47,11 @@ class TestBuildComponentList: assert build_component_list(d, convert=lambda x: x) == ["b", "c", "a"] -class TestUtilsConf: - def test_arglist_to_dict(self): - assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == { - "arg1": "val1", - "arg2": "val2", - } +def test_arglist_to_dict(): + assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == { + "arg1": "val1", + "arg2": "val2", + } class TestFeedExportConfig: diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index 6598bdce7..dc1d96f66 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -18,23 +18,24 @@ except ImportError: ipy = False -class TestUtilsConsole: - def test_get_shell_embed_func(self): - shell = get_shell_embed_func(["invalid"]) - assert shell is None +def test_get_shell_embed_func(): + shell = get_shell_embed_func(["invalid"]) + assert shell is None - shell = get_shell_embed_func(["invalid", "python"]) - assert callable(shell) - assert shell.__name__ == "_embed_standard_shell" + shell = get_shell_embed_func(["invalid", "python"]) + assert callable(shell) + assert shell.__name__ == "_embed_standard_shell" - @pytest.mark.skipif(not bpy, reason="bpython not available in testenv") - def test_get_shell_embed_func2(self): - shell = get_shell_embed_func(["bpython"]) - assert callable(shell) - assert shell.__name__ == "_embed_bpython_shell" - @pytest.mark.skipif(not ipy, reason="IPython not available in testenv") - def test_get_shell_embed_func3(self): - # default shell should be 'ipython' - shell = get_shell_embed_func() - assert shell.__name__ == "_embed_ipython_shell" +@pytest.mark.skipif(not bpy, reason="bpython not available in testenv") +def test_get_shell_embed_func_bpython(): + shell = get_shell_embed_func(["bpython"]) + assert callable(shell) + assert shell.__name__ == "_embed_bpython_shell" + + +@pytest.mark.skipif(not ipy, reason="IPython not available in testenv") +def test_get_shell_embed_func_ipython(): + # default shell should be 'ipython' + shell = get_shell_embed_func() + assert shell.__name__ == "_embed_ipython_shell" diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index e8dd88049..02362693a 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -1,4 +1,5 @@ import warnings +from typing import Any import pytest from w3lib.http import basic_auth_header @@ -8,9 +9,8 @@ from scrapy.utils.curl import curl_to_request_kwargs class TestCurlToRequestKwargs: - maxDiff = 5000 - - def _test_command(self, curl_command, expected_result): + @staticmethod + def _test_command(curl_command: str, expected_result: dict[str, Any]) -> None: result = curl_to_request_kwargs(curl_command) assert result == expected_result try: diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 75b6b0e99..352e49165 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,5 +1,6 @@ import copy import warnings +from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, MutableMapping import pytest @@ -16,7 +17,12 @@ from scrapy.utils.datatypes import ( from scrapy.utils.python import garbage_collect -class CaseInsensitiveDictBase: +class TestCaseInsensitiveDictBase(ABC): + @property + @abstractmethod + def dict_class(self) -> type[MutableMapping]: + raise NotImplementedError + def test_init_dict(self): seq = {"red": 1, "black": 3} d = self.dict_class(seq) @@ -199,7 +205,7 @@ class CaseInsensitiveDictBase: assert h1.get("header1") == h3.get("HEADER1") -class TestCaseInsensitiveDict(CaseInsensitiveDictBase): +class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase): dict_class = CaseInsensitiveDict def test_repr(self): @@ -216,7 +222,7 @@ class TestCaseInsensitiveDict(CaseInsensitiveDictBase): @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestCaselessDict(CaseInsensitiveDictBase): +class TestCaselessDict(TestCaseInsensitiveDictBase): dict_class = CaselessDict def test_deprecation_message(self): diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 662de0dc3..a88b5e008 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,6 +1,7 @@ import inspect import warnings from unittest import mock +from warnings import WarningMessage import pytest @@ -21,7 +22,9 @@ class NewName(SomeBaseClass): class TestWarnWhenSubclassed: - def _mywarnings(self, w, category=MyWarning): + def _mywarnings( + self, w: list[WarningMessage], category: type[Warning] = MyWarning + ) -> list[WarningMessage]: return [x for x in w if x.category is MyWarning] def test_no_warning_on_definition(self): diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index cea564653..20251ca59 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -3,88 +3,92 @@ from unittest import mock from scrapy.utils.display import pformat, pprint - -class TestDisplay: - object = {"a": 1} - colorized_strings = { +value = {"a": 1} +colorized_strings = { + ( ( - ( - "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'" - "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}" - ) - + suffix + "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'" + "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}" ) - for suffix in ( - # https://github.com/pygments/pygments/issues/2313 - "\n", # pygments ≤ 2.13 - "\x1b[37m\x1b[39;49;00m\n", # pygments ≥ 2.14 - ) - } - plain_string = "{'a': 1}" + + suffix + ) + for suffix in ( + # https://github.com/pygments/pygments/issues/2313 + "\n", # pygments ≤ 2.13 + "\x1b[37m\x1b[39;49;00m\n", # pygments ≥ 2.14 + ) +} +plain_string = "{'a': 1}" - @mock.patch("sys.platform", "linux") - @mock.patch("sys.stdout.isatty") - def test_pformat(self, isatty): - isatty.return_value = True - assert pformat(self.object) in self.colorized_strings - @mock.patch("sys.stdout.isatty") - def test_pformat_dont_colorize(self, isatty): - isatty.return_value = True - assert pformat(self.object, colorize=False) == self.plain_string +@mock.patch("sys.platform", "linux") +@mock.patch("sys.stdout.isatty") +def test_pformat(isatty): + isatty.return_value = True + assert pformat(value) in colorized_strings - def test_pformat_not_tty(self): - assert pformat(self.object) == self.plain_string - @mock.patch("sys.platform", "win32") - @mock.patch("platform.version") - @mock.patch("sys.stdout.isatty") - def test_pformat_old_windows(self, isatty, version): - isatty.return_value = True - version.return_value = "10.0.14392" - assert pformat(self.object) in self.colorized_strings +@mock.patch("sys.stdout.isatty") +def test_pformat_dont_colorize(isatty): + isatty.return_value = True + assert pformat(value, colorize=False) == plain_string - @mock.patch("sys.platform", "win32") - @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") - @mock.patch("platform.version") - @mock.patch("sys.stdout.isatty") - def test_pformat_windows_no_terminal_processing( - self, isatty, version, terminal_processing - ): - isatty.return_value = True - version.return_value = "10.0.14393" - terminal_processing.return_value = False - assert pformat(self.object) == self.plain_string - @mock.patch("sys.platform", "win32") - @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") - @mock.patch("platform.version") - @mock.patch("sys.stdout.isatty") - def test_pformat_windows(self, isatty, version, terminal_processing): - isatty.return_value = True - version.return_value = "10.0.14393" - terminal_processing.return_value = True - assert pformat(self.object) in self.colorized_strings +def test_pformat_not_tty(): + assert pformat(value) == plain_string - @mock.patch("sys.platform", "linux") - @mock.patch("sys.stdout.isatty") - def test_pformat_no_pygments(self, isatty): - isatty.return_value = True - import builtins +@mock.patch("sys.platform", "win32") +@mock.patch("platform.version") +@mock.patch("sys.stdout.isatty") +def test_pformat_old_windows(isatty, version): + isatty.return_value = True + version.return_value = "10.0.14392" + assert pformat(value) in colorized_strings - real_import = builtins.__import__ - def mock_import(name, globals, locals, fromlist, level): - if "pygments" in name: - raise ImportError - return real_import(name, globals, locals, fromlist, level) +@mock.patch("sys.platform", "win32") +@mock.patch("scrapy.utils.display._enable_windows_terminal_processing") +@mock.patch("platform.version") +@mock.patch("sys.stdout.isatty") +def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing): + isatty.return_value = True + version.return_value = "10.0.14393" + terminal_processing.return_value = False + assert pformat(value) == plain_string - builtins.__import__ = mock_import - assert pformat(self.object) == self.plain_string - builtins.__import__ = real_import - def test_pprint(self): - with mock.patch("sys.stdout", new=StringIO()) as mock_out: - pprint(self.object) - assert mock_out.getvalue() == "{'a': 1}\n" +@mock.patch("sys.platform", "win32") +@mock.patch("scrapy.utils.display._enable_windows_terminal_processing") +@mock.patch("platform.version") +@mock.patch("sys.stdout.isatty") +def test_pformat_windows(isatty, version, terminal_processing): + isatty.return_value = True + version.return_value = "10.0.14393" + terminal_processing.return_value = True + assert pformat(value) in colorized_strings + + +@mock.patch("sys.platform", "linux") +@mock.patch("sys.stdout.isatty") +def test_pformat_no_pygments(isatty): + isatty.return_value = True + + import builtins + + real_import = builtins.__import__ + + def mock_import(name, globals, locals, fromlist, level): + if "pygments" in name: + raise ImportError + return real_import(name, globals, locals, fromlist, level) + + builtins.__import__ = mock_import + assert pformat(value) == plain_string + builtins.__import__ = real_import + + +def test_pprint(): + with mock.patch("sys.stdout", new=StringIO()) as mock_out: + pprint(value) + assert mock_out.getvalue() == "{'a': 1}\n" diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index c43ed152b..06fdf9cba 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -11,47 +11,51 @@ from tests import tests_datadir SAMPLEDIR = Path(tests_datadir, "compressed") -class TestGunzip: - def test_gunzip_basic(self): - r1 = Response( - "http://www.example.com", - body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(), - ) - assert gzip_magic_number(r1) +def test_gunzip_basic(): + r1 = Response( + "http://www.example.com", + body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(), + ) + assert gzip_magic_number(r1) - r2 = Response("http://www.example.com", body=gunzip(r1.body)) - assert not gzip_magic_number(r2) - assert len(r2.body) == 9950 + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + assert not gzip_magic_number(r2) + assert len(r2.body) == 9950 - def test_gunzip_truncated(self): - text = gunzip((SAMPLEDIR / "truncated-crc-error.gz").read_bytes()) - assert text.endswith(b"") - assert not gzip_magic_number(r2) +def test_gunzip_no_gzip_file_raises(): + with pytest.raises(BadGzipFile): + gunzip((SAMPLEDIR / "feed-sample1.xml").read_bytes()) - def test_is_gzipped_empty(self): - r1 = Response("http://www.example.com") - assert not gzip_magic_number(r1) - def test_gunzip_illegal_eof(self): - text = html_to_unicode( - "charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes()) - )[1] - expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text( - encoding="utf-8" - ) - assert len(text) == len(expected_text) - assert text == expected_text +def test_gunzip_truncated_short(): + r1 = Response( + "http://www.example.com", + body=(SAMPLEDIR / "truncated-crc-error-short.gz").read_bytes(), + ) + assert gzip_magic_number(r1) + + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + assert r2.body.endswith(b"") + assert not gzip_magic_number(r2) + + +def test_is_gzipped_empty(): + r1 = Response("http://www.example.com") + assert not gzip_magic_number(r1) + + +def test_gunzip_illegal_eof(): + text = html_to_unicode( + "charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes()) + )[1] + expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text( + encoding="utf-8" + ) + assert len(text) == len(expected_text) + assert text == expected_text diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index 0c05ef7d6..9bd86f7fb 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -4,18 +4,17 @@ from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached -class TestHttpobjUtils: - def test_urlparse_cached(self): - url = "http://www.example.com/index.html" - request1 = Request(url) - request2 = Request(url) - req1a = urlparse_cached(request1) - req1b = urlparse_cached(request1) - req2 = urlparse_cached(request2) - urlp = urlparse(url) +def test_urlparse_cached(): + url = "http://www.example.com/index.html" + request1 = Request(url) + request2 = Request(url) + req1a = urlparse_cached(request1) + req1b = urlparse_cached(request1) + req2 = urlparse_cached(request2) + urlp = urlparse(url) - assert req1a == req2 - assert req1a == urlp - assert req1a is req1b - assert req1a is not req2 - assert req1a is not req2 + assert req1a == req2 + assert req1a == urlp + assert req1a is req1b + assert req1a is not req2 + assert req1a is not req2 diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index fa0d37866..ac32fff2c 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,3 +1,8 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any + import pytest from scrapy.exceptions import ScrapyDeprecationWarning @@ -5,9 +10,19 @@ from scrapy.http import Response, TextResponse, XmlResponse from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml from tests import get_testdata +if TYPE_CHECKING: + from collections.abc import Iterator + + from scrapy import Selector + + +class TestXmliterBase(ABC): + @abstractmethod + def xmliter( + self, obj: Response | str | bytes, nodename: str, *args: Any + ) -> Iterator[Selector]: + raise NotImplementedError -class XmliterBase: - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter(self): body = b""" @@ -39,7 +54,6 @@ class XmliterBase: ("002", ["Name 2"], ["Type 2"]), ] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unusual_node(self): body = b""" @@ -53,7 +67,6 @@ class XmliterBase: ] assert nodenames == [["matchme..."]] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unicode(self): # example taken from https://github.com/scrapy/scrapy/issues/1665 body = """ @@ -113,7 +126,6 @@ class XmliterBase: ("27", ["A"], ["27"]), ] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_text(self): body = ( '' @@ -125,7 +137,6 @@ class XmliterBase: ["two"], ] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaces(self): body = b""" @@ -163,7 +174,6 @@ class XmliterBase: assert node.xpath("id/text()").getall() == [] assert node.xpath("price/text()").getall() == [] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename(self): body = b""" @@ -191,7 +201,6 @@ class XmliterBase: "http://www.mydummycompany.com/images/item1.jpg" ] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename_missing(self): body = b""" @@ -216,7 +225,6 @@ class XmliterBase: with pytest.raises(StopIteration): next(my_iter) - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_exception(self): body = ( '' @@ -229,13 +237,11 @@ class XmliterBase: with pytest.raises(StopIteration): next(iter) - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_objtype_exception(self): i = self.xmliter(42, "product") with pytest.raises(TypeError): next(i) - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_encoding(self): body = ( b'\n' @@ -250,8 +256,12 @@ class XmliterBase: ) -class TestXmliter(XmliterBase): - xmliter = staticmethod(xmliter) +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class TestXmliter(TestXmliterBase): + def xmliter( + self, obj: Response | str | bytes, nodename: str, *args: Any + ) -> Iterator[Selector]: + return xmliter(obj, nodename) def test_deprecation(self): body = b""" @@ -267,8 +277,11 @@ class TestXmliter(XmliterBase): next(self.xmliter(body, "product")) -class TestLxmlXmliter(XmliterBase): - xmliter = staticmethod(xmliter_lxml) +class TestLxmlXmliter(TestXmliterBase): + def xmliter( + self, obj: Response | str | bytes, nodename: str, *args: Any + ) -> Iterator[Selector]: + return xmliter_lxml(obj, nodename, *args) def test_xmliter_iterate_namespace(self): body = b""" @@ -493,23 +506,32 @@ class TestUtilsCsv: ] -class TestHelper: +class TestBodyOrStr: bbody = b"utf8-body" ubody = bbody.decode("utf8") - txtresponse = TextResponse(url="http://example.org/", body=bbody, encoding="utf-8") - response = Response(url="http://example.org/", body=bbody) - def test_body_or_str(self): - for obj in (self.bbody, self.ubody, self.txtresponse, self.response): - r1 = _body_or_str(obj) - self._assert_type_and_value(r1, self.ubody, obj) - r2 = _body_or_str(obj, unicode=True) - self._assert_type_and_value(r2, self.ubody, obj) - r3 = _body_or_str(obj, unicode=False) - self._assert_type_and_value(r3, self.bbody, obj) - assert type(r1) is type(r2) - assert type(r1) is not type(r3) + @pytest.mark.parametrize( + "obj", + [ + bbody, + ubody, + TextResponse(url="http://example.org/", body=bbody, encoding="utf-8"), + Response(url="http://example.org/", body=bbody), + ], + ) + def test_body_or_str(self, obj: Response | str | bytes) -> None: + r1 = _body_or_str(obj) + self._assert_type_and_value(r1, self.ubody, obj) + r2 = _body_or_str(obj, unicode=True) + self._assert_type_and_value(r2, self.ubody, obj) + r3 = _body_or_str(obj, unicode=False) + self._assert_type_and_value(r3, self.bbody, obj) + assert type(r1) is type(r2) + assert type(r1) is not type(r3) - def _assert_type_and_value(self, a, b, obj): + @staticmethod + def _assert_type_and_value( + a: str | bytes, b: str | bytes, obj: Response | str | bytes + ) -> None: assert type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}" assert a == b diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 56375606c..f40e424ff 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -22,7 +22,9 @@ from scrapy.utils.test import get_crawler from tests.spiders import LogSpider if TYPE_CHECKING: - from collections.abc import Mapping, MutableMapping + from collections.abc import Generator, Mapping, MutableMapping + + from scrapy.crawler import Crawler class TestFailureToExcInfo: @@ -70,33 +72,41 @@ class TestTopLevelFormatter: class TestLogCounterHandler: - def setup_method(self): + @pytest.fixture + def crawler(self) -> Crawler: settings = {"LOG_LEVEL": "WARNING"} - self.logger = logging.getLogger("test") - self.logger.setLevel(logging.NOTSET) - self.logger.propagate = False - self.crawler = get_crawler(settings_dict=settings) - self.handler = LogCounterHandler(self.crawler) - self.logger.addHandler(self.handler) + return get_crawler(settings_dict=settings) - def teardown_method(self): - self.logger.propagate = True - self.logger.removeHandler(self.handler) + @pytest.fixture + def logger(self, crawler: Crawler) -> Generator[logging.Logger]: + logger = logging.getLogger("test") + logger.setLevel(logging.NOTSET) + logger.propagate = False + handler = LogCounterHandler(crawler) + logger.addHandler(handler) - def test_init(self): - assert self.crawler.stats.get_value("log_count/DEBUG") is None - assert self.crawler.stats.get_value("log_count/INFO") is None - assert self.crawler.stats.get_value("log_count/WARNING") is None - assert self.crawler.stats.get_value("log_count/ERROR") is None - assert self.crawler.stats.get_value("log_count/CRITICAL") is None + yield logger - def test_accepted_level(self): - self.logger.error("test log msg") - assert self.crawler.stats.get_value("log_count/ERROR") == 1 + logger.propagate = True + logger.removeHandler(handler) - def test_filtered_out_level(self): - self.logger.debug("test log msg") - assert self.crawler.stats.get_value("log_count/INFO") is None + def test_init(self, crawler: Crawler, logger: logging.Logger) -> None: + assert crawler.stats + assert crawler.stats.get_value("log_count/DEBUG") is None + assert crawler.stats.get_value("log_count/INFO") is None + assert crawler.stats.get_value("log_count/WARNING") is None + assert crawler.stats.get_value("log_count/ERROR") is None + assert crawler.stats.get_value("log_count/CRITICAL") is None + + def test_accepted_level(self, crawler: Crawler, logger: logging.Logger) -> None: + logger.error("test log msg") + assert crawler.stats + assert crawler.stats.get_value("log_count/ERROR") == 1 + + def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None: + logger.debug("test log msg") + assert crawler.stats + assert crawler.stats.get_value("log_count/INFO") is None class TestStreamLogger: @@ -135,7 +145,7 @@ class TestStreamLogger: ) def test_spider_logger_adapter_process( base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict -): +) -> None: logger = logging.getLogger("test") spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra) @@ -149,59 +159,75 @@ def test_spider_logger_adapter_process( class TestLogging: - def setup_method(self): - self.log_stream = StringIO() - handler = logging.StreamHandler(self.log_stream) + @pytest.fixture + def log_stream(self) -> StringIO: + return StringIO() + + @pytest.fixture + def spider(self) -> LogSpider: + return LogSpider() + + @pytest.fixture(autouse=True) + def logger(self, log_stream: StringIO) -> Generator[logging.Logger]: + handler = logging.StreamHandler(log_stream) logger = logging.getLogger("log_spider") logger.addHandler(handler) logger.setLevel(logging.DEBUG) - self.handler = handler - self.logger = logger - self.spider = LogSpider() - def teardown_method(self): - self.logger.removeHandler(self.handler) + yield logger - def test_debug_logging(self): + logger.removeHandler(handler) + + def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo message" - self.spider.log_debug(log_message) - log_contents = self.log_stream.getvalue() + spider.log_debug(log_message) + log_contents = log_stream.getvalue() assert log_contents == f"{log_message}\n" - def test_info_logging(self): + def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Bar message" - self.spider.log_info(log_message) - log_contents = self.log_stream.getvalue() + spider.log_info(log_message) + log_contents = log_stream.getvalue() assert log_contents == f"{log_message}\n" - def test_warning_logging(self): + def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Baz message" - self.spider.log_warning(log_message) - log_contents = self.log_stream.getvalue() + spider.log_warning(log_message) + log_contents = log_stream.getvalue() assert log_contents == f"{log_message}\n" - def test_error_logging(self): + def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo bar message" - self.spider.log_error(log_message) - log_contents = self.log_stream.getvalue() + spider.log_error(log_message) + log_contents = log_stream.getvalue() assert log_contents == f"{log_message}\n" - def test_critical_logging(self): + def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo bar baz message" - self.spider.log_critical(log_message) - log_contents = self.log_stream.getvalue() + spider.log_critical(log_message) + log_contents = log_stream.getvalue() assert log_contents == f"{log_message}\n" class TestLoggingWithExtra: - def setup_method(self): - self.log_stream = StringIO() - handler = logging.StreamHandler(self.log_stream) + regex_pattern = re.compile(r"^]+>$") + + @pytest.fixture + def log_stream(self) -> StringIO: + return StringIO() + + @pytest.fixture + def spider(self) -> LogSpider: + return LogSpider() + + @pytest.fixture(autouse=True) + def logger(self, log_stream: StringIO) -> Generator[logging.Logger]: + handler = logging.StreamHandler(log_stream) formatter = logging.Formatter( '{"levelname": "%(levelname)s", "message": "%(message)s", "spider": "%(spider)s", "important_info": "%(important_info)s"}' ) @@ -209,80 +235,79 @@ class TestLoggingWithExtra: logger = logging.getLogger("log_spider") logger.addHandler(handler) logger.setLevel(logging.DEBUG) - self.handler = handler - self.logger = logger - self.spider = LogSpider() - self.regex_pattern = re.compile(r"^]+>$") - def teardown_method(self): - self.logger.removeHandler(self.handler) + yield logger - def test_debug_logging(self): + logger.removeHandler(handler) + + def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo message" extra = {"important_info": "foo"} - self.spider.log_debug(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_debug(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "DEBUG" assert log_contents["message"] == log_message assert self.regex_pattern.match(log_contents["spider"]) assert log_contents["important_info"] == extra["important_info"] - def test_info_logging(self): + def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Bar message" extra = {"important_info": "bar"} - self.spider.log_info(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_info(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "INFO" assert log_contents["message"] == log_message assert self.regex_pattern.match(log_contents["spider"]) assert log_contents["important_info"] == extra["important_info"] - def test_warning_logging(self): + def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Baz message" extra = {"important_info": "baz"} - self.spider.log_warning(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_warning(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "WARNING" assert log_contents["message"] == log_message assert self.regex_pattern.match(log_contents["spider"]) assert log_contents["important_info"] == extra["important_info"] - def test_error_logging(self): + def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo bar message" extra = {"important_info": "foo bar"} - self.spider.log_error(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_error(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "ERROR" assert log_contents["message"] == log_message assert self.regex_pattern.match(log_contents["spider"]) assert log_contents["important_info"] == extra["important_info"] - def test_critical_logging(self): + def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo bar baz message" extra = {"important_info": "foo bar baz"} - self.spider.log_critical(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_critical(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "CRITICAL" assert log_contents["message"] == log_message assert self.regex_pattern.match(log_contents["spider"]) assert log_contents["important_info"] == extra["important_info"] - def test_overwrite_spider_extra(self): + def test_overwrite_spider_extra( + self, log_stream: StringIO, spider: LogSpider + ) -> None: log_message = "Foo message" extra = {"important_info": "foo", "spider": "shouldn't change"} - self.spider.log_error(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_error(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "ERROR" assert log_contents["message"] == log_message diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index aa250be69..20a3d940c 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -1,18 +1,17 @@ -import contextlib import os -import shutil -import tempfile import warnings from pathlib import Path +import pytest + from scrapy.utils.misc import set_environ from scrapy.utils.project import data_path, get_project_settings -@contextlib.contextmanager -def inside_a_project(): +@pytest.fixture +def proj_path(tmp_path): prev_dir = Path.cwd() - project_dir = tempfile.mkdtemp() + project_dir = tmp_path try: os.chdir(project_dir) @@ -21,21 +20,19 @@ def inside_a_project(): yield project_dir finally: os.chdir(prev_dir) - shutil.rmtree(project_dir) -class TestProjectUtils: - def test_data_path_outside_project(self): - assert str(Path(".scrapy", "somepath")) == data_path("somepath") - abspath = str(Path(os.path.sep, "absolute", "path")) - assert abspath == data_path(abspath) +def test_data_path_outside_project(): + assert str(Path(".scrapy", "somepath")) == data_path("somepath") + abspath = str(Path(os.path.sep, "absolute", "path")) + assert abspath == data_path(abspath) - def test_data_path_inside_project(self): - with inside_a_project() as proj_path: - expected = Path(proj_path, ".scrapy", "somepath") - assert expected.resolve() == Path(data_path("somepath")).resolve() - abspath = str(Path(os.path.sep, "absolute", "path").resolve()) - assert abspath == data_path(abspath) + +def test_data_path_inside_project(proj_path: Path) -> None: + expected = proj_path / ".scrapy" / "somepath" + assert expected.resolve() == Path(data_path("somepath")).resolve() + abspath = str(Path(os.path.sep, "absolute", "path").resolve()) + assert abspath == data_path(abspath) class TestGetProjectSettings: diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 291646ad7..c933e0ac9 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,7 +1,10 @@ +from __future__ import annotations + import functools import operator import platform import sys +from typing import TYPE_CHECKING, TypeVar import pytest from twisted.trial import unittest @@ -20,16 +23,22 @@ from scrapy.utils.python import ( without_none_values, ) +if TYPE_CHECKING: + from collections.abc import Iterable, Mapping -class TestMutableChain: - def test_mutablechain(self): - m = MutableChain(range(2), [2, 3], (4, 5)) - m.extend(range(6, 7)) - m.extend([7, 8]) - m.extend([9, 10], (11, 12)) - assert next(m) == 0 - assert m.__next__() == 1 - assert list(m) == list(range(2, 13)) + +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") + + +def test_mutablechain(): + m = MutableChain(range(2), [2, 3], (4, 5)) + m.extend(range(6, 7)) + m.extend([7, 8]) + m.extend([9, 10], (11, 12)) + assert next(m) == 0 + assert m.__next__() == 1 + assert list(m) == list(range(2, 13)) class TestMutableAsyncChain(unittest.TestCase): @@ -112,144 +121,150 @@ class TestToBytes: assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b" -class TestMemoizedMethod: - def test_memoizemethod_noargs(self): - class A: - @memoizemethod_noargs - def cached(self): - return object() +def test_memoizemethod_noargs(): + class A: + @memoizemethod_noargs + def cached(self): + return object() - def noncached(self): - return object() + def noncached(self): + return object() - a = A() - one = a.cached() - two = a.cached() - three = a.noncached() - assert one is two - assert one is not three + a = A() + one = a.cached() + two = a.cached() + three = a.noncached() + assert one is two + assert one is not three -class TestBinaryIsText: - def test_binaryistext(self): - assert binary_is_text(b"hello") - - def test_utf_16_strings_contain_null_bytes(self): - assert binary_is_text("hello".encode("utf-16")) - - def test_one_with_encoding(self): - assert binary_is_text(b"
Price \xa3
") - - def test_real_binary_bytes(self): - assert not binary_is_text(b"\x02\xa3") +@pytest.mark.parametrize( + ("value", "expected"), + [ + (b"hello", True), + ("hello".encode("utf-16"), True), + (b"
Price \xa3
", True), + (b"\x02\xa3", False), + ], +) +def test_binaryistext(value: bytes, expected: bool) -> None: + assert binary_is_text(value) is expected -class TestUtilsPython: - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_equal_attributes(self): - class Obj: +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +def test_equal_attributes(): + class Obj: + pass + + a = Obj() + b = Obj() + # no attributes given return False + assert not equal_attributes(a, b, []) + # nonexistent attributes + assert not equal_attributes(a, b, ["x", "y"]) + + a.x = 1 + b.x = 1 + # equal attribute + assert equal_attributes(a, b, ["x"]) + + b.y = 2 + # obj1 has no attribute y + assert not equal_attributes(a, b, ["x", "y"]) + + a.y = 2 + # equal attributes + assert equal_attributes(a, b, ["x", "y"]) + + a.y = 1 + # different attributes + assert not equal_attributes(a, b, ["x", "y"]) + + # test callable + a.meta = {} + b.meta = {} + assert equal_attributes(a, b, ["meta"]) + + # compare ['meta']['a'] + a.meta["z"] = 1 + b.meta["z"] = 1 + + get_z = operator.itemgetter("z") + get_meta = operator.attrgetter("meta") + + def compare_z(obj): + return get_z(get_meta(obj)) + + assert equal_attributes(a, b, [compare_z, "x"]) + # fail z equality + a.meta["z"] = 2 + assert not equal_attributes(a, b, [compare_z, "x"]) + + +def test_get_func_args(): + def f1(a, b, c): + pass + + def f2(a, b=None, c=None): + pass + + def f3(a, b=None, *, c=None): + pass + + class A: + def __init__(self, a, b, c): pass - a = Obj() - b = Obj() - # no attributes given return False - assert not equal_attributes(a, b, []) - # nonexistent attributes - assert not equal_attributes(a, b, ["x", "y"]) - - a.x = 1 - b.x = 1 - # equal attribute - assert equal_attributes(a, b, ["x"]) - - b.y = 2 - # obj1 has no attribute y - assert not equal_attributes(a, b, ["x", "y"]) - - a.y = 2 - # equal attributes - assert equal_attributes(a, b, ["x", "y"]) - - a.y = 1 - # different attributes - assert not equal_attributes(a, b, ["x", "y"]) - - # test callable - a.meta = {} - b.meta = {} - assert equal_attributes(a, b, ["meta"]) - - # compare ['meta']['a'] - a.meta["z"] = 1 - b.meta["z"] = 1 - - get_z = operator.itemgetter("z") - get_meta = operator.attrgetter("meta") - - def compare_z(obj): - return get_z(get_meta(obj)) - - assert equal_attributes(a, b, [compare_z, "x"]) - # fail z equality - a.meta["z"] = 2 - assert not equal_attributes(a, b, [compare_z, "x"]) - - def test_get_func_args(self): - def f1(a, b, c): + def method(self, a, b, c): pass - def f2(a, b=None, c=None): + class Callable: + def __call__(self, a, b, c): pass - def f3(a, b=None, *, c=None): - pass + a = A(1, 2, 3) + cal = Callable() + partial_f1 = functools.partial(f1, None) + partial_f2 = functools.partial(f1, b=None) + partial_f3 = functools.partial(partial_f2, None) - class A: - def __init__(self, a, b, c): - pass + assert get_func_args(f1) == ["a", "b", "c"] + assert get_func_args(f2) == ["a", "b", "c"] + assert get_func_args(f3) == ["a", "b", "c"] + assert get_func_args(A) == ["a", "b", "c"] + assert get_func_args(a.method) == ["a", "b", "c"] + assert get_func_args(partial_f1) == ["b", "c"] + assert get_func_args(partial_f2) == ["a", "c"] + assert get_func_args(partial_f3) == ["c"] + assert get_func_args(cal) == ["a", "b", "c"] + assert get_func_args(object) == [] + assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"] + assert get_func_args(" ".join, stripself=True) == ["iterable"] - def method(self, a, b, c): - pass + if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy": + # the correct and correctly extracted signature + assert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"] + elif platform.python_implementation() == "CPython": + # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature + # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951) + assert get_func_args(operator.itemgetter(2), stripself=True) in [ + [], + ["args", "kwargs"], + ] - class Callable: - def __call__(self, a, b, c): - pass - a = A(1, 2, 3) - cal = Callable() - partial_f1 = functools.partial(f1, None) - partial_f2 = functools.partial(f1, b=None) - partial_f3 = functools.partial(partial_f2, None) - - assert get_func_args(f1) == ["a", "b", "c"] - assert get_func_args(f2) == ["a", "b", "c"] - assert get_func_args(f3) == ["a", "b", "c"] - assert get_func_args(A) == ["a", "b", "c"] - assert get_func_args(a.method) == ["a", "b", "c"] - assert get_func_args(partial_f1) == ["b", "c"] - assert get_func_args(partial_f2) == ["a", "c"] - assert get_func_args(partial_f3) == ["c"] - assert get_func_args(cal) == ["a", "b", "c"] - assert get_func_args(object) == [] - assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"] - assert get_func_args(" ".join, stripself=True) == ["iterable"] - - if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy": - # the correct and correctly extracted signature - assert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"] - elif platform.python_implementation() == "CPython": - # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature - # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951) - assert get_func_args(operator.itemgetter(2), stripself=True) in [ - [], - ["args", "kwargs"], - ] - - def test_without_none_values(self): - assert without_none_values([1, None, 3, 4]) == [1, 3, 4] - assert without_none_values((1, None, 3, 4)) == (1, 3, 4) - assert without_none_values({"one": 1, "none": None, "three": 3, "four": 4}) == { - "one": 1, - "three": 3, - "four": 4, - } +@pytest.mark.parametrize( + ("value", "expected"), + [ + ([1, None, 3, 4], [1, 3, 4]), + ((1, None, 3, 4), (1, 3, 4)), + ( + {"one": 1, "none": None, "three": 3, "four": 4}, + {"one": 1, "three": 3, "four": 4}, + ), + ], +) +def test_without_none_values( + value: Mapping[_KT, _VT] | Iterable[_KT], expected: dict[_KT, _VT] | Iterable[_KT] +) -> None: + assert without_none_values(value) == expected diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 5b8509753..9c4cb7159 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -20,45 +20,52 @@ from scrapy.utils.request import ( from scrapy.utils.test import get_crawler -class TestUtilsRequest: - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_request_authenticate(self): - r = Request("http://www.example.com") - request_authenticate(r, "someuser", "somepass") - assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M=" +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +def test_request_authenticate(): + r = Request("http://www.example.com") + request_authenticate(r, "someuser", "somepass") + assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M=" - def test_request_httprepr(self): - r1 = Request("http://www.example.com") - assert ( - request_httprepr(r1) == b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n" - ) - r1 = Request("http://www.example.com/some/page.html?arg=1") - assert ( - request_httprepr(r1) - == b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n" - ) +@pytest.mark.parametrize( + ("r", "expected"), + [ + ( + Request("http://www.example.com"), + b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + ), + ( + Request("http://www.example.com/some/page.html?arg=1"), + b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + ), + ( + Request( + "http://www.example.com", + method="POST", + headers={"Content-type": b"text/html"}, + body=b"Some body", + ), + b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body", + ), + ], +) +def test_request_httprepr(r: Request, expected: bytes) -> None: + assert request_httprepr(r) == expected - r1 = Request( - "http://www.example.com", - method="POST", - headers={"Content-type": b"text/html"}, - body=b"Some body", - ) - assert ( - request_httprepr(r1) - == b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body" - ) - def test_request_httprepr_for_non_http_request(self): - # the representation is not important but it must not fail. - request_httprepr(Request("file:///tmp/foo.txt")) - request_httprepr(Request("ftp://localhost/tmp/foo.txt")) +@pytest.mark.parametrize( + "r", + [ + Request("file:///tmp/foo.txt"), + Request("ftp://localhost/tmp/foo.txt"), + ], +) +def test_request_httprepr_for_non_http_request(r: Request) -> None: + # the representation is not important but it must not fail. + request_httprepr(r) class TestFingerprint: - maxDiff = None - function: staticmethod = staticmethod(fingerprint) cache: ( WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]] @@ -229,35 +236,6 @@ class TestFingerprint: assert actual == expected -REQUEST_OBJECTS_TO_TEST = ( - Request("http://www.example.com/"), - Request("http://www.example.com/query?id=111&cat=222"), - Request("http://www.example.com/query?cat=222&id=111"), - Request("http://www.example.com/hnnoticiaj1.aspx?78132,199"), - Request("http://www.example.com/hnnoticiaj1.aspx?78160,199"), - Request("http://www.example.com/members/offers.html"), - Request( - "http://www.example.com/members/offers.html", - headers={"SESSIONID": b"somehash"}, - ), - Request( - "http://www.example.com/", - headers={"Accept-Language": b"en"}, - ), - Request( - "http://www.example.com/", - headers={ - "Accept-Language": b"en", - "SESSIONID": b"somehash", - }, - ), - Request("http://www.example.com/test.html"), - Request("http://www.example.com/test.html#fragment"), - Request("http://www.example.com", method="POST"), - Request("http://www.example.com", method="POST", body=b"request body"), -) - - class TestRequestFingerprinter: def test_default_implementation(self): crawler = get_crawler() diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 80f2f25d5..179ca49e4 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse import pytest -from scrapy.http import HtmlResponse, Response, TextResponse +from scrapy.http import HtmlResponse, Response from scrapy.utils.python import to_bytes from scrapy.utils.response import ( _remove_html_comments, @@ -15,229 +15,203 @@ from scrapy.utils.response import ( ) -class TestResponseUtils: - dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response") +def test_open_in_browser(): + url = "http:///www.example.com/some/page.html" + body = ( + b" test page test body " + ) - def test_open_in_browser(self): - url = "http:///www.example.com/some/page.html" - body = b" test page test body " + def browser_open(burl: str) -> bool: + path = urlparse(burl).path + if not path or not Path(path).exists(): + path = burl.replace("file://", "") + bbody = Path(path).read_bytes() + assert b'' in bbody + return True - def browser_open(burl): - path = urlparse(burl).path - if not path or not Path(path).exists(): - path = burl.replace("file://", "") - bbody = Path(path).read_bytes() - assert b'' in bbody - return True + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=browser_open), "Browser not called" - response = HtmlResponse(url, body=body) - assert open_in_browser(response, _openfunc=browser_open), "Browser not called" + resp = Response(url, body=body) + with pytest.raises(TypeError): + open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg - resp = Response(url, body=body) - with pytest.raises(TypeError): - open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg - def test_get_meta_refresh(self): - r1 = HtmlResponse( - "http://www.example.com", - body=b""" - - Dummy - blahablsdfsal& - """, - ) - r2 = HtmlResponse( - "http://www.example.com", - body=b""" - - Dummy - blahablsdfsal& - """, - ) - r3 = HtmlResponse( - "http://www.example.com", - body=b""" -