From 197781e3af51cd46ae7761938afa474c40c36b76 Mon Sep 17 00:00:00 2001 From: Yash nagarkar <116726926+yash08123@users.noreply.github.com> Date: Fri, 22 Sep 2023 13:42:20 +0530 Subject: [PATCH 001/269] Cover the removal of is_botocore on the release notes (#6061) --- docs/news.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index fc3cfd9e8..c5b75aae2 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -469,6 +469,10 @@ Deprecation removals has now been removed. (:issue:`5719`) +- The ``scrapy.utils.boto.is_botocore()`` function, deprecated in Scrapy 2.4, + has now been removed. + (:issue:`5719`) + Deprecations ~~~~~~~~~~~~ From 8dc72dfc4d5a651e034a956a03d8e0a5f4c8a94d Mon Sep 17 00:00:00 2001 From: kokobhara <146670393+kokobhara@users.noreply.github.com> Date: Mon, 2 Oct 2023 15:44:05 +0530 Subject: [PATCH 002/269] Cover PythonItemExporter backwaird-incompatible changes in 2.11 (#6081) --- docs/news.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index c5b75aae2..fd8fa3ea3 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -40,6 +40,9 @@ Backward-incompatible changes UTF-32). If you need to deal with JSON documents in an invalid encoding, use ``json.loads(response.text)`` instead. (:issue:`6016`) +- :class:`~scrapy.exporters.PythonItemExporter` used the binary output by + default but it no longer does. (:issue:`6006`, :issue:`6007`) + Deprecation removals ~~~~~~~~~~~~~~~~~~~~ From a6cee787dd45fabba3f39dbb1752baeef649f5b7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 11 Nov 2023 20:00:12 +0400 Subject: [PATCH 003/269] Improve type hints for copy() and replace() in Request and Response. --- .../downloadermiddlewares/httpcompression.py | 6 +-- scrapy/downloadermiddlewares/redirect.py | 1 + scrapy/http/request/__init__.py | 37 +++++++++++++++---- scrapy/http/request/json_request.py | 26 +++++++++++-- scrapy/http/response/__init__.py | 31 +++++++++++++--- 5 files changed, 81 insertions(+), 20 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 56a58a750..d44eb933a 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -2,7 +2,7 @@ from __future__ import annotations import io import zlib -from typing import TYPE_CHECKING, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from scrapy import Request, Spider from scrapy.crawler import Crawler @@ -74,12 +74,12 @@ class HttpCompressionMiddleware: respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) - kwargs = dict(cls=respcls, body=decoded_body) + kwargs: Dict[str, Any] = dict(body=decoded_body) if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the # responsetypes guessing is reliable kwargs["encoding"] = None - response = response.replace(**kwargs) + response = response.replace(cls=respcls, **kwargs) if not content_encoding: del response.headers["Content-Encoding"] diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 814b1a561..7b1401ac8 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -27,6 +27,7 @@ def _build_redirect_request( redirect_request = source_request.replace( url=url, **kwargs, + cls=None, cookies=None, ) if "Cookie" in redirect_request.headers: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a1c5a5e51..4effc2178 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,8 +4,11 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ +from __future__ import annotations + import inspect from typing import ( + TYPE_CHECKING, Any, AnyStr, Callable, @@ -19,7 +22,7 @@ from typing import ( Type, TypeVar, Union, - cast, + overload, ) from w3lib.url import safe_url_string @@ -31,6 +34,11 @@ from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") @@ -173,23 +181,36 @@ class Request(object_ref): def __repr__(self) -> str: return f"<{self.method} {self.url}>" - def copy(self) -> "Request": + def copy(self) -> Self: return self.replace() - def replace(self, *args: Any, **kwargs: Any) -> "Request": + @overload + def replace( + self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any + ) -> RequestTypeVar: + ... + + @overload + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: + ... + + def replace( + self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any + ) -> Request: """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) - cls = kwargs.pop("cls", self.__class__) - return cast(Request, cls(*args, **kwargs)) + if cls is None: + cls = self.__class__ + return cls(*args, **kwargs) @classmethod def from_curl( - cls: Type[RequestTypeVar], + cls, curl_command: str, ignore_unknown_options: bool = True, **kwargs: Any, - ) -> RequestTypeVar: + ) -> Self: """Create a Request object from a string containing a `cURL `_ command. It populates the HTTP method, the URL, the headers, the cookies and the body. It accepts the same @@ -221,7 +242,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]: + def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 1dd9e6c87..5c09835e4 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -5,12 +5,18 @@ This module implements the JsonRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ +from __future__ import annotations + import copy import json import warnings -from typing import Any, Optional, Tuple +from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, overload -from scrapy.http.request import Request +from scrapy.http.request import Request, RequestTypeVar + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class JsonRequest(Request): @@ -44,7 +50,19 @@ class JsonRequest(Request): def dumps_kwargs(self) -> dict: return self._dumps_kwargs - def replace(self, *args: Any, **kwargs: Any) -> Request: + @overload + def replace( + self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any + ) -> RequestTypeVar: + ... + + @overload + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: + ... + + def replace( + self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any + ) -> Request: body_passed = kwargs.get("body", None) is not None data = kwargs.pop("data", None) data_passed = data is not None @@ -54,7 +72,7 @@ class JsonRequest(Request): elif not body_passed and data_passed: kwargs["body"] = self._dumps(data) - return super().replace(*args, **kwargs) + return super().replace(*args, cls=cls, **kwargs) def _dumps(self, data: dict) -> str: """Convert to JSON""" diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 6eae3e8b3..e889a6460 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -19,8 +19,10 @@ from typing import ( Mapping, Optional, Tuple, + Type, + TypeVar, Union, - cast, + overload, ) from urllib.parse import urljoin @@ -33,9 +35,15 @@ from scrapy.link import Link from scrapy.utils.trackref import object_ref if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + from scrapy.selector import SelectorList +ResponseTypeVar = TypeVar("ResponseTypeVar", bound="Response") + + class Response(object_ref): """An object that represents an HTTP response, which is usually downloaded (by the Downloader) and fed to the Spiders for processing. @@ -132,16 +140,29 @@ class Response(object_ref): def __repr__(self) -> str: return f"<{self.status} {self.url}>" - def copy(self) -> Response: + def copy(self) -> Self: """Return a copy of this Response""" return self.replace() - def replace(self, *args: Any, **kwargs: Any) -> Response: + @overload + def replace( + self, *args: Any, cls: Type[ResponseTypeVar], **kwargs: Any + ) -> ResponseTypeVar: + ... + + @overload + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: + ... + + def replace( + self, *args: Any, cls: Optional[Type[Response]] = None, **kwargs: Any + ) -> Response: """Create a new Response with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) - cls = kwargs.pop("cls", self.__class__) - return cast(Response, cls(*args, **kwargs)) + if cls is None: + cls = self.__class__ + return cls(*args, **kwargs) def urljoin(self, url: str) -> str: """Join this Response's url with a possible relative url to form an From 5d55e4f56b77168b961db15e0f03d608fad69e7d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 12 Nov 2023 20:15:06 +0400 Subject: [PATCH 004/269] Add mypy tests. --- tests_typing/test_http_request.mypy-testing | 66 ++++++++++++++++++++ tests_typing/test_http_response.mypy-testing | 45 +++++++++++++ tox.ini | 8 +++ 3 files changed, 119 insertions(+) create mode 100644 tests_typing/test_http_request.mypy-testing create mode 100644 tests_typing/test_http_response.mypy-testing diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing new file mode 100644 index 000000000..a306b15fe --- /dev/null +++ b/tests_typing/test_http_request.mypy-testing @@ -0,0 +1,66 @@ +import pytest + +from scrapy import Request +from scrapy.http import JsonRequest + + +class MyRequest(Request): + pass + + +class MyRequest2(Request): + pass + + +@pytest.mark.mypy_testing +def mypy_test_headers(): + Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Request("data:,", headers=None) + Request("data:,", headers={}) + Request("data:,", headers=[]) + Request("data:,", headers={"foo": "bar"}) + Request("data:,", headers={b"foo": "bar"}) + Request("data:,", headers={"foo": b"bar"}) + Request("data:,", headers=[("foo", "bar")]) + Request("data:,", headers=[(b"foo", "bar")]) + Request("data:,", headers=[("foo", b"bar")]) + + +@pytest.mark.mypy_testing +def mypy_test_copy(): + req = Request("data:,") + reveal_type(req) # R: scrapy.http.request.Request + req_copy = req.copy() + reveal_type(req_copy) # R: scrapy.http.request.Request + + req = MyRequest("data:,") + reveal_type(req) # R: __main__.MyRequest + req_copy = req.copy() + reveal_type(req_copy) # R: __main__.MyRequest + + +@pytest.mark.mypy_testing +def mypy_test_replace(): + req = Request("data:,") + reveal_type(req) # R: scrapy.http.request.Request + req_copy = req.replace(body=b"a") + reveal_type(req_copy) # R: scrapy.http.request.Request + + req = MyRequest("data:,") + reveal_type(req) # R: __main__.MyRequest + req_copy = req.replace(body=b"a") + reveal_type(req_copy) # R: __main__.MyRequest + req_copy2 = req.replace(body=b"a", cls=MyRequest2) + reveal_type(req_copy2) # R: __main__.MyRequest2 + + +@pytest.mark.mypy_testing +def mypy_test_jsonrequest_copy_replace(): + req = JsonRequest("data:,") + reveal_type(req) # R: scrapy.http.request.json_request.JsonRequest + req_copy = req.copy() + reveal_type(req_copy) # R: scrapy.http.request.json_request.JsonRequest + req_copy = req.replace(body=b"a") + reveal_type(req_copy) # R: scrapy.http.request.json_request.JsonRequest + req_copy_my = req.replace(body=b"a", cls=MyRequest) + reveal_type(req_copy_my) # R: __main__.MyRequest diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing new file mode 100644 index 000000000..66ac6ad1d --- /dev/null +++ b/tests_typing/test_http_response.mypy-testing @@ -0,0 +1,45 @@ +import pytest + +from scrapy.http import HtmlResponse, Response, TextResponse + + +@pytest.mark.mypy_testing +def mypy_test_headers(): + Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Response("data:,", headers=None) + Response("data:,", headers={}) + Response("data:,", headers=[]) + Response("data:,", headers={"foo": "bar"}) + Response("data:,", headers={b"foo": "bar"}) + Response("data:,", headers={"foo": b"bar"}) + Response("data:,", headers=[("foo", "bar")]) + Response("data:,", headers=[(b"foo", "bar")]) + Response("data:,", headers=[("foo", b"bar")]) + + +@pytest.mark.mypy_testing +def mypy_test_copy(): + resp = Response("data:,") + reveal_type(resp) # R: scrapy.http.response.Response + resp_copy = resp.copy() + reveal_type(resp_copy) # R: scrapy.http.response.Response + + resp = HtmlResponse("data:,") + reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse + resp_copy = resp.copy() + reveal_type(resp_copy) # R: scrapy.http.response.html.HtmlResponse + + +@pytest.mark.mypy_testing +def mypy_test_replace(): + resp = Response("data:,") + reveal_type(resp) # R: scrapy.http.response.Response + resp_copy = resp.replace(body=b"a") + reveal_type(resp_copy) # R: scrapy.http.response.Response + + resp = HtmlResponse("data:,") + reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse + resp_copy = resp.replace(body=b"a") + reveal_type(resp_copy) # R: scrapy.http.response.html.HtmlResponse + resp_copy2 = resp.replace(body=b"a", cls=TextResponse) + reveal_type(resp_copy2) # R: scrapy.http.response.text.TextResponse diff --git a/tox.ini b/tox.ini index 932c0b805..c3fa54339 100644 --- a/tox.ini +++ b/tox.ini @@ -46,6 +46,14 @@ deps = commands = mypy {posargs: scrapy tests} +[testenv:typing-tests] +deps = + {[testenv]deps} + {[testenv:typing]deps} + pytest-mypy-testing==0.1.1 +commands = + pytest {posargs: tests_typing} + [testenv:pre-commit] basepython = python3 deps = From 204d6e180a7c8bc59f188230fb001339a5a43476 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 12 Nov 2023 20:47:52 +0400 Subject: [PATCH 005/269] Enable typing-tests in CI. --- .github/workflows/checks.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d6fc0f6c5..ed1629b67 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -18,6 +18,9 @@ jobs: - python-version: 3.8 env: TOXENV: typing + - python-version: 3.8 + env: + TOXENV: typing-tests - python-version: "3.11" # Keep in sync with .readthedocs.yml env: TOXENV: docs From 8776b4a6fb64e87c7baf96ae256e04a09246e360 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 12 Nov 2023 20:52:29 +0400 Subject: [PATCH 006/269] Fix env deps for typing-tests. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c3fa54339..21ac4c3ff 100644 --- a/tox.ini +++ b/tox.ini @@ -48,7 +48,7 @@ commands = [testenv:typing-tests] deps = - {[testenv]deps} + -rtests/requirements.txt {[testenv:typing]deps} pytest-mypy-testing==0.1.1 commands = From db5a73f7bb44704b1751a3d005f53cbcd9846415 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 15 Nov 2023 12:02:39 +0400 Subject: [PATCH 007/269] Update the expected mypy output to match the old Python one. --- tests_typing/test_http_request.mypy-testing | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing index a306b15fe..636e6895f 100644 --- a/tests_typing/test_http_request.mypy-testing +++ b/tests_typing/test_http_request.mypy-testing @@ -14,7 +14,7 @@ class MyRequest2(Request): @pytest.mark.mypy_testing def mypy_test_headers(): - Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]" Request("data:,", headers=None) Request("data:,", headers={}) Request("data:,", headers=[]) From ebdea4037a38bb207f90658b9380fda7a2e3e825 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 15 Nov 2023 12:31:31 +0400 Subject: [PATCH 008/269] Update another output line. --- tests_typing/test_http_response.mypy-testing | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing index 66ac6ad1d..2e58b4fbc 100644 --- a/tests_typing/test_http_response.mypy-testing +++ b/tests_typing/test_http_response.mypy-testing @@ -5,7 +5,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse @pytest.mark.mypy_testing def mypy_test_headers(): - Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]" Response("data:,", headers=None) Response("data:,", headers={}) Response("data:,", headers=[]) From 5fccf370b87378fe2db6bdd52b98c1e2a951df3b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 15 Nov 2023 15:38:13 +0100 Subject: [PATCH 009/269] Update the RTD URL for coverage --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 38ca81932..9ca0f817a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -276,7 +276,7 @@ coverage_ignore_pyobjects = [ intersphinx_mapping = { "attrs": ("https://www.attrs.org/en/stable/", None), - "coverage": ("https://coverage.readthedocs.io/en/stable", None), + "coverage": ("https://coverage.readthedocs.io/en/latest", None), "cryptography": ("https://cryptography.io/en/latest/", None), "cssselect": ("https://cssselect.readthedocs.io/en/latest", None), "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None), From 080fecd8900b6b1f94e8e143e90338279ba8d6e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Nov 2023 15:39:30 +0100 Subject: [PATCH 010/269] Drop the Authorization header on cross-domain redirect --- docs/news.rst | 27 ++++++++++++++++++ scrapy/downloadermiddlewares/redirect.py | 10 +++++-- tests/test_downloadermiddleware_redirect.py | 31 +++++++++++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index fd8fa3ea3..b19ec2e99 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,20 @@ Release notes ============= +.. _release-2.11.1: + +Scrapy 2.11.1 (unreleased) +-------------------------- + +**Security bug fix:** + +- The ``Authorization`` header is now dropped on redirects to a different + domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more + information. + + .. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv + + .. _release-2.11.0: Scrapy 2.11.0 (2023-09-18) @@ -2869,6 +2883,19 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.4: + +Scrapy 1.8.4 (unreleased) +------------------------- + +**Security bug fix:** + +- The ``Authorization`` header is now dropped on redirects to a different + domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more + information. + + .. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv + .. _release-1.8.3: diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 65f1d2224..3176ed930 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -17,11 +17,17 @@ def _build_redirect_request(source_request, *, url, **kwargs): **kwargs, cookies=None, ) - if "Cookie" in redirect_request.headers: + has_cookie_header = "Cookie" in redirect_request.headers + has_authorization_header = "Authorization" in redirect_request.headers + if has_cookie_header or has_authorization_header: source_request_netloc = urlparse_cached(source_request).netloc redirect_request_netloc = urlparse_cached(redirect_request).netloc if source_request_netloc != redirect_request_netloc: - del redirect_request.headers["Cookie"] + if has_cookie_header: + del redirect_request.headers["Cookie"] + # https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name + if has_authorization_header: + del redirect_request.headers["Authorization"] return redirect_request diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index dc15b672c..10b8ca9af 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -247,6 +247,37 @@ class RedirectMiddlewareTest(unittest.TestCase): perc_encoded_utf8_url = "http://scrapytest.org/a%C3%A7%C3%A3o" self.assertEqual(perc_encoded_utf8_url, req_result.url) + def test_cross_domain_header_dropping(self): + safe_headers = {"A": "B"} + original_request = Request( + "https://example.com", + headers={"Cookie": "a=b", "Authorization": "a", **safe_headers}, + ) + + internal_response = Response( + "https://example.com", + headers={"Location": "https://example.com/a"}, + status=301, + ) + internal_redirect_request = self.mw.process_response( + original_request, internal_response, self.spider + ) + self.assertIsInstance(internal_redirect_request, Request) + self.assertEqual(original_request.headers, internal_redirect_request.headers) + + external_response = Response( + "https://example.com", + headers={"Location": "https://example.org/a"}, + status=301, + ) + external_redirect_request = self.mw.process_response( + original_request, external_response, self.spider + ) + self.assertIsInstance(external_redirect_request, Request) + self.assertEqual( + safe_headers, external_redirect_request.headers.to_unicode_dict() + ) + class MetaRefreshMiddlewareTest(unittest.TestCase): def setUp(self): From 75e99c75b3c6219df50546877e02f7bbb37324c3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 15 Nov 2023 19:51:04 +0400 Subject: [PATCH 011/269] Improve the docs about Crawler attributes and settings initialization. --- docs/news.rst | 6 ++++-- docs/topics/spiders.rst | 10 ++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index fd8fa3ea3..0c202639e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -32,8 +32,10 @@ Backward-incompatible changes :meth:`scrapy.crawler.Crawler.__init__` and before the settings are finalized and frozen. This change was needed to allow changing the settings in :meth:`scrapy.Spider.from_crawler`. If you want to access the final - setting values in the spider code as early as possible you can do this in - :meth:`~scrapy.Spider.start_requests`. (:issue:`6038`) + setting values and the initialized :class:`~scrapy.crawler.Crawler` + attributes in the spider code as early as possible you can do this in + :meth:`~scrapy.Spider.start_requests` or in a handler of the + :signal:`engine_started` signal. (:issue:`6038`) - The :meth:`TextResponse.json ` method now requires the response to be in a valid JSON encoding (UTF-8, UTF-16, or diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 20452d558..30677fe74 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -142,8 +142,14 @@ scrapy.Spider method, which is handy if you want to modify them based on arguments. As a consequence, these settings aren't the final values as they can be modified later by e.g. :ref:`add-ons - `. The final settings are available in the - :meth:`start_requests` method and later. + `. For the same reason, most of the + :class:`~scrapy.crawler.Crawler` attributes aren't initialized at + this point. + + The final settings and the initialized + :class:`~scrapy.crawler.Crawler` attributes are available in the + :meth:`start_requests` method, handlers of the + :signal:`engine_started` signal and later. :param crawler: crawler to which the spider will be bound :type crawler: :class:`~scrapy.crawler.Crawler` instance From ffbf943e9d0fed636174fab34b2d957b95ee8800 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 2 Oct 2023 20:40:25 +0400 Subject: [PATCH 012/269] Merge pull request #6077 from 11-aryan/11-aryan --- docs/topics/request-response.rst | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 41df51589..adf3d0f4a 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -115,20 +115,9 @@ Request objects cookies for that domain and will be sent again in future requests. That's the typical behaviour of any regular web browser. - To create a request that does not send stored cookies and does not - store received cookies, set the ``dont_merge_cookies`` key to ``True`` - in :attr:`request.meta `. - - Example of a request that sends manually-defined cookies and ignores - cookie storage: - - .. code-block:: python - - Request( - url="http://www.example.com", - cookies={"currency": "USD", "country": "UY"}, - meta={"dont_merge_cookies": True}, - ) + Note that setting the :reqmeta:`dont_merge_cookies` key to ``True`` in + :attr:`request.meta ` causes custom cookies to be + ignored. For more info see :ref:`cookies-mw`. From 59cfdeaa5c83ca1e65be7220296366c135b7676c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Oct 2023 20:20:12 +0400 Subject: [PATCH 013/269] Merge pull request #6083 from wRAR/py3.12-release Adapt to the Python 3.12 final release --- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 20 +++++++++----------- .github/workflows/tests-windows.yml | 8 ++++---- tests/requirements.txt | 6 ++---- tests/test_downloader_handlers.py | 2 +- tests/test_feedexport.py | 3 --- tests/test_pipeline_files.py | 5 ----- 7 files changed, 17 insertions(+), 29 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 3044a1af3..aa9b3851d 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 5ff92a571..62b5f123a 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -17,7 +17,10 @@ jobs: - python-version: "3.11" env: TOXENV: py - - python-version: "3.11" + - python-version: "3.12" + env: + TOXENV: py + - python-version: "3.12" env: TOXENV: asyncio - python-version: pypy3.9 @@ -41,22 +44,17 @@ jobs: env: TOXENV: botocore-pinned - - python-version: "3.11" + - python-version: "3.12" env: TOXENV: extra-deps - - python-version: "3.11" + - python-version: "3.12" env: TOXENV: botocore - - python-version: "3.12.0-rc.2" - env: - TOXENV: py - - python-version: "3.12.0-rc.2" + # keep until uvloop supports 3.12 + - python-version: "3.11" env: TOXENV: asyncio - - python-version: "3.12.0-rc.2" - env: - TOXENV: extra-deps steps: - uses: actions/checkout@v3 @@ -67,7 +65,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || contains(matrix.python-version, '3.12.0') + if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index c8d1928d7..48e0bea76 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -17,13 +17,13 @@ jobs: - python-version: "3.10" env: TOXENV: py - - python-version: "3.10" - env: - TOXENV: asyncio - python-version: "3.11" env: TOXENV: py - - python-version: "3.11" + - python-version: "3.12" + env: + TOXENV: py + - python-version: "3.12" env: TOXENV: asyncio diff --git a/tests/requirements.txt b/tests/requirements.txt index 3ea7f3333..c07fda2d6 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,7 +1,6 @@ # Tests requirements attrs -# https://github.com/giampaolo/pyftpdlib/issues/560 -pyftpdlib; python_version < "3.12" +pyftpdlib >= 1.5.8 pytest pytest-cov==4.0.0 pytest-xdist @@ -10,8 +9,7 @@ testfixtures # uvloop currently doesn't build on 3.12 uvloop; platform_system != "Windows" and python_version < "3.12" -# bpython requires greenlet which currently doesn't build on 3.12 -bpython; python_version < "3.12" # optional for shell wrapper tests +bpython # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests # 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072 brotli==1.0.9; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 57211d97a..f12243e1d 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -127,7 +127,7 @@ class FileTestCase(unittest.TestCase): return self.download_request(request, Spider("foo")).addCallback(_test) def test_non_existent(self): - request = Request(f"file://{self.mktemp()}") + request = Request(path_to_file_uri(self.mktemp())) d = self.download_request(request, Spider("foo")) return self.assertFailure(d, OSError) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 6b82974fa..56967c0d5 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -125,9 +125,6 @@ class FileFeedStorageTest(unittest.TestCase): path.unlink() -@pytest.mark.skipif( - sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet" -) class FTPFeedStorageTest(unittest.TestCase): def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index bf96f17b6..468751446 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,7 +1,6 @@ import dataclasses import os import random -import sys import time from datetime import datetime from io import BytesIO @@ -12,7 +11,6 @@ from unittest import mock from urllib.parse import urlparse import attr -import pytest from itemadapter import ItemAdapter from twisted.internet import defer from twisted.trial import unittest @@ -648,9 +646,6 @@ class TestGCSFilesStore(unittest.TestCase): store.bucket.get_blob.assert_called_with(expected_blob_path) -@pytest.mark.skipif( - sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet" -) class TestFTPFileStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): From 538192916f496eb21846d797a6feff5c05f501cf Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 17 Oct 2023 17:08:23 +0400 Subject: [PATCH 014/269] Merge pull request #6064 from wRAR/signals-proper Refactor installing signals. --- scrapy/crawler.py | 12 +++++++----- scrapy/utils/ossignal.py | 9 +++------ scrapy/utils/testproc.py | 7 ++++--- setup.py | 3 +-- tests/CrawlerProcess/sleeping.py | 24 +++++++++++++++++++++++ tests/requirements.txt | 1 + tests/test_command_shell.py | 26 +++++++++++++++++++++++++ tests/test_crawler.py | 33 ++++++++++++++++++++++++++++++-- 8 files changed, 97 insertions(+), 18 deletions(-) create mode 100644 tests/CrawlerProcess/sleeping.py diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 22fd65be7..6f54e62e9 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -404,8 +404,8 @@ class CrawlerProcess(CrawlerRunner): :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished - :param bool install_signal_handlers: whether to install the shutdown - handlers (default: True) + :param bool install_signal_handlers: whether to install the OS signal + handlers from Twisted and Scrapy (default: True) """ from twisted.internet import reactor @@ -416,15 +416,17 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) - if install_signal_handlers: - install_shutdown_handlers(self._signal_shutdown) resolver_class = load_object(self.settings["DNS_RESOLVER"]) resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) resolver.install_on_reactor() tp = reactor.getThreadPool() tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) reactor.addSystemEventTrigger("before", "shutdown", self.stop) - reactor.run(installSignalHandlers=False) # blocking call + if install_signal_handlers: + reactor.addSystemEventTrigger( + "after", "startup", install_shutdown_handlers, self._signal_shutdown + ) + reactor.run(installSignalHandlers=install_signal_handlers) # blocking call def _graceful_stop_reactor(self) -> Deferred: d = self.stop() diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index 2334ea792..db9a71273 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -19,13 +19,10 @@ def install_shutdown_handlers( function: SignalHandlerT, override_sigint: bool = True ) -> None: """Install the given function as a signal handler for all common shutdown - signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the - SIGINT handler won't be install if there is already a handler in place - (e.g. Pdb) + signals (such as SIGINT, SIGTERM, etc). If ``override_sigint`` is ``False`` the + SIGINT handler won't be installed if there is already a handler in place + (e.g. Pdb) """ - from twisted.internet import reactor - - reactor._handleSignals() signal.signal(signal.SIGTERM, function) if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint: signal.signal(signal.SIGINT, function) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 5f7a7db14..0688e014b 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -2,7 +2,7 @@ from __future__ import annotations import os import sys -from typing import Iterable, Optional, Tuple, cast +from typing import Iterable, List, Optional, Tuple, cast from twisted.internet.defer import Deferred from twisted.internet.error import ProcessTerminated @@ -26,14 +26,15 @@ class ProcessTest: env = os.environ.copy() if settings is not None: env["SCRAPY_SETTINGS_MODULE"] = settings + assert self.command cmd = self.prefix + [self.command] + list(args) pp = TestProcessProtocol() - pp.deferred.addBoth(self._process_finished, cmd, check_code) + pp.deferred.addCallback(self._process_finished, cmd, check_code) reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) return pp.deferred def _process_finished( - self, pp: TestProcessProtocol, cmd: str, check_code: bool + self, pp: TestProcessProtocol, cmd: List[str], check_code: bool ) -> Tuple[int, bytes, bytes]: if pp.exitcode and check_code: msg = f"process {cmd} exit with code {pp.exitcode}" diff --git a/setup.py b/setup.py index 47c0af0b0..405633f55 100644 --- a/setup.py +++ b/setup.py @@ -6,8 +6,7 @@ version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() install_requires = [ - # 23.8.0 incompatibility: https://github.com/scrapy/scrapy/issues/6024 - "Twisted>=18.9.0,<23.8.0", + "Twisted>=18.9.0", "cryptography>=36.0.0", "cssselect>=0.9.1", "itemloaders>=1.0.1", diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py new file mode 100644 index 000000000..420d9d328 --- /dev/null +++ b/tests/CrawlerProcess/sleeping.py @@ -0,0 +1,24 @@ +from twisted.internet.defer import Deferred + +import scrapy +from scrapy.crawler import CrawlerProcess +from scrapy.utils.defer import maybe_deferred_to_future + + +class SleepingSpider(scrapy.Spider): + name = "sleeping" + + start_urls = ["data:,;"] + + async def parse(self, response): + from twisted.internet import reactor + + d = Deferred() + reactor.callLater(3, d.callback, None) + await maybe_deferred_to_future(d) + + +process = CrawlerProcess(settings={}) + +process.crawl(SleepingSpider) +process.start() diff --git a/tests/requirements.txt b/tests/requirements.txt index c07fda2d6..d4bfead40 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,5 +1,6 @@ # Tests requirements attrs +pexpect >= 4.8.0 pyftpdlib >= 1.5.8 pytest pytest-cov==4.0.0 diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 6589381f3..7d87eb62c 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -1,11 +1,15 @@ +import sys +from io import BytesIO from pathlib import Path +from pexpect.popen_spawn import PopenSpawn from twisted.internet import defer from twisted.trial import unittest from scrapy.utils.testproc import ProcessTest from scrapy.utils.testsite import SiteTest from tests import NON_EXISTING_RESOLVABLE, tests_datadir +from tests.mockserver import MockServer class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @@ -133,3 +137,25 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"] _, _, err = yield self.execute(args, check_code=True) self.assertNotIn(b"RuntimeError: There is no current event loop in thread", err) + + +class InteractiveShellTest(unittest.TestCase): + def test_fetch(self): + args = ( + sys.executable, + "-m", + "scrapy.cmdline", + "shell", + ) + logfile = BytesIO() + p = PopenSpawn(args, timeout=5) + p.logfile_read = logfile + p.expect_exact("Available Scrapy objects") + with MockServer() as mockserver: + p.sendline(f"fetch('{mockserver.url('/')}')") + p.sendline("type(response)") + p.expect_exact("HtmlResponse") + p.sendeof() + p.wait() + logfile.seek(0) + self.assertNotIn("Traceback", logfile.read().decode()) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 2b141e894..60b92377d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,13 +1,16 @@ import logging import os import platform +import signal import subprocess import sys import warnings from pathlib import Path +from typing import List import pytest from packaging.version import parse as parse_version +from pexpect.popen_spawn import PopenSpawn from pytest import mark, raises from twisted.internet import defer from twisted.trial import unittest @@ -289,9 +292,12 @@ class ScriptRunnerMixin: script_dir: Path cwd = os.getcwd() - def run_script(self, script_name: str, *script_args): + def get_script_args(self, script_name: str, *script_args: str) -> List[str]: script_path = self.script_dir / script_name - args = [sys.executable, str(script_path)] + list(script_args) + return [sys.executable, str(script_path)] + list(script_args) + + def run_script(self, script_name: str, *script_args: str) -> str: + args = self.get_script_args(script_name, *script_args) p = subprocess.Popen( args, env=get_mockserver_env(), @@ -517,6 +523,29 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("The value of FOO is 42", log) + def test_shutdown_graceful(self): + sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK + args = self.get_script_args("sleeping.py") + p = PopenSpawn(args, timeout=5) + p.expect_exact("Spider opened") + p.expect_exact("Crawled (200)") + p.kill(sig) + p.expect_exact("shutting down gracefully") + p.expect_exact("Spider closed (shutdown)") + p.wait() + + def test_shutdown_forced(self): + sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK + args = self.get_script_args("sleeping.py") + p = PopenSpawn(args, timeout=5) + p.expect_exact("Spider opened") + p.expect_exact("Crawled (200)") + p.kill(sig) + p.expect_exact("shutting down gracefully") + p.kill(sig) + p.expect_exact("forcing unclean shutdown") + p.wait() + class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): script_dir = Path(__file__).parent.resolve() / "CrawlerRunner" From 5e4fb0bc5fc066136b171ce488599b1ddd64c83a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 17 Oct 2023 21:24:44 +0400 Subject: [PATCH 015/269] Re-enable uvloop tests on 3.12 (#6098) --- .github/workflows/checks.yml | 4 ++-- .github/workflows/publish.yml | 2 +- .github/workflows/tests-ubuntu.yml | 5 ----- scrapy/contracts/__init__.py | 6 ++++-- tests/requirements.txt | 3 +-- tox.ini | 2 +- 6 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ee0cb4b1e..f91055ba5 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -8,7 +8,7 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.11" + - python-version: "3.12" env: TOXENV: pylint - python-version: 3.8 @@ -17,7 +17,7 @@ jobs: - python-version: "3.11" # Keep in sync with .readthedocs.yml env: TOXENV: docs - - python-version: "3.11" + - python-version: "3.12" env: TOXENV: twinecheck diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 22b8996b6..095793299 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: 3.12 - run: | pip install --upgrade build twine python -m build diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 62b5f123a..c883f958c 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -51,11 +51,6 @@ jobs: env: TOXENV: botocore - # keep until uvloop supports 3.12 - - python-version: "3.11" - env: - TOXENV: asyncio - steps: - uses: actions/checkout@v3 diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 1ec2a0234..2d9ddd89a 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -41,7 +41,9 @@ class Contract: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") - return list(iterate_spider_output(cb_result)) + return list( # pylint: disable=return-in-finally + iterate_spider_output(cb_result) + ) request.callback = wrapper @@ -68,7 +70,7 @@ class Contract: else: results.addSuccess(self.testcase_post) finally: - return output + return output # pylint: disable=return-in-finally request.callback = wrapper diff --git a/tests/requirements.txt b/tests/requirements.txt index d4bfead40..5b75674f5 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -7,8 +7,7 @@ pytest-cov==4.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures -# uvloop currently doesn't build on 3.12 -uvloop; platform_system != "Windows" and python_version < "3.12" +uvloop; platform_system != "Windows" bpython # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests diff --git a/tox.ini b/tox.ini index 9c2522a43..381da9773 100644 --- a/tox.ini +++ b/tox.ini @@ -57,7 +57,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==2.17.5 + pylint==3.0.1 commands = pylint conftest.py docs extras scrapy setup.py tests From 1045856a50d379d145e514ec9c7aeeed231aefd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 30 Oct 2023 09:35:29 +0100 Subject: [PATCH 016/269] Merge pull request #6112 from wRAR/test-shutdown-forced Make shutdown tests more robust. --- tests/CrawlerProcess/sleeping.py | 2 +- tests/test_command_shell.py | 5 ++++- tests/test_crawler.py | 11 +++++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py index 420d9d328..45479ea4f 100644 --- a/tests/CrawlerProcess/sleeping.py +++ b/tests/CrawlerProcess/sleeping.py @@ -14,7 +14,7 @@ class SleepingSpider(scrapy.Spider): from twisted.internet import reactor d = Deferred() - reactor.callLater(3, d.callback, None) + reactor.callLater(int(self.sleep), d.callback, None) await maybe_deferred_to_future(d) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 7d87eb62c..7918d94b2 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -1,3 +1,4 @@ +import os import sys from io import BytesIO from pathlib import Path @@ -147,8 +148,10 @@ class InteractiveShellTest(unittest.TestCase): "scrapy.cmdline", "shell", ) + env = os.environ.copy() + env["SCRAPY_PYTHON_SHELL"] = "python" logfile = BytesIO() - p = PopenSpawn(args, timeout=5) + p = PopenSpawn(args, env=env, timeout=5) p.logfile_read = logfile p.expect_exact("Available Scrapy objects") with MockServer() as mockserver: diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 60b92377d..0a7f9bac8 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -525,7 +525,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_shutdown_graceful(self): sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py") + args = self.get_script_args("sleeping.py", "-a", "sleep=3") p = PopenSpawn(args, timeout=5) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") @@ -534,14 +534,21 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.expect_exact("Spider closed (shutdown)") p.wait() + @defer.inlineCallbacks def test_shutdown_forced(self): + from twisted.internet import reactor + sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py") + args = self.get_script_args("sleeping.py", "-a", "sleep=10") p = PopenSpawn(args, timeout=5) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") p.kill(sig) p.expect_exact("shutting down gracefully") + # sending the second signal too fast often causes problems + d = defer.Deferred() + reactor.callLater(0.1, d.callback, None) + yield d p.kill(sig) p.expect_exact("forcing unclean shutdown") p.wait() From 150f9d6d888970d5f164387761989aba59e830c0 Mon Sep 17 00:00:00 2001 From: Jessica Allman-LaPorte Date: Fri, 3 Nov 2023 05:02:18 -0400 Subject: [PATCH 017/269] Make shell switching more clear in the tutorial (#6128) --- docs/intro/tutorial.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 19a76fc16..8ea98f29b 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -493,7 +493,15 @@ in the callback, as you can see below: "tags": quote.css("div.tags a.tag::text").getall(), } -If you run this spider, it will output the extracted data with the log:: +To run this spider, exit the scrapy shell by entering:: + + quit() + +Then, run:: + + scrapy crawl quotes + +Now, it should output the extracted data with the log:: 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/> {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'} From 49b284ab8508d3400582781343ea8171980b1e70 Mon Sep 17 00:00:00 2001 From: Kiran <75929997+Kiran1689@users.noreply.github.com> Date: Tue, 14 Nov 2023 00:43:10 +0530 Subject: [PATCH 018/269] Updated README.rst (#6144) --- README.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 1918850d6..14adff648 100644 --- a/README.rst +++ b/README.rst @@ -17,9 +17,10 @@ Scrapy :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu :alt: Ubuntu -.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS - :alt: macOS +.. .. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg + .. :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS + .. :alt: macOS + .. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows @@ -41,7 +42,7 @@ Scrapy Overview ======== -Scrapy is a fast high-level web crawling and web scraping framework, used to +Scrapy is a BSD-licensed fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. @@ -110,4 +111,4 @@ See https://scrapy.org/companies/ for a list. Commercial Support ================== -See https://scrapy.org/support/ for details. +See https://scrapy.org/support/ for details. \ No newline at end of file From 6969041c5f6891a0298d7e68ece762adee1bb222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 22 Nov 2023 15:52:00 +0100 Subject: [PATCH 019/269] Protect against gzip bombs --- .../downloadermiddlewares/httpcompression.py | 26 +++++++++++----- scrapy/utils/_compression.py | 2 ++ scrapy/utils/gz.py | 14 +++++++-- tests/sample_data/compressed/bomb-gzip.bin | Bin 0 -> 27988 bytes ...st_downloadermiddleware_httpcompression.py | 29 ++++++++++++++++-- 5 files changed, 59 insertions(+), 12 deletions(-) create mode 100644 scrapy/utils/_compression.py create mode 100644 tests/sample_data/compressed/bomb-gzip.bin diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index ead426951..5dd67ea87 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -2,9 +2,10 @@ import io import warnings import zlib -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes +from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip @@ -29,24 +30,26 @@ class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - def __init__(self, stats=None): - self.stats = stats + def __init__(self, crawler=None): + self.stats = crawler.stats + self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool("COMPRESSION_ENABLED"): raise NotConfigured try: - return cls(stats=crawler.stats) + return cls(crawler=crawler) except TypeError: warnings.warn( "HttpCompressionMiddleware subclasses must either modify " - "their '__init__' method to support a 'stats' parameter or " - "reimplement the 'from_crawler' method.", + "their '__init__' method to support a 'crawler' parameter or " + "reimplement their 'from_crawler' method.", ScrapyDeprecationWarning, ) result = cls() result.stats = crawler.stats + result._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") return result def process_request(self, request, spider): @@ -59,7 +62,14 @@ class HttpCompressionMiddleware: content_encoding = response.headers.getlist("Content-Encoding") if content_encoding: encoding = content_encoding.pop() - decoded_body = self._decode(response.body, encoding.lower()) + try: + decoded_body = self._decode(response.body, encoding.lower()) + except _DecompressionMaxSizeExceeded: + raise IgnoreRequest( + f"Ignored response {response} because its body " + f"({len(response.body)}B) exceeded DOWNLOAD_MAXSIZE " + f"({self._max_size}B) during decompression." + ) if self.stats: self.stats.inc_value( "httpcompression/response_bytes", @@ -85,7 +95,7 @@ class HttpCompressionMiddleware: def _decode(self, body, encoding): if encoding == b"gzip" or encoding == b"x-gzip": - body = gunzip(body) + body = gunzip(body, max_size=self._max_size) if encoding == b"deflate": try: diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py new file mode 100644 index 000000000..e726a70f5 --- /dev/null +++ b/scrapy/utils/_compression.py @@ -0,0 +1,2 @@ +class _DecompressionMaxSizeExceeded(ValueError): + pass diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index c7f74030e..cd5059a5c 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -5,8 +5,10 @@ from typing import List from scrapy.http import Response +from ._compression import _DecompressionMaxSizeExceeded -def gunzip(data: bytes) -> bytes: + +def gunzip(data: bytes, max_size: int = 0) -> bytes: """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. @@ -14,10 +16,10 @@ def gunzip(data: bytes) -> bytes: f = GzipFile(fileobj=BytesIO(data)) output_list: List[bytes] = [] chunk = b"." + decompressed_size = 0 while chunk: try: chunk = f.read1(8196) - output_list.append(chunk) except (OSError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error @@ -25,6 +27,14 @@ def gunzip(data: bytes) -> bytes: if output_list: break raise + decompressed_size += len(chunk) + if max_size and decompressed_size > max_size: + raise _DecompressionMaxSizeExceeded( + f"The number of bytes decompressed so far " + f"({decompressed_size}B) exceed the specified maximum " + f"({max_size}B)." + ) + output_list.append(chunk) return b"".join(output_list) diff --git a/tests/sample_data/compressed/bomb-gzip.bin b/tests/sample_data/compressed/bomb-gzip.bin new file mode 100644 index 0000000000000000000000000000000000000000..64aa0c3696cc1c6d86d218c70635d88f2035ec9e GIT binary patch literal 27988 zcmeIyElY!86b9f&oiK|G(Y#;~27Xl0-yq1UE0YN_<|{r6TM$G+ga1HfWkC=@i}}T- zAQQ0;tA;J=f*|@M2A1oDJDzYj_mw}*X4m_rN*LQrYP)-t7`Kz1`EpV#FVq|L(0ja} zI9SSs+qBrtti6t3Pxsob#`n?W)Wc%`Y$l!UY&@@AZN0t3&;4p=`TZgaH}D5)fC3Vd zkc1>8Aqh!HLK2dYgd`*(2}wvo5|WUFBqSjTNk~Exl8}TXBq0e&NJ0{lkc1>8Aqh!H zLK2dYgd`*(2}wvo5|WUFBqSjTNk~Exl8}TXBq0e&NJ0{lkc1>8Aqh!HLK2d2h!PI& z=1wx8 S;eSfl{TO{ZZ^qTjoA3)j<4WiN literal 0 HcmV?d00001 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 9dad056de..f834e78f5 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -10,7 +10,7 @@ from scrapy.downloadermiddlewares.httpcompression import ( ACCEPTED_ENCODINGS, HttpCompressionMiddleware, ) -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider @@ -35,12 +35,24 @@ FORMAT = { "html-zstd-streaming-no-content-size.bin", "zstd", ), + **{ + f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id) + for format_id in ( + # "br", + "gzip", # 27 988 → 11 511 612 + # "deflate", + # "zstd", + ) + }, } class HttpCompressionTest(TestCase): def setUp(self): - self.crawler = get_crawler(Spider) + settings = { + "DOWNLOAD_MAXSIZE": 10_000_000, # For compression bomb tests. + } + self.crawler = get_crawler(Spider, settings_dict=settings) self.spider = self.crawler._create_spider("scrapytest.org") self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) self.crawler.stats.open_spider(self.spider) @@ -373,6 +385,19 @@ class HttpCompressionTest(TestCase): self.assertStatsEqual("httpcompression/response_count", None) self.assertStatsEqual("httpcompression/response_bytes", None) + def _test_compression_bomb(self, compression_id): + response = self._getresponse(f"bomb-{compression_id}") + self.assertRaises( + IgnoreRequest, + self.mw.process_response, + response.request, + response, + self.spider, + ) + + def test_compression_bomb_gzip(self): + self._test_compression_bomb("gzip") + class HttpCompressionSubclassTest(TestCase): def test_init_missing_stats(self): From 0bf29a7b1b9b6a641c486780b7b0fa455577bf39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 22 Nov 2023 16:10:50 +0100 Subject: [PATCH 020/269] Update test expectations --- scrapy/downloadermiddlewares/httpcompression.py | 4 +++- .../test_downloadermiddleware_httpcompression.py | 16 ++-------------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 5dd67ea87..0fec05a14 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -30,7 +30,9 @@ class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - def __init__(self, crawler=None): + def __init__(self, *, crawler=None): + if not crawler: + return self.stats = crawler.stats self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index f834e78f5..f5dedd28d 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -127,18 +127,6 @@ class HttpCompressionTest(TestCase): self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_bytes", 74837) - def test_process_response_gzip_no_stats(self): - mw = HttpCompressionMiddleware() - response = self._getresponse("gzip") - request = response.request - - self.assertEqual(response.headers["Content-Encoding"], b"gzip") - newresponse = mw.process_response(request, response, self.spider) - self.assertEqual(mw.stats, None) - assert newresponse is not response - assert newresponse.body.startswith(b" Date: Wed, 22 Nov 2023 17:12:43 +0100 Subject: [PATCH 021/269] Protect against deflate bombs --- .../downloadermiddlewares/httpcompression.py | 20 +++------ scrapy/utils/_compression.py | 39 ++++++++++++++++++ scrapy/utils/gz.py | 2 +- tests/sample_data/compressed/bomb-deflate.bin | Bin 0 -> 27968 bytes ...st_downloadermiddleware_httpcompression.py | 5 ++- 5 files changed, 49 insertions(+), 17 deletions(-) create mode 100644 tests/sample_data/compressed/bomb-deflate.bin diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 0fec05a14..8cec87c47 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,11 +1,10 @@ import io import warnings -import zlib from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.utils._compression import _DecompressionMaxSizeExceeded +from scrapy.utils._compression import _DecompressionMaxSizeExceeded, _inflate from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip @@ -97,23 +96,14 @@ class HttpCompressionMiddleware: def _decode(self, body, encoding): if encoding == b"gzip" or encoding == b"x-gzip": - body = gunzip(body, max_size=self._max_size) - + return gunzip(body, max_size=self._max_size) if encoding == b"deflate": - try: - body = zlib.decompress(body) - except zlib.error: - # ugly hack to work with raw deflate content that may - # be sent by microsoft servers. For more information, see: - # http://carsten.codimi.de/gzip.yaws/ - # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx - # http://www.gzip.org/zlib/zlib_faq.html#faq38 - body = zlib.decompress(body, -15) + return _inflate(body, max_size=self._max_size) if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS: - body = brotli.decompress(body) + return brotli.decompress(body) if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS: # Using its streaming API since its simple API could handle only cases # where there is content size data embedded in the frame reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body)) - body = reader.read() + return reader.read() return body diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index e726a70f5..34bf2e4f7 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -1,2 +1,41 @@ +import zlib +from io import BytesIO +from typing import List + + class _DecompressionMaxSizeExceeded(ValueError): pass + + +def _inflate(data: bytes, *, max_size: int = 0) -> bytes: + decompressor = zlib.decompressobj() + raw_decompressor = zlib.decompressobj(wbits=-15) + input_stream = BytesIO(data) + output_list: List[bytes] = [] + output_chunk = b"." + decompressed_size = 0 + CHUNK_SIZE = 8196 + while output_chunk: + input_chunk = input_stream.read(CHUNK_SIZE) + try: + output_chunk = decompressor.decompress(input_chunk) + except zlib.error: + if decompressor != raw_decompressor: + # ugly hack to work with raw deflate content that may + # be sent by microsoft servers. For more information, see: + # http://carsten.codimi.de/gzip.yaws/ + # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx + # http://www.gzip.org/zlib/zlib_faq.html#faq38 + decompressor = raw_decompressor + output_chunk = decompressor.decompress(input_chunk) + else: + raise + decompressed_size += len(output_chunk) + if max_size and decompressed_size > max_size: + raise _DecompressionMaxSizeExceeded( + f"The number of bytes decompressed so far " + f"({decompressed_size}B) exceed the specified maximum " + f"({max_size}B)." + ) + output_list.append(output_chunk) + return b"".join(output_list) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index cd5059a5c..548134721 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -8,7 +8,7 @@ from scrapy.http import Response from ._compression import _DecompressionMaxSizeExceeded -def gunzip(data: bytes, max_size: int = 0) -> bytes: +def gunzip(data: bytes, *, max_size: int = 0) -> bytes: """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. diff --git a/tests/sample_data/compressed/bomb-deflate.bin b/tests/sample_data/compressed/bomb-deflate.bin new file mode 100644 index 0000000000000000000000000000000000000000..3598aca0777ec2511a721e9655cabb8c21c658bd GIT binary patch literal 27968 zcmeI&u}VS#6b9f+=-?6}`2-Gb=^8GEdrPzhZ7q%$M35jzaB^}JadC@=dVsh@OD>AI zMF>rSC{CdeWcdhg3f~#dd^nu*O@FmB>%Sy!^U2^bI{%2BjpGkTvtGCQb9b0}%gx*A zC^NVm7VfVnqwxEtJUIF4gqj_=18;x=5|WUFBqSjTNk~Exl8}TXBq0e&NJ0{lkc1>8 zAqh!HLK2dYgd`*(2}wvo5|VI-C0ssb8?oTOioa2%K7C$JY75N{+<`Yh0SQS+LK2dY zgd`*(2}wvo5|WUFBqSjTNk~Exl8}TXBq0e&NJ0{lkc1>8Aqh#iPZGYjN(Y-T;OY9R z_8O#jIJamt;d0?};d0?}5|WUFBqSjTNk~Exl8}TXBq0e&NJ0{lkc1>8Aqh#im4waX I@bhBz2V-bE-2eap literal 0 HcmV?d00001 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index f5dedd28d..3af8202cc 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -39,8 +39,8 @@ FORMAT = { f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id) for format_id in ( # "br", + "deflate", # 27 968 → 11 511 612 "gzip", # 27 988 → 11 511 612 - # "deflate", # "zstd", ) }, @@ -383,6 +383,9 @@ class HttpCompressionTest(TestCase): self.spider, ) + def test_compression_bomb_deflate(self): + self._test_compression_bomb("deflate") + def test_compression_bomb_gzip(self): self._test_compression_bomb("gzip") From fba167c5e1f356bcc452e95e92199f1a15135c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 22 Nov 2023 17:32:09 +0100 Subject: [PATCH 022/269] Protect against brotli bombs --- .../downloadermiddlewares/httpcompression.py | 14 +++++----- scrapy/utils/_compression.py | 26 +++++++++++++++++++ tests/sample_data/compressed/bomb-br.bin | 2 ++ ...st_downloadermiddleware_httpcompression.py | 9 ++++++- 4 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 tests/sample_data/compressed/bomb-br.bin diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 8cec87c47..91748e57e 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -4,25 +4,25 @@ import warnings from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.utils._compression import _DecompressionMaxSizeExceeded, _inflate +from scrapy.utils._compression import _DecompressionMaxSizeExceeded, _inflate, _unbrotli from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip ACCEPTED_ENCODINGS = [b"gzip", b"deflate"] try: - import brotli - - ACCEPTED_ENCODINGS.append(b"br") + import brotli # noqa: F401 except ImportError: pass +else: + ACCEPTED_ENCODINGS.append(b"br") try: import zstandard - - ACCEPTED_ENCODINGS.append(b"zstd") except ImportError: pass +else: + ACCEPTED_ENCODINGS.append(b"zstd") class HttpCompressionMiddleware: @@ -100,7 +100,7 @@ class HttpCompressionMiddleware: if encoding == b"deflate": return _inflate(body, max_size=self._max_size) if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS: - return brotli.decompress(body) + return _unbrotli(body, max_size=self._max_size) if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS: # Using its streaming API since its simple API could handle only cases # where there is content size data embedded in the frame diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 34bf2e4f7..9a32ce4f0 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -2,6 +2,11 @@ import zlib from io import BytesIO from typing import List +try: + import brotli +except ImportError: + pass + class _DecompressionMaxSizeExceeded(ValueError): pass @@ -39,3 +44,24 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes: ) output_list.append(output_chunk) return b"".join(output_list) + + +def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: + decompressor = brotli.Decompressor() + input_stream = BytesIO(data) + output_list: List[bytes] = [] + output_chunk = b"." + decompressed_size = 0 + CHUNK_SIZE = 8196 + while output_chunk: + input_chunk = input_stream.read(CHUNK_SIZE) + output_chunk = decompressor.process(input_chunk) + decompressed_size += len(output_chunk) + if max_size and decompressed_size > max_size: + raise _DecompressionMaxSizeExceeded( + f"The number of bytes decompressed so far " + f"({decompressed_size}B) exceed the specified maximum " + f"({max_size}B)." + ) + output_list.append(output_chunk) + return b"".join(output_list) diff --git a/tests/sample_data/compressed/bomb-br.bin b/tests/sample_data/compressed/bomb-br.bin new file mode 100644 index 000000000..50059866f --- /dev/null +++ b/tests/sample_data/compressed/bomb-br.bin @@ -0,0 +1,2 @@ +;nުVp SmoY2 +()-д=_o \ No newline at end of file diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 3af8202cc..8858916bc 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -38,7 +38,7 @@ FORMAT = { **{ f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id) for format_id in ( - # "br", + "br", # 34 → 11 511 612 "deflate", # 27 968 → 11 511 612 "gzip", # 27 988 → 11 511 612 # "zstd", @@ -383,6 +383,13 @@ class HttpCompressionTest(TestCase): self.spider, ) + def test_compression_bomb_br(self): + try: + import brotli # noqa: F401 + except ImportError: + raise SkipTest("no brotli") + self._test_compression_bomb("br") + def test_compression_bomb_deflate(self): self._test_compression_bomb("deflate") From 9cc870387745f45f744ceef6ed226eefcce0e066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 22 Nov 2023 17:53:00 +0100 Subject: [PATCH 023/269] Protect against zstandard bombs --- .../downloadermiddlewares/httpcompression.py | 15 ++++++----- scrapy/utils/_compression.py | 25 ++++++++++++++++++ tests/sample_data/compressed/bomb-zstd.bin | Bin 0 -> 1096 bytes ...st_downloadermiddleware_httpcompression.py | 5 +++- 4 files changed, 37 insertions(+), 8 deletions(-) create mode 100644 tests/sample_data/compressed/bomb-zstd.bin diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 91748e57e..8ee1d95a6 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,10 +1,14 @@ -import io import warnings from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.utils._compression import _DecompressionMaxSizeExceeded, _inflate, _unbrotli +from scrapy.utils._compression import ( + _DecompressionMaxSizeExceeded, + _inflate, + _unbrotli, + _unzstd, +) from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip @@ -18,7 +22,7 @@ else: ACCEPTED_ENCODINGS.append(b"br") try: - import zstandard + import zstandard # noqa: F401 except ImportError: pass else: @@ -102,8 +106,5 @@ class HttpCompressionMiddleware: if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS: return _unbrotli(body, max_size=self._max_size) if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS: - # Using its streaming API since its simple API could handle only cases - # where there is content size data embedded in the frame - reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body)) - return reader.read() + return _unzstd(body, max_size=self._max_size) return body diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 9a32ce4f0..93aa254b2 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -7,6 +7,11 @@ try: except ImportError: pass +try: + import zstandard +except ImportError: + pass + class _DecompressionMaxSizeExceeded(ValueError): pass @@ -65,3 +70,23 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: ) output_list.append(output_chunk) return b"".join(output_list) + + +def _unzstd(data: bytes, *, max_size: int = 0) -> bytes: + decompressor = zstandard.ZstdDecompressor() + stream_reader = decompressor.stream_reader(BytesIO(data)) + output_list: List[bytes] = [] + output_chunk = b"." + decompressed_size = 0 + CHUNK_SIZE = 8196 + while output_chunk: + output_chunk = stream_reader.read(CHUNK_SIZE) + decompressed_size += len(output_chunk) + if max_size and decompressed_size > max_size: + raise _DecompressionMaxSizeExceeded( + f"The number of bytes decompressed so far " + f"({decompressed_size}B) exceed the specified maximum " + f"({max_size}B)." + ) + output_list.append(output_chunk) + return b"".join(output_list) diff --git a/tests/sample_data/compressed/bomb-zstd.bin b/tests/sample_data/compressed/bomb-zstd.bin new file mode 100644 index 0000000000000000000000000000000000000000..4b0efa8a41c88a38dffe7cea9e4a6726bdb137c4 GIT binary patch literal 1096 zcmdPcs{gko!e;q;1{Fqz2O$}m#R@=_sF0kWTTql*T%4Jor;wDNo219Z$k6h?-fpfB z0|SQwBg3EnmI6#5b|Mlx7l~br#Lh!vBdZBP5+5~lG&~1uT5@4vU|?kU`+vT_!f29* VWPM*?)(2)~i{-##pxebr9RRD(6-595 literal 0 HcmV?d00001 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 8858916bc..7babd1318 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -41,7 +41,7 @@ FORMAT = { "br", # 34 → 11 511 612 "deflate", # 27 968 → 11 511 612 "gzip", # 27 988 → 11 511 612 - # "zstd", + "zstd", # 1 096 → 11 511 612 ) }, } @@ -396,6 +396,9 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_gzip(self): self._test_compression_bomb("gzip") + def test_compression_bomb_zstd(self): + self._test_compression_bomb("zstd") + class HttpCompressionSubclassTest(TestCase): def test_init_missing_stats(self): From 3fda2fe103dafa8d4d48b21c63b2321ccec9c378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 22 Nov 2023 18:34:37 +0100 Subject: [PATCH 024/269] Protect against gzip bomb sitemaps --- scrapy/spiders/sitemap.py | 8 +++++++- tests/test_spider.py | 15 +++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index aaf75a519..cc8b13cc3 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -3,6 +3,7 @@ import re from scrapy.http import Request, XmlResponse from scrapy.spiders import Spider +from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.gz import gunzip, gzip_magic_number from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots @@ -71,7 +72,12 @@ class SitemapSpider(Spider): if isinstance(response, XmlResponse): return response.body if gzip_magic_number(response): - return gunzip(response.body) + try: + return gunzip( + response.body, max_size=self.settings.getint("DOWNLOAD_MAXSIZE") + ) + except _DecompressionMaxSizeExceeded: + return None # actual gzipped sitemap files are decompressed above ; # if we are here (response body is not gzipped) # and have a response for .xml.gz, diff --git a/tests/test_spider.py b/tests/test_spider.py index 00da3d485..875ff5454 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -2,6 +2,7 @@ import gzip import inspect import warnings from io import BytesIO +from pathlib import Path from typing import Any from unittest import mock @@ -25,7 +26,7 @@ from scrapy.spiders import ( ) from scrapy.spiders.init import InitSpider from scrapy.utils.test import get_crawler -from tests import get_testdata +from tests import get_testdata, tests_datadir class SpiderTest(unittest.TestCase): @@ -489,7 +490,8 @@ class SitemapSpiderTest(SpiderTest): GZBODY = f.getvalue() def assertSitemapBody(self, response, body): - spider = self.spider_class("example.com") + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") self.assertEqual(spider._get_sitemap_body(response), body) def test_get_sitemap_body(self): @@ -692,6 +694,15 @@ Sitemap: /sitemap-relative-url.xml ["http://www.example.com/sitemap2.xml"], ) + def test_compression_bomb(self): + settings = {"DOWNLOAD_MAXSIZE": 10_000_000} + crawler = get_crawler(settings_dict=settings) + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + response = Response(url="https://example.com", body=body) + self.assertIsNone(spider._get_sitemap_body(response)) + class DeprecationTest(unittest.TestCase): def test_crawl_spider(self): From e0b66c021ae20cdcc24e4bb02ffae56005d3a073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 22 Nov 2023 19:03:24 +0100 Subject: [PATCH 025/269] Mind Spider.download_maxsize and Request.meta['download_maxsize'] --- .../downloadermiddlewares/httpcompression.py | 30 ++++-- scrapy/spiders/sitemap.py | 10 +- ...st_downloadermiddleware_httpcompression.py | 99 ++++++++++++++++--- tests/test_spider.py | 35 ++++++- 4 files changed, 143 insertions(+), 31 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 8ee1d95a6..e6463307e 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,5 +1,6 @@ import warnings +from scrapy import signals from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes @@ -38,6 +39,7 @@ class HttpCompressionMiddleware: return self.stats = crawler.stats self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") + crawler.signals.connect(self.open_spider, signals.spider_opened) @classmethod def from_crawler(cls, crawler): @@ -52,10 +54,15 @@ class HttpCompressionMiddleware: "reimplement their 'from_crawler' method.", ScrapyDeprecationWarning, ) - result = cls() - result.stats = crawler.stats - result._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") - return result + spider = cls() + spider.stats = crawler.stats + spider._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") + crawler.signals.connect(spider.open_spider, signals.spider_opened) + return spider + + def open_spider(self, spider): + if hasattr(spider, "download_maxsize"): + self._max_size = spider.download_maxsize def process_request(self, request, spider): request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS)) @@ -67,8 +74,11 @@ class HttpCompressionMiddleware: content_encoding = response.headers.getlist("Content-Encoding") if content_encoding: encoding = content_encoding.pop() + max_size = request.meta.get("download_maxsize", self._max_size) try: - decoded_body = self._decode(response.body, encoding.lower()) + decoded_body = self._decode( + response.body, encoding.lower(), max_size + ) except _DecompressionMaxSizeExceeded: raise IgnoreRequest( f"Ignored response {response} because its body " @@ -98,13 +108,13 @@ class HttpCompressionMiddleware: return response - def _decode(self, body, encoding): + def _decode(self, body, encoding, max_size): if encoding == b"gzip" or encoding == b"x-gzip": - return gunzip(body, max_size=self._max_size) + return gunzip(body, max_size=max_size) if encoding == b"deflate": - return _inflate(body, max_size=self._max_size) + return _inflate(body, max_size=max_size) if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS: - return _unbrotli(body, max_size=self._max_size) + return _unbrotli(body, max_size=max_size) if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS: - return _unzstd(body, max_size=self._max_size) + return _unzstd(body, max_size=max_size) return body diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index cc8b13cc3..3bca3f5c2 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -72,10 +72,14 @@ class SitemapSpider(Spider): if isinstance(response, XmlResponse): return response.body if gzip_magic_number(response): + max_size = response.meta.get( + "download_maxsize", + getattr( + self, "download_maxsize", self.settings.getint("DOWNLOAD_MAXSIZE") + ), + ) try: - return gunzip( - response.body, max_size=self.settings.getint("DOWNLOAD_MAXSIZE") - ) + return gunzip(response.body, max_size=max_size) except _DecompressionMaxSizeExceeded: return None # actual gzipped sitemap files are decompressed above ; diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 7babd1318..6d71ba71e 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -49,10 +49,7 @@ FORMAT = { class HttpCompressionTest(TestCase): def setUp(self): - settings = { - "DOWNLOAD_MAXSIZE": 10_000_000, # For compression bomb tests. - } - self.crawler = get_crawler(Spider, settings_dict=settings) + self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("scrapytest.org") self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) self.crawler.stats.open_spider(self.spider) @@ -373,31 +370,103 @@ class HttpCompressionTest(TestCase): self.assertStatsEqual("httpcompression/response_count", None) self.assertStatsEqual("httpcompression/response_bytes", None) - def _test_compression_bomb(self, compression_id): + def _test_compression_bomb_setting(self, compression_id): + settings = {"DOWNLOAD_MAXSIZE": 10_000_000} + crawler = get_crawler(Spider, settings_dict=settings) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + response = self._getresponse(f"bomb-{compression_id}") self.assertRaises( IgnoreRequest, - self.mw.process_response, + mw.process_response, response.request, response, - self.spider, + spider, ) - def test_compression_bomb_br(self): + def test_compression_bomb_setting_br(self): try: import brotli # noqa: F401 except ImportError: raise SkipTest("no brotli") - self._test_compression_bomb("br") + self._test_compression_bomb_setting("br") - def test_compression_bomb_deflate(self): - self._test_compression_bomb("deflate") + def test_compression_bomb_setting_deflate(self): + self._test_compression_bomb_setting("deflate") - def test_compression_bomb_gzip(self): - self._test_compression_bomb("gzip") + def test_compression_bomb_setting_gzip(self): + self._test_compression_bomb_setting("gzip") - def test_compression_bomb_zstd(self): - self._test_compression_bomb("zstd") + def test_compression_bomb_setting_zstd(self): + self._test_compression_bomb_setting("zstd") + + def _test_compression_bomb_spider_attr(self, compression_id): + class DownloadMaxSizeSpider(Spider): + download_maxsize = 10_000_000 + + crawler = get_crawler(DownloadMaxSizeSpider) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + + response = self._getresponse(f"bomb-{compression_id}") + self.assertRaises( + IgnoreRequest, + mw.process_response, + response.request, + response, + spider, + ) + + def test_compression_bomb_spider_attr_br(self): + try: + import brotli # noqa: F401 + except ImportError: + raise SkipTest("no brotli") + self._test_compression_bomb_spider_attr("br") + + def test_compression_bomb_spider_attr_deflate(self): + self._test_compression_bomb_spider_attr("deflate") + + def test_compression_bomb_spider_attr_gzip(self): + self._test_compression_bomb_spider_attr("gzip") + + def test_compression_bomb_spider_attr_zstd(self): + self._test_compression_bomb_spider_attr("zstd") + + def _test_compression_bomb_request_meta(self, compression_id): + crawler = get_crawler(Spider) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + + response = self._getresponse(f"bomb-{compression_id}") + response.meta["download_maxsize"] = 10_000_000 + self.assertRaises( + IgnoreRequest, + mw.process_response, + response.request, + response, + spider, + ) + + def test_compression_bomb_request_meta_br(self): + try: + import brotli # noqa: F401 + except ImportError: + raise SkipTest("no brotli") + self._test_compression_bomb_request_meta("br") + + def test_compression_bomb_request_meta_deflate(self): + self._test_compression_bomb_request_meta("deflate") + + def test_compression_bomb_request_meta_gzip(self): + self._test_compression_bomb_request_meta("gzip") + + def test_compression_bomb_request_meta_zstd(self): + self._test_compression_bomb_request_meta("zstd") class HttpCompressionSubclassTest(TestCase): diff --git a/tests/test_spider.py b/tests/test_spider.py index 875ff5454..e8480ceb4 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -509,6 +509,7 @@ class SitemapSpiderTest(SpiderTest): url="http://www.example.com/sitemap", body=self.GZBODY, headers={"content-type": "application/gzip"}, + request=Request("http://www.example.com/sitemap"), ) self.assertSitemapBody(r, self.BODY) @@ -517,7 +518,11 @@ class SitemapSpiderTest(SpiderTest): self.assertSitemapBody(r, self.BODY) def test_get_sitemap_body_xml_url_compressed(self): - r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY) + r = Response( + url="http://www.example.com/sitemap.xml.gz", + body=self.GZBODY, + request=Request("http://www.example.com/sitemap"), + ) self.assertSitemapBody(r, self.BODY) # .xml.gz but body decoded by HttpCompression middleware already @@ -694,13 +699,37 @@ Sitemap: /sitemap-relative-url.xml ["http://www.example.com/sitemap2.xml"], ) - def test_compression_bomb(self): + def test_compression_bomb_setting(self): settings = {"DOWNLOAD_MAXSIZE": 10_000_000} crawler = get_crawler(settings_dict=settings) spider = self.spider_class.from_crawler(crawler, "example.com") body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") body = body_path.read_bytes() - response = Response(url="https://example.com", body=body) + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + self.assertIsNone(spider._get_sitemap_body(response)) + + def test_compression_bomb_spider_attr(self): + class DownloadMaxSizeSpider(self.spider_class): + download_maxsize = 10_000_000 + + crawler = get_crawler() + spider = DownloadMaxSizeSpider.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + self.assertIsNone(spider._get_sitemap_body(response)) + + def test_compression_bomb_request_meta(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_maxsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) self.assertIsNone(spider._get_sitemap_body(response)) From 1087bb7b2eab28543bf9ba13149adb7acd4a3675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 23 Nov 2023 09:11:14 +0100 Subject: [PATCH 026/269] Update the docs --- docs/topics/request-response.rst | 1 + docs/topics/settings.rst | 36 +++++++++++++++++--------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index adf3d0f4a..2d1227cf8 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -702,6 +702,7 @@ Those are: * :reqmeta:`download_fail_on_dataloss` * :reqmeta:`download_latency` * :reqmeta:`download_maxsize` +* :reqmeta:`download_warnsize` * :reqmeta:`download_timeout` * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) * ``ftp_user`` (See :setting:`FTP_USER` for more info) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 7cdfb8768..eb24b834a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -873,40 +873,42 @@ The amount of time (in secs) that the downloader will wait before timing out. Request.meta key. .. setting:: DOWNLOAD_MAXSIZE +.. reqmeta:: download_maxsize DOWNLOAD_MAXSIZE ---------------- -Default: ``1073741824`` (1024MB) +Default: ``1073741824`` (1 GiB) -The maximum response size (in bytes) that downloader will download. +The maximum response body size (in bytes) allowed. Bigger responses are +aborted and ignored. -If you want to disable it set to 0. +This applies both before and after compression. If decompressing a response +body would exceed this limit, decompression is aborted and the response is +ignored. -.. reqmeta:: download_maxsize +Use ``0`` to disable this limit. -.. note:: - - This size can be set per spider using :attr:`download_maxsize` - spider attribute and per-request using :reqmeta:`download_maxsize` - Request.meta key. +This limit can be set per spider using the :attr:`download_maxsize` spider +attribute and per request using the :reqmeta:`download_maxsize` Request.meta +key. .. setting:: DOWNLOAD_WARNSIZE +.. reqmeta:: download_warnsize DOWNLOAD_WARNSIZE ----------------- -Default: ``33554432`` (32MB) +Default: ``33554432`` (32 MiB) -The response size (in bytes) that downloader will start to warn. +If the size of a response exceeds this value, before or after compression, a +warning will be logged about it. -If you want to disable it set to 0. +Use ``0`` to disable this limit. -.. note:: - - This size can be set per spider using :attr:`download_warnsize` - spider attribute and per-request using :reqmeta:`download_warnsize` - Request.meta key. +This limit can be set per spider using the :attr:`download_warnsize` spider +attribute and per request using the :reqmeta:`download_warnsize` Request.meta +key. .. setting:: DOWNLOAD_FAIL_ON_DATALOSS From 03d9866518ab43844ba0309394529240f4cf115e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 23 Nov 2023 10:26:47 +0100 Subject: [PATCH 027/269] Also use DOWNLOAD_WARNSIZE for decompressions --- .../downloadermiddlewares/httpcompression.py | 18 ++- scrapy/spiders/sitemap.py | 35 ++++- scrapy/utils/_compression.py | 12 +- scrapy/utils/gz.py | 4 +- ...st_downloadermiddleware_httpcompression.py | 130 ++++++++++++++++++ tests/test_spider.py | 78 +++++++++++ 6 files changed, 260 insertions(+), 17 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index e6463307e..95bc1849d 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,4 +1,5 @@ import warnings +from logging import getLogger from scrapy import signals from scrapy.exceptions import IgnoreRequest, NotConfigured @@ -13,6 +14,8 @@ from scrapy.utils._compression import ( from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip +logger = getLogger(__name__) + ACCEPTED_ENCODINGS = [b"gzip", b"deflate"] try: @@ -39,6 +42,7 @@ class HttpCompressionMiddleware: return self.stats = crawler.stats self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") + self._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE") crawler.signals.connect(self.open_spider, signals.spider_opened) @classmethod @@ -57,12 +61,15 @@ class HttpCompressionMiddleware: spider = cls() spider.stats = crawler.stats spider._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") + spider._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE") crawler.signals.connect(spider.open_spider, signals.spider_opened) return spider def open_spider(self, spider): if hasattr(spider, "download_maxsize"): self._max_size = spider.download_maxsize + if hasattr(spider, "download_warnsize"): + self._warn_size = spider.download_warnsize def process_request(self, request, spider): request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS)) @@ -75,6 +82,7 @@ class HttpCompressionMiddleware: if content_encoding: encoding = content_encoding.pop() max_size = request.meta.get("download_maxsize", self._max_size) + warn_size = request.meta.get("download_warnsize", self._warn_size) try: decoded_body = self._decode( response.body, encoding.lower(), max_size @@ -82,8 +90,14 @@ class HttpCompressionMiddleware: except _DecompressionMaxSizeExceeded: raise IgnoreRequest( f"Ignored response {response} because its body " - f"({len(response.body)}B) exceeded DOWNLOAD_MAXSIZE " - f"({self._max_size}B) during decompression." + f"({len(response.body)} B) exceeded DOWNLOAD_MAXSIZE " + f"({self._max_size} B) during decompression." + ) + if len(response.body) < warn_size and len(decoded_body) >= warn_size: + logger.warning( + f"{response} body size after decompression " + f"({len(decoded_body)} B) is larger than the " + f"download warning size ({warn_size} B)." ) if self.stats: self.stats.inc_value( diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 3bca3f5c2..0574f0ccb 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -1,5 +1,6 @@ import logging import re +from typing import TYPE_CHECKING, Any from scrapy.http import Request, XmlResponse from scrapy.spiders import Spider @@ -7,6 +8,12 @@ from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.gz import gunzip, gzip_magic_number from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) @@ -16,6 +23,17 @@ class SitemapSpider(Spider): sitemap_follow = [""] sitemap_alternate_links = False + @classmethod + def from_crawler(cls, crawler: "Crawler", *args: Any, **kwargs: Any) -> "Self": + spider = super().from_crawler(crawler, *args, **kwargs) + spider._max_size = getattr( + spider, "download_maxsize", spider.settings.getint("DOWNLOAD_MAXSIZE") + ) + spider._warn_size = getattr( + spider, "download_warnsize", spider.settings.getint("DOWNLOAD_WARNSIZE") + ) + return spider + def __init__(self, *a, **kw): super().__init__(*a, **kw) self._cbs = [] @@ -72,16 +90,19 @@ class SitemapSpider(Spider): if isinstance(response, XmlResponse): return response.body if gzip_magic_number(response): - max_size = response.meta.get( - "download_maxsize", - getattr( - self, "download_maxsize", self.settings.getint("DOWNLOAD_MAXSIZE") - ), - ) + uncompressed_size = len(response.body) + max_size = response.meta.get("download_maxsize", self._max_size) + warn_size = response.meta.get("download_warnsize", self._warn_size) try: - return gunzip(response.body, max_size=max_size) + body = gunzip(response.body, max_size=max_size) except _DecompressionMaxSizeExceeded: return None + if uncompressed_size < warn_size and len(body) >= warn_size: + logger.warning( + f"{response} body size after decompression ({len(body)} B) " + f"is larger than the download warning size ({warn_size} B)." + ) + return body # actual gzipped sitemap files are decompressed above ; # if we are here (response body is not gzipped) # and have a response for .xml.gz, diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 93aa254b2..a70f6c275 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -44,8 +44,8 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes: if max_size and decompressed_size > max_size: raise _DecompressionMaxSizeExceeded( f"The number of bytes decompressed so far " - f"({decompressed_size}B) exceed the specified maximum " - f"({max_size}B)." + f"({decompressed_size} B) exceed the specified maximum " + f"({max_size} B)." ) output_list.append(output_chunk) return b"".join(output_list) @@ -65,8 +65,8 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: if max_size and decompressed_size > max_size: raise _DecompressionMaxSizeExceeded( f"The number of bytes decompressed so far " - f"({decompressed_size}B) exceed the specified maximum " - f"({max_size}B)." + f"({decompressed_size} B) exceed the specified maximum " + f"({max_size} B)." ) output_list.append(output_chunk) return b"".join(output_list) @@ -85,8 +85,8 @@ def _unzstd(data: bytes, *, max_size: int = 0) -> bytes: if max_size and decompressed_size > max_size: raise _DecompressionMaxSizeExceeded( f"The number of bytes decompressed so far " - f"({decompressed_size}B) exceed the specified maximum " - f"({max_size}B)." + f"({decompressed_size} B) exceed the specified maximum " + f"({max_size} B)." ) output_list.append(output_chunk) return b"".join(output_list) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 548134721..e5cf68d62 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -31,8 +31,8 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes: if max_size and decompressed_size > max_size: raise _DecompressionMaxSizeExceeded( f"The number of bytes decompressed so far " - f"({decompressed_size}B) exceed the specified maximum " - f"({max_size}B)." + f"({decompressed_size} B) exceed the specified maximum " + f"({max_size} B)." ) output_list.append(chunk) return b"".join(output_list) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 6d71ba71e..f74fff218 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,9 +1,11 @@ from gzip import GzipFile from io import BytesIO +from logging import WARNING from pathlib import Path from unittest import SkipTest, TestCase from warnings import catch_warnings +from testfixtures import LogCapture from w3lib.encoding import resolve_encoding from scrapy.downloadermiddlewares.httpcompression import ( @@ -468,6 +470,134 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_request_meta_zstd(self): self._test_compression_bomb_request_meta("zstd") + def _test_download_warnsize_setting(self, compression_id): + settings = {"DOWNLOAD_WARNSIZE": 10_000_000} + crawler = get_crawler(Spider, settings_dict=settings) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + response = self._getresponse(f"bomb-{compression_id}") + + with LogCapture( + "scrapy.downloadermiddlewares.httpcompression", + propagate=False, + level=WARNING, + ) as log: + mw.process_response(response.request, response, spider) + log.check( + ( + "scrapy.downloadermiddlewares.httpcompression", + "WARNING", + ( + "<200 http://scrapytest.org/> body size after " + "decompression (11511612 B) is larger than the download " + "warning size (10000000 B)." + ), + ), + ) + + def test_download_warnsize_setting_br(self): + try: + import brotli # noqa: F401 + except ImportError: + raise SkipTest("no brotli") + self._test_download_warnsize_setting("br") + + def test_download_warnsize_setting_deflate(self): + self._test_download_warnsize_setting("deflate") + + def test_download_warnsize_setting_gzip(self): + self._test_download_warnsize_setting("gzip") + + def test_download_warnsize_setting_zstd(self): + self._test_download_warnsize_setting("zstd") + + def _test_download_warnsize_spider_attr(self, compression_id): + class DownloadWarnSizeSpider(Spider): + download_warnsize = 10_000_000 + + crawler = get_crawler(DownloadWarnSizeSpider) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + response = self._getresponse(f"bomb-{compression_id}") + + with LogCapture( + "scrapy.downloadermiddlewares.httpcompression", + propagate=False, + level=WARNING, + ) as log: + mw.process_response(response.request, response, spider) + log.check( + ( + "scrapy.downloadermiddlewares.httpcompression", + "WARNING", + ( + "<200 http://scrapytest.org/> body size after " + "decompression (11511612 B) is larger than the download " + "warning size (10000000 B)." + ), + ), + ) + + def test_download_warnsize_spider_attr_br(self): + try: + import brotli # noqa: F401 + except ImportError: + raise SkipTest("no brotli") + self._test_download_warnsize_spider_attr("br") + + def test_download_warnsize_spider_attr_deflate(self): + self._test_download_warnsize_spider_attr("deflate") + + def test_download_warnsize_spider_attr_gzip(self): + self._test_download_warnsize_spider_attr("gzip") + + def test_download_warnsize_spider_attr_zstd(self): + self._test_download_warnsize_spider_attr("zstd") + + def _test_download_warnsize_request_meta(self, compression_id): + crawler = get_crawler(Spider) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + response = self._getresponse(f"bomb-{compression_id}") + response.meta["download_warnsize"] = 10_000_000 + + with LogCapture( + "scrapy.downloadermiddlewares.httpcompression", + propagate=False, + level=WARNING, + ) as log: + mw.process_response(response.request, response, spider) + log.check( + ( + "scrapy.downloadermiddlewares.httpcompression", + "WARNING", + ( + "<200 http://scrapytest.org/> body size after " + "decompression (11511612 B) is larger than the download " + "warning size (10000000 B)." + ), + ), + ) + + def test_download_warnsize_request_meta_br(self): + try: + import brotli # noqa: F401 + except ImportError: + raise SkipTest("no brotli") + self._test_download_warnsize_request_meta("br") + + def test_download_warnsize_request_meta_deflate(self): + self._test_download_warnsize_request_meta("deflate") + + def test_download_warnsize_request_meta_gzip(self): + self._test_download_warnsize_request_meta("gzip") + + def test_download_warnsize_request_meta_zstd(self): + self._test_download_warnsize_request_meta("zstd") + class HttpCompressionSubclassTest(TestCase): def test_init_missing_stats(self): diff --git a/tests/test_spider.py b/tests/test_spider.py index e8480ceb4..3f595cc93 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -2,6 +2,7 @@ import gzip import inspect import warnings from io import BytesIO +from logging import WARNING from pathlib import Path from typing import Any from unittest import mock @@ -732,6 +733,83 @@ Sitemap: /sitemap-relative-url.xml response = Response(url="https://example.com", body=body, request=request) self.assertIsNone(spider._get_sitemap_body(response)) + def test_download_warnsize_setting(self): + settings = {"DOWNLOAD_WARNSIZE": 10_000_000} + crawler = get_crawler(settings_dict=settings) + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + + def test_download_warnsize_spider_attr(self): + class DownloadWarnSizeSpider(self.spider_class): + download_warnsize = 10_000_000 + + crawler = get_crawler() + spider = DownloadWarnSizeSpider.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_warnsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + + def test_download_warnsize_request_meta(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_warnsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + class DeprecationTest(unittest.TestCase): def test_crawl_spider(self): From b53ed52a22470adbe269f5a7dcc67a0da369eaf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 23 Nov 2023 11:36:45 +0100 Subject: [PATCH 028/269] Update the release notes --- docs/news.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 0c202639e..c4081b99b 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,20 @@ Release notes ============= +.. _release-2.11.1: + +Scrapy 2.11.1 (unreleased) +-------------------------- + +**Security bug fix:** + +- :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` now also apply + to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security + advisory`_ for more information. + + .. _7j7m-v7m3-jqm7 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-7j7m-v7m3-jqm7 + + .. _release-2.11.0: Scrapy 2.11.0 (2023-09-18) @@ -2871,6 +2885,17 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.4: + +Scrapy 1.8.4 (unreleased) +------------------------- + +**Security bug fix:** + +- :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` now also apply + to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security + advisory`_ for more information. + .. _release-1.8.3: From cf80e5670e8317858b9f60008a5d2d97b4988da0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 23 Nov 2023 12:07:15 +0100 Subject: [PATCH 029/269] Solve linting and typing issues --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- scrapy/spiders/sitemap.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 95bc1849d..6c8b659bd 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -93,7 +93,7 @@ class HttpCompressionMiddleware: f"({len(response.body)} B) exceeded DOWNLOAD_MAXSIZE " f"({self._max_size} B) during decompression." ) - if len(response.body) < warn_size and len(decoded_body) >= warn_size: + if len(response.body) < warn_size <= len(decoded_body): logger.warning( f"{response} body size after decompression " f"({len(decoded_body)} B) is larger than the " diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 0574f0ccb..386aa6a6e 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -22,6 +22,8 @@ class SitemapSpider(Spider): sitemap_rules = [("", "parse")] sitemap_follow = [""] sitemap_alternate_links = False + _max_size: int + _warn_size: int @classmethod def from_crawler(cls, crawler: "Crawler", *args: Any, **kwargs: Any) -> "Self": @@ -97,7 +99,7 @@ class SitemapSpider(Spider): body = gunzip(response.body, max_size=max_size) except _DecompressionMaxSizeExceeded: return None - if uncompressed_size < warn_size and len(body) >= warn_size: + if uncompressed_size < warn_size <= len(body): logger.warning( f"{response} body size after decompression ({len(body)} B) " f"is larger than the download warning size ({warn_size} B)." From 8e25f8c157e53c9f5df51e950fed18becfe7797d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 23 Nov 2023 14:12:59 +0100 Subject: [PATCH 030/269] Fix bad message --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 6c8b659bd..f03294d65 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -91,7 +91,7 @@ class HttpCompressionMiddleware: raise IgnoreRequest( f"Ignored response {response} because its body " f"({len(response.body)} B) exceeded DOWNLOAD_MAXSIZE " - f"({self._max_size} B) during decompression." + f"({max_size} B) during decompression." ) if len(response.body) < warn_size <= len(decoded_body): logger.warning( From 5f2827efe7b069e514a87bd0f7a9589f49f0a97a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 24 Nov 2023 10:02:56 +0100 Subject: [PATCH 031/269] Make HttpCompressionMiddleware changes backward-comaptible --- scrapy/downloadermiddlewares/httpcompression.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index f03294d65..1a3f6962a 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -37,8 +37,10 @@ class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - def __init__(self, *, crawler=None): + def __init__(self, stats=None, *, crawler=None): if not crawler: + if stats: + self.stats = stats return self.stats = crawler.stats self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") From 6c278e1862c5453ec45a8f6a5c2472710895cad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 24 Nov 2023 10:15:21 +0100 Subject: [PATCH 032/269] =?UTF-8?q?List[bytes]=20=E2=86=92=20BytesIO?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/utils/_compression.py | 22 ++++++++++++---------- scrapy/utils/gz.py | 12 ++++++------ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index a70f6c275..b17fd7881 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -1,6 +1,5 @@ import zlib from io import BytesIO -from typing import List try: import brotli @@ -21,7 +20,7 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes: decompressor = zlib.decompressobj() raw_decompressor = zlib.decompressobj(wbits=-15) input_stream = BytesIO(data) - output_list: List[bytes] = [] + output_stream = BytesIO() output_chunk = b"." decompressed_size = 0 CHUNK_SIZE = 8196 @@ -47,14 +46,15 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes: f"({decompressed_size} B) exceed the specified maximum " f"({max_size} B)." ) - output_list.append(output_chunk) - return b"".join(output_list) + output_stream.write(output_chunk) + output_stream.seek(0) + return output_stream.read() def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: decompressor = brotli.Decompressor() input_stream = BytesIO(data) - output_list: List[bytes] = [] + output_stream = BytesIO() output_chunk = b"." decompressed_size = 0 CHUNK_SIZE = 8196 @@ -68,14 +68,15 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: f"({decompressed_size} B) exceed the specified maximum " f"({max_size} B)." ) - output_list.append(output_chunk) - return b"".join(output_list) + output_stream.write(output_chunk) + output_stream.seek(0) + return output_stream.read() def _unzstd(data: bytes, *, max_size: int = 0) -> bytes: decompressor = zstandard.ZstdDecompressor() stream_reader = decompressor.stream_reader(BytesIO(data)) - output_list: List[bytes] = [] + output_stream = BytesIO() output_chunk = b"." decompressed_size = 0 CHUNK_SIZE = 8196 @@ -88,5 +89,6 @@ def _unzstd(data: bytes, *, max_size: int = 0) -> bytes: f"({decompressed_size} B) exceed the specified maximum " f"({max_size} B)." ) - output_list.append(output_chunk) - return b"".join(output_list) + output_stream.write(output_chunk) + output_stream.seek(0) + return output_stream.read() diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index e5cf68d62..5d23e8f05 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,7 +1,6 @@ import struct from gzip import GzipFile from io import BytesIO -from typing import List from scrapy.http import Response @@ -14,7 +13,7 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes: This is resilient to CRC checksum errors. """ f = GzipFile(fileobj=BytesIO(data)) - output_list: List[bytes] = [] + output_stream = BytesIO() chunk = b"." decompressed_size = 0 while chunk: @@ -23,8 +22,8 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes: except (OSError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error - # some pages are quite small so output_list is empty - if output_list: + # some pages are quite small so output_stream is empty + if output_stream: break raise decompressed_size += len(chunk) @@ -34,8 +33,9 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes: f"({decompressed_size} B) exceed the specified maximum " f"({max_size} B)." ) - output_list.append(chunk) - return b"".join(output_list) + output_stream.write(chunk) + output_stream.seek(0) + return output_stream.read() def gzip_magic_number(response: Response) -> bool: From 62398e424c0a3d66d0bdd3908e47284cbe797ef9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 24 Nov 2023 10:20:14 +0100 Subject: [PATCH 033/269] =?UTF-8?q?CHUNK=5FSIZE:=208=20KiB=20=E2=86=92=203?= =?UTF-8?q?2=20KiB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/utils/_compression.py | 12 ++++++------ scrapy/utils/gz.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index b17fd7881..5610595d3 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -12,6 +12,9 @@ except ImportError: pass +_CHUNK_SIZE = 65536 # 64 KiB + + class _DecompressionMaxSizeExceeded(ValueError): pass @@ -23,9 +26,8 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes: output_stream = BytesIO() output_chunk = b"." decompressed_size = 0 - CHUNK_SIZE = 8196 while output_chunk: - input_chunk = input_stream.read(CHUNK_SIZE) + input_chunk = input_stream.read(_CHUNK_SIZE) try: output_chunk = decompressor.decompress(input_chunk) except zlib.error: @@ -57,9 +59,8 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: output_stream = BytesIO() output_chunk = b"." decompressed_size = 0 - CHUNK_SIZE = 8196 while output_chunk: - input_chunk = input_stream.read(CHUNK_SIZE) + input_chunk = input_stream.read(_CHUNK_SIZE) output_chunk = decompressor.process(input_chunk) decompressed_size += len(output_chunk) if max_size and decompressed_size > max_size: @@ -79,9 +80,8 @@ def _unzstd(data: bytes, *, max_size: int = 0) -> bytes: output_stream = BytesIO() output_chunk = b"." decompressed_size = 0 - CHUNK_SIZE = 8196 while output_chunk: - output_chunk = stream_reader.read(CHUNK_SIZE) + output_chunk = stream_reader.read(_CHUNK_SIZE) decompressed_size += len(output_chunk) if max_size and decompressed_size > max_size: raise _DecompressionMaxSizeExceeded( diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 5d23e8f05..cf7316e82 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -4,7 +4,7 @@ from io import BytesIO from scrapy.http import Response -from ._compression import _DecompressionMaxSizeExceeded +from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded def gunzip(data: bytes, *, max_size: int = 0) -> bytes: @@ -18,7 +18,7 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes: decompressed_size = 0 while chunk: try: - chunk = f.read1(8196) + chunk = f.read1(_CHUNK_SIZE) except (OSError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error From 8a73c6c90c5984292d39a0a9d4be86e792d81cb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 24 Nov 2023 10:25:01 +0100 Subject: [PATCH 034/269] Fix HttpCompressionMiddleware backward compatibility --- scrapy/downloadermiddlewares/httpcompression.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 1a3f6962a..58ca1017f 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -39,8 +39,9 @@ class HttpCompressionMiddleware: def __init__(self, stats=None, *, crawler=None): if not crawler: - if stats: - self.stats = stats + self.stats = stats + self._max_size = 1073741824 + self._warn_size = 33554432 return self.stats = crawler.stats self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") From a113208a0643263fdd7198238bf9213f9148bc1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 24 Nov 2023 11:35:15 +0100 Subject: [PATCH 035/269] Fix BytesIO non-emptiness check --- scrapy/utils/gz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index cf7316e82..2e487d88b 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -23,7 +23,7 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes: # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error # some pages are quite small so output_stream is empty - if output_stream: + if output_stream.getbuffer().nbytes > 0: break raise decompressed_size += len(chunk) From c947f51077e1ab246f30764cc4cc7a1cc5835d40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 29 Nov 2023 11:54:08 +0100 Subject: [PATCH 036/269] Set an arbitrary upper limit on ReDoS-vulnerable regexps --- docs/news.rst | 28 ++++++++++++++++++++++++++++ scrapy/utils/iterators.py | 12 +++++++----- scrapy/utils/response.py | 4 ++-- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index fd8fa3ea3..121cc0322 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,22 @@ Release notes ============= +.. _release-2.11.1: + +Scrapy 2.11.1 (unreleased) +-------------------------- + +**Security bug fix:** + +- The regular expressions of the ``iternodes`` node iterator of + :class:`~scrapy.spiders.XMLFeedSpider` are no longer susceptible to a + `ReDoS attack`_. Please, see the `cc65-xxvf-f7r9 security + advisory`_ for more information. + + .. _ReDoS attack: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS + .. _cc65-xxvf-f7r9 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9 + + .. _release-2.11.0: Scrapy 2.11.0 (2023-09-18) @@ -2869,6 +2885,18 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.4: + +Scrapy 1.8.4 (unreleased) +------------------------- + +**Security bug fix:** + +- The regular expressions of the ``iternodes`` node iterator of + :class:`~scrapy.spiders.XMLFeedSpider` are no longer susceptible to a + `ReDoS attack`_. Please, see the `cc65-xxvf-f7r9 security + advisory`_ for more information. + .. _release-1.8.3: diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 03d779afb..6b89334e0 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -40,10 +40,10 @@ def xmliter( """ nodename_patt = re.escape(nodename) - DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S) + DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]{1,1024}>\s*", re.S) HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.S) - END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.S) - NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S) + END_TAG_RE = re.compile(r"<\s*/([^\s>]{1,1024})\s*>", re.S) + NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]{,1024})=[^>\s]+)", re.S) text = _body_or_str(obj) document_header_match = re.search(DOCUMENT_HEADER_RE, text) @@ -57,13 +57,15 @@ def xmliter( for tagname in reversed(re.findall(END_TAG_RE, header_end)): assert header_end_idx tag = re.search( - rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S + rf"<\s*{tagname}.{{,1024}}?xmlns[:=][^>]{{,1024}}>", + text[: header_end_idx[1]], + re.S, ) if tag: for x in re.findall(NAMESPACE_RE, tag.group()): namespaces[x[1]] = x[0] - r = re.compile(rf"<{nodename_patt}[\s>].*?", re.DOTALL) + r = re.compile(rf"<{nodename_patt}[\s>].{{,1024}}?", re.DOTALL) for match in r.finditer(text): nodetext = ( document_header diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c540d6278..b0e106c5e 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -91,8 +91,8 @@ def open_in_browser( if isinstance(response, HtmlResponse): if b"' - body = re.sub(b"", b"", body, flags=re.DOTALL) - body = re.sub(rb"(|\s.*?>))", to_bytes(repl), body) + body = re.sub(b"", b"", body, flags=re.DOTALL) + body = re.sub(rb"(|\s.{,1024}?>))", to_bytes(repl), body) ext = ".html" elif isinstance(response, TextResponse): ext = ".txt" From eb8b2c5197f3dd8570bad1945b23886ee57d045f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 29 Nov 2023 12:13:04 +0100 Subject: [PATCH 037/269] Mention open_in_browser in the release notes --- docs/news.rst | 16 ++++++++-------- docs/topics/debug.rst | 18 +++--------------- scrapy/utils/response.py | 17 +++++++++++++++-- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 121cc0322..2bbe833a5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -10,10 +10,10 @@ Scrapy 2.11.1 (unreleased) **Security bug fix:** -- The regular expressions of the ``iternodes`` node iterator of - :class:`~scrapy.spiders.XMLFeedSpider` are no longer susceptible to a - `ReDoS attack`_. Please, see the `cc65-xxvf-f7r9 security - advisory`_ for more information. +- Fixed regular expressions susceptible to a `ReDoS attack`_ affecting the + ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider` and + the :func:`~scrapy.utils.response.open_in_browser` function. Please, see + the `cc65-xxvf-f7r9 security advisory`_ for more information. .. _ReDoS attack: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS .. _cc65-xxvf-f7r9 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9 @@ -2892,10 +2892,10 @@ Scrapy 1.8.4 (unreleased) **Security bug fix:** -- The regular expressions of the ``iternodes`` node iterator of - :class:`~scrapy.spiders.XMLFeedSpider` are no longer susceptible to a - `ReDoS attack`_. Please, see the `cc65-xxvf-f7r9 security - advisory`_ for more information. +- Fixed regular expressions susceptible to a `ReDoS attack`_ affecting the + ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider` and + the :func:`~scrapy.utils.response.open_in_browser` function. Please, see + the `cc65-xxvf-f7r9 security advisory`_ for more information. .. _release-1.8.3: diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 49c5b0410..988e37bbd 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -125,25 +125,15 @@ Fortunately, the :command:`shell` is your bread and butter in this case (see See also: :ref:`topics-shell-inspect-response`. + Open in browser =============== Sometimes you just want to see how a certain response looks in a browser, you -can use the ``open_in_browser`` function for that. Here is an example of how -you would use it: +can use the :func:`~scrapy.utils.response.open_in_browser` function for that: -.. code-block:: python +.. autofunction:: scrapy.utils.response.open_in_browser - from scrapy.utils.response import open_in_browser - - - def parse_details(self, response): - if "item name" not in response.body: - open_in_browser(response) - -``open_in_browser`` will open a browser with the response received by Scrapy at -that point, adjusting the `base tag`_ so that images and styles are displayed -properly. Logging ======= @@ -163,8 +153,6 @@ available in all future runs should they be necessary again: For more information, check the :ref:`topics-logging` section. -.. _base tag: https://www.w3schools.com/tags/tag_base.asp - .. _debug-vscode: Visual Studio Code diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index b0e106c5e..8401d4ed1 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -81,8 +81,21 @@ def open_in_browser( ], _openfunc: Callable[[str], Any] = webbrowser.open, ) -> Any: - """Open the given response in a local web browser, populating the - tag for external links to work + """Open *response* in a local web browser, adjusting the `base tag`_ for + external links to work, e.g. so that images and styles are displayed. + + .. _base tag: https://www.w3schools.com/tags/tag_base.asp + + For example: + + .. code-block:: python + + from scrapy.utils.response import open_in_browser + + + def parse_details(self, response): + if "item name" not in response.body: + open_in_browser(response) """ from scrapy.http import HtmlResponse, TextResponse From 40b3efbbee3919e8f6900daeaed5e14183cabfd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 29 Nov 2023 12:47:04 +0100 Subject: [PATCH 038/269] Remove open_in_browser from the 1.8.4 release notes --- docs/news.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 2bbe833a5..c14815d06 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -2893,9 +2893,8 @@ Scrapy 1.8.4 (unreleased) **Security bug fix:** - Fixed regular expressions susceptible to a `ReDoS attack`_ affecting the - ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider` and - the :func:`~scrapy.utils.response.open_in_browser` function. Please, see - the `cc65-xxvf-f7r9 security advisory`_ for more information. + ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider`. + Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. .. _release-1.8.3: From bb74badd1bd66c59a63268c52342507c76d290b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 11 Dec 2023 17:39:55 +0100 Subject: [PATCH 039/269] =?UTF-8?q?spider=20=E2=86=92=20mw?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/downloadermiddlewares/httpcompression.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 58ca1017f..816be25a1 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -61,12 +61,12 @@ class HttpCompressionMiddleware: "reimplement their 'from_crawler' method.", ScrapyDeprecationWarning, ) - spider = cls() - spider.stats = crawler.stats - spider._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") - spider._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE") - crawler.signals.connect(spider.open_spider, signals.spider_opened) - return spider + mw = cls() + mw.stats = crawler.stats + mw._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") + mw._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE") + crawler.signals.connect(mw.open_spider, signals.spider_opened) + return mw def open_spider(self, spider): if hasattr(spider, "download_maxsize"): From 1533b69032e2fb5e495e88a3fed57c0d98502612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 13 Dec 2023 12:01:35 +0100 Subject: [PATCH 040/269] Test and address ReDoS attack vectors for open_in_browser --- scrapy/utils/response.py | 6 +++--- tests/test_utils_response.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 8401d4ed1..4369e6439 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -103,9 +103,9 @@ def open_in_browser( body = response.body if isinstance(response, HtmlResponse): if b"' - body = re.sub(b"", b"", body, flags=re.DOTALL) - body = re.sub(rb"(|\s.{,1024}?>))", to_bytes(repl), body) + repl = rf'\0' + body = re.sub(b"(?s)|$)", b"", body) + body = re.sub(rb"]*?>)", to_bytes(repl), body, count=1) ext = ".html" elif isinstance(response, TextResponse): ext = ".txt" diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 80e15a60f..942584d92 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,10 +1,12 @@ import unittest import warnings from pathlib import Path +from time import process_time from urllib.parse import urlparse from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Response, TextResponse +from scrapy.settings.default_settings import DOWNLOAD_MAXSIZE from scrapy.utils.python import to_bytes from scrapy.utils.response import ( get_base_url, @@ -198,3 +200,37 @@ class ResponseUtilsTest(unittest.TestCase): assert open_in_browser( r5, _openfunc=check_base_url ), "Inject unique base url with conditional comment" + + def test_open_in_browser_redos_comment(self): + MAX_CPU_TIME = 30 + + # Exploit input from + # https://makenowjust-labs.github.io/recheck/playground/ + # for // (old pattern to remove comments). + body = b"->" + + response = HtmlResponse("https://example.com", body=body) + + start_time = process_time() + + open_in_browser(response, lambda url: True) + + end_time = process_time() + self.assertLess(end_time - start_time, MAX_CPU_TIME) + + def test_open_in_browser_redos_head(self): + MAX_CPU_TIME = 15 + + # Exploit input from + # https://makenowjust-labs.github.io/recheck/playground/ + # for /(|\s.*?>))/ (old pattern to find the head element). + body = b" Date: Tue, 17 Oct 2023 17:49:22 -0300 Subject: [PATCH 041/269] Remove deprecated scrapy.downloadermiddlewares.decompression --- scrapy/downloadermiddlewares/decompression.py | 94 ------------------- ...test_downloadermiddleware_decompression.py | 53 ----------- 2 files changed, 147 deletions(-) delete mode 100644 scrapy/downloadermiddlewares/decompression.py delete mode 100644 tests/test_downloadermiddleware_decompression.py diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py deleted file mode 100644 index 3b8702419..000000000 --- a/scrapy/downloadermiddlewares/decompression.py +++ /dev/null @@ -1,94 +0,0 @@ -""" This module implements the DecompressionMiddleware which tries to recognise -and extract the potentially compressed responses that may arrive. -""" - -import bz2 -import gzip -import logging -import tarfile -import zipfile -from io import BytesIO -from tempfile import mktemp -from warnings import warn - -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.responsetypes import responsetypes - -warn( - "scrapy.downloadermiddlewares.decompression is deprecated", - ScrapyDeprecationWarning, - stacklevel=2, -) - - -logger = logging.getLogger(__name__) - - -class DecompressionMiddleware: - """This middleware tries to recognise and extract the possibly compressed - responses that may arrive.""" - - def __init__(self): - self._formats = { - "tar": self._is_tar, - "zip": self._is_zip, - "gz": self._is_gzip, - "bz2": self._is_bzip2, - } - - def _is_tar(self, response): - archive = BytesIO(response.body) - try: - tar_file = tarfile.open(name=mktemp(), fileobj=archive) - except tarfile.ReadError: - return - - body = tar_file.extractfile(tar_file.members[0]).read() - respcls = responsetypes.from_args(filename=tar_file.members[0].name, body=body) - return response.replace(body=body, cls=respcls) - - def _is_zip(self, response): - archive = BytesIO(response.body) - try: - zip_file = zipfile.ZipFile(archive) - except zipfile.BadZipFile: - return - - namelist = zip_file.namelist() - body = zip_file.read(namelist[0]) - respcls = responsetypes.from_args(filename=namelist[0], body=body) - return response.replace(body=body, cls=respcls) - - def _is_gzip(self, response): - archive = BytesIO(response.body) - try: - body = gzip.GzipFile(fileobj=archive).read() - except OSError: - return - - respcls = responsetypes.from_args(body=body) - return response.replace(body=body, cls=respcls) - - def _is_bzip2(self, response): - try: - body = bz2.decompress(response.body) - except OSError: - return - - respcls = responsetypes.from_args(body=body) - return response.replace(body=body, cls=respcls) - - def process_response(self, request, response, spider): - if not response.body: - return response - - for fmt, func in self._formats.items(): - new_response = func(response) - if new_response: - logger.debug( - "Decompressed response with format: %(responsefmt)s", - {"responsefmt": fmt}, - extra={"spider": spider}, - ) - return new_response - return response diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py deleted file mode 100644 index 95739414e..000000000 --- a/tests/test_downloadermiddleware_decompression.py +++ /dev/null @@ -1,53 +0,0 @@ -from unittest import TestCase, main - -from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware -from scrapy.http import Response, XmlResponse -from scrapy.spiders import Spider -from scrapy.utils.test import assert_samelines -from tests import get_testdata - - -def _test_data(formats): - uncompressed_body = get_testdata("compressed", "feed-sample1.xml") - test_responses = {} - for format in formats: - body = get_testdata("compressed", "feed-sample1." + format) - test_responses[format] = Response("http://foo.com/bar", body=body) - return uncompressed_body, test_responses - - -class DecompressionMiddlewareTest(TestCase): - test_formats = ["tar", "xml.bz2", "xml.gz", "zip"] - uncompressed_body, test_responses = _test_data(test_formats) - - def setUp(self): - self.mw = DecompressionMiddleware() - self.spider = Spider("foo") - - def test_known_compression_formats(self): - for fmt in self.test_formats: - rsp = self.test_responses[fmt] - new = self.mw.process_response(None, rsp, self.spider) - error_msg = f"Failed {fmt}, response type {type(new).__name__}" - assert isinstance(new, XmlResponse), error_msg - assert_samelines(self, new.body, self.uncompressed_body, fmt) - - def test_plain_response(self): - rsp = Response(url="http://test.com", body=self.uncompressed_body) - new = self.mw.process_response(None, rsp, self.spider) - assert new is rsp - assert_samelines(self, new.body, rsp.body) - - def test_empty_response(self): - rsp = Response(url="http://test.com", body=b"") - new = self.mw.process_response(None, rsp, self.spider) - assert new is rsp - assert not rsp.body - assert not new.body - - def tearDown(self): - del self.mw - - -if __name__ == "__main__": - main() From 12b10a7a6427c43968cc18d98a3ed3c6366eeabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 13 Dec 2023 13:35:05 +0100 Subject: [PATCH 042/269] Cover scrapy.downloadermiddlewares.decompression in the release notes --- docs/news.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index c4081b99b..a12bda53f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -16,6 +16,10 @@ Scrapy 2.11.1 (unreleased) .. _7j7m-v7m3-jqm7 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-7j7m-v7m3-jqm7 +- Also in relation with the `7j7m-v7m3-jqm7 security advisory`_, the + deprecated ``scrapy.downloadermiddlewares.decompression`` module has been + removed. + .. _release-2.11.0: @@ -2896,6 +2900,10 @@ Scrapy 1.8.4 (unreleased) to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security advisory`_ for more information. +- Also in relation with the `7j7m-v7m3-jqm7 security advisory`_, use of the + ``scrapy.downloadermiddlewares.decompression`` module is discouraged and + will trigger a warning. + .. _release-1.8.3: From 4f72b49f975a406784779dc19ede31364f92235f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 10:06:13 +0100 Subject: [PATCH 043/269] Fix namespaces nodename support for xmliter_lxml --- scrapy/utils/iterators.py | 23 +++++++++++++++++++++-- tests/test_utils_iterators.py | 5 ----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 6b89334e0..9c53ab524 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -12,11 +12,14 @@ from typing import ( List, Literal, Optional, + Tuple, Union, cast, overload, ) +from lxml import etree + from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode @@ -77,15 +80,31 @@ def xmliter( yield Selector(text=nodetext, type="xml") +def _resolve_xml_namespace(element_name: str, data: bytes) -> Tuple[str, str]: + if ":" not in element_name: + return element_name, None, None + reader: "SupportsReadClose[bytes]" = _StreamReader(data) + node_prefix, element_name = element_name.split(":", maxsplit=1) + ns_iterator = etree.iterparse( + reader, encoding=reader.encoding, events=("start-ns",) + ) + for event, (_prefix, _namespace) in ns_iterator: + if _prefix != node_prefix: + continue + return element_name, _prefix, _namespace + return f"{node_prefix}:{element_name}", None, None + + def xmliter_lxml( obj: Union[Response, str, bytes], nodename: str, namespace: Optional[str] = None, prefix: str = "x", ) -> Generator[Selector, Any, None]: - from lxml import etree + if not namespace: + nodename, prefix, namespace = _resolve_xml_namespace(nodename, obj) - reader = _StreamReader(obj) + reader: "SupportsReadClose[bytes]" = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 3598fa0bb..24f03155b 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,4 +1,3 @@ -from pytest import mark from twisted.trial import unittest from scrapy.http import Response, TextResponse, XmlResponse @@ -247,10 +246,6 @@ class XmliterTestCase(unittest.TestCase): class LxmlXmliterTestCase(XmliterTestCase): xmliter = staticmethod(xmliter_lxml) - @mark.xfail(reason="known bug of the current implementation") - def test_xmliter_namespaced_nodename(self): - super().test_xmliter_namespaced_nodename() - def test_xmliter_iterate_namespace(self): body = b""" From d50f436a73ef13fca8d3d9c302ae1a48e984a4a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 10:08:45 +0100 Subject: [PATCH 044/269] Enable huge_tree for xmliter_lxml --- scrapy/utils/iterators.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 9c53ab524..1c51c0c6a 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -86,7 +86,10 @@ def _resolve_xml_namespace(element_name: str, data: bytes) -> Tuple[str, str]: reader: "SupportsReadClose[bytes]" = _StreamReader(data) node_prefix, element_name = element_name.split(":", maxsplit=1) ns_iterator = etree.iterparse( - reader, encoding=reader.encoding, events=("start-ns",) + reader, + encoding=reader.encoding, + events=("start-ns",), + huge_tree=True, ) for event, (_prefix, _namespace) in ns_iterator: if _prefix != node_prefix: @@ -107,7 +110,10 @@ def xmliter_lxml( reader: "SupportsReadClose[bytes]" = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( - cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding + reader, + tag=tag, + encoding=reader.encoding, + huge_tree=True, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) for _, node in iterable: From 9655b0b8eb4bbc66b0fe540a19265b6342cf371b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 10:19:47 +0100 Subject: [PATCH 045/269] Mark slow tests, with their own tox env and CI job --- .github/workflows/tests-ubuntu.yml | 6 ++++ pytest.ini | 2 ++ tests/test_utils_response.py | 50 +++++++++++++++++------------- tox.ini | 6 ++++ 4 files changed, 42 insertions(+), 22 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 5ff92a571..7562cf22b 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -47,6 +47,9 @@ jobs: - python-version: "3.11" env: TOXENV: botocore + - python-version: "3.11" + env: + TOXENV: slow - python-version: "3.12.0-rc.2" env: @@ -57,6 +60,9 @@ jobs: - python-version: "3.12.0-rc.2" env: TOXENV: extra-deps + - python-version: "3.12.0-rc.2" + env: + TOXENV: slow steps: - uses: actions/checkout@v3 diff --git a/pytest.ini b/pytest.ini index 16983be5e..877fbcd1d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -17,10 +17,12 @@ addopts = --ignore=docs/topics/stats.rst --ignore=docs/topics/telnetconsole.rst --ignore=docs/utils + -m 'not slow' markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed requires_uvloop: marks tests as only enabled when uvloop is known to be working + slow: marks tests as slow, not executed by default filterwarnings = ignore:scrapy.downloadermiddlewares.decompression is deprecated ignore:Module scrapy.utils.reqser is deprecated diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 942584d92..93b9bacaf 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -4,6 +4,8 @@ from pathlib import Path from time import process_time from urllib.parse import urlparse +import pytest + from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Response, TextResponse from scrapy.settings.default_settings import DOWNLOAD_MAXSIZE @@ -201,36 +203,40 @@ class ResponseUtilsTest(unittest.TestCase): r5, _openfunc=check_base_url ), "Inject unique base url with conditional comment" - def test_open_in_browser_redos_comment(self): - MAX_CPU_TIME = 30 - # Exploit input from - # https://makenowjust-labs.github.io/recheck/playground/ - # for // (old pattern to remove comments). - body = b"->" +@pytest.mark.slow +def test_open_in_browser_redos_comment(): + MAX_CPU_TIME = 30 - response = HtmlResponse("https://example.com", body=body) + # Exploit input from + # https://makenowjust-labs.github.io/recheck/playground/ + # for // (old pattern to remove comments). + body = b"->" - start_time = process_time() + response = HtmlResponse("https://example.com", body=body) - open_in_browser(response, lambda url: True) + start_time = process_time() - end_time = process_time() - self.assertLess(end_time - start_time, MAX_CPU_TIME) + open_in_browser(response, lambda url: True) - def test_open_in_browser_redos_head(self): - MAX_CPU_TIME = 15 + end_time = process_time() + assert (end_time - start_time) < MAX_CPU_TIME - # Exploit input from - # https://makenowjust-labs.github.io/recheck/playground/ - # for /(|\s.*?>))/ (old pattern to find the head element). - body = b"|\s.*?>))/ (old pattern to find the head element). + body = b" Date: Fri, 15 Dec 2023 10:23:24 +0100 Subject: [PATCH 046/269] Restore the implementation of xmliter --- scrapy/utils/iterators.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 1c51c0c6a..b67029433 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -43,10 +43,10 @@ def xmliter( """ nodename_patt = re.escape(nodename) - DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]{1,1024}>\s*", re.S) + DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S) HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.S) - END_TAG_RE = re.compile(r"<\s*/([^\s>]{1,1024})\s*>", re.S) - NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]{,1024})=[^>\s]+)", re.S) + END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.S) + NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S) text = _body_or_str(obj) document_header_match = re.search(DOCUMENT_HEADER_RE, text) @@ -60,15 +60,13 @@ def xmliter( for tagname in reversed(re.findall(END_TAG_RE, header_end)): assert header_end_idx tag = re.search( - rf"<\s*{tagname}.{{,1024}}?xmlns[:=][^>]{{,1024}}>", - text[: header_end_idx[1]], - re.S, + rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S ) if tag: for x in re.findall(NAMESPACE_RE, tag.group()): namespaces[x[1]] = x[0] - r = re.compile(rf"<{nodename_patt}[\s>].{{,1024}}?", re.DOTALL) + r = re.compile(rf"<{nodename_patt}[\s>].*?", re.DOTALL) for match in r.finditer(text): nodetext = ( document_header From 150d96764b5a455c75315596ca8ba5ded0f416dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 11:42:55 +0100 Subject: [PATCH 047/269] Deprecate xmliter in favor of xmliter_lxml --- docs/faq.rst | 10 ++++--- docs/news.rst | 32 +++++++++++++++++----- scrapy/spiders/feed.py | 4 +-- scrapy/utils/iterators.py | 23 ++++++++++++++-- tests/test_utils_iterators.py | 35 +++++++++++++++++++++--- tests/test_utils_response.py | 50 +++++++++++++++++------------------ 6 files changed, 110 insertions(+), 44 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 20dd814df..657802fd3 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -297,9 +297,13 @@ build the DOM of the entire feed in memory, and this can be quite slow and consume a lot of memory. In order to avoid parsing all the entire feed at once in memory, you can use -the functions ``xmliter`` and ``csviter`` from ``scrapy.utils.iterators`` -module. In fact, this is what the feed spiders (see :ref:`topics-spiders`) use -under the cover. +the :func:`~scrapy.utils.iterators.xmliter_lxml` and +:func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what +:class:`~scrapy.spiders.XMLFeedSpider` uses. + +.. autofunction:: scrapy.utils.iterators.xmliter_lxml + +.. autofunction:: scrapy.utils.iterators.csviter Does Scrapy manage cookies automatically? ----------------------------------------- diff --git a/docs/news.rst b/docs/news.rst index c14815d06..57b99e94f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -10,12 +10,23 @@ Scrapy 2.11.1 (unreleased) **Security bug fix:** -- Fixed regular expressions susceptible to a `ReDoS attack`_ affecting the - ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider` and - the :func:`~scrapy.utils.response.open_in_browser` function. Please, see - the `cc65-xxvf-f7r9 security advisory`_ for more information. +- Addressed `ReDoS vulnerabilities`_: - .. _ReDoS attack: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS + - ``scrapy.utils.iterators.xmliter`` is now deprecated in favor of + :func:`~scrapy.utils.iterators.xmliter_lxml`, which + :class:`~scrapy.spiders.XMLFeedSpider` now uses. + + To minimize the impact of this change on existing code, + :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating + the node namespace with a prefix in the node name, and big files with + highly nested trees. + + - Fixed regular expressions in the implementation of the + :func:`~scrapy.utils.response.open_in_browser` function. + + Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. + + .. _ReDoS vulnerabilities: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS .. _cc65-xxvf-f7r9 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9 @@ -2892,8 +2903,15 @@ Scrapy 1.8.4 (unreleased) **Security bug fix:** -- Fixed regular expressions susceptible to a `ReDoS attack`_ affecting the - ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider`. +- Due to its `ReDoS vulnerabilities`_, ``scrapy.utils.iterators.xmliter`` is + now deprecated in favor of :func:`~scrapy.utils.iterators.xmliter_lxml`, + which :class:`~scrapy.spiders.XMLFeedSpider` now uses. + + To minimize the impact of this change on existing code, + :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating + the node namespace as a prefix in the node name, and big files with highly + nested trees when using lxml 4.2 or later. + Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 6afadc577..42675c76a 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spiders.rst from scrapy.exceptions import NotConfigured, NotSupported from scrapy.selector import Selector from scrapy.spiders import Spider -from scrapy.utils.iterators import csviter, xmliter +from scrapy.utils.iterators import csviter, xmliter_lxml from scrapy.utils.spider import iterate_spider_output @@ -84,7 +84,7 @@ class XMLFeedSpider(Spider): return self.parse_nodes(response, nodes) def _iternodes(self, response): - for node in xmliter(response, self.itertag): + for node in xmliter_lxml(response, self.itertag): self._register_namespaces(node) yield node diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index b67029433..7574e377a 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -17,9 +17,12 @@ from typing import ( cast, overload, ) +from warnings import warn from lxml import etree +from packaging.version import Version +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode @@ -29,6 +32,12 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_LXML_VERSION = Version(etree.__version__) +_LXML_HUGE_TREE_VERSION = Version("4.2") +_ITERPARSE_KWARGS = {} +if _LXML_VERSION >= _LXML_HUGE_TREE_VERSION: + _ITERPARSE_KWARGS["huge_tree"] = True + def xmliter( obj: Union[Response, str, bytes], nodename: str @@ -41,6 +50,16 @@ def xmliter( - a unicode string - a string encoded as utf-8 """ + warn( + ( + "xmliter is deprecated and its use strongly discouraged because " + "it is vulnerable to ReDoS attacks. Use xmliter_lxml instead. See " + "https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9" + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) + nodename_patt = re.escape(nodename) DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S) @@ -87,7 +106,7 @@ def _resolve_xml_namespace(element_name: str, data: bytes) -> Tuple[str, str]: reader, encoding=reader.encoding, events=("start-ns",), - huge_tree=True, + **_ITERPARSE_KWARGS, ) for event, (_prefix, _namespace) in ns_iterator: if _prefix != node_prefix: @@ -111,7 +130,7 @@ def xmliter_lxml( reader, tag=tag, encoding=reader.encoding, - huge_tree=True, + **_ITERPARSE_KWARGS, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) for _, node in iterable: diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 24f03155b..505cc276c 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,13 +1,14 @@ +import pytest from twisted.trial import unittest +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse, XmlResponse from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml from tests import get_testdata -class XmliterTestCase(unittest.TestCase): - xmliter = staticmethod(xmliter) - +class XmliterBaseTestCase: + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter(self): body = b""" @@ -39,6 +40,7 @@ class XmliterTestCase(unittest.TestCase): attrs, [("001", ["Name 1"], ["Type 1"]), ("002", ["Name 2"], ["Type 2"])] ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unusual_node(self): body = b""" @@ -52,6 +54,7 @@ class XmliterTestCase(unittest.TestCase): ] self.assertEqual(nodenames, [["matchme..."]]) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unicode(self): # example taken from https://github.com/scrapy/scrapy/issues/1665 body = """ @@ -111,6 +114,7 @@ class XmliterTestCase(unittest.TestCase): [("26", ["-"], ["80"]), ("21", ["Ab"], ["76"]), ("27", ["A"], ["27"])], ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_text(self): body = ( '' @@ -122,6 +126,7 @@ class XmliterTestCase(unittest.TestCase): [["one"], ["two"]], ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaces(self): body = b""" @@ -161,6 +166,7 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath("id/text()").getall(), []) self.assertEqual(node.xpath("price/text()").getall(), []) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename(self): body = b""" @@ -189,6 +195,7 @@ class XmliterTestCase(unittest.TestCase): ["http://www.mydummycompany.com/images/item1.jpg"], ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename_missing(self): body = b""" @@ -213,6 +220,7 @@ class XmliterTestCase(unittest.TestCase): with self.assertRaises(StopIteration): next(my_iter) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_exception(self): body = ( '' @@ -225,10 +233,12 @@ class XmliterTestCase(unittest.TestCase): self.assertRaises(StopIteration, next, iter) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_objtype_exception(self): i = self.xmliter(42, "product") self.assertRaises(TypeError, next, i) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_encoding(self): body = ( b'\n' @@ -243,7 +253,24 @@ class XmliterTestCase(unittest.TestCase): ) -class LxmlXmliterTestCase(XmliterTestCase): +class XmliterTestCase(XmliterBaseTestCase, unittest.TestCase): + xmliter = staticmethod(xmliter) + + def test_deprecation(self): + body = b""" + + + + + """ + with pytest.warns( + ScrapyDeprecationWarning, + match="xmliter", + ): + next(self.xmliter(body, "product")) + + +class LxmlXmliterTestCase(XmliterBaseTestCase, unittest.TestCase): xmliter = staticmethod(xmliter_lxml) def test_xmliter_iterate_namespace(self): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 93b9bacaf..1dbe187bf 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -203,40 +203,38 @@ class ResponseUtilsTest(unittest.TestCase): r5, _openfunc=check_base_url ), "Inject unique base url with conditional comment" + @pytest.mark.slow + def test_open_in_browser_redos_comment(self): + MAX_CPU_TIME = 30 -@pytest.mark.slow -def test_open_in_browser_redos_comment(): - MAX_CPU_TIME = 30 + # Exploit input from + # https://makenowjust-labs.github.io/recheck/playground/ + # for // (old pattern to remove comments). + body = b"->" - # Exploit input from - # https://makenowjust-labs.github.io/recheck/playground/ - # for // (old pattern to remove comments). - body = b"->" + response = HtmlResponse("https://example.com", body=body) - response = HtmlResponse("https://example.com", body=body) + start_time = process_time() - start_time = process_time() + open_in_browser(response, lambda url: True) - open_in_browser(response, lambda url: True) + end_time = process_time() + self.assertLess(end_time - start_time, MAX_CPU_TIME) - end_time = process_time() - assert (end_time - start_time) < MAX_CPU_TIME + @pytest.mark.slow + def test_open_in_browser_redos_head(self): + MAX_CPU_TIME = 15 + # Exploit input from + # https://makenowjust-labs.github.io/recheck/playground/ + # for /(|\s.*?>))/ (old pattern to find the head element). + body = b"|\s.*?>))/ (old pattern to find the head element). - body = b" Date: Fri, 15 Dec 2023 11:49:22 +0100 Subject: [PATCH 048/269] Minor naming changes --- scrapy/utils/iterators.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 7574e377a..f239630f4 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -101,18 +101,18 @@ def _resolve_xml_namespace(element_name: str, data: bytes) -> Tuple[str, str]: if ":" not in element_name: return element_name, None, None reader: "SupportsReadClose[bytes]" = _StreamReader(data) - node_prefix, element_name = element_name.split(":", maxsplit=1) + input_prefix, element_name = element_name.split(":", maxsplit=1) ns_iterator = etree.iterparse( reader, encoding=reader.encoding, events=("start-ns",), **_ITERPARSE_KWARGS, ) - for event, (_prefix, _namespace) in ns_iterator: - if _prefix != node_prefix: + for event, (prefix, namespace) in ns_iterator: + if prefix != input_prefix: continue - return element_name, _prefix, _namespace - return f"{node_prefix}:{element_name}", None, None + return element_name, prefix, namespace + return f"{input_prefix}:{element_name}", None, None def xmliter_lxml( From a49c8762dd163b60cc73c4486a662471cfa7ac7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 12:14:53 +0100 Subject: [PATCH 049/269] Avoid calling iterparse twice --- scrapy/utils/iterators.py | 42 +++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index f239630f4..8610e9b77 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -12,7 +12,6 @@ from typing import ( List, Literal, Optional, - Tuple, Union, cast, overload, @@ -97,43 +96,38 @@ def xmliter( yield Selector(text=nodetext, type="xml") -def _resolve_xml_namespace(element_name: str, data: bytes) -> Tuple[str, str]: - if ":" not in element_name: - return element_name, None, None - reader: "SupportsReadClose[bytes]" = _StreamReader(data) - input_prefix, element_name = element_name.split(":", maxsplit=1) - ns_iterator = etree.iterparse( - reader, - encoding=reader.encoding, - events=("start-ns",), - **_ITERPARSE_KWARGS, - ) - for event, (prefix, namespace) in ns_iterator: - if prefix != input_prefix: - continue - return element_name, prefix, namespace - return f"{input_prefix}:{element_name}", None, None - - def xmliter_lxml( obj: Union[Response, str, bytes], nodename: str, namespace: Optional[str] = None, prefix: str = "x", ) -> Generator[Selector, Any, None]: - if not namespace: - nodename, prefix, namespace = _resolve_xml_namespace(nodename, obj) - reader: "SupportsReadClose[bytes]" = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( reader, - tag=tag, encoding=reader.encoding, + events=("end", "start-ns"), **_ITERPARSE_KWARGS, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) - for _, node in iterable: + needs_namespace_resolution = not namespace and ":" in nodename + if needs_namespace_resolution: + prefix, nodename = nodename.split(":", maxsplit=1) + for event, data in iterable: + if event == "start-ns": + if needs_namespace_resolution: + _prefix, _namespace = data + if _prefix != prefix: + continue + namespace = _namespace + needs_namespace_resolution = False + selxpath = f"//{prefix}:{nodename}" + tag = f"{{{namespace}}}{nodename}" + continue + node = data + if node.tag != tag: + continue nodetext = etree.tostring(node, encoding="unicode") node.clear() xs = Selector(text=nodetext, type="xml") From ce9d290eff8b5992023ffa5e833881b83a0669c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 12:23:26 +0100 Subject: [PATCH 050/269] Remove the lxml version check for huge_tree on xmliter_lxml iterparse supports the option since lxml 2.2.1, it was the HTML parser that only got it in 4.2 --- docs/news.rst | 2 +- scrapy/utils/iterators.py | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 525ddbf40..fab8b6f20 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -19,7 +19,7 @@ Scrapy 2.11.1 (unreleased) To minimize the impact of this change on existing code, :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating the node namespace with a prefix in the node name, and big files with - highly nested trees. + highly nested trees when using libxml2 2.7+. - Fixed regular expressions in the implementation of the :func:`~scrapy.utils.response.open_in_browser` function. diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 8610e9b77..b6abe2e0c 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -19,7 +19,6 @@ from typing import ( from warnings import warn from lxml import etree -from packaging.version import Version from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse @@ -31,12 +30,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -_LXML_VERSION = Version(etree.__version__) -_LXML_HUGE_TREE_VERSION = Version("4.2") -_ITERPARSE_KWARGS = {} -if _LXML_VERSION >= _LXML_HUGE_TREE_VERSION: - _ITERPARSE_KWARGS["huge_tree"] = True - def xmliter( obj: Union[Response, str, bytes], nodename: str @@ -108,7 +101,7 @@ def xmliter_lxml( reader, encoding=reader.encoding, events=("end", "start-ns"), - **_ITERPARSE_KWARGS, + huge_tree=True, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) needs_namespace_resolution = not namespace and ":" in nodename From bc138ef8e958f4bac5a4413d40566efc2b59acfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 12:24:04 +0100 Subject: [PATCH 051/269] Minor release notes fix --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index fab8b6f20..f346d1239 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -2912,7 +2912,7 @@ Scrapy 1.8.4 (unreleased) To minimize the impact of this change on existing code, :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating the node namespace as a prefix in the node name, and big files with highly - nested trees when using lxml 4.2 or later. + nested trees when using libxml2 2.7+. Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. From c7c7a488b950806888691f58dda0b06478b98c7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 13:18:23 +0100 Subject: [PATCH 052/269] Fix typing issues --- scrapy/utils/iterators.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index b6abe2e0c..ab48e525f 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -95,10 +95,10 @@ def xmliter_lxml( namespace: Optional[str] = None, prefix: str = "x", ) -> Generator[Selector, Any, None]: - reader: "SupportsReadClose[bytes]" = _StreamReader(obj) + reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( - reader, + cast("SupportsReadClose[bytes]", reader), encoding=reader.encoding, events=("end", "start-ns"), huge_tree=True, @@ -109,6 +109,7 @@ def xmliter_lxml( prefix, nodename = nodename.split(":", maxsplit=1) for event, data in iterable: if event == "start-ns": + assert isinstance(data, tuple) if needs_namespace_resolution: _prefix, _namespace = data if _prefix != prefix: @@ -118,6 +119,7 @@ def xmliter_lxml( selxpath = f"//{prefix}:{nodename}" tag = f"{{{namespace}}}{nodename}" continue + assert isinstance(data, etree._Element) node = data if node.tag != tag: continue From 27781a85e738052e0441c81d773b3ec124194594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 13:52:12 +0100 Subject: [PATCH 053/269] Fix bad closing tags in XMLFeedSpider tests --- tests/test_spider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index 00da3d485..5c4007d87 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -149,10 +149,10 @@ class XMLFeedSpiderTest(SpiderTest): body = b""" - http://www.example.com/Special-Offers.html2009-08-16 + http://www.example.com/Special-Offers.html2009-08-16 - http://www.example.com/2009-08-16 + http://www.example.com/2009-08-16 """ response = XmlResponse(url="http://example.com/sitemap.xml", body=body) From 1fab844f7dd5fe622899c41ad8a0d28dd27c5089 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 20 Dec 2023 15:57:51 +0400 Subject: [PATCH 054/269] Pin the Python version for typing-tests. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 21ac4c3ff..f0788c0af 100644 --- a/tox.ini +++ b/tox.ini @@ -47,6 +47,7 @@ commands = mypy {posargs: scrapy tests} [testenv:typing-tests] +basepython = python3.8 deps = -rtests/requirements.txt {[testenv:typing]deps} From a72394a388a8c41ab07f4511b096d85e6de168fe Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 20 Dec 2023 16:14:53 +0400 Subject: [PATCH 055/269] Add tests for replace() with kwargs. --- tests_typing/test_http_request.mypy-testing | 14 ++++++++++++++ tests_typing/test_http_response.mypy-testing | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing index 636e6895f..665db9088 100644 --- a/tests_typing/test_http_request.mypy-testing +++ b/tests_typing/test_http_request.mypy-testing @@ -1,3 +1,5 @@ +from typing import Any, Dict + import pytest from scrapy import Request @@ -33,6 +35,9 @@ def mypy_test_copy(): req_copy = req.copy() reveal_type(req_copy) # R: scrapy.http.request.Request + +@pytest.mark.mypy_testing +def mypy_test_copy_subclass(): req = MyRequest("data:,") reveal_type(req) # R: __main__.MyRequest req_copy = req.copy() @@ -45,13 +50,22 @@ def mypy_test_replace(): reveal_type(req) # R: scrapy.http.request.Request req_copy = req.replace(body=b"a") reveal_type(req_copy) # R: scrapy.http.request.Request + kwargs: Dict[str, Any] = {} + req_copy2 = req.replace(body=b"a", **kwargs) + reveal_type(req_copy2) # R: Any + +@pytest.mark.mypy_testing +def mypy_test_replace_subclass(): req = MyRequest("data:,") reveal_type(req) # R: __main__.MyRequest req_copy = req.replace(body=b"a") reveal_type(req_copy) # R: __main__.MyRequest req_copy2 = req.replace(body=b"a", cls=MyRequest2) reveal_type(req_copy2) # R: __main__.MyRequest2 + kwargs: Dict[str, Any] = {} + req_copy3 = req.replace(body=b"a", cls=MyRequest2, **kwargs) + reveal_type(req_copy3) # R: __main__.MyRequest2 @pytest.mark.mypy_testing diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing index 2e58b4fbc..d58ac1027 100644 --- a/tests_typing/test_http_response.mypy-testing +++ b/tests_typing/test_http_response.mypy-testing @@ -1,3 +1,5 @@ +from typing import Any, Dict + import pytest from scrapy.http import HtmlResponse, Response, TextResponse @@ -24,6 +26,9 @@ def mypy_test_copy(): resp_copy = resp.copy() reveal_type(resp_copy) # R: scrapy.http.response.Response + +@pytest.mark.mypy_testing +def mypy_test_copy_subclass(): resp = HtmlResponse("data:,") reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse resp_copy = resp.copy() @@ -36,10 +41,19 @@ def mypy_test_replace(): reveal_type(resp) # R: scrapy.http.response.Response resp_copy = resp.replace(body=b"a") reveal_type(resp_copy) # R: scrapy.http.response.Response + kwargs: Dict[str, Any] = {} + resp_copy2 = resp.replace(body=b"a", **kwargs) + reveal_type(resp_copy2) # R: Any + +@pytest.mark.mypy_testing +def mypy_test_replace_subclass(): resp = HtmlResponse("data:,") reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse resp_copy = resp.replace(body=b"a") reveal_type(resp_copy) # R: scrapy.http.response.html.HtmlResponse resp_copy2 = resp.replace(body=b"a", cls=TextResponse) reveal_type(resp_copy2) # R: scrapy.http.response.text.TextResponse + kwargs: Dict[str, Any] = {} + resp_copy3 = resp.replace(body=b"a", cls=TextResponse, **kwargs) + reveal_type(resp_copy3) # R: scrapy.http.response.text.TextResponse From f56b5fc39ef3b322b8d0ad17fb424440bd79da0b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 20 Dec 2023 16:19:11 +0400 Subject: [PATCH 056/269] Bump typing deps. --- tox.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index f0788c0af..25b30d759 100644 --- a/tox.ini +++ b/tox.ini @@ -33,14 +33,14 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.6.1 - typing-extensions==4.8.0 + mypy==1.7.1 + typing-extensions==4.9.0 types-attrs==19.1.0 types-lxml==2023.10.21 - types-Pillow==10.1.0.0 - types-Pygments==2.16.0.0 + types-Pillow==10.1.0.2 + types-Pygments==2.17.0.0 types-pyOpenSSL==23.3.0.0 - types-setuptools==68.2.0.0 + types-setuptools==69.0.0.0 # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 w3lib >= 2.1.2 commands = From b095dd218fe64f2541079d691e3c2c68d2e03ff9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 30 Nov 2023 10:54:09 +0100 Subject: [PATCH 057/269] Extend Request.meta documentation (#5565) --- docs/topics/request-response.rst | 47 ++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index adf3d0f4a..8edf710bc 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -193,18 +193,47 @@ Request objects :meth:`replace`. .. attribute:: Request.meta + :value: {} - A dict that contains arbitrary metadata for this request. This dict is - empty for new Requests, and is usually populated by different Scrapy - components (extensions, middlewares, etc). So the data contained in this - dict depends on the extensions you have enabled. + A dictionary of arbitrary metadata for the request. - See :ref:`topics-request-meta` for a list of special meta keys - recognized by Scrapy. + You may extend request metadata as you see fit. - This dict is :doc:`shallow copied ` when the request is - cloned using the ``copy()`` or ``replace()`` methods, and can also be - accessed, in your spider, from the ``response.meta`` attribute. + Request metadata can also be accessed through the + :attr:`~scrapy.http.Response.meta` attribute of a response. + + To pass data from one spider callback to another, consider using + :attr:`cb_kwargs` instead. However, request metadata may be the right + choice in certain scenarios, such as to maintain some debugging data + across all follow-up requests (e.g. the source URL). + + A common use of request metadata is to define request-specific + parameters for Scrapy components (extensions, middlewares, etc.). For + example, if you set ``dont_retry`` to ``True``, + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` will never + retry that request, even if it fails. See :ref:`topics-request-meta`. + + You may also use request metadata in your custom Scrapy components, for + example, to keep request state information relevant to your component. + For example, + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses the + ``retry_times`` metadata key to keep track of how many times a request + has been retried so far. + + Copying all the metadata of a previous request into a new, follow-up + request in a spider callback is a bad practice, because request + metadata may include metadata set by Scrapy components that is not + meant to be copied into other requests. For example, copying the + ``retry_times`` metadata key into follow-up requests can lower the + amount of retries allowed for those follow-up requests. + + You should only copy all request metadata from one request to another + if the new request is meant to replace the old request, as is often the + case when returning a request from a :ref:`downloader middleware + ` method. + + Also mind that the :meth:`copy` and :meth:`replace` request methods + :doc:`shallow-copy ` request metadata. .. attribute:: Request.cb_kwargs From 369712ee50f7438c2863359f053cb1b221a42169 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 30 Nov 2023 11:01:22 +0100 Subject: [PATCH 058/269] =?UTF-8?q?SPM=20=E2=86=92=20Zyte=20API=20(#6163)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/practices.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index f64da22d8..b1b8c9e9c 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -288,9 +288,8 @@ Here are some tips to keep in mind when dealing with these kinds of sites: * use a pool of rotating IPs. For example, the free `Tor project`_ or paid services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a super proxy that you can attach your own proxies to. -* use a highly distributed downloader that circumvents bans internally, so you - can just focus on parsing clean pages. One example of such downloaders is - `Zyte Smart Proxy Manager`_ +* use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy + plugin `__ If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. @@ -301,4 +300,4 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _Common Crawl: https://commoncrawl.org/ .. _testspiders: https://github.com/scrapinghub/testspiders .. _scrapoxy: https://scrapoxy.io/ -.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ +.. _Zyte API: https://docs.zyte.com/zyte-api/get-started.html From 48a9a58ff27d24910b55a0ad5e6b014589c71115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 20 Dec 2023 12:47:18 +0100 Subject: [PATCH 059/269] =?UTF-8?q?Link=20to=20Zyte=E2=80=99s=20export=20g?= =?UTF-8?q?uides?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/feed-exports.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 700775e4b..f64bbac06 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -13,6 +13,11 @@ Scrapy provides this functionality out of the box with the Feed Exports, which allows you to generate feeds with the scraped items, using multiple serialization formats and storage backends. +This page provides detailed documentation for all feed export features. If you +are looking for a step-by-step guide, check out `Zyte’s export guides`_. + +.. _Zyte’s export guides: https://docs.zyte.com/web-scraping/guides/export/index.html#exporting-scraped-data + .. _topics-feed-format: Serialization formats From 2534a28ef032ae03e567859a498307b07ad34f64 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 25 Dec 2023 15:03:08 +0400 Subject: [PATCH 060/269] Bump mypy. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 25b30d759..8996b12a4 100644 --- a/tox.ini +++ b/tox.ini @@ -33,7 +33,7 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.7.1 + mypy==1.8.0 typing-extensions==4.9.0 types-attrs==19.1.0 types-lxml==2023.10.21 From badc7c5be9dcfaf7c8acbb87fa530f0477ec3c35 Mon Sep 17 00:00:00 2001 From: Chan Sau Yee <15137352+y26805@users.noreply.github.com> Date: Fri, 29 Dec 2023 20:32:51 +0900 Subject: [PATCH 061/269] Update black reference in docs (#6192) --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 2b3249601..d728338da 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -178,7 +178,7 @@ Scrapy: * We use `black `_ for code formatting. There is a hook in the pre-commit config that will automatically format your code before every commit. You can also - run black manually with ``tox -e black``. + run black manually with ``tox -e pre-commit``. * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. From 6127f7d27824de1f9847f7bb07f9755c955d9c3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 28 Dec 2023 12:25:01 +0100 Subject: [PATCH 062/269] Update quotes.toscrape.com page copies (#6190) --- docs/_tests/quotes.html | 2 +- docs/_tests/quotes1.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/_tests/quotes.html b/docs/_tests/quotes.html index 71aff8847..f4002ecd1 100644 --- a/docs/_tests/quotes.html +++ b/docs/_tests/quotes.html @@ -273,7 +273,7 @@ Quotes by: GoodReads.com

- Made with ❤ by Scrapinghub + Made with ❤ by Zyte

diff --git a/docs/_tests/quotes1.html b/docs/_tests/quotes1.html index 71aff8847..f4002ecd1 100644 --- a/docs/_tests/quotes1.html +++ b/docs/_tests/quotes1.html @@ -273,7 +273,7 @@ Quotes by: GoodReads.com

- Made with ❤ by Scrapinghub + Made with ❤ by Zyte

From 09a7efef7c75558c9ea198a00fc11ab26fb16ce5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 12 Jan 2024 18:30:41 +0400 Subject: [PATCH 063/269] Remove a defer.returnValue call. --- tests/test_feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 56967c0d5..ae5810fb8 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2300,7 +2300,7 @@ class BatchDeliveriesTest(FeedExportTestBase): content[feed["format"]].append(file.read_bytes()) finally: self.tearDown() - defer.returnValue(content) + return content @defer.inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): From c5dad41190551578c2973c34520952f26f75dc7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 2 Feb 2024 14:03:16 +0100 Subject: [PATCH 064/269] Speed up tests, remove comments without regexps --- .github/workflows/tests-ubuntu.yml | 3 -- scrapy/utils/response.py | 14 +++++++- tests/test_utils_response.py | 51 ++++++++++++++++++++++++++---- tox.ini | 6 ---- 4 files changed, 57 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 388ba9572..338c99584 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -50,9 +50,6 @@ jobs: - python-version: "3.12" env: TOXENV: botocore - - python-version: "3.12" - env: - TOXENV: slow steps: - uses: actions/checkout@v3 diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 4369e6439..fabfb1167 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -74,6 +74,18 @@ def response_httprepr(response: Response) -> bytes: return b"".join(values) +def _remove_html_comments(body): + start = body.find(b"", start + 1) + if end == -1: + return body[:start] + else: + body = body[:start] + body[end + 3 :] + start = body.find(b"|$)", b"", body) body = re.sub(rb"]*?>)", to_bytes(repl), body, count=1) ext = ".html" elif isinstance(response, TextResponse): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 1dbe187bf..db3c31b89 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -8,9 +8,9 @@ import pytest from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Response, TextResponse -from scrapy.settings.default_settings import DOWNLOAD_MAXSIZE from scrapy.utils.python import to_bytes from scrapy.utils.response import ( + _remove_html_comments, get_base_url, get_meta_refresh, open_in_browser, @@ -203,14 +203,13 @@ class ResponseUtilsTest(unittest.TestCase): r5, _openfunc=check_base_url ), "Inject unique base url with conditional comment" - @pytest.mark.slow def test_open_in_browser_redos_comment(self): - MAX_CPU_TIME = 30 + MAX_CPU_TIME = 0.001 # Exploit input from # https://makenowjust-labs.github.io/recheck/playground/ # for // (old pattern to remove comments). - body = b"->" + body = b"->" response = HtmlResponse("https://example.com", body=body) @@ -221,14 +220,13 @@ class ResponseUtilsTest(unittest.TestCase): end_time = process_time() self.assertLess(end_time - start_time, MAX_CPU_TIME) - @pytest.mark.slow def test_open_in_browser_redos_head(self): - MAX_CPU_TIME = 15 + MAX_CPU_TIME = 0.001 # Exploit input from # https://makenowjust-labs.github.io/recheck/playground/ # for /(|\s.*?>))/ (old pattern to find the head element). - body = b"b", + b"ab", + ), + ( + b"ac", + b"ac", + ), + ( + b"acccd", + b"acd", + ), + ( + b"ad", + b"ad", + ), + ), +) +def test_remove_html_comments(input_body, output_body): + assert ( + _remove_html_comments(input_body) == output_body + ), f"{_remove_html_comments(input_body)=} == {output_body=}" diff --git a/tox.ini b/tox.ini index e87d6a175..381da9773 100644 --- a/tox.ini +++ b/tox.ini @@ -221,9 +221,3 @@ setenv = {[pinned]setenv} commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} - - -[testenv:slow] -basepython = python3 -commands = - {[testenv]commands} -m 'slow' From 810aaa637da12a1f393291eb2b13aa0c8a163efb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 2 Feb 2024 14:04:28 +0100 Subject: [PATCH 065/269] Undo an unintended change --- .github/workflows/tests-ubuntu.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 338c99584..c883f958c 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -50,6 +50,7 @@ jobs: - python-version: "3.12" env: TOXENV: botocore + steps: - uses: actions/checkout@v3 From 5e5a92026e43023b80f7733844a2703c3f966009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 2 Feb 2024 14:06:45 +0100 Subject: [PATCH 066/269] Remove slow leftovers --- pytest.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index 877fbcd1d..16983be5e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -17,12 +17,10 @@ addopts = --ignore=docs/topics/stats.rst --ignore=docs/topics/telnetconsole.rst --ignore=docs/utils - -m 'not slow' markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed requires_uvloop: marks tests as only enabled when uvloop is known to be working - slow: marks tests as slow, not executed by default filterwarnings = ignore:scrapy.downloadermiddlewares.decompression is deprecated ignore:Module scrapy.utils.reqser is deprecated From 1c9d308accd38a91ffa92e3aff8912cd792070eb Mon Sep 17 00:00:00 2001 From: Andy <128531452+Andy-W-Developer@users.noreply.github.com> Date: Tue, 6 Feb 2024 00:52:01 +1300 Subject: [PATCH 067/269] Cover the deprecation and removal of response_httprepr in the release notes (#6216) --- docs/news.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 65d9c5181..d90e32560 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -62,6 +62,9 @@ Deprecation removals 1.0.0, use :attr:`CrawlerRunner.spider_loader ` instead. (:issue:`6010`) +- The :func:`scrapy.utils.response.response_httprepr` function, deprecated in + Scrapy 2.6.0, has now been removed. (:issue:`6111`) + Deprecations ~~~~~~~~~~~~ @@ -1157,6 +1160,9 @@ Deprecations Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider` first to set the :class:`~scrapy.Spider` object. +- :func:`scrapy.utils.response.response_httprepr` is now deprecated. + (:issue:`4972`) + New features ~~~~~~~~~~~~ From a55e933c11899997757bd4107738f9472d1d3c2e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Feb 2024 20:08:40 +0400 Subject: [PATCH 068/269] Release notes for 2.11.1 (#6150) --- docs/news.rst | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 0c202639e..c26cef22c 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,64 @@ Release notes ============= +.. _release-2.11.1: + +Scrapy 2.11.1 (YYYY-MM-DD) +-------------------------- + +Highlights: + +- Support for Twisted >= 23.8.0. + +- Documentation improvements. + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- The Twisted dependency is no longer restricted to < 23.8.0. (:issue:`6024`, + :issue:`6064`, :issue:`6142`) + +Bug fixes +~~~~~~~~~ + +- The OS signal handling code was refactored to no longer use private Twisted + functions. (:issue:`6024`, :issue:`6064`, :issue:`6112`) + +Documentation +~~~~~~~~~~~~~ + +- Improved documentation for :class:`~scrapy.crawler.Crawler` initialization + changes made in the 2.11.0 release. (:issue:`6057`, :issue:`6147`) + +- Extended documentation for :attr:`Request.meta `. + (:issue:`5565`) + +- Fixed the :reqmeta:`dont_merge_cookies` documentation. (:issue:`5936`, + :issue:`6077`) + +- Added a link to Zyte's export guides to the :ref:`feed exports + ` documentation. (:issue:`6183`) + +- Added a missing note about backward-incompatible changes in + :class:`~scrapy.exporters.PythonItemExporter` to the 2.11.0 release notes. + (:issue:`6060`, :issue:`6081`) + +- Added a missing note about removing the deprecated + ``scrapy.utils.boto.is_botocore()`` function to the 2.8.0 release notes. + (:issue:`6056`, :issue:`6061`) + +- Other documentation improvements. (:issue:`6128`, :issue:`6144`, + :issue:`6163`, :issue:`6190`, :issue:`6192`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added Python 3.12 to the CI configuration, re-enabled tests that were + disabled when the pre-release support was added. (:issue:`5985`, + :issue:`6083`, :issue:`6098`) + +- Fixed a test issue on PyPy 7.3.14. (:issue:`6204`, :issue:`6205`) + .. _release-2.11.0: Scrapy 2.11.0 (2023-09-18) From 6b88b3346c393f07c4e4481405c3fd1ab4cc58a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 14 Feb 2024 18:16:40 +0100 Subject: [PATCH 069/269] Set the release date of versions 2.11.1 and 1.8.4 --- docs/news.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 16e7e79a7..518632a5b 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.11.1: -Scrapy 2.11.1 (unreleased) +Scrapy 2.11.1 (2024-02-14) -------------------------- Highlights: @@ -2972,7 +2972,7 @@ affect subclasses: .. _release-1.8.4: -Scrapy 1.8.4 (unreleased) +Scrapy 1.8.4 (2024-02-14) ------------------------- **Security bug fixes:** From 502addc717b6b971425a9385359a382b8d0187a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 14 Feb 2024 18:17:48 +0100 Subject: [PATCH 070/269] =?UTF-8?q?Bump=20version:=202.11.0=20=E2=86=92=20?= =?UTF-8?q?2.11.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f76bf783d..6ce6e2a59 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.11.0 +current_version = 2.11.1 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 46b81d815..6ceb272ee 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.11.0 +2.11.1 From 2f1d345e74d19e33016f9e69fcda0bda9afb568d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 14 Feb 2024 18:59:01 +0100 Subject: [PATCH 071/269] Solve test issues --- ...st_downloadermiddleware_httpcompression.py | 24 +++++++++++++++++++ tests/test_utils_response.py | 4 ++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index f74fff218..9deb81c37 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -402,6 +402,10 @@ class HttpCompressionTest(TestCase): self._test_compression_bomb_setting("gzip") def test_compression_bomb_setting_zstd(self): + try: + import zstandard # noqa: F401 + except ImportError: + raise SkipTest("no zstd support (zstandard)") self._test_compression_bomb_setting("zstd") def _test_compression_bomb_spider_attr(self, compression_id): @@ -436,6 +440,10 @@ class HttpCompressionTest(TestCase): self._test_compression_bomb_spider_attr("gzip") def test_compression_bomb_spider_attr_zstd(self): + try: + import zstandard # noqa: F401 + except ImportError: + raise SkipTest("no zstd support (zstandard)") self._test_compression_bomb_spider_attr("zstd") def _test_compression_bomb_request_meta(self, compression_id): @@ -468,6 +476,10 @@ class HttpCompressionTest(TestCase): self._test_compression_bomb_request_meta("gzip") def test_compression_bomb_request_meta_zstd(self): + try: + import zstandard # noqa: F401 + except ImportError: + raise SkipTest("no zstd support (zstandard)") self._test_compression_bomb_request_meta("zstd") def _test_download_warnsize_setting(self, compression_id): @@ -510,6 +522,10 @@ class HttpCompressionTest(TestCase): self._test_download_warnsize_setting("gzip") def test_download_warnsize_setting_zstd(self): + try: + import zstandard # noqa: F401 + except ImportError: + raise SkipTest("no zstd support (zstandard)") self._test_download_warnsize_setting("zstd") def _test_download_warnsize_spider_attr(self, compression_id): @@ -554,6 +570,10 @@ class HttpCompressionTest(TestCase): self._test_download_warnsize_spider_attr("gzip") def test_download_warnsize_spider_attr_zstd(self): + try: + import zstandard # noqa: F401 + except ImportError: + raise SkipTest("no zstd support (zstandard)") self._test_download_warnsize_spider_attr("zstd") def _test_download_warnsize_request_meta(self, compression_id): @@ -596,6 +616,10 @@ class HttpCompressionTest(TestCase): self._test_download_warnsize_request_meta("gzip") def test_download_warnsize_request_meta_zstd(self): + try: + import zstandard # noqa: F401 + except ImportError: + raise SkipTest("no zstd support (zstandard)") self._test_download_warnsize_request_meta("zstd") diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index db3c31b89..37ef89e76 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -204,7 +204,7 @@ class ResponseUtilsTest(unittest.TestCase): ), "Inject unique base url with conditional comment" def test_open_in_browser_redos_comment(self): - MAX_CPU_TIME = 0.001 + MAX_CPU_TIME = 0.02 # Exploit input from # https://makenowjust-labs.github.io/recheck/playground/ @@ -221,7 +221,7 @@ class ResponseUtilsTest(unittest.TestCase): self.assertLess(end_time - start_time, MAX_CPU_TIME) def test_open_in_browser_redos_head(self): - MAX_CPU_TIME = 0.001 + MAX_CPU_TIME = 0.02 # Exploit input from # https://makenowjust-labs.github.io/recheck/playground/ From bccb4cf18ba38c8bf09d61d19e0ffabaf15554b1 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Wed, 14 Feb 2024 12:29:29 -0600 Subject: [PATCH 072/269] fix: LxmlLinkExtractor unique_list missing key --- scrapy/linkextractors/lxmlhtml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 23cbd0116..98781ba7f 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -248,5 +248,5 @@ class LxmlLinkExtractor: links = self._extract_links(doc, response.url, response.encoding, base_url) all_links.extend(self._process_links(links)) if self.link_extractor.unique: - return unique_list(all_links) + return unique_list(all_links, key=self.link_extractor.link_key) return all_links From 660e3b19532c50eac7d135549e594b7f98285184 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Thu, 15 Feb 2024 16:55:08 -0600 Subject: [PATCH 073/269] update: docs/topics/items.rst --- docs/topics/items.rst | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 3c38ac2dc..97ed7a900 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -399,12 +399,7 @@ In code that receives an item, such as methods of :ref:`item pipelines `, it is a good practice to use the :class:`~itemadapter.ItemAdapter` class and the :func:`~itemadapter.is_item` function to write code that works for -any :ref:`supported item type `: - -.. autoclass:: itemadapter.ItemAdapter - -.. autofunction:: itemadapter.is_item - +any supported item type. Other classes related to items ============================== From 3e7b704c08aacd51a8a7589e32c73c7754582eed Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Thu, 15 Feb 2024 16:57:44 -0600 Subject: [PATCH 074/269] update: docs/topics/selectors.rst --- docs/topics/selectors.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 4a64d530b..c841400b6 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -1032,11 +1032,6 @@ whereas the CSS lookup is translated into XPath and thus runs more efficiently, so performance-wise its uses are limited to situations that are not easily described with CSS selectors. -Parsel also simplifies adding your own XPath extensions. - -.. autofunction:: parsel.xpathfuncs.set_xpathfunc - - .. _topics-selectors-ref: Built-in Selectors reference From 9bb973dc54766a0f8d10eca0947d11f195c1a1be Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Fri, 16 Feb 2024 19:25:38 +0800 Subject: [PATCH 075/269] Refactor LogStats extension to log IPM and RPM to stats on spider_close (#4111) --- scrapy/extensions/logstats.py | 44 +++++++++++++++++++------ tests/test_logstats.py | 62 +++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 10 deletions(-) create mode 100644 tests/test_logstats.py diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 78874a6db..9f63e9c4b 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -9,7 +9,10 @@ logger = logging.getLogger(__name__) class LogStats: - """Log basic scraping stats periodically""" + """Log basic scraping stats periodically like: + * RPM - Requests per Minute + * IPM - Items per Minute + """ def __init__(self, stats, interval=60.0): self.stats = stats @@ -35,24 +38,45 @@ class LogStats: self.task.start(self.interval) def log(self, spider): - items = self.stats.get_value("item_scraped_count", 0) - pages = self.stats.get_value("response_received_count", 0) - irate = (items - self.itemsprev) * self.multiplier - prate = (pages - self.pagesprev) * self.multiplier - self.pagesprev, self.itemsprev = pages, items + self.calculate_stats() msg = ( "Crawled %(pages)d pages (at %(pagerate)d pages/min), " "scraped %(items)d items (at %(itemrate)d items/min)" ) log_args = { - "pages": pages, - "pagerate": prate, - "items": items, - "itemrate": irate, + "pages": self.pages, + "pagerate": self.prate, + "items": self.items, + "itemrate": self.irate, } logger.info(msg, log_args, extra={"spider": spider}) + def calculate_stats(self): + self.items = self.stats.get_value("item_scraped_count", 0) + self.pages = self.stats.get_value("response_received_count", 0) + self.irate = (self.items - self.itemsprev) * self.multiplier + self.prate = (self.pages - self.pagesprev) * self.multiplier + self.pagesprev, self.itemsprev = self.pages, self.items + def spider_closed(self, spider, reason): if self.task and self.task.running: self.task.stop() + + rpm_final, ipm_final = self.calculate_final_stats(spider) + self.stats.set_value("responses_per_minute", rpm_final) + self.stats.set_value("items_per_minute", ipm_final) + + def calculate_final_stats(self, spider): + start_time = self.stats.get_value("start_time") + finished_time = self.stats.get_value("finished_time") + + if not start_time or not finished_time: + return None, None + + mins_elapsed = (finished_time - start_time).seconds / 60 + + items = self.stats.get_value("item_scraped_count", 0) + pages = self.stats.get_value("response_received_count", 0) + + return (pages / mins_elapsed), (items / mins_elapsed) diff --git a/tests/test_logstats.py b/tests/test_logstats.py new file mode 100644 index 000000000..d87285df7 --- /dev/null +++ b/tests/test_logstats.py @@ -0,0 +1,62 @@ +import unittest +from datetime import datetime + +from scrapy.extensions.logstats import LogStats +from scrapy.utils.test import get_crawler +from tests.spiders import SimpleSpider + + +class TestLogStats(unittest.TestCase): + def setUp(self): + self.crawler = get_crawler(SimpleSpider) + self.spider = self.crawler._create_spider("spidey") + self.stats = self.crawler.stats + + self.stats.set_value("response_received_count", 4802) + self.stats.set_value("item_scraped_count", 3201) + + def test_stats_calculations(self): + logstats = LogStats.from_crawler(self.crawler) + + with self.assertRaises(AttributeError): + logstats.pagesprev + logstats.itemsprev + + logstats.spider_opened(self.spider) + self.assertEqual(logstats.pagesprev, 4802) + self.assertEqual(logstats.itemsprev, 3201) + + logstats.calculate_stats() + self.assertEqual(logstats.items, 3201) + self.assertEqual(logstats.pages, 4802) + self.assertEqual(logstats.irate, 0.0) + self.assertEqual(logstats.prate, 0.0) + self.assertEqual(logstats.pagesprev, 4802) + self.assertEqual(logstats.itemsprev, 3201) + + # Simulate what happens after a minute + self.stats.set_value("response_received_count", 5187) + self.stats.set_value("item_scraped_count", 3492) + logstats.calculate_stats() + self.assertEqual(logstats.items, 3492) + self.assertEqual(logstats.pages, 5187) + self.assertEqual(logstats.irate, 291.0) + self.assertEqual(logstats.prate, 385.0) + self.assertEqual(logstats.pagesprev, 5187) + self.assertEqual(logstats.itemsprev, 3492) + + # Simulate when spider closes after running for 30 mins + self.stats.set_value("start_time", datetime.fromtimestamp(1655100172)) + self.stats.set_value("finished_time", datetime.fromtimestamp(1655101972)) + logstats.spider_closed(self.spider, "test reason") + self.assertEqual(self.stats.get_value("responses_per_minute"), 172.9) + self.assertEqual(self.stats.get_value("items_per_minute"), 116.4) + + def test_stats_calculations_no_time(self): + """The stat values should be None since the start and finish time are + not available. + """ + logstats = LogStats.from_crawler(self.crawler) + logstats.spider_closed(self.spider, "test reason") + self.assertIsNone(self.stats.get_value("responses_per_minute")) + self.assertIsNone(self.stats.get_value("items_per_minute")) From 36f72877ba8863a7fc39383e79f478400f6c09e9 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Fri, 16 Feb 2024 10:39:16 -0600 Subject: [PATCH 076/269] update: docs/topics/selectors.rst --- docs/topics/selectors.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index c841400b6..e32fc2b70 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -1032,6 +1032,9 @@ whereas the CSS lookup is translated into XPath and thus runs more efficiently, so performance-wise its uses are limited to situations that are not easily described with CSS selectors. +Parsel also simplifies adding your own XPath extensions with +:func:`~parsel.xpathfuncs.set_xpathfunc`. + .. _topics-selectors-ref: Built-in Selectors reference From 5e51417a485f296354e9639f85fb0b51a4a3e533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 16 Feb 2024 20:10:52 +0100 Subject: [PATCH 077/269] Add tests, fix canonicalize passing --- scrapy/linkextractors/lxmlhtml.py | 2 +- tests/test_linkextractors.py | 112 ++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 98781ba7f..7abdaaec4 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -153,7 +153,7 @@ class LxmlLinkExtractor: unique=unique, process=process_value, strip=strip, - canonicalized=canonicalize, + canonicalized=not canonicalize, ) self.allow_res = [ x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 18e9608c1..f23b8988e 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -745,6 +745,118 @@ class Base: lx = self.extractor_cls() self.assertIsInstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls) + def test_link_extractor_aggregation(self): + """When a parameter like restrict_css is used, the underlying + implementation calls its internal link extractor once per selector + matching the specified restrictions, and then aggregates the + extracted links. + + Test that aggregation respects the unique and canonicalize + parameters. + """ + # unique=True (default), canonicalize=False (default) + lx = self.extractor_cls(restrict_css=("div",)) + response = HtmlResponse( + "https://example.com", + body=b""" +
+ a1 + b1 +
+
+ a2 + b2 +
+ """, + ) + actual = lx.extract_links(response) + self.assertEqual( + actual, + [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/b?b=2&a=1", text="b2"), + ], + ) + + # unique=True (default), canonicalize=True + lx = self.extractor_cls(restrict_css=("div",), canonicalize=True) + response = HtmlResponse( + "https://example.com", + body=b""" +
+ a1 + b1 +
+
+ a2 + b2 +
+ """, + ) + actual = lx.extract_links(response) + self.assertEqual( + actual, + [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + ], + ) + + # unique=False, canonicalize=False (default) + lx = self.extractor_cls(restrict_css=("div",), unique=False) + response = HtmlResponse( + "https://example.com", + body=b""" +
+ a1 + b1 +
+
+ a2 + b2 +
+ """, + ) + actual = lx.extract_links(response) + self.assertEqual( + actual, + [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/a", text="a2"), + Link(url="https://example.com/b?b=2&a=1", text="b2"), + ], + ) + + # unique=False, canonicalize=True + lx = self.extractor_cls( + restrict_css=("div",), unique=False, canonicalize=True + ) + response = HtmlResponse( + "https://example.com", + body=b""" +
+ a1 + b1 +
+
+ a2 + b2 +
+ """, + ) + actual = lx.extract_links(response) + self.assertEqual( + actual, + [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/a", text="a2"), + Link(url="https://example.com/b?a=1&b=2", text="b2"), + ], + ) + class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): extractor_cls = LxmlLinkExtractor From c4e4b9b56e7fe10c5e7472b152dd47253a97af5b Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 20 Feb 2024 14:50:16 +0500 Subject: [PATCH 078/269] Add a SECURITY.md file (#6051) --- .bumpversion.cfg | 4 ++++ SECURITY.md | 12 ++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 SECURITY.md diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 6ce6e2a59..968a34d96 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -5,3 +5,7 @@ tag = True tag_name = {new_version} [bumpversion:file:scrapy/VERSION] + +[bumpversion:file:SECURITY.md] +parse = (?P\d+)\.(?P\d+)\.x +serialize = {major}.{minor}.x diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..51305d95e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,12 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +| ------- | ------------------ | +| 2.11.x | :white_check_mark: | +| < 2.11.x | :x: | + +## Reporting a Vulnerability + +Please report the vulnerability using https://github.com/scrapy/scrapy/security/advisories/new. From ee1189512f652fae72f013c9d4759976b8b69994 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Tue, 20 Feb 2024 08:47:29 -0300 Subject: [PATCH 079/269] Replace urlparse with urlparse_cached where possible (#6229) --- docs/topics/media-pipeline.rst | 8 ++++---- scrapy/core/http2/stream.py | 6 +++--- scrapy/downloadermiddlewares/redirect.py | 4 ++-- tests/CrawlerRunner/ip_address.py | 3 ++- tests/test_http_cookies.py | 10 +++++----- tests/test_http_request.py | 11 ++++++----- tests/test_scheduler_base.py | 5 +++-- 7 files changed, 25 insertions(+), 22 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index da0587aa4..c96dd0f99 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -532,14 +532,14 @@ See here the methods that you can override in your custom Files Pipeline: .. code-block:: python from pathlib import PurePosixPath - from urllib.parse import urlparse + from scrapy.utils.httpobj import urlparse_cached from scrapy.pipelines.files import FilesPipeline class MyFilesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None, *, item=None): - return "files/" + PurePosixPath(urlparse(request.url).path).name + return "files/" + PurePosixPath(urlparse_cached(request).path).name Similarly, you can use the ``item`` to determine the file path based on some item property. @@ -690,14 +690,14 @@ See here the methods that you can override in your custom Images Pipeline: .. code-block:: python from pathlib import PurePosixPath - from urllib.parse import urlparse + from scrapy.utils.httpobj import urlparse_cached from scrapy.pipelines.images import ImagesPipeline class MyImagesPipeline(ImagesPipeline): def file_path(self, request, response=None, info=None, *, item=None): - return "files/" + PurePosixPath(urlparse(request.url).path).name + return "files/" + PurePosixPath(urlparse_cached(request).path).name Similarly, you can use the ``item`` to determine the file path based on some item property. diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 39d5921f4..0f282d83d 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -2,7 +2,6 @@ import logging from enum import Enum from io import BytesIO from typing import TYPE_CHECKING, Dict, List, Optional, Tuple -from urllib.parse import urlparse from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError @@ -15,6 +14,7 @@ from twisted.web.client import ResponseFailed from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes +from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol @@ -185,7 +185,7 @@ class Stream: def check_request_url(self) -> bool: # Make sure that we are sending the request to the correct URL - url = urlparse(self._request.url) + url = urlparse_cached(self._request) return ( url.netloc == str(self._protocol.metadata["uri"].host, "utf-8") or url.netloc == str(self._protocol.metadata["uri"].netloc, "utf-8") @@ -194,7 +194,7 @@ class Stream: ) def _get_request_headers(self) -> List[Tuple[str, str]]: - url = urlparse(self._request.url) + url = urlparse_cached(self._request) path = url.path if url.query: diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 83afdf7d7..24089afea 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any, List, Union, cast -from urllib.parse import urljoin, urlparse +from urllib.parse import urljoin from w3lib.url import safe_url_string @@ -125,7 +125,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): assert response.headers["Location"] is not None location = safe_url_string(response.headers["Location"]) if response.headers["Location"].startswith(b"//"): - request_scheme = urlparse(request.url).scheme + request_scheme = urlparse_cached(request).scheme location = request_scheme + "://" + location.lstrip("/") redirected_url = urljoin(request.url, location) diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 23260ab0d..5bf7512bc 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -9,6 +9,7 @@ from twisted.python.runtime import platform from scrapy import Request, Spider from scrapy.crawler import CrawlerRunner +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import configure_logging from tests.mockserver import MockDNSServer, MockServer @@ -30,7 +31,7 @@ class LocalhostSpider(Spider): yield Request(self.url) def parse(self, response): - netloc = urlparse(response.url).netloc + netloc = urlparse_cached(response).netloc host = netloc.split(":")[0] self.logger.info(f"Host: {host}") self.logger.info(f"Type: {type(response.ip_address)}") diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 9e43b72b0..8b5554914 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -1,8 +1,8 @@ from unittest import TestCase -from urllib.parse import urlparse from scrapy.http import Request, Response from scrapy.http.cookies import WrappedRequest, WrappedResponse +from scrapy.utils.httpobj import urlparse_cached class WrappedRequestTest(TestCase): @@ -17,12 +17,12 @@ class WrappedRequestTest(TestCase): self.assertEqual(self.wrapped.full_url, self.request.url) def test_get_host(self): - self.assertEqual(self.wrapped.get_host(), urlparse(self.request.url).netloc) - self.assertEqual(self.wrapped.host, urlparse(self.request.url).netloc) + self.assertEqual(self.wrapped.get_host(), urlparse_cached(self.request).netloc) + self.assertEqual(self.wrapped.host, urlparse_cached(self.request).netloc) def test_get_type(self): - self.assertEqual(self.wrapped.get_type(), urlparse(self.request.url).scheme) - self.assertEqual(self.wrapped.type, urlparse(self.request.url).scheme) + self.assertEqual(self.wrapped.get_type(), urlparse_cached(self.request).scheme) + self.assertEqual(self.wrapped.type, urlparse_cached(self.request).scheme) def test_is_unverifiable(self): self.assertFalse(self.wrapped.is_unverifiable()) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 6dc9ec8b7..04fcaa231 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -5,7 +5,7 @@ import warnings import xmlrpc.client from typing import Any, Dict, List from unittest import mock -from urllib.parse import parse_qs, unquote_to_bytes, urlparse +from urllib.parse import parse_qs, unquote_to_bytes from scrapy.http import ( FormRequest, @@ -16,6 +16,7 @@ from scrapy.http import ( XmlRpcRequest, ) from scrapy.http.request import NO_CALLBACK +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode @@ -617,8 +618,8 @@ class FormRequestTest(RequestTest): method="GET", formdata=(("foo", "bar"), ("foo", "baz")), ) - self.assertEqual(urlparse(req.url).hostname, "www.example.com") - self.assertEqual(urlparse(req.url).query, "foo=bar&foo=baz") + self.assertEqual(urlparse_cached(req).hostname, "www.example.com") + self.assertEqual(urlparse_cached(req).query, "foo=bar&foo=baz") def test_from_response_override_duplicate_form_key(self): response = _buildresponse( @@ -666,8 +667,8 @@ class FormRequestTest(RequestTest): response, formdata={"one": ["two", "three"], "six": "seven"} ) self.assertEqual(r1.method, "GET") - self.assertEqual(urlparse(r1.url).hostname, "www.example.com") - self.assertEqual(urlparse(r1.url).path, "/this/get.php") + self.assertEqual(urlparse_cached(r1).hostname, "www.example.com") + self.assertEqual(urlparse_cached(r1).path, "/this/get.php") fs = _qs(r1) self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"}) self.assertEqual(set(fs[b"one"]), {b"two", b"three"}) diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 76ca777a8..5db2e4e50 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -1,6 +1,6 @@ from typing import Dict, Optional from unittest import TestCase -from urllib.parse import urljoin, urlparse +from urllib.parse import urljoin from testfixtures import LogCapture from twisted.internet import defer @@ -9,6 +9,7 @@ from twisted.trial.unittest import TestCase as TwistedTestCase from scrapy.core.scheduler import BaseScheduler from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.request import fingerprint from scrapy.utils.test import get_crawler from tests.mockserver import MockServer @@ -57,7 +58,7 @@ class TestSpider(Spider): self.start_urls = map(mockserver.url, PATHS) def parse(self, response): - return {"path": urlparse(response.url).path} + return {"path": urlparse_cached(response).path} class InterfaceCheckMixin: From f096f17fa4ac1307fa1c81ae082bb52e9f86653a Mon Sep 17 00:00:00 2001 From: Elias Ram Date: Tue, 20 Feb 2024 20:32:02 +0100 Subject: [PATCH 080/269] test #6 added tests for check command --- tests/test_command_check.py | 55 +++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 592494aba..d503628b8 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -1,3 +1,8 @@ +import sys +from io import StringIO +from unittest.mock import Mock, PropertyMock, patch + +from scrapy.commands.check import Command from tests.test_commands import CommandTest @@ -94,3 +99,53 @@ class CheckSpider(scrapy.Spider): raise Exception('SCRAPY_CHECK not set') """ self._test_contract(parse_def=parse_def) + + @patch("scrapy.commands.check.ContractsManager") + def test_run_with_opts_list_prints_spider(self, cm_cls_mock): + output = StringIO() + sys.stdout = output + cmd = Command() + cmd.settings = Mock(getwithbase=Mock(return_value={})) + cm_cls_mock.return_value = cm_mock = Mock() + spider_loader_mock = Mock() + cmd.crawler_process = Mock(spider_loader=spider_loader_mock) + spider_name = "FakeSpider" + spider_cls_mock = Mock() + type(spider_cls_mock).name = PropertyMock(return_value=spider_name) + spider_loader_mock.load.side_effect = lambda x: {spider_name: spider_cls_mock}[ + x + ] + tested_methods = ["fakeMethod1", "fakeMethod2"] + cm_mock.tested_methods_from_spidercls.side_effect = lambda x: { + spider_cls_mock: tested_methods + }[x] + + cmd.run([spider_name], Mock(list=True)) + + self.assertEqual( + "FakeSpider\n * fakeMethod1\n * fakeMethod2\n", output.getvalue() + ) + sys.stdout = sys.__stdout__ + + @patch("scrapy.commands.check.ContractsManager") + def test_run_without_opts_list_does_not_crawl_spider_with_no_tested_methods( + self, cm_cls_mock + ): + cmd = Command() + cmd.settings = Mock(getwithbase=Mock(return_value={})) + cm_cls_mock.return_value = cm_mock = Mock() + spider_loader_mock = Mock() + cmd.crawler_process = Mock(spider_loader=spider_loader_mock) + spider_name = "FakeSpider" + spider_cls_mock = Mock() + spider_loader_mock.load.side_effect = lambda x: {spider_name: spider_cls_mock}[ + x + ] + tested_methods = [] + cm_mock.tested_methods_from_spidercls.side_effect = lambda x: { + spider_cls_mock: tested_methods + }[x] + + cmd.run([spider_name], Mock(list=False)) + + cmd.crawler_process.crawl.assert_not_called() From e8e6d28479a0479361cd3de0fb854c243ed684b1 Mon Sep 17 00:00:00 2001 From: Elias Ram Date: Tue, 20 Feb 2024 20:41:18 +0100 Subject: [PATCH 081/269] test #8 added tests for LxmlLinkExtractor --- tests/test_linkextractors.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 18e9608c1..66a30c635 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -2,6 +2,7 @@ import pickle import re import unittest from typing import Optional +from unittest.mock import Mock from packaging.version import Version from pytest import mark @@ -851,3 +852,36 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): ), ], ) + + def test_link_allowed_is_false_with_empty_url(self): + mock_link = Mock() + mock_link.url = "" + expected = False + + actual = LxmlLinkExtractor()._link_allowed(mock_link) + + self.assertEqual(expected, actual) + + def test_link_allowed_is_false_with_bad_url_prefix(self): + mock_link = Mock() + mock_link.url = "htp://should_be_http.com" + expected = False + + actual = LxmlLinkExtractor()._link_allowed(mock_link) + + self.assertEqual(expected, actual) + + def test_link_allowed_is_false_with_missing_url_prefix(self): + mock_link = Mock() + mock_link.url = "should_have_prefix.com" + expected = False + + actual = LxmlLinkExtractor()._link_allowed(mock_link) + + self.assertEqual(expected, actual) + + def test_link_allowed_raises_with_none_url(self): + mock_link = Mock() + mock_link.url = None + + self.assertRaises(AttributeError, LxmlLinkExtractor()._link_allowed, mock_link) From e27d320c3cde3c965e61a674e8880043943cf17a Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Tue, 20 Feb 2024 23:58:39 +0100 Subject: [PATCH 082/269] test #3 Increased branch coverage for form.py --- tests/test_http_request.py | 56 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 6dc9ec8b7..510ae74ba 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1642,6 +1642,62 @@ class JsonRequestTest(RequestTest): self.assertEqual(kwargs["ensure_ascii"], True) self.assertEqual(kwargs["allow_nan"], True) + def test_form_response_with_invalid_formdata_type_error(self): + """Test that a form response with invalid form data throws a type error""" + response = _buildresponse( + """ +
+ +
+ """ + ) + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formdata=123) + + self.assertIn( + "formdata should be a dict or iterable of tuples", str(context.exception) + ) + + def test_form_response_with_custom_invalid_formdata_value_error(self): + """Test that a form response with invalid form data throws a value error""" + response = _buildresponse( + """ +
+ +
+ """ + ) + + class CustomFormdata: + def __iter__(self): + raise ValueError("Custom iteration error for testing") + + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formdata=CustomFormdata()) + + self.assertIn( + "formdata should be a dict or iterable of tuples", str(context.exception) + ) + + def test_get_form_with_xpath_no_form_parent(self): + """Test that _get_from raised a ValueError when an XPath selects an element + not nested within a
and no parent is found""" + response = _buildresponse( + """ +
+

This paragraph is not inside a form.

+
+ + +
+ """ + ) + + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') + + self.assertIn("No
element found with", str(context.exception)) + def tearDown(self): warnings.resetwarnings() super().tearDown() From e2a0c85f1167c7c32219eaf095c1818b2c74702c Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Wed, 21 Feb 2024 02:21:24 +0100 Subject: [PATCH 083/269] doc #3 Clarified test comments --- tests/test_http_request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 510ae74ba..95e4a7be0 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1643,7 +1643,7 @@ class JsonRequestTest(RequestTest): self.assertEqual(kwargs["allow_nan"], True) def test_form_response_with_invalid_formdata_type_error(self): - """Test that a form response with invalid form data throws a type error""" + """Test that a ValueError is raised for non-iterable and non-dict formdata input""" response = _buildresponse( """ @@ -1659,7 +1659,7 @@ class JsonRequestTest(RequestTest): ) def test_form_response_with_custom_invalid_formdata_value_error(self): - """Test that a form response with invalid form data throws a value error""" + """Test that a ValueError is raised for fault-inducing iterable formdata input""" response = _buildresponse( """ From 12b4417c56d8aa76cbe3a36c026962612453ee6e Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Wed, 21 Feb 2024 02:30:50 +0100 Subject: [PATCH 084/269] test #22 Improve json_request.py coverage --- tests/test_http_request.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 6dc9ec8b7..d1c435468 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1642,6 +1642,26 @@ class JsonRequestTest(RequestTest): self.assertEqual(kwargs["ensure_ascii"], True) self.assertEqual(kwargs["allow_nan"], True) + def test_replacement_both_body_and_data_warns(self): + """Test that we can get a warning if both body and data are passed for branch coverage""" + body1 = None + body2 = b"body" + data1 = { + "name1": "value1", + } + data2 = { + "name2": "value2", + } + r1 = self.request_class(url="http://www.example.com/", data=data1, body=body1) + + with mock.patch("warnings.warn") as mock_warn: + r1.replace(data=data2, body=body2) + mock_warn.assert_called_once() + (warning_message,), _ = mock_warn.call_args + self.assertIn( + "Both body and data passed. data will be ignored", warning_message + ) + def tearDown(self): warnings.resetwarnings() super().tearDown() From bc036542a82ec054dd6a36b4b928a9bc6ae48e63 Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Wed, 21 Feb 2024 02:35:45 +0100 Subject: [PATCH 085/269] refactor #3 Moved tests to FormRequestTest --- tests/test_http_request.py | 112 ++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 95e4a7be0..39afc5fd1 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1426,6 +1426,62 @@ class FormRequestTest(RequestTest): r = self.request_class.from_response(response) self.assertEqual(r.method, expected) + def test_form_response_with_invalid_formdata_type_error(self): + """Test that a ValueError is raised for non-iterable and non-dict formdata input""" + response = _buildresponse( + """ + + +
+ """ + ) + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formdata=123) + + self.assertIn( + "formdata should be a dict or iterable of tuples", str(context.exception) + ) + + def test_form_response_with_custom_invalid_formdata_value_error(self): + """Test that a ValueError is raised for fault-inducing iterable formdata input""" + response = _buildresponse( + """ +
+ +
+ """ + ) + + class CustomFormdata: + def __iter__(self): + raise ValueError("Custom iteration error for testing") + + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formdata=CustomFormdata()) + + self.assertIn( + "formdata should be a dict or iterable of tuples", str(context.exception) + ) + + def test_get_form_with_xpath_no_form_parent(self): + """Test that _get_from raised a ValueError when an XPath selects an element + not nested within a
and no parent is found""" + response = _buildresponse( + """ +
+

This paragraph is not inside a form.

+
+ + +
+ """ + ) + + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') + + self.assertIn("No
element found with", str(context.exception)) + def _buildresponse(body, **kwargs): kwargs.setdefault("body", body) @@ -1642,62 +1698,6 @@ class JsonRequestTest(RequestTest): self.assertEqual(kwargs["ensure_ascii"], True) self.assertEqual(kwargs["allow_nan"], True) - def test_form_response_with_invalid_formdata_type_error(self): - """Test that a ValueError is raised for non-iterable and non-dict formdata input""" - response = _buildresponse( - """ - - -
- """ - ) - with self.assertRaises(ValueError) as context: - FormRequest.from_response(response, formdata=123) - - self.assertIn( - "formdata should be a dict or iterable of tuples", str(context.exception) - ) - - def test_form_response_with_custom_invalid_formdata_value_error(self): - """Test that a ValueError is raised for fault-inducing iterable formdata input""" - response = _buildresponse( - """ -
- -
- """ - ) - - class CustomFormdata: - def __iter__(self): - raise ValueError("Custom iteration error for testing") - - with self.assertRaises(ValueError) as context: - FormRequest.from_response(response, formdata=CustomFormdata()) - - self.assertIn( - "formdata should be a dict or iterable of tuples", str(context.exception) - ) - - def test_get_form_with_xpath_no_form_parent(self): - """Test that _get_from raised a ValueError when an XPath selects an element - not nested within a
and no parent is found""" - response = _buildresponse( - """ -
-

This paragraph is not inside a form.

-
- - -
- """ - ) - - with self.assertRaises(ValueError) as context: - FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') - - self.assertIn("No
element found with", str(context.exception)) - def tearDown(self): warnings.resetwarnings() super().tearDown() From 6fc78270427c41e401a01a46551d27dd4ddf846c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 21 Feb 2024 14:27:42 +0100 Subject: [PATCH 086/269] Do not close the underlying file from compression plugins (#6239) --- docs/topics/feed-exports.rst | 8 +++++++- scrapy/extensions/postprocessing.py | 3 --- tests/test_feedexport.py | 2 ++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index f64bbac06..922b765db 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -390,7 +390,13 @@ Each plugin is a class that must implement the following methods: .. method:: close(self) - Close the target file object. + Clean up the plugin. + + For example, you might want to close a file wrapper that you might have + used to compress data written into the file received in the ``__init__`` + method. + + .. warning:: Do not close the file from the ``__init__`` method. To pass a parameter to your plugin, use :ref:`feed options `. You can then access those parameters from the ``__init__`` method of your plugin. diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index 79e3b1656..17969c5b0 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -42,7 +42,6 @@ class GzipPlugin: def close(self) -> None: self.gzipfile.close() - self.file.close() class Bz2Plugin: @@ -69,7 +68,6 @@ class Bz2Plugin: def close(self) -> None: self.bz2file.close() - self.file.close() class LZMAPlugin: @@ -111,7 +109,6 @@ class LZMAPlugin: def close(self) -> None: self.lzmafile.close() - self.file.close() # io.IOBase is subclassed here, so that exporters can use the PostProcessingManager diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 277555608..d7560b5ff 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1731,6 +1731,7 @@ class FeedExportTest(FeedExportTestBase): def store(self, file): Storage.store_file = file + Storage.file_was_closed = file.closed file.close() settings = { @@ -1746,6 +1747,7 @@ class FeedExportTest(FeedExportTestBase): } yield self.exported_no_data(settings) self.assertIs(Storage.open_file, Storage.store_file) + self.assertFalse(Storage.file_was_closed) class FeedPostProcessedExportsTest(FeedExportTestBase): From f19045403a44011a62162d73fedacf5038af098f Mon Sep 17 00:00:00 2001 From: Elias Ram Date: Wed, 21 Feb 2024 16:03:51 +0100 Subject: [PATCH 087/269] test #8 made tests cleaner --- tests/test_linkextractors.py | 34 ++++++---------------------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 66a30c635..55ea9eed2 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -2,7 +2,6 @@ import pickle import re import unittest from typing import Optional -from unittest.mock import Mock from packaging.version import Version from pytest import mark @@ -854,34 +853,13 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): ) def test_link_allowed_is_false_with_empty_url(self): - mock_link = Mock() - mock_link.url = "" - expected = False - - actual = LxmlLinkExtractor()._link_allowed(mock_link) - - self.assertEqual(expected, actual) + bad_link = Link("") + self.assertFalse(LxmlLinkExtractor()._link_allowed(bad_link)) def test_link_allowed_is_false_with_bad_url_prefix(self): - mock_link = Mock() - mock_link.url = "htp://should_be_http.com" - expected = False - - actual = LxmlLinkExtractor()._link_allowed(mock_link) - - self.assertEqual(expected, actual) + bad_link = Link("htp://should_be_http.example") + self.assertFalse(LxmlLinkExtractor()._link_allowed(bad_link)) def test_link_allowed_is_false_with_missing_url_prefix(self): - mock_link = Mock() - mock_link.url = "should_have_prefix.com" - expected = False - - actual = LxmlLinkExtractor()._link_allowed(mock_link) - - self.assertEqual(expected, actual) - - def test_link_allowed_raises_with_none_url(self): - mock_link = Mock() - mock_link.url = None - - self.assertRaises(AttributeError, LxmlLinkExtractor()._link_allowed, mock_link) + bad_link = Link("should_have_prefix.example") + self.assertFalse(LxmlLinkExtractor()._link_allowed(bad_link)) From b7a7ae7dbbaddd9d14b50c1257370af51e1ac1b5 Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Wed, 21 Feb 2024 21:04:45 +0100 Subject: [PATCH 088/269] refactor #22 Change comment and warning catching --- tests/test_http_request.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index d1c435468..a45293695 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1643,7 +1643,7 @@ class JsonRequestTest(RequestTest): self.assertEqual(kwargs["allow_nan"], True) def test_replacement_both_body_and_data_warns(self): - """Test that we can get a warning if both body and data are passed for branch coverage""" + """Test that we get a warning if both body and data are passed""" body1 = None body2 = b"body" data1 = { @@ -1654,12 +1654,11 @@ class JsonRequestTest(RequestTest): } r1 = self.request_class(url="http://www.example.com/", data=data1, body=body1) - with mock.patch("warnings.warn") as mock_warn: + with warnings.catch_warnings(record=True) as _warnings: r1.replace(data=data2, body=body2) - mock_warn.assert_called_once() - (warning_message,), _ = mock_warn.call_args self.assertIn( - "Both body and data passed. data will be ignored", warning_message + "Both body and data passed. data will be ignored", + str(_warnings[0].message), ) def tearDown(self): From 877398a3dee8e92300ef52177b986be16a55f277 Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Wed, 21 Feb 2024 21:13:57 +0100 Subject: [PATCH 089/269] refactor #3 Remove inner class in form test --- tests/test_http_request.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 39afc5fd1..71d442a81 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1452,12 +1452,8 @@ class FormRequestTest(RequestTest): """ ) - class CustomFormdata: - def __iter__(self): - raise ValueError("Custom iteration error for testing") - with self.assertRaises(ValueError) as context: - FormRequest.from_response(response, formdata=CustomFormdata()) + FormRequest.from_response(response, formdata=("a",)) self.assertIn( "formdata should be a dict or iterable of tuples", str(context.exception) From c513e7d6e58efc99c5dfc8ad135dc23df528c5fd Mon Sep 17 00:00:00 2001 From: Can Kupeli Date: Wed, 21 Feb 2024 23:52:35 +0100 Subject: [PATCH 090/269] implemented tests for branch coverage for function printSummary --- tests/test_command_check.py | 52 +++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 592494aba..70098a4e7 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -1,3 +1,6 @@ +from unittest.mock import Mock, call, patch + +from scrapy.commands.check import TextTestResult from tests.test_commands import CommandTest @@ -94,3 +97,52 @@ class CheckSpider(scrapy.Spider): raise Exception('SCRAPY_CHECK not set') """ self._test_contract(parse_def=parse_def) + + def test_printSummary_with_unsuccessful_test_result_without_errors_and_without_failures( + self, + ): + result = TextTestResult(Mock(), descriptions=False, verbosity=1) + start_time = 1.0 + stop_time = 2.0 + result.testsRun = 5 + result.failures = [] + result.errors = [] + result.unexpectedSuccesses = ["a", "b"] + with patch.object(result.stream, "write") as mock_write: + result.printSummary(start_time, stop_time) + mock_write.assert_has_calls([call("FAILED"), call("\n")]) + + def test_printSummary_with_unsuccessful_test_result_with_only_failures(self): + result = TextTestResult(Mock(), descriptions=False, verbosity=1) + start_time = 1.0 + stop_time = 2.0 + result.testsRun = 5 + result.failures = [(self, "failure")] + result.errors = [] + with patch.object(result.stream, "writeln") as mock_write: + result.printSummary(start_time, stop_time) + mock_write.assert_called_with(" (failures=1)") + + def test_printSummary_with_unsuccessful_test_result_with_only_errors(self): + result = TextTestResult(Mock(), descriptions=False, verbosity=1) + start_time = 1.0 + stop_time = 2.0 + result.testsRun = 5 + result.failures = [] + result.errors = [(self, "error")] + with patch.object(result.stream, "writeln") as mock_write: + result.printSummary(start_time, stop_time) + mock_write.assert_called_with(" (errors=1)") + + def test_printSummary_with_unsuccessful_test_result_with_both_failures_and_errors( + self, + ): + result = TextTestResult(Mock(), descriptions=False, verbosity=1) + start_time = 1.0 + stop_time = 2.0 + result.testsRun = 5 + result.failures = [(self, "failure")] + result.errors = [(self, "error")] + with patch.object(result.stream, "writeln") as mock_write: + result.printSummary(start_time, stop_time) + mock_write.assert_called_with(" (failures=1, errors=1)") From e208f82076182e71a4eb8470eb527363158f4b0c Mon Sep 17 00:00:00 2001 From: vishesh10 Date: Thu, 22 Feb 2024 16:46:24 +0530 Subject: [PATCH 091/269] Add support for multiple-compressed responses (#6063) --- .../downloadermiddlewares/httpcompression.py | 34 ++++++++-- .../compressed/html-gzip-deflate-gzip.bin | Bin 0 -> 8014 bytes .../compressed/html-gzip-deflate.bin | Bin 0 -> 7991 bytes ...st_downloadermiddleware_httpcompression.py | 58 ++++++++++++++++++ 4 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 tests/sample_data/compressed/html-gzip-deflate-gzip.bin create mode 100644 tests/sample_data/compressed/html-gzip-deflate.bin diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 709333948..1e340abb6 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,6 +1,7 @@ from __future__ import annotations import warnings +from itertools import chain from logging import getLogger from typing import TYPE_CHECKING, List, Optional, Union @@ -102,18 +103,18 @@ class HttpCompressionMiddleware: if isinstance(response, Response): content_encoding = response.headers.getlist("Content-Encoding") if content_encoding: - encoding = content_encoding.pop() max_size = request.meta.get("download_maxsize", self._max_size) warn_size = request.meta.get("download_warnsize", self._warn_size) try: - decoded_body = self._decode( - response.body, encoding.lower(), max_size + decoded_body, content_encoding = self._handle_encoding( + response.body, content_encoding, max_size ) except _DecompressionMaxSizeExceeded: raise IgnoreRequest( f"Ignored response {response} because its body " - f"({len(response.body)} B) exceeded DOWNLOAD_MAXSIZE " - f"({max_size} B) during decompression." + f"({len(response.body)} B compressed) exceeded " + f"DOWNLOAD_MAXSIZE ({max_size} B) during " + f"decompression." ) if len(response.body) < warn_size <= len(decoded_body): logger.warning( @@ -121,6 +122,7 @@ class HttpCompressionMiddleware: f"({len(decoded_body)} B) is larger than the " f"download warning size ({warn_size} B)." ) + response.headers["Content-Encoding"] = content_encoding if self.stats: self.stats.inc_value( "httpcompression/response_bytes", @@ -144,6 +146,28 @@ class HttpCompressionMiddleware: return response + def _handle_encoding(self, body, content_encoding, max_size): + to_decode, to_keep = self._split_encodings(content_encoding) + for encoding in to_decode: + body = self._decode(body, encoding, max_size) + return body, to_keep + + def _split_encodings(self, content_encoding): + to_keep = [ + encoding.strip().lower() + for encoding in chain.from_iterable( + encodings.split(b",") for encodings in content_encoding + ) + ] + to_decode = [] + while to_keep: + encoding = to_keep.pop() + if encoding not in ACCEPTED_ENCODINGS: + to_keep.append(encoding) + return to_decode, to_keep + to_decode.append(encoding) + return to_decode, to_keep + def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes: if encoding == b"gzip" or encoding == b"x-gzip": return gunzip(body, max_size=max_size) diff --git a/tests/sample_data/compressed/html-gzip-deflate-gzip.bin b/tests/sample_data/compressed/html-gzip-deflate-gzip.bin new file mode 100644 index 0000000000000000000000000000000000000000..d66f4c5a03df80085208d8990df59e1702c63b50 GIT binary patch literal 8014 zcmV-UAF<#ciwFP!000000|7T5$l!RKHC1(7lmGWskY!}j3`dRLA?64iIN z(Bm6+BEw8}=7FI~PdU-NoI$sYi==Cu_J|W)>riui_5yjZAmBi- za14I9JL=!~z9y>ZG1)S5g}nXRqrXC3vsTsJxlC(lv|L*QFL5BGB|t7A5yV z8)A3oNJKx+@#>nN_hRjk)tc|^i3jX>&B*BY@$Ev^66`9i!wQBVG74zW%9*8cbkc6?RKdAV?G-_trm>*sW}ar*a&5LpX`354B2 zJVft*L#}$lU@P1H7kM35$fXWsR*Qqj@tWcB+nQ6xzO@V ze?a^36_WPKCXE`Ns`Negn^k~p+;;T*8$ie;1bp6St z(w-Ipec=t_y1GQ*KveU{lv3?#F=k0kbRY>0A4F~1xD=?@M3uRBHJka-ks|i%Zhnt8W>_q2-r0 z^-u!V5+n|%pa!682{6Q>_06P`h@o0wBYJVSaF=L&RRKcrX_*L8RPz(U)gdu0GGdDR zmc_)V1l1k>bRUH7XWSo5PdMZvi$h1BEM-6_=In`%3tnn}lq#Bg^$gCISUqXVBb1=* zd=tDfApw0l+JMM+?{BO)BF&g7SF7Y^v$HLN3tN#W8E16ETYD2oYs&_?H2}6vR3UK( z_{)kAF7wb}OL;&Fv4dgPP)f^a3N7o&phv&nA!Y*U!Q7&zuaeqQZM719B2RW`lsx3o zn}CT^%CI5x5-raR$+spIROp~2V919=AR9CgHJ-+tBW9%yxo|vdE`0z*Kq8?Yy!_nCd0S zh&z!TM(H1d`T$FT@GA2%irjKTzDys+x{4~d>X-f>r5ug~b{GdfW>#P(#egxpc%qf6 z>DP)e>+TYNEZc`qf&?IRFykM(p%xoY;UW7o?z&cs=h!}l(6Jr)`SWdK%$PEnW3qZN(FbDwU!XPf{z^VG<~FX z&xfzFGk;zP0l}tQw4^=M2HBeW_}kj`4nrkCy!EuEN`yGtlr9ASNb>n1Y8?{hyUM@n!6o3f z6SG?oNxgcFz!&||FlnvZyt!%0ht!C`uw6er024X2R##+YlH+!Jlk?0>v5gEL1%8^X z#)~KRTHSOzRcp-VW^=~w(7#UAm9^kQx;Vr2PGE?Fi@jVzteIk@LTaV7eJLKzoOm^= zPO$GM*vFVAZU!MA4#ARF2CfGRScR@PcD$ zx534=Hc-ue8xs{+-&UcPpC)Xd7P2{PfTKW?JyEN9_`8l>uA(3n5>A+qtPXu`_XUVQ zSssHW4u@`d^A|e*kKaHzuJTpiGM-8+Ly1c<2iC3JFUo< zuG0h_{fJ>0^G(Byut|Y%JNI)2#|sQRp@sAHH!(o6B>-kztNl)s#BD}o67^i9h602L z{~(>?#Z}a6wPtl!Tjv=t0QYO=sSl_g)aK1@h0XK2QUM>Bq8mX8455VR*bA`qLcE{P zM=BU97HPKOP!*(gOI%nFIMs&%*7TC1QXyPlr!Br|VtiyOO^K>`nL%uB-=Xf-N=YvA zinAzm&>c!yr$(75NdAB;6JUoc<4hR$PCiEiLK4#-XXaeqq`;M74STk+$|gl@#TbsT z#1dxlq1DSMz+?M2UO`98lP=Pz75o=3rSO~9friQqcBbNCTAjSQS6nH$OTla{J~mI8 zOpmodeX0B7fihrZv|maZ?9PR=U{+SH8C)Sl+Y#C*^iBvRu}u;!hnbWd*Hx)Tu`HOM zpUC<>_^F2Vzj7;UJsc)}GSxslTdO>3RL8-k77`?Shp`IDGp?Wdh{L469oaO;TUu@%0*;vkQn?|LgUZR| zJe2~e7&)3r#>c=AgZ8%M%8nW>k}Mg<)`AFoN#oKU=p_!*+2@p3Xb%dplB7c6#m^F^ zJVQQCvP+cc#vyt!(Qj zKjP3Qeu8C3XBwpx(-o(>uH$xE#F%SBX{0P~fp#%uib4XImbHj=Ouiy|w#+_)3nwU9 z?5uu@tThe(elbD&;ORsRtBxK!DNJFn5ZE(U86;msFc&>QK`mydl9MQdHFgNK>8sXa zv@MeNjEsL}qsm5UR>g7L?ljS^nA3SIHTN^E_RnREelLy@BWanbG zquU4T#~@qxdWf@1^+a{AB@u`3=0~=CRJK)7#IEy#nH@7?UUhTzK$}MJpb^ABP%5Ij zeyxZ=Q?VVUj2q`eTk7^z$5IzT(`v2hR{h8uiWx+0IY-wX5YPI@DHNT$RJ+Z-4#@T@#!%X`l_lvrgE4Q>mq6fs`- zQ7EgmGApdZRm8D;n2*b)*`R7UB^_0^4*|O6d*eZcO^iioi|TN;?fXLNS(GU>@ALV< zOzmx4P+Q!J`I<6umbbnlFN+XJ<38VdIe!3dHSCX)tCWekKgVA0xc0cP8n<2zCq7r$_&$>#yt9zxNF&WewuA9^hBUskUknYcv^aAdSlL~f@qSeOj>tl zVEbAjyk$2R1oR-)T=4d+VlRSjUk3G+hNh@*o6aZ)?S0D5mer{6QycM9yAffGQ z_VfUD341ZN6LUGQYHkxKjMO(-%?t3jeq9#h`Z1?@3ws{3yVFxHzwrI1@#QmG5?T?+ zWe&>tDBWs5H5zyh=5c6GdF9-uF_%_A9hj$#cHxkcnxlmfpP$c**KPEpW0H~ineFyY zip9_NO-^rd@+Oq-OJxNMwHMm<$;d^yBBwL|p^?v{WPtrA`x8ZGOECmwOIY;NFG+by z*}{S=lWfcneHg3#RF^R3UctdXf+7_Hil z6d-11AO0ks`-HCq9g(iP!1M31F>XJUY8eXcUucF_sfX>x2&+L~Kz@>3Tj#Dd5|E8C z!<QWOGi~F~SlKb3yxK$ymy9d}zWl`l#B5^3K`vs+c}trIdW$lDBndDt@2D zAAMtE7xhRUTw*`pAYj%+w8T`suGak0TPL=(UI-C=oz|u!rVBXp`0dGpB-Si#RhMN! zN#F3di4856Utsk=PRy-c8(&Yetg~DcF>F{alA!UE-UGbm35gyo?ME3=LZtoedinK} z6hGfNW|0!~!P=Y(Hfb9Rrsa?tjUZx|fK6pV5+eD0P$K-ABaiq#O107{&BfC~vH7m{M zEC2n~Nqh5o@x<3SMxK1^wLTzxrAByM(n>h?42QnZb`}w21xROuo`^hOdWvF$Qh{9Q zuUVQKHFUQ|cPmodPVCV>dHRw@%=A7S({K@6t9j}1jnLDs(f|zo z3`P&G#bJ_kPC8u~c~e~-VMN95EkcP*^IyNYgJlT>P(9T7xV23Qk31N3wYT{-^xSRE z-rC&nn)djd=@^j1ma_7W{g+F!!pfFtyYJ7&&acShdj6RrC(azS1Aca`2Ha;IU!Nfk zvc|%Ey0hC$z1r_)=lD+cE|Ibe(Y4t|zsOZJv|kr^n26bl`3s&!o`n6q4vW3NJb4j! z%;VdBEn!t8X6XBI#fIGOHL-%w{H-s@{yke0RhYnZ)B5&M@b;AWAVAzR6@Po)Oth!Jcvi1Eobj`;5XZY}OZ_ zlnNN&{hQE{J|j%T9HpVJVMTq+=L_|;jY@jbEU0Dae0XnZGhk)CR^a`&lb3}-6`qfD zZ4E#J{ai|AvuO5gy?GADo|;yexy_EptN~b8j5h}^V0N15{~ZeKMWs3+hCyF(2JfW~ zGrMcxT|fH6=X5l_rE)(3FaHslsKQ+Q)E4SQHVNePoJfs)VDdO!qXRU4jK31&1mJGq z3c`x?vVs=|8V;E~P95$V4U$%ReJ7K79qpcwB$o71pyfVeo>e;9TH}b>ptL4rm!DLG z;+pZ)_acn`lI&_=*Qy?F^+*B#lJSWhhoD$$4B%6e5IVhmNh+JbfN5}+>_?r1*G@F>u%?zVnIk-zqoT}p!1WZdl~ft~!*?-K1S@Pm>jaowvWTzM3M z3BO3Q?cBgXz-ghNjex$)Dyy}b^}|<#e!ZPG_y=}x|A_)rabLSx-L3Fveo_CZ*(l&Y zD-PmNR`wb_gMTng=Jr(dN|pQ^@_U)=5Ad>nB1uM>`*TdVu_VwMedMxb*nyPzg`whV zkp!u@25{G8YUy=6CQ1wNN)g&6{F=>QtTR>ZABWAWo}llHD)P_;%+YYq&gY@8@cO_s zgsbW7fYdvsb3{I${S2^W_|oxnh~E+hdmPVOOZg(**+rdNR_}~QcPu)+`Bfzp9$76y z6T+np{EKgEE;^}z?xk*7^H=Rw_Ys&}O=pgIx*((K=_d(n<;Q-?yDPf$immw+a9f!H z&u^NUNJVA#re0KVsW!U&hkgFmIcdB0{_ z#Bl%6wRy!kdZG*EcO>Zk^tvU+>`RLvYZ(XNwi&{ID`}XlJ5jhWcb(`*Lva1(KUTBV zhfaSma5(B6EQ06UYGaH6pkq)+gpv0f;?D?Z<(D_BQI+_A-n$!-Rx+%bNmE5C^N6sZ z?!#B=k!8R0OqYJba7(V4f z>$L&v0%`mTRKf(w9UN05)wyYYrqA$yrzw6Dy=dAet5K{Gj|r9|XeeqX^HPspIr%#f z-Aw!@o>~Pv&;!?>3M34&iT~68Cnva*Wr1jYc~RHe=R+*6RqTW*oNBxbKRqZb9+QW% z+~M1SJh1f^$Bx-%y(xrahiea@_nRT6?LEfN`chIKuKs7I=8dB3HQ$^*Vq_U4V;+(w z&xG)L+F8d!V#+C|rY7Nm0t0<2|A#65JFnvXdz1cP70V_MJz=!LJ0Xg_V*T+sRYv;R zk6NoD&MlCTj4_HDj!_nEA@WQ$2I`K>-@yWCZO+TFwM`)3rU>GemRv%4iXAX)Dm$R- za*9iw5l!Q?@hHkwSSVMuBT)PUJq%QA%gVpgTKg&{h&TAP(v%`-=@0$|LgRkkMv*{8b~W^|czkMAmIW zmlU}Cp)R`JX^iwkhun`=j|IJMt*!gDfG2X2|n?_@z@CA|)6 z3`06Xrc9BdI_ki=huE7)wOrRUO1{@QOe>+%_%A2EQQWFNd=WDq<&a z+_E-HUS;ZI-9l`!iYI!%9(=|zhQ77j+{T2b5cw5!bbDUf=X&-JWBK8lV86803mq~1R=iDU`^3$m7bIF+9 z1>vegA@&$FPKA@#v43`Eu62Rt3%EZq4I{GSipQk~{<(ySVwZW*v7nY8DCIaGya}hV zb^LoCY$}$>rHhKM+9xQUJjFN)ix3ju;lz;|4_8E@aq8(rEQ`u5(1-rvY8O*hm9TM) z?v19F-4lL0hz@HJwCY-)YAfY8wL@DPP!sFt@}Zl+BIlB3eakrVV!u0lfz`qaC+TFa zo}g~k#v4}w^-$=`d?OmxcKMV^k9F?V$tm<+F`dJ-+9d2#zYW^bZ}=`Ae|+H{TIepN zl-j37Gk^rkAH_*WbxV5sq#3&T_Y;W{Hh&k5?X!wpJ5ztb&+T^4JdK=dGS%b;M3lok z@2Yrle?&a;Kd8JjC2RWrb~TWLyojo%L=OInJrbKQS=DL^zTULK{mimaP~o>9B`!G9 z1rZ|Y`F3|MTRC2aZin8z6E*r5Y@{V=g`>eKzeWSR-;Hm$VN7-k9`}SlnRez0?PW_HBspUFdbM}- z`4+T860?A9h}Ey4oA{2x3_H41NP%T%chNw{(GIq9EQ*Y9xGp-=#g3S%a<~fBOt0%KB-SuxU8W zdyDjj-LJI#r2m8#xQ%lF^Upn6O;t$R)PT|G8FgluVbv@@`NnC&62)cDzqs-1ki!e2 zQLpvFTNk`&ItA-gK|IUd6W$WE`OV2^VtdeoRF$(4HPoZ{lnn~If%mEl(;L+Y<6f`68G1^B8dYC&;MZOJ$>6i> zf)IOGg!+1{gSn9E?Bai$VjF=Y#bq>rVp4u97H}lC1h(a+7xTRT1R8SZT zjjA;JE2E<+!gMD)?ge)hV97F(Hu!bu2fU$dP(%Q$vFOokK4LJ5o~bmuh}Tv7!&50^ z2RR8~I!V7qCEjqF?N_u*)FCy>W4a0ARS>d}F&eizQK56ySvR6MAU@p&Rt5gNuE;ok z+%4f_^-8>64gPq2)FWXPIDL;JVuG2lZBs)hSEL#BEj;puW0iGW@UO~-99A96VP4=A z7vEFbl7a>Z20k~mk2MEdGaQ{L>X)uj`mt@oggAT5M?lu3kE=B-y^pm{{fl6rkLNJ7 zKt^x8KUavhaoJ{Qng2|8e0|)w2p2y6!scm*n_g9g=;I|AEpsSX7>LxdbZi~UQ;pL$t#qTaOdBB3>-Lu`E&L;goo6E^}JZg0%i1 zzRKR8DoM>^WCz;RRey$cp8V!!MKh-uH3Z4v=PJU%cenZLe%~3n_3%XWvv!dd!_z1K Q142u<+9f5~j5i+u01a&HYybcN literal 0 HcmV?d00001 diff --git a/tests/sample_data/compressed/html-gzip-deflate.bin b/tests/sample_data/compressed/html-gzip-deflate.bin new file mode 100644 index 0000000000000000000000000000000000000000..5066842ef7d137786f43682c99c63e24eae261ec GIT binary patch literal 7991 zcmV-7AIRW%oHbQ-T$BIzRgh+cATb04X;45K35n4uFiN_IbaxEt?(QBT-6bI%gMrdW zNC-&B1is_*d;R`>&U5a$aqfN7MqxiOdBgVM33^wKK@!z>x6tDob|S-J=M>3F*C}#i zpg=;RAY+OFu42tH27|;~He|t~fdY4cUpv;8DWh17mkiDKZ8oR6255#$`;^y|kJ#;? zxZmCH0IwUPfE&w;W6mC=J92Aq@D5>dKjnXAi8OEWb_!_iS#~fwGCbtuyjp-AtQ6Qh!9x|h6#k-K|Dn7e?zW%!eA@g{ug;2 zSIDIfWLArV$MKrs@!OhH#^cL0WYl^0UI%=kR9k=B_A)Qm$E~I3!7;!$o1qk8bai$0 z(9m&vV&{Fq&xAfh} zCx_;QKOonFU+l7@A9jL(uMon{A@W9RA`k8t>d`GQ2X~JEpZi1Q=MQO|SK8vM+5Rqb zNB)|77G#M1rOm7EfR2YdZP>Ajwtq{<%8JqQr+_AJuiL*C-u)d>S!0bV?h74vhZC@3 z=mNR;@3eoAl-w)#i(S#AN(b?X)64iR@2lOVp%xs(en(=nItgp9 zteho0ETa9jxz^azT6KFto8yn`)^wyQe)r2mxqRX9r0n;eZthlqzi)lf$gpawo<-4_ zWs@Ru?>0`==(vF)ue8i7)q!`hc=KVs(4C}xBq~%nyIW($VR27?5HdSAd$p+PG+Q<3 z=JX?E5BbG)ru1mgZQ`p<*2uyA&2`VF8!ev;n7CS9nkbTpoDqag%nX{F-B*ajmZ_E} zzAMc%WH8aFop{dxa@|W+WG@82m*hw`(NKl)veJk1wX8NmWJ8o0yIn~1i|&iVSO4r6 zhg*`A_INbYkH?l5sY&xufWq||LC>6Om9^Nn9Za7uH?qn{#^c%s->hatVnbxHDVago zvvYQfHn3dM0Lrh!Y|4ihp`{x%M@qkdR4w;u5)PNxb4msS?e+R8M%i zx|G#;|7Sj>+Kr(nNvl=s#h7&K(^Q)&bFZS6yX0k5*0Z+GlJo|jkF3i4$;^OQ57^5= z(o-G30nwx@Q5gl<(wK&8d+RxS>(ZVU0e#^O;<~y-;6POK z$dpp;YB6R>O>`g$4Ie~p+PD;`*F=@Mcjah1+f^y4DG$KRv4gK@@*42rs>OBK-tJk# zBo0IOuH@KtboRk*yOW*Pu;72iq4OfT!*m6U0IviN6^JWD09zwx?v-QI^BnyiyP^*g zk*m=WcxE50YsMf3%ODSkfHa8!#d4t`9g9oS+^cUH#i8YwHuX>f))FKRr=SL)YY8yK zqV>(Bl8B*NU?X~Qw{Vwed{qHL@oAX|QdIL3!qp)$Eiz(?`mjtgFDf0Qbkd-V*?mRLP$$|IDZ?0gfvG9dwdI@*B9ckgek zI3mrMDOaoHX0x*`f(u)bC>dvT!&`e3NNdXmxitW`O;jOq2l&g15H9o3U`u&G3bBJ> z*HB8!XbLUs$)HES-XUfJ>A~EhrmvFPQf;*oej-nHXp}tU(VKvYQ_8R*^Aau349T}9 z6;$Y;Bw)ygL?9b95H+60oFitX4Y_bUYc72NL_jIPp-ix({Hqu5WWBCKa0J%=WL4po z_59n&8e0u^R4PobS19xvc?i?yj1BpKWtr@ZO6|O|N0{m*$cQ_U9Y*ONg8BeUf$%Ex zGK$=CL%vKO#=43sx9XSvAEg|Q1a=q)K4w;6CdGg;yLh6Ns_EB?G3)LUe=OUFPl5y> zbTM8LgD@UoQj|SD`V1>%D(IVWwuNEWsDO0RdliTp;sd1$<>lv6pwe66%xrI~f1Y6I zld8TzRFtsfVAg~R7eJzpoSC7ilS&15o3)k@yn>G$@HBm-be|jwv^!mBhuF5Ldwqin+(Ejek3mwEyKHNNTa*p-HyOc_60M+ zaCzS!OmznYn@;+HVl+_cx)fJtzwTb{g>KDC&4>kLHi|*)Ehrrf9Rb0nTePG-)CSp_ z`uN-0^$tTNK)m&|rb>i3+LSH?|48!rA!;2G=DW(j>cJ)8wiB~k5J|myjldWE(J*PP z+q}7H%7@g5z_49EJpdCqwN_VTWs>7|dz16bOtFm&9|eAzt;UNd_FCO^J5_7U=4Nxo z?$EzZ)s?m2L%KM_^-f@jf{VReLado$q(W+?w0$Wa&762Os!p))C)mfBCT<2H9}dBi zR|c*J3Rs1%H+HG@lRIb;>$kzhwKh=Aej5`NSl?Em zmY*hUpBAz?Y=EOcl08wYdHB1IU9O@a6%tOEkgN`UZTAI;KUp4wB@ZFu1!;_7t`2cm zwHX^~U1`AuLc#`lD?;bD3af$7>?>0C^s>S`E0j~d|B$UF(yGz|dMKp=i#7A+Eatv0 z+Y@oB!$ceSO?)(1$=G2M#J}2v6p4R9z1o+ruH!j!9y_hbm#)(U9{q@681qfTjIc?8 za69*N2FD8wJfVg2^*1p&3R;9@OT|ZiUVBx>5ljn4%j&2@Iix=-3Oe^g_I!&qpd4Di&$B;ZPN%bxT}W z4>;9_0oL@AqEaDTU#Bg;X<~e2Dou&1d6_|MZr`Eq)=Eh(@`|%4bkH42S*J#sC`kT* zD-&RcE8|QU_f9@X140thA7|!V-lV{lVGVn>vC1YzY{eLku*4E(@uAhrD8OU;H(o(U z%abnBs1^JdFQxFC)`5n~40fjCVOpKMx>sB&xJ$uoEj~6+nM{whKz*tE`iR4%za7~$$6H!%9RiM-1X8&nl7q_0gEe*twdt$YVze!jDP!O7)T)*T{@n^Bs`>R0NXRS0ODLK^4 zQZ0+L9I3`&^y=k5k(1>-dmlV!_*)w-t5wh_*^>v{=w#<&wxinz>&GBl_j-u4O7%o_ zuO$(O@8(Cgd{nkoQN*tEf|(sNVqSG~^+20O@SqXIKTs;7x_+&QKvS_Dri>fsLtEZT#Eb{4E8b2D$k6=_^BS(=abXFVCRhuW!R+ttjZes(H`C{g>|LktN7X2U zH}sx~x?Dzg;m><8L&a^R&eRVqJj;|X8M-iQO};NXDLa_^tO3RcU*0#Ye4l;VDXzm+ z!7=(y%fq)?kGKd}lSCz4Oz>&1Q;*g}RFU2H1G5Iz)bCJTu@uwiusx{ahA8f zA}@;&NaH@=dO3dpZZ+(WlB<-7xj)BV@3{83uo|~s4deWO@RJ}YoA_s~?5Z92w)t)+ zt~g*Nk>n`}lyzawX;Hn;?NLGNLG;PyZ6LhBou#BapR?V*KwCT|i6Uq$J zpvFAFMZ=2322km{z&X(1v@KYP{Q@astGvdgr!uIq4b_shiwi9zXuWD`+ zD2&uMS@@Lc#dNanGl7tgyg z2?~~d)P82jl<>(=w_blhtVD-x?dg5@uk)?Xw5*Y-UKp*~juaqfW*`0}p8JHa1RasC zyTJ4Bu`zBxlxi6Y?O$kyR;h>W#t5rHUqF76TwCX^H4>1GF~girA&aurTV!)i*fGKq z4s${KW64;`aeQdPGWw|6hVstY@~W6VV5O9N-jcU_LK}p~6w}}lcmtSD@KTgc8 zT^nCdv#hgR6ftaAE|Q?}limZo<_U=&EbT`bQ9`8s?Rxq3lN3MSIcAX(^ugMB=T>94 z{U+4FQEyUwVpSgHnN9>91Lu8#Q#dMt3Vx+)nJ# zK6(0*M$Gg+9Mf58F^D( z9brVp?kz%zO!HsAxr1d11W-NH`M9-B36DG&bhWqnHT2wV&fePG@0#}boaq>l!XNM!(2aHMCzBc$kRUiTMkjMV^HHy$*}LzdU&ncg*A4el1~DBxdOQam9w* z?lrN3(EP10$o@TB6IGbNbkq9wQ1JGY_#i;wV74aUvQ6}*qT1xj22WOE$VHbf)e)Yr*=2gk%Lvj9F(8Ly?r%OLTOq6~U8;Jr;x z|G9We8Si~}H%!JH#=gm9ES?eBOu?RV1Ouf-Kl_ZrEo{~opOgw1;QgD>kv=0##2lre zuVF=f%;yXBw2ex7(k!TD>3n!^X)|DDy;k7;x09EJK^2~lbZrel1N~e|WwU7ZY`u97 z$DW#2n7Pf4$E*QZSBy6YE?{<==>HuG>_w$IA%;O;aR%?D4l}!J;9WoZ!{>A~zNKi8U84guevH2o;{@Pt;0nTu^s<5%1{w~TJx(3& z8V!OsS>3JhXMR!tsM#psKPwL6P*(ODJ%fKROy>4f z^h%Zd9P)da><{p=ej-Umnfr50xUnSA8hzxlW!Qm~_=Tb3YLNt~xCU_7WNPVkJSIvD z@JbQdCH$JrU#v4#?H`BDtDd0mj4JZb1ggv5Y~{y(%DXGN^NOwc6mVOa0ncxmnMg%t_NHD`aH%%B z{D*!1*5q&a7hdmARB!x*=>UdQ*P)Hf{}CqEa)NB*OLU7$r%)1!WIPj~+L7?cLrjGu z-%A~N^rNr#EVW#8sKpuBe9^_E5DV5m^=j1^cA471qN-ZcF>}AH;#I4BnTG;jvw5cR z>AinxCK^Sn1P{D2G;?=Em#$&ri{mXXMwko4GA66iR0qDAGI7s_({ql_ur<5` z`O5FIlm5Xp^U1L=NoA5ZD_yb1>txaQI?o)gSU2yDell%OI$gCBnE7FftFhq_HH7gK zvQ8#vJb+$q2nyuNyjXTh#O%D99&e^X)xkCtVWSJQ78pL|LhH2w>jG)~3RJ=b$sHV1 zBh|TSex}dxf2S#a6TN8KC#zAc5swL$BWNgUCi7B{T{-zX5Zz4tCZ1XaJJ18yp9&-l zvWfrG|0gH7lVyQueR)yW+UG+ou2t-WDV%D&3_m?6D;|@FvfSa@fjqGF7RQd+X1ytd zV~1-Gp!b_0rtLk(&-zkQAFlpqr{;~K>own;K4N4UBx4?uCeMWMdfHjXLSo7(rluz0 zfdT`4D*uNm{yVSY{d<%CU=_5Rv9(Pg-=+xSmX=&Xd5Rq{Y$`jT>vD=qoDogqwDBm)Rahuj zwIfjc13e5>Y|F~O(^~r~CWtrqwbGO#Y9(Tc9r@j2MgIauWSK=mJfWs$vB*exer}OH z|7tJNF**>7XChJ*bL;$dp~@rjPop4H@^oMQ!K3h!_~eG<*+*|y6uLoHxlpo>n={v~tv z4BZfGvX2QN7D*)5KQCpwzi^g&;gNUVi&_gBnP7pt#> zZf=OA8ZjCtbTSq&t#o~}gMoVqhdA3dExN#$;_qZZVkNx}Y79dK@d%MZHt6!bA4~h+&xgOLmAz@KQ)oN|wNLRk|Dk2|ziV(Uw5Bj?;D5%SZh>~qPO-38&QLm~DUG){$+*Rg+g zX0COC<_ow#F%2WK$fb*lui7Ul zo;<}k3X2dD-{HiO8V^@QqH*f!L@bNSEzpPl;c6FCR+X@EjP8x5mfaJ6JBSWz5wz-B zpK2@RH?>1s8c-AK=klSOz#`|8W_`;z@?yU`e1X-%3Mc7guAZQ7)y5lF0rgPm%X}jm z)^_=nNso2z)yXOJUNN1+wAv)>Q@;(`(r@@K9)En{9$M%wrIgyIMKgc|%OAx_M|Dej z`lK1U`S%lv5;lJqjqS6FTsu>L!O!h>&peHsYckd321Jy@JnyP_aeqWS@;|7&GbL;K z{&qExgS?2UrbG_@iaipWFIm-U3clX7!Trp#QBdKxA0;k0(ghJB>G^hdE?YTXhHi)6 zy%RP17i^>@X@#S~DZfSoyx)y)xM56o3Lf`_Kbdys3GHP`FFGe^NcJoR1<`HjQWFdH zFaUDNKc>nA!3wN1+YY;@T_n}L!`!|HomkMhsQL^18i*iBaA3)Ur=i?~E)K%yeyw1& zMWw*6ubnD=zMM@csBC~{lV4(DUl2%VDZNK+NhCR9`Fgc?^!XOFLlU!qZHU#cpqu!P z!VEjQR7iny@WSXC$;@tStk;l`SA*b?S zP+Xv44!3lHWTGJ6PHH54ecz=$m05$X9)J4`Bg*<|m#}F#&3lXVhTX5U{G|Vc7PyUb z0Q1j1T1{0*+SGv2=oxipm|@i{Kl#RK!V<-0&%e0w>yX0>qEWB)!dn--XgUS!R6#t; z-4ostwE4}+XJUKMgH)BX5jE7K_~YGksK)T!bE_E>ch=;8LLNy8-3wcydeP{pA08b~ z%1vMzE|d)lyMgzr3)36b2;*L_z8QK-f*Ms{bKuuq{K??6?1B(`SA_a{tb@6b>g?iw zn_?S*BgJJjfMQa9E97+z@bA@{rZ@A+42VK83%I3ONmNi63yrEY`zxcPDZ+FoJMIN{ z6=2CSkv8~s=m)%^Y*0i1tFh?OY(8QziJqx6yNK6S`@>TyV+T12U^+>^MkU^Gn(bG# zOVlAX%450-;#Cl`kTDv!I#Ho>)mb;9I3PaV237_ByspSNe%vkLWA#eBUJd?uebggi z6*zs5BVvM?ux(RACs(8y^({Q|hGUg=T=1{Th8$KM%VA#N6c^u9+LD3>2nIekw2w6h zTQeM;DC(E4QTnlM!h|?`%tt`hq>rmLEWMAlPW_8uppWM;v_M90ygyfnwsF~JXqo>^ zcYJ-^xCj?M{leyHhnrqih3MlY7%g)sSQv=Zv2<)5%2SQxP=Im(-$gIq_-1-Vz^C05 zFlr+*WGx-rB}25sBU_IXw<2CFy|FA&<>8cgY%X(J(So%8Aim1ppDIbsVq^!})K!0m tb)Nj@W<@in7&QdR;O8pB!FRX$>we!Ex%Kcw^s{!67Q@pg{{uoxxY|2R=d%C+ literal 0 HcmV?d00001 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 9deb81c37..ae5569d0a 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -27,6 +27,8 @@ FORMAT = { "x-gzip": ("html-gzip.bin", "gzip"), "rawdeflate": ("html-rawdeflate.bin", "deflate"), "zlibdeflate": ("html-zlibdeflate.bin", "deflate"), + "gzip-deflate": ("html-gzip-deflate.bin", "gzip, deflate"), + "gzip-deflate-gzip": ("html-gzip-deflate-gzip.bin", "gzip, deflate, gzip"), "br": ("html-br.bin", "br"), # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin "zstd-static-content-size": ("html-zstd-static-content-size.bin", "zstd"), @@ -205,6 +207,62 @@ class HttpCompressionTest(TestCase): assert newresponse is not response self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"uuencode"]) + def test_multi_compression_single_header(self): + response = self._getresponse("gzip-deflate") + request = response.request + newresponse = self.mw.process_response(request, response, self.spider) + assert newresponse is not response + assert "Content-Encoding" not in newresponse.headers + assert newresponse.body.startswith(b" Date: Mon, 26 Feb 2024 10:53:06 -0800 Subject: [PATCH 092/269] Remove usage of deprecated mktemp (#5285) --- .bandit.yml | 1 - tests/test_commands.py | 5 ++--- tests/test_downloader_handlers.py | 28 ++++++++++++---------------- tests/test_http2_client_protocol.py | 4 ++-- tests/test_pipeline_crawl.py | 4 ++-- tests/test_spiderloader/__init__.py | 4 ++-- tests/test_spiderstate.py | 5 ++--- tests/test_squeues_request.py | 2 +- tests/test_webclient.py | 7 +++---- 9 files changed, 26 insertions(+), 34 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 2aae8a0aa..8c6a08e1b 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -4,7 +4,6 @@ skips: - B105 - B301 - B303 -- B306 - B307 - B311 - B320 diff --git a/tests/test_commands.py b/tests/test_commands.py index 36f800850..2f36baa87 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -6,13 +6,12 @@ import platform import re import subprocess import sys -import tempfile from contextlib import contextmanager from itertools import chain from pathlib import Path from shutil import copytree, rmtree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION -from tempfile import mkdtemp +from tempfile import TemporaryFile, mkdtemp from threading import Timer from typing import Dict, Generator, Optional, Union from unittest import skipIf @@ -82,7 +81,7 @@ class ProjectTest(unittest.TestCase): rmtree(self.temp_path) def call(self, *new_args, **kwargs): - with tempfile.TemporaryFile() as out: + with TemporaryFile() as out: args = (sys.executable, "-m", "scrapy.cmdline") + new_args return subprocess.call( args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **kwargs diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index dd07d33f1..d3fd63847 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -2,8 +2,8 @@ import contextlib import os import shutil import sys -import tempfile from pathlib import Path +from tempfile import mkdtemp, mkstemp from typing import Optional, Type from unittest import SkipTest, mock @@ -107,13 +107,14 @@ class LoadTestCase(unittest.TestCase): class FileTestCase(unittest.TestCase): def setUp(self): # add a special char to check that they are handled correctly - self.tmpname = Path(self.mktemp() + "^") + self.fd, self.tmpname = mkstemp(suffix="^") Path(self.tmpname).write_text("0123456789", encoding="utf-8") handler = build_from_crawler(FileDownloadHandler, get_crawler()) self.download_request = handler.download_request def tearDown(self): - self.tmpname.unlink() + os.close(self.fd) + os.remove(self.tmpname) def test_download(self): def _test(response): @@ -122,12 +123,12 @@ class FileTestCase(unittest.TestCase): self.assertEqual(response.body, b"0123456789") self.assertEqual(response.protocol, None) - request = Request(path_to_file_uri(str(self.tmpname))) + request = Request(path_to_file_uri(self.tmpname)) assert request.url.upper().endswith("%5E") return self.download_request(request, Spider("foo")).addCallback(_test) def test_non_existent(self): - request = Request(path_to_file_uri(self.mktemp())) + request = Request(path_to_file_uri(mkdtemp())) d = self.download_request(request, Spider("foo")) return self.assertFailure(d, OSError) @@ -224,8 +225,7 @@ class HttpTestCase(unittest.TestCase): certfile = "keys/localhost.crt" def setUp(self): - self.tmpname = Path(self.mktemp()) - self.tmpname.mkdir() + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) r.putChild(b"redirect", util.Redirect(b"/file")) @@ -651,8 +651,7 @@ class Https11CustomCiphers(unittest.TestCase): certfile = "keys/localhost.crt" def setUp(self): - self.tmpname = Path(self.mktemp()) - self.tmpname.mkdir() + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) self.site = server.Site(r, timeout=None) @@ -1015,8 +1014,7 @@ class BaseFTPTestCase(unittest.TestCase): from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dirs and test file - self.directory = Path(self.mktemp()) - self.directory.mkdir() + self.directory = Path(mkdtemp()) userdir = self.directory / self.username userdir.mkdir() for filename, content in self.test_files: @@ -1092,7 +1090,7 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): - f, local_fname = tempfile.mkstemp() + f, local_fname = mkstemp() fname_bytes = to_bytes(local_fname) local_fname = Path(local_fname) os.close(f) @@ -1113,7 +1111,7 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def _test_response_class(self, filename, response_class): - f, local_fname = tempfile.mkstemp() + f, local_fname = mkstemp() local_fname = Path(local_fname) os.close(f) meta = {} @@ -1163,9 +1161,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase): from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dir and test file - self.directory = Path(self.mktemp()) - self.directory.mkdir() - + self.directory = Path(mkdtemp()) for filename, content in self.test_files: (self.directory / filename).write_bytes(content) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 8fdf3d56f..995c02a1a 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -5,6 +5,7 @@ import shutil import string from ipaddress import IPv4Address from pathlib import Path +from tempfile import mkdtemp from typing import Dict from unittest import mock, skipIf from urllib.parse import urlencode @@ -185,8 +186,7 @@ class Https2ClientProtocolTestCase(TestCase): certificate_file = Path(__file__).parent / "keys" / "localhost.crt" def _init_resource(self): - self.temp_directory = self.mktemp() - Path(self.temp_directory).mkdir() + self.temp_directory = mkdtemp() r = File(self.temp_directory) r.putChild(b"get-data-html-small", GetDataHtmlSmall()) r.putChild(b"get-data-html-large", GetDataHtmlLarge()) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index c41ab483f..be9811980 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,5 +1,6 @@ import shutil from pathlib import Path +from tempfile import mkdtemp from typing import Optional, Set from testfixtures import LogCapture @@ -67,8 +68,7 @@ class FileDownloadCrawlTestCase(TestCase): self.mockserver.__enter__() # prepare a directory for storing files - self.tmpmediastore = Path(self.mktemp()) - self.tmpmediastore.mkdir() + self.tmpmediastore = Path(mkdtemp()) self.settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", "ITEM_PIPELINES": {self.pipeline_class: 1}, diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 04025d30d..f950739f2 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -3,6 +3,7 @@ import sys import tempfile import warnings from pathlib import Path +from tempfile import mkdtemp from twisted.trial import unittest from zope.interface.verify import verifyObject @@ -139,8 +140,7 @@ class SpiderLoaderTest(unittest.TestCase): class DuplicateSpiderNameLoaderTest(unittest.TestCase): def setUp(self): orig_spiders_dir = module_dir / "test_spiders" - self.tmpdir = Path(self.mktemp()) - self.tmpdir.mkdir() + self.tmpdir = Path(mkdtemp()) self.spiders_dir = self.tmpdir / "test_spiders_xxx" _copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(str(self.tmpdir)) diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index f97125b76..59d18d92e 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -1,6 +1,6 @@ import shutil from datetime import datetime, timezone -from pathlib import Path +from tempfile import mkdtemp from twisted.trial import unittest @@ -12,8 +12,7 @@ from scrapy.utils.test import get_crawler class SpiderStateTest(unittest.TestCase): def test_store_load(self): - jobdir = self.mktemp() - Path(jobdir).mkdir() + jobdir = mkdtemp() try: spider = Spider(name="default") dt = datetime.now(tz=timezone.utc) diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index b444c32b7..499ca46b8 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -25,7 +25,7 @@ class BaseQueueTestCase(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-") self.qpath = self.tempfilename() - self.qdir = self.mkdtemp() + self.qdir = tempfile.mkdtemp() self.crawler = get_crawler(Spider) def tearDown(self): diff --git a/tests/test_webclient.py b/tests/test_webclient.py index d4b6ba15b..53558814d 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -4,6 +4,7 @@ Tests borrowed from the twisted.web.client tests. """ import shutil from pathlib import Path +from tempfile import mkdtemp import OpenSSL.SSL from twisted.internet import defer, reactor @@ -274,8 +275,7 @@ class WebClientTestCase(unittest.TestCase): return reactor.listenTCP(0, site, interface="127.0.0.1") def setUp(self): - self.tmpname = Path(self.mktemp()) - self.tmpname.mkdir() + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) r.putChild(b"redirect", util.Redirect(b"/file")) @@ -440,8 +440,7 @@ class WebClientSSLTestCase(unittest.TestCase): return f"https://127.0.0.1:{self.portno}/{path}" def setUp(self): - self.tmpname = Path(self.mktemp()) - self.tmpname.mkdir() + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) r.putChild(b"payload", PayloadResource()) From 2d46b4acf5855faaf2d6baa36615f08bd8aefccf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 27 Feb 2024 09:28:02 +0100 Subject: [PATCH 093/269] Complete coverage for the AutoThrottle extension (#6245) --- docs/topics/autothrottle.rst | 2 +- scrapy/extensions/throttle.py | 5 + tests/test_extension_throttle.py | 340 +++++++++++++++++++++++++++++++ 3 files changed, 346 insertions(+), 1 deletion(-) create mode 100644 tests/test_extension_throttle.py diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 8e6aae65c..5370d77b3 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -131,7 +131,7 @@ AUTOTHROTTLE_TARGET_CONCURRENCY Default: ``1.0`` Average number of requests Scrapy should be sending in parallel to remote -websites. +websites. It must be higher than ``0.0``. By default, AutoThrottle adjusts the delay to send a single concurrent request to each of the remote websites. Set this option to diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 396800775..d217c7a69 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -16,6 +16,11 @@ class AutoThrottle: self.target_concurrency = crawler.settings.getfloat( "AUTOTHROTTLE_TARGET_CONCURRENCY" ) + if self.target_concurrency <= 0.0: + raise NotConfigured( + f"AUTOTHROTTLE_TARGET_CONCURRENCY " + f"({self.target_concurrency!r}) must be higher than 0." + ) crawler.signals.connect(self._spider_opened, signal=signals.spider_opened) crawler.signals.connect( self._response_downloaded, signal=signals.response_downloaded diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py new file mode 100644 index 000000000..dae4ea966 --- /dev/null +++ b/tests/test_extension_throttle.py @@ -0,0 +1,340 @@ +from logging import INFO +from unittest.mock import Mock + +import pytest + +from scrapy import Request, Spider +from scrapy.exceptions import NotConfigured +from scrapy.extensions.throttle import AutoThrottle +from scrapy.http.response import Response +from scrapy.settings.default_settings import ( + AUTOTHROTTLE_MAX_DELAY, + AUTOTHROTTLE_START_DELAY, + DOWNLOAD_DELAY, +) +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler as _get_crawler + +UNSET = object() + + +class TestSpider(Spider): + name = "test" + + +def get_crawler(settings=None, spidercls=None): + settings = settings or {} + settings["AUTOTHROTTLE_ENABLED"] = True + return _get_crawler(settings_dict=settings, spidercls=spidercls) + + +@pytest.mark.parametrize( + ("value", "expected"), + ( + (UNSET, False), + (False, False), + (True, True), + ), +) +def test_enabled(value, expected): + settings = {} + if value is not UNSET: + settings["AUTOTHROTTLE_ENABLED"] = value + crawler = _get_crawler(settings_dict=settings) + if expected: + build_from_crawler(AutoThrottle, crawler) + else: + with pytest.raises(NotConfigured): + build_from_crawler(AutoThrottle, crawler) + + +@pytest.mark.parametrize( + "value", + ( + 0.0, + -1.0, + ), +) +def test_target_concurrency_invalid(value): + settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": value} + crawler = get_crawler(settings) + with pytest.raises(NotConfigured): + build_from_crawler(AutoThrottle, crawler) + + +@pytest.mark.parametrize( + ("spider", "setting", "expected"), + ( + (UNSET, UNSET, DOWNLOAD_DELAY), + (1.0, UNSET, 1.0), + (UNSET, 1.0, 1.0), + (1.0, 2.0, 1.0), + (3.0, 2.0, 3.0), + ), +) +def test_mindelay_definition(spider, setting, expected): + settings = {} + if setting is not UNSET: + settings["DOWNLOAD_DELAY"] = setting + + class _TestSpider(Spider): + name = "test" + + if spider is not UNSET: + _TestSpider.download_delay = spider + + crawler = get_crawler(settings, _TestSpider) + at = build_from_crawler(AutoThrottle, crawler) + at._spider_opened(_TestSpider()) + assert at.mindelay == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + ( + (UNSET, AUTOTHROTTLE_MAX_DELAY), + (1.0, 1.0), + ), +) +def test_maxdelay_definition(value, expected): + settings = {} + if value is not UNSET: + settings["AUTOTHROTTLE_MAX_DELAY"] = value + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + at._spider_opened(TestSpider()) + assert at.maxdelay == expected + + +@pytest.mark.parametrize( + ("min_spider", "min_setting", "start_setting", "expected"), + ( + (UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), + (UNSET, AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY), + (UNSET, AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), + (UNSET, UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0), + (UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 1.0), + ( + AUTOTHROTTLE_START_DELAY + 1.0, + AUTOTHROTTLE_START_DELAY + 2.0, + UNSET, + AUTOTHROTTLE_START_DELAY + 1.0, + ), + ( + AUTOTHROTTLE_START_DELAY + 2.0, + UNSET, + AUTOTHROTTLE_START_DELAY + 1.0, + AUTOTHROTTLE_START_DELAY + 2.0, + ), + ( + AUTOTHROTTLE_START_DELAY + 1.0, + UNSET, + AUTOTHROTTLE_START_DELAY + 2.0, + AUTOTHROTTLE_START_DELAY + 2.0, + ), + ), +) +def test_startdelay_definition(min_spider, min_setting, start_setting, expected): + settings = {} + if min_setting is not UNSET: + settings["DOWNLOAD_DELAY"] = min_setting + if start_setting is not UNSET: + settings["AUTOTHROTTLE_START_DELAY"] = start_setting + + class _TestSpider(Spider): + name = "test" + + if min_spider is not UNSET: + _TestSpider.download_delay = min_spider + + crawler = get_crawler(settings, _TestSpider) + at = build_from_crawler(AutoThrottle, crawler) + spider = _TestSpider() + at._spider_opened(spider) + assert spider.download_delay == expected + + +@pytest.mark.parametrize( + ("meta", "slot"), + ( + ({}, None), + ({"download_latency": 1.0}, None), + ({"download_slot": "foo"}, None), + ({"download_slot": "foo"}, "foo"), + ({"download_latency": 1.0, "download_slot": "foo"}, None), + ), +) +def test_skipped(meta, slot): + crawler = get_crawler() + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + request = Request("https://example.com", meta=meta) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + if slot is not None: + crawler.engine.downloader.slots[slot] = object() + at._adjust_delay = None # Raise exception if called. + + at._response_downloaded(None, request, spider) + + +@pytest.mark.parametrize( + ("download_latency", "target_concurrency", "slot_delay", "expected"), + ( + (2.0, 2.0, 1.0, 1.0), + (1.0, 2.0, 1.0, 0.75), + (4.0, 2.0, 1.0, 2.0), + (2.0, 1.0, 1.0, 2.0), + (2.0, 4.0, 1.0, 0.75), + (2.0, 2.0, 0.5, 1.0), + (2.0, 2.0, 2.0, 1.5), + ), +) +def test_adjustment(download_latency, target_concurrency, slot_delay, expected): + settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": download_latency, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = slot_delay + crawler.engine.downloader.slots["foo"] = slot + + at._response_downloaded(response, request, spider) + + assert slot.delay == expected, f"{slot.delay} != {expected}" + + +@pytest.mark.parametrize( + ("mindelay", "maxdelay", "expected"), + ( + (0.5, 2.0, 1.0), + (0.25, 0.5, 0.5), + (2.0, 4.0, 2.0), + ), +) +def test_adjustment_limits(mindelay, maxdelay, expected): + download_latency, target_concurrency, slot_delay = (2.0, 2.0, 1.0) + # expected adjustment without limits with these values: 1.0 + settings = { + "AUTOTHROTTLE_MAX_DELAY": maxdelay, + "AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency, + "DOWNLOAD_DELAY": mindelay, + } + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": download_latency, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = slot_delay + crawler.engine.downloader.slots["foo"] = slot + + at._response_downloaded(response, request, spider) + + assert slot.delay == expected, f"{slot.delay} != {expected}" + + +@pytest.mark.parametrize( + ("download_latency", "target_concurrency", "slot_delay", "expected"), + ( + (2.0, 2.0, 1.0, 1.0), + (1.0, 2.0, 1.0, 1.0), # Instead of 0.75 + (4.0, 2.0, 1.0, 2.0), + ), +) +def test_adjustment_bad_response( + download_latency, target_concurrency, slot_delay, expected +): + settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": download_latency, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url, status=400) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = slot_delay + crawler.engine.downloader.slots["foo"] = slot + + at._response_downloaded(response, request, spider) + + assert slot.delay == expected, f"{slot.delay} != {expected}" + + +def test_debug(caplog): + settings = {"AUTOTHROTTLE_DEBUG": True} + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": 1.0, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url, body=b"foo") + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = 2.0 + slot.transferring = (None, None) + crawler.engine.downloader.slots["foo"] = slot + + caplog.clear() + with caplog.at_level(INFO): + at._response_downloaded(response, request, spider) + + assert caplog.record_tuples == [ + ( + "scrapy.extensions.throttle", + INFO, + "slot: foo | conc: 2 | delay: 1500 ms (-500) | latency: 1000 ms | size: 3 bytes", + ), + ] + + +def test_debug_disabled(caplog): + crawler = get_crawler() + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": 1.0, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url, body=b"foo") + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = 2.0 + slot.transferring = (None, None) + crawler.engine.downloader.slots["foo"] = slot + + caplog.clear() + with caplog.at_level(INFO): + at._response_downloaded(response, request, spider) + + assert caplog.record_tuples == [] From d87f949526470fc4847c99f58e1dcc40d4e9ed00 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 27 Feb 2024 17:08:13 -0300 Subject: [PATCH 094/269] Use defusedxml.xmlrpc --- scrapy/http/request/rpc.py | 4 ++++ setup.py | 1 + 2 files changed, 5 insertions(+) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index bde860a66..2bf5ba4b6 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -7,9 +7,13 @@ See documentation in docs/topics/request-response.rst import xmlrpc.client as xmlrpclib from typing import Any, Optional +from defusedxml import xmlrpc + from scrapy.http.request import Request from scrapy.utils.python import get_func_args +xmlrpc.monkey_patch() + DUMPS_ARGS = get_func_args(xmlrpclib.dumps) diff --git a/setup.py b/setup.py index 405633f55..2d6d26b0c 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ install_requires = [ "packaging", "tldextract", "lxml>=4.4.1", + "defusedxml>=0.7.1", ] extras_require = { ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"], From 7f945ad6db728234987629b2299750abee5c1781 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 27 Feb 2024 17:39:29 -0300 Subject: [PATCH 095/269] Import defusedxml.xmlrpc using alias --- scrapy/http/request/rpc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 2bf5ba4b6..5a2107f76 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -7,12 +7,12 @@ See documentation in docs/topics/request-response.rst import xmlrpc.client as xmlrpclib from typing import Any, Optional -from defusedxml import xmlrpc +import defusedxml.xmlrpc as xml_rpc from scrapy.http.request import Request from scrapy.utils.python import get_func_args -xmlrpc.monkey_patch() +xml_rpc.monkey_patch() DUMPS_ARGS = get_func_args(xmlrpclib.dumps) From 008ebb65fc2f0e7cfe9fa43d7ac938a94b3098fb Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 27 Feb 2024 18:10:28 -0300 Subject: [PATCH 096/269] Change immport style --- scrapy/http/request/rpc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 5a2107f76..84b433990 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -7,12 +7,12 @@ See documentation in docs/topics/request-response.rst import xmlrpc.client as xmlrpclib from typing import Any, Optional -import defusedxml.xmlrpc as xml_rpc +import defusedxml.xmlrpc from scrapy.http.request import Request from scrapy.utils.python import get_func_args -xml_rpc.monkey_patch() +defusedxml.xmlrpc.monkey_patch() DUMPS_ARGS = get_func_args(xmlrpclib.dumps) From 282767f23b2e71969bea3bd5492abde88d2054c6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 27 Feb 2024 19:49:06 +0500 Subject: [PATCH 097/269] Bump black. --- .flake8 | 2 +- .pre-commit-config.yaml | 4 ++-- docs/topics/addons.rst | 3 +-- scrapy/commands/__init__.py | 1 + scrapy/commands/shell.py | 1 + scrapy/core/downloader/handlers/__init__.py | 6 +++--- scrapy/core/downloader/handlers/http10.py | 1 + scrapy/core/downloader/middleware.py | 1 + scrapy/core/engine.py | 1 + scrapy/core/http2/stream.py | 12 ++++++------ scrapy/core/scraper.py | 1 + scrapy/core/spidermw.py | 1 + scrapy/downloadermiddlewares/defaultheaders.py | 1 + scrapy/downloadermiddlewares/downloadtimeout.py | 1 + scrapy/downloadermiddlewares/retry.py | 1 + scrapy/exceptions.py | 1 + scrapy/extension.py | 1 + scrapy/extensions/corestats.py | 1 + scrapy/extensions/memusage.py | 1 + scrapy/extensions/postprocessing.py | 1 + scrapy/http/request/__init__.py | 17 +++++++++++------ scrapy/http/request/rpc.py | 1 + scrapy/http/response/__init__.py | 1 + scrapy/http/response/text.py | 1 + scrapy/link.py | 1 + scrapy/linkextractors/__init__.py | 1 + scrapy/linkextractors/lxmlhtml.py | 1 + scrapy/loader/__init__.py | 1 + scrapy/mail.py | 1 + scrapy/pipelines/__init__.py | 1 + scrapy/pipelines/files.py | 1 + scrapy/pipelines/images.py | 1 + scrapy/responsetypes.py | 1 + scrapy/selector/unified.py | 1 + scrapy/settings/__init__.py | 1 - scrapy/shell.py | 1 + scrapy/spidermiddlewares/httperror.py | 1 + scrapy/spidermiddlewares/offsite.py | 1 + scrapy/spidermiddlewares/referer.py | 1 + scrapy/spiders/__init__.py | 1 + scrapy/spiders/feed.py | 1 + scrapy/statscollectors.py | 1 + scrapy/utils/defer.py | 7 +++---- scrapy/utils/deprecate.py | 6 ++---- scrapy/utils/iterators.py | 11 +++++------ scrapy/utils/misc.py | 1 + scrapy/utils/python.py | 7 +++---- scrapy/utils/request.py | 7 ++++--- scrapy/utils/response.py | 7 ++++--- scrapy/utils/signal.py | 1 + scrapy/utils/sitemap.py | 1 + scrapy/utils/spider.py | 15 +++++---------- scrapy/utils/url.py | 1 + tests/mocks/dummydbm.py | 1 + tests/spiders.py | 1 + tests/test_exporters.py | 4 +++- tests/test_pipeline_media.py | 6 +++--- tests/test_responsetypes.py | 5 ++++- tests/test_webclient.py | 1 + 59 files changed, 102 insertions(+), 60 deletions(-) diff --git a/.flake8 b/.flake8 index 544d72956..62ccad9cf 100644 --- a/.flake8 +++ b/.flake8 @@ -1,7 +1,7 @@ [flake8] max-line-length = 119 -ignore = W503, E203 +ignore = E203, E501, E701, E704, W503 exclude = docs/conf.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0cff5cc73..83bc65b67 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: hooks: - id: flake8 - repo: https://github.com/psf/black.git - rev: 23.9.1 + rev: 24.2.0 hooks: - id: black - repo: https://github.com/pycqa/isort @@ -21,4 +21,4 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==23.9.1 + - black==24.2.0 diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 1bf2172bd..d2fc41003 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -150,8 +150,7 @@ Access the crawler instance: def from_crawler(cls, crawler): return cls(crawler) - def update_settings(self, settings): - ... + def update_settings(self, settings): ... Use a fallback component: diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 2aa569cdd..27993710e 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -1,6 +1,7 @@ """ Base class for Scrapy commands """ + import argparse import os from pathlib import Path diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 12e37babc..f72a23c6a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -3,6 +3,7 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ + from argparse import Namespace from threading import Thread from typing import List, Type diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 416669b7f..ade51ca63 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -21,9 +21,9 @@ logger = logging.getLogger(__name__) class DownloadHandlers: def __init__(self, crawler: "Crawler"): self._crawler: "Crawler" = crawler - self._schemes: Dict[ - str, Union[str, Callable] - ] = {} # stores acceptable schemes on instancing + self._schemes: Dict[str, Union[str, Callable]] = ( + {} + ) # stores acceptable schemes on instancing self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes self._notconfigured: Dict[str, str] = {} # remembers failed handlers handlers: Dict[str, Union[str, Callable]] = without_none_values( diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index b6ac7a251..d168c2b2e 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -1,5 +1,6 @@ """Download handlers for http and https schemes """ + from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index dca13c01e..52ebe4e22 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,6 +3,7 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ + from typing import Any, Callable, Generator, List, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 545cd401f..2db085081 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -4,6 +4,7 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spider. For more information see docs/topics/architecture.rst """ + import logging from time import time from typing import ( diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 0f282d83d..4132fc385 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -111,17 +111,17 @@ class Stream: # Metadata of an HTTP/2 connection stream # initialized when stream is instantiated self.metadata: Dict = { - "request_content_length": 0 - if self._request.body is None - else len(self._request.body), + "request_content_length": ( + 0 if self._request.body is None else len(self._request.body) + ), # Flag to keep track whether the stream has initiated the request "request_sent": False, # Flag to track whether we have logged about exceeding download warnsize "reached_warnsize": False, # Each time we send a data frame, we will decrease value by the amount send. - "remaining_content_length": 0 - if self._request.body is None - else len(self._request.body), + "remaining_content_length": ( + 0 if self._request.body is None else len(self._request.body) + ), # Flag to keep track whether client (self) have closed this stream "stream_closed_local": False, # Flag to keep track whether the server has closed the stream diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 8fb16b8a9..272841e01 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -1,5 +1,6 @@ """This module implements the Scraper component which parses responses and extracts information from them""" + from __future__ import annotations import logging diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 031a0be36..1ccfd08a2 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,6 +3,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ + import logging from inspect import isasyncgenfunction, iscoroutine from itertools import islice diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 8aec37cf1..58fd415b9 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -3,6 +3,7 @@ DefaultHeaders downloader middleware See documentation in docs/topics/downloader-middleware.rst """ + from __future__ import annotations from typing import TYPE_CHECKING, Iterable, Tuple, Union diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index 1c904c05b..fd7c03a38 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -3,6 +3,7 @@ Download timeout middleware See documentation in docs/topics/downloader-middleware.rst """ + from __future__ import annotations from typing import TYPE_CHECKING, Union diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 3c494de78..46587a898 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -9,6 +9,7 @@ RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ + from __future__ import annotations import warnings diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 6d188c489..e7ecdbe0c 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -4,6 +4,7 @@ Scrapy core exceptions These exceptions are documented in docs/topics/exceptions.rst. Please don't add new exceptions here without documenting them there. """ + from typing import Any # Internal diff --git a/scrapy/extension.py b/scrapy/extension.py index 4e365cfa1..6be14450c 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -3,6 +3,7 @@ The Extension Manager See documentation in docs/topics/extensions.rst """ + from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 302a615f2..717c249d9 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -1,6 +1,7 @@ """ Extension for collecting core stats like items scraped and start/finish times """ + from datetime import datetime, timezone from scrapy import signals diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index ca766c938..4d4501c44 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -3,6 +3,7 @@ MemoryUsage extension See documentation in docs/topics/extensions.rst """ + import logging import socket import sys diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index 17969c5b0..f8b59827b 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -1,6 +1,7 @@ """ Extension for processing data before they are exported to feeds. """ + from bz2 import BZ2File from gzip import GzipFile from io import IOBase diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a1c5a5e51..6269ee86a 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,6 +4,7 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ + import inspect from typing import ( Any, @@ -231,12 +232,16 @@ class Request(object_ref): """ d = { "url": self.url, # urls are safe (safe_string_url) - "callback": _find_method(spider, self.callback) - if callable(self.callback) - else self.callback, - "errback": _find_method(spider, self.errback) - if callable(self.errback) - else self.errback, + "callback": ( + _find_method(spider, self.callback) + if callable(self.callback) + else self.callback + ), + "errback": ( + _find_method(spider, self.errback) + if callable(self.errback) + else self.errback + ), "headers": dict(self.headers), } for attr in self.attributes: diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 84b433990..e20e7c438 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -4,6 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ + import xmlrpc.client as xmlrpclib from typing import Any, Optional diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 6eae3e8b3..d73dfce4b 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,6 +4,7 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ + from __future__ import annotations from ipaddress import IPv4Address, IPv6Address diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 6596d8a5c..2816610fb 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -4,6 +4,7 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ + from __future__ import annotations import json diff --git a/scrapy/link.py b/scrapy/link.py index 0868ae5ef..4bdbc1823 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,6 +4,7 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ + from typing import Any diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 3774430a7..73a63651c 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -5,6 +5,7 @@ This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ + import re # common file extensions that are not followed if they occur in links diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 23cbd0116..d76db20ba 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,6 +1,7 @@ """ Link extractor based on lxml.html """ + import logging import operator from functools import partial diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 1042a3d48..529fa279e 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -3,6 +3,7 @@ Item Loader See documentation in docs/topics/loaders.rst """ + import itemloaders from scrapy.item import Item diff --git a/scrapy/mail.py b/scrapy/mail.py index 237327451..4b18b6003 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -3,6 +3,7 @@ Mail sending helpers See documentation in docs/topics/email.rst """ + import logging from email import encoders as Encoders from email.mime.base import MIMEBase diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index c97d71fb6..f9544d329 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -3,6 +3,7 @@ Item pipeline See documentation in docs/item-pipeline.rst """ + from typing import Any, List from twisted.internet.defer import Deferred diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 73064ad10..1d7625299 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -3,6 +3,7 @@ Files Pipeline See documentation in topics/media-pipeline.rst """ + import base64 import functools import hashlib diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 02c4b1361..8169583f8 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -3,6 +3,7 @@ Images Pipeline See documentation in topics/media-pipeline.rst """ + import functools import hashlib import warnings diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 0d127d851..702e50536 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -2,6 +2,7 @@ This module implements a class which returns the appropriate Response class based on different criteria. """ + from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 863fb6032..75d5e9fbd 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -1,6 +1,7 @@ """ XPath selectors based on lxml """ + from typing import Any, Optional, Type, Union from parsel import Selector as _ParselSelector diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b5d8fdb12..d270a72f4 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -58,7 +58,6 @@ def get_settings_priority(priority: Union[int, str]) -> int: class SettingsAttribute: - """Class for storing data related to settings attributes. This class is intended for internal usage, you should try Settings class diff --git a/scrapy/shell.py b/scrapy/shell.py index bb3b1461c..05909977a 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -3,6 +3,7 @@ See documentation in docs/topics/shell.rst """ + import os import signal diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 94450b35b..35c869a75 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -3,6 +3,7 @@ HttpError Spider Middleware See documentation in docs/topics/spider-middleware.rst """ + from __future__ import annotations import logging diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index a5214702d..dd2fccfcb 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -3,6 +3,7 @@ Offsite Spider Middleware See documentation in docs/topics/spider-middleware.rst """ + from __future__ import annotations import logging diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a29e0ebb5..a0b6851e5 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,6 +2,7 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ + from __future__ import annotations import warnings diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index e16d71727..72c2aaba7 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -3,6 +3,7 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ + from __future__ import annotations import logging diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 599af7360..5caf8c79e 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -4,6 +4,7 @@ for scraping from an XML feed. See documentation in docs/topics/spiders.rst """ + from scrapy.exceptions import NotConfigured, NotSupported from scrapy.selector import Selector from scrapy.spiders import Spider diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 15193aac5..ab571a3ab 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -1,6 +1,7 @@ """ Scrapy extension for collecting scraping stats """ + import logging import pprint from typing import TYPE_CHECKING, Any, Dict, Optional diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index bf3c5ef5b..c391db9fd 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -1,6 +1,7 @@ """ Helper functions for dealing with Twisted deferreds """ + import asyncio import inspect from asyncio import Future @@ -304,13 +305,11 @@ _T = TypeVar("_T") @overload -def deferred_from_coro(o: _CT) -> Deferred: - ... +def deferred_from_coro(o: _CT) -> Deferred: ... @overload -def deferred_from_coro(o: _T) -> _T: - ... +def deferred_from_coro(o: _T) -> _T: ... def deferred_from_coro(o: _T) -> Union[Deferred, _T]: diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index ea577c44a..e0f2ac763 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -138,13 +138,11 @@ DEPRECATION_RULES: List[Tuple[str, str]] = [] @overload -def update_classpath(path: str) -> str: - ... +def update_classpath(path: str) -> str: ... @overload -def update_classpath(path: Any) -> Any: - ... +def update_classpath(path: Any) -> Any: ... def update_classpath(path: Any) -> Any: diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c56be5ea2..93a2ba7a1 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -225,18 +225,17 @@ def csviter( @overload -def _body_or_str(obj: Union[Response, str, bytes]) -> str: - ... +def _body_or_str(obj: Union[Response, str, bytes]) -> str: ... @overload -def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: - ... +def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: ... @overload -def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[False]) -> bytes: - ... +def _body_or_str( + obj: Union[Response, str, bytes], unicode: Literal[False] +) -> bytes: ... def _body_or_str( diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index b38190cb3..7b43760a8 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,4 +1,5 @@ """Helper functions which don't fit anywhere else""" + import ast import hashlib import inspect diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 68ca96b69..7b408c49c 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,6 +1,7 @@ """ This module contains essential stuff that should've come with Python itself ;) """ + import collections.abc import gc import inspect @@ -285,13 +286,11 @@ def equal_attributes( @overload -def without_none_values(iterable: Mapping) -> dict: - ... +def without_none_values(iterable: Mapping) -> dict: ... @overload -def without_none_values(iterable: Iterable) -> Iterable: - ... +def without_none_values(iterable: Iterable) -> Iterable: ... def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]: diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index db0b44cf4..e99d1eeb5 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -44,7 +44,9 @@ def _serialize_headers( yield from request.headers.getlist(header) -_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" +_fingerprint_cache: ( + "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" +) _fingerprint_cache = WeakKeyDictionary() @@ -114,8 +116,7 @@ def fingerprint( class RequestFingerprinterProtocol(Protocol): - def fingerprint(self, request: Request) -> bytes: - ... + def fingerprint(self, request: Request) -> bytes: ... class RequestFingerprinter: diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 33cd692bf..63a484b42 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -2,6 +2,7 @@ This module provides some useful functions for working with scrapy.http.Response objects """ + import os import re import tempfile @@ -29,9 +30,9 @@ def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: return _baseurl_cache[response] -_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = ( - WeakKeyDictionary() -) +_metaref_cache: ( + "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" +) = WeakKeyDictionary() def get_meta_refresh( diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 21a12a19e..a25100c03 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,4 +1,5 @@ """Helper functions for working with signals""" + import collections.abc import logging from typing import Any as TypingAny diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 3d2ecc9a7..8bf941eb2 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -4,6 +4,7 @@ Module for processing Sitemaps. Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ + from typing import Any, Dict, Generator, Iterator, Optional from urllib.parse import urljoin diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 704df8657..855bc8f87 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -39,13 +39,11 @@ def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: # type: ig @overload -def iterate_spider_output(result: CoroutineType) -> Deferred: - ... +def iterate_spider_output(result: CoroutineType) -> Deferred: ... @overload -def iterate_spider_output(result: _T) -> Iterable: - ... +def iterate_spider_output(result: _T) -> Iterable: ... def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]: @@ -83,8 +81,7 @@ def spidercls_for_request( default_spidercls: Type[Spider], log_none: bool = ..., log_multiple: bool = ..., -) -> Type[Spider]: - ... +) -> Type[Spider]: ... @overload @@ -94,8 +91,7 @@ def spidercls_for_request( default_spidercls: Literal[None], log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: - ... +) -> Optional[Type[Spider]]: ... @overload @@ -105,8 +101,7 @@ def spidercls_for_request( *, log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: - ... +) -> Optional[Type[Spider]]: ... def spidercls_for_request( diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 22b4197f9..9d97cb12f 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -5,6 +5,7 @@ library. Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ + import re from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index 2869ff8f7..bde3de228 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -1,4 +1,5 @@ """DBM-like dummy module""" + import collections from typing import Any, DefaultDict diff --git a/tests/spiders.py b/tests/spiders.py index 3df153a12..94969db99 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -1,6 +1,7 @@ """ Some spiders used for testing and benchmarking """ + import asyncio import time from urllib.parse import urlencode diff --git a/tests/test_exporters.py b/tests/test_exporters.py index c11913365..59b724495 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -121,7 +121,9 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(name, "John\xa3") ie = self._get_exporter(fields_to_export={"name": "名稱"}) - self.assertEqual(list(ie._get_serialized_fields(self.i)), [("名稱", "John\xa3")]) + self.assertEqual( + list(ie._get_serialized_fields(self.i)), [("名稱", "John\xa3")] + ) def test_field_custom_serializer(self): i = self.custom_field_item_class(name="John\xa3", age="22") diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 820484565..d477b59be 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -22,9 +22,9 @@ from scrapy.utils.test import get_crawler try: from PIL import Image # noqa: imported just to check for the import error except ImportError: - skip_pillow: Optional[ - str - ] = "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" + skip_pillow: Optional[str] = ( + "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" + ) else: skip_pillow = None diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 713a83d52..2633cca5b 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -33,7 +33,10 @@ class ResponseTypesTest(unittest.TestCase): ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response), ("attachment;filename=data高.doc".encode("gbk"), Response), ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse), - ("attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"), XmlResponse), + ( + "attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"), + XmlResponse, + ), ] for source, cls in mappings: retcls = responsetypes.from_content_disposition(source) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 53558814d..cce119001 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -2,6 +2,7 @@ from twisted.internet import defer Tests borrowed from the twisted.web.client tests. """ + import shutil from pathlib import Path from tempfile import mkdtemp From 6e5918345b8eb10674e11d9c4c5db8c9028be674 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 27 Feb 2024 19:50:54 +0500 Subject: [PATCH 098/269] Bump bandit, flake8 and isort. --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 83bc65b67..a911d4cfe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,11 @@ repos: - repo: https://github.com/PyCQA/bandit - rev: 1.7.5 + rev: 1.7.7 hooks: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 + rev: 7.0.0 hooks: - id: flake8 - repo: https://github.com/psf/black.git @@ -13,7 +13,7 @@ repos: hooks: - id: black - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/adamchainz/blacken-docs From 68104b9f48802d1ecc1c892c61aaa197500435b5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 27 Feb 2024 20:21:27 +0500 Subject: [PATCH 099/269] Update .bandit.yml, add problem names. --- .bandit.yml | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 8c6a08e1b..6e8331c0f 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,20 +1,19 @@ skips: -- B101 -- B113 # https://github.com/PyCQA/bandit/issues/1010 -- B105 -- B301 -- B303 -- B307 -- B311 -- B320 -- B321 -- B324 -- B402 # https://github.com/scrapy/scrapy/issues/4180 -- B403 -- B404 -- B406 -- B410 -- B503 -- B603 -- B605 +- B101 # assert_used +- B105 # hardcoded_password_string +- B301 # pickle +- B307 # eval +- B311 # random +- B320 # xml_bad_etree +- B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180 +- B324 # hashlib "Use of weak SHA1 hash for security" +- B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 +- B403 # import_pickle +- B404 # import_subprocess +- B406 # import_xml_sax +- B410 # import_lxml +- B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 +- B503 # ssl_with_bad_defaults +- B603 # subprocess_without_shell_equals_true +- B605 # start_process_with_a_shell exclude_dirs: ['tests'] From d2c05d9d96394e111ead1aa097402cdbc18c0859 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 27 Feb 2024 20:43:57 +0500 Subject: [PATCH 100/269] Bump mypy and type stubs. --- scrapy/utils/spider.py | 3 +-- scrapy/utils/ssl.py | 2 +- tox.ini | 14 +++++++------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 855bc8f87..cbbb01d85 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -34,8 +34,7 @@ _T = TypeVar("_T") # https://stackoverflow.com/questions/60222982 @overload -def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: # type: ignore[misc] - ... +def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: ... # type: ignore[overload-overlap] @overload diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index e74769c65..d520ef809 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,6 +1,6 @@ from typing import Any, Optional -import OpenSSL._util as pyOpenSSLutil # type: ignore[import-untyped] +import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL import OpenSSL.version from OpenSSL.crypto import X509Name diff --git a/tox.ini b/tox.ini index 359ff0f73..e787c7bf3 100644 --- a/tox.ini +++ b/tox.ini @@ -29,14 +29,14 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.6.1 - typing-extensions==4.8.0 + mypy==1.8.0 + typing-extensions==4.10.0 types-attrs==19.1.0 - types-lxml==2023.10.21 - types-Pillow==10.1.0.0 - types-Pygments==2.16.0.0 - types-pyOpenSSL==23.3.0.0 - types-setuptools==68.2.0.0 + types-lxml==2024.2.9 + types-Pillow==10.2.0.20240213 + types-Pygments==2.17.0.20240106 + types-pyOpenSSL==24.0.0.20240130 + types-setuptools==69.1.0.20240223 # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 w3lib >= 2.1.2 commands = From 4f9dd998dcf01c003bc2a053b6bd78091d12ecd5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 27 Feb 2024 22:01:36 +0500 Subject: [PATCH 101/269] Bump pylint, cleanup the ignored tags. --- docs/conf.py | 2 +- pylintrc | 17 +----------- .../downloadermiddlewares/httpcompression.py | 2 +- scrapy/http/headers.py | 4 ++- scrapy/utils/ossignal.py | 6 ++++- scrapy/utils/signal.py | 5 +++- tests/test_commands.py | 27 ------------------- tests/test_item.py | 4 ++- tests/test_linkextractors.py | 3 --- tests/test_loader_deprecated.py | 6 ++--- tests/test_settings/__init__.py | 2 +- tests/test_utils_datatypes.py | 2 +- tests/test_utils_signal.py | 6 ----- tests/test_utils_spider.py | 2 +- tox.ini | 2 +- 15 files changed, 25 insertions(+), 65 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9ca0f817a..399078010 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -227,7 +227,7 @@ latex_documents = [ # A list of regular expressions that match URIs that should not be checked when # doing a linkcheck build. linkcheck_ignore = [ - "http://localhost:\d+", + r"http://localhost:\d+", "http://hg.scrapy.org", "http://directory.google.com/", ] diff --git a/pylintrc b/pylintrc index c8654b8d3..78004e78a 100644 --- a/pylintrc +++ b/pylintrc @@ -4,21 +4,14 @@ jobs=1 # >1 hides results [MESSAGES CONTROL] disable=abstract-method, - anomalous-backslash-in-string, arguments-differ, arguments-renamed, attribute-defined-outside-init, bad-classmethod-argument, - bad-mcs-classmethod-argument, bare-except, broad-except, broad-exception-raised, c-extension-no-member, - catching-non-exception, - cell-var-from-loop, - comparison-with-callable, - consider-using-dict-items, - consider-using-in, consider-using-with, cyclic-import, dangerous-default-value, @@ -32,7 +25,6 @@ disable=abstract-method, implicit-str-concat, import-error, import-outside-toplevel, - import-self, inconsistent-return-statements, inherit-non-class, invalid-name, @@ -44,7 +36,6 @@ disable=abstract-method, logging-fstring-interpolation, logging-not-lazy, lost-exception, - method-hidden, missing-docstring, no-else-raise, no-else-return, @@ -52,7 +43,7 @@ disable=abstract-method, no-method-argument, no-name-in-module, no-self-argument, - no-value-for-parameter, + no-value-for-parameter, # https://github.com/pylint-dev/pylint/issues/3268 not-callable, pointless-exception-statement, pointless-statement, @@ -77,14 +68,10 @@ disable=abstract-method, too-many-public-methods, too-many-return-statements, unbalanced-tuple-unpacking, - undefined-variable, - undefined-loop-variable, - unexpected-special-method-signature, unnecessary-comprehension, unnecessary-dunder-call, unnecessary-pass, unreachable, - unsubscriptable-object, unused-argument, unused-import, unused-private-member, @@ -92,8 +79,6 @@ disable=abstract-method, unused-wildcard-import, use-dict-literal, used-before-assignment, - useless-object-inheritance, # Required for Python 2 support useless-return, - useless-super-delegation, wildcard-import, wrong-import-position diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 1e340abb6..f0ad24f72 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -169,7 +169,7 @@ class HttpCompressionMiddleware: return to_decode, to_keep def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes: - if encoding == b"gzip" or encoding == b"x-gzip": + if encoding in {b"gzip", b"x-gzip"}: return gunzip(body, max_size=max_size) if encoding == b"deflate": return _inflate(body, max_size=max_size) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 21eb9fb73..73aee7178 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -113,7 +113,9 @@ class Headers(CaselessDict): return ((k, self.getlist(k)) for k in self.keys()) def values(self) -> List[Optional[bytes]]: # type: ignore[override] - return [self[k] for k in self.keys()] + return [ + self[k] for k in self.keys() # pylint: disable=consider-using-dict-items + ] def to_string(self) -> bytes: # cast() can be removed if the headers_dict_to_raw() hint is improved diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index db9a71273..5985a847e 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -24,7 +24,11 @@ def install_shutdown_handlers( (e.g. Pdb) """ signal.signal(signal.SIGTERM, function) - if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint: + if ( + signal.getsignal(signal.SIGINT) # pylint: disable=comparison-with-callable + == signal.default_int_handler + or override_sigint + ): signal.signal(signal.SIGINT, function) # Catch Ctrl-Break in windows if hasattr(signal, "SIGBREAK"): diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index a25100c03..89cfbd2ec 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -98,7 +98,10 @@ def send_catch_log_deferred( robustApply, receiver, signal=signal, sender=sender, *arguments, **named ) d.addErrback(logerror, receiver) - d.addBoth(lambda result: (receiver, result)) + # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html + d.addBoth( + lambda result: (receiver, result) # pylint: disable=cell-var-from-loop + ) dfds.append(d) d = DeferredList(dfds) d.addCallback(lambda out: [x[1] for x in out]) diff --git a/tests/test_commands.py b/tests/test_commands.py index 2f36baa87..febad21da 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -991,38 +991,11 @@ class MySpider(scrapy.Spider): class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = "myspider.pyw" - def setUp(self): - super().setUp() - def test_start_requests_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") self.assertIn("start_requests", log) self.assertIn("badspider.pyw", log) - def test_run_good_spider(self): - super().test_run_good_spider() - - def test_runspider(self): - super().test_runspider() - - def test_runspider_dnscache_disabled(self): - super().test_runspider_dnscache_disabled() - - def test_runspider_log_level(self): - super().test_runspider_log_level() - - def test_runspider_log_short_names(self): - super().test_runspider_log_short_names() - - def test_runspider_no_spider_found(self): - super().test_runspider_no_spider_found() - - def test_output(self): - super().test_output() - - def test_overwrite_output(self): - super().test_overwrite_output() - def test_runspider_unable_to_load(self): raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") diff --git a/tests/test_item.py b/tests/test_item.py index ce2b4fd15..daf5d4f59 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -290,7 +290,9 @@ class ItemMetaTest(unittest.TestCase): class ItemMetaClassCellRegression(unittest.TestCase): def test_item_meta_classcell_regression(self): class MyItem(Item, metaclass=ItemMeta): - def __init__(self, *args, **kwargs): + def __init__( + self, *args, **kwargs + ): # pylint: disable=useless-parent-delegation # This call to super() trigger the __classcell__ propagation # requirement. When not done properly raises an error: # TypeError: __class__ set to diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 55ea9eed2..6b4df90d8 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -818,9 +818,6 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): ], ) - def test_restrict_xpaths_with_html_entities(self): - super().test_restrict_xpaths_with_html_entities() - @mark.skipif( Version(w3lib_version) < Version("2.0.0"), reason=( diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index d7f773d5c..99cdf88d9 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -678,11 +678,11 @@ class SelectJmesTestCase(unittest.TestCase): } def test_output(self): - for tl in self.test_list_equals: - expr, test_list, expected = self.test_list_equals[tl] + for k, v in self.test_list_equals.items(): + expr, test_list, expected = v test = SelectJmes(expr)(test_list) self.assertEqual( - test, expected, msg=f'test "{tl}" got {test} expected {expected}' + test, expected, msg=f'test "{k}" got {test} expected {expected}' ) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 3fde5e8c5..9ee248538 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -426,7 +426,7 @@ class SettingsTest(unittest.TestCase): mydict = settings.get("TEST_DICT") self.assertIsInstance(mydict, BaseSettings) self.assertIn("key", mydict) - self.assertEqual(mydict["key"], "val") + self.assertEqual(mydict["key"], "val") # pylint: disable=unsubscriptable-object self.assertEqual(mydict.getpriority("key"), 0) @mock.patch("scrapy.settings.default_settings", default_settings) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 9e5f88f48..be5c6de81 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -353,7 +353,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): for i, r in enumerate(refs): self.assertIn(r, cache) self.assertEqual(cache[r], i) - del r # delete reference to the last object in the list + del r # delete reference to the last object in the list # pylint: disable=undefined-loop-variable # delete half of the objects, make sure that is reflected in the cache for _ in range(max // 2): diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 65b99e0c4..60232f10b 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -75,9 +75,6 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): await defer.succeed(42) return "OK" - def test_send_catch_log(self): - return super().test_send_catch_log() - @mark.only_asyncio() class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): @@ -87,9 +84,6 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): await asyncio.sleep(0.2) return await get_from_asyncio_queue("OK") - def test_send_catch_log(self): - return super().test_send_catch_log() - class SendCatchLogTest2(unittest.TestCase): def test_error_logged_if_deferred_not_supported(self): diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index 460ae40c3..dd1d26448 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -26,7 +26,7 @@ class UtilsSpidersTestCase(unittest.TestCase): self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o]) def test_iter_spider_classes(self): - import tests.test_utils_spider + import tests.test_utils_spider # pylint: disable=import-self it = iter_spider_classes(tests.test_utils_spider) self.assertEqual(set(it), {MySpider1, MySpider2}) diff --git a/tox.ini b/tox.ini index e787c7bf3..4ed9b3bd7 100644 --- a/tox.ini +++ b/tox.ini @@ -53,7 +53,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==3.0.1 + pylint==3.1.0 commands = pylint conftest.py docs extras scrapy setup.py tests From 63acd0720970c87450fdbcb9aa6967118c9c1cf2 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 28 Feb 2024 16:14:08 -0300 Subject: [PATCH 102/269] Fix and re-enable unnecessary-comprehension and use-dict-literal pylint tags --- pylintrc | 2 -- scrapy/downloadermiddlewares/httpcompression.py | 2 +- scrapy/spiders/crawl.py | 2 +- scrapy/utils/python.py | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pylintrc b/pylintrc index 78004e78a..c60e4e16a 100644 --- a/pylintrc +++ b/pylintrc @@ -68,7 +68,6 @@ disable=abstract-method, too-many-public-methods, too-many-return-statements, unbalanced-tuple-unpacking, - unnecessary-comprehension, unnecessary-dunder-call, unnecessary-pass, unreachable, @@ -77,7 +76,6 @@ disable=abstract-method, unused-private-member, unused-variable, unused-wildcard-import, - use-dict-literal, used-before-assignment, useless-return, wildcard-import, diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index f0ad24f72..aa3abe853 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -135,7 +135,7 @@ class HttpCompressionMiddleware: respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) - kwargs = dict(cls=respcls, body=decoded_body) + kwargs = {"cls": respcls, "body": decoded_body} if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the # responsetypes guessing is reliable diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index ebb4f5984..2a3913da5 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -85,7 +85,7 @@ class CrawlSpider(Spider): url=link.url, callback=self._callback, errback=self._errback, - meta=dict(rule=rule_index, link_text=link.text), + meta={"rule": rule_index, "link_text": link.text}, ) def _requests_to_follow(self, response): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7b408c49c..1e7364e49 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -162,7 +162,7 @@ def re_rsearch( pattern = re.compile(pattern) for chunk, offset in _chunk_iter(): - matches = [match for match in pattern.finditer(chunk)] + matches = list(pattern.finditer(chunk)) if matches: start, end = matches[-1].span() return offset + start, offset + end From 26a16f2c43dc96fe33d0b0fc8846402e9ae97e9a Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 28 Feb 2024 16:36:19 -0300 Subject: [PATCH 103/269] Fix tests --- tests/test_crawl.py | 10 ++--- tests/test_downloadermiddleware_cookies.py | 2 +- tests/test_downloadermiddleware_httpauth.py | 4 +- tests/test_exporters.py | 24 +++++------ tests/test_linkextractors.py | 38 ++++++++-------- tests/test_loader_deprecated.py | 20 ++++----- tests/test_mail.py | 2 +- tests/test_pipeline_crawl.py | 2 +- tests/test_pipeline_files.py | 2 +- tests/test_pipeline_images.py | 18 ++++---- tests/test_pipeline_media.py | 48 ++++++++++----------- tests/test_scheduler.py | 26 +++++------ tests/test_spidermiddleware_offsite.py | 17 +++++--- tests/test_utils_iterators.py | 16 +++---- tests/test_utils_template.py | 2 +- 15 files changed, 117 insertions(+), 114 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 96d43b2b9..6cde4ed8c 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -76,11 +76,11 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def _test_delay(self, total, delay, randomize=False): - crawl_kwargs = dict( - maxlatency=delay * 2, - mockserver=self.mockserver, - total=total, - ) + crawl_kwargs = { + "maxlatency": delay * 2, + "mockserver": self.mockserver, + "total": total, + } tolerance = 1 - (0.6 if randomize else 0.2) settings = {"DOWNLOAD_DELAY": delay, "RANDOMIZE_DOWNLOAD_DELAY": randomize} diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 4a81a638e..425fabcc7 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -320,7 +320,7 @@ class CookiesMiddlewareTest(TestCase): @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_from_default_request_headers_middleware(self): - DEFAULT_REQUEST_HEADERS = dict(Cookie="default=value; asdf=qwerty") + DEFAULT_REQUEST_HEADERS = {"Cookie": "default=value; asdf=qwerty"} mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items()) # overwrite with values from 'cookies' request argument req1 = Request("http://example.org", cookies={"default": "something"}) diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index fc110e6cc..500af6536 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -59,7 +59,7 @@ class HttpAuthMiddlewareTest(unittest.TestCase): self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) def test_auth_already_set(self): - req = Request("http://example.com/", headers=dict(Authorization="Digest 123")) + req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.headers["Authorization"], b"Digest 123") @@ -79,6 +79,6 @@ class HttpAuthAnyMiddlewareTest(unittest.TestCase): self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) def test_auth_already_set(self): - req = Request("http://example.com/", headers=dict(Authorization="Digest 123")) + req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.headers["Authorization"], b"Digest 123") diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 59b724495..fa9389044 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -152,7 +152,7 @@ class PythonItemExporterTest(BaseItemExporterTest): def test_nested_item(self): i1 = self.item_class(name="Joseph", age="22") - i2 = dict(name="Maria", age=i1) + i2 = {"name": "Maria", "age": i1} i3 = self.item_class(name="Jesus", age=i2) ie = self._get_exporter() exported = ie.export_item(i3) @@ -185,7 +185,7 @@ class PythonItemExporterTest(BaseItemExporterTest): def test_export_item_dict_list(self): i1 = self.item_class(name="Joseph", age="22") - i2 = dict(name="Maria", age=[i1]) + i2 = {"name": "Maria", "age": [i1]} i3 = self.item_class(name="Jesus", age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) @@ -373,7 +373,7 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_join_multivalue_not_strings(self): self.assertExportResult( - item=dict(name="John", friends=[4, 8]), + item={"name": "John", "friends": [4, 8]}, include_headers_line=False, expected='"[4, 8]",John\r\n', ) @@ -388,14 +388,14 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_errors_default(self): with self.assertRaises(UnicodeEncodeError): self.assertExportResult( - item=dict(text="W\u0275\u200Brd"), + item={"text": "W\u0275\u200Brd"}, expected=None, encoding="windows-1251", ) def test_errors_xmlcharrefreplace(self): self.assertExportResult( - item=dict(text="W\u0275\u200Brd"), + item={"text": "W\u0275\u200Brd"}, include_headers_line=False, expected="Wɵ​rd\r\n", encoding="windows-1251", @@ -455,8 +455,8 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_item(self): - i1 = dict(name="foo\xa3hoo", age="22") - i2 = dict(name="bar", age=i1) + i1 = {"name": "foo\xa3hoo", "age": "22"} + i2 = {"name": "bar", "age": i1} i3 = self.item_class(name="buz", age=i2) self.assertExportResult( @@ -478,8 +478,8 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_list_item(self): - i1 = dict(name="foo") - i2 = dict(name="bar", v2={"egg": ["spam"]}) + i1 = {"name": "foo"} + i2 = {"name": "bar", "v2": {"egg": ["spam"]}} i3 = self.item_class(name="buz", age=[i1, i2]) self.assertExportResult( @@ -534,7 +534,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): def test_nested_item(self): i1 = self.item_class(name="Joseph", age="22") - i2 = dict(name="Maria", age=i1) + i2 = {"name": "Maria", "age": i1} i3 = self.item_class(name="Jesus", age=i2) self.ie.start_exporting() self.ie.export_item(i3) @@ -622,9 +622,9 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertEqual(exported, [expected]) def test_nested_dict_item(self): - i1 = dict(name="Joseph\xa3", age="22") + i1 = {"name": "Joseph\xa3", "age": "22"} i2 = self.item_class(name="Maria", age=i1) - i3 = dict(name="Jesus", age=i2) + i3 = {"name": "Jesus", "age": i2} self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 6b4df90d8..217c7a299 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -37,7 +37,7 @@ class Base: page4_url = "http://example.com/page%204.html" self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -55,7 +55,7 @@ class Base: def test_extract_filter_allow(self): lx = self.extractor_cls(allow=("sample",)) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -70,7 +70,7 @@ class Base: def test_extract_filter_allow_with_duplicates(self): lx = self.extractor_cls(allow=("sample",), unique=False) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -93,7 +93,7 @@ class Base: def test_extract_filter_allow_with_duplicates_canonicalize(self): lx = self.extractor_cls(allow=("sample",), unique=False, canonicalize=True) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -116,7 +116,7 @@ class Base: def test_extract_filter_allow_no_duplicates_canonicalize(self): lx = self.extractor_cls(allow=("sample",), unique=True, canonicalize=True) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -127,7 +127,7 @@ class Base: def test_extract_filter_allow_and_deny(self): lx = self.extractor_cls(allow=("sample",), deny=("3",)) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -137,7 +137,7 @@ class Base: def test_extract_filter_allowed_domains(self): lx = self.extractor_cls(allow_domains=("google.com",)) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://www.google.com/something", text=""), ], @@ -148,7 +148,7 @@ class Base: lx = self.extractor_cls(allow="sample") self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -162,7 +162,7 @@ class Base: lx = self.extractor_cls(allow="sample", deny="3") self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -171,7 +171,7 @@ class Base: lx = self.extractor_cls(allow_domains="google.com") self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://www.google.com/something", text=""), ], @@ -179,7 +179,7 @@ class Base: lx = self.extractor_cls(deny_domains="example.com") self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://www.google.com/something", text=""), ], @@ -265,7 +265,7 @@ class Base: def test_restrict_xpaths(self): lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]',)) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -337,7 +337,7 @@ class Base: restrict_css=("#subwrapper + a",), ) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -705,7 +705,7 @@ class Base: response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item1.html", @@ -758,7 +758,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item1.html", text="Item 1", nofollow=False @@ -779,7 +779,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): # Simple text inclusion test lx = self.extractor_cls(restrict_text="dog") self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item2.html", @@ -791,7 +791,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): # Unique regex test lx = self.extractor_cls(restrict_text=r"of.*dog") self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item2.html", @@ -803,7 +803,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): # Multiple regex test lx = self.extractor_cls(restrict_text=[r"of.*dog", r"of.*cat"]) self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item1.html", @@ -834,7 +834,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item2.html", diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 99cdf88d9..528efa142 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -565,37 +565,37 @@ class NoInputReprocessingFromDictTest(unittest.TestCase): """ def test_avoid_reprocessing_with_initial_values_single(self): - il = NoInputReprocessingDictLoader(item=dict(title="foo")) + il = NoInputReprocessingDictLoader(item={"title": "foo"}) il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="foo")) + self.assertEqual(il_loaded, {"title": "foo"}) self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo") + NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"} ) def test_avoid_reprocessing_with_initial_values_list(self): - il = NoInputReprocessingDictLoader(item=dict(title=["foo", "bar"])) + il = NoInputReprocessingDictLoader(item={"title": ["foo", "bar"]}) il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="foo")) + self.assertEqual(il_loaded, {"title": "foo"}) self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo") + NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"} ) def test_avoid_reprocessing_without_initial_values_single(self): il = NoInputReprocessingDictLoader() il.add_value("title", "foo") il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="FOO")) + self.assertEqual(il_loaded, {"title": "FOO"}) self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO") + NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"} ) def test_avoid_reprocessing_without_initial_values_list(self): il = NoInputReprocessingDictLoader() il.add_value("title", ["foo", "bar"]) il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="FOO")) + self.assertEqual(il_loaded, {"title": "FOO"}) self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO") + NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"} ) diff --git a/tests/test_mail.py b/tests/test_mail.py index 2535e58db..ff1505397 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -91,7 +91,7 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(attach.get_payload(decode=True), b"content") def _catch_mail_sent(self, **kwargs): - self.catched_msg = dict(**kwargs) + self.catched_msg = {**kwargs} def test_send_utf8(self): subject = "sübjèçt" diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index be9811980..5a9a217ce 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -140,7 +140,7 @@ class FileDownloadCrawlTestCase(TestCase): self.assertEqual(logs.count(file_dl_failure), 3) # check that no files were written to the media store - self.assertEqual([x for x in self.tmpmediastore.iterdir()], []) + self.assertEqual(list(self.tmpmediastore.iterdir()), []) @defer.inlineCallbacks def test_download_media(self): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e7000e314..0babde4d9 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -221,7 +221,7 @@ class FilesPipelineTestCase(unittest.TestCase): file_path = CustomFilesPipeline.from_settings( Settings({"FILES_STORE": self.tempdir}) ).file_path - item = dict(path="path-to-store-file") + item = {"path": "path-to-store-file"} request = Request("http://example.com") self.assertEqual(file_path(request, item=item), "full/path-to-store-file") diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 2e2e06b89..18a2454b3 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -132,7 +132,7 @@ class ImagesPipelineTestCase(unittest.TestCase): thumb_path = CustomImagesPipeline.from_settings( Settings({"IMAGES_STORE": self.tempdir}) ).thumb_path - item = dict(path="path-to-store-file") + item = {"path": "path-to-store-file"} request = Request("http://example.com") self.assertEqual( thumb_path(request, "small", item=item), "thumb/small/path-to-store-file" @@ -433,14 +433,14 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): ] # This should match what is defined in ImagesPipeline. - default_pipeline_settings = dict( - MIN_WIDTH=0, - MIN_HEIGHT=0, - EXPIRES=90, - THUMBS={}, - IMAGES_URLS_FIELD="image_urls", - IMAGES_RESULT_FIELD="images", - ) + default_pipeline_settings = { + "MIN_WIDTH": 0, + "MIN_HEIGHT": 0, + "EXPIRES": 90, + "THUMBS": {}, + "IMAGES_URLS_FIELD": "image_urls", + "IMAGES_RESULT_FIELD": "images", + } def setUp(self): self.tempdir = mkdtemp() diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index d477b59be..d4dde4a40 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -59,7 +59,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): assert self.pipe.media_to_download(request, self.info) is None def test_default_get_media_requests(self): - item = dict(name="name") + item = {"name": "name"} assert self.pipe.get_media_requests(item, self.info) is None def test_default_media_downloaded(self): @@ -73,7 +73,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): assert self.pipe.media_failed(fail, request, self.info) is fail def test_default_item_completed(self): - item = dict(name="name") + item = {"name": "name"} assert self.pipe.item_completed([], item, self.info) is item # Check that failures are logged by default @@ -98,7 +98,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): @inlineCallbacks def test_default_process_item(self): - item = dict(name="name") + item = {"name": "name"} new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item @@ -226,11 +226,11 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): rsp = Response("http://url1") req = Request( "http://url1", - meta=dict(response=rsp), + meta={"response": rsp}, callback=self._callback, errback=self._errback, ) - item = dict(requests=req) + item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(True, rsp)]) self.assertEqual( @@ -250,11 +250,11 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): fail = Failure(Exception()) req = Request( "http://url1", - meta=dict(response=fail), + meta={"response": fail}, callback=self._callback, errback=self._errback, ) - item = dict(requests=req) + item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(False, fail)]) self.assertEqual( @@ -272,10 +272,10 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): def test_mix_of_success_and_failure(self): self.pipe.LOG_FAILED_RESULTS = False rsp1 = Response("http://url1") - req1 = Request("http://url1", meta=dict(response=rsp1)) + req1 = Request("http://url1", meta={"response": rsp1}) fail = Failure(Exception()) - req2 = Request("http://url2", meta=dict(response=fail)) - item = dict(requests=[req1, req2]) + req2 = Request("http://url2", meta={"response": fail}) + item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(True, rsp1), (False, fail)]) m = self.pipe._mockcalled @@ -294,7 +294,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): def test_get_media_requests(self): # returns single Request (without callback) req = Request("http://url") - item = dict(requests=req) # pass a single item + item = {"requests": req} # pass a single item new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item self.assertIn(self.fingerprint(req), self.info.downloaded) @@ -302,7 +302,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): # returns iterable of Requests req1 = Request("http://url1") req2 = Request("http://url2") - item = dict(requests=iter([req1, req2])) + item = {"requests": iter([req1, req2])} new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item assert self.fingerprint(req1) in self.info.downloaded @@ -311,17 +311,17 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): @inlineCallbacks def test_results_are_cached_across_multiple_items(self): rsp1 = Response("http://url1") - req1 = Request("http://url1", meta=dict(response=rsp1)) - item = dict(requests=req1) + req1 = Request("http://url1", meta={"response": rsp1}) + item = {"requests": req1} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) self.assertEqual(new_item["results"], [(True, rsp1)]) # rsp2 is ignored, rsp1 must be in results because request fingerprints are the same req2 = Request( - req1.url, meta=dict(response=Response("http://donot.download.me")) + req1.url, meta={"response": Response("http://donot.download.me")} ) - item = dict(requests=req2) + item = {"requests": req2} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) self.assertEqual(self.fingerprint(req1), self.fingerprint(req2)) @@ -330,11 +330,11 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): @inlineCallbacks def test_results_are_cached_for_requests_of_single_item(self): rsp1 = Response("http://url1") - req1 = Request("http://url1", meta=dict(response=rsp1)) + req1 = Request("http://url1", meta={"response": rsp1}) req2 = Request( - req1.url, meta=dict(response=Response("http://donot.download.me")) + req1.url, meta={"response": Response("http://donot.download.me")} ) - item = dict(requests=[req1, req2]) + item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) @@ -359,16 +359,16 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): def rsp2_func(): self.fail("it must cache rsp1 result and must not try to redownload") - req1 = Request("http://url", meta=dict(response=rsp1_func)) - req2 = Request(req1.url, meta=dict(response=rsp2_func)) - item = dict(requests=[req1, req2]) + req1 = Request("http://url", meta={"response": rsp1_func}) + req2 = Request(req1.url, meta={"response": rsp2_func}) + item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) @inlineCallbacks def test_use_media_to_download_result(self): - req = Request("http://url", meta=dict(result="ITSME", response=self.fail)) - item = dict(requests=req) + req = Request("http://url", meta={"result": "ITSME", "response": self.fail}) + item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(True, "ITSME")]) self.assertEqual( diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index f8465a5ff..37099dae6 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -45,15 +45,15 @@ class MockDownloader: class MockCrawler(Crawler): def __init__(self, priority_queue_cls, jobdir): - settings = dict( - SCHEDULER_DEBUG=False, - SCHEDULER_DISK_QUEUE="scrapy.squeues.PickleLifoDiskQueue", - SCHEDULER_MEMORY_QUEUE="scrapy.squeues.LifoMemoryQueue", - SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, - JOBDIR=jobdir, - DUPEFILTER_CLASS="scrapy.dupefilters.BaseDupeFilter", - REQUEST_FINGERPRINTER_IMPLEMENTATION="2.7", - ) + settings = { + "SCHEDULER_DEBUG": False, + "SCHEDULER_DISK_QUEUE": "scrapy.squeues.PickleLifoDiskQueue", + "SCHEDULER_MEMORY_QUEUE": "scrapy.squeues.LifoMemoryQueue", + "SCHEDULER_PRIORITY_QUEUE": priority_queue_cls, + "JOBDIR": jobdir, + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + } super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) self.stats = load_object(self.settings["STATS_CLASS"])(self) @@ -338,10 +338,10 @@ class TestIntegrationWithDownloaderAwareInMemory(TestCase): class TestIncompatibility(unittest.TestCase): def _incompatible(self): - settings = dict( - SCHEDULER_PRIORITY_QUEUE="scrapy.pqueues.DownloaderAwarePriorityQueue", - CONCURRENT_REQUESTS_PER_IP=1, - ) + settings = { + "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.DownloaderAwarePriorityQueue", + "CONCURRENT_REQUESTS_PER_IP": 1, + } crawler = get_crawler(Spider, settings) scheduler = Scheduler.from_crawler(crawler) spider = Spider(name="spider") diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index ea45b7698..837f1c2c8 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -16,10 +16,10 @@ class TestOffsiteMiddleware(TestCase): self.mw.spider_opened(self.spider) def _get_spiderargs(self): - return dict( - name="foo", - allowed_domains=["scrapytest.org", "scrapy.org", "scrapy.test.org"], - ) + return { + "name": "foo", + "allowed_domains": ["scrapytest.org", "scrapy.org", "scrapy.test.org"], + } def test_process_spider_output(self): res = Response("http://scrapytest.org") @@ -50,7 +50,7 @@ class TestOffsiteMiddleware(TestCase): class TestOffsiteMiddleware2(TestOffsiteMiddleware): def _get_spiderargs(self): - return dict(name="foo", allowed_domains=None) + return {"name": "foo", "allowed_domains": None} def test_process_spider_output(self): res = Response("http://scrapytest.org") @@ -61,13 +61,16 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware): class TestOffsiteMiddleware3(TestOffsiteMiddleware2): def _get_spiderargs(self): - return dict(name="foo") + return {"name": "foo"} class TestOffsiteMiddleware4(TestOffsiteMiddleware3): def _get_spiderargs(self): bad_hostname = urlparse("http:////scrapytest.org").hostname - return dict(name="foo", allowed_domains=["scrapytest.org", None, bad_hostname]) + return { + "name": "foo", + "allowed_domains": ["scrapytest.org", None, bad_hostname], + } def test_process_spider_output(self): res = Response("http://scrapytest.org") diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index ee22e6675..ec377bb19 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -355,7 +355,7 @@ class UtilsCsvTestCase(unittest.TestCase): response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) - result = [row for row in csv] + result = list(csv) self.assertEqual( result, [ @@ -377,7 +377,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, delimiter="\t") self.assertEqual( - [row for row in csv], + list(csv), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -394,7 +394,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv1 = csviter(response1, quotechar="'") self.assertEqual( - [row for row in csv1], + list(csv1), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -407,7 +407,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv2 = csviter(response2, delimiter="|", quotechar="'") self.assertEqual( - [row for row in csv2], + list(csv2), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -422,7 +422,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response) self.assertEqual( - [row for row in csv], + list(csv), [ {"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"}, { @@ -441,7 +441,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, delimiter="\t") self.assertEqual( - [row for row in csv], + list(csv), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -458,7 +458,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, headers=[h.decode("utf-8") for h in headers]) self.assertEqual( - [row for row in csv], + list(csv), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -475,7 +475,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response) self.assertEqual( - [row for row in csv], + list(csv), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index cbe80e157..fc42c0d2f 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -16,7 +16,7 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): rmtree(self.tmp_path) def test_simple_render(self): - context = dict(project_name="proj", name="spi", classname="TheSpider") + context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"} template = "from ${project_name}.spiders.${name} import ${classname}" rendered = "from proj.spiders.spi import TheSpider" From 706eb8d4275be993867122e5e41c31321488309e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 29 Feb 2024 14:33:55 +0500 Subject: [PATCH 104/269] Fix a merge error. --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index aebdfb3e4..2352be0fe 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -135,7 +135,7 @@ class HttpCompressionMiddleware: respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) - kwargs: Dict[str, Any] = {"cls": respcls, "body": decoded_body} + kwargs: Dict[str, Any] = {"body": decoded_body} if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the # responsetypes guessing is reliable From 2169810414a700fcbfe33eafe1e85e46e7f62413 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 29 Feb 2024 06:41:14 -0300 Subject: [PATCH 105/269] fix: Proxy tests don't use custom certificate authority --- tests/test_proxy_connect.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 46d42e9f6..93f006c76 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -27,7 +27,7 @@ from mitmproxy.tools.main import mitmdump sys.argv[0] = "mitmdump" sys.exit(mitmdump()) """ - cert_path = Path(__file__).parent.resolve() / "keys" / "mitmproxy-ca.pem" + cert_path = Path(__file__).parent.resolve() / "keys" self.proc = Popen( [ sys.executable, @@ -40,8 +40,8 @@ sys.exit(mitmdump()) "0", "--proxyauth", f"{self.auth_user}:{self.auth_pass}", - "--certs", - str(cert_path), + "--set", + f"confdir={cert_path}", "--ssl-insecure", ], stdout=PIPE, From 2bfd9a2257c79ae56955e95b46f2bc7b23e1eabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 29 Feb 2024 11:11:42 +0100 Subject: [PATCH 106/269] bandit: allow-list false positives --- .bandit.yml | 11 +---------- scrapy/commands/bench.py | 6 ++++-- scrapy/commands/edit.py | 2 +- scrapy/commands/genspider.py | 2 +- scrapy/core/downloader/__init__.py | 2 +- scrapy/exporters.py | 4 ++-- scrapy/extensions/httpcache.py | 6 +++--- scrapy/extensions/spiderstate.py | 4 ++-- scrapy/settings/default_settings.py | 2 +- scrapy/shell.py | 2 +- scrapy/squeues.py | 2 +- scrapy/utils/benchserver.py | 2 +- scrapy/utils/engine.py | 2 +- 13 files changed, 20 insertions(+), 27 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 6e8331c0f..4fcd75c57 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,19 +1,10 @@ skips: -- B101 # assert_used -- B105 # hardcoded_password_string -- B301 # pickle -- B307 # eval -- B311 # random +- B101 # assert_used, needed for mypy - B320 # xml_bad_etree - B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180 - B324 # hashlib "Use of weak SHA1 hash for security" - B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 -- B403 # import_pickle -- B404 # import_subprocess -- B406 # import_xml_sax - B410 # import_lxml - B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 - B503 # ssl_with_bad_defaults -- B603 # subprocess_without_shell_equals_true -- B605 # start_process_with_a_shell exclude_dirs: ['tests'] diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index e1ccdc451..aaf5a439f 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,4 +1,4 @@ -import subprocess +import subprocess # nosec import sys import time from urllib.parse import urlencode @@ -29,7 +29,9 @@ class _BenchServer: from scrapy.utils.test import get_testenv pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"] - self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE, env=get_testenv()) + self.proc = subprocess.Popen( + pargs, stdout=subprocess.PIPE, env=get_testenv() + ) # nosec self.proc.stdout.readline() def __exit__(self, exc_type, exc_value, traceback): diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 03a8ed5c7..e85d2c9ec 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -37,4 +37,4 @@ class Command(ScrapyCommand): sfile = sys.modules[spidercls.__module__].__file__ sfile = sfile.replace(".pyc", ".py") - self.exitcode = os.system(f'{editor} "{sfile}"') + self.exitcode = os.system(f'{editor} "{sfile}"') # nosec diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 68cbe8ff6..567ebcdc0 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -113,7 +113,7 @@ class Command(ScrapyCommand): if template_file: self._genspider(module, name, url, opts.template, template_file) if opts.edit: - self.exitcode = os.system(f'scrapy edit "{name}"') + self.exitcode = os.system(f'scrapy edit "{name}"') # nosec def _genspider(self, module, name, url, template_name, template_file): """Generate the spider module, based on the given template""" diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index c84525160..666282856 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -40,7 +40,7 @@ class Slot: def download_delay(self) -> float: if self.randomize_delay: - return random.uniform(0.5 * self.delay, 1.5 * self.delay) + return random.uniform(0.5 * self.delay, 1.5 * self.delay) # nosec return self.delay def close(self) -> None: diff --git a/scrapy/exporters.py b/scrapy/exporters.py index f85f1dad8..79fd4e56f 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -5,10 +5,10 @@ Item Exporters are used to export/serialize items into different formats. import csv import io import marshal -import pickle +import pickle # nosec import pprint from collections.abc import Mapping -from xml.sax.saxutils import XMLGenerator +from xml.sax.saxutils import XMLGenerator # nosec from itemadapter import ItemAdapter, is_item diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 7e4f047a8..335728502 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,6 +1,6 @@ import gzip import logging -import pickle +import pickle # nosec from email.utils import mktime_tz, parsedate_tz from importlib import import_module from pathlib import Path @@ -274,7 +274,7 @@ class DbmCacheStorage: if 0 < self.expiration_secs < time() - float(ts): return # expired - return pickle.loads(db[f"{key}_data"]) + return pickle.loads(db[f"{key}_data"]) # nosec class FilesystemCacheStorage: @@ -352,7 +352,7 @@ class FilesystemCacheStorage: if 0 < self.expiration_secs < time() - mtime: return # expired with self._open(metapath, "rb") as f: - return pickle.load(f) + return pickle.load(f) # nosec def parse_cachecontrol(header): diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 929a3be70..43359401b 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -1,4 +1,4 @@ -import pickle +import pickle # nosec from pathlib import Path from scrapy import signals @@ -31,7 +31,7 @@ class SpiderState: def spider_opened(self, spider): if self.jobdir and Path(self.statefn).exists(): with Path(self.statefn).open("rb") as f: - spider.state = pickle.load(f) + spider.state = pickle.load(f) # nosec else: spider.state = {} diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 49ab1b5ef..2b3d95a0e 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -177,7 +177,7 @@ FILES_STORE_S3_ACL = "private" FILES_STORE_GCS_ACL = "" FTP_USER = "anonymous" -FTP_PASSWORD = "guest" +FTP_PASSWORD = "guest" # nosec FTP_PASSIVE_MODE = True GCS_PROJECT_ID = None diff --git a/scrapy/shell.py b/scrapy/shell.py index 05909977a..63ea33892 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -50,7 +50,7 @@ class Shell: else: self.populate_vars() if self.code: - print(eval(self.code, globals(), self.vars)) + print(eval(self.code, globals(), self.vars)) # nosec else: """ Detect interactive shell setting in scrapy.cfg diff --git a/scrapy/squeues.py b/scrapy/squeues.py index f665ad88c..e20f60f06 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -3,7 +3,7 @@ Scheduler queues """ import marshal -import pickle +import pickle # nosec from os import PathLike from pathlib import Path from typing import Union diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 38884a9f0..f6f704d4b 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -14,7 +14,7 @@ class Root(Resource): def render(self, request): total = _getarg(request, b"total", 100, int) show = _getarg(request, b"show", 10, int) - nlist = [random.randint(1, total) for _ in range(show)] + nlist = [random.randint(1, total) for _ in range(show)] # nosec request.write(b"") args = request.args.copy() for nl in nlist: diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index a5f2a8c6e..0b2722663 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -30,7 +30,7 @@ def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: checks: List[Tuple[str, Any]] = [] for test in tests: try: - checks += [(test, eval(test))] + checks += [(test, eval(test))] # nosec except Exception as e: checks += [(test, f"{type(e).__name__} (exception)")] From 31cbbb57584fe2a7c42d30acf2aa4707039457b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 29 Feb 2024 11:31:39 +0100 Subject: [PATCH 107/269] bandit: ignore md5 usage for download slot names --- scrapy/pqueues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index b62d2fe58..593667f1f 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -20,7 +20,7 @@ def _path_safe(text): pathable_slot = "".join([c if c.isalnum() or c in "-._" else "_" for c in text]) # as we replace some letters we can get collision for different slots # add we add unique part - unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() + unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # nosec return "-".join([pathable_slot, unique_slot]) From 032e6a091a27b406aa48293f752d4782f8cac159 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 29 Feb 2024 16:24:52 +0500 Subject: [PATCH 108/269] Reformat the new changes with new black. --- scrapy/http/request/json_request.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 5c09835e4..59b11c692 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -53,12 +53,10 @@ class JsonRequest(Request): @overload def replace( self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any - ) -> RequestTypeVar: - ... + ) -> RequestTypeVar: ... @overload - def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: - ... + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any From 1311e7db05204fe2cae7d1c5caf8b0ffe9371cd0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 29 Feb 2024 16:31:06 +0500 Subject: [PATCH 109/269] Regenerate the expired mitmproxy-ca.pem. --- tests/keys/mitmproxy-ca.pem | 93 ++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 48 deletions(-) diff --git a/tests/keys/mitmproxy-ca.pem b/tests/keys/mitmproxy-ca.pem index cdef75f99..61a690cc8 100644 --- a/tests/keys/mitmproxy-ca.pem +++ b/tests/keys/mitmproxy-ca.pem @@ -1,50 +1,47 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCYp6U4G9YWITYB -/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7ZDiT -NUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMgz1BJ -u6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniIggUH -JrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE6HFB -eIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/0zls -90iyQ3E/AgMBAAECggEBAJA1dyAdM85uC04vKVNUJM1GDp0xS+0syBReJaKRI3nJ -epoCj+RqxGag1pdaYLI0G84NTPqECz9LOyLdqpPgEfKRIxWlf9oWmSnfnXskArd8 -VfVcWYl6tEPv1TToTZIBmCbYLBFVbLxG/GrbK6uokdhUsqbdXwEKok2IEaSTRlDn -v8BVXte00d9VEKKpmI6EY3f45uPQPHuJNcitP2HGW1mT/C6XoZR6wj+VvoRgUGQT -I7PuktbYpQlLV+oX0uZz9frPGhjydUq0Jti5v3QAJEb+7D0cKrkZW+7fYDx4YkRU -oDiuWEyO2kfpff52Qxs+xUXMiAyw6/8+TamKoAi1TIECgYEAyAzoztW6W4CjL2au -/hN5VmbAvuBxq1m1G5KgXM1myX9V2CgH6OKwzJQNSCEfKMNOjqxB99T7C3tMCjgG -gmbUzylTeciQFF+crrl2Rn/6qZS9dCo1hagb3K5eXMhLXoP425Y4sypNPPqULhPn -YrUDFNAf89rRLqP1KMPLZ+uO7EECgYEAw1lWPxGV+X85iQxYN9xoX85htfJSBXTf -dLirQ4bkykOxSA6ZzFuhDO/G373Q1rze4tmEO790uOCeaiXGgeWC1A+2PMO957i5 -9FqhDIkmerfdIttdEUMM9rQwuTcLnixGZkT5GHDzjtNinaIVB+pv7twRAESqN9dC -QXh7IF7g/X8CgYBMhQOX+hCqZ24D95cAAJrs/ajEWj2geVPZFCDa3oZulJJVeBpu -bieKWScra9/rS6mE0Ub6cTEFl0fisMNspcDI7NnNP3Y9FMVt3+rp1JIgw5AkGvEW -CtN9egUGIGcT5A8Qj0lo3slkhcSgS2S6UNq431MZh51z5askyJ/JREULAQKBgFrR -OatwfYzUfOcd+hVePpfr1rlDwqYOw6P8BoMKP2tZNR4Oy6maH7Fn98kk8eYjQGuu -PC+avqUEqCEpFrRlAwGbnFl7ltoXozvatmyhhmYe/Iur+ASCa5B2DQDOenQ6mTAK -eNPIDzMjSwGFzMk1UHx3it/ZDFmRlZfibzuJYIf5AoGBAIaPHk4qadK/XpcD4Wwx -BOsDEIz27DGWdwWfd5r3EcV4zX/wNzH0G1Z8eydNjUqKzufMZgFwpcTu0Evesl1/ -B8kC8sLHxQoG5SvBu4dBxMwKIU9O9uFnX5SUYZUDpCtUYyZ+GtGom41Jwg5ENrwy -HzPh2taMnCA0h1fNLFFBkw88 ------END PRIVATE KEY----- +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAuq7ujTpHoSwQn9/hFJT837jU/T7xLuyXjkAEfL7uVDuPWSdF +AJy+QJsuL6INMKMlxLUb1RRzxQgAmtYN1dIEbTPplffCNbfYm3FXg0mZlxg1UBg9 +rE1bPwuz/B+M76S35EIiKQlpaCFErLQi5oyhw9FIBvYLZxxgfeDfDPiXQlBvtHix +n9TFqNoLNZkAX+auh2Wj2SSjM74pBQWsuVZLkF5CAwYuMQkpEplCV/QHNX3ZeNdQ +YNFvpA7CxENa3sTZyHpeTmWoOdJXgJFveAWL4ZhSvkSd0HDuPPJp4JckETNESa9B +qOwFfj36SM+5dRiCwiuzwAQ+oaFjisMXuRbVFQIDAQABAoIBABqFaJmCupNgnboA +xcq1QdmMuiGCNCRs8zj/ykNoopYv4fUR+aEVI4gtI5obxRDwVJjF+/7BCZNnyCI3 +H78NN5jGA7zM9nfINwsaRor9xUasZ0KKNxTH5pslz/uVBeIzvfY9GPpIfoOPGmEI +tF6Zgw+9JyTqBoOvCdxIOpfupxqB8TQ0z4UbFUuBiEkGuJ+o8C1rX8Wze0JUl0qG +BOwhQtaCn/yrm+dTXZ5XaelJY5mcwgFy22Jiynmm6TbLhyZlACd2Q/MGak7o1TJL +QgvvGMlcVrK7MZ3TJN+wzwWfwAAjXnT3Xvd5pD5yunZJoNe8YyFOCMlh0swNG5Zt +0tGeX78CgYEA+m6gYGKTNWFnqlqMZRfGTqiqVZeVQKftcLdA1dkscnffRP9bvKOW +9TbgzoGHiyZnjZBDFTUuy67El8RXIMsxYy3GYuGRdUSLS63Fr2af4pBQIYvW6OmG +UZlcAP6ZAhUzn409XGlXaac3F30hFeKdC62+V5ZMnfPlVhHRCoKaaqsCgYEAvtV4 +FuU5sFKyhKPPV3rzaNZtL0swvtBIuODH1oAWhPNySQvCu+45W0EOOAPPpsYP4wGX +G+otOSp4RLdlVXNhkh1rpJzeK77KZ5ZY+ShkuHD+uL/iRARwl/gh2Ve1aqUrm1LE +9ldchmQGvLalN9HalzeW//xHA3X9SF4Vo16Dvz8CgYABeZlUOABp9hLoO/RLvCIc +4H1wV543bUXGvi2RlN/gJLiZ7W8a41PGSfZ1AOpNdYJyoQDkJRYLeRILWsqwlMHL +tb9PYci7ihXP8kwRxmb2rKbsK6iuYoG6BU83akh4bKuLKwfLfYtYQfXfG4uQV29Z +XEKcvXPiEkethBlZGH/UVQKBgCa9Pvum3OcmYob6mgSwOOl3XgLTyLlzns+pEehB +aFDk+rZJZOaxnYMg2boVS/oXCvKSSBKqnzOTo4aPlEqceZonzspD7fYDbSNKKhWq +VYf7qDno+g3EuPagsH5mh7V2gjutub4oTegaNiPpD/Ec8Lrx1f1xQRk6wogGUW4w +qZ4RAoGBAPg1LezV8mlesF5mhj+KubYP4l1Zf9geAeQprjDbFsA0BEAS2KsWgmwR +Ye1fmek7jDjCPLQ4Amq030mLJuQGEM3cZPqjKX2sBZ8fQcgw7pWJWMvKMTBA4Aah +zQx1KXwHJANMWq/0QSFDq/LGJ2OYMlV2F0tH3P5Kp7ZASTyc78ux +-----END RSA PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIDoTCCAomgAwIBAgIGDodLQx9+MA0GCSqGSIb3DQEBCwUAMCgxEjAQBgNVBAMM -CW1pdG1wcm94eTESMBAGA1UECgwJbWl0bXByb3h5MB4XDTIwMDgxMjE3MDMyNloX -DTIzMDgxNDE3MDMyNlowKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAlt -aXRtcHJveHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCYp6U4G9YW -ITYB/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7 -ZDiTNUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMg -z1BJu6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniI -ggUHJrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE -6HFBeIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/ -0zls90iyQ3E/AgMBAAGjgdAwgc0wDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhC -AQEEBAMCAgQweAYDVR0lBHEwbwYIKwYBBQUHAwEGCCsGAQUFBwMCBggrBgEFBQcD -BAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQBgjcCARYGCisGAQQBgjcKAwEG -CisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG+EIEATAOBgNVHQ8BAf8EBAMC -AQYwHQYDVR0OBBYEFBCsLPpFz3l9rOOfGmfs+VRc3jhJMA0GCSqGSIb3DQEBCwUA -A4IBAQADTpA15na6U5qqDCe0rr39fkS1/dY804Xnz7g/L3AsxPE1KOMijuJa8sKd -kKwba1173FwMupfK39zY8jUxL8Qprdi92RO6CpoFUsL/icpA///lYhzUSqt32qwe -gRNW3mtYBimOk6KH1NOfQnJolWpJh+g1OEsitQKEeKwIn5Hz+8/yS5tbwLgdnMlY -1/it1H70JSdE7nfJueqN4cFfBsm6XaHZzacJJmN7WP88fd+zztnSQsBFbLlnjnqj -envCDIwCrMywKNMqEBMwmBEGSAF47fVNYj6KzDAtMvBdDkYaHWpBf4tnFfk6v0wj -wiKjdLjCmJgjGAQjRw5VYJ8JI0XO +MIIDNTCCAh2gAwIBAgIUcGDiCmOuhfxMGFS/otcGGFkOSAEwDQYJKoZIhvcNAQEL +BQAwKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAltaXRtcHJveHkwHhcN +MjQwMjI3MTMwNTQ4WhcNMzQwMjI2MTMwNTQ4WjAoMRIwEAYDVQQDDAltaXRtcHJv +eHkxEjAQBgNVBAoMCW1pdG1wcm94eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBALqu7o06R6EsEJ/f4RSU/N+41P0+8S7sl45ABHy+7lQ7j1knRQCcvkCb +Li+iDTCjJcS1G9UUc8UIAJrWDdXSBG0z6ZX3wjW32JtxV4NJmZcYNVAYPaxNWz8L +s/wfjO+kt+RCIikJaWghRKy0IuaMocPRSAb2C2ccYH3g3wz4l0JQb7R4sZ/Uxaja +CzWZAF/mrodlo9kkozO+KQUFrLlWS5BeQgMGLjEJKRKZQlf0BzV92XjXUGDRb6QO +wsRDWt7E2ch6Xk5lqDnSV4CRb3gFi+GYUr5EndBw7jzyaeCXJBEzREmvQajsBX49 ++kjPuXUYgsIrs8AEPqGhY4rDF7kW1RUCAwEAAaNXMFUwDwYDVR0TAQH/BAUwAwEB +/zATBgNVHSUEDDAKBggrBgEFBQcDATAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FOjFT0G7itqsrCij2InhRSfB0sEkMA0GCSqGSIb3DQEBCwUAA4IBAQCVMa5/xlH4 +GUbrWNMdxr9LL7Dh+vK0wYCfAsc/kO2zCq8iVt/MaqVLel/bKcQhvE5RZHvyep13 +x7378OfCqqHkDDDNroWIvij84ZtMUaM53tF13G/ZGOlNsoLNynWs9IVVvqGKsH7o +/buJ1RNArI/0irF0UD7qrMmo1p6SYanZhqdh2PphNy9NS3FsfrfnuWvf+/TRp9Ts +L8058B0p/LIL0OB5trYFircC3iKSOuRl0ERD2ufgSqsSVEYm1mc6UIxv+d1iFD+Q +8CRUF88icQXrec1TCbhh0CfdDxz+FYSTnW0DR0L75coa/CBmRxAjnrkLoXRr3Y1d +sTjU4zDdBcBw -----END CERTIFICATE----- From 4cd94aa668c60f92b1d9f4e5cf27752e1fe9c9cd Mon Sep 17 00:00:00 2001 From: "Yuri H. Galvao" Date: Fri, 1 Mar 2024 04:07:38 -0600 Subject: [PATCH 110/269] Restore brotlipy support (#6261) --- scrapy/utils/_compression.py | 32 +++++++++++++++++++++++++++++++- tox.ini | 1 + 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 5610595d3..14531df3f 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -1,10 +1,40 @@ import zlib from io import BytesIO +from warnings import warn + +from scrapy.exceptions import ScrapyDeprecationWarning try: import brotli except ImportError: pass +else: + try: + brotli.Decompressor.process + except AttributeError: + + warn( + ( + "You have brotlipy installed, and Scrapy will use it, but " + "Scrapy support for brotlipy is deprecated and will stop " + "working in a future version of Scrapy. brotlipy itself is " + "deprecated, it has been superseded by brotlicffi (not " + "currently supported by Scrapy). Please, uninstall brotlipy " + "and install brotli instead. brotlipy has the same import " + "name as brotli, so keeping both installed is strongly " + "discouraged." + ), + ScrapyDeprecationWarning, + ) + + def _brotli_decompress(decompressor, data): + return decompressor.decompress(data) + + else: + + def _brotli_decompress(decompressor, data): + return decompressor.process(data) + try: import zstandard @@ -61,7 +91,7 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: decompressed_size = 0 while output_chunk: input_chunk = input_stream.read(_CHUNK_SIZE) - output_chunk = decompressor.process(input_chunk) + output_chunk = _brotli_decompress(decompressor, input_chunk) decompressed_size += len(output_chunk) if max_size and decompressed_size > max_size: raise _DecompressionMaxSizeExceeded( diff --git a/tox.ini b/tox.ini index 4ed9b3bd7..237aa489c 100644 --- a/tox.ini +++ b/tox.ini @@ -135,6 +135,7 @@ deps = google-cloud-storage==1.29.0 Pillow==7.1.0 robotexclusionrulesparser==1.6.2 + brotlipy install_command = {[pinned]install_command} setenv = {[pinned]setenv} From aa1bf6907964f0281264052cabc28197c5d28107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 1 Mar 2024 12:48:00 +0100 Subject: [PATCH 111/269] Mark hashlib usages as not intended for security (#6264) --- .bandit.yml | 1 - scrapy/pipelines/files.py | 28 ++++++++++++++++++++++------ scrapy/pipelines/images.py | 9 ++++----- scrapy/utils/misc.py | 10 +++++++++- scrapy/utils/request.py | 2 +- 5 files changed, 36 insertions(+), 14 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 4fcd75c57..db2fbb84c 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -2,7 +2,6 @@ skips: - B101 # assert_used, needed for mypy - B320 # xml_bad_etree - B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180 -- B324 # hashlib "Use of weak SHA1 hash for security" - B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 - B410 # import_lxml - B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 1d7625299..d04218089 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -16,7 +16,7 @@ from ftplib import FTP from io import BytesIO from os import PathLike from pathlib import Path -from typing import DefaultDict, Optional, Set, Union +from typing import IO, DefaultDict, Optional, Set, Union from urllib.parse import urlparse from itemadapter import ItemAdapter @@ -31,7 +31,6 @@ from scrapy.utils.boto import is_botocore_available from scrapy.utils.datatypes import CaseInsensitiveDict from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info -from scrapy.utils.misc import md5sum from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str @@ -42,6 +41,23 @@ def _to_string(path: Union[str, PathLike]) -> str: return str(path) # convert a Path object to string +def _md5sum(file: IO) -> str: + """Calculate the md5 checksum of a file-like object without reading its + whole content in memory. + + >>> from io import BytesIO + >>> _md5sum(BytesIO(b'file content to hash')) + '784406af91dd5a54fbb9c84c2236595a' + """ + m = hashlib.md5() # nosec + while True: + d = file.read(8096) + if not d: + break + m.update(d) + return m.hexdigest() + + class FileException(Exception): """General media error exception""" @@ -70,7 +86,7 @@ class FSFilesStore: return {} with absolute_path.open("rb") as f: - checksum = md5sum(f) + checksum = _md5sum(f) return {"last_modified": last_modified, "checksum": checksum} @@ -299,7 +315,7 @@ class FTPFilesStore: ftp.set_pasv(False) file_path = f"{self.basedir}/{path}" last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip()) - m = hashlib.md5() + m = hashlib.md5() # nosec ftp.retrbinary(f"RETR {file_path}", m.update) return {"last_modified": last_modified, "checksum": m.hexdigest()} # The file doesn't exist @@ -531,7 +547,7 @@ class FilesPipeline(MediaPipeline): def file_downloaded(self, response, request, info, *, item=None): path = self.file_path(request, response=response, info=info, item=item) buf = BytesIO(response.body) - checksum = md5sum(buf) + checksum = _md5sum(buf) buf.seek(0) self.store.persist_file(path, buf, info) return checksum @@ -542,7 +558,7 @@ class FilesPipeline(MediaPipeline): return item def file_path(self, request, response=None, info=None, *, item=None): - media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() + media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec media_ext = Path(request.url).suffix # Handles empty and wild extensions by trying to guess the # mime type then extension or default to empty string otherwise diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 8169583f8..137aa7a9a 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -17,11 +17,10 @@ from itemadapter import ItemAdapter from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.files import FileException, FilesPipeline +from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum # TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings -from scrapy.utils.misc import md5sum from scrapy.utils.python import get_func_args, to_bytes @@ -128,7 +127,7 @@ class ImagesPipeline(FilesPipeline): for path, image, buf in self.get_images(response, request, info, item=item): if checksum is None: buf.seek(0) - checksum = md5sum(buf) + checksum = _md5sum(buf) width, height = image.size self.store.persist_file( path, @@ -228,9 +227,9 @@ class ImagesPipeline(FilesPipeline): return item def file_path(self, request, response=None, info=None, *, item=None): - image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() + image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec return f"full/{image_guid}.jpg" def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None): - thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() + thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec return f"thumbs/{thumb_id}/{thumb_guid}.jpg" diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 7b43760a8..7f83d06fb 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -113,7 +113,15 @@ def md5sum(file: IO) -> str: >>> md5sum(BytesIO(b'file content to hash')) '784406af91dd5a54fbb9c84c2236595a' """ - m = hashlib.md5() + warnings.warn( + ( + "The scrapy.utils.misc.md5sum function is deprecated, and will be " + "removed in a future version of Scrapy." + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) + m = hashlib.md5() # nosec while True: d = file.read(8096) if not d: diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index e99d1eeb5..1f07d58eb 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -111,7 +111,7 @@ def fingerprint( "headers": headers, } fingerprint_json = json.dumps(fingerprint_data, sort_keys=True) - cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest() + cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest() # nosec return cache[cache_key] From bf149356fc6e519e92fb55150a60b40b14e45ae8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 1 Mar 2024 16:02:03 +0100 Subject: [PATCH 112/269] Bandit: allow-list lxml usages (#6265) --- .bandit.yml | 2 -- scrapy/http/request/form.py | 17 ++++++----------- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/selector/unified.py | 3 ++- scrapy/utils/_compression.py | 1 - scrapy/utils/iterators.py | 5 +++-- scrapy/utils/sitemap.py | 4 ++-- scrapy/utils/versions.py | 2 +- 8 files changed, 15 insertions(+), 21 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index db2fbb84c..b7f1817e0 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,9 +1,7 @@ skips: - B101 # assert_used, needed for mypy -- B320 # xml_bad_etree - B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180 - B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 -- B410 # import_lxml - B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 - B503 # ssl_with_bad_defaults exclude_dirs: ['tests'] diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 0f80a0ab7..3206d79cd 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -10,21 +10,16 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit -from lxml.html import ( - FormElement, - HTMLParser, - InputElement, - MultipleSelectOptions, - SelectElement, - TextareaElement, -) -from parsel.selector import create_root_node +from lxml.html import FormElement # nosec +from lxml.html import InputElement # nosec +from lxml.html import MultipleSelectOptions # nosec +from lxml.html import SelectElement # nosec +from lxml.html import TextareaElement # nosec from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request from scrapy.http.response.text import TextResponse from scrapy.utils.python import is_listlike, to_bytes -from scrapy.utils.response import get_base_url if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -120,7 +115,7 @@ def _get_form( formxpath: Optional[str], ) -> FormElement: """Find the wanted form element within the given response.""" - root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response)) + root = response.selector.root forms = root.xpath("//form") if not forms: raise ValueError(f"No element found in {response}") diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index d76db20ba..55bc0fc43 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -7,7 +7,7 @@ import operator from functools import partial from urllib.parse import urljoin, urlparse -from lxml import etree +from lxml import etree # nosec from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 75d5e9fbd..aa9581fcd 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -8,6 +8,7 @@ from parsel import Selector as _ParselSelector from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.utils.python import to_bytes +from scrapy.utils.response import get_base_url from scrapy.utils.trackref import object_ref __all__ = ["Selector", "SelectorList"] @@ -88,7 +89,7 @@ class Selector(_ParselSelector, object_ref): if response is not None: text = response.text - kwargs.setdefault("base_url", response.url) + kwargs.setdefault("base_url", get_base_url(response)) self.response = response diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 14531df3f..7c40d0a02 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -12,7 +12,6 @@ else: try: brotli.Decompressor.process except AttributeError: - warn( ( "You have brotlipy installed, and Scrapy will use it, but " diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 93a2ba7a1..49493e9c6 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -18,7 +18,7 @@ from typing import ( ) from warnings import warn -from lxml import etree +from lxml import etree # nosec from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse @@ -26,7 +26,7 @@ from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode if TYPE_CHECKING: - from lxml._types import SupportsReadClose + from lxml._types import SupportsReadClose # nosec logger = logging.getLogger(__name__) @@ -101,6 +101,7 @@ def xmliter_lxml( cast("SupportsReadClose[bytes]", reader), encoding=reader.encoding, events=("end", "start-ns"), + resolve_entities=False, huge_tree=True, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 8bf941eb2..7dcee3a2f 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -8,7 +8,7 @@ SitemapSpider, its API is subject to change without notice. from typing import Any, Dict, Generator, Iterator, Optional from urllib.parse import urljoin -import lxml.etree +import lxml.etree # nosec class Sitemap: @@ -19,7 +19,7 @@ class Sitemap: xmlp = lxml.etree.XMLParser( recover=True, remove_comments=True, resolve_entities=False ) - self._root = lxml.etree.fromstring(xmltext, parser=xmlp) + self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # nosec rt = self._root.tag self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index 9b637bdb0..42e5e9be4 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -4,7 +4,7 @@ from typing import List, Tuple import cryptography import cssselect -import lxml.etree +import lxml.etree # nosec import parsel import twisted import w3lib From 6b75d8f3b3107957f3ae381ce3882ac3778f34c4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 5 Mar 2024 22:23:48 +0500 Subject: [PATCH 113/269] Bump pytest-mypy-testing. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c43bd73d1..7192b6808 100644 --- a/tox.ini +++ b/tox.ini @@ -47,7 +47,7 @@ basepython = python3.8 deps = -rtests/requirements.txt {[testenv:typing]deps} - pytest-mypy-testing==0.1.1 + pytest-mypy-testing==0.1.3 commands = pytest {posargs: tests_typing} From cab1016bb6f719b15043f65502c63fbaa191df36 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 20:24:17 -0300 Subject: [PATCH 114/269] Add brotlicffi support --- scrapy/utils/_compression.py | 2 ++ ...st_downloadermiddleware_httpcompression.py | 35 +++++++++++++++---- tox.ini | 1 + 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 7c40d0a02..7896f4c01 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -6,6 +6,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning try: import brotli +except ImportError: + import brotlicffi as brotli except ImportError: pass else: diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index ae5569d0a..7c36f748e 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -130,7 +130,10 @@ class HttpCompressionTest(TestCase): def test_process_response_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") response = self._getresponse("br") @@ -448,7 +451,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_setting_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_setting("br") @@ -486,7 +492,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_spider_attr_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_spider_attr("br") @@ -522,7 +531,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_request_meta_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_request_meta("br") @@ -568,7 +580,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_setting_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_setting("br") @@ -616,7 +631,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_spider_attr_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_spider_attr("br") @@ -662,7 +680,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_request_meta_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_request_meta("br") diff --git a/tox.ini b/tox.ini index 237aa489c..6b804b78c 100644 --- a/tox.ini +++ b/tox.ini @@ -162,6 +162,7 @@ basepython = {[testenv:pypy3]basepython} deps = {[pinned]deps} PyPyDispatcher==2.1.0 + brotlicffi commands = pytest --durations=10 scrapy tests install_command = {[pinned]install_command} From 3421823dce94a693ee86915110d899d8da6f3e9f Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 20:26:23 -0300 Subject: [PATCH 115/269] Nested try-except block --- scrapy/utils/_compression.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 7896f4c01..477573588 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -5,9 +5,10 @@ from warnings import warn from scrapy.exceptions import ScrapyDeprecationWarning try: - import brotli -except ImportError: - import brotlicffi as brotli + try: + import brotli + except ImportError: + import brotlicffi as brotli except ImportError: pass else: From a52429ae08ec15d70f7f3e2079d32933bb639d6b Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 21:12:25 -0300 Subject: [PATCH 116/269] Update disclaimer --- scrapy/utils/_compression.py | 4 ++-- tox.ini | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 477573588..4b3fd342d 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -20,8 +20,8 @@ else: "You have brotlipy installed, and Scrapy will use it, but " "Scrapy support for brotlipy is deprecated and will stop " "working in a future version of Scrapy. brotlipy itself is " - "deprecated, it has been superseded by brotlicffi (not " - "currently supported by Scrapy). Please, uninstall brotlipy " + "deprecated, it has been superseded by brotlicffi " + "Please, uninstall brotlipy " "and install brotli instead. brotlipy has the same import " "name as brotli, so keeping both installed is strongly " "discouraged." diff --git a/tox.ini b/tox.ini index 6b804b78c..9cf3c92ad 100644 --- a/tox.ini +++ b/tox.ini @@ -124,7 +124,7 @@ deps = robotexclusionrulesparser Pillow Twisted[http2] - brotli + brotli; implementation_name != 'pypy' zstandard [testenv:extra-deps-pinned] From 16864ea602ebc3d1a764aaf6c101a5f20ff57bee Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 21:18:24 -0300 Subject: [PATCH 117/269] Remove PyPy condition --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9cf3c92ad..6b804b78c 100644 --- a/tox.ini +++ b/tox.ini @@ -124,7 +124,7 @@ deps = robotexclusionrulesparser Pillow Twisted[http2] - brotli; implementation_name != 'pypy' + brotli zstandard [testenv:extra-deps-pinned] From 532cd2eabd8b280e64a1087c49b8f5eb5f05530f Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 21:30:20 -0300 Subject: [PATCH 118/269] Use brotlicffi for PyPy --- tests/requirements.txt | 3 +-- tox.ini | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 5b75674f5..ca5f6ddbd 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -11,8 +11,7 @@ uvloop; platform_system != "Windows" bpython # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests -# 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072 -brotli==1.0.9; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests +brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" diff --git a/tox.ini b/tox.ini index 6b804b78c..237aa489c 100644 --- a/tox.ini +++ b/tox.ini @@ -162,7 +162,6 @@ basepython = {[testenv:pypy3]basepython} deps = {[pinned]deps} PyPyDispatcher==2.1.0 - brotlicffi commands = pytest --durations=10 scrapy tests install_command = {[pinned]install_command} From 7f1fbdba3cc6f118cbf11285ed26e488f854aed1 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 22:11:11 -0300 Subject: [PATCH 119/269] Check brotlicffi for ACCEPTED_ENCODINGS --- scrapy/downloadermiddlewares/httpcompression.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index aa3abe853..0e5e215ac 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -29,7 +29,10 @@ logger = getLogger(__name__) ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"] try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: pass else: From 7be919138d84ec00feb80a78e13721dff998c10c Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 6 Mar 2024 05:49:31 -0300 Subject: [PATCH 120/269] Update scrapy/utils/_compression.py Co-authored-by: Andrey Rakhmatullin --- scrapy/utils/_compression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 4b3fd342d..349fd9ac0 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -20,7 +20,7 @@ else: "You have brotlipy installed, and Scrapy will use it, but " "Scrapy support for brotlipy is deprecated and will stop " "working in a future version of Scrapy. brotlipy itself is " - "deprecated, it has been superseded by brotlicffi " + "deprecated, it has been superseded by brotlicffi. " "Please, uninstall brotlipy " "and install brotli instead. brotlipy has the same import " "name as brotli, so keeping both installed is strongly " From 99f7165c63a8a2dba72090f65ba1093d476d669a Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 6 Mar 2024 09:02:01 -0300 Subject: [PATCH 121/269] Update scrapy/utils/_compression.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/utils/_compression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 349fd9ac0..84c255c28 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -22,7 +22,7 @@ else: "working in a future version of Scrapy. brotlipy itself is " "deprecated, it has been superseded by brotlicffi. " "Please, uninstall brotlipy " - "and install brotli instead. brotlipy has the same import " + "and install brotli or brotlicffi instead. brotlipy has the same import " "name as brotli, so keeping both installed is strongly " "discouraged." ), From 6ecc9e0a34be6317d1b35a3ca1fc13cb98129732 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 6 Mar 2024 17:21:08 +0500 Subject: [PATCH 122/269] Add typing for scrapy/commands (#6268) --- scrapy/commands/__init__.py | 48 +++++++----- scrapy/commands/bench.py | 18 +++-- scrapy/commands/check.py | 18 +++-- scrapy/commands/crawl.py | 14 +++- scrapy/commands/edit.py | 14 ++-- scrapy/commands/fetch.py | 23 +++--- scrapy/commands/genspider.py | 31 +++++--- scrapy/commands/list.py | 8 +- scrapy/commands/parse.py | 128 +++++++++++++++++++++++--------- scrapy/commands/runspider.py | 12 +-- scrapy/commands/settings.py | 13 ++-- scrapy/commands/shell.py | 19 ++--- scrapy/commands/startproject.py | 22 +++--- scrapy/commands/version.py | 13 ++-- scrapy/commands/view.py | 15 +++- scrapy/shell.py | 70 +++++++++++------ scrapy/utils/console.py | 48 ++++++++---- scrapy/utils/response.py | 26 +++---- 18 files changed, 355 insertions(+), 185 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 27993710e..9fe803d3c 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -3,61 +3,62 @@ Base class for Scrapy commands """ import argparse +import builtins import os from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Iterable, List, Optional from twisted.python import failure -from scrapy.crawler import CrawlerProcess +from scrapy.crawler import Crawler, CrawlerProcess from scrapy.exceptions import UsageError from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli class ScrapyCommand: - requires_project = False + requires_project: bool = False crawler_process: Optional[CrawlerProcess] = None # default settings to be used for this command instead of global defaults default_settings: Dict[str, Any] = {} - exitcode = 0 + exitcode: int = 0 def __init__(self) -> None: self.settings: Any = None # set in scrapy.cmdline - def set_crawler(self, crawler): + def set_crawler(self, crawler: Crawler) -> None: if hasattr(self, "_crawler"): raise RuntimeError("crawler already set") - self._crawler = crawler + self._crawler: Crawler = crawler - def syntax(self): + def syntax(self) -> str: """ Command syntax (preferably one-line). Do not include command name. """ return "" - def short_desc(self): + def short_desc(self) -> str: """ A short description of the command """ return "" - def long_desc(self): + def long_desc(self) -> str: """A long description of the command. Return short description when not available. It cannot contain newlines since contents will be formatted by optparser which removes newlines and wraps text. """ return self.short_desc() - def help(self): + def help(self) -> str: """An extensive help for the command. It will be shown when using the "help" command. It can contain newlines since no post-formatting will be applied to its contents. """ return self.long_desc() - def add_options(self, parser): + def add_options(self, parser: argparse.ArgumentParser) -> None: """ Populate option parse with options available for this command """ @@ -92,7 +93,7 @@ class ScrapyCommand: ) group.add_argument("--pdb", action="store_true", help="enable pdb on failure") - def process_options(self, args, opts): + def process_options(self, args: List[str], opts: argparse.Namespace) -> None: try: self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline") except ValueError: @@ -129,8 +130,8 @@ class BaseRunSpiderCommand(ScrapyCommand): Common class used to share functionality between the crawl, parse and runspider commands """ - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-a", dest="spargs", @@ -162,8 +163,8 @@ class BaseRunSpiderCommand(ScrapyCommand): help="format to use for dumping items", ) - def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) + def process_options(self, args: List[str], opts: argparse.Namespace) -> None: + super().process_options(args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: @@ -183,7 +184,13 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): Help Formatter for scrapy command line help messages. """ - def __init__(self, prog, indent_increment=2, max_help_position=24, width=None): + def __init__( + self, + prog: str, + indent_increment: int = 2, + max_help_position: int = 24, + width: Optional[int] = None, + ): super().__init__( prog, indent_increment=indent_increment, @@ -191,11 +198,12 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): width=width, ) - def _join_parts(self, part_strings): - parts = self.format_part_strings(part_strings) + def _join_parts(self, part_strings: Iterable[str]) -> str: + # scrapy.commands.list shadows builtins.list + parts = self.format_part_strings(builtins.list(part_strings)) return super()._join_parts(parts) - def format_part_strings(self, part_strings): + def format_part_strings(self, part_strings: List[str]) -> List[str]: """ Underline and title case command line help message headers. """ diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index aaf5a439f..2e6bb5d86 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,10 +1,14 @@ +import argparse import subprocess # nosec import sys import time +from typing import Any, Iterable, List from urllib.parse import urlencode import scrapy +from scrapy import Request from scrapy.commands import ScrapyCommand +from scrapy.http import Response from scrapy.linkextractors import LinkExtractor @@ -15,26 +19,28 @@ class Command(ScrapyCommand): "CLOSESPIDER_TIMEOUT": 10, } - def short_desc(self): + def short_desc(self) -> str: return "Run quick benchmark test" - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: with _BenchServer(): + assert self.crawler_process self.crawler_process.crawl(_BenchSpider, total=100000) self.crawler_process.start() class _BenchServer: - def __enter__(self): + def __enter__(self) -> None: from scrapy.utils.test import get_testenv pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"] self.proc = subprocess.Popen( pargs, stdout=subprocess.PIPE, env=get_testenv() ) # nosec + assert self.proc.stdout self.proc.stdout.readline() - def __exit__(self, exc_type, exc_value, traceback): + def __exit__(self, exc_type, exc_value, traceback) -> None: self.proc.kill() self.proc.wait() time.sleep(0.2) @@ -49,11 +55,11 @@ class _BenchSpider(scrapy.Spider): baseurl = "http://localhost:8998" link_extractor = LinkExtractor() - def start_requests(self): + def start_requests(self) -> Iterable[Request]: qargs = {"total": self.total, "show": self.show} url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}" return [scrapy.Request(url, dont_filter=True)] - def parse(self, response): + def parse(self, response: Response) -> Any: # type: ignore[override] for link in self.link_extractor.extract_links(response): yield scrapy.Request(link.url, callback=self.parse) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index de54ca4d3..22c8abf7a 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -1,5 +1,7 @@ +import argparse import time from collections import defaultdict +from typing import List from unittest import TextTestResult as _TextTestResult from unittest import TextTestRunner @@ -10,9 +12,10 @@ from scrapy.utils.misc import load_object, set_environ class TextTestResult(_TextTestResult): - def printSummary(self, start, stop): + def printSummary(self, start: float, stop: float) -> None: write = self.stream.write - writeln = self.stream.writeln + # _WritelnDecorator isn't implemented in typeshed yet + writeln = self.stream.writeln # type: ignore[attr-defined] run = self.testsRun plural = "s" if run != 1 else "" @@ -42,14 +45,14 @@ class Command(ScrapyCommand): requires_project = True default_settings = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Check spider contracts" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-l", "--list", @@ -66,7 +69,7 @@ class Command(ScrapyCommand): help="print contract tests for all spiders", ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: # load contracts contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS")) conman = ContractsManager(load_object(c) for c in contracts) @@ -76,6 +79,7 @@ class Command(ScrapyCommand): # contract requests contract_reqs = defaultdict(list) + assert self.crawler_process spider_loader = self.crawler_process.spider_loader with set_environ(SCRAPY_CHECK="true"): diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 2f0f1c7b9..6e023af81 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,3 +1,8 @@ +import argparse +from typing import List, cast + +from twisted.python.failure import Failure + from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError @@ -5,13 +10,13 @@ from scrapy.exceptions import UsageError class Command(BaseRunSpiderCommand): requires_project = True - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Run a spider" - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if len(args) < 1: raise UsageError() elif len(args) > 1: @@ -20,10 +25,11 @@ class Command(BaseRunSpiderCommand): ) spname = args[0] + assert self.crawler_process crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) if getattr(crawl_defer, "result", None) is not None and issubclass( - crawl_defer.result.type, Exception + cast(Failure, crawl_defer.result).type, Exception ): self.exitcode = 1 else: diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index e85d2c9ec..04012bee8 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -1,5 +1,7 @@ +import argparse import os import sys +from typing import List from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError @@ -9,32 +11,34 @@ class Command(ScrapyCommand): requires_project = True default_settings = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return "" - def short_desc(self): + def short_desc(self) -> str: return "Edit spider" - def long_desc(self): + def long_desc(self) -> str: return ( "Edit a spider using the editor defined in the EDITOR environment" " variable or else the EDITOR setting" ) - def _err(self, msg): + def _err(self, msg: str) -> None: sys.stderr.write(msg + os.linesep) self.exitcode = 1 - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if len(args) != 1: raise UsageError() editor = self.settings["EDITOR"] + assert self.crawler_process try: spidercls = self.crawler_process.spider_loader.load(args[0]) except KeyError: return self._err(f"Spider not found: {args[0]}") sfile = sys.modules[spidercls.__module__].__file__ + assert sfile sfile = sfile.replace(".pyc", ".py") self.exitcode = os.system(f'{editor} "{sfile}"') # nosec diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index cdb7ad4ae..1acf2d26f 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,13 +1,13 @@ import sys -from argparse import Namespace -from typing import List, Type +from argparse import ArgumentParser, Namespace +from typing import Dict, List, Type from w3lib.url import is_url from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import DefaultSpider, spidercls_for_request @@ -15,20 +15,20 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request class Command(ScrapyCommand): requires_project = False - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Fetch a URL using the Scrapy downloader" - def long_desc(self): + def long_desc(self) -> str: return ( "Fetch a URL using the Scrapy downloader and print its content" " to stdout. You may want to use --nolog to disable logging" ) - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: ArgumentParser) -> None: + super().add_options(parser) parser.add_argument("--spider", dest="spider", help="use this spider") parser.add_argument( "--headers", @@ -44,20 +44,21 @@ class Command(ScrapyCommand): help="do not handle HTTP 3xx status codes and print response as-is", ) - def _print_headers(self, headers, prefix): + def _print_headers(self, headers: Dict[bytes, List[bytes]], prefix: bytes) -> None: for key, values in headers.items(): for value in values: self._print_bytes(prefix + b" " + key + b": " + value) - def _print_response(self, response, opts): + def _print_response(self, response: Response, opts: Namespace) -> None: if opts.headers: + assert response.request self._print_headers(response.request.headers, b">") print(">") self._print_headers(response.headers, b"<") else: self._print_bytes(response.body) - def _print_bytes(self, bytes_): + def _print_bytes(self, bytes_: bytes) -> None: sys.stdout.buffer.write(bytes_ + b"\n") def run(self, args: List[str], opts: Namespace) -> None: diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 567ebcdc0..2649fb23d 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -1,9 +1,10 @@ +import argparse import os import shutil import string from importlib import import_module from pathlib import Path -from typing import Optional, cast +from typing import List, Optional, Union, cast from urllib.parse import urlparse import scrapy @@ -12,7 +13,7 @@ from scrapy.exceptions import UsageError from scrapy.utils.template import render_templatefile, string_camelcase -def sanitize_module_name(module_name): +def sanitize_module_name(module_name: str) -> str: """Sanitize the given module name, by replacing dashes and points with underscores and prefixing it with a letter if it doesn't start with one @@ -23,7 +24,7 @@ def sanitize_module_name(module_name): return module_name -def extract_domain(url): +def extract_domain(url: str) -> str: """Extract domain name from URL string""" o = urlparse(url) if o.scheme == "" and o.netloc == "": @@ -31,7 +32,7 @@ def extract_domain(url): return o.netloc -def verify_url_scheme(url): +def verify_url_scheme(url: str) -> str: """Check url for scheme and insert https if none found.""" parsed = urlparse(url) if parsed.scheme == "" and parsed.netloc == "": @@ -43,14 +44,14 @@ class Command(ScrapyCommand): requires_project = False default_settings = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Generate new spider using pre-defined templates" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-l", "--list", @@ -86,7 +87,7 @@ class Command(ScrapyCommand): help="If the spider already exists, overwrite it with the template", ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if opts.list: self._list_templates() return @@ -115,7 +116,14 @@ class Command(ScrapyCommand): if opts.edit: self.exitcode = os.system(f'scrapy edit "{name}"') # nosec - def _genspider(self, module, name, url, template_name, template_file): + def _genspider( + self, + module: str, + name: str, + url: str, + template_name: str, + template_file: Union[str, os.PathLike], + ) -> None: """Generate the spider module, based on the given template""" capitalized_module = "".join(s.capitalize() for s in module.split("_")) domain = extract_domain(url) @@ -130,6 +138,7 @@ class Command(ScrapyCommand): } if self.settings.get("NEWSPIDER_MODULE"): spiders_module = import_module(self.settings["NEWSPIDER_MODULE"]) + assert spiders_module.__file__ spiders_dir = Path(spiders_module.__file__).parent.resolve() else: spiders_module = None @@ -152,7 +161,7 @@ class Command(ScrapyCommand): print('Use "scrapy genspider --list" to see all available templates.') return None - def _list_templates(self): + def _list_templates(self) -> None: print("Available templates:") for file in sorted(Path(self.templates_dir).iterdir()): if file.suffix == ".tmpl": diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 2f5032360..dcc51a694 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,3 +1,6 @@ +import argparse +from typing import List + from scrapy.commands import ScrapyCommand @@ -5,9 +8,10 @@ class Command(ScrapyCommand): requires_project = True default_settings = {"LOG_ENABLED": False} - def short_desc(self): + def short_desc(self) -> str: return "List available spiders" - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: + assert self.crawler_process for s in sorted(self.crawler_process.spider_loader.list()): print(s) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index c9f8586d3..2453c0d39 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,16 +1,32 @@ +import argparse import functools import inspect import json import logging -from typing import Dict +from types import CoroutineType +from typing import ( + Any, + AsyncGenerator, + Callable, + Dict, + Iterable, + List, + Optional, + Tuple, + TypeVar, + Union, + overload, +) from itemadapter import ItemAdapter, is_item -from twisted.internet.defer import maybeDeferred +from twisted.internet.defer import Deferred, maybeDeferred +from twisted.python.failure import Failure from w3lib.url import is_url from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError -from scrapy.http import Request +from scrapy.http import Request, Response +from scrapy.spiders import Spider from scrapy.utils import display from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.defer import aiter_errback, deferred_from_coro @@ -20,24 +36,26 @@ from scrapy.utils.spider import spidercls_for_request logger = logging.getLogger(__name__) +_T = TypeVar("_T") + class Command(BaseRunSpiderCommand): requires_project = True spider = None - items: Dict[int, list] = {} - requests: Dict[int, list] = {} + items: Dict[int, List[Any]] = {} + requests: Dict[int, List[Request]] = {} first_response = None - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Parse URL (using its spider) and print the results" - def add_options(self, parser): - BaseRunSpiderCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "--spider", dest="spider", @@ -106,7 +124,7 @@ class Command(BaseRunSpiderCommand): ) @property - def max_level(self): + def max_level(self) -> int: max_items, max_requests = 0, 0 if self.items: max_items = max(self.items) @@ -114,13 +132,21 @@ class Command(BaseRunSpiderCommand): max_requests = max(self.requests) return max(max_items, max_requests) - def handle_exception(self, _failure): + def handle_exception(self, _failure: Failure) -> None: logger.error( "An error is caught while iterating the async iterable", exc_info=failure_to_exc_info(_failure), ) - def iterate_spider_output(self, result): + @overload + def iterate_spider_output( + self, result: Union[AsyncGenerator, CoroutineType] + ) -> Deferred: ... + + @overload + def iterate_spider_output(self, result: _T) -> Iterable: ... + + def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]: if inspect.isasyncgen(result): d = deferred_from_coro( collect_asyncgen(aiter_errback(result, self.handle_exception)) @@ -133,15 +159,15 @@ class Command(BaseRunSpiderCommand): return d return arg_to_iter(deferred_from_coro(result)) - def add_items(self, lvl, new_items): + def add_items(self, lvl: int, new_items: List[Any]) -> None: old_items = self.items.get(lvl, []) self.items[lvl] = old_items + new_items - def add_requests(self, lvl, new_reqs): + def add_requests(self, lvl: int, new_reqs: List[Request]) -> None: old_reqs = self.requests.get(lvl, []) self.requests[lvl] = old_reqs + new_reqs - def print_items(self, lvl=None, colour=True): + def print_items(self, lvl: Optional[int] = None, colour: bool = True) -> None: if lvl is None: items = [item for lst in self.items.values() for item in lst] else: @@ -150,7 +176,7 @@ class Command(BaseRunSpiderCommand): print("# Scraped Items ", "-" * 60) display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour) - def print_requests(self, lvl=None, colour=True): + def print_requests(self, lvl: Optional[int] = None, colour: bool = True) -> None: if lvl is None: if self.requests: requests = self.requests[max(self.requests)] @@ -162,7 +188,7 @@ class Command(BaseRunSpiderCommand): print("# Requests ", "-" * 65) display.pprint(requests, colorize=colour) - def print_results(self, opts): + def print_results(self, opts: argparse.Namespace) -> None: colour = not opts.nocolour if opts.verbose: @@ -179,7 +205,14 @@ class Command(BaseRunSpiderCommand): if not opts.nolinks: self.print_requests(colour=colour) - def _get_items_and_requests(self, spider_output, opts, depth, spider, callback): + def _get_items_and_requests( + self, + spider_output: Iterable[Any], + opts: argparse.Namespace, + depth: int, + spider: Spider, + callback: Callable, + ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, Callable]: items, requests = [], [] for x in spider_output: if is_item(x): @@ -188,14 +221,21 @@ class Command(BaseRunSpiderCommand): requests.append(x) return items, requests, opts, depth, spider, callback - def run_callback(self, response, callback, cb_kwargs=None): + def run_callback( + self, + response: Response, + callback: Callable, + cb_kwargs: Optional[Dict[str, Any]] = None, + ) -> Deferred: cb_kwargs = cb_kwargs or {} d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) return d - def get_callback_from_rules(self, spider, response): + def get_callback_from_rules( + self, spider: Spider, response: Response + ) -> Union[Callable, str, None]: if getattr(spider, "rules", None): - for rule in spider.rules: + for rule in spider.rules: # type: ignore[attr-defined] if rule.link_extractor.matches(response.url): return rule.callback or "parse" else: @@ -204,8 +244,10 @@ class Command(BaseRunSpiderCommand): "please specify a callback to use for parsing", {"spider": spider.name}, ) + return None - def set_spidercls(self, url, opts): + def set_spidercls(self, url: str, opts: argparse.Namespace) -> None: + assert self.crawler_process spider_loader = self.crawler_process.spider_loader if opts.spider: try: @@ -219,13 +261,14 @@ class Command(BaseRunSpiderCommand): if not self.spidercls: logger.error("Unable to find spider for: %(url)s", {"url": url}) - def _start_requests(spider): + def _start_requests(spider: Spider) -> Iterable[Request]: yield self.prepare_request(spider, Request(url), opts) if self.spidercls: self.spidercls.start_requests = _start_requests - def start_parsing(self, url, opts): + def start_parsing(self, url: str, opts: argparse.Namespace) -> None: + assert self.crawler_process self.crawler_process.crawl(self.spidercls, **opts.spargs) self.pcrawler = list(self.crawler_process.crawlers)[0] self.crawler_process.start() @@ -233,7 +276,12 @@ class Command(BaseRunSpiderCommand): if not self.first_response: logger.error("No response downloaded for: %(url)s", {"url": url}) - def scraped_data(self, args): + def scraped_data( + self, + args: Tuple[ + List[Any], List[Request], argparse.Namespace, int, Spider, Callable + ], + ) -> List[Any]: items, requests, opts, depth, spider, callback = args if opts.pipelines: itemproc = self.pcrawler.engine.scraper.itemproc @@ -252,8 +300,14 @@ class Command(BaseRunSpiderCommand): return scraped_data - def _get_callback(self, *, spider, opts, response=None): - cb = None + def _get_callback( + self, + *, + spider: Spider, + opts: argparse.Namespace, + response: Optional[Response] = None, + ) -> Callable: + cb: Union[str, Callable, None] = None if response: cb = response.meta["_callback"] if not cb: @@ -270,6 +324,7 @@ class Command(BaseRunSpiderCommand): cb = "parse" if not callable(cb): + assert cb is not None cb_method = getattr(spider, cb, None) if callable(cb_method): cb = cb_method @@ -277,10 +332,13 @@ class Command(BaseRunSpiderCommand): raise ValueError( f"Cannot find callback {cb!r} in spider: {spider.name}" ) + assert callable(cb) return cb - def prepare_request(self, spider, request, opts): - def callback(response, **cb_kwargs): + def prepare_request( + self, spider: Spider, request: Request, opts: argparse.Namespace + ) -> Request: + def callback(response: Response, **cb_kwargs: Any) -> Deferred: # memorize first request if not self.first_response: self.first_response = response @@ -288,7 +346,7 @@ class Command(BaseRunSpiderCommand): cb = self._get_callback(spider=spider, opts=opts, response=response) # parse items and requests - depth = response.meta["_depth"] + depth: int = response.meta["_depth"] d = self.run_callback(response, cb, cb_kwargs) d.addCallback(self._get_items_and_requests, opts, depth, spider, callback) @@ -311,13 +369,13 @@ class Command(BaseRunSpiderCommand): request.callback = callback return request - def process_options(self, args, opts): - BaseRunSpiderCommand.process_options(self, args, opts) + def process_options(self, args: List[str], opts: argparse.Namespace) -> None: + super().process_options(args, opts) self.process_request_meta(opts) self.process_request_cb_kwargs(opts) - def process_request_meta(self, opts): + def process_request_meta(self, opts: argparse.Namespace) -> None: if opts.meta: try: opts.meta = json.loads(opts.meta) @@ -328,7 +386,7 @@ class Command(BaseRunSpiderCommand): print_help=False, ) - def process_request_cb_kwargs(self, opts): + def process_request_cb_kwargs(self, opts: argparse.Namespace) -> None: if opts.cbkwargs: try: opts.cbkwargs = json.loads(opts.cbkwargs) @@ -339,7 +397,7 @@ class Command(BaseRunSpiderCommand): print_help=False, ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: # parse arguments if not len(args) == 1 or not is_url(args[0]): raise UsageError() diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 58ed89a81..77850e7b5 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,9 +1,10 @@ +import argparse import sys from importlib import import_module from os import PathLike from pathlib import Path from types import ModuleType -from typing import Union +from typing import List, Union from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError @@ -27,16 +28,16 @@ class Command(BaseRunSpiderCommand): requires_project = False default_settings = {"SPIDER_LOADER_WARN_ONLY": True} - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Run a self-contained spider (without creating a project)" - def long_desc(self): + def long_desc(self) -> str: return "Run the spider defined in the given file" - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if len(args) != 1: raise UsageError() filename = Path(args[0]) @@ -51,6 +52,7 @@ class Command(BaseRunSpiderCommand): raise UsageError(f"No spider found in file: {filename}\n") spidercls = spclasses.pop() + assert self.crawler_process self.crawler_process.crawl(spidercls, **opts.spargs) self.crawler_process.start() diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 318187204..dbda73b44 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,4 +1,6 @@ +import argparse import json +from typing import List from scrapy.commands import ScrapyCommand from scrapy.settings import BaseSettings @@ -8,14 +10,14 @@ class Command(ScrapyCommand): requires_project = False default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} - def syntax(self): + def syntax(self) -> str: return "[options]" - def short_desc(self): + def short_desc(self) -> str: return "Get settings values" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "--get", dest="get", metavar="SETTING", help="print raw setting value" ) @@ -44,7 +46,8 @@ class Command(ScrapyCommand): help="print setting value, interpreted as a list", ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: + assert self.crawler_process settings = self.crawler_process.settings if opts.get: s = settings.get(opts.get) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index f72a23c6a..668c95a7b 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -4,9 +4,9 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ -from argparse import Namespace +from argparse import ArgumentParser, Namespace from threading import Thread -from typing import List, Type +from typing import Any, Dict, List, Type from scrapy import Spider from scrapy.commands import ScrapyCommand @@ -24,20 +24,20 @@ class Command(ScrapyCommand): "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", } - def syntax(self): + def syntax(self) -> str: return "[url|file]" - def short_desc(self): + def short_desc(self) -> str: return "Interactive scraping console" - def long_desc(self): + def long_desc(self) -> str: return ( "Interactive console for scraping the given url or file. " "Use ./file.html syntax or full path for local file." ) - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-c", dest="code", @@ -52,7 +52,7 @@ class Command(ScrapyCommand): help="do not handle HTTP 3xx status codes and print response as-is", ) - def update_vars(self, vars): + def update_vars(self, vars: Dict[str, Any]) -> None: """You can use this function to update the Scrapy objects that will be available in the shell """ @@ -88,7 +88,8 @@ class Command(ScrapyCommand): shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) shell.start(url=url, redirect=not opts.no_redirect) - def _start_crawler_thread(self): + def _start_crawler_thread(self) -> None: + assert self.crawler_process t = Thread( target=self.crawler_process.start, kwargs={"stop_after_crawl": False, "install_signal_handlers": False}, diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index fde609c6f..58c1aa28f 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,3 +1,4 @@ +import argparse import os import re import string @@ -5,13 +6,14 @@ from importlib.util import find_spec from pathlib import Path from shutil import copy2, copystat, ignore_patterns, move from stat import S_IWUSR as OWNER_WRITE_PERMISSION +from typing import List, Tuple, Union import scrapy from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.template import render_templatefile, string_camelcase -TEMPLATES_TO_RENDER = ( +TEMPLATES_TO_RENDER: Tuple[Tuple[str, ...], ...] = ( ("scrapy.cfg",), ("${project_name}", "settings.py.tmpl"), ("${project_name}", "items.py.tmpl"), @@ -22,7 +24,7 @@ TEMPLATES_TO_RENDER = ( IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn") -def _make_writable(path): +def _make_writable(path: Union[str, os.PathLike]) -> None: current_permissions = os.stat(path).st_mode os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION) @@ -31,14 +33,14 @@ class Command(ScrapyCommand): requires_project = False default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} - def syntax(self): + def syntax(self) -> str: return " [project_dir]" - def short_desc(self): + def short_desc(self) -> str: return "Create new project" - def _is_valid_name(self, project_name): - def _module_exists(module_name): + def _is_valid_name(self, project_name: str) -> bool: + def _module_exists(module_name: str) -> bool: spec = find_spec(module_name) return spec is not None and spec.loader is not None @@ -53,7 +55,7 @@ class Command(ScrapyCommand): return True return False - def _copytree(self, src: Path, dst: Path): + def _copytree(self, src: Path, dst: Path) -> None: """ Since the original function always creates the directory, to resolve the issue a new function had to be created. It's a simple copy and @@ -84,7 +86,7 @@ class Command(ScrapyCommand): copystat(src, dst) _make_writable(dst) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if len(args) not in (1, 2): raise UsageError() @@ -105,7 +107,9 @@ class Command(ScrapyCommand): return self._copytree(Path(self.templates_dir), project_dir.resolve()) - move(project_dir / "module", project_dir / project_name) + # On 3.8 shutil.move doesn't fully support Path args, but it supports our use case + # See https://bugs.python.org/issue32689 + move(project_dir / "module", project_dir / project_name) # type: ignore[arg-type] for paths in TEMPLATES_TO_RENDER: tplfile = Path( project_dir, diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 47582866b..f057e8544 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -1,3 +1,6 @@ +import argparse +from typing import List + import scrapy from scrapy.commands import ScrapyCommand from scrapy.utils.versions import scrapy_components_versions @@ -6,14 +9,14 @@ from scrapy.utils.versions import scrapy_components_versions class Command(ScrapyCommand): default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} - def syntax(self): + def syntax(self) -> str: return "[-v]" - def short_desc(self): + def short_desc(self) -> str: return "Print Scrapy version" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "--verbose", "-v", @@ -22,7 +25,7 @@ class Command(ScrapyCommand): help="also display twisted/python/platform info (useful for bug reports)", ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if opts.verbose: versions = scrapy_components_versions() width = max(len(n) for (n, _) in versions) diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index ebdfa10a8..21679e3aa 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -1,21 +1,28 @@ import argparse +import logging from scrapy.commands import fetch +from scrapy.http import Response, TextResponse from scrapy.utils.response import open_in_browser +logger = logging.getLogger(__name__) + class Command(fetch.Command): - def short_desc(self): + def short_desc(self) -> str: return "Open URL in browser, as seen by Scrapy" - def long_desc(self): + def long_desc(self) -> str: return ( "Fetch a URL using the Scrapy downloader and show its contents in a browser" ) - def add_options(self, parser): + def add_options(self, parser: argparse.ArgumentParser) -> None: super().add_options(parser) parser.add_argument("--headers", help=argparse.SUPPRESS) - def _print_response(self, response, opts): + def _print_response(self, response: Response, opts: argparse.Namespace) -> None: + if not isinstance(response, TextResponse): + logger.error("Cannot view a non-text response.") + return open_in_browser(response) diff --git a/scrapy/shell.py b/scrapy/shell.py index 63ea33892..fac42e8a2 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -6,6 +6,7 @@ See documentation in docs/topics/shell.rst import os import signal +from typing import Any, Callable, Dict, Optional, Tuple, Union from itemadapter import is_item from twisted.internet import defer, threads @@ -26,18 +27,32 @@ from scrapy.utils.response import open_in_browser class Shell: - relevant_classes = (Crawler, Spider, Request, Response, Settings) + relevant_classes: Tuple[type, ...] = (Crawler, Spider, Request, Response, Settings) - def __init__(self, crawler, update_vars=None, code=None): - self.crawler = crawler - self.update_vars = update_vars or (lambda x: None) - self.item_class = load_object(crawler.settings["DEFAULT_ITEM_CLASS"]) - self.spider = None - self.inthread = not threadable.isInIOThread() - self.code = code - self.vars = {} + def __init__( + self, + crawler: Crawler, + update_vars: Optional[Callable[[Dict[str, Any]], None]] = None, + code: Optional[str] = None, + ): + self.crawler: Crawler = crawler + self.update_vars: Callable[[Dict[str, Any]], None] = update_vars or ( + lambda x: None + ) + self.item_class: type = load_object(crawler.settings["DEFAULT_ITEM_CLASS"]) + self.spider: Optional[Spider] = None + self.inthread: bool = not threadable.isInIOThread() + self.code: Optional[str] = code + self.vars: Dict[str, Any] = {} - def start(self, url=None, request=None, response=None, spider=None, redirect=True): + def start( + self, + url: Optional[str] = None, + request: Optional[Request] = None, + response: Optional[Response] = None, + spider: Optional[Spider] = None, + redirect: bool = True, + ) -> None: # disable accidental Ctrl-C key press from shutting down the engine signal.signal(signal.SIGINT, signal.SIG_IGN) if url: @@ -77,7 +92,7 @@ class Shell: self.vars, shells=shells, banner=self.vars.pop("banner", "") ) - def _schedule(self, request, spider): + def _schedule(self, request: Request, spider: Optional[Spider]) -> defer.Deferred: if is_asyncio_reactor_installed(): # set the asyncio event loop for the current thread event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"] @@ -85,10 +100,11 @@ class Shell: spider = self._open_spider(request, spider) d = _request_deferred(request) d.addCallback(lambda x: (x, spider)) + assert self.crawler.engine self.crawler.engine.crawl(request) return d - def _open_spider(self, request, spider): + def _open_spider(self, request: Request, spider: Optional[Spider]) -> Spider: if self.spider: return self.spider @@ -96,11 +112,18 @@ class Shell: spider = self.crawler.spider or self.crawler._create_spider() self.crawler.spider = spider + assert self.crawler.engine self.crawler.engine.open_spider(spider, close_if_idle=False) self.spider = spider return spider - def fetch(self, request_or_url, spider=None, redirect=True, **kwargs): + def fetch( + self, + request_or_url: Union[Request, str], + spider: Optional[Spider] = None, + redirect: bool = True, + **kwargs: Any, + ) -> None: from twisted.internet import reactor if isinstance(request_or_url, Request): @@ -123,7 +146,12 @@ class Shell: pass self.populate_vars(response, request, spider) - def populate_vars(self, response=None, request=None, spider=None): + def populate_vars( + self, + response: Optional[Response] = None, + request: Optional[Request] = None, + spider: Optional[Spider] = None, + ) -> None: import scrapy self.vars["scrapy"] = scrapy @@ -141,10 +169,10 @@ class Shell: if not self.code: self.vars["banner"] = self.get_help() - def print_help(self): + def print_help(self) -> None: print(self.get_help()) - def get_help(self): + def get_help(self) -> str: b = [] b.append("Available Scrapy objects:") b.append( @@ -168,11 +196,11 @@ class Shell: return "\n".join(f"[s] {line}" for line in b) - def _is_relevant(self, value): + def _is_relevant(self, value: Any) -> bool: return isinstance(value, self.relevant_classes) or is_item(value) -def inspect_response(response, spider): +def inspect_response(response: Response, spider: Spider) -> None: """Open a shell to inspect the given response""" # Shell.start removes the SIGINT handler, so save it and re-add it after # the shell has closed @@ -181,7 +209,7 @@ def inspect_response(response, spider): signal.signal(signal.SIGINT, sigint_handler) -def _request_deferred(request): +def _request_deferred(request: Request) -> defer.Deferred: """Wrap a request inside a Deferred. This function is harmful, do not use it until you know what you are doing. @@ -195,12 +223,12 @@ def _request_deferred(request): request_callback = request.callback request_errback = request.errback - def _restore_callbacks(result): + def _restore_callbacks(result: Any) -> Any: request.callback = request_callback request.errback = request_errback return result - d = defer.Deferred() + d: defer.Deferred = defer.Deferred() d.addBoth(_restore_callbacks) if request.callback: d.addCallbacks(request.callback, request.errback) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 100f040bb..bf1803115 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,17 +1,27 @@ from functools import wraps +from typing import Any, Callable, Dict, Iterable, Optional + +EmbedFuncT = Callable[..., None] +KnownShellsT = Dict[str, Callable[..., EmbedFuncT]] -def _embed_ipython_shell(namespace={}, banner=""): +def _embed_ipython_shell( + namespace: Dict[str, Any] = {}, banner: str = "" +) -> EmbedFuncT: """Start an IPython Shell""" try: from IPython.terminal.embed import InteractiveShellEmbed from IPython.terminal.ipapp import load_default_config except ImportError: - from IPython.frontend.terminal.embed import InteractiveShellEmbed - from IPython.frontend.terminal.ipapp import load_default_config + from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] + InteractiveShellEmbed, + ) + from IPython.frontend.terminal.ipapp import ( # type: ignore[no-redef] + load_default_config, + ) @wraps(_embed_ipython_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: config = load_default_config() # Always use .instance() to ensure _instance propagation to all parents # this is needed for completion works well for new imports @@ -26,30 +36,36 @@ def _embed_ipython_shell(namespace={}, banner=""): return wrapper -def _embed_bpython_shell(namespace={}, banner=""): +def _embed_bpython_shell( + namespace: Dict[str, Any] = {}, banner: str = "" +) -> EmbedFuncT: """Start a bpython shell""" import bpython @wraps(_embed_bpython_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: bpython.embed(locals_=namespace, banner=banner) return wrapper -def _embed_ptpython_shell(namespace={}, banner=""): +def _embed_ptpython_shell( + namespace: Dict[str, Any] = {}, banner: str = "" +) -> EmbedFuncT: """Start a ptpython shell""" import ptpython.repl @wraps(_embed_ptpython_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: print(banner) ptpython.repl.embed(locals=namespace) return wrapper -def _embed_standard_shell(namespace={}, banner=""): +def _embed_standard_shell( + namespace: Dict[str, Any] = {}, banner: str = "" +) -> EmbedFuncT: """Start a standard python shell""" import code @@ -63,13 +79,13 @@ def _embed_standard_shell(namespace={}, banner=""): readline.parse_and_bind("tab:complete") @wraps(_embed_standard_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: code.interact(banner=banner, local=namespace) return wrapper -DEFAULT_PYTHON_SHELLS = { +DEFAULT_PYTHON_SHELLS: KnownShellsT = { "ptpython": _embed_ptpython_shell, "ipython": _embed_ipython_shell, "bpython": _embed_bpython_shell, @@ -77,7 +93,9 @@ DEFAULT_PYTHON_SHELLS = { } -def get_shell_embed_func(shells=None, known_shells=None): +def get_shell_embed_func( + shells: Optional[Iterable[str]] = None, known_shells: Optional[KnownShellsT] = None +) -> Any: """Return the first acceptable shell-embed function from a given list of shell names. """ @@ -95,7 +113,11 @@ def get_shell_embed_func(shells=None, known_shells=None): continue -def start_python_console(namespace=None, banner="", shells=None): +def start_python_console( + namespace: Optional[Dict[str, Any]] = None, + banner: str = "", + shells: Optional[Iterable[str]] = None, +) -> None: """Start Python console bound to the given namespace. Readline support and tab completion will be used on Unix, if available. """ diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 63a484b42..a0b06f75c 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -3,24 +3,27 @@ This module provides some useful functions for working with scrapy.http.Response objects """ +from __future__ import annotations + import os import re import tempfile import webbrowser -from typing import Any, Callable, Iterable, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Iterable, Tuple, Union from weakref import WeakKeyDictionary from twisted.web import http from w3lib import html -import scrapy -from scrapy.http.response import Response from scrapy.utils.python import to_bytes, to_unicode -_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary() +if TYPE_CHECKING: + from scrapy.http import Response, TextResponse + +_baseurl_cache: WeakKeyDictionary[Response, str] = WeakKeyDictionary() -def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: +def get_base_url(response: TextResponse) -> str: """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: text = response.text[0:4096] @@ -30,13 +33,13 @@ def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: return _baseurl_cache[response] -_metaref_cache: ( - "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" -) = WeakKeyDictionary() +_metaref_cache: WeakKeyDictionary[ + Response, Union[Tuple[None, None], Tuple[float, str]] +] = WeakKeyDictionary() def get_meta_refresh( - response: "scrapy.http.response.text.TextResponse", + response: TextResponse, ignore_tags: Iterable[str] = ("script", "noscript"), ) -> Union[Tuple[None, None], Tuple[float, str]]: """Parse the http-equiv refresh parameter from the given response""" @@ -68,10 +71,7 @@ def _remove_html_comments(body): def open_in_browser( - response: Union[ - "scrapy.http.response.html.HtmlResponse", - "scrapy.http.response.text.TextResponse", - ], + response: TextResponse, _openfunc: Callable[[str], Any] = webbrowser.open, ) -> Any: """Open *response* in a local web browser, adjusting the `base tag`_ for From 8985a04bd1328cd6156a7c33a5db74ad8c81802f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 8 Mar 2024 10:57:03 +0500 Subject: [PATCH 123/269] Full typing for scrapy/exporters.py. (#6275) --- scrapy/exporters.py | 150 ++++++++++++++++++-------------- scrapy/extensions/feedexport.py | 11 +-- 2 files changed, 89 insertions(+), 72 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 79fd4e56f..fb4998099 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -3,16 +3,18 @@ Item Exporters are used to export/serialize items into different formats. """ import csv -import io import marshal import pickle # nosec import pprint -from collections.abc import Mapping +from io import BytesIO, TextIOWrapper +from json import JSONEncoder +from typing import Any, Callable, Dict, Iterable, Mapping, Optional, Tuple, Union from xml.sax.saxutils import XMLGenerator # nosec +from xml.sax.xmlreader import AttributesImpl # nosec from itemadapter import ItemAdapter, is_item -from scrapy.item import Item +from scrapy.item import Field, Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder @@ -29,36 +31,42 @@ __all__ = [ class BaseItemExporter: - def __init__(self, *, dont_fail=False, **kwargs): - self._kwargs = kwargs + def __init__(self, *, dont_fail: bool = False, **kwargs: Any): + self._kwargs: Dict[str, Any] = kwargs self._configure(kwargs, dont_fail=dont_fail) - def _configure(self, options, dont_fail=False): + def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None: """Configure the exporter by popping options from the ``options`` dict. If dont_fail is set, it won't raise an exception on unexpected options (useful for using with keyword arguments in subclasses ``__init__`` methods) """ - self.encoding = options.pop("encoding", None) - self.fields_to_export = options.pop("fields_to_export", None) - self.export_empty_fields = options.pop("export_empty_fields", False) - self.indent = options.pop("indent", None) + self.encoding: Optional[str] = options.pop("encoding", None) + self.fields_to_export: Union[Mapping[str, str], Iterable[str], None] = ( + options.pop("fields_to_export", None) + ) + self.export_empty_fields: bool = options.pop("export_empty_fields", False) + self.indent: Optional[int] = options.pop("indent", None) if not dont_fail and options: raise TypeError(f"Unexpected options: {', '.join(options.keys())}") - def export_item(self, item): + def export_item(self, item: Any) -> None: raise NotImplementedError - def serialize_field(self, field, name, value): - serializer = field.get("serializer", lambda x: x) + def serialize_field( + self, field: Union[Mapping[str, Any], Field], name: str, value: Any + ) -> Any: + serializer: Callable[[Any], Any] = field.get("serializer", lambda x: x) return serializer(value) - def start_exporting(self): + def start_exporting(self) -> None: pass - def finish_exporting(self): + def finish_exporting(self) -> None: pass - def _get_serialized_fields(self, item, default_value=None, include_empty=None): + def _get_serialized_fields( + self, item: Any, default_value: Any = None, include_empty: Optional[bool] = None + ) -> Iterable[Tuple[str, Any]]: """Return the fields to export as an iterable of tuples (name, serialized_value) """ @@ -100,22 +108,22 @@ class BaseItemExporter: class JsonLinesItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(dont_fail=True, **kwargs) - self.file = file + self.file: BytesIO = file self._kwargs.setdefault("ensure_ascii", not self.encoding) - self.encoder = ScrapyJSONEncoder(**self._kwargs) + self.encoder: JSONEncoder = ScrapyJSONEncoder(**self._kwargs) - def export_item(self, item): + def export_item(self, item: Any) -> None: itemdict = dict(self._get_serialized_fields(item)) data = self.encoder.encode(itemdict) + "\n" self.file.write(to_bytes(data, self.encoding)) class JsonItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(dont_fail=True, **kwargs) - self.file = file + self.file: BytesIO = file # there is a small difference between the behaviour or JsonItemExporter.indent # and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent # the addition of newlines everywhere @@ -127,26 +135,26 @@ class JsonItemExporter(BaseItemExporter): self.encoder = ScrapyJSONEncoder(**self._kwargs) self.first_item = True - def _beautify_newline(self): + def _beautify_newline(self) -> None: if self.indent is not None: self.file.write(b"\n") - def _add_comma_after_first(self): + def _add_comma_after_first(self) -> None: if self.first_item: self.first_item = False else: self.file.write(b",") self._beautify_newline() - def start_exporting(self): + def start_exporting(self) -> None: self.file.write(b"[") self._beautify_newline() - def finish_exporting(self): + def finish_exporting(self) -> None: self._beautify_newline() self.file.write(b"]") - def export_item(self, item): + def export_item(self, item: Any) -> None: itemdict = dict(self._get_serialized_fields(item)) data = to_bytes(self.encoder.encode(itemdict), self.encoding) self._add_comma_after_first() @@ -154,7 +162,7 @@ class JsonItemExporter(BaseItemExporter): class XmlItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): self.item_element = kwargs.pop("item_element", "item") self.root_element = kwargs.pop("root_element", "items") super().__init__(**kwargs) @@ -162,22 +170,22 @@ class XmlItemExporter(BaseItemExporter): self.encoding = "utf-8" self.xg = XMLGenerator(file, encoding=self.encoding) - def _beautify_newline(self, new_item=False): + def _beautify_newline(self, new_item: bool = False) -> None: if self.indent is not None and (self.indent > 0 or new_item): self.xg.characters("\n") - def _beautify_indent(self, depth=1): + def _beautify_indent(self, depth: int = 1) -> None: if self.indent: self.xg.characters(" " * self.indent * depth) - def start_exporting(self): + def start_exporting(self) -> None: self.xg.startDocument() - self.xg.startElement(self.root_element, {}) + self.xg.startElement(self.root_element, AttributesImpl({})) self._beautify_newline(new_item=True) - def export_item(self, item): + def export_item(self, item: Any) -> None: self._beautify_indent(depth=1) - self.xg.startElement(self.item_element, {}) + self.xg.startElement(self.item_element, AttributesImpl({})) self._beautify_newline() for name, value in self._get_serialized_fields(item, default_value=""): self._export_xml_field(name, value, depth=2) @@ -185,13 +193,13 @@ class XmlItemExporter(BaseItemExporter): self.xg.endElement(self.item_element) self._beautify_newline(new_item=True) - def finish_exporting(self): + def finish_exporting(self) -> None: self.xg.endElement(self.root_element) self.xg.endDocument() - def _export_xml_field(self, name, serialized_value, depth): + def _export_xml_field(self, name: str, serialized_value: Any, depth: int) -> None: self._beautify_indent(depth=depth) - self.xg.startElement(name, {}) + self.xg.startElement(name, AttributesImpl({})) if hasattr(serialized_value, "items"): self._beautify_newline() for subname, value in serialized_value.items(): @@ -213,17 +221,17 @@ class XmlItemExporter(BaseItemExporter): class CsvItemExporter(BaseItemExporter): def __init__( self, - file, - include_headers_line=True, - join_multivalued=",", - errors=None, - **kwargs, + file: BytesIO, + include_headers_line: bool = True, + join_multivalued: str = ",", + errors: Optional[str] = None, + **kwargs: Any, ): super().__init__(dont_fail=True, **kwargs) if not self.encoding: self.encoding = "utf-8" self.include_headers_line = include_headers_line - self.stream = io.TextIOWrapper( + self.stream = TextIOWrapper( file, line_buffering=False, write_through=True, @@ -235,11 +243,13 @@ class CsvItemExporter(BaseItemExporter): self._headers_not_written = True self._join_multivalued = join_multivalued - def serialize_field(self, field, name, value): - serializer = field.get("serializer", self._join_if_needed) + def serialize_field( + self, field: Union[Mapping[str, Any], Field], name: str, value: Any + ) -> Any: + serializer: Callable[[Any], Any] = field.get("serializer", self._join_if_needed) return serializer(value) - def _join_if_needed(self, value): + def _join_if_needed(self, value: Any) -> Any: if isinstance(value, (list, tuple)): try: return self._join_multivalued.join(value) @@ -247,7 +257,7 @@ class CsvItemExporter(BaseItemExporter): pass return value - def export_item(self, item): + def export_item(self, item: Any) -> None: if self._headers_not_written: self._headers_not_written = False self._write_headers_and_set_fields_to_export(item) @@ -256,36 +266,38 @@ class CsvItemExporter(BaseItemExporter): values = list(self._build_row(x for _, x in fields)) self.csv_writer.writerow(values) - def finish_exporting(self): + def finish_exporting(self) -> None: self.stream.detach() # Avoid closing the wrapped file. - def _build_row(self, values): + def _build_row(self, values: Iterable[Any]) -> Iterable[Any]: for s in values: try: yield to_unicode(s, self.encoding) except TypeError: yield s - def _write_headers_and_set_fields_to_export(self, item): + def _write_headers_and_set_fields_to_export(self, item: Any) -> None: if self.include_headers_line: if not self.fields_to_export: # use declared field names, or keys if the item is a dict self.fields_to_export = ItemAdapter(item).field_names() + fields: Iterable[str] if isinstance(self.fields_to_export, Mapping): fields = self.fields_to_export.values() else: + assert self.fields_to_export fields = self.fields_to_export row = list(self._build_row(fields)) self.csv_writer.writerow(row) class PickleItemExporter(BaseItemExporter): - def __init__(self, file, protocol=4, **kwargs): + def __init__(self, file: BytesIO, protocol: int = 4, **kwargs: Any): super().__init__(**kwargs) - self.file = file - self.protocol = protocol + self.file: BytesIO = file + self.protocol: int = protocol - def export_item(self, item): + def export_item(self, item: Any) -> None: d = dict(self._get_serialized_fields(item)) pickle.dump(d, self.file, self.protocol) @@ -299,20 +311,20 @@ class MarshalItemExporter(BaseItemExporter): opened in binary mode, a :class:`~io.BytesIO` object, etc) """ - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(**kwargs) - self.file = file + self.file: BytesIO = file - def export_item(self, item): + def export_item(self, item: Any) -> None: marshal.dump(dict(self._get_serialized_fields(item)), self.file) class PprintItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(**kwargs) - self.file = file + self.file: BytesIO = file - def export_item(self, item): + def export_item(self, item: Any) -> None: itemdict = dict(self._get_serialized_fields(item)) self.file.write(to_bytes(pprint.pformat(itemdict) + "\n")) @@ -327,16 +339,20 @@ class PythonItemExporter(BaseItemExporter): .. _msgpack: https://pypi.org/project/msgpack/ """ - def _configure(self, options, dont_fail=False): + def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None: super()._configure(options, dont_fail) if not self.encoding: self.encoding = "utf-8" - def serialize_field(self, field, name, value): - serializer = field.get("serializer", self._serialize_value) + def serialize_field( + self, field: Union[Mapping[str, Any], Field], name: str, value: Any + ) -> Any: + serializer: Callable[[Any], Any] = field.get( + "serializer", self._serialize_value + ) return serializer(value) - def _serialize_value(self, value): + def _serialize_value(self, value: Any) -> Any: if isinstance(value, Item): return self.export_item(value) if is_item(value): @@ -347,10 +363,10 @@ class PythonItemExporter(BaseItemExporter): return to_unicode(value, encoding=self.encoding) return value - def _serialize_item(self, item): + def _serialize_item(self, item: Any) -> Iterable[Tuple[Union[str, bytes], Any]]: for key, value in ItemAdapter(item).items(): yield key, self._serialize_value(value) - def export_item(self, item): - result = dict(self._get_serialized_fields(item)) + def export_item(self, item: Any) -> Dict[Union[str, bytes], Any]: # type: ignore[override] + result: Dict[Union[str, bytes], Any] = dict(self._get_serialized_fields(item)) return result diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e5e363b52..3b0dd804e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -11,7 +11,7 @@ import warnings from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union +from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Type, Union from urllib.parse import unquote, urlparse from twisted.internet import defer, threads @@ -21,6 +21,7 @@ from zope.interface import Interface, implementer from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exporters import BaseItemExporter from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings @@ -324,12 +325,12 @@ class FeedSlot: filter, feed_options, spider, - exporters, + exporters: Dict[str, Type[BaseItemExporter]], settings, crawler, ): self.file = None - self.exporter = None + self.exporter: Optional[BaseItemExporter] = None self.storage = storage # feed params self.batch_id = batch_id @@ -341,7 +342,7 @@ class FeedSlot: # exporter params self.feed_options = feed_options self.spider = spider - self.exporters = exporters + self.exporters: Dict[str, Type[BaseItemExporter]] = exporters self.settings = settings self.crawler = crawler # flags @@ -373,7 +374,7 @@ class FeedSlot: def _get_instance(self, objcls, *args, **kwargs): return build_from_crawler(objcls, self.crawler, *args, **kwargs) - def _get_exporter(self, file, format, *args, **kwargs): + def _get_exporter(self, file, format, *args, **kwargs) -> BaseItemExporter: return self._get_instance(self.exporters[format], file, *args, **kwargs) def finish_exporting(self): From 421e08dd4a4d5ed2acf3cd0fca4c7bfa5a6d3eb2 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 8 Mar 2024 10:59:56 +0500 Subject: [PATCH 124/269] Full typing for scrapy/extensions, part 1. (#6276) --- scrapy/extensions/closespider.py | 52 +++++++++++++++++++++----------- scrapy/extensions/corestats.py | 33 +++++++++++++------- scrapy/extensions/debug.py | 25 ++++++++++----- scrapy/extensions/logstats.py | 50 ++++++++++++++++++------------ scrapy/extensions/memdebug.py | 22 ++++++++++---- scrapy/extensions/memusage.py | 51 ++++++++++++++++++++----------- scrapy/mail.py | 12 ++++++-- scrapy/utils/engine.py | 13 ++++---- 8 files changed, 171 insertions(+), 87 deletions(-) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 4307b4170..812b3553c 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -4,20 +4,31 @@ conditions are met. See documentation in docs/topics/extensions.rst """ +from __future__ import annotations + import logging from collections import defaultdict +from typing import TYPE_CHECKING, Any, DefaultDict, Dict -from scrapy import signals +from twisted.python.failure import Failure + +from scrapy import Request, Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) class CloseSpider: - def __init__(self, crawler): - self.crawler = crawler + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler - self.close_on = { + self.close_on: Dict[str, Any] = { "timeout": crawler.settings.getfloat("CLOSESPIDER_TIMEOUT"), "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"), "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"), @@ -28,7 +39,7 @@ class CloseSpider: if not any(self.close_on.values()): raise NotConfigured - self.counter = defaultdict(int) + self.counter: DefaultDict[str, int] = defaultdict(int) if self.close_on.get("errorcount"): crawler.signals.connect(self.error_count, signal=signals.spider_error) @@ -39,8 +50,8 @@ class CloseSpider: if self.close_on.get("itemcount"): crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) if self.close_on.get("timeout_no_item"): - self.timeout_no_item = self.close_on["timeout_no_item"] - self.items_in_period = 0 + self.timeout_no_item: int = self.close_on["timeout_no_item"] + self.items_in_period: int = 0 crawler.signals.connect( self.spider_opened_no_item, signal=signals.spider_opened ) @@ -50,22 +61,25 @@ class CloseSpider: crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def error_count(self, failure, response, spider): + def error_count(self, failure: Failure, response: Response, spider: Spider) -> None: self.counter["errorcount"] += 1 if self.counter["errorcount"] == self.close_on["errorcount"]: + assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_errorcount") - def page_count(self, response, request, spider): + def page_count(self, response: Response, request: Request, spider: Spider) -> None: self.counter["pagecount"] += 1 if self.counter["pagecount"] == self.close_on["pagecount"]: + assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_pagecount") - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: from twisted.internet import reactor + assert self.crawler.engine self.task = reactor.callLater( self.close_on["timeout"], self.crawler.engine.close_spider, @@ -73,21 +87,22 @@ class CloseSpider: reason="closespider_timeout", ) - def item_scraped(self, item, spider): + def item_scraped(self, item: Any, spider: Spider) -> None: self.counter["itemcount"] += 1 if self.counter["itemcount"] == self.close_on["itemcount"]: + assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_itemcount") - def spider_closed(self, spider): - task = getattr(self, "task", False) + def spider_closed(self, spider: Spider) -> None: + task = getattr(self, "task", None) if task and task.active(): task.cancel() - task_no_item = getattr(self, "task_no_item", False) + task_no_item = getattr(self, "task_no_item", None) if task_no_item and task_no_item.running: task_no_item.stop() - def spider_opened_no_item(self, spider): + def spider_opened_no_item(self, spider: Spider) -> None: from twisted.internet import task self.task_no_item = task.LoopingCall(self._count_items_produced, spider) @@ -98,10 +113,10 @@ class CloseSpider: f"{self.timeout_no_item} seconds." ) - def item_scraped_no_item(self, item, spider): + def item_scraped_no_item(self, item: Any, spider: Spider) -> None: self.items_in_period += 1 - def _count_items_produced(self, spider): + def _count_items_produced(self, spider: Spider) -> None: if self.items_in_period >= 1: self.items_in_period = 0 else: @@ -109,4 +124,5 @@ class CloseSpider: f"Closing spider since no items were produced in the last " f"{self.timeout_no_item} seconds." ) + assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_timeout_no_item") diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 717c249d9..f3ac19623 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -2,18 +2,28 @@ Extension for collecting core stats like items scraped and start/finish times """ -from datetime import datetime, timezone +from __future__ import annotations -from scrapy import signals +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional + +from scrapy import Spider, signals +from scrapy.crawler import Crawler +from scrapy.statscollectors import StatsCollector + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class CoreStats: - def __init__(self, stats): - self.stats = stats - self.start_time = None + def __init__(self, stats: StatsCollector): + self.stats: StatsCollector = stats + self.start_time: Optional[datetime] = None @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: + assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) @@ -22,11 +32,12 @@ class CoreStats: crawler.signals.connect(o.response_received, signal=signals.response_received) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: self.start_time = datetime.now(tz=timezone.utc) self.stats.set_value("start_time", self.start_time, spider=spider) - def spider_closed(self, spider, reason): + def spider_closed(self, spider: Spider, reason: str) -> None: + assert self.start_time is not None finish_time = datetime.now(tz=timezone.utc) elapsed_time = finish_time - self.start_time elapsed_time_seconds = elapsed_time.total_seconds() @@ -36,13 +47,13 @@ class CoreStats: self.stats.set_value("finish_time", finish_time, spider=spider) self.stats.set_value("finish_reason", reason, spider=spider) - def item_scraped(self, item, spider): + def item_scraped(self, item: Any, spider: Spider) -> None: self.stats.inc_value("item_scraped_count", spider=spider) - def response_received(self, spider): + def response_received(self, spider: Spider) -> None: self.stats.inc_value("response_received_count", spider=spider) - def item_dropped(self, item, spider, exception): + def item_dropped(self, item: Any, spider: Spider, exception: BaseException) -> None: reason = exception.__class__.__name__ self.stats.inc_value("item_dropped_count", spider=spider) self.stats.inc_value(f"item_dropped_reasons_count/{reason}", spider=spider) diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 1b6c7777f..26726b662 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -4,22 +4,31 @@ Extensions for debugging Scrapy See documentation in docs/topics/extensions.rst """ +from __future__ import annotations + import logging import signal import sys import threading import traceback from pdb import Pdb +from types import FrameType +from typing import TYPE_CHECKING, Optional +from scrapy.crawler import Crawler from scrapy.utils.engine import format_engine_status from scrapy.utils.trackref import format_live_refs +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) class StackTraceDump: - def __init__(self, crawler=None): - self.crawler = crawler + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler try: signal.signal(signal.SIGUSR2, self.dump_stacktrace) signal.signal(signal.SIGQUIT, self.dump_stacktrace) @@ -28,10 +37,11 @@ class StackTraceDump: pass @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def dump_stacktrace(self, signum, frame): + def dump_stacktrace(self, signum: int, frame: Optional[FrameType]) -> None: + assert self.crawler.engine log_args = { "stackdumps": self._thread_stacks(), "enginestatus": format_engine_status(self.crawler.engine), @@ -44,7 +54,7 @@ class StackTraceDump: extra={"crawler": self.crawler}, ) - def _thread_stacks(self): + def _thread_stacks(self) -> str: id2name = dict((th.ident, th.name) for th in threading.enumerate()) dumps = "" for id_, frame in sys._current_frames().items(): @@ -55,12 +65,13 @@ class StackTraceDump: class Debugger: - def __init__(self): + def __init__(self) -> None: try: signal.signal(signal.SIGUSR2, self._enter_debugger) except AttributeError: # win32 platforms don't support SIGUSR signals pass - def _enter_debugger(self, signum, frame): + def _enter_debugger(self, signum: int, frame: Optional[FrameType]) -> None: + assert frame Pdb().set_trace(frame.f_back) diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 9f63e9c4b..2388afa75 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -1,9 +1,18 @@ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING, Optional, Tuple, Union from twisted.internet import task -from scrapy import signals +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.statscollectors import StatsCollector + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) @@ -14,30 +23,31 @@ class LogStats: * IPM - Items per Minute """ - def __init__(self, stats, interval=60.0): - self.stats = stats - self.interval = interval - self.multiplier = 60.0 / self.interval - self.task = None + def __init__(self, stats: StatsCollector, interval: float = 60.0): + self.stats: StatsCollector = stats + self.interval: float = interval + self.multiplier: float = 60.0 / self.interval + self.task: Optional[task.LoopingCall] = None @classmethod - def from_crawler(cls, crawler): - interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") + def from_crawler(cls, crawler: Crawler) -> Self: + interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL") if not interval: raise NotConfigured + assert crawler.stats o = cls(crawler.stats, interval) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_opened(self, spider): - self.pagesprev = 0 - self.itemsprev = 0 + def spider_opened(self, spider: Spider) -> None: + self.pagesprev: int = 0 + self.itemsprev: int = 0 self.task = task.LoopingCall(self.log, spider) self.task.start(self.interval) - def log(self, spider): + def log(self, spider: Spider) -> None: self.calculate_stats() msg = ( @@ -52,14 +62,14 @@ class LogStats: } logger.info(msg, log_args, extra={"spider": spider}) - def calculate_stats(self): - self.items = self.stats.get_value("item_scraped_count", 0) - self.pages = self.stats.get_value("response_received_count", 0) - self.irate = (self.items - self.itemsprev) * self.multiplier - self.prate = (self.pages - self.pagesprev) * self.multiplier + def calculate_stats(self) -> None: + self.items: int = self.stats.get_value("item_scraped_count", 0) + self.pages: int = self.stats.get_value("response_received_count", 0) + self.irate: float = (self.items - self.itemsprev) * self.multiplier + self.prate: float = (self.pages - self.pagesprev) * self.multiplier self.pagesprev, self.itemsprev = self.pages, self.items - def spider_closed(self, spider, reason): + def spider_closed(self, spider: Spider, reason: str) -> None: if self.task and self.task.running: self.task.stop() @@ -67,7 +77,9 @@ class LogStats: self.stats.set_value("responses_per_minute", rpm_final) self.stats.set_value("items_per_minute", ipm_final) - def calculate_final_stats(self, spider): + def calculate_final_stats( + self, spider: Spider + ) -> Union[Tuple[None, None], Tuple[float, float]]: start_time = self.stats.get_value("start_time") finished_time = self.stats.get_value("finished_time") diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index 03ede0681..f304e1bf2 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -4,26 +4,36 @@ MemoryDebugger extension See documentation in docs/topics/extensions.rst """ -import gc +from __future__ import annotations -from scrapy import signals +import gc +from typing import TYPE_CHECKING + +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.statscollectors import StatsCollector from scrapy.utils.trackref import live_refs +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class MemoryDebugger: - def __init__(self, stats): - self.stats = stats + def __init__(self, stats: StatsCollector): + self.stats: StatsCollector = stats @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("MEMDEBUG_ENABLED"): raise NotConfigured + assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_closed(self, spider, reason): + def spider_closed(self, spider: Spider, reason: str) -> None: gc.collect() self.stats.set_value( "memdebug/gc_garbage_count", len(gc.garbage), spider=spider diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 4d4501c44..9de06b24d 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -4,24 +4,32 @@ MemoryUsage extension See documentation in docs/topics/extensions.rst """ +from __future__ import annotations + import logging import socket import sys from importlib import import_module from pprint import pformat +from typing import TYPE_CHECKING, List from twisted.internet import task from scrapy import signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender from scrapy.utils.engine import get_engine_status +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) class MemoryUsage: - def __init__(self, crawler): + def __init__(self, crawler: Crawler): if not crawler.settings.getbool("MEMUSAGE_ENABLED"): raise NotConfigured try: @@ -30,32 +38,33 @@ class MemoryUsage: except ImportError: raise NotConfigured - self.crawler = crawler - self.warned = False - self.notify_mails = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") - self.limit = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 - self.warning = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 - self.check_interval = crawler.settings.getfloat( + self.crawler: Crawler = crawler + self.warned: bool = False + self.notify_mails: List[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") + self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 + self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 + self.check_interval: float = crawler.settings.getfloat( "MEMUSAGE_CHECK_INTERVAL_SECONDS" ) - self.mail = MailSender.from_settings(crawler.settings) + self.mail: MailSender = MailSender.from_settings(crawler.settings) crawler.signals.connect(self.engine_started, signal=signals.engine_started) crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def get_virtual_size(self): - size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss + def get_virtual_size(self) -> int: + size: int = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss if sys.platform != "darwin": # on macOS ru_maxrss is in bytes, on Linux it is in KB size *= 1024 return size - def engine_started(self): + def engine_started(self) -> None: + assert self.crawler.stats self.crawler.stats.set_value("memusage/startup", self.get_virtual_size()) - self.tasks = [] + self.tasks: List[task.LoopingCall] = [] tsk = task.LoopingCall(self.update) self.tasks.append(tsk) tsk.start(self.check_interval, now=True) @@ -68,15 +77,18 @@ class MemoryUsage: self.tasks.append(tsk) tsk.start(self.check_interval, now=True) - def engine_stopped(self): + def engine_stopped(self) -> None: for tsk in self.tasks: if tsk.running: tsk.stop() - def update(self): + def update(self) -> None: + assert self.crawler.stats self.crawler.stats.max_value("memusage/max", self.get_virtual_size()) - def _check_limit(self): + def _check_limit(self) -> None: + assert self.crawler.engine + assert self.crawler.stats peak_mem_usage = self.get_virtual_size() if peak_mem_usage > self.limit: self.crawler.stats.set_value("memusage/limit_reached", 1) @@ -106,9 +118,10 @@ class MemoryUsage: {"virtualsize": peak_mem_usage / 1024 / 1024}, ) - def _check_warning(self): + def _check_warning(self) -> None: if self.warned: # warn only once return + assert self.crawler.stats if self.get_virtual_size() > self.warning: self.crawler.stats.set_value("memusage/warning_reached", 1) mem = self.warning / 1024 / 1024 @@ -126,8 +139,10 @@ class MemoryUsage: self.crawler.stats.set_value("memusage/warning_notified", 1) self.warned = True - def _send_report(self, rcpts, subject): + def _send_report(self, rcpts: List[str], subject: str) -> None: """send notification mail with some additional useful info""" + assert self.crawler.engine + assert self.crawler.stats stats = self.crawler.stats s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n" s += f"Maximum memory usage : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n" diff --git a/scrapy/mail.py b/scrapy/mail.py index 4b18b6003..dce33fcdf 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -4,6 +4,8 @@ Mail sending helpers See documentation in docs/topics/email.rst """ +from __future__ import annotations + import logging from email import encoders as Encoders from email.mime.base import MIMEBase @@ -12,14 +14,20 @@ from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO +from typing import TYPE_CHECKING from twisted import version as twisted_version from twisted.internet import defer, ssl from twisted.python.versions import Version +from scrapy.settings import BaseSettings from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) @@ -56,7 +64,7 @@ class MailSender: self.debug = debug @classmethod - def from_settings(cls, settings): + def from_settings(cls, settings: BaseSettings) -> Self: return cls( smtphost=settings["MAIL_HOST"], mailfrom=settings["MAIL_FROM"], @@ -203,7 +211,7 @@ class MailSender: to_addrs, msg, d, - **factory_keywords + **factory_keywords, ) factory.noisy = False return factory diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 0b2722663..fdcf484d4 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -1,14 +1,15 @@ """Some debugging functions for working with the Scrapy engine""" +from __future__ import annotations + # used in global tests code from time import time # noqa: F401 -from typing import TYPE_CHECKING, Any, List, Tuple +from typing import Any, List, Tuple -if TYPE_CHECKING: - from scrapy.core.engine import ExecutionEngine +from scrapy.core.engine import ExecutionEngine -def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: +def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]: """Return a report of the current engine status""" tests = [ "time()-engine.start_time", @@ -37,7 +38,7 @@ def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: return checks -def format_engine_status(engine: "ExecutionEngine") -> str: +def format_engine_status(engine: ExecutionEngine) -> str: checks = get_engine_status(engine) s = "Execution engine status\n\n" for test, result in checks: @@ -47,5 +48,5 @@ def format_engine_status(engine: "ExecutionEngine") -> str: return s -def print_engine_status(engine: "ExecutionEngine") -> None: +def print_engine_status(engine: ExecutionEngine) -> None: print(format_engine_status(engine)) From 6e84648c0717642b069249225857019a87de54b9 Mon Sep 17 00:00:00 2001 From: pengqiseven <134899215+pengqiseven@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:03:06 +0800 Subject: [PATCH 125/269] Fix some comments (#6285) Signed-off-by: pengqiseven Co-authored-by: pengqiseven --- sep/sep-018.rst | 2 +- tests/test_utils_python.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sep/sep-018.rst b/sep/sep-018.rst index 9ac62c090..13ab501ed 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -140,7 +140,7 @@ Example: The data flow with Spider Middleware v2 is as follows: -1. When a response arrives from the engine, it it passed through all the spider +1. When a response arrives from the engine, it is passed through all the spider middlewares (in descending order). The result of each middleware ``process_response`` is kept and then returned along with the spider callback result diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 80d2e8da1..1d1d19146 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -175,7 +175,7 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertTrue(equal_attributes(a, b, ["x", "y"])) a.y = 1 - # differente attributes + # different attributes self.assertFalse(equal_attributes(a, b, ["x", "y"])) # test callable From 642af407049a5ce8e76b7999c68333670c8c8622 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 11 Mar 2024 14:09:09 +0500 Subject: [PATCH 126/269] Full typing for scrapy/extensions, part 2. (#6279) --- docs/topics/telnetconsole.rst | 4 +- scrapy/extensions/periodic_log.py | 89 +++++++++++++++++------------ scrapy/extensions/postprocessing.py | 4 +- scrapy/extensions/spiderstate.py | 26 ++++++--- scrapy/extensions/statsmailer.py | 31 +++++++--- scrapy/extensions/telnet.py | 37 +++++++----- scrapy/extensions/throttle.py | 49 ++++++++++------ scrapy/mail.py | 12 ++-- scrapy/utils/reactor.py | 8 +-- 9 files changed, 165 insertions(+), 95 deletions(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 832829b75..0e4a8fa6c 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -172,8 +172,8 @@ TELNETCONSOLE_PORT Default: ``[6023, 6073]`` -The port range to use for the telnet console. If set to ``None`` or ``0``, a -dynamically assigned port is used. +The port range to use for the telnet console. If set to ``None``, a dynamically +assigned port is used. .. setting:: TELNETCONSOLE_HOST diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 2d557f123..9567f948a 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -1,12 +1,22 @@ +from __future__ import annotations + import logging from datetime import datetime, timezone +from json import JSONEncoder +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from twisted.internet import task -from scrapy import signals +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.statscollectors import StatsCollector from scrapy.utils.serialize import ScrapyJSONEncoder +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) @@ -15,32 +25,34 @@ class PeriodicLog: def __init__( self, - stats, - interval=60.0, - ext_stats={}, - ext_delta={}, - ext_timing_enabled=False, + stats: StatsCollector, + interval: float = 60.0, + ext_stats: Dict[str, Any] = {}, + ext_delta: Dict[str, Any] = {}, + ext_timing_enabled: bool = False, ): - self.stats = stats - self.interval = interval - self.multiplier = 60.0 / self.interval - self.task = None - self.encoder = ScrapyJSONEncoder(sort_keys=True, indent=4) - self.ext_stats_enabled = bool(ext_stats) - self.ext_stats_include = ext_stats.get("include", []) - self.ext_stats_exclude = ext_stats.get("exclude", []) - self.ext_delta_enabled = bool(ext_delta) - self.ext_delta_include = ext_delta.get("include", []) - self.ext_delta_exclude = ext_delta.get("exclude", []) - self.ext_timing_enabled = ext_timing_enabled + self.stats: StatsCollector = stats + self.interval: float = interval + self.multiplier: float = 60.0 / self.interval + self.task: Optional[task.LoopingCall] = None + self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) + self.ext_stats_enabled: bool = bool(ext_stats) + self.ext_stats_include: List[str] = ext_stats.get("include", []) + self.ext_stats_exclude: List[str] = ext_stats.get("exclude", []) + self.ext_delta_enabled: bool = bool(ext_delta) + self.ext_delta_include: List[str] = ext_delta.get("include", []) + self.ext_delta_exclude: List[str] = ext_delta.get("exclude", []) + self.ext_timing_enabled: bool = ext_timing_enabled @classmethod - def from_crawler(cls, crawler): - interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") + def from_crawler(cls, crawler: Crawler) -> Self: + interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL") if not interval: raise NotConfigured try: - ext_stats = crawler.settings.getdict("PERIODIC_LOG_STATS") + ext_stats: Optional[Dict[str, Any]] = crawler.settings.getdict( + "PERIODIC_LOG_STATS" + ) except (TypeError, ValueError): ext_stats = ( {"enabled": True} @@ -48,7 +60,9 @@ class PeriodicLog: else None ) try: - ext_delta = crawler.settings.getdict("PERIODIC_LOG_DELTA") + ext_delta: Optional[Dict[str, Any]] = crawler.settings.getdict( + "PERIODIC_LOG_DELTA" + ) except (TypeError, ValueError): ext_delta = ( {"enabled": True} @@ -56,11 +70,14 @@ class PeriodicLog: else None ) - ext_timing_enabled = crawler.settings.getbool( + ext_timing_enabled: bool = crawler.settings.getbool( "PERIODIC_LOG_TIMING_ENABLED", False ) if not (ext_stats or ext_delta or ext_timing_enabled): raise NotConfigured + assert crawler.stats + assert ext_stats is not None + assert ext_delta is not None o = cls( crawler.stats, interval, @@ -72,16 +89,16 @@ class PeriodicLog: crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_opened(self, spider): - self.time_prev = datetime.now(tz=timezone.utc) - self.delta_prev = {} - self.stats_prev = {} + def spider_opened(self, spider: Spider) -> None: + self.time_prev: datetime = datetime.now(tz=timezone.utc) + self.delta_prev: Dict[str, Union[int, float]] = {} + self.stats_prev: Dict[str, Union[int, float]] = {} self.task = task.LoopingCall(self.log) self.task.start(self.interval) - def log(self): - data = {} + def log(self) -> None: + data: Dict[str, Any] = {} if self.ext_timing_enabled: data.update(self.log_timing()) if self.ext_delta_enabled: @@ -90,8 +107,8 @@ class PeriodicLog: data.update(self.log_crawler_stats()) logger.info(self.encoder.encode(data)) - def log_delta(self): - num_stats = { + def log_delta(self) -> Dict[str, Any]: + num_stats: Dict[str, Union[int, float]] = { k: v for k, v in self.stats._stats.items() if isinstance(v, (int, float)) @@ -101,7 +118,7 @@ class PeriodicLog: self.delta_prev = num_stats return {"delta": delta} - def log_timing(self): + def log_timing(self) -> Dict[str, Any]: now = datetime.now(tz=timezone.utc) time = { "log_interval": self.interval, @@ -113,7 +130,7 @@ class PeriodicLog: self.time_prev = now return {"time": time} - def log_crawler_stats(self): + def log_crawler_stats(self) -> Dict[str, Any]: stats = { k: v for k, v in self.stats._stats.items() @@ -121,7 +138,9 @@ class PeriodicLog: } return {"stats": stats} - def param_allowed(self, stat_name, include, exclude): + def param_allowed( + self, stat_name: str, include: List[str], exclude: List[str] + ) -> bool: if not include and not exclude: return True for p in exclude: @@ -134,7 +153,7 @@ class PeriodicLog: return True return False - def spider_closed(self, spider, reason): + def spider_closed(self, spider: Spider, reason: str) -> None: self.log() if self.task and self.task.running: self.task.stop() diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index f8b59827b..7ffbd8bc3 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -6,7 +6,7 @@ from bz2 import BZ2File from gzip import GzipFile from io import IOBase from lzma import LZMAFile -from typing import Any, BinaryIO, Dict, List +from typing import Any, BinaryIO, Dict, List, cast from scrapy.utils.misc import load_object @@ -142,7 +142,7 @@ class PostProcessingManager(IOBase): :return: returns number of bytes written :rtype: int """ - return self.head_plugin.write(data) + return cast(int, self.head_plugin.write(data)) def tell(self) -> int: return self.file.tell() diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 43359401b..c6eb20277 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -1,19 +1,27 @@ +from __future__ import annotations + import pickle # nosec from pathlib import Path +from typing import TYPE_CHECKING, Optional -from scrapy import signals +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class SpiderState: """Store and load spider state during a scraping job""" - def __init__(self, jobdir=None): - self.jobdir = jobdir + def __init__(self, jobdir: Optional[str] = None): + self.jobdir: Optional[str] = jobdir @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: jobdir = job_dir(crawler.settings) if not jobdir: raise NotConfigured @@ -23,18 +31,20 @@ class SpiderState: crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened) return obj - def spider_closed(self, spider): + def spider_closed(self, spider: Spider) -> None: if self.jobdir: with Path(self.statefn).open("wb") as f: + assert hasattr(spider, "state") # set in spider_opened pickle.dump(spider.state, f, protocol=4) - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: if self.jobdir and Path(self.statefn).exists(): with Path(self.statefn).open("rb") as f: - spider.state = pickle.load(f) # nosec + spider.state = pickle.load(f) # type: ignore[attr-defined] # nosec else: - spider.state = {} + spider.state = {} # type: ignore[attr-defined] @property def statefn(self) -> str: + assert self.jobdir return str(Path(self.jobdir, "spider.state")) diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 58610c25e..20b8f910c 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -4,28 +4,41 @@ StatsMailer extension sends an email when a spider finishes scraping. Use STATSMAILER_RCPTS setting to enable and give the recipient mail address """ -from scrapy import signals +from __future__ import annotations + +from typing import TYPE_CHECKING, List, Optional + +from twisted.internet.defer import Deferred + +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender +from scrapy.statscollectors import StatsCollector + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class StatsMailer: - def __init__(self, stats, recipients, mail): - self.stats = stats - self.recipients = recipients - self.mail = mail + def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender): + self.stats: StatsCollector = stats + self.recipients: List[str] = recipients + self.mail: MailSender = mail @classmethod - def from_crawler(cls, crawler): - recipients = crawler.settings.getlist("STATSMAILER_RCPTS") + def from_crawler(cls, crawler: Crawler) -> Self: + recipients: List[str] = crawler.settings.getlist("STATSMAILER_RCPTS") if not recipients: raise NotConfigured - mail = MailSender.from_settings(crawler.settings) + mail: MailSender = MailSender.from_settings(crawler.settings) + assert crawler.stats o = cls(crawler.stats, recipients, mail) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_closed(self, spider): + def spider_closed(self, spider: Spider) -> Optional[Deferred]: spider_stats = self.stats.get_stats(spider) body = "Global stats\n\n" body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items()) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index c92b7f5fe..00c69434c 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -4,13 +4,17 @@ Scrapy Telnet Console extension See documentation in docs/topics/telnetconsole.rst """ +from __future__ import annotations + import binascii import logging import os import pprint import traceback +from typing import TYPE_CHECKING, Any, Dict, List from twisted.internet import protocol +from twisted.internet.tcp import Port try: from twisted.conch import manhole, telnet @@ -22,12 +26,16 @@ except (ImportError, SyntaxError): TWISTED_CONCH_AVAILABLE = False from scrapy import signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.utils.decorators import defers from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp from scrapy.utils.trackref import print_live_refs +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) # signal to update telnet variables @@ -36,7 +44,7 @@ update_telnet_vars = object() class TelnetConsole(protocol.ServerFactory): - def __init__(self, crawler): + def __init__(self, crawler: Crawler): if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"): raise NotConfigured if not TWISTED_CONCH_AVAILABLE: @@ -44,14 +52,14 @@ class TelnetConsole(protocol.ServerFactory): "TELNETCONSOLE_ENABLED setting is True but required twisted " "modules failed to import:\n" + _TWISTED_CONCH_TRACEBACK ) - self.crawler = crawler - self.noisy = False - self.portrange = [ + self.crawler: Crawler = crawler + self.noisy: bool = False + self.portrange: List[int] = [ int(x) for x in crawler.settings.getlist("TELNETCONSOLE_PORT") ] - self.host = crawler.settings["TELNETCONSOLE_HOST"] - self.username = crawler.settings["TELNETCONSOLE_USERNAME"] - self.password = crawler.settings["TELNETCONSOLE_PASSWORD"] + self.host: str = crawler.settings["TELNETCONSOLE_HOST"] + self.username: str = crawler.settings["TELNETCONSOLE_USERNAME"] + self.password: str = crawler.settings["TELNETCONSOLE_PASSWORD"] if not self.password: self.password = binascii.hexlify(os.urandom(8)).decode("utf8") @@ -61,11 +69,11 @@ class TelnetConsole(protocol.ServerFactory): self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def start_listening(self): - self.port = listen_tcp(self.portrange, self.host, self) + def start_listening(self) -> None: + self.port: Port = listen_tcp(self.portrange, self.host, self) h = self.port.getHost() logger.info( "Telnet console listening on %(host)s:%(port)d", @@ -73,10 +81,10 @@ class TelnetConsole(protocol.ServerFactory): extra={"crawler": self.crawler}, ) - def stop_listening(self): + def stop_listening(self) -> None: self.port.stopListening() - def protocol(self): + def protocol(self) -> telnet.TelnetTransport: # type: ignore[override] class Portal: """An implementation of IPortal""" @@ -95,9 +103,10 @@ class TelnetConsole(protocol.ServerFactory): return telnet.TelnetTransport(telnet.AuthenticatingTelnetProtocol, Portal()) - def _get_telnet_vars(self): + def _get_telnet_vars(self) -> Dict[str, Any]: # Note: if you add entries here also update topics/telnetconsole.rst - telnet_vars = { + assert self.crawler.engine + telnet_vars: Dict[str, Any] = { "engine": self.crawler.engine, "spider": self.crawler.engine.spider, "slot": self.crawler.engine.slot, diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index d217c7a69..bf4e6bb63 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -1,19 +1,29 @@ -import logging +from __future__ import annotations -from scrapy import signals +import logging +from typing import TYPE_CHECKING, Optional, Tuple + +from scrapy import Request, Spider, signals +from scrapy.core.downloader import Slot +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) class AutoThrottle: - def __init__(self, crawler): - self.crawler = crawler + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler if not crawler.settings.getbool("AUTOTHROTTLE_ENABLED"): raise NotConfigured - self.debug = crawler.settings.getbool("AUTOTHROTTLE_DEBUG") - self.target_concurrency = crawler.settings.getfloat( + self.debug: bool = crawler.settings.getbool("AUTOTHROTTLE_DEBUG") + self.target_concurrency: float = crawler.settings.getfloat( "AUTOTHROTTLE_TARGET_CONCURRENCY" ) if self.target_concurrency <= 0.0: @@ -27,27 +37,29 @@ class AutoThrottle: ) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def _spider_opened(self, spider): + def _spider_opened(self, spider: Spider) -> None: self.mindelay = self._min_delay(spider) self.maxdelay = self._max_delay(spider) - spider.download_delay = self._start_delay(spider) + spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined] - def _min_delay(self, spider): + def _min_delay(self, spider: Spider) -> float: s = self.crawler.settings return getattr(spider, "download_delay", s.getfloat("DOWNLOAD_DELAY")) - def _max_delay(self, spider): + def _max_delay(self, spider: Spider) -> float: return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY") - def _start_delay(self, spider): + def _start_delay(self, spider: Spider) -> float: return max( self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") ) - def _response_downloaded(self, response, request, spider): + def _response_downloaded( + self, response: Response, request: Request, spider: Spider + ) -> None: key, slot = self._get_slot(request, spider) latency = request.meta.get("download_latency") if latency is None or slot is None: @@ -74,11 +86,16 @@ class AutoThrottle: extra={"spider": spider}, ) - def _get_slot(self, request, spider): - key = request.meta.get("download_slot") + def _get_slot( + self, request: Request, spider: Spider + ) -> Tuple[Optional[str], Optional[Slot]]: + key: Optional[str] = request.meta.get("download_slot") + if key is None: + return None, None + assert self.crawler.engine return key, self.crawler.engine.downloader.slots.get(key) - def _adjust_delay(self, slot, latency, response): + def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None: """Define delay adjustment policy""" # If a server needs `latency` seconds to respond then diff --git a/scrapy/mail.py b/scrapy/mail.py index dce33fcdf..7cb5ef454 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -14,10 +14,11 @@ from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from twisted import version as twisted_version -from twisted.internet import defer, ssl +from twisted.internet import ssl +from twisted.internet.defer import Deferred from twisted.python.versions import Version from scrapy.settings import BaseSettings @@ -85,9 +86,10 @@ class MailSender: mimetype="text/plain", charset=None, _callback=None, - ): + ) -> Optional[Deferred]: from twisted.internet import reactor + msg: MIMEBase if attachs: msg = MIMEMultipart() else: @@ -134,7 +136,7 @@ class MailSender: "mailattachs": len(attachs), }, ) - return + return None dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8")) dfd.addCallbacks( @@ -178,7 +180,7 @@ class MailSender: from twisted.internet import reactor msg = BytesIO(msg) - d = defer.Deferred() + d = Deferred() factory = self._create_sender_factory(to_addrs, msg, d) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index ad3d1d8bc..6cde49bfe 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -2,17 +2,19 @@ import asyncio import sys from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from contextlib import suppress -from typing import Any, Callable, Dict, Optional, Sequence, Type +from typing import Any, Callable, Dict, List, Optional, Sequence, Type from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error from twisted.internet.base import DelayedCall +from twisted.internet.protocol import ServerFactory +from twisted.internet.tcp import Port from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import load_object -def listen_tcp(portrange, host, factory): +def listen_tcp(portrange: List[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor @@ -20,8 +22,6 @@ def listen_tcp(portrange, host, factory): raise ValueError(f"invalid portrange: {portrange}") if not portrange: return reactor.listenTCP(0, factory, interface=host) - if not hasattr(portrange, "__iter__"): - return reactor.listenTCP(portrange, factory, interface=host) if len(portrange) == 1: return reactor.listenTCP(portrange[0], factory, interface=host) for x in range(portrange[0], portrange[1] + 1): From ab5ea32ffd9cbea22d0fb10ece5258cea207dd61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 11 Mar 2024 23:26:19 +0100 Subject: [PATCH 127/269] Fix WindowsRunSpiderCommandTest skip outside Windows for older Twisted. --- tests/test_commands.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index febad21da..ff308c5ac 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -987,10 +987,14 @@ class MySpider(scrapy.Spider): self.assertIn("The value of FOO is 42", log) -@skipIf(platform.system() != "Windows", "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = "myspider.pyw" + def setUp(self): + # https://github.com/scrapy/scrapy/issues/6286 + if platform.system() != "Windows": + raise unittest.SkipTest("Windows required for .pyw files") + def test_start_requests_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") self.assertIn("start_requests", log) From 188d9a8bb363ab3ff37dbb6020354afbc72ec02d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 11 Mar 2024 23:28:57 +0100 Subject: [PATCH 128/269] Remove unnecessary comment --- tests/test_commands.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index ff308c5ac..ae8289ba7 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -991,7 +991,6 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = "myspider.pyw" def setUp(self): - # https://github.com/scrapy/scrapy/issues/6286 if platform.system() != "Windows": raise unittest.SkipTest("Windows required for .pyw files") From e72de11f55dc5f37d449385f20d2ce4c504914d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 12 Mar 2024 09:29:10 +0100 Subject: [PATCH 129/269] Add super --- tests/test_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index ae8289ba7..b9d468c66 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -993,6 +993,7 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): def setUp(self): if platform.system() != "Windows": raise unittest.SkipTest("Windows required for .pyw files") + return super().setUp() def test_start_requests_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") From d7581c6b41e97fc09c011b089cf34ddd62f41876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 12 Mar 2024 09:44:29 +0100 Subject: [PATCH 130/269] Allow disabling the AutoThrottle extension for a given slot (#6246) --- docs/topics/autothrottle.rst | 12 +++++++++++ docs/topics/settings.rst | 12 +++++++++-- scrapy/core/downloader/__init__.py | 19 ++++++++++++++---- scrapy/extensions/throttle.py | 2 +- tests/test_core_downloader.py | 3 ++- tests/test_downloaderslotssettings.py | 29 ++++++++++++++++++++++++++- tests/test_extension_throttle.py | 19 ++++++++++-------- 7 files changed, 79 insertions(+), 17 deletions(-) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 5370d77b3..8a13b8976 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -47,6 +47,18 @@ effect, but there are some important differences: AutoThrottle doesn't have these issues. +Disabling throttling on a downloader slot +========================================= + +It is possible to disable AutoThrottle for a specific download slot at run time +by setting its ``throttle`` attribute to ``False``, e.g. using +:setting:`DOWNLOAD_SLOTS`. + +Note, however, that AutoThrottle still determines the starting delay of every +slot by setting the ``download_delay`` attribute on the running spider. You +might want to set a custom value for the ``delay`` attribute of the slot, e.g. +using :setting:`DOWNLOAD_SLOTS`. + Throttling algorithm ==================== diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 439aedc18..2bd9cf1ed 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -835,7 +835,7 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2: .. setting:: DOWNLOAD_SLOTS DOWNLOAD_SLOTS ----------------- +-------------- Default: ``{}`` @@ -844,7 +844,12 @@ Allows to define concurrency/delay parameters on per slot (domain) basis: .. code-block:: python DOWNLOAD_SLOTS = { - "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False}, + "quotes.toscrape.com": { + "concurrency": 1, + "delay": 2, + "randomize_delay": False, + "throttle": False, + }, "books.toscrape.com": {"delay": 3, "randomize_delay": False}, } @@ -856,6 +861,9 @@ Allows to define concurrency/delay parameters on per slot (domain) basis: - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` + There is no global setting for ``throttle``, whose default value is + ``None``. + .. setting:: DOWNLOAD_TIMEOUT diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 666282856..ecd3e8b56 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -2,7 +2,7 @@ import random from collections import deque from datetime import datetime from time import time -from typing import TYPE_CHECKING, Any, Deque, Dict, Set, Tuple, cast +from typing import TYPE_CHECKING, Any, Deque, Dict, Optional, Set, Tuple, cast from twisted.internet import task from twisted.internet.defer import Deferred @@ -24,10 +24,18 @@ if TYPE_CHECKING: class Slot: """Downloader slot""" - def __init__(self, concurrency: int, delay: float, randomize_delay: bool): + def __init__( + self, + concurrency: int, + delay: float, + randomize_delay: bool, + *, + throttle: Optional[bool] = None, + ): self.concurrency: int = concurrency self.delay: float = delay self.randomize_delay: bool = randomize_delay + self.throttle = throttle self.active: Set[Request] = set() self.queue: Deque[Tuple[Request, Deferred]] = deque() @@ -52,13 +60,15 @@ class Slot: return ( f"{cls_name}(concurrency={self.concurrency!r}, " f"delay={self.delay:.2f}, " - f"randomize_delay={self.randomize_delay!r})" + f"randomize_delay={self.randomize_delay!r}, " + f"throttle={self.throttle!r})" ) def __str__(self) -> str: return ( f"" @@ -127,7 +137,8 @@ class Downloader: slot_settings.get("delay", delay), ) randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) - new_slot = Slot(conc, delay, randomize_delay) + throttle = slot_settings.get("throttle", None) + new_slot = Slot(conc, delay, randomize_delay, throttle=throttle) self.slots[key] = new_slot return key, self.slots[key] diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index bf4e6bb63..217e61a81 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -62,7 +62,7 @@ class AutoThrottle: ) -> None: key, slot = self._get_slot(request, spider) latency = request.meta.get("download_latency") - if latency is None or slot is None: + if latency is None or slot is None or slot.throttle is False: return olddelay = slot.delay diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 9a6e9e4ff..81cff4947 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -7,5 +7,6 @@ class SlotTest(unittest.TestCase): def test_repr(self): slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) self.assertEqual( - repr(slot), "Slot(concurrency=8, delay=0.10, randomize_delay=True)" + repr(slot), + "Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)", ) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 9d4072d19..ea8c5b4f0 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -3,8 +3,10 @@ import time from twisted.internet import defer from twisted.trial.unittest import TestCase +from scrapy import Request +from scrapy.core.downloader import Downloader, Slot from scrapy.crawler import CrawlerRunner -from scrapy.http import Request +from scrapy.utils.test import get_crawler from tests.mockserver import MockServer from tests.spiders import MetaSpider @@ -20,6 +22,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): "concurrency": 1, "delay": 2, "randomize_delay": False, + "throttle": False, }, "books.toscrape.com": {"delay": 3, "randomize_delay": False}, }, @@ -70,3 +73,27 @@ class CrawlTestCase(TestCase): } self.assertTrue(max(list(error_delta.values())) < tolerance) + + +def test_params(): + params = { + "concurrency": 1, + "delay": 2, + "randomize_delay": False, + "throttle": False, + } + settings = { + "DOWNLOAD_SLOTS": { + "example.com": params, + }, + } + crawler = get_crawler(settings_dict=settings) + downloader = Downloader(crawler) + downloader._slot_gc_loop.stop() # Prevent an unclean reactor. + request = Request("https://example.com") + _, actual = downloader._get_slot(request, spider=None) + expected = Slot(**params) + for param in params: + assert getattr(expected, param) == getattr( + actual, param + ), f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}" diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index dae4ea966..722a05c26 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -157,16 +157,17 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) @pytest.mark.parametrize( - ("meta", "slot"), + ("meta", "slot", "throttle"), ( - ({}, None), - ({"download_latency": 1.0}, None), - ({"download_slot": "foo"}, None), - ({"download_slot": "foo"}, "foo"), - ({"download_latency": 1.0, "download_slot": "foo"}, None), + ({}, None, None), + ({"download_latency": 1.0}, None, None), + ({"download_slot": "foo"}, None, None), + ({"download_slot": "foo"}, "foo", None), + ({"download_latency": 1.0, "download_slot": "foo"}, None, None), + ({"download_latency": 1.0, "download_slot": "foo"}, "foo", False), ), ) -def test_skipped(meta, slot): +def test_skipped(meta, slot, throttle): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) spider = TestSpider() @@ -177,7 +178,9 @@ def test_skipped(meta, slot): crawler.engine.downloader = Mock() crawler.engine.downloader.slots = {} if slot is not None: - crawler.engine.downloader.slots[slot] = object() + _slot = Mock() + _slot.throttle = throttle + crawler.engine.downloader.slots[slot] = _slot at._adjust_delay = None # Raise exception if called. at._response_downloaded(None, request, spider) From 4460d3ed9631f8409c24f78f1abb36345967b5d5 Mon Sep 17 00:00:00 2001 From: Lucas Belo <144740771+lucas-belo@users.noreply.github.com> Date: Wed, 13 Mar 2024 03:22:48 -0300 Subject: [PATCH 131/269] Remove tests/requirements.txt and refactor extra deps (#6272) Co-authored-by: lucasbelo777 --- conftest.py | 18 ++++++------------ tests/requirements.txt | 17 ----------------- tox.ini | 32 ++++++++++++++++++++++++++++---- 3 files changed, 34 insertions(+), 33 deletions(-) delete mode 100644 tests/requirements.txt diff --git a/conftest.py b/conftest.py index 2bfa46f5a..2ab3dffd4 100644 --- a/conftest.py +++ b/conftest.py @@ -1,10 +1,6 @@ -import platform -import sys from pathlib import Path import pytest -from twisted import version as twisted_version -from twisted.python.versions import Version from twisted.web.http import H2_ENABLED from scrapy.utils.reactor import install_reactor @@ -85,14 +81,12 @@ def only_not_asyncio(request, reactor_pytest): def requires_uvloop(request): if not request.node.get_closest_marker("requires_uvloop"): return - if sys.implementation.name == "pypy": - pytest.skip("uvloop does not support pypy properly") - if platform.system() == "Windows": - pytest.skip("uvloop does not support Windows") - if twisted_version == Version("twisted", 21, 2, 0): - pytest.skip("https://twistedmatrix.com/trac/ticket/10106") - if sys.version_info >= (3, 12): - pytest.skip("uvloop doesn't support Python 3.12 yet") + try: + import uvloop + + del uvloop + except ImportError: + pytest.skip("uvloop is not installed") def pytest_configure(config): diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index ca5f6ddbd..000000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Tests requirements -attrs -pexpect >= 4.8.0 -pyftpdlib >= 1.5.8 -pytest -pytest-cov==4.0.0 -pytest-xdist -sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 -testfixtures -uvloop; platform_system != "Windows" - -bpython # optional for shell wrapper tests -brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests -brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests -zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests -ipython -pywin32; sys_platform == "win32" diff --git a/tox.ini b/tox.ini index 237aa489c..b5effb527 100644 --- a/tox.ini +++ b/tox.ini @@ -7,9 +7,23 @@ envlist = pre-commit,pylint,typing,py minversion = 1.7.0 +[test-requirements] +deps = + attrs + pexpect >= 4.8.0 + pyftpdlib >= 1.5.8 + pygments + pytest + pytest-cov==4.0.0 + pytest-xdist + sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 + testfixtures + pywin32; sys_platform == "win32" + [testenv] deps = - -rtests/requirements.txt + {[test-requirements]deps} + # mitmproxy does not support PyPy mitmproxy; implementation_name != 'pypy' passenv = @@ -81,7 +95,7 @@ deps = w3lib==1.17.0 zope.interface==5.1.0 lxml==4.4.1 - -rtests/requirements.txt + {[test-requirements]deps} # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies # above, hence we do not install it in pinned environments at the moment @@ -124,8 +138,12 @@ deps = robotexclusionrulesparser Pillow Twisted[http2] - brotli - zstandard + uvloop; platform_system != "Windows" + bpython # optional for shell wrapper tests + brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests + brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests + zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests + ipython [testenv:extra-deps-pinned] basepython = python3.8 @@ -136,6 +154,12 @@ deps = Pillow==7.1.0 robotexclusionrulesparser==1.6.2 brotlipy + uvloop==0.14.0; platform_system != "Windows" + bpython==0.7.1 + zstandard==0.1; implementation_name != 'pypy' + ipython==2.0.0 + brotli==0.5.2; implementation_name != 'pypy' + brotlicffi==0.8.0; implementation_name == 'pypy' install_command = {[pinned]install_command} setenv = {[pinned]setenv} From 8d917c0b55cbeebd284f7446ccc0f90af60729ac Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 14 Mar 2024 17:44:57 +0500 Subject: [PATCH 132/269] Run black. --- tests/test_command_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index effafae54..b0f1cd38a 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -1,6 +1,6 @@ import sys from io import StringIO -from unittest.mock import Mock, call, PropertyMock, patch +from unittest.mock import Mock, PropertyMock, call, patch from scrapy.commands.check import Command, TextTestResult from tests.test_commands import CommandTest From f7bf3f726e3f19bf68b5e7e460f116850896eb42 Mon Sep 17 00:00:00 2001 From: igeni Date: Mon, 1 Apr 2024 16:37:23 +0300 Subject: [PATCH 133/269] modified string's concat to f-strings (#6296) --- tests/test_selector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_selector.py b/tests/test_selector.py index 85527bba9..1b5f3f018 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -80,10 +80,10 @@ class SelectorTestCase(unittest.TestCase): meta = ( '' ) - head = "" + meta + "" + head = f"{meta}" body_content = '\xa3' - body = "" + body_content + "" - html = "" + head + body + "" + body = f"{body_content}" + html = f"{head}{body}" encoding = "utf-8" html_utf8 = html.encode(encoding) From 7b37dcd80d3783d3a21ff524e572019ff2b9e0ed Mon Sep 17 00:00:00 2001 From: Lorenzo Verardo Date: Thu, 4 Apr 2024 12:22:50 +0200 Subject: [PATCH 134/269] Handle robots.txt files not UTF-8 encoded --- scrapy/robotstxt.py | 2 +- tests/test_robotstxt_interface.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 6ea2bfd97..ad06137e2 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -23,7 +23,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): if to_native_str_type: robotstxt_body = to_unicode(robotstxt_body) else: - robotstxt_body = robotstxt_body.decode("utf-8") + robotstxt_body = robotstxt_body.decode("utf-8", errors="ignore") except UnicodeDecodeError: # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it. # Switch to 'allow all' state. diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index d7a923085..6ad30deed 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,5 +1,7 @@ from twisted.trial import unittest +from scrapy.robotstxt import decode_robotstxt + def reppy_available(): # check if reppy parser is installed @@ -141,6 +143,25 @@ class BaseRobotParserTest: ) +class DecodeRobotsTxtTest(unittest.TestCase): + def test_native_string_conversion(self): + robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8") + decoded_content = decode_robotstxt( + robotstxt_body, spider=None, to_native_str_type=True + ) + self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") + + def test_decode_utf8(self): + robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8") + decoded_content = decode_robotstxt(robotstxt_body, spider=None) + self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") + + def test_decode_non_utf8(self): + robotstxt_body = b"User-agent: *\n\xFFDisallow: /\n" + decoded_content = decode_robotstxt(robotstxt_body, spider=None) + self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") + + class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import PythonRobotParser From 48c5a8c98f545e35708a580e724d0b8e1ada5e6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Ruiz?= Date: Mon, 8 Apr 2024 11:47:53 +0200 Subject: [PATCH 135/269] Fix WrappedRequest.get_header raising TypeError if default is None (#6310) --- scrapy/http/cookies.py | 3 ++- tests/test_http_cookies.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 2595f328e..72855bad5 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -170,7 +170,8 @@ class WrappedRequest: return name in self.request.headers def get_header(self, name, default=None): - return to_unicode(self.request.headers.get(name, default), errors="replace") + value = self.request.headers.get(name, default) + return to_unicode(value, errors="replace") if value is not None else None def header_items(self): return [ diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 8b5554914..932644320 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -43,6 +43,13 @@ class WrappedRequestTest(TestCase): def test_get_header(self): self.assertEqual(self.wrapped.get_header("content-type"), "text/html") self.assertEqual(self.wrapped.get_header("xxxxx", "def"), "def") + self.assertEqual(self.wrapped.get_header("xxxxx"), None) + wrapped = WrappedRequest( + Request( + "http://www.example.com/page.html", headers={"empty-binary-header": b""} + ) + ) + self.assertEqual(wrapped.get_header("empty-binary-header"), "") def test_header_items(self): self.assertEqual(self.wrapped.header_items(), [("Content-Type", ["text/html"])]) From 1d11ea3a54607b436f9a88f07911902a4882f0e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 11 Apr 2024 12:19:32 +0200 Subject: [PATCH 136/269] Update practices.rst --- docs/topics/practices.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index b1b8c9e9c..cd359b147 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -289,7 +289,8 @@ Here are some tips to keep in mind when dealing with these kinds of sites: services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a super proxy that you can attach your own proxies to. * use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy - plugin `__ + plugin `__ and additional + features, like `AI web scraping `__ If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. From 5f67c01d1d2cc8a0104361b0323d755e22ed93dc Mon Sep 17 00:00:00 2001 From: TechVest <166724172+TechVest@users.noreply.github.com> Date: Wed, 17 Apr 2024 16:56:26 +0800 Subject: [PATCH 137/269] chore: fix some typos in comments (#6317) Signed-off-by: TechVest --- .git-blame-ignore-revs | 4 ++-- docs/topics/broad-crawls.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index dbcebfa0a..a9fc3dd68 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,7 +1,7 @@ # .git-blame-ignore-revs # adding black formatter to all the code e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d -# re applying black to the code with default line length +# reapplying black to the code with default line length 303f0a70fcf8067adf0a909c2096a5009162383a -# reaplying black again and removing line length on pre-commit black config +# reapplying black again and removing line length on pre-commit black config c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962 \ No newline at end of file diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 8be89feb2..750aae554 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -116,7 +116,7 @@ Reduce log level When doing broad crawls you are often only interested in the crawl rates you get and any errors found. These stats are reported by Scrapy when using the ``INFO`` log level. In order to save CPU (and log storage requirements) you -should not use ``DEBUG`` log level when preforming large broad crawls in +should not use ``DEBUG`` log level when performing large broad crawls in production. Using ``DEBUG`` level when developing your (broad) crawler may be fine though. From b1fe97dc6c8509d58b29c61cf7801eeee1b409a9 Mon Sep 17 00:00:00 2001 From: kokobhara <146670393+kokobhara@users.noreply.github.com> Date: Wed, 17 Apr 2024 16:44:57 +0530 Subject: [PATCH 138/269] Fix test expectations (#6316) --- tests/test_utils_python.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 1d1d19146..4c60deafe 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -239,8 +239,11 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"]) if platform.python_implementation() == "CPython": - # doesn't work on CPython: https://bugs.python.org/issue42785 - self.assertEqual(get_func_args(operator.itemgetter(2)), []) + # This didn't work on older versions of CPython: https://github.com/python/cpython/issues/86951 + self.assertIn( + get_func_args(operator.itemgetter(2), stripself=True), + [[], ["args", "kwargs"]], + ) elif platform.python_implementation() == "PyPy": self.assertEqual( get_func_args(operator.itemgetter(2), stripself=True), ["obj"] From a166e9739962ec7cca8a655e5f20a18a1bce7d14 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 12:39:22 +0500 Subject: [PATCH 139/269] Remove the auto-generated copyright years from the docs footer. (#6322) --- docs/conf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 399078010..dcd2c9a3a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,7 +10,6 @@ # serve to show the default. import sys -from datetime import datetime from pathlib import Path # If your extensions are in another directory, add it here. If the directory @@ -48,7 +47,7 @@ master_doc = "index" # General information about the project. project = "Scrapy" -copyright = f"2008–{datetime.now().year}, Scrapy developers" +copyright = "Scrapy developers" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From 57acad3c38602f4399c307c2c002f9eddde97cbc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 12:43:45 +0500 Subject: [PATCH 140/269] Full typing for scrapy/extensions, part 3. (#6325) --- scrapy/extensions/feedexport.py | 331 +++++++++++++++++++--------- scrapy/extensions/httpcache.py | 153 ++++++++----- scrapy/extensions/postprocessing.py | 4 +- scrapy/utils/iterators.py | 6 +- tox.ini | 14 +- 5 files changed, 328 insertions(+), 180 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 3b0dd804e..97f39afe7 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -4,6 +4,8 @@ Feed Exports extension See documentation in docs/topics/feed-exports.rst """ +from __future__ import annotations + import logging import re import sys @@ -11,18 +13,36 @@ import warnings from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Type, Union +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Protocol, + Tuple, + Type, + TypeVar, + Union, + cast, +) from urllib.parse import unquote, urlparse -from twisted.internet import defer, threads -from twisted.internet.defer import DeferredList +from twisted.internet import threads +from twisted.internet.defer import Deferred, DeferredList, maybeDeferred +from twisted.python.failure import Failure from w3lib.url import file_uri_to_path from zope.interface import Interface, implementer from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import BaseItemExporter from scrapy.extensions.postprocessing import PostProcessingManager +from scrapy.settings import BaseSettings, Settings from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.defer import maybe_deferred_to_future @@ -32,6 +52,12 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values +if TYPE_CHECKING: + from _typeshed import OpenBinaryMode + + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) try: @@ -41,8 +67,19 @@ try: except ImportError: IS_BOTO3_AVAILABLE = False +UriParamsCallableT = Callable[[Dict[str, Any], Spider], Optional[Dict[str, Any]]] -def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): +_StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol") + + +def build_storage( + builder: Callable[..., _StorageT], + uri: str, + *args: Any, + feed_options: Optional[Dict[str, Any]] = None, + preargs: Iterable[Any] = (), + **kwargs: Any, +) -> _StorageT: kwargs["feed_options"] = feed_options return builder(*preargs, uri, *args, **kwargs) @@ -56,10 +93,10 @@ class ItemFilter: :type feed_options: dict """ - feed_options: Optional[dict] - item_classes: Tuple + feed_options: Optional[Dict[str, Any]] + item_classes: Tuple[type, ...] - def __init__(self, feed_options: Optional[dict]) -> None: + def __init__(self, feed_options: Optional[Dict[str, Any]]) -> None: self.feed_options = feed_options if feed_options is not None: self.item_classes = tuple( @@ -98,28 +135,49 @@ class IFeedStorage(Interface): """Store the given file stream""" +class FeedStorageProtocol(Protocol): + """Reimplementation of ``IFeedStorage`` that can be used in type hints.""" + + def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None): + """Initialize the storage with the parameters given in the URI and the + feed-specific options (see :setting:`FEEDS`)""" + + def open(self, spider: Spider) -> IO[bytes]: + """Open the storage for the given spider. It must return a file-like + object that will be used for the exporters""" + + def store(self, file: IO[bytes]) -> Optional[Deferred]: + """Store the given file stream""" + + @implementer(IFeedStorage) class BlockingFeedStorage: - def open(self, spider): + def open(self, spider: Spider) -> IO[bytes]: path = spider.crawler.settings["FEED_TEMPDIR"] if path and not Path(path).is_dir(): raise OSError("Not a Directory: " + str(path)) return NamedTemporaryFile(prefix="feed-", dir=path) - def store(self, file): + def store(self, file: IO[bytes]) -> Optional[Deferred]: return threads.deferToThread(self._store_in_thread, file) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: raise NotImplementedError @implementer(IFeedStorage) class StdoutFeedStorage: - def __init__(self, uri, _stdout=None, *, feed_options=None): + def __init__( + self, + uri: str, + _stdout: Optional[IO[bytes]] = None, + *, + feed_options: Optional[Dict[str, Any]] = None, + ): if not _stdout: _stdout = sys.stdout.buffer - self._stdout = _stdout + self._stdout: IO[bytes] = _stdout if feed_options and feed_options.get("overwrite", False) is True: logger.warning( "Standard output (stdout) storage does not support " @@ -128,54 +186,58 @@ class StdoutFeedStorage: "it to False." ) - def open(self, spider): + def open(self, spider: Spider) -> IO[bytes]: return self._stdout - def store(self, file): + def store(self, file: IO[bytes]) -> Optional[Deferred]: pass @implementer(IFeedStorage) class FileFeedStorage: - def __init__(self, uri, *, feed_options=None): - self.path = file_uri_to_path(uri) + def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None): + self.path: str = file_uri_to_path(uri) feed_options = feed_options or {} - self.write_mode = "wb" if feed_options.get("overwrite", False) else "ab" + self.write_mode: OpenBinaryMode = ( + "wb" if feed_options.get("overwrite", False) else "ab" + ) - def open(self, spider) -> IO[Any]: + def open(self, spider: Spider) -> IO[bytes]: dirname = Path(self.path).parent if dirname and not dirname.exists(): dirname.mkdir(parents=True) return Path(self.path).open(self.write_mode) - def store(self, file): + def store(self, file: IO[bytes]) -> Optional[Deferred]: file.close() + return None class S3FeedStorage(BlockingFeedStorage): def __init__( self, - uri, - access_key=None, - secret_key=None, - acl=None, - endpoint_url=None, + uri: str, + access_key: Optional[str] = None, + secret_key: Optional[str] = None, + acl: Optional[str] = None, + endpoint_url: Optional[str] = None, *, - feed_options=None, - session_token=None, - region_name=None, + feed_options: Optional[Dict[str, Any]] = None, + session_token: Optional[str] = None, + region_name: Optional[str] = None, ): if not is_botocore_available(): raise NotConfigured("missing botocore library") u = urlparse(uri) - self.bucketname = u.hostname - self.access_key = u.username or access_key - self.secret_key = u.password or secret_key - self.session_token = session_token - self.keyname = u.path[1:] # remove first "/" - self.acl = acl - self.endpoint_url = endpoint_url - self.region_name = region_name + assert u.hostname + self.bucketname: str = u.hostname + self.access_key: Optional[str] = u.username or access_key + self.secret_key: Optional[str] = u.password or secret_key + self.session_token: Optional[str] = session_token + self.keyname: str = u.path[1:] # remove first "/" + self.acl: Optional[str] = acl + self.endpoint_url: Optional[str] = endpoint_url + self.region_name: Optional[str] = region_name if IS_BOTO3_AVAILABLE: import boto3.session @@ -218,7 +280,13 @@ class S3FeedStorage(BlockingFeedStorage): ) @classmethod - def from_crawler(cls, crawler, uri, *, feed_options=None): + def from_crawler( + cls, + crawler: Crawler, + uri: str, + *, + feed_options: Optional[Dict[str, Any]] = None, + ) -> Self: return build_storage( cls, uri, @@ -231,8 +299,9 @@ class S3FeedStorage(BlockingFeedStorage): feed_options=feed_options, ) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) + kwargs: Dict[str, Any] if IS_BOTO3_AVAILABLE: kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} self.s3_client.upload_fileobj( @@ -247,22 +316,23 @@ class S3FeedStorage(BlockingFeedStorage): class GCSFeedStorage(BlockingFeedStorage): - def __init__(self, uri, project_id, acl): - self.project_id = project_id - self.acl = acl + def __init__(self, uri: str, project_id: Optional[str], acl: Optional[str]): + self.project_id: Optional[str] = project_id + self.acl: Optional[str] = acl u = urlparse(uri) - self.bucket_name = u.hostname - self.blob_name = u.path[1:] # remove first "/" + assert u.hostname + self.bucket_name: str = u.hostname + self.blob_name: str = u.path[1:] # remove first "/" @classmethod - def from_crawler(cls, crawler, uri): + def from_crawler(cls, crawler: Crawler, uri: str) -> Self: return cls( uri, crawler.settings["GCS_PROJECT_ID"], crawler.settings["FEED_STORAGE_GCS_ACL"] or None, ) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) from google.cloud.storage import Client @@ -292,7 +362,13 @@ class FTPFeedStorage(BlockingFeedStorage): self.overwrite: bool = not feed_options or feed_options.get("overwrite", True) @classmethod - def from_crawler(cls, crawler, uri, *, feed_options=None): + def from_crawler( + cls, + crawler: Crawler, + uri: str, + *, + feed_options: Optional[Dict[str, Any]] = None, + ) -> Self: return build_storage( cls, uri, @@ -300,7 +376,7 @@ class FTPFeedStorage(BlockingFeedStorage): feed_options=feed_options, ) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: ftp_store_file( path=self.path, file=file, @@ -316,46 +392,51 @@ class FTPFeedStorage(BlockingFeedStorage): class FeedSlot: def __init__( self, - storage, - uri, - format, - store_empty, - batch_id, - uri_template, - filter, - feed_options, - spider, + storage: FeedStorageProtocol, + uri: str, + format: str, + store_empty: bool, + batch_id: int, + uri_template: str, + filter: ItemFilter, + feed_options: Dict[str, Any], + spider: Spider, exporters: Dict[str, Type[BaseItemExporter]], - settings, - crawler, + settings: BaseSettings, + crawler: Crawler, ): - self.file = None + self.file: Optional[IO[bytes]] = None self.exporter: Optional[BaseItemExporter] = None - self.storage = storage + self.storage: FeedStorageProtocol = storage # feed params - self.batch_id = batch_id - self.format = format - self.store_empty = store_empty - self.uri_template = uri_template - self.uri = uri - self.filter = filter + self.batch_id: int = batch_id + self.format: str = format + self.store_empty: bool = store_empty + self.uri_template: str = uri_template + self.uri: str = uri + self.filter: ItemFilter = filter # exporter params - self.feed_options = feed_options - self.spider = spider + self.feed_options: Dict[str, Any] = feed_options + self.spider: Spider = spider self.exporters: Dict[str, Type[BaseItemExporter]] = exporters - self.settings = settings - self.crawler = crawler + self.settings: BaseSettings = settings + self.crawler: Crawler = crawler # flags - self.itemcount = 0 - self._exporting = False - self._fileloaded = False + self.itemcount: int = 0 + self._exporting: bool = False + self._fileloaded: bool = False - def start_exporting(self): + def start_exporting(self) -> None: if not self._fileloaded: self.file = self.storage.open(self.spider) if "postprocessing" in self.feed_options: - self.file = PostProcessingManager( - self.feed_options["postprocessing"], self.file, self.feed_options + self.file = cast( + IO[bytes], + PostProcessingManager( + self.feed_options["postprocessing"], + self.file, + self.feed_options, + ), ) self.exporter = self._get_exporter( file=self.file, @@ -368,17 +449,23 @@ class FeedSlot: self._fileloaded = True if not self._exporting: + assert self.exporter self.exporter.start_exporting() self._exporting = True - def _get_instance(self, objcls, *args, **kwargs): + def _get_instance( + self, objcls: Type[BaseItemExporter], *args: Any, **kwargs: Any + ) -> BaseItemExporter: return build_from_crawler(objcls, self.crawler, *args, **kwargs) - def _get_exporter(self, file, format, *args, **kwargs) -> BaseItemExporter: + def _get_exporter( + self, file: IO[bytes], format: str, *args: Any, **kwargs: Any + ) -> BaseItemExporter: return self._get_instance(self.exporters[format], file, *args, **kwargs) - def finish_exporting(self): + def finish_exporting(self) -> None: if self._exporting: + assert self.exporter self.exporter.finish_exporting() self._exporting = False @@ -390,22 +477,22 @@ _FeedSlot = create_deprecated_class( class FeedExporter: - _pending_deferreds: List[defer.Deferred] = [] + _pending_deferreds: List[Deferred] = [] @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: exporter = cls(crawler) crawler.signals.connect(exporter.open_spider, signals.spider_opened) crawler.signals.connect(exporter.close_spider, signals.spider_closed) crawler.signals.connect(exporter.item_scraped, signals.item_scraped) return exporter - def __init__(self, crawler): - self.crawler = crawler - self.settings = crawler.settings + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler + self.settings: Settings = crawler.settings self.feeds = {} - self.slots = [] - self.filters = {} + self.slots: List[FeedSlot] = [] + self.filters: Dict[str, ItemFilter] = {} if not self.settings["FEEDS"] and not self.settings["FEED_URI"]: raise NotConfigured @@ -437,8 +524,12 @@ class FeedExporter: ) self.filters[uri] = self._load_filter(feed_options) - self.storages = self._load_components("FEED_STORAGES") - self.exporters = self._load_components("FEED_EXPORTERS") + self.storages: Dict[str, Type[FeedStorageProtocol]] = self._load_components( + "FEED_STORAGES" + ) + self.exporters: Dict[str, Type[BaseItemExporter]] = self._load_components( + "FEED_EXPORTERS" + ) for uri, feed_options in self.feeds.items(): if not self._storage_supported(uri, feed_options): raise NotConfigured @@ -447,7 +538,7 @@ class FeedExporter: if not self._exporter_supported(feed_options["format"]): raise NotConfigured - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> None: for uri, feed_options in self.feeds.items(): uri_params = self._get_uri_params(spider, feed_options["uri_params"]) self.slots.append( @@ -460,7 +551,7 @@ class FeedExporter: ) ) - async def close_spider(self, spider): + async def close_spider(self, spider: Spider) -> None: for slot in self.slots: self._close_slot(slot, spider) @@ -473,8 +564,9 @@ class FeedExporter: self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed) ) - def _close_slot(self, slot, spider): - def get_file(slot_): + def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred]: + def get_file(slot_: FeedSlot) -> IO[bytes]: + assert slot_.file if isinstance(slot_.file, PostProcessingManager): slot_.file.close() return slot_.file.file @@ -492,7 +584,7 @@ class FeedExporter: return None logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" - d = defer.maybeDeferred(slot.storage.store, get_file(slot)) + d: Deferred = maybeDeferred(slot.storage.store, get_file(slot)) d.addCallback( self._handle_store_success, logmsg, spider, type(slot.storage).__name__ @@ -510,20 +602,33 @@ class FeedExporter: return d - def _handle_store_error(self, f, logmsg, spider, slot_type): + def _handle_store_error( + self, f: Failure, logmsg: str, spider: Spider, slot_type: str + ) -> None: logger.error( "Error storing %s", logmsg, exc_info=failure_to_exc_info(f), extra={"spider": spider}, ) + assert self.crawler.stats self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") - def _handle_store_success(self, f, logmsg, spider, slot_type): + def _handle_store_success( + self, f: Failure, logmsg: str, spider: Spider, slot_type: str + ) -> None: logger.info("Stored %s", logmsg, extra={"spider": spider}) + assert self.crawler.stats self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}") - def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template): + def _start_new_batch( + self, + batch_id: int, + uri: str, + feed_options: Dict[str, Any], + spider: Spider, + uri_template: str, + ) -> FeedSlot: """ Redirect the output data stream to a new file. Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified @@ -546,11 +651,11 @@ class FeedExporter: spider=spider, exporters=self.exporters, settings=self.settings, - crawler=getattr(self, "crawler", None), + crawler=self.crawler, ) return slot - def item_scraped(self, item, spider): + def item_scraped(self, item: Any, spider: Spider) -> None: slots = [] for slot in self.slots: if not slot.filter.accepts(item): @@ -560,6 +665,7 @@ class FeedExporter: continue slot.start_exporting() + assert slot.exporter slot.exporter.export_item(item) slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one @@ -584,7 +690,7 @@ class FeedExporter: slots.append(slot) self.slots = slots - def _load_components(self, setting_prefix): + def _load_components(self, setting_prefix: str) -> Dict[str, Any]: conf = without_none_values(self.settings.getwithbase(setting_prefix)) d = {} for k, v in conf.items(): @@ -594,12 +700,13 @@ class FeedExporter: pass return d - def _exporter_supported(self, format): + def _exporter_supported(self, format: str) -> bool: if format in self.exporters: return True logger.error("Unknown feed format: %(format)s", {"format": format}) + return False - def _settings_are_valid(self): + def _settings_are_valid(self) -> bool: """ If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain %(batch_time)s or %(batch_id)d to distinguish different files of partial output @@ -617,7 +724,7 @@ class FeedExporter: return False return True - def _storage_supported(self, uri, feed_options): + def _storage_supported(self, uri: str, feed_options: Dict[str, Any]) -> bool: scheme = urlparse(uri).scheme if scheme in self.storages or PureWindowsPath(uri).drive: try: @@ -630,8 +737,11 @@ class FeedExporter: ) else: logger.error("Unknown feed storage scheme: %(scheme)s", {"scheme": scheme}) + return False - def _get_storage(self, uri, feed_options): + def _get_storage( + self, uri: str, feed_options: Dict[str, Any] + ) -> FeedStorageProtocol: """Fork of create_instance specific to feed storage classes It supports not passing the *feed_options* parameters to classes that @@ -640,11 +750,14 @@ class FeedExporter: feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"]) crawler = getattr(self, "crawler", None) - def build_instance(builder, *preargs): + def build_instance( + builder: Type[FeedStorageProtocol], *preargs: Any + ) -> FeedStorageProtocol: return build_storage( builder, uri, feed_options=feed_options, preargs=preargs ) + instance: FeedStorageProtocol if crawler and hasattr(feedcls, "from_crawler"): instance = build_instance(feedcls.from_crawler, crawler) method_name = "from_crawler" @@ -661,9 +774,9 @@ class FeedExporter: def _get_uri_params( self, spider: Spider, - uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]], + uri_params_function: Union[str, UriParamsCallableT, None], slot: Optional[FeedSlot] = None, - ) -> dict: + ) -> Dict[str, Any]: params = {} for k in dir(spider): params[k] = getattr(spider, k) @@ -671,7 +784,7 @@ class FeedExporter: params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-") params["batch_time"] = utc_now.isoformat().replace(":", "-") params["batch_id"] = slot.batch_id + 1 if slot is not None else 1 - uripar_function = ( + uripar_function: UriParamsCallableT = ( load_object(uri_params_function) if uri_params_function else lambda params, _: params @@ -679,7 +792,9 @@ class FeedExporter: new_params = uripar_function(params, spider) return new_params if new_params is not None else params - def _load_filter(self, feed_options): + def _load_filter(self, feed_options: Dict[str, Any]) -> ItemFilter: # load the item filter if declared else load the default filter class - item_filter_class = load_object(feed_options.get("item_filter", ItemFilter)) + item_filter_class: Type[ItemFilter] = load_object( + feed_options.get("item_filter", ItemFilter) + ) return item_filter_class(feed_options) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 335728502..dd5bce24f 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,10 +1,13 @@ import gzip import logging +import os import pickle # nosec from email.utils import mktime_tz, parsedate_tz from importlib import import_module from pathlib import Path from time import time +from types import ModuleType +from typing import IO, TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union, cast from weakref import WeakKeyDictionary from w3lib.http import headers_dict_to_raw, headers_raw_to_dict @@ -12,49 +15,65 @@ from w3lib.http import headers_dict_to_raw, headers_raw_to_dict from scrapy.http import Headers, Response from scrapy.http.request import Request from scrapy.responsetypes import responsetypes +from scrapy.settings import BaseSettings from scrapy.spiders import Spider from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.request import RequestFingerprinter + +if TYPE_CHECKING: + # typing.Concatenate requires Python 3.10 + from typing_extensions import Concatenate + logger = logging.getLogger(__name__) class DummyPolicy: - def __init__(self, settings): - self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") - self.ignore_http_codes = [ + def __init__(self, settings: BaseSettings): + self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") + self.ignore_http_codes: List[int] = [ int(x) for x in settings.getlist("HTTPCACHE_IGNORE_HTTP_CODES") ] - def should_cache_request(self, request): + def should_cache_request(self, request: Request) -> bool: return urlparse_cached(request).scheme not in self.ignore_schemes - def should_cache_response(self, response, request): + def should_cache_response(self, response: Response, request: Request) -> bool: return response.status not in self.ignore_http_codes - def is_cached_response_fresh(self, cachedresponse, request): + def is_cached_response_fresh( + self, cachedresponse: Response, request: Request + ) -> bool: return True - def is_cached_response_valid(self, cachedresponse, response, request): + def is_cached_response_valid( + self, cachedresponse: Response, response: Response, request: Request + ) -> bool: return True class RFC2616Policy: MAXAGE = 3600 * 24 * 365 # one year - def __init__(self, settings): - self.always_store = settings.getbool("HTTPCACHE_ALWAYS_STORE") - self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") - self._cc_parsed = WeakKeyDictionary() - self.ignore_response_cache_controls = [ + def __init__(self, settings: BaseSettings): + self.always_store: bool = settings.getbool("HTTPCACHE_ALWAYS_STORE") + self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") + self._cc_parsed: WeakKeyDictionary[ + Union[Request, Response], Dict[bytes, Optional[bytes]] + ] = WeakKeyDictionary() + self.ignore_response_cache_controls: List[bytes] = [ to_bytes(cc) for cc in settings.getlist("HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS") ] - def _parse_cachecontrol(self, r): + def _parse_cachecontrol( + self, r: Union[Request, Response] + ) -> Dict[bytes, Optional[bytes]]: if r not in self._cc_parsed: cch = r.headers.get(b"Cache-Control", b"") + assert cch is not None parsed = parse_cachecontrol(cch) if isinstance(r, Response): for key in self.ignore_response_cache_controls: @@ -62,7 +81,7 @@ class RFC2616Policy: self._cc_parsed[r] = parsed return self._cc_parsed[r] - def should_cache_request(self, request): + def should_cache_request(self, request: Request) -> bool: if urlparse_cached(request).scheme in self.ignore_schemes: return False cc = self._parse_cachecontrol(request) @@ -72,7 +91,7 @@ class RFC2616Policy: # Any other is eligible for caching return True - def should_cache_response(self, response, request): + def should_cache_response(self, response: Response, request: Request) -> bool: # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 # Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 # Status code 206 is not included because cache can not deal with partial contents @@ -100,7 +119,9 @@ class RFC2616Policy: # info and can not be revalidated return False - def is_cached_response_fresh(self, cachedresponse, request): + def is_cached_response_fresh( + self, cachedresponse: Response, request: Request + ) -> bool: cc = self._parse_cachecontrol(cachedresponse) ccreq = self._parse_cachecontrol(request) if b"no-cache" in cc or b"no-cache" in ccreq: @@ -141,7 +162,9 @@ class RFC2616Policy: self._set_conditional_validators(request, cachedresponse) return False - def is_cached_response_valid(self, cachedresponse, response, request): + def is_cached_response_valid( + self, cachedresponse: Response, response: Response, request: Request + ) -> bool: # Use the cached response if the new response is a server error, # as long as the old response didn't specify must-revalidate. if response.status >= 500: @@ -152,7 +175,9 @@ class RFC2616Policy: # Use the cached response if the server says it hasn't changed. return response.status == 304 - def _set_conditional_validators(self, request, cachedresponse): + def _set_conditional_validators( + self, request: Request, cachedresponse: Response + ) -> None: if b"Last-Modified" in cachedresponse.headers: request.headers[b"If-Modified-Since"] = cachedresponse.headers[ b"Last-Modified" @@ -161,13 +186,15 @@ class RFC2616Policy: if b"ETag" in cachedresponse.headers: request.headers[b"If-None-Match"] = cachedresponse.headers[b"ETag"] - def _get_max_age(self, cc): + def _get_max_age(self, cc: Dict[bytes, Optional[bytes]]) -> Optional[int]: try: - return max(0, int(cc[b"max-age"])) + return max(0, int(cc[b"max-age"])) # type: ignore[arg-type] except (KeyError, ValueError): return None - def _compute_freshness_lifetime(self, response, request, now): + def _compute_freshness_lifetime( + self, response: Response, request: Request, now: float + ) -> float: # Reference nsHttpResponseHead::ComputeFreshnessLifetime # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#706 cc = self._parse_cachecontrol(response) @@ -198,10 +225,12 @@ class RFC2616Policy: # Insufficient information to compute freshness lifetime return 0 - def _compute_current_age(self, response, request, now): + def _compute_current_age( + self, response: Response, request: Request, now: float + ) -> float: # Reference nsHttpResponseHead::ComputeCurrentAge # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#658 - currentage = 0 + currentage: float = 0 # If Date header is not set we assume it is a fast connection, and # clock is in sync with the server date = rfc1123_to_epoch(response.headers.get(b"Date")) or now @@ -210,7 +239,7 @@ class RFC2616Policy: if b"Age" in response.headers: try: - age = int(response.headers[b"Age"]) + age = int(response.headers[b"Age"]) # type: ignore[arg-type] currentage = max(currentage, age) except ValueError: pass @@ -219,13 +248,13 @@ class RFC2616Policy: class DbmCacheStorage: - def __init__(self, settings): - self.cachedir = data_path(settings["HTTPCACHE_DIR"], createdir=True) - self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS") - self.dbmodule = import_module(settings["HTTPCACHE_DBM_MODULE"]) - self.db = None + def __init__(self, settings: BaseSettings): + self.cachedir: str = data_path(settings["HTTPCACHE_DIR"], createdir=True) + self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") + self.dbmodule: ModuleType = import_module(settings["HTTPCACHE_DBM_MODULE"]) + self.db: Any = None # the real type is private - def open_spider(self, spider: Spider): + def open_spider(self, spider: Spider) -> None: dbpath = Path(self.cachedir, f"{spider.name}.db") self.db = self.dbmodule.open(str(dbpath), "c") @@ -235,15 +264,16 @@ class DbmCacheStorage: extra={"spider": spider}, ) - self._fingerprinter = spider.crawler.request_fingerprinter + assert spider.crawler.request_fingerprinter + self._fingerprinter: RequestFingerprinter = spider.crawler.request_fingerprinter - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> None: self.db.close() - def retrieve_response(self, spider, request): + def retrieve_response(self, spider: Spider, request: Request) -> Optional[Response]: data = self._read_data(spider, request) if data is None: - return # not cached + return None # not cached url = data["url"] status = data["status"] headers = Headers(data["headers"]) @@ -252,7 +282,9 @@ class DbmCacheStorage: response = respcls(url=url, headers=headers, status=status, body=body) return response - def store_response(self, spider, request, response): + def store_response( + self, spider: Spider, request: Request, response: Response + ) -> None: key = self._fingerprinter.fingerprint(request).hex() data = { "status": response.status, @@ -263,28 +295,31 @@ class DbmCacheStorage: self.db[f"{key}_data"] = pickle.dumps(data, protocol=4) self.db[f"{key}_time"] = str(time()) - def _read_data(self, spider, request): + def _read_data(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]: key = self._fingerprinter.fingerprint(request).hex() db = self.db tkey = f"{key}_time" if tkey not in db: - return # not found + return None # not found ts = db[tkey] if 0 < self.expiration_secs < time() - float(ts): - return # expired + return None # expired - return pickle.loads(db[f"{key}_data"]) # nosec + return cast(Dict[str, Any], pickle.loads(db[f"{key}_data"])) # nosec class FilesystemCacheStorage: - def __init__(self, settings): - self.cachedir = data_path(settings["HTTPCACHE_DIR"]) - self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS") - self.use_gzip = settings.getbool("HTTPCACHE_GZIP") - self._open = gzip.open if self.use_gzip else open + def __init__(self, settings: BaseSettings): + self.cachedir: str = data_path(settings["HTTPCACHE_DIR"]) + self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") + self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP") + # https://github.com/python/mypy/issues/10740 + self._open: Callable[Concatenate[Union[str, os.PathLike], str, ...], IO] = ( + gzip.open if self.use_gzip else open # type: ignore[assignment] + ) - def open_spider(self, spider: Spider): + def open_spider(self, spider: Spider) -> None: logger.debug( "Using filesystem cache storage in %(cachedir)s", {"cachedir": self.cachedir}, @@ -294,27 +329,29 @@ class FilesystemCacheStorage: assert spider.crawler.request_fingerprinter self._fingerprinter = spider.crawler.request_fingerprinter - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> None: pass - def retrieve_response(self, spider: Spider, request: Request): + def retrieve_response(self, spider: Spider, request: Request) -> Optional[Response]: """Return response if present in cache, or None otherwise.""" metadata = self._read_meta(spider, request) if metadata is None: - return # not cached + return None # not cached rpath = Path(self._get_request_path(spider, request)) with self._open(rpath / "response_body", "rb") as f: body = f.read() with self._open(rpath / "response_headers", "rb") as f: rawheaders = f.read() - url = metadata.get("response_url") + url = metadata["response_url"] status = metadata["status"] headers = Headers(headers_raw_to_dict(rawheaders)) respcls = responsetypes.from_args(headers=headers, url=url, body=body) response = respcls(url=url, headers=headers, status=status, body=body) return response - def store_response(self, spider: Spider, request: Request, response): + def store_response( + self, spider: Spider, request: Request, response: Response + ) -> None: """Store the given response in the cache.""" rpath = Path(self._get_request_path(spider, request)) if not rpath.exists(): @@ -343,19 +380,19 @@ class FilesystemCacheStorage: key = self._fingerprinter.fingerprint(request).hex() return str(Path(self.cachedir, spider.name, key[0:2], key)) - def _read_meta(self, spider: Spider, request: Request): + def _read_meta(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]: rpath = Path(self._get_request_path(spider, request)) metapath = rpath / "pickled_meta" if not metapath.exists(): - return # not found + return None # not found mtime = metapath.stat().st_mtime if 0 < self.expiration_secs < time() - mtime: - return # expired + return None # expired with self._open(metapath, "rb") as f: - return pickle.load(f) # nosec + return cast(Dict[str, Any], pickle.load(f)) # nosec -def parse_cachecontrol(header): +def parse_cachecontrol(header: bytes) -> Dict[bytes, Optional[bytes]]: """Parse Cache-Control header https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 @@ -375,9 +412,9 @@ def parse_cachecontrol(header): return directives -def rfc1123_to_epoch(date_str): +def rfc1123_to_epoch(date_str: Union[str, bytes, None]) -> Optional[int]: try: - date_str = to_unicode(date_str, encoding="ascii") - return mktime_tz(parsedate_tz(date_str)) + date_str = to_unicode(date_str, encoding="ascii") # type: ignore[arg-type] + return mktime_tz(parsedate_tz(date_str)) # type: ignore[arg-type] except Exception: return None diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index 7ffbd8bc3..ac12ad829 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -6,7 +6,7 @@ from bz2 import BZ2File from gzip import GzipFile from io import IOBase from lzma import LZMAFile -from typing import Any, BinaryIO, Dict, List, cast +from typing import IO, Any, BinaryIO, Dict, List, cast from scrapy.utils.misc import load_object @@ -126,7 +126,7 @@ class PostProcessingManager(IOBase): """ def __init__( - self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any] + self, plugins: List[Any], file: IO[bytes], feed_options: Dict[str, Any] ) -> None: self.plugins = self._load_plugins(plugins) self.file = file diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 49493e9c6..cd6e9d04e 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -3,7 +3,6 @@ import logging import re from io import StringIO from typing import ( - TYPE_CHECKING, Any, Callable, Dict, @@ -25,9 +24,6 @@ from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode -if TYPE_CHECKING: - from lxml._types import SupportsReadClose # nosec - logger = logging.getLogger(__name__) @@ -98,7 +94,7 @@ def xmliter_lxml( reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( - cast("SupportsReadClose[bytes]", reader), + reader, encoding=reader.encoding, events=("end", "start-ns"), resolve_entities=False, diff --git a/tox.ini b/tox.ini index b5effb527..d7527bb04 100644 --- a/tox.ini +++ b/tox.ini @@ -43,14 +43,14 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.8.0 - typing-extensions==4.10.0 + mypy==1.10.0 + typing-extensions==4.11.0 types-attrs==19.1.0 - types-lxml==2024.2.9 - types-Pillow==10.2.0.20240213 - types-Pygments==2.17.0.20240106 - types-pyOpenSSL==24.0.0.20240130 - types-setuptools==69.1.0.20240223 + types-lxml==2024.4.14 + types-Pillow==10.2.0.20240423 + types-Pygments==2.17.0.20240310 + types-pyOpenSSL==24.0.0.20240417 + types-setuptools==69.5.0.20240423 # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 w3lib >= 2.1.2 commands = From d7da298e0637d105dcec379f6cbb3196e752ae72 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 19:14:59 +0500 Subject: [PATCH 141/269] Typing for build_from_*. (#6326) --- scrapy/core/engine.py | 3 ++- scrapy/core/scheduler.py | 2 ++ scrapy/crawler.py | 4 +++- scrapy/utils/misc.py | 28 +++++++++++++++++++--------- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 2db085081..93a0c51bc 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -366,7 +366,8 @@ class ExecutionEngine: self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.spider = spider if hasattr(scheduler, "open"): - yield scheduler.open(spider) + if d := scheduler.open(spider): + yield d yield self.scraper.open_spider(spider) assert self.crawler.stats self.crawler.stats.open_spider(spider) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index f41b83a67..b2209e53f 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -322,6 +322,7 @@ class Scheduler(BaseScheduler): def _mq(self): """Create a new priority queue instance, with in-memory storage""" + assert self.crawler return build_from_crawler( self.pqclass, self.crawler, @@ -331,6 +332,7 @@ class Scheduler(BaseScheduler): def _dq(self): """Create a new priority queue instance, with disk storage""" + assert self.crawler assert self.dqdir state = self._read_dqs_state(self.dqdir) q = build_from_crawler( diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1db9ace28..ccfe78891 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -445,7 +445,9 @@ class CrawlerProcess(CrawlerRunner): d.addBoth(self._stop_reactor) resolver_class = load_object(self.settings["DNS_RESOLVER"]) - resolver = build_from_crawler(resolver_class, self, reactor=reactor) + # We pass self, which is CrawlerProcess, instead of Crawler here, + # which works because the default resolvers only use crawler.settings. + resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[arg-type] resolver.install_on_reactor() tp = reactor.getThreadPool() tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 7f83d06fb..faf52e44a 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,5 +1,7 @@ """Helper functions which don't fit anywhere else""" +from __future__ import annotations + import ast import hashlib import inspect @@ -22,6 +24,8 @@ from typing import ( Iterable, List, Optional, + Type, + TypeVar, Union, cast, ) @@ -32,9 +36,11 @@ from scrapy.utils.datatypes import LocalWeakReferencedCache if TYPE_CHECKING: from scrapy import Spider - + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes +T = TypeVar("T") def arg_to_iter(arg: Any) -> Iterable[Any]: @@ -177,7 +183,9 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): return instance -def build_from_crawler(objcls, crawler, /, *args, **kwargs): +def build_from_crawler( + objcls: Type[T], crawler: Crawler, /, *args: Any, **kwargs: Any +) -> T: """Construct a class instance using its ``from_crawler`` constructor. ``*args`` and ``**kwargs`` are forwarded to the constructor. @@ -185,20 +193,22 @@ def build_from_crawler(objcls, crawler, /, *args, **kwargs): Raises ``TypeError`` if the resulting instance is ``None``. """ if hasattr(objcls, "from_crawler"): - instance = objcls.from_crawler(crawler, *args, **kwargs) + instance = objcls.from_crawler(crawler, *args, **kwargs) # type: ignore[attr-defined] method_name = "from_crawler" elif hasattr(objcls, "from_settings"): - instance = objcls.from_settings(crawler.settings, *args, **kwargs) + instance = objcls.from_settings(crawler.settings, *args, **kwargs) # type: ignore[attr-defined] method_name = "from_settings" else: instance = objcls(*args, **kwargs) method_name = "__new__" if instance is None: raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") - return instance + return cast(T, instance) -def build_from_settings(objcls, settings, /, *args, **kwargs): +def build_from_settings( + objcls: Type[T], settings: BaseSettings, /, *args: Any, **kwargs: Any +) -> T: """Construct a class instance using its ``from_settings`` constructor. ``*args`` and ``**kwargs`` are forwarded to the constructor. @@ -206,14 +216,14 @@ def build_from_settings(objcls, settings, /, *args, **kwargs): Raises ``TypeError`` if the resulting instance is ``None``. """ if hasattr(objcls, "from_settings"): - instance = objcls.from_settings(settings, *args, **kwargs) + instance = objcls.from_settings(settings, *args, **kwargs) # type: ignore[attr-defined] method_name = "from_settings" else: instance = objcls(*args, **kwargs) method_name = "__new__" if instance is None: raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") - return instance + return cast(T, instance) @contextmanager @@ -290,7 +300,7 @@ def is_generator_with_return_value(callable: Callable) -> bool: return bool(_generator_callbacks_cache[callable]) -def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None: +def warn_on_generator_with_return_value(spider: Spider, callable: Callable) -> None: """ Logs a warning if a callable is a generator function and includes a 'return' statement with a value different than None From 9eea22fb0ca99193b7f38f9f9398b278d64ea977 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 18:59:31 +0500 Subject: [PATCH 142/269] Full typing for scrapy/cmdline.py. --- scrapy/cmdline.py | 47 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 6580ba9ce..4df5698a6 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -4,18 +4,22 @@ import inspect import os import sys from importlib.metadata import entry_points +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter from scrapy.crawler import CrawlerProcess from scrapy.exceptions import UsageError +from scrapy.settings import BaseSettings, Settings from scrapy.utils.misc import walk_modules from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect class ScrapyArgumentParser(argparse.ArgumentParser): - def _parse_optional(self, arg_string): + def _parse_optional( + self, arg_string: str + ) -> Optional[Tuple[Optional[argparse.Action], str, Optional[str]]]: # if starts with -: it means that is a parameter not a argument if arg_string[:2] == "-:": return None @@ -23,7 +27,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser): return super()._parse_optional(arg_string) -def _iter_command_classes(module_name): +def _iter_command_classes(module_name: str) -> Iterable[Type[ScrapyCommand]]: # TODO: add `name` attribute to commands and merge this function with # scrapy.utils.spider.iter_spider_classes for module in walk_modules(module_name): @@ -37,8 +41,8 @@ def _iter_command_classes(module_name): yield obj -def _get_commands_from_module(module, inproject): - d = {} +def _get_commands_from_module(module: str, inproject: bool) -> Dict[str, ScrapyCommand]: + d: Dict[str, ScrapyCommand] = {} for cmd in _iter_command_classes(module): if inproject or not cmd.requires_project: cmdname = cmd.__module__.split(".")[-1] @@ -46,8 +50,10 @@ def _get_commands_from_module(module, inproject): return d -def _get_commands_from_entry_points(inproject, group="scrapy.commands"): - cmds = {} +def _get_commands_from_entry_points( + inproject: bool, group: str = "scrapy.commands" +) -> Dict[str, ScrapyCommand]: + cmds: Dict[str, ScrapyCommand] = {} if sys.version_info >= (3, 10): eps = entry_points(group=group) else: @@ -61,7 +67,9 @@ def _get_commands_from_entry_points(inproject, group="scrapy.commands"): return cmds -def _get_commands_dict(settings, inproject): +def _get_commands_dict( + settings: BaseSettings, inproject: bool +) -> Dict[str, ScrapyCommand]: cmds = _get_commands_from_module("scrapy.commands", inproject) cmds.update(_get_commands_from_entry_points(inproject)) cmds_module = settings["COMMANDS_MODULE"] @@ -70,16 +78,17 @@ def _get_commands_dict(settings, inproject): return cmds -def _pop_command_name(argv): +def _pop_command_name(argv: List[str]) -> Optional[str]: i = 0 for arg in argv[1:]: if not arg.startswith("-"): del argv[i] return arg i += 1 + return None -def _print_header(settings, inproject): +def _print_header(settings: BaseSettings, inproject: bool) -> None: version = scrapy.__version__ if inproject: print(f"Scrapy {version} - active project: {settings['BOT_NAME']}\n") @@ -88,7 +97,7 @@ def _print_header(settings, inproject): print(f"Scrapy {version} - no active project\n") -def _print_commands(settings, inproject): +def _print_commands(settings: BaseSettings, inproject: bool) -> None: _print_header(settings, inproject) print("Usage:") print(" scrapy [options] [args]\n") @@ -103,13 +112,17 @@ def _print_commands(settings, inproject): print('Use "scrapy -h" to see more info about a command') -def _print_unknown_command(settings, cmdname, inproject): +def _print_unknown_command( + settings: BaseSettings, cmdname: str, inproject: bool +) -> None: _print_header(settings, inproject) print(f"Unknown command: {cmdname}\n") print('Use "scrapy" to see available commands') -def _run_print_help(parser, func, *a, **kw): +def _run_print_help( + parser: argparse.ArgumentParser, func: Callable, *a: Any, **kw: Any +) -> None: try: func(*a, **kw) except UsageError as e: @@ -120,7 +133,9 @@ def _run_print_help(parser, func, *a, **kw): sys.exit(2) -def execute(argv=None, settings=None): +def execute( + argv: Optional[List[str]] = None, settings: Optional[Settings] = None +) -> None: if argv is None: argv = sys.argv @@ -162,14 +177,16 @@ def execute(argv=None, settings=None): sys.exit(cmd.exitcode) -def _run_command(cmd, args, opts): +def _run_command(cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace) -> None: if opts.profile: _run_command_profiled(cmd, args, opts) else: cmd.run(args, opts) -def _run_command_profiled(cmd, args, opts): +def _run_command_profiled( + cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace +) -> None: if opts.profile: sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") loc = locals() From fc1a83e7c42dc5142eb9190fdca887385254a7a6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 19:07:31 +0500 Subject: [PATCH 143/269] Full typing for scrapy/item.py. --- scrapy/item.py | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/scrapy/item.py b/scrapy/item.py index d3eb90b7b..e04e994ef 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -4,14 +4,20 @@ Scrapy Item See documentation in docs/topics/item.rst """ +from __future__ import annotations + from abc import ABCMeta from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat -from typing import Dict +from typing import TYPE_CHECKING, Any, Dict, Iterator, KeysView, NoReturn, Tuple from scrapy.utils.trackref import object_ref +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class Field(dict): """Container of field metadata""" @@ -23,7 +29,9 @@ class ItemMeta(ABCMeta): .. _metaclass: https://realpython.com/python-metaclasses """ - def __new__(mcs, class_name, bases, attrs): + def __new__( + mcs, class_name: str, bases: Tuple[type, ...], attrs: Dict[str, Any] + ) -> ItemMeta: classcell = attrs.pop("__classcell__", None) new_bases = tuple(base._class for base in bases if hasattr(base, "_class")) _class = super().__new__(mcs, "x_" + class_name, new_bases, attrs) @@ -44,7 +52,7 @@ class ItemMeta(ABCMeta): return super().__new__(mcs, class_name, bases, new_attrs) -class Item(MutableMapping, object_ref, metaclass=ItemMeta): +class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta): """ Base class for scraped items. @@ -69,51 +77,51 @@ class Item(MutableMapping, object_ref, metaclass=ItemMeta): fields: Dict[str, Field] - def __init__(self, *args, **kwargs): - self._values = {} + def __init__(self, *args: Any, **kwargs: Any): + self._values: Dict[str, Any] = {} if args or kwargs: # avoid creating dict for most common case for k, v in dict(*args, **kwargs).items(): self[k] = v - def __getitem__(self, key): + def __getitem__(self, key: str) -> Any: return self._values[key] - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: Any) -> None: if key in self.fields: self._values[key] = value else: raise KeyError(f"{self.__class__.__name__} does not support field: {key}") - def __delitem__(self, key): + def __delitem__(self, key: str) -> None: del self._values[key] - def __getattr__(self, name): + def __getattr__(self, name: str) -> NoReturn: if name in self.fields: raise AttributeError(f"Use item[{name!r}] to get field value") raise AttributeError(name) - def __setattr__(self, name, value): + def __setattr__(self, name: str, value: Any) -> None: if not name.startswith("_"): raise AttributeError(f"Use item[{name!r}] = {value!r} to set field value") super().__setattr__(name, value) - def __len__(self): + def __len__(self) -> int: return len(self._values) - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self._values) __hash__ = object_ref.__hash__ - def keys(self): + def keys(self) -> KeysView[str]: return self._values.keys() - def __repr__(self): + def __repr__(self) -> str: return pformat(dict(self)) - def copy(self): + def copy(self) -> Self: return self.__class__(self) - def deepcopy(self): + def deepcopy(self) -> Self: """Return a :func:`~copy.deepcopy` of this item.""" return deepcopy(self) From 08a265b6ff9bc47774173238b06715154b39e534 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 19:10:00 +0500 Subject: [PATCH 144/269] Full typing for scrapy/extension.py. --- scrapy/extension.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/extension.py b/scrapy/extension.py index 6be14450c..8221b675e 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -4,7 +4,10 @@ The Extension Manager See documentation in docs/topics/extensions.rst """ +from typing import Any, List + from scrapy.middleware import MiddlewareManager +from scrapy.settings import Settings from scrapy.utils.conf import build_component_list @@ -12,5 +15,5 @@ class ExtensionManager(MiddlewareManager): component_name = "extension" @classmethod - def _get_mwlist_from_settings(cls, settings): + def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: return build_component_list(settings.getwithbase("EXTENSIONS")) From 38020e0b0481d2b15792757669272d6d3bf4b14f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 20:12:30 +0500 Subject: [PATCH 145/269] Full typing for scrapy/mail.py. --- scrapy/mail.py | 98 +++++++++++++++++++++++++++++++------------------- 1 file changed, 62 insertions(+), 36 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 7cb5ef454..56adba934 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -14,11 +14,23 @@ from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO -from typing import TYPE_CHECKING, Optional +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Optional, + Sequence, + Tuple, + Union, +) from twisted import version as twisted_version from twisted.internet import ssl from twisted.internet.defer import Deferred +from twisted.python.failure import Failure from twisted.python.versions import Version from scrapy.settings import BaseSettings @@ -26,6 +38,9 @@ from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes if TYPE_CHECKING: + # imports twisted.internet.reactor + from twisted.mail.smtp import ESMTPSenderFactory + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -37,7 +52,7 @@ logger = logging.getLogger(__name__) COMMASPACE = ", " -def _to_bytes_or_none(text): +def _to_bytes_or_none(text: Union[str, bytes, None]) -> Optional[bytes]: if text is None: return None return to_bytes(text) @@ -46,23 +61,23 @@ def _to_bytes_or_none(text): class MailSender: def __init__( self, - smtphost="localhost", - mailfrom="scrapy@localhost", - smtpuser=None, - smtppass=None, - smtpport=25, - smtptls=False, - smtpssl=False, - debug=False, + smtphost: str = "localhost", + mailfrom: str = "scrapy@localhost", + smtpuser: Optional[str] = None, + smtppass: Optional[str] = None, + smtpport: int = 25, + smtptls: bool = False, + smtpssl: bool = False, + debug: bool = False, ): - self.smtphost = smtphost - self.smtpport = smtpport - self.smtpuser = _to_bytes_or_none(smtpuser) - self.smtppass = _to_bytes_or_none(smtppass) - self.smtptls = smtptls - self.smtpssl = smtpssl - self.mailfrom = mailfrom - self.debug = debug + self.smtphost: str = smtphost + self.smtpport: int = smtpport + self.smtpuser: Optional[bytes] = _to_bytes_or_none(smtpuser) + self.smtppass: Optional[bytes] = _to_bytes_or_none(smtppass) + self.smtptls: bool = smtptls + self.smtpssl: bool = smtpssl + self.mailfrom: str = mailfrom + self.debug: bool = debug @classmethod def from_settings(cls, settings: BaseSettings) -> Self: @@ -78,14 +93,14 @@ class MailSender: def send( self, - to, - subject, - body, - cc=None, - attachs=(), - mimetype="text/plain", - charset=None, - _callback=None, + to: Union[str, List[str]], + subject: str, + body: str, + cc: Union[str, List[str], None] = None, + attachs: Sequence[Tuple[str, str, IO]] = (), + mimetype: str = "text/plain", + charset: Optional[str] = None, + _callback: Optional[Callable[..., None]] = None, ) -> Optional[Deferred]: from twisted.internet import reactor @@ -142,13 +157,15 @@ class MailSender: dfd.addCallbacks( callback=self._sent_ok, errback=self._sent_failed, - callbackArgs=[to, cc, subject, len(attachs)], - errbackArgs=[to, cc, subject, len(attachs)], + callbackArgs=(to, cc, subject, len(attachs)), + errbackArgs=(to, cc, subject, len(attachs)), ) reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd) return dfd - def _sent_ok(self, result, to, cc, subject, nattachs): + def _sent_ok( + self, result: Any, to: List[str], cc: List[str], subject: str, nattachs: int + ) -> None: logger.info( "Mail sent OK: To=%(mailto)s Cc=%(mailcc)s " 'Subject="%(mailsubject)s" Attachs=%(mailattachs)d', @@ -160,7 +177,14 @@ class MailSender: }, ) - def _sent_failed(self, failure, to, cc, subject, nattachs): + def _sent_failed( + self, + failure: Failure, + to: List[str], + cc: List[str], + subject: str, + nattachs: int, + ) -> Failure: errstr = str(failure.value) logger.error( "Unable to send mail: To=%(mailto)s Cc=%(mailcc)s " @@ -176,13 +200,13 @@ class MailSender: ) return failure - def _sendmail(self, to_addrs, msg): + def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred: from twisted.internet import reactor - msg = BytesIO(msg) - d = Deferred() + msg_io = BytesIO(msg) + d: Deferred = Deferred() - factory = self._create_sender_factory(to_addrs, msg, d) + factory = self._create_sender_factory(to_addrs, msg_io, d) if self.smtpssl: reactor.connectSSL( @@ -193,10 +217,12 @@ class MailSender: return d - def _create_sender_factory(self, to_addrs, msg, d): + def _create_sender_factory( + self, to_addrs: List[str], msg: IO, d: Deferred + ) -> ESMTPSenderFactory: from twisted.mail.smtp import ESMTPSenderFactory - factory_keywords = { + factory_keywords: Dict[str, Any] = { "heloFallback": True, "requireAuthentication": False, "requireTransportSecurity": self.smtptls, From 0c8e21b8acfcac2d6d057f3c67b2252d0fa660e3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 20:57:31 +0500 Subject: [PATCH 146/269] Full typing for scrapy/pqueues.py. --- scrapy/core/downloader/__init__.py | 2 +- scrapy/pqueues.py | 138 +++++++++++++++++++++-------- 2 files changed, 100 insertions(+), 40 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index ecd3e8b56..f88da41ea 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -143,7 +143,7 @@ class Downloader: return key, self.slots[key] - def _get_slot_key(self, request: Request, spider: Spider) -> str: + def _get_slot_key(self, request: Request, spider: Any) -> str: if self.DOWNLOAD_SLOT in request.meta: return cast(str, request.meta[self.DOWNLOAD_SLOT]) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 593667f1f..213ad590d 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,12 +1,32 @@ +from __future__ import annotations + import hashlib import logging +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + List, + Optional, + Protocol, + Tuple, + Type, + cast, +) +from scrapy import Request +from scrapy.core.downloader import Downloader +from scrapy.crawler import Crawler from scrapy.utils.misc import build_from_crawler +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) -def _path_safe(text): +def _path_safe(text: str) -> str: """ Return a filesystem-safe version of a string ``text`` @@ -24,6 +44,18 @@ def _path_safe(text): return "-".join([pathable_slot, unique_slot]) +class QueueProtocol(Protocol): + """Protocol for downstream queues of ``ScrapyPriorityQueue``.""" + + def push(self, request: Request) -> None: ... + + def pop(self) -> Optional[Request]: ... + + def close(self) -> None: ... + + def __len__(self) -> int: ... + + class ScrapyPriorityQueue: """A priority queue implemented using multiple internal queues (typically, FIFO queues). It uses one internal queue for each priority value. The internal @@ -51,18 +83,30 @@ class ScrapyPriorityQueue: """ @classmethod - def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()): + def from_crawler( + cls, + crawler: Crawler, + downstream_queue_cls: Type[QueueProtocol], + key: str, + startprios: Iterable[int] = (), + ) -> Self: return cls(crawler, downstream_queue_cls, key, startprios) - def __init__(self, crawler, downstream_queue_cls, key, startprios=()): - self.crawler = crawler - self.downstream_queue_cls = downstream_queue_cls - self.key = key - self.queues = {} - self.curprio = None + def __init__( + self, + crawler: Crawler, + downstream_queue_cls: Type[QueueProtocol], + key: str, + startprios: Iterable[int] = (), + ): + self.crawler: Crawler = crawler + self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls + self.key: str = key + self.queues: Dict[int, QueueProtocol] = {} + self.curprio: Optional[int] = None self.init_prios(startprios) - def init_prios(self, startprios): + def init_prios(self, startprios: Iterable[int]) -> None: if not startprios: return @@ -71,17 +115,17 @@ class ScrapyPriorityQueue: self.curprio = min(startprios) - def qfactory(self, key): + def qfactory(self, key: int) -> QueueProtocol: return build_from_crawler( self.downstream_queue_cls, self.crawler, self.key + "/" + str(key), ) - def priority(self, request): + def priority(self, request: Request) -> int: return -request.priority - def push(self, request): + def push(self, request: Request) -> None: priority = self.priority(request) if priority not in self.queues: self.queues[priority] = self.qfactory(priority) @@ -90,9 +134,9 @@ class ScrapyPriorityQueue: if self.curprio is None or priority < self.curprio: self.curprio = priority - def pop(self): + def pop(self) -> Optional[Request]: if self.curprio is None: - return + return None q = self.queues[self.curprio] m = q.pop() if not q: @@ -102,7 +146,7 @@ class ScrapyPriorityQueue: self.curprio = min(prios) if prios else None return m - def peek(self): + def peek(self) -> Optional[Request]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -112,30 +156,32 @@ class ScrapyPriorityQueue: if self.curprio is None: return None queue = self.queues[self.curprio] - return queue.peek() + # Protocols can't declare optional members + return cast(Request, queue.peek()) # type: ignore[attr-defined] - def close(self): - active = [] + def close(self) -> List[int]: + active: List[int] = [] for p, q in self.queues.items(): active.append(p) q.close() return active - def __len__(self): + def __len__(self) -> int: return sum(len(x) for x in self.queues.values()) if self.queues else 0 class DownloaderInterface: - def __init__(self, crawler): - self.downloader = crawler.engine.downloader + def __init__(self, crawler: Crawler): + assert crawler.engine + self.downloader: Downloader = crawler.engine.downloader - def stats(self, possible_slots): + def stats(self, possible_slots: Iterable[str]) -> List[Tuple[int, str]]: return [(self._active_downloads(slot), slot) for slot in possible_slots] - def get_slot_key(self, request): + def get_slot_key(self, request: Request) -> str: return self.downloader._get_slot_key(request, None) - def _active_downloads(self, slot): + def _active_downloads(self, slot: str) -> int: """Return a number of requests in a Downloader for a given slot""" if slot not in self.downloader.slots: return 0 @@ -149,10 +195,22 @@ class DownloaderAwarePriorityQueue: """ @classmethod - def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()): + def from_crawler( + cls, + crawler: Crawler, + downstream_queue_cls: Type[QueueProtocol], + key: str, + startprios: Optional[Dict[str, Iterable[int]]] = None, + ) -> Self: return cls(crawler, downstream_queue_cls, key, startprios) - def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()): + def __init__( + self, + crawler: Crawler, + downstream_queue_cls: Type[QueueProtocol], + key: str, + slot_startprios: Optional[Dict[str, Iterable[int]]] = None, + ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP' @@ -169,16 +227,18 @@ class DownloaderAwarePriorityQueue: "queue class can be resumed." ) - self._downloader_interface = DownloaderInterface(crawler) - self.downstream_queue_cls = downstream_queue_cls - self.key = key - self.crawler = crawler + self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler) + self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls + self.key: str = key + self.crawler: Crawler = crawler - self.pqueues = {} # slot -> priority queue + self.pqueues: Dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue for slot, startprios in (slot_startprios or {}).items(): self.pqueues[slot] = self.pqfactory(slot, startprios) - def pqfactory(self, slot, startprios=()): + def pqfactory( + self, slot: str, startprios: Iterable[int] = () + ) -> ScrapyPriorityQueue: return ScrapyPriorityQueue( self.crawler, self.downstream_queue_cls, @@ -186,11 +246,11 @@ class DownloaderAwarePriorityQueue: startprios, ) - def pop(self): + def pop(self) -> Optional[Request]: stats = self._downloader_interface.stats(self.pqueues) if not stats: - return + return None slot = min(stats)[1] queue = self.pqueues[slot] @@ -199,14 +259,14 @@ class DownloaderAwarePriorityQueue: del self.pqueues[slot] return request - def push(self, request): + def push(self, request: Request) -> None: slot = self._downloader_interface.get_slot_key(request) if slot not in self.pqueues: self.pqueues[slot] = self.pqfactory(slot) queue = self.pqueues[slot] queue.push(request) - def peek(self): + def peek(self) -> Optional[Request]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -220,13 +280,13 @@ class DownloaderAwarePriorityQueue: queue = self.pqueues[slot] return queue.peek() - def close(self): + def close(self) -> Dict[str, List[int]]: active = {slot: queue.close() for slot, queue in self.pqueues.items()} self.pqueues.clear() return active - def __len__(self): + def __len__(self) -> int: return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 - def __contains__(self, slot): + def __contains__(self, slot: str) -> bool: return slot in self.pqueues From 21fa0761818c158ae9fc35b49ea8d2300f0fa510 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 22:41:58 +0500 Subject: [PATCH 147/269] Fix MutableMapping import for Python 3.8. --- scrapy/item.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scrapy/item.py b/scrapy/item.py index e04e994ef..2daea64cc 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -7,10 +7,18 @@ See documentation in docs/topics/item.rst from __future__ import annotations from abc import ABCMeta -from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat -from typing import TYPE_CHECKING, Any, Dict, Iterator, KeysView, NoReturn, Tuple +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterator, + KeysView, + MutableMapping, + NoReturn, + Tuple, +) from scrapy.utils.trackref import object_ref From ad35ffdb0da052d0df194ce5dc1ba7e8d823190f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 23:10:03 +0500 Subject: [PATCH 148/269] Full typing for scrapy/resolver.py. --- scrapy/resolver.py | 67 +++++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index e2e8beff4..ba7cd716b 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,8 +1,12 @@ -from typing import Any +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Type from twisted.internet import defer -from twisted.internet.base import ThreadedResolver +from twisted.internet.base import ReactorBase, ThreadedResolver +from twisted.internet.defer import Deferred from twisted.internet.interfaces import ( + IAddress, IHostnameResolver, IHostResolution, IResolutionReceiver, @@ -12,6 +16,12 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + # TODO: cache misses dnscache: LocalCache[str, Any] = LocalCache(10000) @@ -22,65 +32,66 @@ class CachingThreadedResolver(ThreadedResolver): Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests. """ - def __init__(self, reactor, cache_size, timeout): + def __init__(self, reactor: ReactorBase, cache_size: int, timeout: float): super().__init__(reactor) dnscache.limit = cache_size self.timeout = timeout @classmethod - def from_crawler(cls, crawler, reactor): + def from_crawler(cls, crawler: Crawler, reactor: ReactorBase) -> Self: if crawler.settings.getbool("DNSCACHE_ENABLED"): cache_size = crawler.settings.getint("DNSCACHE_SIZE") else: cache_size = 0 return cls(reactor, cache_size, crawler.settings.getfloat("DNS_TIMEOUT")) - def install_on_reactor(self): + def install_on_reactor(self) -> None: self.reactor.installResolver(self) - def getHostByName(self, name: str, timeout=None): + def getHostByName(self, name: str, timeout: Sequence[int] = ()) -> Deferred[str]: if name in dnscache: return defer.succeed(dnscache[name]) # in Twisted<=16.6, getHostByName() is always called with # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple), # so the input argument above is simply overridden # to enforce Scrapy's DNS_TIMEOUT setting's value - timeout = (self.timeout,) + # The timeout arg is typed as Sequence[int] but supports floats. + timeout = (self.timeout,) # type: ignore[assignment] d = super().getHostByName(name, timeout) if dnscache.limit: d.addCallback(self._cache_result, name) return d - def _cache_result(self, result, name): + def _cache_result(self, result: Any, name: str) -> Any: dnscache[name] = result return result @implementer(IHostResolution) class HostResolution: - def __init__(self, name): - self.name = name + def __init__(self, name: str): + self.name: str = name - def cancel(self): + def cancel(self) -> None: raise NotImplementedError() @provider(IResolutionReceiver) class _CachingResolutionReceiver: - def __init__(self, resolutionReceiver, hostName): - self.resolutionReceiver = resolutionReceiver - self.hostName = hostName - self.addresses = [] + def __init__(self, resolutionReceiver: IResolutionReceiver, hostName: str): + self.resolutionReceiver: IResolutionReceiver = resolutionReceiver + self.hostName: str = hostName + self.addresses: List[IAddress] = [] - def resolutionBegan(self, resolution): + def resolutionBegan(self, resolution: IHostResolution) -> None: self.resolutionReceiver.resolutionBegan(resolution) self.resolution = resolution - def addressResolved(self, address): + def addressResolved(self, address: IAddress) -> None: self.resolutionReceiver.addressResolved(address) self.addresses.append(address) - def resolutionComplete(self): + def resolutionComplete(self) -> None: self.resolutionReceiver.resolutionComplete() if self.addresses: dnscache[self.hostName] = self.addresses @@ -93,30 +104,30 @@ class CachingHostnameResolver: does not support setting a timeout value for DNS requests. """ - def __init__(self, reactor, cache_size): - self.reactor = reactor - self.original_resolver = reactor.nameResolver + def __init__(self, reactor: ReactorBase, cache_size: int): + self.reactor: ReactorBase = reactor + self.original_resolver: IHostnameResolver = reactor.nameResolver dnscache.limit = cache_size @classmethod - def from_crawler(cls, crawler, reactor): + def from_crawler(cls, crawler: Crawler, reactor: ReactorBase) -> Self: if crawler.settings.getbool("DNSCACHE_ENABLED"): cache_size = crawler.settings.getint("DNSCACHE_SIZE") else: cache_size = 0 return cls(reactor, cache_size) - def install_on_reactor(self): + def install_on_reactor(self) -> None: self.reactor.installNameResolver(self) def resolveHostName( self, - resolutionReceiver, + resolutionReceiver: IResolutionReceiver, hostName: str, - portNumber=0, - addressTypes=None, - transportSemantics="TCP", - ): + portNumber: int = 0, + addressTypes: Optional[Sequence[Type[IAddress]]] = None, + transportSemantics: str = "TCP", + ) -> IHostResolution: try: addresses = dnscache[hostName] except KeyError: From b749db92e5b15c974b0d77280c22b63000ad4263 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 23:17:59 +0500 Subject: [PATCH 149/269] Full typing for scrapy/robotstxt.py. --- scrapy/robotstxt.py | 65 +++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index ad06137e2..a33f73306 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -3,9 +3,10 @@ from __future__ import annotations import logging import sys from abc import ABCMeta, abstractmethod -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Optional, Union from warnings import warn +from scrapy import Spider from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import to_unicode @@ -18,12 +19,14 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): +def decode_robotstxt( + robotstxt_body: bytes, spider: Optional[Spider], to_native_str_type: bool = False +) -> str: try: if to_native_str_type: - robotstxt_body = to_unicode(robotstxt_body) + body_decoded = to_unicode(robotstxt_body) else: - robotstxt_body = robotstxt_body.decode("utf-8", errors="ignore") + body_decoded = robotstxt_body.decode("utf-8", errors="ignore") except UnicodeDecodeError: # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it. # Switch to 'allow all' state. @@ -33,8 +36,8 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): exc_info=sys.exc_info(), extra={"spider": spider}, ) - robotstxt_body = "" - return robotstxt_body + body_decoded = "" + return body_decoded class RobotParser(metaclass=ABCMeta): @@ -66,82 +69,80 @@ class RobotParser(metaclass=ABCMeta): class PythonRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): + def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): from urllib.robotparser import RobotFileParser - self.spider = spider - robotstxt_body = decode_robotstxt( - robotstxt_body, spider, to_native_str_type=True - ) - self.rp = RobotFileParser() - self.rp.parse(robotstxt_body.splitlines()) + self.spider: Optional[Spider] = spider + body_decoded = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True) + self.rp: RobotFileParser = RobotFileParser() + self.rp.parse(body_decoded.splitlines()) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider o = cls(robotstxt_body, spider) return o - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(user_agent, url) class ReppyRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): + def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2) from reppy.robots import Robots - self.spider = spider + self.spider: Optional[Spider] = spider self.rp = Robots.parse("", robotstxt_body) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider o = cls(robotstxt_body, spider) return o - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: return self.rp.allowed(url, user_agent) class RerpRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): + def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): from robotexclusionrulesparser import RobotExclusionRulesParser - self.spider = spider - self.rp = RobotExclusionRulesParser() - robotstxt_body = decode_robotstxt(robotstxt_body, spider) - self.rp.parse(robotstxt_body) + self.spider: Optional[Spider] = spider + self.rp: RobotExclusionRulesParser = RobotExclusionRulesParser() + body_decoded = decode_robotstxt(robotstxt_body, spider) + self.rp.parse(body_decoded) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider o = cls(robotstxt_body, spider) return o - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.is_allowed(user_agent, url) class ProtegoRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): + def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): from protego import Protego - self.spider = spider - robotstxt_body = decode_robotstxt(robotstxt_body, spider) - self.rp = Protego.parse(robotstxt_body) + self.spider: Optional[Spider] = spider + body_decoded = decode_robotstxt(robotstxt_body, spider) + self.rp = Protego.parse(body_decoded) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider o = cls(robotstxt_body, spider) return o - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(url, user_agent) From 5f7fd2a653407da3eb3e53c853209d9bfb6b275f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 23:39:31 +0500 Subject: [PATCH 150/269] Full typing for scrapy/squeues.py. --- scrapy/squeues.py | 58 +++++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index e20f60f06..4676b058e 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -2,20 +2,28 @@ Scheduler queues """ +from __future__ import annotations + import marshal import pickle # nosec from os import PathLike from pathlib import Path -from typing import Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union from queuelib import queue +from scrapy import Request +from scrapy.crawler import Crawler from scrapy.utils.request import request_from_dict +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self -def _with_mkdir(queue_class): + +def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]: class DirectoriesCreated(queue_class): - def __init__(self, path: Union[str, PathLike], *args, **kwargs): + def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any): dirname = Path(path).parent if not dirname.exists(): dirname.mkdir(parents=True, exist_ok=True) @@ -24,18 +32,23 @@ def _with_mkdir(queue_class): return DirectoriesCreated -def _serializable_queue(queue_class, serialize, deserialize): +def _serializable_queue( + queue_class: Type[queue.BaseQueue], + serialize: Callable[[Any], bytes], + deserialize: Callable[[bytes], Any], +) -> Type[queue.BaseQueue]: class SerializableQueue(queue_class): - def push(self, obj): + def push(self, obj: Any) -> None: s = serialize(obj) super().push(s) - def pop(self): + def pop(self) -> Optional[Any]: s = super().pop() if s: return deserialize(s) + return None - def peek(self): + def peek(self) -> Optional[Any]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -50,31 +63,36 @@ def _serializable_queue(queue_class, serialize, deserialize): ) from ex if s: return deserialize(s) + return None return SerializableQueue -def _scrapy_serialization_queue(queue_class): +def _scrapy_serialization_queue( + queue_class: Type[queue.BaseQueue], +) -> Type[queue.BaseQueue]: class ScrapyRequestQueue(queue_class): - def __init__(self, crawler, key): + def __init__(self, crawler: Crawler, key: str): self.spider = crawler.spider super().__init__(key) @classmethod - def from_crawler(cls, crawler, key, *args, **kwargs): + def from_crawler( + cls, crawler: Crawler, key: str, *args: Any, **kwargs: Any + ) -> Self: return cls(crawler, key) - def push(self, request): - request = request.to_dict(spider=self.spider) - return super().push(request) + def push(self, request: Request) -> None: + request_dict = request.to_dict(spider=self.spider) + super().push(request_dict) - def pop(self): + def pop(self) -> Optional[Request]: request = super().pop() if not request: return None return request_from_dict(request, spider=self.spider) - def peek(self): + def peek(self) -> Optional[Request]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -89,13 +107,15 @@ def _scrapy_serialization_queue(queue_class): return ScrapyRequestQueue -def _scrapy_non_serialization_queue(queue_class): +def _scrapy_non_serialization_queue( + queue_class: Type[queue.BaseQueue], +) -> Type[queue.BaseQueue]: class ScrapyRequestQueue(queue_class): @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: return cls() - def peek(self): + def peek(self) -> Optional[Any]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -113,7 +133,7 @@ def _scrapy_non_serialization_queue(queue_class): return ScrapyRequestQueue -def _pickle_serialize(obj): +def _pickle_serialize(obj: Any) -> bytes: try: return pickle.dumps(obj, protocol=4) # Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s) From 203fa9667fb69f6251c0a44b14cf0450ce769a32 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 23:47:55 +0500 Subject: [PATCH 151/269] Add queue typing to scrapy/core/scheduler.py. --- scrapy/core/scheduler.py | 33 +++++++++++++++++---------------- scrapy/pqueues.py | 3 ++- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index b2209e53f..ab59c0d14 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,13 +4,15 @@ import json import logging from abc import abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, cast +from typing import TYPE_CHECKING, Any, Optional, Type, cast +from queuelib.queue import BaseQueue from twisted.internet.defer import Deferred from scrapy.crawler import Crawler from scrapy.dupefilters import BaseDupeFilter from scrapy.http.request import Request +from scrapy.pqueues import ScrapyPriorityQueue from scrapy.spiders import Spider from scrapy.statscollectors import StatsCollector from scrapy.utils.job import job_dir @@ -121,9 +123,6 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): raise NotImplementedError() -SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler") - - class Scheduler(BaseScheduler): """ Default Scrapy scheduler. This implementation also handles duplication @@ -179,24 +178,24 @@ class Scheduler(BaseScheduler): self, dupefilter: BaseDupeFilter, jobdir: Optional[str] = None, - dqclass=None, - mqclass=None, + dqclass: Optional[Type[BaseQueue]] = None, + mqclass: Optional[Type[BaseQueue]] = None, logunser: bool = False, stats: Optional[StatsCollector] = None, - pqclass=None, + pqclass: Optional[Type[ScrapyPriorityQueue]] = None, crawler: Optional[Crawler] = None, ): self.df: BaseDupeFilter = dupefilter self.dqdir: Optional[str] = self._dqdir(jobdir) - self.pqclass = pqclass - self.dqclass = dqclass - self.mqclass = mqclass + self.pqclass: Optional[Type[ScrapyPriorityQueue]] = pqclass + self.dqclass: Optional[Type[BaseQueue]] = dqclass + self.mqclass: Optional[Type[BaseQueue]] = mqclass self.logunser: bool = logunser self.stats: Optional[StatsCollector] = stats self.crawler: Optional[Crawler] = crawler @classmethod - def from_crawler(cls: Type[SchedulerTV], crawler: Crawler) -> SchedulerTV: + def from_crawler(cls, crawler: Crawler) -> Self: """ Factory method, initializes the scheduler with arguments taken from the crawl settings """ @@ -221,9 +220,9 @@ class Scheduler(BaseScheduler): (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory (3) return the result of the dupefilter's ``open`` method """ - self.spider = spider - self.mqs = self._mq() - self.dqs = self._dq() if self.dqdir else None + self.spider: Spider = spider + self.mqs: ScrapyPriorityQueue = self._mq() + self.dqs: Optional[ScrapyPriorityQueue] = self._dq() if self.dqdir else None return self.df.open() def close(self, reason: str) -> Optional[Deferred]: @@ -320,9 +319,10 @@ class Scheduler(BaseScheduler): return self.dqs.pop() return None - def _mq(self): + def _mq(self) -> ScrapyPriorityQueue: """Create a new priority queue instance, with in-memory storage""" assert self.crawler + assert self.pqclass return build_from_crawler( self.pqclass, self.crawler, @@ -330,10 +330,11 @@ class Scheduler(BaseScheduler): key="", ) - def _dq(self): + def _dq(self) -> ScrapyPriorityQueue: """Create a new priority queue instance, with disk storage""" assert self.crawler assert self.dqdir + assert self.pqclass state = self._read_dqs_state(self.dqdir) q = build_from_crawler( self.pqclass, diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 213ad590d..773825c5e 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -16,13 +16,14 @@ from typing import ( from scrapy import Request from scrapy.core.downloader import Downloader -from scrapy.crawler import Crawler from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) From bd0d4cee885744c7ea38185ec42f0137e7632b79 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 May 2024 16:12:44 +0500 Subject: [PATCH 152/269] Fixes for queuelib. --- scrapy/core/scheduler.py | 4 +++- scrapy/squeues.py | 21 +++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index ab59c0d14..f30a5d9c9 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -6,7 +6,6 @@ from abc import abstractmethod from pathlib import Path from typing import TYPE_CHECKING, Any, Optional, Type, cast -from queuelib.queue import BaseQueue from twisted.internet.defer import Deferred from scrapy.crawler import Crawler @@ -19,6 +18,9 @@ from scrapy.utils.job import job_dir from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: + # requires queuelib >= 1.6.2 + from queuelib.queue import BaseQueue + # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 4676b058e..6f80ee388 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -22,7 +22,7 @@ if TYPE_CHECKING: def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]: - class DirectoriesCreated(queue_class): + class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc] def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any): dirname = Path(path).parent if not dirname.exists(): @@ -37,7 +37,7 @@ def _serializable_queue( serialize: Callable[[Any], bytes], deserialize: Callable[[bytes], Any], ) -> Type[queue.BaseQueue]: - class SerializableQueue(queue_class): + class SerializableQueue(queue_class): # type: ignore[valid-type,misc] def push(self, obj: Any) -> None: s = serialize(obj) super().push(s) @@ -71,7 +71,7 @@ def _serializable_queue( def _scrapy_serialization_queue( queue_class: Type[queue.BaseQueue], ) -> Type[queue.BaseQueue]: - class ScrapyRequestQueue(queue_class): + class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc] def __init__(self, crawler: Crawler, key: str): self.spider = crawler.spider super().__init__(key) @@ -110,7 +110,7 @@ def _scrapy_serialization_queue( def _scrapy_non_serialization_queue( queue_class: Type[queue.BaseQueue], ) -> Type[queue.BaseQueue]: - class ScrapyRequestQueue(queue_class): + class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc] @classmethod def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: return cls() @@ -142,17 +142,18 @@ def _pickle_serialize(obj: Any) -> bytes: raise ValueError(str(e)) from e +# queue.*Queue aren't subclasses of queue.BaseQueue _PickleFifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads + _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type] ) _PickleLifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads + _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type] ) _MarshalFifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads + _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type] ) _MarshalLifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads + _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type] ) # public queue classes @@ -160,5 +161,5 @@ PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQu PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue) MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue) MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue) -FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) -LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) +FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) # type: ignore[arg-type] +LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) # type: ignore[arg-type] From 1f394306e14ccab9d14ffb9adc64a7c5c08d9af6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 May 2024 20:37:30 +0500 Subject: [PATCH 153/269] Use the Self type hint in from_crawler/from_settings. --- scrapy/core/downloader/contextfactory.py | 7 +++++- scrapy/core/downloader/handlers/ftp.py | 10 +++++++- scrapy/core/downloader/handlers/http10.py | 10 +++++++- scrapy/core/downloader/handlers/http11.py | 10 +++++++- scrapy/core/downloader/handlers/http2.py | 14 +++++------ scrapy/core/downloader/handlers/s3.py | 10 +++++++- scrapy/http/request/__init__.py | 13 ++++++---- scrapy/pipelines/files.py | 20 ++++++++++++---- scrapy/pipelines/images.py | 29 ++++++++++++++++++----- scrapy/pipelines/media.py | 12 ++++++++-- scrapy/spiders/crawl.py | 12 +++++++--- scrapy/spiders/sitemap.py | 4 +++- scrapy/utils/request.py | 9 +++++-- 13 files changed, 124 insertions(+), 36 deletions(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index dba4d8cdc..6a82634f1 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import warnings from typing import TYPE_CHECKING, Any, List, Optional @@ -25,6 +27,9 @@ from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from twisted.internet._sslverify import ClientTLSOptions + # typing.Self requires Python 3.11 + from typing_extensions import Self + @implementer(IPolicyForHTTPS) class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): @@ -62,7 +67,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): method: int = SSL.SSLv23_METHOD, *args: Any, **kwargs: Any, - ): + ) -> Self: tls_verbose_logging: bool = settings.getbool( "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" ) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 4081545ce..69add8558 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -28,18 +28,26 @@ In case of status 200 request, response.headers will come with two keys: 'Size' - with size of the downloaded data """ +from __future__ import annotations + import re from io import BytesIO +from typing import TYPE_CHECKING from urllib.parse import unquote from twisted.internet.protocol import ClientCreator, Protocol from twisted.protocols.ftp import CommandFailed, FTPClient +from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class ReceivedDataProtocol(Protocol): def __init__(self, filename=None): @@ -76,7 +84,7 @@ class FTPDownloadHandler: self.passive_mode = settings["FTP_PASSIVE_MODE"] @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) def download_request(self, request, spider): diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index d168c2b2e..256dc36a1 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -1,9 +1,17 @@ """Download handlers for http and https schemes """ +from __future__ import annotations + +from typing import TYPE_CHECKING + from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class HTTP10DownloadHandler: lazy = False @@ -17,7 +25,7 @@ class HTTP10DownloadHandler: self._crawler = crawler @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler) -> Self: return cls(crawler.settings, crawler) def download_request(self, request, spider): diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c3704de3d..15f8abc64 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -1,11 +1,14 @@ """Download handlers for http and https schemes""" +from __future__ import annotations + import ipaddress import logging import re from contextlib import suppress from io import BytesIO from time import time +from typing import TYPE_CHECKING from urllib.parse import urldefrag, urlunparse from twisted.internet import defer, protocol, ssl @@ -32,6 +35,11 @@ from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.python import to_bytes, to_unicode +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) @@ -56,7 +64,7 @@ class HTTP11DownloadHandler: self._disconnect_timeout = 1 @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler) -> Self: return cls(crawler.settings, crawler) def download_request(self, request, spider): diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index b2579362c..e9a6b6fa3 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,5 +1,7 @@ +from __future__ import annotations + from time import time -from typing import Optional, Type, TypeVar +from typing import TYPE_CHECKING, Optional from urllib.parse import urldefrag from twisted.internet.base import DelayedCall @@ -16,9 +18,9 @@ from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.python import to_bytes -H2DownloadHandlerOrSubclass = TypeVar( - "H2DownloadHandlerOrSubclass", bound="H2DownloadHandler" -) +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class H2DownloadHandler: @@ -31,9 +33,7 @@ class H2DownloadHandler: self._context_factory = load_context_factory_from_settings(settings, crawler) @classmethod - def from_crawler( - cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler - ) -> H2DownloadHandlerOrSubclass: + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) def download_request(self, request: Request, spider: Spider) -> Deferred: diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 1f7533759..99fbb49ce 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,9 +1,17 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.exceptions import NotConfigured from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class S3DownloadHandler: def __init__( @@ -57,7 +65,7 @@ class S3DownloadHandler: self._download_http = _http_handler.download_request @classmethod - def from_crawler(cls, crawler, **kwargs): + def from_crawler(cls, crawler, **kwargs) -> Self: return cls(crawler.settings, crawler=crawler, **kwargs) def download_request(self, request, spider): diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 6269ee86a..191b3cef4 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -5,8 +5,11 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ +from __future__ import annotations + import inspect from typing import ( + TYPE_CHECKING, Any, AnyStr, Callable, @@ -17,8 +20,6 @@ from typing import ( NoReturn, Optional, Tuple, - Type, - TypeVar, Union, cast, ) @@ -32,7 +33,9 @@ from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax -RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn: @@ -186,11 +189,11 @@ class Request(object_ref): @classmethod def from_curl( - cls: Type[RequestTypeVar], + cls, curl_command: str, ignore_unknown_options: bool = True, **kwargs: Any, - ) -> RequestTypeVar: + ) -> Self: """Create a Request object from a string containing a `cURL `_ command. It populates the HTTP method, the URL, the headers, the cookies and the body. It accepts the same diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index d04218089..d00f44502 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -4,6 +4,8 @@ Files Pipeline See documentation in topics/media-pipeline.rst """ +from __future__ import annotations + import base64 import functools import hashlib @@ -16,7 +18,7 @@ from ftplib import FTP from io import BytesIO from os import PathLike from pathlib import Path -from typing import IO, DefaultDict, Optional, Set, Union +from typing import IO, TYPE_CHECKING, DefaultDict, Optional, Set, Type, Union, cast from urllib.parse import urlparse from itemadapter import ItemAdapter @@ -34,6 +36,10 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) @@ -385,8 +391,8 @@ class FilesPipeline(MediaPipeline): super().__init__(download_func=download_func, settings=settings) @classmethod - def from_settings(cls, settings): - s3store = cls.STORE_SCHEMES["s3"] + def from_settings(cls, settings) -> Self: + s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] @@ -396,11 +402,15 @@ class FilesPipeline(MediaPipeline): s3store.AWS_VERIFY = settings["AWS_VERIFY"] s3store.POLICY = settings["FILES_STORE_S3_ACL"] - gcs_store = cls.STORE_SCHEMES["gs"] + gcs_store: Type[GCSFilesStore] = cast( + Type[GCSFilesStore], cls.STORE_SCHEMES["gs"] + ) gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] gcs_store.POLICY = settings["FILES_STORE_GCS_ACL"] or None - ftp_store = cls.STORE_SCHEMES["ftp"] + ftp_store: Type[FTPFilesStore] = cast( + Type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] + ) ftp_store.FTP_USERNAME = settings["FTP_USER"] ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE") diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 137aa7a9a..e7ef06fb3 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -4,25 +4,38 @@ Images Pipeline See documentation in topics/media-pipeline.rst """ +from __future__ import annotations + import functools import hashlib import warnings from contextlib import suppress from io import BytesIO from os import PathLike -from typing import Dict, Tuple, Union +from typing import TYPE_CHECKING, Dict, Tuple, Type, Union, cast from itemadapter import ItemAdapter from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum +from scrapy.pipelines.files import ( + FileException, + FilesPipeline, + FTPFilesStore, + GCSFilesStore, + S3FilesStore, + _md5sum, +) # TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.utils.python import get_func_args, to_bytes +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class NoimagesDrop(DropItem): """Product with no images exception""" @@ -96,8 +109,8 @@ class ImagesPipeline(FilesPipeline): self._deprecated_convert_image = None @classmethod - def from_settings(cls, settings): - s3store = cls.STORE_SCHEMES["s3"] + def from_settings(cls, settings) -> Self: + s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] @@ -107,11 +120,15 @@ class ImagesPipeline(FilesPipeline): s3store.AWS_VERIFY = settings["AWS_VERIFY"] s3store.POLICY = settings["IMAGES_STORE_S3_ACL"] - gcs_store = cls.STORE_SCHEMES["gs"] + gcs_store: Type[GCSFilesStore] = cast( + Type[GCSFilesStore], cls.STORE_SCHEMES["gs"] + ) gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None - ftp_store = cls.STORE_SCHEMES["ftp"] + ftp_store: Type[FTPFilesStore] = cast( + Type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] + ) ftp_store.FTP_USERNAME = settings["FTP_USER"] ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE") diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index fc156ab41..fd5e70cb9 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,6 +1,9 @@ +from __future__ import annotations + import functools import logging from collections import defaultdict +from typing import TYPE_CHECKING from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure @@ -12,6 +15,11 @@ from scrapy.utils.defer import defer_result, mustbe_deferred from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) @@ -67,9 +75,9 @@ class MediaPipeline: return formatted_key @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler) -> Self: try: - pipe = cls.from_settings(crawler.settings) + pipe = cls.from_settings(crawler.settings) # type: ignore[attr-defined] except AttributeError: pipe = cls() pipe.crawler = crawler diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 2a3913da5..ba8b7b366 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -5,8 +5,10 @@ for scraping typical web sites that requires crawling pages. See documentation in docs/topics/spiders.rst """ +from __future__ import annotations + import copy -from typing import AsyncIterable, Awaitable, Sequence +from typing import TYPE_CHECKING, AsyncIterable, Awaitable, Sequence from scrapy.http import HtmlResponse, Request, Response from scrapy.linkextractors import LinkExtractor @@ -14,6 +16,10 @@ from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + def _identity(x): return x @@ -140,9 +146,9 @@ class CrawlSpider(Spider): self._rules[-1]._compile(self) @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler, *args, **kwargs) -> Self: spider = super().from_crawler(crawler, *args, **kwargs) - spider._follow_links = crawler.settings.getbool( + spider._follow_links = crawler.settings.getbool( # type: ignore[attr-defined] "CRAWLSPIDER_FOLLOW_LINKS", True ) return spider diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index cd83a1464..f0e630c42 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging import re from typing import TYPE_CHECKING, Any @@ -26,7 +28,7 @@ class SitemapSpider(Spider): _warn_size: int @classmethod - def from_crawler(cls, crawler: "Crawler", *args: Any, **kwargs: Any) -> "Self": + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = super().from_crawler(crawler, *args, **kwargs) spider._max_size = getattr( spider, "download_maxsize", spider.settings.getint("DOWNLOAD_MAXSIZE") diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 1f07d58eb..c86f9fe39 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -3,6 +3,8 @@ This module provides some useful functions for working with scrapy.http.Request objects """ +from __future__ import annotations + import hashlib import json import warnings @@ -32,6 +34,9 @@ from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + from scrapy.crawler import Crawler @@ -133,10 +138,10 @@ class RequestFingerprinter: """ @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler) -> Self: return cls(crawler) - def __init__(self, crawler: Optional["Crawler"] = None): + def __init__(self, crawler: Optional[Crawler] = None): if crawler: implementation = crawler.settings.get( "REQUEST_FINGERPRINTER_IMPLEMENTATION" From 8a08283580176049cb539423795830b9faea91a9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 5 May 2024 22:32:46 +0500 Subject: [PATCH 154/269] Full typing for scrapy/http/cookies.py. --- scrapy/http/cookies.py | 123 +++++++++++++++++++++-------------- scrapy/http/response/text.py | 4 +- 2 files changed, 76 insertions(+), 51 deletions(-) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 72855bad5..8af89c74f 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,36 +1,56 @@ +from __future__ import annotations + import re import time from http.cookiejar import Cookie from http.cookiejar import CookieJar as _CookieJar -from http.cookiejar import DefaultCookiePolicy -from typing import Sequence +from http.cookiejar import CookiePolicy, DefaultCookiePolicy +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterator, + List, + Optional, + Sequence, + Tuple, + cast, +) from scrapy import Request from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + # Defined in the http.cookiejar module, but undocumented: # https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527 IPV4_RE = re.compile(r"\.\d+$", re.ASCII) class CookieJar: - def __init__(self, policy=None, check_expired_frequency=10000): - self.policy = policy or DefaultCookiePolicy() - self.jar = _CookieJar(self.policy) - self.jar._cookies_lock = _DummyLock() - self.check_expired_frequency = check_expired_frequency - self.processed = 0 + def __init__( + self, + policy: Optional[CookiePolicy] = None, + check_expired_frequency: int = 10000, + ): + self.policy: CookiePolicy = policy or DefaultCookiePolicy() + self.jar: _CookieJar = _CookieJar(self.policy) + self.jar._cookies_lock = _DummyLock() # type: ignore[attr-defined] + self.check_expired_frequency: int = check_expired_frequency + self.processed: int = 0 - def extract_cookies(self, response, request): + def extract_cookies(self, response: Response, request: Request) -> None: wreq = WrappedRequest(request) wrsp = WrappedResponse(response) - return self.jar.extract_cookies(wrsp, wreq) + self.jar.extract_cookies(wrsp, wreq) # type: ignore[arg-type] def add_cookie_header(self, request: Request) -> None: wreq = WrappedRequest(request) - self.policy._now = self.jar._now = int(time.time()) + self.policy._now = self.jar._now = int(time.time()) # type: ignore[attr-defined] # the cookiejar implementation iterates through all domains # instead we restrict to potential matches on the domain @@ -47,10 +67,10 @@ class CookieJar: cookies = [] for host in hosts: - if host in self.jar._cookies: - cookies += self.jar._cookies_for_domain(host, wreq) + if host in self.jar._cookies: # type: ignore[attr-defined] + cookies += self.jar._cookies_for_domain(host, wreq) # type: ignore[attr-defined] - attrs = self.jar._cookie_attrs(cookies) + attrs = self.jar._cookie_attrs(cookies) # type: ignore[attr-defined] if attrs: if not wreq.has_header("Cookie"): wreq.add_unredirected_header("Cookie", "; ".join(attrs)) @@ -61,37 +81,42 @@ class CookieJar: self.jar.clear_expired_cookies() @property - def _cookies(self): - return self.jar._cookies + def _cookies(self) -> Dict[str, Dict[str, Dict[str, Cookie]]]: + return self.jar._cookies # type: ignore[attr-defined,no-any-return] - def clear_session_cookies(self, *args, **kwargs): - return self.jar.clear_session_cookies(*args, **kwargs) + def clear_session_cookies(self) -> None: + return self.jar.clear_session_cookies() - def clear(self, domain=None, path=None, name=None): - return self.jar.clear(domain, path, name) + def clear( + self, + domain: Optional[str] = None, + path: Optional[str] = None, + name: Optional[str] = None, + ) -> None: + self.jar.clear(domain, path, name) - def __iter__(self): + def __iter__(self) -> Iterator[Cookie]: return iter(self.jar) - def __len__(self): + def __len__(self) -> int: return len(self.jar) - def set_policy(self, pol): - return self.jar.set_policy(pol) + def set_policy(self, pol: CookiePolicy) -> None: + self.jar.set_policy(pol) def make_cookies(self, response: Response, request: Request) -> Sequence[Cookie]: wreq = WrappedRequest(request) wrsp = WrappedResponse(response) - return self.jar.make_cookies(wrsp, wreq) + return self.jar.make_cookies(wrsp, wreq) # type: ignore[arg-type] - def set_cookie(self, cookie): + def set_cookie(self, cookie: Cookie) -> None: self.jar.set_cookie(cookie) def set_cookie_if_ok(self, cookie: Cookie, request: Request) -> None: - self.jar.set_cookie_if_ok(cookie, WrappedRequest(request)) + self.jar.set_cookie_if_ok(cookie, WrappedRequest(request)) # type: ignore[arg-type] -def potential_domain_matches(domain): +def potential_domain_matches(domain: str) -> List[str]: """Potential domain matches for a cookie >>> potential_domain_matches('www.example.com') @@ -111,10 +136,10 @@ def potential_domain_matches(domain): class _DummyLock: - def acquire(self): + def acquire(self) -> None: pass - def release(self): + def release(self) -> None: pass @@ -124,19 +149,19 @@ class WrappedRequest: see http://docs.python.org/library/urllib2.html#urllib2.Request """ - def __init__(self, request): + def __init__(self, request: Request): self.request = request - def get_full_url(self): + def get_full_url(self) -> str: return self.request.url - def get_host(self): + def get_host(self) -> str: return urlparse_cached(self.request).netloc - def get_type(self): + def get_type(self) -> str: return urlparse_cached(self.request).scheme - def is_unverifiable(self): + def is_unverifiable(self) -> bool: """Unverifiable should indicate whether the request is unverifiable, as defined by RFC 2965. It defaults to False. An unverifiable request is one whose URL the user did not have the @@ -144,36 +169,36 @@ class WrappedRequest: HTML document, and the user had no option to approve the automatic fetching of the image, this should be true. """ - return self.request.meta.get("is_unverifiable", False) + return cast(bool, self.request.meta.get("is_unverifiable", False)) @property - def full_url(self): + def full_url(self) -> str: return self.get_full_url() @property - def host(self): + def host(self) -> str: return self.get_host() @property - def type(self): + def type(self) -> str: return self.get_type() @property - def unverifiable(self): + def unverifiable(self) -> bool: return self.is_unverifiable() @property - def origin_req_host(self): - return urlparse_cached(self.request).hostname + def origin_req_host(self) -> str: + return cast(str, urlparse_cached(self.request).hostname) - def has_header(self, name): + def has_header(self, name: str) -> bool: return name in self.request.headers - def get_header(self, name, default=None): + def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]: value = self.request.headers.get(name, default) return to_unicode(value, errors="replace") if value is not None else None - def header_items(self): + def header_items(self) -> List[Tuple[str, List[str]]]: return [ ( to_unicode(k, errors="replace"), @@ -182,18 +207,18 @@ class WrappedRequest: for k, v in self.request.headers.items() ] - def add_unredirected_header(self, name, value): + def add_unredirected_header(self, name: str, value: str) -> None: self.request.headers.appendlist(name, value) class WrappedResponse: - def __init__(self, response): + def __init__(self, response: Response): self.response = response - def info(self): + def info(self) -> Self: return self - def get_all(self, name, default=None): + def get_all(self, name: str, default: Any = None) -> List[str]: return [ to_unicode(v, errors="replace") for v in self.response.headers.getlist(name) ] diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 2816610fb..522ffc0d5 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -159,12 +159,12 @@ class TextResponse(Response): def jmespath(self, query: str, **kwargs: Any) -> SelectorList: from scrapy.selector import SelectorList - if not hasattr(self.selector, "jmespath"): # type: ignore[attr-defined] + if not hasattr(self.selector, "jmespath"): raise AttributeError( "Please install parsel >= 1.8.1 to get jmespath support" ) - return cast(SelectorList, self.selector.jmespath(query, **kwargs)) # type: ignore[attr-defined] + return cast(SelectorList, self.selector.jmespath(query, **kwargs)) def xpath(self, query: str, **kwargs: Any) -> SelectorList: from scrapy.selector import SelectorList From c76dfc383f34514a5a2841d2e32ac4e0c8c751a3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 5 May 2024 22:52:15 +0500 Subject: [PATCH 155/269] Full typing for scrapy/linkextractors. --- scrapy/commands/bench.py | 3 +- scrapy/linkextractors/__init__.py | 8 +- scrapy/linkextractors/lxmlhtml.py | 149 +++++++++++++++++------------- 3 files changed, 91 insertions(+), 69 deletions(-) diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 2e6bb5d86..7523f3cfe 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -8,7 +8,7 @@ from urllib.parse import urlencode import scrapy from scrapy import Request from scrapy.commands import ScrapyCommand -from scrapy.http import Response +from scrapy.http import Response, TextResponse from scrapy.linkextractors import LinkExtractor @@ -61,5 +61,6 @@ class _BenchSpider(scrapy.Spider): return [scrapy.Request(url, dont_filter=True)] def parse(self, response: Response) -> Any: # type: ignore[override] + assert isinstance(Response, TextResponse) for link in self.link_extractor.extract_links(response): yield scrapy.Request(link.url, callback=self.parse) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 73a63651c..38dbe8135 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -7,6 +7,7 @@ For more info see docs/topics/link-extractors.rst """ import re +from typing import Iterable # common file extensions that are not followed if they occur in links IGNORED_EXTENSIONS = [ @@ -110,14 +111,11 @@ IGNORED_EXTENSIONS = [ ] -_re_type = type(re.compile("", 0)) - - -def _matches(url, regexs): +def _matches(url: str, regexs: Iterable[re.Pattern[str]]) -> bool: return any(r.search(url) for r in regexs) -def _is_valid_url(url): +def _is_valid_url(url: str) -> bool: return url.split("://", 1)[0] in {"http", "https", "file", "ftp"} diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 71c6d08fc..3fa7d1e3a 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -5,21 +5,19 @@ Link extractor based on lxml.html import logging import operator from functools import partial +from typing import Any, Callable, Iterable, List, Optional, Set, Tuple, Union, cast from urllib.parse import urljoin, urlparse from lxml import etree # nosec +from lxml.html import HtmlElement # nosec from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string +from scrapy import Selector +from scrapy.http import TextResponse from scrapy.link import Link -from scrapy.linkextractors import ( - IGNORED_EXTENSIONS, - _is_valid_url, - _matches, - _re_type, - re, -) +from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches, re from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url @@ -33,41 +31,56 @@ XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml" _collect_string_content = etree.XPath("string()") -def _nons(tag): +def _nons(tag: Any) -> Any: if isinstance(tag, str): if tag[0] == "{" and tag[1 : len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE: return tag.split("}")[-1] return tag -def _identity(x): +def _identity(x: Any) -> Any: return x -def _canonicalize_link_url(link): +def _canonicalize_link_url(link: Link) -> str: return canonicalize_url(link.url, keep_fragments=True) class LxmlParserLinkExtractor: def __init__( self, - tag="a", - attr="href", - process=None, - unique=False, - strip=True, - canonicalized=False, + tag: Union[str, Callable[[str], bool]] = "a", + attr: Union[str, Callable[[str], bool]] = "href", + process: Optional[Callable[[Any], Any]] = None, + unique: bool = False, + strip: bool = True, + canonicalized: bool = False, ): - self.scan_tag = tag if callable(tag) else partial(operator.eq, tag) - self.scan_attr = attr if callable(attr) else partial(operator.eq, attr) - self.process_attr = process if callable(process) else _identity - self.unique = unique - self.strip = strip - self.link_key = ( - operator.attrgetter("url") if canonicalized else _canonicalize_link_url + # mypy doesn't infer types for operator.* and also for partial() + self.scan_tag: Callable[[str], bool] = ( + tag + if callable(tag) + else cast(Callable[[str], bool], partial(operator.eq, tag)) + ) + self.scan_attr: Callable[[str], bool] = ( + attr + if callable(attr) + else cast(Callable[[str], bool], partial(operator.eq, attr)) + ) + self.process_attr: Callable[[Any], Any] = ( + process if callable(process) else _identity + ) + self.unique: bool = unique + self.strip: bool = strip + self.link_key: Callable[[Link], str] = ( + cast(Callable[[Link], str], operator.attrgetter("url")) + if canonicalized + else _canonicalize_link_url ) - def _iter_links(self, document): + def _iter_links( + self, document: HtmlElement + ) -> Iterable[Tuple[HtmlElement, str, str]]: for el in document.iter(etree.Element): if not self.scan_tag(_nons(el.tag)): continue @@ -75,10 +88,16 @@ class LxmlParserLinkExtractor: for attrib in attribs: if not self.scan_attr(attrib): continue - yield (el, attrib, attribs[attrib]) + yield el, attrib, attribs[attrib] - def _extract_links(self, selector, response_url, response_encoding, base_url): - links = [] + def _extract_links( + self, + selector: Selector, + response_url: str, + response_encoding: str, + base_url: str, + ) -> List[Link]: + links: List[Link] = [] # hacky way to get the underlying lxml parsed document for el, attr, attr_val in self._iter_links(selector.root): # pseudo lxml.html.HtmlElement.make_links_absolute(base_url) @@ -108,44 +127,48 @@ class LxmlParserLinkExtractor: links.append(link) return self._deduplicate_if_needed(links) - def extract_links(self, response): + def extract_links(self, response: TextResponse) -> List[Link]: base_url = get_base_url(response) return self._extract_links( response.selector, response.url, response.encoding, base_url ) - def _process_links(self, links): + def _process_links(self, links: List[Link]) -> List[Link]: """Normalize and filter extracted links The subclass should override it if necessary """ return self._deduplicate_if_needed(links) - def _deduplicate_if_needed(self, links): + def _deduplicate_if_needed(self, links: List[Link]) -> List[Link]: if self.unique: return unique_list(links, key=self.link_key) return links +_RegexT = Union[str, re.Pattern[str]] +_RegexOrSeveralT = Union[_RegexT, Iterable[_RegexT]] + + class LxmlLinkExtractor: _csstranslator = HTMLTranslator() def __init__( self, - allow=(), - deny=(), - allow_domains=(), - deny_domains=(), - restrict_xpaths=(), - tags=("a", "area"), - attrs=("href",), - canonicalize=False, - unique=True, - process_value=None, - deny_extensions=None, - restrict_css=(), - strip=True, - restrict_text=None, + allow: _RegexOrSeveralT = (), + deny: _RegexOrSeveralT = (), + allow_domains: Union[str, Iterable[str]] = (), + deny_domains: Union[str, Iterable[str]] = (), + restrict_xpaths: Union[str, Iterable[str]] = (), + tags: Union[str, Iterable[str]] = ("a", "area"), + attrs: Union[str, Iterable[str]] = ("href",), + canonicalize: bool = False, + unique: bool = True, + process_value: Optional[Callable[[Any], Any]] = None, + deny_extensions: Union[str, Iterable[str], None] = None, + restrict_css: Union[str, Iterable[str]] = (), + strip: bool = True, + restrict_text: Optional[_RegexOrSeveralT] = None, ): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) self.link_extractor = LxmlParserLinkExtractor( @@ -156,31 +179,31 @@ class LxmlLinkExtractor: strip=strip, canonicalized=not canonicalize, ) - self.allow_res = [ - x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow) - ] - self.deny_res = [ - x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny) - ] + self.allow_res: List[re.Pattern[str]] = self._compile_regexes(allow) + self.deny_res: List[re.Pattern[str]] = self._compile_regexes(deny) - self.allow_domains = set(arg_to_iter(allow_domains)) - self.deny_domains = set(arg_to_iter(deny_domains)) + self.allow_domains: Set[str] = set(arg_to_iter(allow_domains)) + self.deny_domains: Set[str] = set(arg_to_iter(deny_domains)) - self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) + self.restrict_xpaths: Tuple[str, ...] = tuple(arg_to_iter(restrict_xpaths)) self.restrict_xpaths += tuple( map(self._csstranslator.css_to_xpath, arg_to_iter(restrict_css)) ) if deny_extensions is None: deny_extensions = IGNORED_EXTENSIONS - self.canonicalize = canonicalize - self.deny_extensions = {"." + e for e in arg_to_iter(deny_extensions)} - self.restrict_text = [ - x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(restrict_text) + self.canonicalize: bool = canonicalize + self.deny_extensions: Set[str] = {"." + e for e in arg_to_iter(deny_extensions)} + self.restrict_text: List[re.Pattern[str]] = self._compile_regexes(restrict_text) + + @staticmethod + def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> List[re.Pattern[str]]: + return [ + x if isinstance(x, re.Pattern) else re.compile(x) + for x in arg_to_iter(value) ] - def _link_allowed(self, link): + def _link_allowed(self, link: Link) -> bool: if not _is_valid_url(link.url): return False if self.allow_res and not _matches(link.url, self.allow_res): @@ -202,7 +225,7 @@ class LxmlLinkExtractor: return False return True - def matches(self, url): + def matches(self, url: str) -> bool: if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): return False if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): @@ -216,7 +239,7 @@ class LxmlLinkExtractor: denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] return any(allowed) and not any(denied) - def _process_links(self, links): + def _process_links(self, links: List[Link]) -> List[Link]: links = [x for x in links if self._link_allowed(x)] if self.canonicalize: for link in links: @@ -224,10 +247,10 @@ class LxmlLinkExtractor: links = self.link_extractor._process_links(links) return links - def _extract_links(self, *args, **kwargs): + def _extract_links(self, *args: Any, **kwargs: Any) -> List[Link]: return self.link_extractor._extract_links(*args, **kwargs) - def extract_links(self, response): + def extract_links(self, response: TextResponse) -> List[Link]: """Returns a list of :class:`~scrapy.link.Link` objects from the specified :class:`response `. From 4b47a5dc32232431494b38f3a1d9ddb3dbaa6247 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 5 May 2024 22:55:21 +0500 Subject: [PATCH 156/269] Skip coverage checks for TYPE_CHECKING blocks. --- .coveragerc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.coveragerc b/.coveragerc index ad0ee0f6c..f9ad353d5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,3 +4,9 @@ include = scrapy/* omit = tests/* disable_warnings = include-ignored + +[report] +# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185 +exclude_lines = + pragma: no cover + if TYPE_CHECKING: From 40e4a5960477299e36a0f7363db4a3c03576f64e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 5 May 2024 23:00:16 +0500 Subject: [PATCH 157/269] Fix Python 3.8. --- scrapy/linkextractors/__init__.py | 4 ++-- scrapy/linkextractors/lxmlhtml.py | 23 +++++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 38dbe8135..d59005edd 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -7,7 +7,7 @@ For more info see docs/topics/link-extractors.rst """ import re -from typing import Iterable +from typing import Iterable, Pattern # common file extensions that are not followed if they occur in links IGNORED_EXTENSIONS = [ @@ -111,7 +111,7 @@ IGNORED_EXTENSIONS = [ ] -def _matches(url: str, regexs: Iterable[re.Pattern[str]]) -> bool: +def _matches(url: str, regexs: Iterable[Pattern[str]]) -> bool: return any(r.search(url) for r in regexs) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 3fa7d1e3a..33a10cd6c 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -5,7 +5,18 @@ Link extractor based on lxml.html import logging import operator from functools import partial -from typing import Any, Callable, Iterable, List, Optional, Set, Tuple, Union, cast +from typing import ( + Any, + Callable, + Iterable, + List, + Optional, + Pattern, + Set, + Tuple, + Union, + cast, +) from urllib.parse import urljoin, urlparse from lxml import etree # nosec @@ -146,7 +157,7 @@ class LxmlParserLinkExtractor: return links -_RegexT = Union[str, re.Pattern[str]] +_RegexT = Union[str, Pattern[str]] _RegexOrSeveralT = Union[_RegexT, Iterable[_RegexT]] @@ -179,8 +190,8 @@ class LxmlLinkExtractor: strip=strip, canonicalized=not canonicalize, ) - self.allow_res: List[re.Pattern[str]] = self._compile_regexes(allow) - self.deny_res: List[re.Pattern[str]] = self._compile_regexes(deny) + self.allow_res: List[Pattern[str]] = self._compile_regexes(allow) + self.deny_res: List[Pattern[str]] = self._compile_regexes(deny) self.allow_domains: Set[str] = set(arg_to_iter(allow_domains)) self.deny_domains: Set[str] = set(arg_to_iter(deny_domains)) @@ -194,10 +205,10 @@ class LxmlLinkExtractor: deny_extensions = IGNORED_EXTENSIONS self.canonicalize: bool = canonicalize self.deny_extensions: Set[str] = {"." + e for e in arg_to_iter(deny_extensions)} - self.restrict_text: List[re.Pattern[str]] = self._compile_regexes(restrict_text) + self.restrict_text: List[Pattern[str]] = self._compile_regexes(restrict_text) @staticmethod - def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> List[re.Pattern[str]]: + def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> List[Pattern[str]]: return [ x if isinstance(x, re.Pattern) else re.compile(x) for x in arg_to_iter(value) From aa025d7eacb461ccb0c724584532d402bb400bd1 Mon Sep 17 00:00:00 2001 From: Sanchay Kumar <51812506+kumar-sanchay@users.noreply.github.com> Date: Mon, 6 May 2024 14:59:35 +0530 Subject: [PATCH 158/269] Indicate that Selector.type can be json (#6334) --- scrapy/selector/unified.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index aa9581fcd..e852aadc7 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -47,7 +47,7 @@ class Selector(_ParselSelector, object_ref): ``response`` isn't available. Using ``text`` and ``response`` together is undefined behavior. - ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` + ``type`` defines the selector type, it can be ``"html"``, ``"xml"``, ``"json"`` or ``None`` (default). If ``type`` is ``None``, the selector automatically chooses the best type From 2cba7896d26dda51ff2e598300363531ebd328b8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 6 May 2024 14:31:24 +0500 Subject: [PATCH 159/269] Small fix for _get_slot_key(). --- scrapy/core/downloader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index f88da41ea..98e1af6fb 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -143,7 +143,7 @@ class Downloader: return key, self.slots[key] - def _get_slot_key(self, request: Request, spider: Any) -> str: + def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str: if self.DOWNLOAD_SLOT in request.meta: return cast(str, request.meta[self.DOWNLOAD_SLOT]) From c4d2748ff572adad8150f1bc09b2d52e61d9dfc8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 May 2024 17:07:19 +0500 Subject: [PATCH 160/269] Small typing improvements in scrapy/core/downloader/contextfactory.py. --- scrapy/core/downloader/contextfactory.py | 7 +++++-- scrapy/core/downloader/handlers/http10.py | 2 +- scrapy/core/downloader/handlers/http11.py | 2 +- scrapy/core/downloader/handlers/http2.py | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 6a82634f1..0e77cd2fe 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -21,6 +21,7 @@ from scrapy.core.downloader.tls import ( ScrapyClientTLSOptions, openssl_methods, ) +from scrapy.crawler import Crawler from scrapy.settings import BaseSettings from scrapy.utils.misc import build_from_crawler, load_object @@ -102,7 +103,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context: - ctx = self.getCertificateOptions().getContext() + ctx: SSL.Context = self.getCertificateOptions().getContext() ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT return ctx @@ -165,7 +166,9 @@ class AcceptableProtocolsContextFactory: return options -def load_context_factory_from_settings(settings, crawler): +def load_context_factory_from_settings( + settings: BaseSettings, crawler: Crawler +) -> IPolicyForHTTPS: ssl_method = openssl_methods[settings.get("DOWNLOADER_CLIENT_TLS_METHOD")] context_factory_cls = load_object(settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]) # try method-aware context factory diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 256dc36a1..2507a4231 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -16,7 +16,7 @@ if TYPE_CHECKING: class HTTP10DownloadHandler: lazy = False - def __init__(self, settings, crawler=None): + def __init__(self, settings, crawler): self.HTTPClientFactory = load_object(settings["DOWNLOADER_HTTPCLIENTFACTORY"]) self.ClientContextFactory = load_object( settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 15f8abc64..52561f7e6 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -46,7 +46,7 @@ logger = logging.getLogger(__name__) class HTTP11DownloadHandler: lazy = False - def __init__(self, settings, crawler=None): + def __init__(self, settings, crawler): self._crawler = crawler from twisted.internet import reactor diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index e9a6b6fa3..efe45c459 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -24,7 +24,7 @@ if TYPE_CHECKING: class H2DownloadHandler: - def __init__(self, settings: Settings, crawler: Optional[Crawler] = None): + def __init__(self, settings: Settings, crawler: Crawler): self._crawler = crawler from twisted.internet import reactor From ec4d40702227f5486c184de9439da4896121a33b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 May 2024 17:36:16 +0500 Subject: [PATCH 161/269] Full typing for smaller download handlers. --- scrapy/core/downloader/handlers/file.py | 4 +++- scrapy/core/downloader/handlers/http11.py | 6 ++--- scrapy/core/downloader/handlers/http2.py | 3 ++- scrapy/core/downloader/handlers/s3.py | 27 ++++++++++++++--------- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 4824167da..17dd7483b 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -2,6 +2,8 @@ from pathlib import Path from w3lib.url import file_uri_to_path +from scrapy import Request, Spider +from scrapy.http import Response from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers @@ -10,7 +12,7 @@ class FileDownloadHandler: lazy = False @defers - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Response: filepath = file_uri_to_path(request.url) body = Path(filepath).read_bytes() respcls = responsetypes.from_args(filename=filepath, body=body) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 52561f7e6..2e7ea559a 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -27,11 +27,11 @@ from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer from zope.interface import implementer -from scrapy import signals +from scrapy import Request, Spider, signals from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.exceptions import StopDownload -from scrapy.http import Headers +from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes from scrapy.utils.python import to_bytes, to_unicode @@ -67,7 +67,7 @@ class HTTP11DownloadHandler: def from_crawler(cls, crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Response: """Return a deferred for the HTTP download""" agent = ScrapyAgent( contextFactory=self._contextFactory, diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index efe45c459..16fc1e3ae 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -8,6 +8,7 @@ from twisted.internet.base import DelayedCall from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.web.client import URI +from twisted.web.iweb import IPolicyForHTTPS from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse @@ -54,7 +55,7 @@ class ScrapyH2Agent: def __init__( self, - context_factory, + context_factory: IPolicyForHTTPS, pool: H2ConnectionPool, connect_timeout: int = 10, bind_address: Optional[bytes] = None, diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 99fbb49ce..c88dd2cdc 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,9 +1,13 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Optional, Type +from scrapy import Request, Spider from scrapy.core.downloader.handlers.http import HTTPDownloadHandler +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response +from scrapy.settings import BaseSettings from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler @@ -16,14 +20,14 @@ if TYPE_CHECKING: class S3DownloadHandler: def __init__( self, - settings, + settings: BaseSettings, *, - crawler=None, - aws_access_key_id=None, - aws_secret_access_key=None, - aws_session_token=None, - httpdownloadhandler=HTTPDownloadHandler, - **kw, + crawler: Crawler, + aws_access_key_id: Optional[str] = None, + aws_secret_access_key: Optional[str] = None, + aws_session_token: Optional[str] = None, + httpdownloadhandler: Type[HTTPDownloadHandler] = HTTPDownloadHandler, + **kw: Any, ): if not is_botocore_available(): raise NotConfigured("missing botocore library") @@ -51,6 +55,8 @@ class S3DownloadHandler: if kw: raise TypeError(f"Unexpected keyword arguments: {kw}") if not self.anon: + assert aws_access_key_id is not None + assert aws_secret_access_key is not None SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"] self._signer = SignerCls( botocore.credentials.Credentials( @@ -65,10 +71,10 @@ class S3DownloadHandler: self._download_http = _http_handler.download_request @classmethod - def from_crawler(cls, crawler, **kwargs) -> Self: + def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self: return cls(crawler.settings, crawler=crawler, **kwargs) - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Response: p = urlparse_cached(request) scheme = "https" if request.meta.get("is_secure") else "http" bucket = p.hostname @@ -85,6 +91,7 @@ class S3DownloadHandler: headers=request.headers.to_unicode_dict(), data=request.body, ) + assert self._signer self._signer.add_auth(awsrequest) request = request.replace(url=url, headers=awsrequest.headers.items()) return self._download_http(request, spider) From e8e13ebb78dd1d2db425b285335f31dba7c1fd39 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 May 2024 18:03:06 +0500 Subject: [PATCH 162/269] Full typing for scrapy/core/downloader/handlers/ftp.py. --- scrapy/core/downloader/handlers/ftp.py | 40 +++++++++++++++----------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 69add8558..ed94b2221 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -32,15 +32,19 @@ from __future__ import annotations import re from io import BytesIO -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional from urllib.parse import unquote +from twisted.internet.defer import Deferred from twisted.internet.protocol import ClientCreator, Protocol from twisted.protocols.ftp import CommandFailed, FTPClient +from twisted.python.failure import Failure +from scrapy import Request, Spider from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.responsetypes import responsetypes +from scrapy.settings import BaseSettings from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes @@ -50,20 +54,20 @@ if TYPE_CHECKING: class ReceivedDataProtocol(Protocol): - def __init__(self, filename=None): - self.__filename = filename - self.body = open(filename, "wb") if filename else BytesIO() - self.size = 0 + def __init__(self, filename: Optional[str] = None): + self.__filename: Optional[str] = filename + self.body: BinaryIO = open(filename, "wb") if filename else BytesIO() + self.size: int = 0 - def dataReceived(self, data): + def dataReceived(self, data: bytes) -> None: self.body.write(data) self.size += len(data) @property - def filename(self): + def filename(self) -> Optional[str]: return self.__filename - def close(self): + def close(self) -> None: self.body.close() if self.filename else self.body.seek(0) @@ -73,12 +77,12 @@ _CODE_RE = re.compile(r"\d+") class FTPDownloadHandler: lazy = False - CODE_MAPPING = { + CODE_MAPPING: Dict[str, int] = { "550": 404, "default": 503, } - def __init__(self, settings): + def __init__(self, settings: BaseSettings): self.default_user = settings["FTP_USER"] self.default_password = settings["FTP_PASSWORD"] self.passive_mode = settings["FTP_PASSIVE_MODE"] @@ -87,7 +91,7 @@ class FTPDownloadHandler: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Deferred: from twisted.internet import reactor parsed_url = urlparse_cached(request) @@ -99,10 +103,10 @@ class FTPDownloadHandler: creator = ClientCreator( reactor, FTPClient, user, password, passive=passive_mode ) - dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21) + dfd: Deferred = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21) return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path)) - def gotClient(self, client, request, filepath): + def gotClient(self, client: FTPClient, request: Request, filepath: str) -> Deferred: self.client = client protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename")) return client.retrieveFile(filepath, protocol).addCallbacks( @@ -112,15 +116,18 @@ class FTPDownloadHandler: errbackArgs=(request,), ) - def _build_response(self, result, request, protocol): + def _build_response( + self, result: Any, request: Request, protocol: ReceivedDataProtocol + ) -> Response: self.result = result protocol.close() headers = {"local filename": protocol.filename or "", "size": protocol.size} body = to_bytes(protocol.filename or protocol.body.read()) respcls = responsetypes.from_args(url=request.url, body=body) - return respcls(url=request.url, status=200, body=body, headers=headers) + # hints for Headers-related types may need to be fixed to not use AnyStr + return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type] - def _failed(self, result, request): + def _failed(self, result: Failure, request: Request) -> Response: message = result.getErrorMessage() if result.type == CommandFailed: m = _CODE_RE.search(message) @@ -130,4 +137,5 @@ class FTPDownloadHandler: return Response( url=request.url, status=httpcode, body=to_bytes(message) ) + assert result.type raise result.type(result.value) From af3e38ab1f3dc7095ae27f572cbde0ea652d7664 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 May 2024 18:09:25 +0500 Subject: [PATCH 163/269] Full typing for scrapy/core/downloader/handlers/http10.py. --- scrapy/core/downloader/handlers/http10.py | 27 +++++++++++++++-------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 2507a4231..9117cb818 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -3,8 +3,13 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Type +from twisted.internet.defer import Deferred + +from scrapy import Request, Spider +from scrapy.crawler import Crawler +from scrapy.settings import BaseSettings from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode @@ -12,29 +17,33 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory + from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory class HTTP10DownloadHandler: lazy = False - def __init__(self, settings, crawler): - self.HTTPClientFactory = load_object(settings["DOWNLOADER_HTTPCLIENTFACTORY"]) - self.ClientContextFactory = load_object( + def __init__(self, settings: BaseSettings, crawler: Crawler): + self.HTTPClientFactory: Type[ScrapyHTTPClientFactory] = load_object( + settings["DOWNLOADER_HTTPCLIENTFACTORY"] + ) + self.ClientContextFactory: Type[ScrapyClientContextFactory] = load_object( settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] ) - self._settings = settings - self._crawler = crawler + self._settings: BaseSettings = settings + self._crawler: Crawler = crawler @classmethod - def from_crawler(cls, crawler) -> Self: + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Deferred: """Return a deferred for the HTTP download""" factory = self.HTTPClientFactory(request) self._connect(factory) return factory.deferred - def _connect(self, factory): + def _connect(self, factory: ScrapyHTTPClientFactory) -> Deferred: from twisted.internet import reactor host, port = to_unicode(factory.host), factory.port From 045387e07faba4a18ecff919457d69938e89b710 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 May 2024 19:56:23 +0500 Subject: [PATCH 164/269] More typing for scrapy/core/downloader/handlers/http11.py. --- scrapy/core/downloader/handlers/http11.py | 286 +++++++++++++--------- scrapy/core/downloader/handlers/s3.py | 5 +- 2 files changed, 169 insertions(+), 122 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 2e7ea559a..40ae1921a 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,31 +8,33 @@ import re from contextlib import suppress from io import BytesIO from time import time -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast from urllib.parse import urldefrag, urlunparse -from twisted.internet import defer, protocol, ssl +from twisted.internet import ssl +from twisted.internet.base import ReactorBase +from twisted.internet.defer import CancelledError, Deferred, succeed from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError +from twisted.internet.interfaces import IConsumer +from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.python.failure import Failure -from twisted.web.client import ( - URI, - Agent, - HTTPConnectionPool, - ResponseDone, - ResponseFailed, -) +from twisted.web.client import URI, Agent, HTTPConnectionPool +from twisted.web.client import Response as TxResponse +from twisted.web.client import ResponseDone, ResponseFailed from twisted.web.http import PotentialDataLoss, _DataLoss from twisted.web.http_headers import Headers as TxHeaders -from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS from zope.interface import implementer from scrapy import Request, Spider, signals from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse +from scrapy.crawler import Crawler from scrapy.exceptions import StopDownload from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes +from scrapy.settings import BaseSettings from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: @@ -46,28 +48,30 @@ logger = logging.getLogger(__name__) class HTTP11DownloadHandler: lazy = False - def __init__(self, settings, crawler): + def __init__(self, settings: BaseSettings, crawler: Crawler): self._crawler = crawler from twisted.internet import reactor - self._pool = HTTPConnectionPool(reactor, persistent=True) + self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True) self._pool.maxPersistentPerHost = settings.getint( "CONCURRENT_REQUESTS_PER_DOMAIN" ) self._pool._factory.noisy = False - self._contextFactory = load_context_factory_from_settings(settings, crawler) - self._default_maxsize = settings.getint("DOWNLOAD_MAXSIZE") - self._default_warnsize = settings.getint("DOWNLOAD_WARNSIZE") - self._fail_on_dataloss = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS") - self._disconnect_timeout = 1 + self._contextFactory: IPolicyForHTTPS = load_context_factory_from_settings( + settings, crawler + ) + self._default_maxsize: int = settings.getint("DOWNLOAD_MAXSIZE") + self._default_warnsize: int = settings.getint("DOWNLOAD_WARNSIZE") + self._fail_on_dataloss: bool = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS") + self._disconnect_timeout: int = 1 @classmethod - def from_crawler(cls, crawler) -> Self: + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider: Spider) -> Response: + def download_request(self, request: Request, spider: Spider) -> Deferred: """Return a deferred for the HTTP download""" agent = ScrapyAgent( contextFactory=self._contextFactory, @@ -79,10 +83,10 @@ class HTTP11DownloadHandler: ) return agent.download_request(request) - def close(self): + def close(self) -> Deferred: from twisted.internet import reactor - d = self._pool.closeCachedConnections() + d: Deferred = self._pool.closeCachedConnections() # closeCachedConnections will hang on network or server issues, so # we'll manually timeout the deferred. # @@ -93,7 +97,7 @@ class HTTP11DownloadHandler: # issue a callback after `_disconnect_timeout` seconds. delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, []) - def cancel_delayed_call(result): + def cancel_delayed_call(result: Any) -> Any: if delayed_call.active(): delayed_call.cancel() return result @@ -123,39 +127,41 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def __init__( self, - reactor, - host, - port, - proxyConf, - contextFactory, - timeout=30, - bindAddress=None, + reactor: ReactorBase, + host: str, + port: int, + proxyConf: Tuple[str, int, Optional[bytes]], + contextFactory: IPolicyForHTTPS, + timeout: float = 30, + bindAddress: Optional[Tuple[str, int]] = None, ): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) - self._tunnelReadyDeferred = defer.Deferred() - self._tunneledHost = host - self._tunneledPort = port - self._contextFactory = contextFactory - self._connectBuffer = bytearray() + self._tunnelReadyDeferred: Deferred = Deferred() + self._tunneledHost: str = host + self._tunneledPort: int = port + self._contextFactory: IPolicyForHTTPS = contextFactory + self._connectBuffer: bytearray = bytearray() - def requestTunnel(self, protocol): + def requestTunnel(self, protocol: Protocol) -> Protocol: """Asks the proxy to open a tunnel.""" + assert protocol.transport tunnelReq = tunnel_request_data( self._tunneledHost, self._tunneledPort, self._proxyAuthHeader ) protocol.transport.write(tunnelReq) self._protocolDataReceived = protocol.dataReceived - protocol.dataReceived = self.processProxyResponse + protocol.dataReceived = self.processProxyResponse # type: ignore[method-assign] self._protocol = protocol return protocol - def processProxyResponse(self, rcvd_bytes): + def processProxyResponse(self, data: bytes) -> None: """Processes the response from the proxy. If the tunnel is successfully created, notifies the client that we are ready to send requests. If not raises a TunnelError. """ - self._connectBuffer += rcvd_bytes + assert self._protocol.transport + self._connectBuffer += data # make sure that enough (all) bytes are consumed # and that we've got all HTTP headers (ending with a blank line) # from the proxy so that we don't send those bytes to the TLS layer @@ -163,23 +169,24 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): # see https://github.com/scrapy/scrapy/issues/2491 if b"\r\n\r\n" not in self._connectBuffer: return - self._protocol.dataReceived = self._protocolDataReceived + self._protocol.dataReceived = self._protocolDataReceived # type: ignore[method-assign] respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer) if respm and int(respm.group("status")) == 200: # set proper Server Name Indication extension - sslOptions = self._contextFactory.creatorForNetloc( + sslOptions = self._contextFactory.creatorForNetloc( # type: ignore[call-arg,misc] self._tunneledHost, self._tunneledPort ) self._protocol.transport.startTLS(sslOptions, self._protocolFactory) self._tunnelReadyDeferred.callback(self._protocol) else: + extra: Any if respm: extra = { "status": int(respm.group("status")), "reason": respm.group("reason").strip(), } else: - extra = rcvd_bytes[: self._truncatedLength] + extra = data[: self._truncatedLength] self._tunnelReadyDeferred.errback( TunnelError( "Could not open CONNECT tunnel with proxy " @@ -187,11 +194,11 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): ) ) - def connectFailed(self, reason): + def connectFailed(self, reason: Failure) -> None: """Propagates the errback to the appropriate deferred.""" self._tunnelReadyDeferred.errback(reason) - def connect(self, protocolFactory): + def connect(self, protocolFactory: Factory) -> Deferred: self._protocolFactory = protocolFactory connectDeferred = super().connect(protocolFactory) connectDeferred.addCallback(self.requestTunnel) @@ -199,7 +206,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): return self._tunnelReadyDeferred -def tunnel_request_data(host, port, proxy_auth_header=None): +def tunnel_request_data( + host: str, port: int, proxy_auth_header: Optional[bytes] = None +) -> bytes: r""" Return binary content of a CONNECT request. @@ -230,18 +239,20 @@ class TunnelingAgent(Agent): def __init__( self, - reactor, - proxyConf, - contextFactory=None, - connectTimeout=None, - bindAddress=None, - pool=None, + reactor: ReactorBase, + proxyConf: Tuple[str, int, Optional[bytes]], + contextFactory: Optional[IPolicyForHTTPS] = None, + connectTimeout: Optional[float] = None, + bindAddress: Optional[bytes] = None, + pool: Optional[HTTPConnectionPool] = None, ): + # TODO make this arg required instead + assert contextFactory is not None super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) - self._proxyConf = proxyConf - self._contextFactory = contextFactory + self._proxyConf: Tuple[str, int, Optional[bytes]] = proxyConf + self._contextFactory: IPolicyForHTTPS = contextFactory - def _getEndpoint(self, uri): + def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint: return TunnelingTCP4ClientEndpoint( reactor=self._reactor, host=uri.host, @@ -253,8 +264,15 @@ class TunnelingAgent(Agent): ) def _requestWithEndpoint( - self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath - ): + self, + key: Any, + endpoint: TCP4ClientEndpoint, + method: bytes, + parsedURI: bytes, + headers: Optional[TxHeaders], + bodyProducer: Optional[IBodyProducer], + requestPath: bytes, + ) -> Deferred: # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy @@ -272,7 +290,12 @@ class TunnelingAgent(Agent): class ScrapyProxyAgent(Agent): def __init__( - self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None + self, + reactor: ReactorBase, + proxyURI: bytes, + connectTimeout: Optional[float] = None, + bindAddress: Optional[bytes] = None, + pool: Optional[HTTPConnectionPool] = None, ): super().__init__( reactor=reactor, @@ -280,9 +303,15 @@ class ScrapyProxyAgent(Agent): bindAddress=bindAddress, pool=pool, ) - self._proxyURI = URI.fromBytes(proxyURI) + self._proxyURI: URI = URI.fromBytes(proxyURI) - def request(self, method, uri, headers=None, bodyProducer=None): + def request( + self, + method: bytes, + uri: bytes, + headers: Optional[TxHeaders] = None, + bodyProducer: Optional[IBodyProducer] = None, + ) -> Deferred: """ Issue a new request via the configured proxy. """ @@ -306,26 +335,29 @@ class ScrapyAgent: def __init__( self, - contextFactory=None, - connectTimeout=10, - bindAddress=None, - pool=None, - maxsize=0, - warnsize=0, - fail_on_dataloss=True, - crawler=None, + contextFactory: Optional[IPolicyForHTTPS] = None, + connectTimeout: float = 10, + bindAddress: Optional[bytes] = None, + pool: Optional[HTTPConnectionPool] = None, + maxsize: int = 0, + warnsize: int = 0, + fail_on_dataloss: bool = True, + crawler: Optional[Crawler] = None, ): - self._contextFactory = contextFactory - self._connectTimeout = connectTimeout - self._bindAddress = bindAddress - self._pool = pool - self._maxsize = maxsize - self._warnsize = warnsize - self._fail_on_dataloss = fail_on_dataloss - self._txresponse = None - self._crawler = crawler + # TODO make these args required instead + assert contextFactory is not None + assert crawler is not None + self._contextFactory: IPolicyForHTTPS = contextFactory + self._connectTimeout: float = connectTimeout + self._bindAddress: Optional[bytes] = bindAddress + self._pool: Optional[HTTPConnectionPool] = pool + self._maxsize: int = maxsize + self._warnsize: int = warnsize + self._fail_on_dataloss: bool = fail_on_dataloss + self._txresponse: Optional[TxResponse] = None + self._crawler: Crawler = crawler - def _get_agent(self, request, timeout): + def _get_agent(self, request: Request, timeout: float) -> Agent: from twisted.internet import reactor bindaddress = request.meta.get("bindaddress") or self._bindAddress @@ -333,10 +365,10 @@ class ScrapyAgent: if proxy: proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] - proxyHost = to_unicode(proxyHost) + proxyHost_str = to_unicode(proxyHost) if scheme == b"https": proxyAuth = request.headers.get(b"Proxy-Authorization", None) - proxyConf = (proxyHost, proxyPort, proxyAuth) + proxyConf = (proxyHost_str, proxyPort, proxyAuth) return self._TunnelingAgent( reactor=reactor, proxyConf=proxyConf, @@ -346,7 +378,9 @@ class ScrapyAgent: pool=self._pool, ) proxyScheme = proxyScheme or b"http" - proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, "", "", "")) + proxyURI = urlunparse( + (proxyScheme, proxyNetloc, proxyParams, b"", b"", b"") + ) return self._ProxyAgent( reactor=reactor, proxyURI=to_bytes(proxyURI, encoding="ascii"), @@ -363,7 +397,7 @@ class ScrapyAgent: pool=self._pool, ) - def download_request(self, request): + def download_request(self, request: Request) -> Deferred: from twisted.internet import reactor timeout = request.meta.get("download_timeout") or self._connectTimeout @@ -380,7 +414,7 @@ class ScrapyAgent: else: bodyproducer = None start_time = time() - d = agent.request( + d: Deferred = agent.request( method, to_bytes(url, encoding="ascii"), headers, bodyproducer ) # set download latency @@ -393,7 +427,9 @@ class ScrapyAgent: d.addBoth(self._cb_timeout, request, url, timeout) return d - def _cb_timeout(self, result, request, url, timeout): + def _cb_timeout( + self, result: Any, request: Request, url: str, timeout: float + ) -> Any: if self._timeout_cl.active(): self._timeout_cl.cancel() return result @@ -404,19 +440,21 @@ class ScrapyAgent: raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") - def _cb_latency(self, result, request, start_time): + def _cb_latency(self, result: Any, request: Request, start_time: float) -> Any: request.meta["download_latency"] = time() - start_time return result @staticmethod - def _headers_from_twisted_response(response): + def _headers_from_twisted_response(response: TxResponse) -> Headers: headers = Headers() if response.length != UNKNOWN_LENGTH: headers[b"Content-Length"] = str(response.length).encode() headers.update(response.headers.getAllRawHeaders()) return headers - def _cb_bodyready(self, txresponse, request): + def _cb_bodyready( + self, txresponse: TxResponse, request: Request + ) -> Union[Dict[str, Any], Deferred]: headers_received_result = self._crawler.signals.send_catch_log( signal=signals.headers_received, headers=self._headers_from_twisted_response(txresponse), @@ -472,7 +510,7 @@ class ScrapyAgent: logger.warning(warning_msg, warning_args) txresponse._transport.loseConnection() - raise defer.CancelledError(warning_msg % warning_args) + raise CancelledError(warning_msg % warning_args) if warnsize and expected_size > warnsize: logger.warning( @@ -481,11 +519,11 @@ class ScrapyAgent: {"size": expected_size, "warnsize": warnsize, "request": request}, ) - def _cancel(_): + def _cancel(_: Any) -> None: # Abort connection immediately. txresponse._transport._producer.abortConnection() - d = defer.Deferred(_cancel) + d: Deferred = Deferred(_cancel) txresponse.deliverBody( _ResponseReader( finished=d, @@ -503,7 +541,9 @@ class ScrapyAgent: return d - def _cb_bodydone(self, result, request, url): + def _cb_bodydone( + self, result: Dict[str, Any], request: Request, url: str + ) -> Union[Response, Failure]: headers = self._headers_from_twisted_response(result["txresponse"]) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) try: @@ -523,53 +563,57 @@ class ScrapyAgent: ) if result.get("failure"): result["failure"].value.response = response - return result["failure"] + return cast(Failure, result["failure"]) return response @implementer(IBodyProducer) class _RequestBodyProducer: - def __init__(self, body): + def __init__(self, body: bytes): self.body = body self.length = len(body) - def startProducing(self, consumer): + def startProducing(self, consumer: IConsumer) -> Deferred: consumer.write(self.body) - return defer.succeed(None) + return succeed(None) - def pauseProducing(self): + def pauseProducing(self) -> None: pass - def stopProducing(self): + def stopProducing(self) -> None: pass -class _ResponseReader(protocol.Protocol): +class _ResponseReader(Protocol): def __init__( self, - finished, - txresponse, - request, - maxsize, - warnsize, - fail_on_dataloss, - crawler, + finished: Deferred, + txresponse: TxResponse, + request: Request, + maxsize: int, + warnsize: int, + fail_on_dataloss: bool, + crawler: Crawler, ): - self._finished = finished - self._txresponse = txresponse - self._request = request - self._bodybuf = BytesIO() - self._maxsize = maxsize - self._warnsize = warnsize - self._fail_on_dataloss = fail_on_dataloss - self._fail_on_dataloss_warned = False - self._reached_warnsize = False - self._bytes_received = 0 - self._certificate = None - self._ip_address = None - self._crawler = crawler + self._finished: Deferred = finished + self._txresponse: TxResponse = txresponse + self._request: Request = request + self._bodybuf: BytesIO = BytesIO() + self._maxsize: int = maxsize + self._warnsize: int = warnsize + self._fail_on_dataloss: bool = fail_on_dataloss + self._fail_on_dataloss_warned: bool = False + self._reached_warnsize: bool = False + self._bytes_received: int = 0 + self._certificate: Optional[ssl.Certificate] = None + self._ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None] = ( + None + ) + self._crawler: Crawler = crawler - def _finish_response(self, flags=None, failure=None): + def _finish_response( + self, flags: Optional[List[str]] = None, failure: Optional[Failure] = None + ) -> None: self._finished.callback( { "txresponse": self._txresponse, @@ -581,7 +625,8 @@ class _ResponseReader(protocol.Protocol): } ) - def connectionMade(self): + def connectionMade(self) -> None: + assert self.transport if self._certificate is None: with suppress(AttributeError): self._certificate = ssl.Certificate( @@ -593,11 +638,12 @@ class _ResponseReader(protocol.Protocol): self.transport._producer.getPeer().host ) - def dataReceived(self, bodyBytes): + def dataReceived(self, bodyBytes: bytes) -> None: # This maybe called several times after cancel was called with buffered data. if self._finished.called: return + assert self.transport self._bodybuf.write(bodyBytes) self._bytes_received += len(bodyBytes) @@ -644,7 +690,7 @@ class _ResponseReader(protocol.Protocol): {"warnsize": self._warnsize, "request": self._request}, ) - def connectionLost(self, reason): + def connectionLost(self, reason: Failure = connectionDone) -> None: if self._finished.called: return diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index c88dd2cdc..9a0811a50 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -2,11 +2,12 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, Optional, Type +from twisted.internet.defer import Deferred + from scrapy import Request, Spider from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response from scrapy.settings import BaseSettings from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached @@ -74,7 +75,7 @@ class S3DownloadHandler: def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self: return cls(crawler.settings, crawler=crawler, **kwargs) - def download_request(self, request: Request, spider: Spider) -> Response: + def download_request(self, request: Request, spider: Spider) -> Deferred: p = urlparse_cached(request) scheme = "https" if request.meta.get("is_secure") else "http" bucket = p.hostname From a828da98c3834ae70a1258278890485803ac7a5c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 6 May 2024 22:34:36 +0500 Subject: [PATCH 165/269] Re-run pre-commit. --- scrapy/core/downloader/handlers/http10.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 9117cb818..da9559525 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -20,6 +20,7 @@ if TYPE_CHECKING: from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory + class HTTP10DownloadHandler: lazy = False From 6bbfb537f9f1ba5dd1c51fc860022adb1d326117 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 8 May 2024 00:39:05 +0500 Subject: [PATCH 166/269] Update MANIFEST.in. --- MANIFEST.in | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 4920dc0c3..06971e39c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,8 @@ -include README.rst -include AUTHORS -include INSTALL -include LICENSE -include MANIFEST.in +include CODE_OF_CONDUCT.md +include CONTRIBUTING.md +include INSTALL.md include NEWS +include SECURITY.md include scrapy/VERSION include scrapy/mime.types @@ -12,16 +11,13 @@ include scrapy/py.typed include codecov.yml include conftest.py include pytest.ini -include requirements-*.txt include tox.ini recursive-include scrapy/templates * -recursive-include scrapy license.txt recursive-include docs * prune docs/build recursive-include extras * -recursive-include bin * recursive-include tests * global-exclude __pycache__ *.py[cod] From 180bc9bad7aceb3a9e10c1411212914bc32fb721 Mon Sep 17 00:00:00 2001 From: aisha-partha <153170327+aisha-partha@users.noreply.github.com> Date: Wed, 8 May 2024 22:06:46 +0530 Subject: [PATCH 167/269] =?UTF-8?q?Closes=20#6342.=20Setting=20METAREFRESH?= =?UTF-8?q?=5FIGNORE=5FTAGS=20to=20[=E2=80=98noscript=E2=80=99]=20by=20def?= =?UTF-8?q?ault?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/downloader-middleware.rst | 4 ++-- scrapy/settings/default_settings.py | 2 +- tests/test_downloadermiddleware_redirect.py | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1abbc4968..3f90cf2ed 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -876,13 +876,13 @@ Whether the Meta Refresh middleware will be enabled. METAREFRESH_IGNORE_TAGS ^^^^^^^^^^^^^^^^^^^^^^^ -Default: ``[]`` +Default: ``['noscript']`` Meta tags within these tags are ignored. .. versionchanged:: 2.0 The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from - ``['script', 'noscript']`` to ``[]``. + ``[]`` to ``['noscript']``. .. setting:: METAREFRESH_MAXDELAY diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 2b3d95a0e..d7ac7ec35 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -239,7 +239,7 @@ MEMUSAGE_NOTIFY_MAIL = [] MEMUSAGE_WARNING_MB = 0 METAREFRESH_ENABLED = True -METAREFRESH_IGNORE_TAGS = [] +METAREFRESH_IGNORE_TAGS = ["noscript"] METAREFRESH_MAXDELAY = 100 NEWSPIDER_MODULE = "" diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 10b8ca9af..83ff25982 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -395,9 +395,8 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): """content="0;URL='http://example.org/newpage'">""" ) rsp = HtmlResponse(req.url, body=body.encode()) - req2 = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req2, Request) - self.assertEqual(req2.url, "http://example.org/newpage") + response = self.mw.process_response(req, rsp, self.spider) + assert isinstance(response, Response) def test_ignore_tags_1_x_list(self): """Test that Scrapy 1.x behavior remains possible""" From 3590a1f66b30edfc836b95bbd0a3611eee9371e2 Mon Sep 17 00:00:00 2001 From: aisha-partha <153170327+aisha-partha@users.noreply.github.com> Date: Wed, 8 May 2024 23:23:17 +0530 Subject: [PATCH 168/269] Closes #6342. Update documentation on METAREFRESH_IGNORE_TAGS default value change --- docs/topics/downloader-middleware.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 3f90cf2ed..2663a3cf3 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -881,6 +881,10 @@ Default: ``['noscript']`` Meta tags within these tags are ignored. .. versionchanged:: 2.0 + The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from + ``['script', 'noscript']`` to ``[]``. + +.. versionchanges:: VERSION The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from ``[]`` to ``['noscript']``. From 2e13a9b8e19ae1c0f595435d7d3ce096c3b29ced Mon Sep 17 00:00:00 2001 From: aisha-partha <153170327+aisha-partha@users.noreply.github.com> Date: Wed, 8 May 2024 23:33:13 +0530 Subject: [PATCH 169/269] Update METAREFRESH_IGNORE_TAGS default value in documentation --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 2663a3cf3..01bde772c 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -876,7 +876,7 @@ Whether the Meta Refresh middleware will be enabled. METAREFRESH_IGNORE_TAGS ^^^^^^^^^^^^^^^^^^^^^^^ -Default: ``['noscript']`` +Default: ``[]`` Meta tags within these tags are ignored. From fe163d98ea81b4aff82bb9b194c4b0268c44f1db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 8 May 2024 20:41:20 +0200 Subject: [PATCH 170/269] Fix typo --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 01bde772c..d4cd062fe 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -884,7 +884,7 @@ Meta tags within these tags are ignored. The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from ``['script', 'noscript']`` to ``[]``. -.. versionchanges:: VERSION +.. versionchanged:: VERSION The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from ``[]`` to ``['noscript']``. From ae7bb849f50af0b91eea4f022d93ad201e545c06 Mon Sep 17 00:00:00 2001 From: Sanchay Kumar <51812506+kumar-sanchay@users.noreply.github.com> Date: Fri, 10 May 2024 15:13:49 +0530 Subject: [PATCH 171/269] Make certain args of ScrapyAgent and TunnelingAgent required (#6349) --- scrapy/core/downloader/handlers/http11.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 40ae1921a..5e84be6ba 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -239,15 +239,14 @@ class TunnelingAgent(Agent): def __init__( self, + *, reactor: ReactorBase, proxyConf: Tuple[str, int, Optional[bytes]], - contextFactory: Optional[IPolicyForHTTPS] = None, + contextFactory: IPolicyForHTTPS, connectTimeout: Optional[float] = None, bindAddress: Optional[bytes] = None, pool: Optional[HTTPConnectionPool] = None, ): - # TODO make this arg required instead - assert contextFactory is not None super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) self._proxyConf: Tuple[str, int, Optional[bytes]] = proxyConf self._contextFactory: IPolicyForHTTPS = contextFactory @@ -335,18 +334,16 @@ class ScrapyAgent: def __init__( self, - contextFactory: Optional[IPolicyForHTTPS] = None, + *, + contextFactory: IPolicyForHTTPS, connectTimeout: float = 10, bindAddress: Optional[bytes] = None, pool: Optional[HTTPConnectionPool] = None, maxsize: int = 0, warnsize: int = 0, fail_on_dataloss: bool = True, - crawler: Optional[Crawler] = None, + crawler: Crawler, ): - # TODO make these args required instead - assert contextFactory is not None - assert crawler is not None self._contextFactory: IPolicyForHTTPS = contextFactory self._connectTimeout: float = connectTimeout self._bindAddress: Optional[bytes] = bindAddress From c9ef5209365bb820ba8f2a3cd9df9fdeca0c9591 Mon Sep 17 00:00:00 2001 From: Sanchay Kumar <51812506+kumar-sanchay@users.noreply.github.com> Date: Fri, 10 May 2024 17:56:45 +0530 Subject: [PATCH 172/269] Add Downloader.get_slot_key() without a spider parameter (#6352) --- docs/news.rst | 14 ++++++++++++++ scrapy/core/downloader/__init__.py | 14 ++++++++++++-- scrapy/pqueues.py | 2 +- tests/test_scheduler.py | 6 +++--- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index fafea0bf8..7db4e59a1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,20 @@ Release notes ============= + +.. _release-VERSION: + +Scrapy VERSION (YYYY-MM-DD) +--------------------------- + +Deprecations +~~~~~~~~~~~~ + +- :func:`scrapy.core.downloader.Downloader._get_slot_key` is now deprecated. + Consider using its corresponding public method get_slot_key() instead. + (:issue:`6340`) + + .. _release-2.11.1: Scrapy 2.11.1 (2024-02-14) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 98e1af6fb..0ab3bdb77 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,4 +1,5 @@ import random +import warnings from collections import deque from datetime import datetime from time import time @@ -10,6 +11,7 @@ from twisted.internet.defer import Deferred from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response from scrapy.resolver import dnscache from scrapy.settings import BaseSettings @@ -125,7 +127,7 @@ class Downloader: return len(self.active) >= self.total_concurrency def _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]: - key = self._get_slot_key(request, spider) + key = self.get_slot_key(request) if key not in self.slots: slot_settings = self.per_slot_settings.get(key, {}) conc = ( @@ -143,7 +145,7 @@ class Downloader: return key, self.slots[key] - def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str: + def get_slot_key(self, request: Request) -> str: if self.DOWNLOAD_SLOT in request.meta: return cast(str, request.meta[self.DOWNLOAD_SLOT]) @@ -153,6 +155,14 @@ class Downloader: return key + def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str: + warnings.warn( + "Use of this protected method is deprecated. Consider using its corresponding public method get_slot_key() instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return self.get_slot_key(request) + def _enqueue_request(self, request: Request, spider: Spider) -> Deferred: key, slot = self._get_slot(request, spider) request.meta[self.DOWNLOAD_SLOT] = key diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 773825c5e..58a47ef0f 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -180,7 +180,7 @@ class DownloaderInterface: return [(self._active_downloads(slot), slot) for slot in possible_slots] def get_slot_key(self, request: Request) -> str: - return self.downloader._get_slot_key(request, None) + return self.downloader.get_slot_key(request) def _active_downloads(self, slot: str) -> int: """Return a number of requests in a Downloader for a given slot""" diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 37099dae6..02b50baa3 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -25,7 +25,7 @@ class MockDownloader: def __init__(self): self.slots = {} - def _get_slot_key(self, request, spider): + def get_slot_key(self, request): if Downloader.DOWNLOAD_SLOT in request.meta: return request.meta[Downloader.DOWNLOAD_SLOT] @@ -273,14 +273,14 @@ class DownloaderAwareSchedulerTestMixin: while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() # pylint: disable=protected-access - slot = downloader._get_slot_key(request, None) + slot = downloader.get_slot_key(request) dequeued_slots.append(slot) downloader.increment(slot) requests.append(request) for request in requests: # pylint: disable=protected-access - slot = downloader._get_slot_key(request, None) + slot = downloader.get_slot_key(request) downloader.decrement(slot) self.assertTrue( From 93f06285309bd46e96fd147bf41e564c94b5bf2b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 13 May 2024 13:55:45 +0400 Subject: [PATCH 173/269] Improve typing for Spider.parse(). (#6274) --- .github/workflows/checks.yml | 3 ++ scrapy/commands/bench.py | 2 +- scrapy/spiders/__init__.py | 19 +++++-- tests_typing/test_spiders.mypy-testing | 68 ++++++++++++++++++++++++++ tox.ini | 9 ++++ 5 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 tests_typing/test_spiders.mypy-testing diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d6fc0f6c5..ed1629b67 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -18,6 +18,9 @@ jobs: - python-version: 3.8 env: TOXENV: typing + - python-version: 3.8 + env: + TOXENV: typing-tests - python-version: "3.11" # Keep in sync with .readthedocs.yml env: TOXENV: docs diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 7523f3cfe..0c4ebcd23 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -60,7 +60,7 @@ class _BenchSpider(scrapy.Spider): url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}" return [scrapy.Request(url, dont_filter=True)] - def parse(self, response: Response) -> Any: # type: ignore[override] + def parse(self, response: Response) -> Any: assert isinstance(Response, TextResponse) for link in self.link_extractor.extract_links(response): yield scrapy.Request(link.url, callback=self.parse) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 72c2aaba7..2416d2a4d 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -17,12 +17,17 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: + from collections.abc import Callable + + # typing.Concatenate requires Python 3.10 # typing.Self requires Python 3.11 - from typing_extensions import Self + from typing_extensions import Concatenate, Self from scrapy.crawler import Crawler from scrapy.settings import BaseSettings + CallbackT = Callable[Concatenate[Response, ...], Any] + class Spider(object_ref): """Base class for scrapy spiders. All spiders must inherit from this @@ -79,10 +84,14 @@ class Spider(object_ref): def _parse(self, response: Response, **kwargs: Any) -> Any: return self.parse(response, **kwargs) - def parse(self, response: Response, **kwargs: Any) -> Any: - raise NotImplementedError( - f"{self.__class__.__name__}.parse callback is not defined" - ) + if TYPE_CHECKING: + parse: CallbackT + else: + + def parse(self, response: Response, **kwargs: Any) -> Any: + raise NotImplementedError( + f"{self.__class__.__name__}.parse callback is not defined" + ) @classmethod def update_settings(cls, settings: BaseSettings) -> None: diff --git a/tests_typing/test_spiders.mypy-testing b/tests_typing/test_spiders.mypy-testing new file mode 100644 index 000000000..162e31d0c --- /dev/null +++ b/tests_typing/test_spiders.mypy-testing @@ -0,0 +1,68 @@ +from typing import Any + +import pytest + +from scrapy.http import HtmlResponse, Response +from scrapy.spiders import Spider + + +class SimpleSpider(Spider): + pass + + +class SameOverrideSpider(Spider): + def parse(self, response: Response, **kwargs: Any) -> Any: + pass + + +class NoKwargsSpider(Spider): + def parse(self, response: Response) -> Any: + pass + + +class SpecificKwargsSpider(Spider): + def parse(self, response: Response, page: int) -> Any: + pass + + +class NarrowOverrideSpider(Spider): + # without type: ignore this produces several note lines in addition to an error line, + # which is unsupported by pytest-mypy-testing + def parse(self, response: HtmlResponse, **kwargs: Any) -> Any: # type: ignore[override] + pass + + +@pytest.mark.mypy_testing +def test_spider_parse() -> None: + spider = Spider() + reveal_type(spider.parse) # R: def (scrapy.http.response.Response, *Any, **Any) -> Any + + +@pytest.mark.mypy_testing +def test_spider_parse_override_simple() -> None: + spider = SimpleSpider() + reveal_type(spider.parse) # R: def (scrapy.http.response.Response, *Any, **Any) -> Any + + +@pytest.mark.mypy_testing +def test_spider_parse_override_same() -> None: + spider = SameOverrideSpider() + reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response, **kwargs: Any) -> Any + + +@pytest.mark.mypy_testing +def test_spider_parse_override_no_kwargs() -> None: + spider = NoKwargsSpider() + reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response) -> Any + + +@pytest.mark.mypy_testing +def test_spider_parse_override_specific_kwargs() -> None: + spider = SpecificKwargsSpider() + reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response, page: builtins.int) -> Any + + +@pytest.mark.mypy_testing +def test_spider_parse_override_narrow() -> None: + spider = NarrowOverrideSpider() + reveal_type(spider.parse) # R: def (response: scrapy.http.response.html.HtmlResponse, **kwargs: Any) -> Any diff --git a/tox.ini b/tox.ini index d7527bb04..ede139756 100644 --- a/tox.ini +++ b/tox.ini @@ -56,6 +56,15 @@ deps = commands = mypy {posargs: scrapy tests} +[testenv:typing-tests] +basepython = python3.8 +deps = + {[test-requirements]deps} + {[testenv:typing]deps} + pytest-mypy-testing==0.1.3 +commands = + pytest {posargs: tests_typing} + [testenv:pre-commit] basepython = python3 deps = From 4ed5c5ae91318768efa338680df67337ed0f67fd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 13 May 2024 14:01:52 +0400 Subject: [PATCH 174/269] Use ParamSpec for callables. (#6353) --- scrapy/cmdline.py | 15 ++- scrapy/core/downloader/handlers/ftp.py | 10 +- scrapy/core/engine.py | 2 +- scrapy/core/scraper.py | 10 +- scrapy/core/spidermw.py | 6 +- scrapy/mail.py | 8 +- scrapy/pipelines/files.py | 3 +- scrapy/pipelines/media.py | 27 ++--- scrapy/shell.py | 4 +- scrapy/utils/decorators.py | 29 ++++-- scrapy/utils/defer.py | 135 +++++++++++++++++-------- scrapy/utils/python.py | 23 ++++- scrapy/utils/reactor.py | 33 ++++-- 13 files changed, 204 insertions(+), 101 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 4df5698a6..da0e51386 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,10 +1,12 @@ +from __future__ import annotations + import argparse import cProfile import inspect import os import sys from importlib.metadata import entry_points -from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type +from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple, Type import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter @@ -15,6 +17,12 @@ from scrapy.utils.misc import walk_modules from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect +if TYPE_CHECKING: + # typing.ParamSpec requires Python 3.10 + from typing_extensions import ParamSpec + + _P = ParamSpec("_P") + class ScrapyArgumentParser(argparse.ArgumentParser): def _parse_optional( @@ -121,7 +129,10 @@ def _print_unknown_command( def _run_print_help( - parser: argparse.ArgumentParser, func: Callable, *a: Any, **kw: Any + parser: argparse.ArgumentParser, + func: Callable[_P, None], + *a: _P.args, + **kw: _P.kwargs, ) -> None: try: func(*a, **kw) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index ed94b2221..77dcf3c38 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -109,12 +109,10 @@ class FTPDownloadHandler: def gotClient(self, client: FTPClient, request: Request, filepath: str) -> Deferred: self.client = client protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename")) - return client.retrieveFile(filepath, protocol).addCallbacks( - callback=self._build_response, - callbackArgs=(request, protocol), - errback=self._failed, - errbackArgs=(request,), - ) + d = client.retrieveFile(filepath, protocol) + d.addCallback(self._build_response, request, protocol) + d.addErrback(self._failed, request) + return d def _build_response( self, result: Any, request: Request, protocol: ReceivedDataProtocol diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 93a0c51bc..6bf3f3e26 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -347,7 +347,7 @@ class ExecutionEngine: assert self.spider is not None dwld = self.downloader.fetch(request, self.spider) - dwld.addCallbacks(_on_success) + dwld.addCallback(_on_success) dwld.addBoth(_on_complete) return dwld diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 272841e01..566e6628b 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -8,7 +8,6 @@ from collections import deque from typing import ( TYPE_CHECKING, Any, - AsyncGenerator, AsyncIterable, Deque, Generator, @@ -18,6 +17,7 @@ from typing import ( Tuple, Type, Union, + cast, ) from itemadapter import is_item @@ -184,7 +184,9 @@ class Scraper: result, request, spider ) # returns spider's processed output dfd.addErrback(self.handle_spider_error, request, result, spider) - dfd.addCallback(self.handle_spider_output, request, result, spider) + dfd.addCallback( + self.handle_spider_output, request, cast(Response, result), spider + ) return dfd def _scrape2( @@ -256,12 +258,12 @@ class Scraper: self, result: Union[Iterable, AsyncIterable], request: Request, - response: Union[Response, Failure], + response: Response, spider: Spider, ) -> Deferred: if not result: return defer_succeed(None) - it: Union[Generator, AsyncGenerator] + it: Union[Iterable, AsyncIterable] if isinstance(result, AsyncIterable): it = aiter_errback( result, self.handle_spider_error, request, response, spider diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 1ccfd08a2..2cef2e1dd 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -303,10 +303,8 @@ class SpiderMiddlewareManager(MiddlewareManager): dfd = mustbe_deferred( self._process_spider_input, scrape_func, response, request, spider ) - dfd.addCallbacks( - callback=deferred_f_from_coro_f(process_callback_output), - errback=process_spider_exception, - ) + dfd.addCallback(deferred_f_from_coro_f(process_callback_output)) + dfd.addErrback(process_spider_exception) return dfd def process_start_requests( diff --git a/scrapy/mail.py b/scrapy/mail.py index 56adba934..fd6302550 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -154,12 +154,8 @@ class MailSender: return None dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8")) - dfd.addCallbacks( - callback=self._sent_ok, - errback=self._sent_failed, - callbackArgs=(to, cc, subject, len(attachs)), - errbackArgs=(to, cc, subject, len(attachs)), - ) + dfd.addCallback(self._sent_ok, to, cc, subject, len(attachs)) + dfd.addErrback(self._sent_failed, to, cc, subject, len(attachs)) reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd) return dfd diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index d00f44502..47457f2a8 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -459,7 +459,8 @@ class FilesPipeline(MediaPipeline): path = self.file_path(request, info=info, item=item) dfd = defer.maybeDeferred(self.store.stat_file, path, info) - dfd.addCallbacks(_onsuccess, lambda _: None) + dfd.addCallback(_onsuccess) + dfd.addErrback(lambda _: None) dfd.addErrback( lambda f: logger.error( self.__class__.__name__ + ".store.stat_file", diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index fd5e70cb9..5f6c5cb07 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -106,10 +106,17 @@ class MediaPipeline: # Return cached result if request was already seen if fp in info.downloaded: - return defer_result(info.downloaded[fp]).addCallbacks(cb, eb) + d = defer_result(info.downloaded[fp]) + d.addCallback(cb) + if eb: + d.addErrback(eb) + return d # Otherwise, wait for result - wad = Deferred().addCallbacks(cb, eb) + wad = Deferred() + wad.addCallback(cb) + if eb: + wad.addErrback(eb) info.waiting[fp].append(wad) # Check if request is downloading right now to avoid doing it twice @@ -140,23 +147,11 @@ class MediaPipeline: if self.download_func: # this ugly code was left only to support tests. TODO: remove dfd = mustbe_deferred(self.download_func, request, info.spider) - dfd.addCallbacks( - callback=self.media_downloaded, - callbackArgs=(request, info), - callbackKeywords={"item": item}, - errback=self.media_failed, - errbackArgs=(request, info), - ) else: self._modify_media_request(request) dfd = self.crawler.engine.download(request) - dfd.addCallbacks( - callback=self.media_downloaded, - callbackArgs=(request, info), - callbackKeywords={"item": item}, - errback=self.media_failed, - errbackArgs=(request, info), - ) + dfd.addCallback(self.media_downloaded, request, info, item=item) + dfd.addErrback(self.media_failed, request, info) return dfd def _cache_result_and_execute_waiters(self, result, fp, info): diff --git a/scrapy/shell.py b/scrapy/shell.py index fac42e8a2..2c22d3d8f 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -231,7 +231,9 @@ def _request_deferred(request: Request) -> defer.Deferred: d: defer.Deferred = defer.Deferred() d.addBoth(_restore_callbacks) if request.callback: - d.addCallbacks(request.callback, request.errback) + d.addCallback(request.callback) + if request.errback: + d.addErrback(request.errback) request.callback, request.errback = d.callback, d.errback return d diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 04186559f..7e82dd519 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -1,21 +1,34 @@ +from __future__ import annotations + import warnings from functools import wraps -from typing import Any, Callable +from typing import TYPE_CHECKING, Any, Callable, TypeVar from twisted.internet import defer, threads from twisted.internet.defer import Deferred from scrapy.exceptions import ScrapyDeprecationWarning +if TYPE_CHECKING: + # typing.ParamSpec requires Python 3.10 + from typing_extensions import ParamSpec -def deprecated(use_instead: Any = None) -> Callable: + _P = ParamSpec("_P") + + +_T = TypeVar("_T") + + +def deprecated( + use_instead: Any = None, +) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: """This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted when the function is used.""" - def deco(func: Callable) -> Callable: + def deco(func: Callable[_P, _T]) -> Callable[_P, _T]: @wraps(func) - def wrapped(*args: Any, **kwargs: Any) -> Any: + def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> Any: message = f"Call to deprecated function {func.__name__}." if use_instead: message += f" Use {use_instead} instead." @@ -30,23 +43,23 @@ def deprecated(use_instead: Any = None) -> Callable: return deco -def defers(func: Callable) -> Callable[..., Deferred]: +def defers(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: """Decorator to make sure a function always returns a deferred""" @wraps(func) - def wrapped(*a: Any, **kw: Any) -> Deferred: + def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]: return defer.maybeDeferred(func, *a, **kw) return wrapped -def inthread(func: Callable) -> Callable[..., Deferred]: +def inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: """Decorator to call a function in a thread and return a deferred with the result """ @wraps(func) - def wrapped(*a: Any, **kw: Any) -> Deferred: + def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]: return threads.deferToThread(func, *a, **kw) return wrapped diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index c391db9fd..abb7e1726 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -2,21 +2,22 @@ Helper functions for dealing with Twisted deferreds """ +from __future__ import annotations + import asyncio import inspect from asyncio import Future from functools import wraps from types import CoroutineType from typing import ( + TYPE_CHECKING, Any, - AsyncGenerator, AsyncIterable, AsyncIterator, Awaitable, Callable, Coroutine, Dict, - Generator, Iterable, Iterator, List, @@ -37,6 +38,14 @@ from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed +if TYPE_CHECKING: + # typing.Concatenate and typing.ParamSpec require Python 3.10 + from typing_extensions import Concatenate, ParamSpec + + _P = ParamSpec("_P") + +_T = TypeVar("_T") + def defer_fail(_failure: Failure) -> Deferred: """Same as twisted.internet.defer.fail but delay calling errback until @@ -74,7 +83,31 @@ def defer_result(result: Any) -> Deferred: return defer_succeed(result) -def mustbe_deferred(f: Callable, *args: Any, **kw: Any) -> Deferred: +@overload +def mustbe_deferred( + f: Callable[_P, Deferred[_T]], *args: _P.args, **kw: _P.kwargs +) -> Deferred[_T]: ... + + +@overload +def mustbe_deferred( + f: Callable[_P, Coroutine[Deferred[Any], Any, _T]], + *args: _P.args, + **kw: _P.kwargs, +) -> Deferred[_T]: ... + + +@overload +def mustbe_deferred( + f: Callable[_P, _T], *args: _P.args, **kw: _P.kwargs +) -> Deferred[_T]: ... + + +def mustbe_deferred( + f: Callable[_P, Union[Deferred[_T], Coroutine[Deferred[Any], Any, _T], _T]], + *args: _P.args, + **kw: _P.kwargs, +) -> Deferred[_T]: """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -92,7 +125,11 @@ def mustbe_deferred(f: Callable, *args: Any, **kw: Any) -> Deferred: def parallel( - iterable: Iterable, count: int, callable: Callable, *args: Any, **named: Any + iterable: Iterable[_T], + count: int, + callable: Callable[Concatenate[_T, _P], Any], + *args: _P.args, + **named: _P.kwargs, ) -> Deferred: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -104,7 +141,7 @@ def parallel( return DeferredList([coop.coiterate(work) for _ in range(count)]) -class _AsyncCooperatorAdapter(Iterator): +class _AsyncCooperatorAdapter(Iterator[Deferred]): """A class that wraps an async iterable into a normal iterator suitable for using in Cooperator.coiterate(). As it's only needed for parallel_async(), it calls the callable directly in the callback, instead of providing a more @@ -152,28 +189,30 @@ class _AsyncCooperatorAdapter(Iterator): def __init__( self, - aiterable: AsyncIterable, - callable: Callable, - *callable_args: Any, - **callable_kwargs: Any, + aiterable: AsyncIterable[_T], + callable: Callable[Concatenate[_T, _P], Any], + *callable_args: _P.args, + **callable_kwargs: _P.kwargs, ): - self.aiterator: AsyncIterator = aiterable.__aiter__() - self.callable: Callable = callable + self.aiterator: AsyncIterator[_T] = aiterable.__aiter__() + self.callable: Callable[Concatenate[_T, _P], Any] = callable self.callable_args: Tuple[Any, ...] = callable_args self.callable_kwargs: Dict[str, Any] = callable_kwargs self.finished: bool = False self.waiting_deferreds: List[Deferred] = [] - self.anext_deferred: Optional[Deferred] = None + self.anext_deferred: Optional[Deferred[_T]] = None - def _callback(self, result: Any) -> None: + def _callback(self, result: _T) -> None: # This gets called when the result from aiterator.__anext__() is available. # It calls the callable on it and sends the result to the oldest waiting Deferred # (by chaining if the result is a Deferred too or by firing if not). self.anext_deferred = None - result = self.callable(result, *self.callable_args, **self.callable_kwargs) + callable_result = self.callable( + result, *self.callable_args, **self.callable_kwargs + ) d = self.waiting_deferreds.pop(0) - if isinstance(result, Deferred): - result.chainDeferred(d) + if isinstance(callable_result, Deferred): + callable_result.chainDeferred(d) else: d.callback(None) if self.waiting_deferreds: @@ -207,11 +246,11 @@ class _AsyncCooperatorAdapter(Iterator): def parallel_async( - async_iterable: AsyncIterable, + async_iterable: AsyncIterable[_T], count: int, - callable: Callable, - *args: Any, - **named: Any, + callable: Callable[Concatenate[_T, _P], Any], + *args: _P.args, + **named: _P.kwargs, ) -> Deferred: """Like parallel but for async iterators""" coop = Cooperator() @@ -221,7 +260,10 @@ def parallel_async( def process_chain( - callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any + callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], + input: Any, + *a: _P.args, + **kw: _P.kwargs, ) -> Deferred: """Return a Deferred built by chaining the given callbacks""" d: Deferred = Deferred() @@ -232,23 +274,17 @@ def process_chain( def process_chain_both( - callbacks: Iterable[Callable], - errbacks: Iterable[Callable], + callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], + errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]], input: Any, - *a: Any, - **kw: Any, + *a: _P.args, + **kw: _P.kwargs, ) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" d: Deferred = Deferred() for cb, eb in zip(callbacks, errbacks): - d.addCallbacks( - callback=cb, - errback=eb, - callbackArgs=a, - callbackKeywords=kw, - errbackArgs=a, - errbackKeywords=kw, - ) + d.addCallback(cb, *a, **kw) + d.addErrback(eb, *a, **kw) if isinstance(input, failure.Failure): d.errback(input) else: @@ -257,20 +293,27 @@ def process_chain_both( def process_parallel( - callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any + callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], + input: Any, + *a: _P.args, + **kw: _P.kwargs, ) -> Deferred: """Return a Deferred with the output of all successful calls to the given callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] d: Deferred = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) - d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) + d.addCallback(lambda r: [x[1] for x in r]) + d.addErrback(lambda f: f.value.subFailure) return d def iter_errback( - iterable: Iterable, errback: Callable, *a: Any, **kw: Any -) -> Generator: + iterable: Iterable[_T], + errback: Callable[Concatenate[Failure, _P], Any], + *a: _P.args, + **kw: _P.kwargs, +) -> Iterable[_T]: """Wraps an iterable calling an errback if an error is caught while iterating it. """ @@ -285,8 +328,11 @@ def iter_errback( async def aiter_errback( - aiterable: AsyncIterable, errback: Callable, *a: Any, **kw: Any -) -> AsyncGenerator: + aiterable: AsyncIterable[_T], + errback: Callable[Concatenate[Failure, _P], Any], + *a: _P.args, + **kw: _P.kwargs, +) -> AsyncIterable[_T]: """Wraps an async iterable calling an errback if an error is caught while iterating it. Similar to scrapy.utils.defer.iter_errback() """ @@ -301,7 +347,6 @@ async def aiter_errback( _CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future]) -_T = TypeVar("_T") @overload @@ -327,7 +372,9 @@ def deferred_from_coro(o: _T) -> Union[Deferred, _T]: return o -def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: +def deferred_f_from_coro_f( + coro_f: Callable[_P, Coroutine[Any, Any, _T]] +) -> Callable[_P, Deferred[_T]]: """Converts a coroutine function into a function that returns a Deferred. The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it. @@ -335,13 +382,15 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: """ @wraps(coro_f) - def f(*coro_args: Any, **coro_kwargs: Any) -> Any: + def f(*coro_args: _P.args, **coro_kwargs: _P.kwargs) -> Any: return deferred_from_coro(coro_f(*coro_args, **coro_kwargs)) return f -def maybeDeferred_coro(f: Callable, *args: Any, **kw: Any) -> Deferred: +def maybeDeferred_coro( + f: Callable[_P, Any], *args: _P.args, **kw: _P.kwargs +) -> Deferred: """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" try: result = f(*args, **kw) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 1e7364e49..5d2d490b2 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -2,6 +2,8 @@ This module contains essential stuff that should've come with Python itself ;) """ +from __future__ import annotations + import collections.abc import gc import inspect @@ -11,6 +13,7 @@ import weakref from functools import partial, wraps from itertools import chain from typing import ( + TYPE_CHECKING, Any, AsyncGenerator, AsyncIterable, @@ -25,12 +28,21 @@ from typing import ( Optional, Pattern, Tuple, + TypeVar, Union, overload, ) from scrapy.utils.asyncgen import as_async_generator +if TYPE_CHECKING: + # typing.Concatenate and typing.ParamSpec require Python 3.10 + from typing_extensions import Concatenate, ParamSpec + + _P = ParamSpec("_P") + +_T = TypeVar("_T") + def flatten(x: Iterable) -> list: """flatten(sequence) -> list @@ -169,14 +181,19 @@ def re_rsearch( return None -def memoizemethod_noargs(method: Callable) -> Callable: +_SelfT = TypeVar("_SelfT") + + +def memoizemethod_noargs( + method: Callable[Concatenate[_SelfT, _P], _T] +) -> Callable[Concatenate[_SelfT, _P], _T]: """Decorator to cache the result of a method (without arguments) using a weak reference to its object """ - cache: weakref.WeakKeyDictionary[Any, Any] = weakref.WeakKeyDictionary() + cache: weakref.WeakKeyDictionary[_SelfT, _T] = weakref.WeakKeyDictionary() @wraps(method) - def new_method(self: Any, *args: Any, **kwargs: Any) -> Any: + def new_method(self: _SelfT, *args: _P.args, **kwargs: _P.kwargs) -> _T: if self not in cache: cache[self] = method(self, *args, **kwargs) return cache[self] diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 6cde49bfe..5af6d22eb 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,8 +1,21 @@ +from __future__ import annotations + import asyncio import sys from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from contextlib import suppress -from typing import Any, Callable, Dict, List, Optional, Sequence, Type +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generic, + List, + Optional, + Tuple, + Type, + TypeVar, +) from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error @@ -13,6 +26,14 @@ from twisted.internet.tcp import Port from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import load_object +if TYPE_CHECKING: + # typing.ParamSpec requires Python 3.10 + from typing_extensions import ParamSpec + + _P = ParamSpec("_P") + +_T = TypeVar("_T") + def listen_tcp(portrange: List[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] """Like reactor.listenTCP but tries different ports in a range.""" @@ -32,14 +53,14 @@ def listen_tcp(portrange: List[int], host: str, factory: ServerFactory) -> Port: raise -class CallLaterOnce: +class CallLaterOnce(Generic[_T]): """Schedule a function to be called in the next reactor loop, but only if it hasn't been already scheduled since the last time it ran. """ - def __init__(self, func: Callable, *a: Any, **kw: Any): - self._func: Callable = func - self._a: Sequence[Any] = a + def __init__(self, func: Callable[_P, _T], *a: _P.args, **kw: _P.kwargs): + self._func: Callable[_P, _T] = func + self._a: Tuple[Any, ...] = a self._kw: Dict[str, Any] = kw self._call: Optional[DelayedCall] = None @@ -53,7 +74,7 @@ class CallLaterOnce: if self._call: self._call.cancel() - def __call__(self) -> Any: + def __call__(self) -> _T: self._call = None return self._func(*self._a, **self._kw) From b8e333c8ce78c195e265fa453e7bc0b11c5ca4af Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 13 May 2024 14:07:51 +0400 Subject: [PATCH 175/269] Full typing for scrapy/spiders. (#6356) --- scrapy/spiders/__init__.py | 6 +- scrapy/spiders/crawl.py | 138 +++++++++++++++++++++++++------------ scrapy/spiders/feed.py | 48 ++++++++----- scrapy/spiders/init.py | 14 ++-- scrapy/spiders/sitemap.py | 49 ++++++++----- scrapy/utils/sitemap.py | 4 +- 6 files changed, 172 insertions(+), 87 deletions(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 2416d2a4d..370801f28 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -39,7 +39,7 @@ class Spider(object_ref): def __init__(self, name: Optional[str] = None, **kwargs: Any): if name is not None: - self.name = name + self.name: str = name elif not getattr(self, "name", None): raise ValueError(f"{type(self).__name__} must have a name") self.__dict__.update(kwargs) @@ -67,8 +67,8 @@ class Spider(object_ref): return spider def _set_crawler(self, crawler: Crawler) -> None: - self.crawler = crawler - self.settings = crawler.settings + self.crawler: Crawler = crawler + self.settings: BaseSettings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) def start_requests(self) -> Iterable[Request]: diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index ba8b7b366..48c830d2a 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -8,9 +8,27 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import copy -from typing import TYPE_CHECKING, AsyncIterable, Awaitable, Sequence +from typing import ( + TYPE_CHECKING, + Any, + AsyncIterable, + Awaitable, + Callable, + Dict, + Iterable, + List, + Optional, + Sequence, + Set, + TypeVar, + Union, + cast, +) + +from twisted.python.failure import Failure from scrapy.http import HtmlResponse, Request, Response +from scrapy.link import Link from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen @@ -20,20 +38,32 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler -def _identity(x): + +_T = TypeVar("_T") +ProcessLinksT = Callable[[List[Link]], List[Link]] +ProcessRequestT = Callable[[Request, Response], Optional[Request]] + + +def _identity(x: _T) -> _T: return x -def _identity_process_request(request, response): +def _identity_process_request( + request: Request, response: Response +) -> Optional[Request]: return request -def _get_method(method, spider): +def _get_method( + method: Union[Callable, str, None], spider: Spider +) -> Optional[Callable]: if callable(method): return method if isinstance(method, str): return getattr(spider, method, None) + return None _default_link_extractor = LinkExtractor() @@ -42,37 +72,46 @@ _default_link_extractor = LinkExtractor() class Rule: def __init__( self, - link_extractor=None, - callback=None, - cb_kwargs=None, - follow=None, - process_links=None, - process_request=None, - errback=None, + link_extractor: Optional[LinkExtractor] = None, + callback: Union[Callable, str, None] = None, + cb_kwargs: Optional[Dict[str, Any]] = None, + follow: Optional[bool] = None, + process_links: Union[ProcessLinksT, str, None] = None, + process_request: Union[ProcessRequestT, str, None] = None, + errback: Union[Callable[[Failure], Any], str, None] = None, ): - self.link_extractor = link_extractor or _default_link_extractor - self.callback = callback - self.errback = errback - self.cb_kwargs = cb_kwargs or {} - self.process_links = process_links or _identity - self.process_request = process_request or _identity_process_request - self.follow = follow if follow is not None else not callback + self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor + self.callback: Union[Callable, str, None] = callback + self.errback: Union[Callable[[Failure], Any], str, None] = errback + self.cb_kwargs: Dict[str, Any] = cb_kwargs or {} + self.process_links: Union[ProcessLinksT, str] = process_links or _identity + self.process_request: Union[ProcessRequestT, str] = ( + process_request or _identity_process_request + ) + self.follow: bool = follow if follow is not None else not callback - def _compile(self, spider): + def _compile(self, spider: Spider) -> None: + # this replaces method names with methods and we can't express this in type hints self.callback = _get_method(self.callback, spider) - self.errback = _get_method(self.errback, spider) - self.process_links = _get_method(self.process_links, spider) - self.process_request = _get_method(self.process_request, spider) + self.errback = cast(Callable[[Failure], Any], _get_method(self.errback, spider)) + self.process_links = cast( + ProcessLinksT, _get_method(self.process_links, spider) + ) + self.process_request = cast( + ProcessRequestT, _get_method(self.process_request, spider) + ) class CrawlSpider(Spider): rules: Sequence[Rule] = () + _rules: List[Rule] + _follow_links: bool - def __init__(self, *a, **kw): + def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) self._compile_rules() - def _parse(self, response, **kwargs): + def _parse(self, response: Response, **kwargs: Any) -> Any: return self._parse_response( response=response, callback=self.parse_start_url, @@ -80,13 +119,13 @@ class CrawlSpider(Spider): follow=True, ) - def parse_start_url(self, response, **kwargs): + def parse_start_url(self, response: Response, **kwargs: Any) -> Any: return [] - def process_results(self, response: Response, results: list): + def process_results(self, response: Response, results: Any) -> Any: return results - def _build_request(self, rule_index, link): + def _build_request(self, rule_index: int, link: Link) -> Request: return Request( url=link.url, callback=self._callback, @@ -94,32 +133,43 @@ class CrawlSpider(Spider): meta={"rule": rule_index, "link_text": link.text}, ) - def _requests_to_follow(self, response): + def _requests_to_follow(self, response: Response) -> Iterable[Optional[Request]]: if not isinstance(response, HtmlResponse): return - seen = set() + seen: Set[Link] = set() for rule_index, rule in enumerate(self._rules): - links = [ + links: List[Link] = [ lnk for lnk in rule.link_extractor.extract_links(response) if lnk not in seen ] - for link in rule.process_links(links): + for link in cast(ProcessLinksT, rule.process_links)(links): seen.add(link) request = self._build_request(rule_index, link) - yield rule.process_request(request, response) + yield cast(ProcessRequestT, rule.process_request)(request, response) - def _callback(self, response, **cb_kwargs): - rule = self._rules[response.meta["rule"]] + def _callback(self, response: Response, **cb_kwargs: Any) -> Any: + rule = self._rules[cast(int, response.meta["rule"])] return self._parse_response( - response, rule.callback, {**rule.cb_kwargs, **cb_kwargs}, rule.follow + response, + cast(Callable, rule.callback), + {**rule.cb_kwargs, **cb_kwargs}, + rule.follow, ) - def _errback(self, failure): - rule = self._rules[failure.request.meta["rule"]] - return self._handle_failure(failure, rule.errback) + def _errback(self, failure: Failure) -> Iterable[Any]: + rule = self._rules[cast(int, failure.request.meta["rule"])] # type: ignore[attr-defined] + return self._handle_failure( + failure, cast(Callable[[Failure], Any], rule.errback) + ) - async def _parse_response(self, response, callback, cb_kwargs, follow=True): + async def _parse_response( + self, + response: Response, + callback: Optional[Callable], + cb_kwargs: Dict[str, Any], + follow: bool = True, + ) -> AsyncIterable[Any]: if callback: cb_res = callback(response, **cb_kwargs) or () if isinstance(cb_res, AsyncIterable): @@ -134,21 +184,23 @@ class CrawlSpider(Spider): for request_or_item in self._requests_to_follow(response): yield request_or_item - def _handle_failure(self, failure, errback): + def _handle_failure( + self, failure: Failure, errback: Optional[Callable[[Failure], Any]] + ) -> Iterable[Any]: if errback: results = errback(failure) or () yield from iterate_spider_output(results) - def _compile_rules(self): + def _compile_rules(self) -> None: self._rules = [] for rule in self.rules: self._rules.append(copy.copy(rule)) self._rules[-1]._compile(self) @classmethod - def from_crawler(cls, crawler, *args, **kwargs) -> Self: + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = super().from_crawler(crawler, *args, **kwargs) - spider._follow_links = crawler.settings.getbool( # type: ignore[attr-defined] + spider._follow_links = crawler.settings.getbool( "CRAWLSPIDER_FOLLOW_LINKS", True ) return spider diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 5caf8c79e..9dd8a5d68 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -5,7 +5,10 @@ for scraping from an XML feed. See documentation in docs/topics/spiders.rst """ +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + from scrapy.exceptions import NotConfigured, NotSupported +from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.spiders import Spider from scrapy.utils.iterators import csviter, xmliter_lxml @@ -22,11 +25,13 @@ class XMLFeedSpider(Spider): use iternodes, since it's a faster and cleaner. """ - iterator = "iternodes" - itertag = "item" - namespaces = () + iterator: str = "iternodes" + itertag: str = "item" + namespaces: Sequence[Tuple[str, str]] = () - def process_results(self, response, results): + def process_results( + self, response: Response, results: Iterable[Any] + ) -> Iterable[Any]: """This overridable method is called for each result (item or request) returned by the spider, and it's intended to perform any last time processing required before returning the results to the framework core, @@ -36,20 +41,20 @@ class XMLFeedSpider(Spider): """ return results - def adapt_response(self, response): + def adapt_response(self, response: Response) -> Response: """You can override this function in order to make any changes you want to into the feed before parsing it. This function must return a response. """ return response - def parse_node(self, response, selector): + def parse_node(self, response: Response, selector: Selector) -> Any: """This method must be overridden with your custom spider functionality""" if hasattr(self, "parse_item"): # backward compatibility return self.parse_item(response, selector) raise NotImplementedError - def parse_nodes(self, response, nodes): + def parse_nodes(self, response: Response, nodes: Iterable[Selector]) -> Any: """This method is called for the nodes matching the provided tag name (itertag). Receives the response and an Selector for each node. Overriding this method is mandatory. Otherwise, you spider won't work. @@ -61,20 +66,25 @@ class XMLFeedSpider(Spider): ret = iterate_spider_output(self.parse_node(response, selector)) yield from self.process_results(response, ret) - def _parse(self, response, **kwargs): + def _parse(self, response: Response, **kwargs: Any) -> Any: if not hasattr(self, "parse_node"): raise NotConfigured( "You must define parse_node method in order to scrape this XML feed" ) response = self.adapt_response(response) + nodes: Iterable[Selector] if self.iterator == "iternodes": nodes = self._iternodes(response) elif self.iterator == "xml": + if not isinstance(response, TextResponse): + raise ValueError("Response content isn't text") selector = Selector(response, type="xml") self._register_namespaces(selector) nodes = selector.xpath(f"//{self.itertag}") elif self.iterator == "html": + if not isinstance(response, TextResponse): + raise ValueError("Response content isn't text") selector = Selector(response, type="html") self._register_namespaces(selector) nodes = selector.xpath(f"//{self.itertag}") @@ -83,12 +93,12 @@ class XMLFeedSpider(Spider): return self.parse_nodes(response, nodes) - def _iternodes(self, response): + def _iternodes(self, response: Response) -> Iterable[Selector]: for node in xmliter_lxml(response, self.itertag): self._register_namespaces(node) yield node - def _register_namespaces(self, selector): + def _register_namespaces(self, selector: Selector) -> None: for prefix, uri in self.namespaces: selector.register_namespace(prefix, uri) @@ -102,27 +112,29 @@ class CSVFeedSpider(Spider): and the file's headers. """ - delimiter = ( + delimiter: Optional[str] = ( None # When this is None, python's csv module's default delimiter is used ) - quotechar = ( + quotechar: Optional[str] = ( None # When this is None, python's csv module's default quotechar is used ) - headers = None + headers: Optional[List[str]] = None - def process_results(self, response, results): + def process_results( + self, response: Response, results: Iterable[Any] + ) -> Iterable[Any]: """This method has the same purpose as the one in XMLFeedSpider""" return results - def adapt_response(self, response): + def adapt_response(self, response: Response) -> Response: """This method has the same purpose as the one in XMLFeedSpider""" return response - def parse_row(self, response, row): + def parse_row(self, response: Response, row: Dict[str, str]) -> Any: """This method must be overridden with your custom spider functionality""" raise NotImplementedError - def parse_rows(self, response): + def parse_rows(self, response: Response) -> Any: """Receives a response and a dict (representing each row) with a key for each provided (or detected) header of the CSV file. This spider also gives the opportunity to override adapt_response and @@ -135,7 +147,7 @@ class CSVFeedSpider(Spider): ret = iterate_spider_output(self.parse_row(response, row)) yield from self.process_results(response, ret) - def _parse(self, response, **kwargs): + def _parse(self, response: Response, **kwargs: Any) -> Any: if not hasattr(self, "parse_row"): raise NotConfigured( "You must define parse_row method in order to scrape this CSV feed" diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index 3cb215b0f..a0898a0cf 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,3 +1,7 @@ +from typing import Any, Iterable, Optional, cast + +from scrapy import Request +from scrapy.http import Response from scrapy.spiders import Spider from scrapy.utils.spider import iterate_spider_output @@ -5,17 +9,17 @@ from scrapy.utils.spider import iterate_spider_output class InitSpider(Spider): """Base Spider with initialization facilities""" - def start_requests(self): - self._postinit_reqs = super().start_requests() - return iterate_spider_output(self.init_request()) + def start_requests(self) -> Iterable[Request]: + self._postinit_reqs: Iterable[Request] = super().start_requests() + return cast(Iterable[Request], iterate_spider_output(self.init_request())) - def initialized(self, response=None): + def initialized(self, response: Optional[Response] = None) -> Any: """This method must be set as the callback of your last initialization request. See self.init_request() docstring for more info. """ return self.__dict__.pop("_postinit_reqs") - def init_request(self): + def init_request(self) -> Any: """This function should return one initialization request, with the self.initialized method as callback. When the self.initialized method is called this spider is considered initialized. If you need to perform diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index f0e630c42..d082fbfdb 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -2,9 +2,21 @@ from __future__ import annotations import logging import re -from typing import TYPE_CHECKING, Any +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Sequence, + Tuple, + Union, + cast, +) -from scrapy.http import Request, XmlResponse +from scrapy.http import Request, Response, XmlResponse from scrapy.spiders import Spider from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.gz import gunzip, gzip_magic_number @@ -20,10 +32,12 @@ logger = logging.getLogger(__name__) class SitemapSpider(Spider): - sitemap_urls = () - sitemap_rules = [("", "parse")] - sitemap_follow = [""] - sitemap_alternate_links = False + sitemap_urls: Sequence[str] = () + sitemap_rules: Sequence[ + Tuple[Union[re.Pattern[str], str], Union[str, Callable]] + ] = [("", "parse")] + sitemap_follow: Sequence[Union[re.Pattern[str], str]] = [""] + sitemap_alternate_links: bool = False _max_size: int _warn_size: int @@ -38,27 +52,29 @@ class SitemapSpider(Spider): ) return spider - def __init__(self, *a, **kw): + def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) - self._cbs = [] + self._cbs: List[Tuple[re.Pattern[str], Callable]] = [] for r, c in self.sitemap_rules: if isinstance(c, str): - c = getattr(self, c) + c = cast(Callable, getattr(self, c)) self._cbs.append((regex(r), c)) - self._follow = [regex(x) for x in self.sitemap_follow] + self._follow: List[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] - def start_requests(self): + def start_requests(self) -> Iterable[Request]: for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) - def sitemap_filter(self, entries): + def sitemap_filter( + self, entries: Iterable[Dict[str, Any]] + ) -> Iterable[Dict[str, Any]]: """This method can be used to filter sitemap entries by their attributes, for example, you can filter locs with lastmod greater than a given date (see docs). """ yield from entries - def _parse_sitemap(self, response): + def _parse_sitemap(self, response: Response) -> Iterable[Request]: if response.url.endswith("/robots.txt"): for url in sitemap_urls_from_robots(response.text, base_url=response.url): yield Request(url, callback=self._parse_sitemap) @@ -86,7 +102,7 @@ class SitemapSpider(Spider): yield Request(loc, callback=c) break - def _get_sitemap_body(self, response): + def _get_sitemap_body(self, response: Response) -> Optional[bytes]: """Return the sitemap body contained in the given response, or None if the response is not a sitemap. """ @@ -117,15 +133,16 @@ class SitemapSpider(Spider): # in other word, here, we have plain XML if response.url.endswith(".xml") or response.url.endswith(".xml.gz"): return response.body + return None -def regex(x): +def regex(x: Union[re.Pattern[str], str]) -> re.Pattern[str]: if isinstance(x, str): return re.compile(x) return x -def iterloc(it, alt=False): +def iterloc(it: Iterable[Dict[str, Any]], alt: bool = False) -> Iterable[str]: for d in it: yield d["loc"] diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 7dcee3a2f..cf429043d 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -5,7 +5,7 @@ Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ -from typing import Any, Dict, Generator, Iterator, Optional +from typing import Any, Dict, Generator, Iterator, Optional, Union from urllib.parse import urljoin import lxml.etree # nosec @@ -15,7 +15,7 @@ class Sitemap: """Class to parse Sitemap (type=urlset) and Sitemap Index (type=sitemapindex) files""" - def __init__(self, xmltext: str): + def __init__(self, xmltext: Union[str, bytes]): xmlp = lxml.etree.XMLParser( recover=True, remove_comments=True, resolve_entities=False ) From b88f22c6c5de4ca8828b2abe860516c2468c4fe3 Mon Sep 17 00:00:00 2001 From: Bagaudin Magomedov <51474632+bloodforcream@users.noreply.github.com> Date: Mon, 13 May 2024 15:33:24 +0300 Subject: [PATCH 176/269] Add SpiderLoggerAdapter, change Spider.logger to return SpiderLoggerAdapter (#6324) --- scrapy/spiders/__init__.py | 7 +- scrapy/utils/log.py | 25 ++++- tests/spiders.py | 23 +++++ tests/test_utils_log.py | 185 +++++++++++++++++++++++++++++++++++++ 4 files changed, 237 insertions(+), 3 deletions(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 370801f28..bef041325 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -25,6 +25,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.settings import BaseSettings + from scrapy.utils.log import SpiderLoggerAdapter CallbackT = Callable[Concatenate[Response, ...], Any] @@ -47,9 +48,11 @@ class Spider(object_ref): self.start_urls: List[str] = [] @property - def logger(self) -> logging.LoggerAdapter: + def logger(self) -> SpiderLoggerAdapter: + from scrapy.utils.log import SpiderLoggerAdapter + logger = logging.getLogger(self.name) - return logging.LoggerAdapter(logger, {"spider": self}) + return SpiderLoggerAdapter(logger, {"spider": self}) def log(self, message: Any, level: int = logging.DEBUG, **kw: Any) -> None: """Log the given message at the given log level diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 2a38f151a..430a91e95 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -4,7 +4,17 @@ import logging import sys from logging.config import dictConfig from types import TracebackType -from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + List, + MutableMapping, + Optional, + Tuple, + Type, + Union, + cast, +) from twisted.python import log as twisted_log from twisted.python.failure import Failure @@ -238,3 +248,16 @@ def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]: args = logkws if not logkws.get("args") else logkws["args"] return (level, message, args) + + +class SpiderLoggerAdapter(logging.LoggerAdapter): + def process( + self, msg: str, kwargs: MutableMapping[str, Any] + ) -> Tuple[str, MutableMapping[str, Any]]: + """Method that augments logging with additional 'extra' data""" + if isinstance(kwargs.get("extra"), MutableMapping): + kwargs["extra"].update(self.extra) + else: + kwargs["extra"] = self.extra + + return msg, kwargs diff --git a/tests/spiders.py b/tests/spiders.py index 94969db99..ea419afbd 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -4,6 +4,7 @@ Some spiders used for testing and benchmarking import asyncio import time +from typing import Optional from urllib.parse import urlencode from twisted.internet import defer @@ -78,6 +79,28 @@ class DelaySpider(MetaSpider): self.t2_err = time.time() +class LogSpider(MetaSpider): + name = "log_spider" + + def log_debug(self, message: str, extra: Optional[dict] = None): + self.logger.debug(message, extra=extra) + + def log_info(self, message: str, extra: Optional[dict] = None): + self.logger.info(message, extra=extra) + + def log_warning(self, message: str, extra: Optional[dict] = None): + self.logger.warning(message, extra=extra) + + def log_error(self, message: str, extra: Optional[dict] = None): + self.logger.error(message, extra=extra) + + def log_critical(self, message: str, extra: Optional[dict] = None): + self.logger.critical(message, extra=extra) + + def parse(self, response): + pass + + class SlowSpider(DelaySpider): name = "slow" diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index eae744df5..a8d080822 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,18 +1,26 @@ +import json import logging +import re import sys import unittest +from io import StringIO +from typing import Any, Dict, Mapping, MutableMapping +from unittest import TestCase +import pytest from testfixtures import LogCapture from twisted.python.failure import Failure from scrapy.extensions import telnet from scrapy.utils.log import ( LogCounterHandler, + SpiderLoggerAdapter, StreamLogger, TopLevelFormatter, failure_to_exc_info, ) from scrapy.utils.test import get_crawler +from tests.spiders import LogSpider class FailureToExcInfoTest(unittest.TestCase): @@ -106,3 +114,180 @@ class StreamLoggerTest(unittest.TestCase): with LogCapture() as log: print("test log msg") log.check(("test", "ERROR", "test log msg")) + + +@pytest.mark.parametrize( + ("base_extra", "log_extra", "expected_extra"), + ( + ( + {"spider": "test"}, + {"extra": {"log_extra": "info"}}, + {"extra": {"log_extra": "info", "spider": "test"}}, + ), + ( + {"spider": "test"}, + {"extra": None}, + {"extra": {"spider": "test"}}, + ), + ( + {"spider": "test"}, + {"extra": {"spider": "test2"}}, + {"extra": {"spider": "test"}}, + ), + ), +) +def test_spider_logger_adapter_process( + base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: Dict +): + logger = logging.getLogger("test") + spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra) + + log_message = "test_log_message" + result_message, result_kwargs = spider_logger_adapter.process( + log_message, log_extra + ) + + assert result_message == log_message + assert result_kwargs == expected_extra + + +class LoggingTestCase(TestCase): + def setUp(self): + self.log_stream = StringIO() + handler = logging.StreamHandler(self.log_stream) + logger = logging.getLogger("log_spider") + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + self.handler = handler + self.logger = logger + self.spider = LogSpider() + + def tearDown(self): + self.logger.removeHandler(self.handler) + + def test_debug_logging(self): + log_message = "Foo message" + self.spider.log_debug(log_message) + log_contents = self.log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + def test_info_logging(self): + log_message = "Bar message" + self.spider.log_info(log_message) + log_contents = self.log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + def test_warning_logging(self): + log_message = "Baz message" + self.spider.log_warning(log_message) + log_contents = self.log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + def test_error_logging(self): + log_message = "Foo bar message" + self.spider.log_error(log_message) + log_contents = self.log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + def test_critical_logging(self): + log_message = "Foo bar baz message" + self.spider.log_critical(log_message) + log_contents = self.log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + +class LoggingWithExtraTestCase(TestCase): + def setUp(self): + self.log_stream = StringIO() + handler = logging.StreamHandler(self.log_stream) + formatter = logging.Formatter( + '{"levelname": "%(levelname)s", "message": "%(message)s", "spider": "%(spider)s", "important_info": "%(important_info)s"}' + ) + handler.setFormatter(formatter) + logger = logging.getLogger("log_spider") + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + self.handler = handler + self.logger = logger + self.spider = LogSpider() + self.regex_pattern = re.compile(r"^]+>$") + + def tearDown(self): + self.logger.removeHandler(self.handler) + + def test_debug_logging(self): + log_message = "Foo message" + extra = {"important_info": "foo"} + self.spider.log_debug(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "DEBUG" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_info_logging(self): + log_message = "Bar message" + extra = {"important_info": "bar"} + self.spider.log_info(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "INFO" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_warning_logging(self): + log_message = "Baz message" + extra = {"important_info": "baz"} + self.spider.log_warning(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "WARNING" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_error_logging(self): + log_message = "Foo bar message" + extra = {"important_info": "foo bar"} + self.spider.log_error(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "ERROR" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_critical_logging(self): + log_message = "Foo bar baz message" + extra = {"important_info": "foo bar baz"} + self.spider.log_critical(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "CRITICAL" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_overwrite_spider_extra(self): + log_message = "Foo message" + extra = {"important_info": "foo", "spider": "shouldn't change"} + self.spider.log_error(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "ERROR" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] From d2f1e00a6afa46c26a334a24217d5bf605bab9fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 14 May 2024 18:54:11 +0200 Subject: [PATCH 177/269] Merge 2.11.2 changes (#6363) --- .bumpversion.cfg | 2 +- docs/faq.rst | 40 +- docs/news.rst | 116 +- docs/topics/benchmarking.rst | 4 +- docs/topics/downloader-middleware.rst | 44 +- docs/topics/settings.rst | 2 +- docs/topics/signals.rst | 11 +- docs/topics/spider-middleware.rst | 40 +- docs/topics/spiders.rst | 3 +- scrapy/VERSION | 2 +- scrapy/core/engine.py | 17 +- scrapy/downloadermiddlewares/httpproxy.py | 19 +- scrapy/downloadermiddlewares/offsite.py | 77 ++ scrapy/downloadermiddlewares/redirect.py | 62 +- scrapy/settings/default_settings.py | 2 +- scrapy/spidermiddlewares/offsite.py | 8 +- scrapy/utils/python.py | 2 +- tests/test_downloadermiddleware.py | 8 +- tests/test_downloadermiddleware_offsite.py | 184 +++ tests/test_downloadermiddleware_redirect.py | 1359 +++++++++++++++---- tests/test_engine.py | 40 +- tests/test_utils_project.py | 21 +- tox.ini | 17 +- 23 files changed, 1726 insertions(+), 354 deletions(-) create mode 100644 scrapy/downloadermiddlewares/offsite.py create mode 100644 tests/test_downloadermiddleware_offsite.py diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 968a34d96..599cd0cff 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.11.1 +current_version = 2.11.2 commit = True tag = True tag_name = {new_version} diff --git a/docs/faq.rst b/docs/faq.rst index 7090f0bcd..d394406e8 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -138,39 +138,37 @@ See previous question. How can I prevent memory errors due to many allowed domains? ------------------------------------------------------------ -If you have a spider with a long list of -:attr:`~scrapy.Spider.allowed_domains` (e.g. 50,000+), consider -replacing the default -:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware -with a :ref:`custom spider middleware ` that requires -less memory. For example: +If you have a spider with a long list of :attr:`~scrapy.Spider.allowed_domains` +(e.g. 50,000+), consider replacing the default +:class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` downloader +middleware with a :ref:`custom downloader middleware +` that requires less memory. For example: - If your domain names are similar enough, use your own regular expression - instead joining the strings in - :attr:`~scrapy.Spider.allowed_domains` into a complex regular - expression. + instead joining the strings in :attr:`~scrapy.Spider.allowed_domains` into + a complex regular expression. - If you can `meet the installation requirements`_, use pyre2_ instead of Python’s re_ to compile your URL-filtering regular expression. See :issue:`1908`. -See also other suggestions at `StackOverflow`_. +See also `other suggestions at StackOverflow +`__. .. note:: Remember to disable - :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable - your custom implementation: + :class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` when you + enable your custom implementation: .. code-block:: python - SPIDER_MIDDLEWARES = { - "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None, - "myproject.middlewares.CustomOffsiteMiddleware": 500, + DOWNLOADER_MIDDLEWARES = { + "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": None, + "myproject.middlewares.CustomOffsiteMiddleware": 50, } .. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation .. _pyre2: https://github.com/andreasvc/pyre2 .. _re: https://docs.python.org/library/re.html -.. _StackOverflow: https://stackoverflow.com/q/36440681/939364 Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- @@ -206,12 +204,10 @@ I get "Filtered offsite request" messages. How can I fix them? Those messages (logged with ``DEBUG`` level) don't necessarily mean there is a problem, so you may not need to fix them. -Those messages are thrown by the Offsite Spider Middleware, which is a spider -middleware (enabled by default) whose purpose is to filter out requests to -domains outside the ones covered by the spider. - -For more info see: -:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware`. +Those messages are thrown by +:class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`, which is a +downloader middleware (enabled by default) whose purpose is to filter out +requests to domains outside the ones covered by the spider. What is the recommended way to deploy a Scrapy crawler in production? --------------------------------------------------------------------- diff --git a/docs/news.rst b/docs/news.rst index 7db4e59a1..758b22d80 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,7 +3,6 @@ Release notes ============= - .. _release-VERSION: Scrapy VERSION (YYYY-MM-DD) @@ -12,11 +11,122 @@ Scrapy VERSION (YYYY-MM-DD) Deprecations ~~~~~~~~~~~~ -- :func:`scrapy.core.downloader.Downloader._get_slot_key` is now deprecated. - Consider using its corresponding public method get_slot_key() instead. +- :meth:`scrapy.core.downloader.Downloader._get_slot_key` is deprecated, use + :meth:`scrapy.core.downloader.Downloader.get_slot_key` instead. (:issue:`6340`) +.. _release-2.11.2: + +Scrapy 2.11.2 (2024-05-14) +-------------------------- + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- Redirects to non-HTTP protocols are no longer followed. Please, see the + `23j4-mw76-5v7h security advisory`_ for more information. (:issue:`457`) + + .. _23j4-mw76-5v7h security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-23j4-mw76-5v7h + +- The ``Authorization`` header is now dropped on redirects to a different + scheme (``http://`` or ``https://``) or port, even if the domain is the + same. Please, see the `4qqq-9vqf-3h3f security advisory`_ for more + information. + + .. _4qqq-9vqf-3h3f security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-4qqq-9vqf-3h3f + +- When using system proxy settings that are different for ``http://`` and + ``https://``, redirects to a different URL scheme will now also trigger the + corresponding change in proxy settings for the redirected request. Please, + see the `jm3v-qxmh-hxwv security advisory`_ for more information. + (:issue:`767`) + + .. _jm3v-qxmh-hxwv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-jm3v-qxmh-hxwv + +- :attr:`Spider.allowed_domains ` is now + enforced for all requests, and not only requests from spider callbacks. + (:issue:`1042`, :issue:`2241`, :issue:`6358`) + +- :func:`~scrapy.utils.iterators.xmliter_lxml` no longer resolves XML + entities. (:issue:`6265`) + +- defusedxml_ is now used to make + :class:`scrapy.http.request.rpc.XmlRpcRequest` more secure. + (:issue:`6250`, :issue:`6251`) + + .. _defusedxml: https://github.com/tiran/defusedxml + +Bug fixes +~~~~~~~~~ + +- Restored support for brotlipy_, which had been dropped in Scrapy 2.11.1 in + favor of brotli_. (:issue:`6261`) + + .. _brotli: https://github.com/google/brotli + + .. note:: brotlipy is deprecated, both in Scrapy and upstream. Use brotli + instead if you can. + +- Make :setting:`METAREFRESH_IGNORE_TAGS` ``["noscript"]`` by default. This + prevents + :class:`~scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware` from + following redirects that would not be followed by web browsers with + JavaScript enabled. (:issue:`6342`, :issue:`6347`) + +- During :ref:`feed export `, do not close the + underlying file from :ref:`built-in post-processing plugins + `. + (:issue:`5932`, :issue:`6178`, :issue:`6239`) + +- :class:`LinkExtractor ` + now properly applies the ``unique`` and ``canonicalize`` parameters. + (:issue:`3273`, :issue:`6221`) + +- Do not initialize the scheduler disk queue if :setting:`JOBDIR` is an empty + string. (:issue:`6121`, :issue:`6124`) + +- Fix :attr:`Spider.logger ` not logging custom extra + information. (:issue:`6323`, :issue:`6324`) + +- ``robots.txt`` files with a non-UTF-8 encoding no longer prevent parsing + the UTF-8-compatible (e.g. ASCII) parts of the document. + (:issue:`6292`, :issue:`6298`) + +- :meth:`scrapy.http.cookies.WrappedRequest.get_header` no longer raises an + exception if ``default`` is ``None``. + (:issue:`6308`, :issue:`6310`) + +- :class:`~scrapy.selector.Selector` now uses + :func:`scrapy.utils.response.get_base_url` to determine the base URL of a + given :class:`~scrapy.http.Response`. (:issue:`6265`) + +- The :meth:`media_to_download` method of :ref:`media pipelines + ` now logs exceptions before stripping them. + (:issue:`5067`, :issue:`5068`) + +- When passing a callback to the :command:`parse` command, build the callback + callable with the right signature. + (:issue:`6182`) + +Documentation +~~~~~~~~~~~~~ + +- Add a FAQ entry about :ref:`creating blank requests `. + (:issue:`6203`, :issue:`6208`) + +- Document that :attr:`scrapy.selector.Selector.type` can be ``"json"``. + (:issue:`6328`, :issue:`6334`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Make builds reproducible. (:issue:`5019`, :issue:`6322`) + +- Packaging and test fixes. + (:issue:`6286`, :issue:`6290`, :issue:`6312`, :issue:`6316`, :issue:`6344`) + + .. _release-2.11.1: Scrapy 2.11.1 (2024-02-14) diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 0643df6a6..b704e54ed 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -24,7 +24,8 @@ You should see an output like this:: 'scrapy.extensions.telnet.TelnetConsole', 'scrapy.extensions.corestats.CoreStats'] 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled downloader middlewares: - ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', + ['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware', + 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', @@ -37,7 +38,6 @@ You should see an output like this:: 'scrapy.downloadermiddlewares.stats.DownloaderStats'] 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled spider middlewares: ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', - 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', 'scrapy.spidermiddlewares.referer.RefererMiddleware', 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', 'scrapy.spidermiddlewares.depth.DepthMiddleware'] diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index d4cd062fe..c31f7fe43 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -763,6 +763,44 @@ HttpProxyMiddleware Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` environment variables, and it will also ignore ``no_proxy`` environment variable. +OffsiteMiddleware +----------------- + +.. module:: scrapy.downloadermiddlewares.offsite + :synopsis: Offsite Middleware + +.. class:: OffsiteMiddleware + + .. versionadded:: 2.11.2 + + Filters out Requests for URLs outside the domains covered by the spider. + + This middleware filters out every request whose host names aren't in the + spider's :attr:`~scrapy.Spider.allowed_domains` attribute. + All subdomains of any domain in the list are also allowed. + E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` + but not ``www2.example.com`` nor ``example.com``. + + When your spider returns a request for a domain not belonging to those + covered by the spider, this middleware will log a debug message similar to + this one:: + + DEBUG: Filtered offsite request to 'offsite.example': + + To avoid filling the log with too much noise, it will only print one of + these messages for each new domain filtered. So, for example, if another + request for ``offsite.example`` is filtered, no log message will be + printed. But if a request for ``other.example`` is filtered, a message + will be printed (but only for the first request filtered). + + If the spider doesn't define an + :attr:`~scrapy.Spider.allowed_domains` attribute, or the + attribute is empty, the offsite middleware will allow all requests. + + If the request has the :attr:`~scrapy.Request.dont_filter` attribute + set, the offsite middleware will allow the request even if its domain is not + listed in allowed domains. + RedirectMiddleware ------------------ @@ -882,7 +920,11 @@ Meta tags within these tags are ignored. .. versionchanged:: 2.0 The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from - ``['script', 'noscript']`` to ``[]``. + ``["script", "noscript"]`` to ``[]``. + +.. versionchanged:: 2.11.2 + The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from + ``[]`` to ``["noscript"]``. .. versionchanged:: VERSION The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2bd9cf1ed..904bd7ecc 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -674,6 +674,7 @@ Default: .. code-block:: python { + "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50, "scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware": 100, "scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware": 300, "scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware": 350, @@ -1613,7 +1614,6 @@ Default: { "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50, - "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": 500, "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 9bfd1761c..13e636055 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -343,11 +343,18 @@ request_scheduled .. signal:: request_scheduled .. function:: request_scheduled(request, spider) - Sent when the engine schedules a :class:`~scrapy.Request`, to be - downloaded later. + Sent when the engine is asked to schedule a :class:`~scrapy.Request`, to be + downloaded later, before the request reaches the :ref:`scheduler + `. + + Raise :exc:`~scrapy.exceptions.IgnoreRequest` to drop a request before it + reaches the scheduler. This signal does not support returning deferreds from its handlers. + .. versionadded:: 2.11.2 + Allow dropping requests with :exc:`~scrapy.exceptions.IgnoreRequest`. + :param request: the request that reached the scheduler :type request: :class:`~scrapy.Request` object diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 3f16efea5..8ddf17a14 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -51,8 +51,8 @@ value. For example, if you want to disable the off-site middleware: .. code-block:: python SPIDER_MIDDLEWARES = { - "myproject.middlewares.CustomSpiderMiddleware": 543, - "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None, + "scrapy.spidermiddlewares.referer.RefererMiddleware": None, + "myproject.middlewares.CustomRefererSpiderMiddleware": 700, } Finally, keep in mind that some middlewares may need to be enabled through a @@ -313,42 +313,6 @@ Default: ``False`` Pass all responses, regardless of its status code. -OffsiteMiddleware ------------------ - -.. module:: scrapy.spidermiddlewares.offsite - :synopsis: Offsite Spider Middleware - -.. class:: OffsiteMiddleware - - Filters out Requests for URLs outside the domains covered by the spider. - - This middleware filters out every request whose host names aren't in the - spider's :attr:`~scrapy.Spider.allowed_domains` attribute. - All subdomains of any domain in the list are also allowed. - E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` - but not ``www2.example.com`` nor ``example.com``. - - When your spider returns a request for a domain not belonging to those - covered by the spider, this middleware will log a debug message similar to - this one:: - - DEBUG: Filtered offsite request to 'www.othersite.com': - - To avoid filling the log with too much noise, it will only print one of - these messages for each new domain filtered. So, for example, if another - request for ``www.othersite.com`` is filtered, no log message will be - printed. But if a request for ``someothersite.com`` is filtered, a message - will be printed (but only for the first request filtered). - - If the spider doesn't define an - :attr:`~scrapy.Spider.allowed_domains` attribute, or the - attribute is empty, the offsite middleware will allow all requests. - - If the request has the :attr:`~scrapy.Request.dont_filter` attribute - set, the offsite middleware will allow the request even if its domain is not - listed in allowed domains. - RefererMiddleware ----------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 30677fe74..8a0102a51 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -75,7 +75,8 @@ scrapy.Spider An optional list of strings containing domains that this spider is allowed to crawl. Requests for URLs not belonging to the domain names specified in this list (or their subdomains) won't be followed if - :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled. + :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` is + enabled. Let's say your target url is ``https://www.example.com/1.html``, then add ``'example.com'`` to the list. diff --git a/scrapy/VERSION b/scrapy/VERSION index 6ceb272ee..9e5bb77a3 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.11.1 +2.11.2 diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 6bf3f3e26..4eca03800 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -28,7 +28,7 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper -from scrapy.exceptions import CloseSpider, DontCloseSpider +from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response from scrapy.logformatter import LogFormatter from scrapy.settings import BaseSettings, Settings @@ -36,6 +36,7 @@ from scrapy.signalmanager import SignalManager from scrapy.spiders import Spider from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object +from scrapy.utils.python import global_object_name from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: @@ -292,9 +293,19 @@ class ExecutionEngine: self.slot.nextcall.schedule() # type: ignore[union-attr] def _schedule_request(self, request: Request, spider: Spider) -> None: - self.signals.send_catch_log( - signals.request_scheduled, request=request, spider=spider + request_scheduled_result = self.signals.send_catch_log( + signals.request_scheduled, + request=request, + spider=spider, + dont_log=IgnoreRequest, ) + for handler, result in request_scheduled_result: + if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest): + logger.debug( + f"Signal handler {global_object_name(handler)} dropped " + f"request {request} before it reached the scheduler." + ) + return if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log( signals.request_dropped, request=request, spider=spider diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 335896ac1..5b56ad449 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -60,26 +60,33 @@ class HttpProxyMiddleware: def process_request( self, request: Request, spider: Spider ) -> Union[Request, Response, None]: - creds, proxy_url = None, None + creds, proxy_url, scheme = None, None, None if "proxy" in request.meta: if request.meta["proxy"] is not None: creds, proxy_url = self._get_proxy(request.meta["proxy"], "") elif self.proxies: parsed = urlparse_cached(request) - scheme = parsed.scheme + _scheme = parsed.scheme if ( # 'no_proxy' is only supported by http schemes - scheme not in ("http", "https") + _scheme not in ("http", "https") or (parsed.hostname and not proxy_bypass(parsed.hostname)) - ) and scheme in self.proxies: + ) and _scheme in self.proxies: + scheme = _scheme creds, proxy_url = self.proxies[scheme] - self._set_proxy_and_creds(request, proxy_url, creds) + self._set_proxy_and_creds(request, proxy_url, creds, scheme) return None def _set_proxy_and_creds( - self, request: Request, proxy_url: Optional[str], creds: Optional[bytes] + self, + request: Request, + proxy_url: Optional[str], + creds: Optional[bytes], + scheme: Optional[str], ) -> None: + if scheme: + request.meta["_scheme_proxy"] = True if proxy_url: request.meta["proxy"] = proxy_url elif request.meta.get("proxy") is not None: diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py new file mode 100644 index 000000000..1e5026925 --- /dev/null +++ b/scrapy/downloadermiddlewares/offsite.py @@ -0,0 +1,77 @@ +import logging +import re +import warnings + +from scrapy import signals +from scrapy.exceptions import IgnoreRequest +from scrapy.utils.httpobj import urlparse_cached + +logger = logging.getLogger(__name__) + + +class OffsiteMiddleware: + @classmethod + def from_crawler(cls, crawler): + o = cls(crawler.stats) + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled) + return o + + def __init__(self, stats): + self.stats = stats + self.domains_seen = set() + + def spider_opened(self, spider): + self.host_regex = self.get_host_regex(spider) + + def request_scheduled(self, request, spider): + self.process_request(request, spider) + + def process_request(self, request, spider): + if request.dont_filter or self.should_follow(request, spider): + return None + domain = urlparse_cached(request).hostname + if domain and domain not in self.domains_seen: + self.domains_seen.add(domain) + logger.debug( + "Filtered offsite request to %(domain)r: %(request)s", + {"domain": domain, "request": request}, + extra={"spider": spider}, + ) + self.stats.inc_value("offsite/domains", spider=spider) + self.stats.inc_value("offsite/filtered", spider=spider) + raise IgnoreRequest + + def should_follow(self, request, spider): + regex = self.host_regex + # hostname can be None for wrong urls (like javascript links) + host = urlparse_cached(request).hostname or "" + return bool(regex.search(host)) + + def get_host_regex(self, spider): + """Override this method to implement a different offsite policy""" + allowed_domains = getattr(spider, "allowed_domains", None) + if not allowed_domains: + return re.compile("") # allow all by default + url_pattern = re.compile(r"^https?://.*$") + port_pattern = re.compile(r":\d+$") + domains = [] + for domain in allowed_domains: + if domain is None: + continue + if url_pattern.match(domain): + message = ( + "allowed_domains accepts only domains, not URLs. " + f"Ignoring URL entry {domain} in allowed_domains." + ) + warnings.warn(message) + elif port_pattern.search(domain): + message = ( + "allowed_domains accepts only domains without ports. " + f"Ignoring entry {domain} in allowed_domains." + ) + warnings.warn(message) + else: + domains.append(re.escape(domain)) + regex = rf'^(.*\.)?({"|".join(domains)})$' + return re.compile(regex) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 24089afea..aa08827c4 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -29,17 +29,49 @@ def _build_redirect_request( **kwargs, cookies=None, ) + if "_scheme_proxy" in redirect_request.meta: + source_request_scheme = urlparse_cached(source_request).scheme + redirect_request_scheme = urlparse_cached(redirect_request).scheme + if source_request_scheme != redirect_request_scheme: + redirect_request.meta.pop("_scheme_proxy") + redirect_request.meta.pop("proxy", None) + redirect_request.meta.pop("_auth_proxy", None) + redirect_request.headers.pop(b"Proxy-Authorization", None) has_cookie_header = "Cookie" in redirect_request.headers has_authorization_header = "Authorization" in redirect_request.headers if has_cookie_header or has_authorization_header: - source_request_netloc = urlparse_cached(source_request).netloc - redirect_request_netloc = urlparse_cached(redirect_request).netloc - if source_request_netloc != redirect_request_netloc: - if has_cookie_header: - del redirect_request.headers["Cookie"] - # https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name - if has_authorization_header: - del redirect_request.headers["Authorization"] + default_ports = {"http": 80, "https": 443} + + parsed_source_request = urlparse_cached(source_request) + source_scheme, source_host, source_port = ( + parsed_source_request.scheme, + parsed_source_request.hostname, + parsed_source_request.port + or default_ports.get(parsed_source_request.scheme), + ) + + parsed_redirect_request = urlparse_cached(redirect_request) + redirect_scheme, redirect_host, redirect_port = ( + parsed_redirect_request.scheme, + parsed_redirect_request.hostname, + parsed_redirect_request.port + or default_ports.get(parsed_redirect_request.scheme), + ) + + if has_cookie_header and ( + redirect_scheme not in {source_scheme, "https"} + or source_host != redirect_host + ): + del redirect_request.headers["Cookie"] + + # https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name + if has_authorization_header and ( + source_scheme != redirect_scheme + or source_host != redirect_host + or source_port != redirect_port + ): + del redirect_request.headers["Authorization"] + return redirect_request @@ -129,9 +161,11 @@ class RedirectMiddleware(BaseRedirectMiddleware): location = request_scheme + "://" + location.lstrip("/") redirected_url = urljoin(request.url, location) + redirected = _build_redirect_request(request, url=redirected_url) + if urlparse_cached(redirected).scheme not in {"http", "https"}: + return response if response.status in (301, 307, 308) or request.method == "HEAD": - redirected = _build_redirect_request(request, url=redirected_url) return self._redirect(redirected, request, spider, response.status) redirected = self._redirect_request_using_get(request, redirected_url) @@ -153,12 +187,16 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): request.meta.get("dont_redirect", False) or request.method == "HEAD" or not isinstance(response, HtmlResponse) + or urlparse_cached(request).scheme not in {"http", "https"} ): return response interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags) - if url and cast(float, interval) < self._maxdelay: - redirected = self._redirect_request_using_get(request, url) + if not url: + return response + redirected = self._redirect_request_using_get(request, url) + if urlparse_cached(redirected).scheme not in {"http", "https"}: + return response + if cast(float, interval) < self._maxdelay: return self._redirect(redirected, request, spider, "meta refresh") - return response diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d7ac7ec35..932475fb5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -101,6 +101,7 @@ DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side + "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50, "scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware": 100, "scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware": 300, "scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware": 350, @@ -301,7 +302,6 @@ SPIDER_MIDDLEWARES = {} SPIDER_MIDDLEWARES_BASE = { # Engine side "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50, - "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": 500, "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index dd2fccfcb..50c93ac9f 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -13,15 +13,21 @@ from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable, Set from scrapy import Spider, signals from scrapy.crawler import Crawler +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.statscollectors import StatsCollector from scrapy.utils.httpobj import urlparse_cached +warnings.warn( + "The scrapy.spidermiddlewares.offsite module is deprecated, use " + "scrapy.downloadermiddlewares.offsite instead.", + ScrapyDeprecationWarning, +) + if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self - logger = logging.getLogger(__name__) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 5d2d490b2..578cde2ac 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -331,7 +331,7 @@ def global_object_name(obj: Any) -> str: >>> global_object_name(Request) 'scrapy.http.request.Request' """ - return f"{obj.__module__}.{obj.__name__}" + return f"{obj.__module__}.{obj.__qualname__}" if hasattr(sys, "pypy_version_info"): diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 062e8a8b4..0155c62eb 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -22,13 +22,11 @@ class ManagerTestCase(TestCase): self.crawler = get_crawler(Spider, self.settings_dict) self.spider = self.crawler._create_spider("foo") self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler) - # some mw depends on stats collector - self.crawler.stats.open_spider(self.spider) - return self.mwman.open_spider(self.spider) + self.crawler.engine = self.crawler._create_engine() + return self.crawler.engine.open_spider(self.spider, start_requests=()) def tearDown(self): - self.crawler.stats.close_spider(self.spider, "") - return self.mwman.close_spider(self.spider) + return self.crawler.engine.close_spider(self.spider) def _download(self, request, response=None): """Executes downloader mw manager's download method and returns diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py new file mode 100644 index 000000000..d4669f450 --- /dev/null +++ b/tests/test_downloadermiddleware_offsite.py @@ -0,0 +1,184 @@ +import pytest + +from scrapy import Request, Spider +from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware +from scrapy.exceptions import IgnoreRequest +from scrapy.utils.test import get_crawler + +UNSET = object() + + +@pytest.mark.parametrize( + ("allowed_domain", "url", "allowed"), + ( + ("example.com", "http://example.com/1", True), + ("example.com", "http://example.org/1", False), + ("example.com", "http://sub.example.com/1", True), + ("sub.example.com", "http://sub.example.com/1", True), + ("sub.example.com", "http://example.com/1", False), + ("example.com", "http://example.com:8000/1", True), + ("example.com", "http://example.org/example.com", False), + ("example.com", "http://example.org/foo.example.com", False), + ("example.com", "http://example.com.example", False), + ("a.example", "http://nota.example", False), + ("b.a.example", "http://notb.a.example", False), + ), +) +def test_process_request_domain_filtering(allowed_domain, url, allowed): + crawler = get_crawler(Spider) + spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request(url) + if allowed: + assert mw.process_request(request, spider) is None + else: + with pytest.raises(IgnoreRequest): + mw.process_request(request, spider) + + +@pytest.mark.parametrize( + ("value", "filtered"), + ( + (UNSET, True), + (None, True), + (False, True), + (True, False), + ), +) +def test_process_request_dont_filter(value, filtered): + crawler = get_crawler(Spider) + spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + kwargs = {} + if value is not UNSET: + kwargs["dont_filter"] = value + request = Request("https://b.example", **kwargs) + if filtered: + with pytest.raises(IgnoreRequest): + mw.process_request(request, spider) + else: + assert mw.process_request(request, spider) is None + + +@pytest.mark.parametrize( + "value", + ( + UNSET, + None, + [], + ), +) +def test_process_request_no_allowed_domains(value): + crawler = get_crawler(Spider) + kwargs = {} + if value is not UNSET: + kwargs["allowed_domains"] = value + spider = crawler._create_spider(name="a", **kwargs) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request("https://example.com") + assert mw.process_request(request, spider) is None + + +def test_process_request_invalid_domains(): + crawler = get_crawler(Spider) + allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] + spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request("https://a.example") + assert mw.process_request(request, spider) is None + for letter in ("b", "c"): + request = Request(f"https://{letter}.example") + with pytest.raises(IgnoreRequest): + mw.process_request(request, spider) + + +@pytest.mark.parametrize( + ("allowed_domain", "url", "allowed"), + ( + ("example.com", "http://example.com/1", True), + ("example.com", "http://example.org/1", False), + ("example.com", "http://sub.example.com/1", True), + ("sub.example.com", "http://sub.example.com/1", True), + ("sub.example.com", "http://example.com/1", False), + ("example.com", "http://example.com:8000/1", True), + ("example.com", "http://example.org/example.com", False), + ("example.com", "http://example.org/foo.example.com", False), + ("example.com", "http://example.com.example", False), + ("a.example", "http://nota.example", False), + ("b.a.example", "http://notb.a.example", False), + ), +) +def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): + crawler = get_crawler(Spider) + spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request(url) + if allowed: + assert mw.request_scheduled(request, spider) is None + else: + with pytest.raises(IgnoreRequest): + mw.request_scheduled(request, spider) + + +@pytest.mark.parametrize( + ("value", "filtered"), + ( + (UNSET, True), + (None, True), + (False, True), + (True, False), + ), +) +def test_request_scheduled_dont_filter(value, filtered): + crawler = get_crawler(Spider) + spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + kwargs = {} + if value is not UNSET: + kwargs["dont_filter"] = value + request = Request("https://b.example", **kwargs) + if filtered: + with pytest.raises(IgnoreRequest): + mw.request_scheduled(request, spider) + else: + assert mw.request_scheduled(request, spider) is None + + +@pytest.mark.parametrize( + "value", + ( + UNSET, + None, + [], + ), +) +def test_request_scheduled_no_allowed_domains(value): + crawler = get_crawler(Spider) + kwargs = {} + if value is not UNSET: + kwargs["allowed_domains"] = value + spider = crawler._create_spider(name="a", **kwargs) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request("https://example.com") + assert mw.request_scheduled(request, spider) is None + + +def test_request_scheduled_invalid_domains(): + crawler = get_crawler(Spider) + allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] + spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request("https://a.example") + assert mw.request_scheduled(request, spider) is None + for letter in ("b", "c"): + request = Request(f"https://{letter}.example") + with pytest.raises(IgnoreRequest): + mw.request_scheduled(request, spider) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 83ff25982..4bfd34fe2 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1,5 +1,9 @@ import unittest +from itertools import chain, product +import pytest + +from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.downloadermiddlewares.redirect import ( MetaRefreshMiddleware, RedirectMiddleware, @@ -7,22 +11,1030 @@ from scrapy.downloadermiddlewares.redirect import ( from scrapy.exceptions import IgnoreRequest from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider +from scrapy.utils.misc import set_environ from scrapy.utils.test import get_crawler -class RedirectMiddlewareTest(unittest.TestCase): +class Base: + class Test(unittest.TestCase): + def test_priority_adjust(self): + req = Request("http://a.com") + rsp = self.get_response(req, "http://a.com/redirected") + req2 = self.mw.process_response(req, rsp, self.spider) + self.assertGreater(req2.priority, req.priority) + + def test_dont_redirect(self): + url = "http://www.example.com/301" + url2 = "http://www.example.com/redirected" + req = Request(url, meta={"dont_redirect": True}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp, self.spider) + assert isinstance(r, Response) + assert r is rsp + + # Test that it redirects when dont_redirect is False + req = Request(url, meta={"dont_redirect": False}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp, self.spider) + assert isinstance(r, Request) + + def test_post(self): + url = "http://www.example.com/302" + url2 = "http://www.example.com/redirected2" + req = Request( + url, + method="POST", + body="test", + headers={"Content-Type": "text/plain", "Content-length": "4"}, + ) + rsp = self.get_response(req, url2) + + req2 = self.mw.process_response(req, rsp, self.spider) + assert isinstance(req2, Request) + self.assertEqual(req2.url, url2) + self.assertEqual(req2.method, "GET") + assert ( + "Content-Type" not in req2.headers + ), "Content-Type header must not be present in redirected request" + assert ( + "Content-Length" not in req2.headers + ), "Content-Length header must not be present in redirected request" + assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" + + def test_max_redirect_times(self): + self.mw.max_redirect_times = 1 + req = Request("http://scrapytest.org/302") + rsp = self.get_response(req, "/redirected") + + req = self.mw.process_response(req, rsp, self.spider) + assert isinstance(req, Request) + assert "redirect_times" in req.meta + self.assertEqual(req.meta["redirect_times"], 1) + self.assertRaises( + IgnoreRequest, self.mw.process_response, req, rsp, self.spider + ) + + def test_ttl(self): + self.mw.max_redirect_times = 100 + req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1}) + rsp = self.get_response(req, "/a") + + req = self.mw.process_response(req, rsp, self.spider) + assert isinstance(req, Request) + self.assertRaises( + IgnoreRequest, self.mw.process_response, req, rsp, self.spider + ) + + def test_redirect_urls(self): + req1 = Request("http://scrapytest.org/first") + rsp1 = self.get_response(req1, "/redirected") + req2 = self.mw.process_response(req1, rsp1, self.spider) + rsp2 = self.get_response(req1, "/redirected2") + req3 = self.mw.process_response(req2, rsp2, self.spider) + + self.assertEqual(req2.url, "http://scrapytest.org/redirected") + self.assertEqual( + req2.meta["redirect_urls"], ["http://scrapytest.org/first"] + ) + self.assertEqual(req3.url, "http://scrapytest.org/redirected2") + self.assertEqual( + req3.meta["redirect_urls"], + ["http://scrapytest.org/first", "http://scrapytest.org/redirected"], + ) + + def test_redirect_reasons(self): + req1 = Request("http://scrapytest.org/first") + rsp1 = self.get_response(req1, "/redirected1") + req2 = self.mw.process_response(req1, rsp1, self.spider) + rsp2 = self.get_response(req2, "/redirected2") + req3 = self.mw.process_response(req2, rsp2, self.spider) + self.assertEqual(req2.meta["redirect_reasons"], [self.reason]) + self.assertEqual(req3.meta["redirect_reasons"], [self.reason, self.reason]) + + def test_cross_origin_header_dropping(self): + safe_headers = {"A": "B"} + cookie_header = {"Cookie": "a=b"} + authorization_header = {"Authorization": "Bearer 123456"} + + original_request = Request( + "https://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers. + internal_response = self.get_response( + original_request, "https://example.com/a" + ) + internal_redirect_request = self.mw.process_response( + original_request, internal_response, self.spider + ) + self.assertIsInstance(internal_redirect_request, Request) + self.assertEqual( + original_request.headers, internal_redirect_request.headers + ) + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers also when the scheme is http. + http_request = Request( + "http://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + http_response = self.get_response(http_request, "http://example.com/a") + http_redirect_request = self.mw.process_response( + http_request, http_response, self.spider + ) + self.assertIsInstance(http_redirect_request, Request) + self.assertEqual(http_request.headers, http_redirect_request.headers) + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_explicit_port_response = self.get_response( + original_request, "https://example.com:443/a" + ) + to_explicit_port_redirect_request = self.mw.process_response( + original_request, to_explicit_port_response, self.spider + ) + self.assertIsInstance(to_explicit_port_redirect_request, Request) + self.assertEqual( + original_request.headers, to_explicit_port_redirect_request.headers + ) + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_implicit_port_response = self.get_response( + original_request, "https://example.com/a" + ) + to_implicit_port_redirect_request = self.mw.process_response( + original_request, to_implicit_port_response, self.spider + ) + self.assertIsInstance(to_implicit_port_redirect_request, Request) + self.assertEqual( + original_request.headers, to_implicit_port_redirect_request.headers + ) + + # A port change drops the Authorization header because the origin + # changes, but keeps the Cookie header because the domain remains the + # same. + different_port_response = self.get_response( + original_request, "https://example.com:8080/a" + ) + different_port_redirect_request = self.mw.process_response( + original_request, different_port_response, self.spider + ) + self.assertIsInstance(different_port_redirect_request, Request) + self.assertEqual( + {**safe_headers, **cookie_header}, + different_port_redirect_request.headers.to_unicode_dict(), + ) + + # A domain change drops both the Authorization and the Cookie header. + external_response = self.get_response( + original_request, "https://example.org/a" + ) + external_redirect_request = self.mw.process_response( + original_request, external_response, self.spider + ) + self.assertIsInstance(external_redirect_request, Request) + self.assertEqual( + safe_headers, external_redirect_request.headers.to_unicode_dict() + ) + + # A scheme upgrade (http → https) drops the Authorization header + # because the origin changes, but keeps the Cookie header because the + # domain remains the same. + upgrade_response = self.get_response(http_request, "https://example.com/a") + upgrade_redirect_request = self.mw.process_response( + http_request, upgrade_response, self.spider + ) + self.assertIsInstance(upgrade_redirect_request, Request) + self.assertEqual( + {**safe_headers, **cookie_header}, + upgrade_redirect_request.headers.to_unicode_dict(), + ) + + # A scheme downgrade (https → http) drops the Authorization header + # because the origin changes, and the Cookie header because its value + # cannot indicate whether the cookies were secure (HTTPS-only) or not. + # + # Note: If the Cookie header is set by the cookie management + # middleware, as recommended in the docs, the dropping of Cookie on + # scheme downgrade is not an issue, because the cookie management + # middleware will add again the Cookie header to the new request if + # appropriate. + downgrade_response = self.get_response( + original_request, "http://example.com/a" + ) + downgrade_redirect_request = self.mw.process_response( + original_request, downgrade_response, self.spider + ) + self.assertIsInstance(downgrade_redirect_request, Request) + self.assertEqual( + safe_headers, + downgrade_redirect_request.headers.to_unicode_dict(), + ) + + def test_meta_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_meta_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_meta_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_meta_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_meta_proxy_http_to_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_meta_proxy_https_to_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_proxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request2.meta["proxy"], "https://b.example") + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_proxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_unproxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertNotIn("Proxy-Authorization", request1.headers) + self.assertNotIn("_auth_proxy", request1.meta) + self.assertNotIn("proxy", request1.meta) + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request2.meta["proxy"], "https://b.example") + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + def test_system_proxy_unproxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertNotIn("Proxy-Authorization", request1.headers) + self.assertNotIn("_auth_proxy", request1.meta) + self.assertNotIn("proxy", request1.meta) + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + def test_system_proxy_proxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request1.meta["proxy"], "https://b.example") + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request3.meta["proxy"], "https://b.example") + + def test_system_proxy_proxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request1.meta["proxy"], "https://b.example") + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request3.meta["proxy"], "https://b.example") + + def test_system_proxy_unproxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertNotIn("Proxy-Authorization", request1.headers) + self.assertNotIn("_auth_proxy", request1.meta) + self.assertNotIn("proxy", request1.meta) + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + def test_system_proxy_unproxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertNotIn("Proxy-Authorization", request1.headers) + self.assertNotIn("_auth_proxy", request1.meta) + self.assertNotIn("proxy", request1.meta) + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + +class RedirectMiddlewareTest(Base.Test): + mwcls = RedirectMiddleware + reason = 302 + def setUp(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") - self.mw = RedirectMiddleware.from_crawler(self.crawler) + self.mw = self.mwcls.from_crawler(self.crawler) - def test_priority_adjust(self): - req = Request("http://a.com") - rsp = Response( - "http://a.com", headers={"Location": "http://a.com/redirected"}, status=301 - ) - req2 = self.mw.process_response(req, rsp, self.spider) - assert req2.priority > req.priority + def get_response(self, request, location, status=302): + headers = {"Location": location} + return Response(request.url, status=status, headers=headers) def test_redirect_3xx_permanent(self): def _test(method, status=301): @@ -52,51 +1064,6 @@ class RedirectMiddlewareTest(unittest.TestCase): _test("POST", status=308) _test("HEAD", status=308) - def test_dont_redirect(self): - url = "http://www.example.com/301" - url2 = "http://www.example.com/redirected" - req = Request(url, meta={"dont_redirect": True}) - rsp = Response(url, headers={"Location": url2}, status=301) - - r = self.mw.process_response(req, rsp, self.spider) - assert isinstance(r, Response) - assert r is rsp - - # Test that it redirects when dont_redirect is False - req = Request(url, meta={"dont_redirect": False}) - rsp = Response(url2, status=200) - - r = self.mw.process_response(req, rsp, self.spider) - assert isinstance(r, Response) - assert r is rsp - - def test_redirect_302(self): - url = "http://www.example.com/302" - url2 = "http://www.example.com/redirected2" - req = Request( - url, - method="POST", - body="test", - headers={"Content-Type": "text/plain", "Content-length": "4"}, - ) - rsp = Response(url, headers={"Location": url2}, status=302) - - req2 = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req2, Request) - self.assertEqual(req2.url, url2) - self.assertEqual(req2.method, "GET") - assert ( - "Content-Type" not in req2.headers - ), "Content-Type header must not be present in redirected request" - assert ( - "Content-Length" not in req2.headers - ), "Content-Length header must not be present in redirected request" - assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" - - # response without Location header but with status code is 3XX should be ignored - del rsp.headers["Location"] - assert self.mw.process_response(req, rsp, self.spider) is rsp - def test_redirect_302_head(self): url = "http://www.example.com/302" url2 = "http://www.example.com/redirected2" @@ -108,10 +1075,6 @@ class RedirectMiddlewareTest(unittest.TestCase): self.assertEqual(req2.url, url2) self.assertEqual(req2.method, "HEAD") - # response without Location header but with status code is 3XX should be ignored - del rsp.headers["Location"] - assert self.mw.process_response(req, rsp, self.spider) is rsp - def test_redirect_302_relative(self): url = "http://www.example.com/302" url2 = "///i8n.example2.com/302" @@ -124,81 +1087,6 @@ class RedirectMiddlewareTest(unittest.TestCase): self.assertEqual(req2.url, url3) self.assertEqual(req2.method, "HEAD") - # response without Location header but with status code is 3XX should be ignored - del rsp.headers["Location"] - assert self.mw.process_response(req, rsp, self.spider) is rsp - - def test_max_redirect_times(self): - self.mw.max_redirect_times = 1 - req = Request("http://scrapytest.org/302") - rsp = Response( - "http://scrapytest.org/302", headers={"Location": "/redirected"}, status=302 - ) - - req = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req, Request) - assert "redirect_times" in req.meta - self.assertEqual(req.meta["redirect_times"], 1) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) - - def test_ttl(self): - self.mw.max_redirect_times = 100 - req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1}) - rsp = Response( - "http://www.scrapytest.org/302", - headers={"Location": "/redirected"}, - status=302, - ) - - req = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req, Request) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) - - def test_redirect_urls(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = Response( - "http://scrapytest.org/first", - headers={"Location": "/redirected"}, - status=302, - ) - req2 = self.mw.process_response(req1, rsp1, self.spider) - rsp2 = Response( - "http://scrapytest.org/redirected", - headers={"Location": "/redirected2"}, - status=302, - ) - req3 = self.mw.process_response(req2, rsp2, self.spider) - - self.assertEqual(req2.url, "http://scrapytest.org/redirected") - self.assertEqual(req2.meta["redirect_urls"], ["http://scrapytest.org/first"]) - self.assertEqual(req3.url, "http://scrapytest.org/redirected2") - self.assertEqual( - req3.meta["redirect_urls"], - ["http://scrapytest.org/first", "http://scrapytest.org/redirected"], - ) - - def test_redirect_reasons(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = Response( - "http://scrapytest.org/first", - headers={"Location": "/redirected1"}, - status=301, - ) - req2 = self.mw.process_response(req1, rsp1, self.spider) - rsp2 = Response( - "http://scrapytest.org/redirected1", - headers={"Location": "/redirected2"}, - status=301, - ) - req3 = self.mw.process_response(req2, rsp2, self.spider) - - self.assertEqual(req2.meta["redirect_reasons"], [301]) - self.assertEqual(req3.meta["redirect_reasons"], [301, 301]) - def test_spider_handling(self): smartspider = self.crawler._create_spider("smarty") smartspider.handle_httpstatus_list = [404, 301, 302] @@ -247,53 +1135,84 @@ class RedirectMiddlewareTest(unittest.TestCase): perc_encoded_utf8_url = "http://scrapytest.org/a%C3%A7%C3%A3o" self.assertEqual(perc_encoded_utf8_url, req_result.url) - def test_cross_domain_header_dropping(self): - safe_headers = {"A": "B"} - original_request = Request( - "https://example.com", - headers={"Cookie": "a=b", "Authorization": "a", **safe_headers}, - ) - - internal_response = Response( - "https://example.com", - headers={"Location": "https://example.com/a"}, - status=301, - ) - internal_redirect_request = self.mw.process_response( - original_request, internal_response, self.spider - ) - self.assertIsInstance(internal_redirect_request, Request) - self.assertEqual(original_request.headers, internal_redirect_request.headers) - - external_response = Response( - "https://example.com", - headers={"Location": "https://example.org/a"}, - status=301, - ) - external_redirect_request = self.mw.process_response( - original_request, external_response, self.spider - ) - self.assertIsInstance(external_redirect_request, Request) - self.assertEqual( - safe_headers, external_redirect_request.headers.to_unicode_dict() - ) + def test_no_location(self): + request = Request("https://example.com") + response = Response(request.url, status=302) + assert self.mw.process_response(request, response, self.spider) is response -class MetaRefreshMiddlewareTest(unittest.TestCase): +SCHEME_PARAMS = ("url", "location", "target") +HTTP_SCHEMES = ("http", "https") +NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo") +REDIRECT_SCHEME_CASES = ( + # http/https → http/https redirects + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + f"{output_scheme}://example.com/b", + ) + for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2) + ), + # http/https → data/file/ftp/s3/foo does not redirect + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + None, + ) + for input_scheme in HTTP_SCHEMES + for output_scheme in NON_HTTP_SCHEMES + ), + # http/https → relative redirects + *( + ( + f"{scheme}://example.com/a", + location, + f"{scheme}://example.com/b", + ) + for scheme in HTTP_SCHEMES + for location in ("//example.com/b", "/b") + ), + # Note: We do not test data/file/ftp/s3 schemes for the initial URL + # because their download handlers cannot return a status code of 3xx. +) + + +@pytest.mark.parametrize(SCHEME_PARAMS, REDIRECT_SCHEME_CASES) +def test_redirect_schemes(url, location, target): + crawler = get_crawler(Spider) + spider = crawler._create_spider("foo") + mw = RedirectMiddleware.from_crawler(crawler) + request = Request(url) + response = Response(url, headers={"Location": location}, status=301) + redirect = mw.process_response(request, response, spider) + if target is None: + assert redirect == response + else: + assert isinstance(redirect, Request) + assert redirect.url == target + + +def meta_refresh_body(url, interval=5): + html = f"""""" + return html.encode("utf-8") + + +class MetaRefreshMiddlewareTest(Base.Test): + mwcls = MetaRefreshMiddleware + reason = "meta refresh" + def setUp(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider("foo") - self.mw = MetaRefreshMiddleware.from_crawler(crawler) + self.mw = self.mwcls.from_crawler(crawler) def _body(self, interval=5, url="http://example.org/newpage"): - html = f"""""" - return html.encode("utf-8") + return meta_refresh_body(url, interval) - def test_priority_adjust(self): - req = Request("http://a.com") - rsp = HtmlResponse(req.url, body=self._body()) - req2 = self.mw.process_response(req, rsp, self.spider) - assert req2.priority > req.priority + def get_response(self, request, location): + return HtmlResponse(request.url, body=self._body(url=location)) def test_meta_refresh(self): req = Request(url="http://example.org") @@ -332,62 +1251,6 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): ), "Content-Length header must not be present in redirected request" assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" - def test_max_redirect_times(self): - self.mw.max_redirect_times = 1 - req = Request("http://scrapytest.org/max") - rsp = HtmlResponse(req.url, body=self._body()) - - req = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req, Request) - assert "redirect_times" in req.meta - self.assertEqual(req.meta["redirect_times"], 1) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) - - def test_ttl(self): - self.mw.max_redirect_times = 100 - req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1}) - rsp = HtmlResponse(req.url, body=self._body()) - - req = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req, Request) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) - - def test_redirect_urls(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = HtmlResponse(req1.url, body=self._body(url="/redirected")) - req2 = self.mw.process_response(req1, rsp1, self.spider) - assert isinstance(req2, Request), req2 - rsp2 = HtmlResponse(req2.url, body=self._body(url="/redirected2")) - req3 = self.mw.process_response(req2, rsp2, self.spider) - assert isinstance(req3, Request), req3 - self.assertEqual(req2.url, "http://scrapytest.org/redirected") - self.assertEqual(req2.meta["redirect_urls"], ["http://scrapytest.org/first"]) - self.assertEqual(req3.url, "http://scrapytest.org/redirected2") - self.assertEqual( - req3.meta["redirect_urls"], - ["http://scrapytest.org/first", "http://scrapytest.org/redirected"], - ) - - def test_redirect_reasons(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = HtmlResponse( - "http://scrapytest.org/first", body=self._body(url="/redirected") - ) - req2 = self.mw.process_response(req1, rsp1, self.spider) - rsp2 = HtmlResponse( - "http://scrapytest.org/redirected", body=self._body(url="/redirected1") - ) - req3 = self.mw.process_response(req2, rsp2, self.spider) - - self.assertEqual(req2.meta["redirect_reasons"], ["meta refresh"]) - self.assertEqual( - req3.meta["redirect_reasons"], ["meta refresh", "meta refresh"] - ) - def test_ignore_tags_default(self): req = Request(url="http://example.org") body = ( @@ -413,5 +1276,45 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): assert isinstance(response, Response) +@pytest.mark.parametrize( + SCHEME_PARAMS, + ( + *REDIRECT_SCHEME_CASES, + # data/file/ftp/s3/foo → * does not redirect + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + None, + ) + for input_scheme in NON_HTTP_SCHEMES + for output_scheme in chain(HTTP_SCHEMES, NON_HTTP_SCHEMES) + ), + # data/file/ftp/s3/foo → relative does not redirect + *( + ( + f"{scheme}://example.com/a", + location, + None, + ) + for scheme in NON_HTTP_SCHEMES + for location in ("//example.com/b", "/b") + ), + ), +) +def test_meta_refresh_schemes(url, location, target): + crawler = get_crawler(Spider) + spider = crawler._create_spider("foo") + mw = MetaRefreshMiddleware.from_crawler(crawler) + request = Request(url) + response = HtmlResponse(url, body=meta_refresh_body(location)) + redirect = mw.process_response(request, response, spider) + if target is None: + assert redirect == response + else: + assert isinstance(redirect, Request) + assert redirect.url == target + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_engine.py b/tests/test_engine.py index 8d7afb6a1..33544e8db 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -15,8 +15,10 @@ import subprocess import sys from collections import defaultdict from dataclasses import dataclass +from logging import DEBUG from pathlib import Path from threading import Timer +from unittest.mock import Mock from urllib.parse import urlparse import attr @@ -27,11 +29,13 @@ from twisted.trial import unittest from twisted.web import server, static, util from scrapy import signals -from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import CloseSpider +from scrapy.core.engine import ExecutionEngine, Slot +from scrapy.core.scheduler import BaseScheduler +from scrapy.exceptions import CloseSpider, IgnoreRequest from scrapy.http import Request from scrapy.item import Field, Item from scrapy.linkextractors import LinkExtractor +from scrapy.signals import request_scheduled from scrapy.spiders import Spider from scrapy.utils.signal import disconnect_all from scrapy.utils.test import get_crawler @@ -467,6 +471,38 @@ class EngineTest(unittest.TestCase): self.assertNotIn(b"Traceback", stderr) +def test_request_scheduled_signal(caplog): + class TestScheduler(BaseScheduler): + def __init__(self): + self.enqueued = [] + + def enqueue_request(self, request: Request) -> bool: + self.enqueued.append(request) + return True + + def signal_handler(request: Request, spider: Spider) -> None: + if "drop" in request.url: + raise IgnoreRequest + + spider = TestSpider() + crawler = get_crawler(spider.__class__) + engine = ExecutionEngine(crawler, lambda _: None) + engine.downloader._slot_gc_loop.stop() + scheduler = TestScheduler() + engine.slot = Slot((), None, Mock(), scheduler) + crawler.signals.connect(signal_handler, request_scheduled) + keep_request = Request("https://keep.example") + engine._schedule_request(keep_request, spider) + drop_request = Request("https://drop.example") + caplog.set_level(DEBUG) + engine._schedule_request(drop_request, spider) + assert scheduler.enqueued == [ + keep_request + ], f"{scheduler.enqueued!r} != [{keep_request!r}]" + assert "dropped request " in caplog.text + crawler.signals.disconnect(signal_handler, request_scheduled) + + if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == "runserver": start_test_site(debug=True) diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 90bd350a5..3831f4c21 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -6,6 +6,7 @@ import unittest import warnings from pathlib import Path +from scrapy.utils.misc import set_environ from scrapy.utils.project import data_path, get_project_settings @@ -38,20 +39,6 @@ class ProjectUtilsTest(unittest.TestCase): self.assertEqual(abspath, data_path(abspath)) -@contextlib.contextmanager -def set_env(**update): - modified = set(update.keys()) & set(os.environ.keys()) - update_after = {k: os.environ[k] for k in modified} - remove_after = frozenset(k for k in update if k not in os.environ) - try: - os.environ.update(update) - yield - finally: - os.environ.update(update_after) - for k in remove_after: - os.environ.pop(k) - - class GetProjectSettingsTestCase(unittest.TestCase): def test_valid_envvar(self): value = "tests.test_cmdline.settings" @@ -60,7 +47,7 @@ class GetProjectSettingsTestCase(unittest.TestCase): } with warnings.catch_warnings(): warnings.simplefilter("error") - with set_env(**envvars): + with set_environ(**envvars): settings = get_project_settings() assert settings.get("SETTINGS_MODULE") == value @@ -69,7 +56,7 @@ class GetProjectSettingsTestCase(unittest.TestCase): envvars = { "SCRAPY_FOO": "bar", } - with set_env(**envvars): + with set_environ(**envvars): settings = get_project_settings() assert settings.get("SCRAPY_FOO") is None @@ -80,7 +67,7 @@ class GetProjectSettingsTestCase(unittest.TestCase): "SCRAPY_FOO": "bar", "SCRAPY_SETTINGS_MODULE": value, } - with set_env(**envvars): + with set_environ(**envvars): settings = get_project_settings() assert settings.get("SETTINGS_MODULE") == value assert settings.get("SCRAPY_FOO") is None diff --git a/tox.ini b/tox.ini index ede139756..cde4243f3 100644 --- a/tox.ini +++ b/tox.ini @@ -26,6 +26,9 @@ deps = # mitmproxy does not support PyPy mitmproxy; implementation_name != 'pypy' + # https://github.com/pallets/werkzeug/pull/2768 breaks flask, required by + # mitmproxy. + werkzeug < 3; python_version < '3.9' and implementation_name != 'pypy' passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -90,6 +93,7 @@ commands = twine check dist/* [pinned] +basepython = python3.8 deps = cryptography==36.0.0 cssselect==0.9.1 @@ -116,7 +120,7 @@ commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} [testenv:pinned] -basepython = python3.8 +basepython = {[pinned]basepython} deps = {[pinned]deps} PyDispatcher==2.0.5 @@ -126,7 +130,7 @@ setenv = commands = {[pinned]commands} [testenv:windows-pinned] -basepython = python3 +basepython = {[pinned]basepython} deps = {[pinned]deps} PyDispatcher==2.0.5 @@ -155,7 +159,7 @@ deps = ipython [testenv:extra-deps-pinned] -basepython = python3.8 +basepython = {[pinned]basepython} deps = {[pinned]deps} boto3==1.20.0 @@ -179,6 +183,7 @@ commands = {[testenv]commands} --reactor=asyncio [testenv:asyncio-pinned] +basepython = {[pinned]basepython} deps = {[testenv:pinned]deps} commands = {[pinned]commands} --reactor=asyncio install_command = {[pinned]install_command} @@ -191,12 +196,12 @@ commands = pytest {posargs:--durations=10 docs scrapy tests} [testenv:pypy3-pinned] -basepython = {[testenv:pypy3]basepython} +basepython = pypy3.8 deps = {[pinned]deps} PyPyDispatcher==2.1.0 commands = - pytest --durations=10 scrapy tests + pytest {posargs:--durations=10 scrapy tests} install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -244,7 +249,7 @@ commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} [testenv:botocore-pinned] -basepython = python3.8 +basepython = {[pinned]basepython} deps = {[pinned]deps} botocore==1.4.87 From 812fd2368f705d033f5f39c152130b12a0fe9b1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 May 2024 11:48:43 +0200 Subject: [PATCH 178/269] Allow user-defined secure cookies (#6357) --- docs/topics/request-response.rst | 3 +- scrapy/downloadermiddlewares/cookies.py | 14 ++- scrapy/utils/_compression.py | 9 +- tests/test_downloadermiddleware_cookies.py | 111 ++++++++++++++++++++- 4 files changed, 126 insertions(+), 11 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index eb70ebce8..3c2843bc1 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -94,13 +94,14 @@ Request objects .. code-block:: python request_with_cookies = Request( - url="http://www.example.com", + url="https://www.example.com", cookies=[ { "name": "currency", "value": "USD", "domain": "example.com", "path": "/currency", + "secure": True, }, ], ) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 85781efd6..6ada3b474 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -33,6 +33,7 @@ logger = logging.getLogger(__name__) _split_domain = TLDExtract(include_psl_private_domains=True) +_UNSET = object() def _is_public_domain(domain: str) -> bool: @@ -133,6 +134,7 @@ class CookiesMiddleware: Decode from bytes if necessary. """ decoded = {} + flags = set() for key in ("name", "value", "path", "domain"): if cookie.get(key) is None: if key in ("name", "value"): @@ -152,10 +154,16 @@ class CookiesMiddleware: cookie, ) decoded[key] = cookie[key].decode("latin1", errors="replace") - + for flag in ("secure",): + value = cookie.get(flag, _UNSET) + if value is _UNSET or not value: + continue + flags.add(flag) cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}" for key, value in decoded.items(): # path, domain cookie_str += f"; {key.capitalize()}={value}" + for flag in flags: # secure + cookie_str += f"; {flag.capitalize()}" return cookie_str def _get_request_cookies( @@ -168,9 +176,11 @@ class CookiesMiddleware: return [] cookies: Iterable[Dict[str, Any]] if isinstance(request.cookies, dict): - cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) + cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items()) else: cookies = request.cookies + for cookie in cookies: + cookie.setdefault("secure", urlparse_cached(request).scheme == "https") formatted = filter(None, (self._format_cookie(c, request) for c in cookies)) response = Response(request.url, headers={"Set-Cookie": formatted}) return jar.make_cookies(response, request) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 84c255c28..591737b8e 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -20,11 +20,10 @@ else: "You have brotlipy installed, and Scrapy will use it, but " "Scrapy support for brotlipy is deprecated and will stop " "working in a future version of Scrapy. brotlipy itself is " - "deprecated, it has been superseded by brotlicffi. " - "Please, uninstall brotlipy " - "and install brotli or brotlicffi instead. brotlipy has the same import " - "name as brotli, so keeping both installed is strongly " - "discouraged." + "deprecated, it has been superseded by brotlicffi. Please, " + "uninstall brotlipy and install brotli or brotlicffi instead. " + "brotlipy has the same import name as brotli, so keeping both " + "installed is strongly discouraged." ), ScrapyDeprecationWarning, ) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 425fabcc7..1f7e6615c 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -14,6 +14,8 @@ from scrapy.spiders import Spider from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler +UNSET = object() + def _cookie_to_set_cookie_value(cookie): """Given a cookie defined as a dictionary with name and value keys, and @@ -414,19 +416,19 @@ class CookiesMiddlewareTest(TestCase): "scrapy.downloadermiddlewares.cookies", "WARNING", "Invalid cookie found in request :" - " {'value': 'bar'} ('name' is missing)", + " {'value': 'bar', 'secure': False} ('name' is missing)", ), ( "scrapy.downloadermiddlewares.cookies", "WARNING", "Invalid cookie found in request :" - " {'name': 'foo'} ('value' is missing)", + " {'name': 'foo', 'secure': False} ('value' is missing)", ), ( "scrapy.downloadermiddlewares.cookies", "WARNING", "Invalid cookie found in request :" - " {'name': 'foo', 'value': None} ('value' is missing)", + " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)", ), ) self.assertCookieValEqual(req1.headers["Cookie"], "key=value1") @@ -732,3 +734,106 @@ class CookiesMiddlewareTest(TestCase): "co.uk", cookies=True, ) + + def _test_cookie_redirect_scheme_change( + self, secure, from_scheme, to_scheme, cookies1, cookies2, cookies3 + ): + """When a redirect causes the URL scheme to change from *from_scheme* + to *to_scheme*, while domain and port remain the same, and given a + cookie on the initial request with its secure attribute set to + *secure*, check if the cookie should be set on the Cookie header of the + initial request (*cookies1*), if it should be kept by the redirect + middleware (*cookies2*), and if it should be present on the Cookie + header in the redirected request (*cookie3*).""" + cookie_kwargs = {} + if secure is not UNSET: + cookie_kwargs["secure"] = secure + input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}] + + request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies) + self.mw.process_request(request1, self.spider) + cookies = request1.headers.get("Cookie") + self.assertEqual(cookies, b"a=b" if cookies1 else None) + + response = Response( + f"{from_scheme}://a.example", + headers={"Location": f"{to_scheme}://a.example"}, + status=301, + ) + self.assertEqual( + self.mw.process_response(request1, response, self.spider), + response, + ) + + request2 = self.redirect_middleware.process_response( + request1, + response, + self.spider, + ) + self.assertIsInstance(request2, Request) + cookies = request2.headers.get("Cookie") + self.assertEqual(cookies, b"a=b" if cookies2 else None) + + self.mw.process_request(request2, self.spider) + cookies = request2.headers.get("Cookie") + self.assertEqual(cookies, b"a=b" if cookies3 else None) + + def test_cookie_redirect_secure_undefined_downgrade(self): + self._test_cookie_redirect_scheme_change( + secure=UNSET, + from_scheme="https", + to_scheme="http", + cookies1=True, + cookies2=True, # xfail, due to a bug in the redirect middleware fixed elsewhere + cookies3=False, + ) + + def test_cookie_redirect_secure_undefined_upgrade(self): + self._test_cookie_redirect_scheme_change( + secure=UNSET, + from_scheme="http", + to_scheme="https", + cookies1=True, + cookies2=True, + cookies3=True, + ) + + def test_cookie_redirect_secure_false_downgrade(self): + self._test_cookie_redirect_scheme_change( + secure=False, + from_scheme="https", + to_scheme="http", + cookies1=True, + cookies2=True, # xfail, due to a bug in the redirect middleware fixed elsewhere + cookies3=True, + ) + + def test_cookie_redirect_secure_false_upgrade(self): + self._test_cookie_redirect_scheme_change( + secure=False, + from_scheme="http", + to_scheme="https", + cookies1=True, + cookies2=True, + cookies3=True, + ) + + def test_cookie_redirect_secure_true_downgrade(self): + self._test_cookie_redirect_scheme_change( + secure=True, + from_scheme="https", + to_scheme="http", + cookies1=True, + cookies2=True, # xfail, due to a bug in the redirect middleware fixed elsewhere + cookies3=False, + ) + + def test_cookie_redirect_secure_true_upgrade(self): + self._test_cookie_redirect_scheme_change( + secure=True, + from_scheme="http", + to_scheme="https", + cookies1=False, + cookies2=False, + cookies3=True, + ) From 631fc65fadb874629787ae5f7fdd876b9ec96a29 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 16 May 2024 18:42:09 +0400 Subject: [PATCH 179/269] Update expectations of cookies after redirects. (#6367) --- tests/test_downloadermiddleware_cookies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 1f7e6615c..5eccd396a 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -784,7 +784,7 @@ class CookiesMiddlewareTest(TestCase): from_scheme="https", to_scheme="http", cookies1=True, - cookies2=True, # xfail, due to a bug in the redirect middleware fixed elsewhere + cookies2=False, cookies3=False, ) @@ -804,7 +804,7 @@ class CookiesMiddlewareTest(TestCase): from_scheme="https", to_scheme="http", cookies1=True, - cookies2=True, # xfail, due to a bug in the redirect middleware fixed elsewhere + cookies2=False, cookies3=True, ) @@ -824,7 +824,7 @@ class CookiesMiddlewareTest(TestCase): from_scheme="https", to_scheme="http", cookies1=True, - cookies2=True, # xfail, due to a bug in the redirect middleware fixed elsewhere + cookies2=False, cookies3=False, ) From b99526b740890e63f1d05074b3e358a9ae59b77f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 19 May 2024 15:45:51 +0500 Subject: [PATCH 180/269] Full typing for scrapy/contracts. --- scrapy/contracts/__init__.py | 75 ++++++++++++++++++++++++------------ scrapy/contracts/default.py | 19 ++++----- 2 files changed, 60 insertions(+), 34 deletions(-) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index d46eb7c51..b300b8457 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -3,10 +3,23 @@ import sys from functools import wraps from inspect import getmembers from types import CoroutineType -from typing import AsyncGenerator, Dict, Optional, Type -from unittest import TestCase +from typing import ( + Any, + AsyncGenerator, + Callable, + Dict, + Iterable, + List, + Optional, + Tuple, + Type, +) +from unittest import TestCase, TestResult -from scrapy.http import Request +from twisted.python.failure import Failure + +from scrapy import Spider +from scrapy.http import Request, Response from scrapy.utils.python import get_spec from scrapy.utils.spider import iterate_spider_output @@ -15,18 +28,20 @@ class Contract: """Abstract class for contracts""" request_cls: Optional[Type[Request]] = None + name: str - def __init__(self, method, *args): + def __init__(self, method: Callable, *args: Any): self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook") self.testcase_post = _create_testcase(method, f"@{self.name} post-hook") - self.args = args + self.args: Tuple[Any, ...] = args - def add_pre_hook(self, request, results): + def add_pre_hook(self, request: Request, results: TestResult) -> Request: if hasattr(self, "pre_process"): cb = request.callback + assert cb is not None @wraps(cb) - def wrapper(response, **cb_kwargs): + def wrapper(response: Response, **cb_kwargs: Any) -> List[Any]: try: results.startTest(self.testcase_pre) self.pre_process(response) @@ -49,12 +64,13 @@ class Contract: return request - def add_post_hook(self, request, results): + def add_post_hook(self, request: Request, results: TestResult) -> Request: if hasattr(self, "post_process"): cb = request.callback + assert cb is not None @wraps(cb) - def wrapper(response, **cb_kwargs): + def wrapper(response: Response, **cb_kwargs: Any) -> List[Any]: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") @@ -76,18 +92,18 @@ class Contract: return request - def adjust_request_args(self, args): + def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: return args class ContractsManager: - contracts: Dict[str, Contract] = {} + contracts: Dict[str, Type[Contract]] = {} - def __init__(self, contracts): + def __init__(self, contracts: Iterable[Type[Contract]]): for contract in contracts: self.contracts[contract.name] = contract - def tested_methods_from_spidercls(self, spidercls): + def tested_methods_from_spidercls(self, spidercls: Type[Spider]) -> List[str]: is_method = re.compile(r"^\s*@", re.MULTILINE).search methods = [] for key, value in getmembers(spidercls): @@ -96,21 +112,26 @@ class ContractsManager: return methods - def extract_contracts(self, method): - contracts = [] + def extract_contracts(self, method: Callable) -> List[Contract]: + contracts: List[Contract] = [] + assert method.__doc__ is not None for line in method.__doc__.split("\n"): line = line.strip() if line.startswith("@"): - name, args = re.match(r"@(\w+)\s*(.*)", line).groups() + m = re.match(r"@(\w+)\s*(.*)", line) + assert m is not None + name, args = m.groups() args = re.split(r"\s+", args) contracts.append(self.contracts[name](method, *args)) return contracts - def from_spider(self, spider, results): - requests = [] + def from_spider( + self, spider: Spider, results: TestResult + ) -> List[Optional[Request]]: + requests: List[Optional[Request]] = [] for method in self.tested_methods_from_spidercls(type(spider)): bound_method = spider.__getattribute__(method) try: @@ -121,7 +142,7 @@ class ContractsManager: return requests - def from_method(self, method, results): + def from_method(self, method: Callable, results: TestResult) -> Optional[Request]: contracts = self.extract_contracts(method) if contracts: request_cls = Request @@ -154,14 +175,18 @@ class ContractsManager: self._clean_req(request, method, results) return request + return None - def _clean_req(self, request, method, results): + def _clean_req( + self, request: Request, method: Callable, results: TestResult + ) -> None: """stop the request from returning objects and records any errors""" cb = request.callback + assert cb is not None @wraps(cb) - def cb_wrapper(response, **cb_kwargs): + def cb_wrapper(response: Response, **cb_kwargs: Any) -> None: try: output = cb(response, **cb_kwargs) output = list(iterate_spider_output(output)) @@ -169,7 +194,7 @@ class ContractsManager: case = _create_testcase(method, "callback") results.addError(case, sys.exc_info()) - def eb_wrapper(failure): + def eb_wrapper(failure: Failure) -> None: case = _create_testcase(method, "errback") exc_info = failure.type, failure.value, failure.getTracebackObject() results.addError(case, exc_info) @@ -178,11 +203,11 @@ class ContractsManager: request.errback = eb_wrapper -def _create_testcase(method, desc): - spider = method.__self__.name +def _create_testcase(method: Callable, desc: str) -> TestCase: + spider = method.__self__.name # type: ignore[attr-defined] class ContractTestCase(TestCase): - def __str__(_self): + def __str__(_self) -> str: return f"[{spider}] {method.__name__} ({desc})" name = f"{spider}_{method.__name__}" diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index eac702cef..71ca4168a 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -1,4 +1,5 @@ import json +from typing import Any, Callable, Dict, List, Optional from itemadapter import ItemAdapter, is_item @@ -15,7 +16,7 @@ class UrlContract(Contract): name = "url" - def adjust_request_args(self, args): + def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: args["url"] = self.args[0] return args @@ -29,7 +30,7 @@ class CallbackKeywordArgumentsContract(Contract): name = "cb_kwargs" - def adjust_request_args(self, args): + def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: args["cb_kwargs"] = json.loads(" ".join(self.args)) return args @@ -48,14 +49,14 @@ class ReturnsContract(Contract): """ name = "returns" - object_type_verifiers = { + object_type_verifiers: Dict[Optional[str], Callable[[Any], bool]] = { "request": lambda x: isinstance(x, Request), "requests": lambda x: isinstance(x, Request), "item": is_item, "items": is_item, } - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) if len(self.args) not in [1, 2, 3]: @@ -66,16 +67,16 @@ class ReturnsContract(Contract): self.obj_type_verifier = self.object_type_verifiers[self.obj_name] try: - self.min_bound = int(self.args[1]) + self.min_bound: float = int(self.args[1]) except IndexError: self.min_bound = 1 try: - self.max_bound = int(self.args[2]) + self.max_bound: float = int(self.args[2]) except IndexError: self.max_bound = float("inf") - def post_process(self, output): + def post_process(self, output: List[Any]) -> None: occurrences = 0 for x in output: if self.obj_type_verifier(x): @@ -85,7 +86,7 @@ class ReturnsContract(Contract): if not assertion: if self.min_bound == self.max_bound: - expected = self.min_bound + expected = str(self.min_bound) else: expected = f"{self.min_bound}..{self.max_bound}" @@ -101,7 +102,7 @@ class ScrapesContract(Contract): name = "scrapes" - def post_process(self, output): + def post_process(self, output: List[Any]) -> None: for x in output: if is_item(x): missing = [arg for arg in self.args if arg not in ItemAdapter(x)] From e676cd3ce0d488f56498b766912725066bcee4d9 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 22 May 2024 07:55:53 -0300 Subject: [PATCH 181/269] docs: Remove top-level reactor imports from CrawlerProces/CrawlerRunner examples --- docs/topics/practices.rst | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index cd359b147..7731180fe 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -92,7 +92,6 @@ reactor after ``MySpider`` has finished running. .. code-block:: python - from twisted.internet import reactor import scrapy from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging @@ -107,6 +106,33 @@ reactor after ``MySpider`` has finished running. runner = CrawlerRunner() d = runner.crawl(MySpider) + from twisted.internet import reactor + + d.addBoth(lambda _: reactor.stop()) + reactor.run() # the script will block here until the crawling is finished + +Same example but using a non-default reactor, is only necessary call ``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically. + +.. code-block:: python + + import scrapy + from scrapy.crawler import CrawlerRunner + from scrapy.utils.log import configure_logging + + + class MySpider(scrapy.Spider): + # Your spider definition + ... + + + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + from scrapy.utils.reactor import install_reactor + + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + runner = CrawlerRunner() + d = runner.crawl(MySpider) + from twisted.internet import reactor + d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished @@ -151,7 +177,6 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: .. code-block:: python import scrapy - from twisted.internet import reactor from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging from scrapy.utils.project import get_project_settings @@ -173,6 +198,8 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: runner.crawl(MySpider1) runner.crawl(MySpider2) d = runner.join() + from twisted.internet import reactor + d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until all crawling jobs are finished @@ -181,7 +208,7 @@ Same example but running the spiders sequentially by chaining the deferreds: .. code-block:: python - from twisted.internet import reactor, defer + from twisted.internet import defer from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging from scrapy.utils.project import get_project_settings @@ -209,6 +236,8 @@ Same example but running the spiders sequentially by chaining the deferreds: reactor.stop() + from twisted.internet import reactor + crawl() reactor.run() # the script will block here until the last crawl call is finished From 8210fae25a9d812447df617155001b9861e0d834 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 22 May 2024 18:50:50 -0300 Subject: [PATCH 182/269] Update docs/topics/practices.rst Co-authored-by: Andrey Rakhmatullin --- docs/topics/practices.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 7731180fe..710be7aa2 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -111,7 +111,9 @@ reactor after ``MySpider`` has finished running. d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished -Same example but using a non-default reactor, is only necessary call ``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically. +Same example but using a non-default reactor, it's only necessary call +``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` + already does this automatically. .. code-block:: python From dc6a495fee41949d50178b9e46d6f41e83425ca2 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 22 May 2024 18:51:02 -0300 Subject: [PATCH 183/269] Update docs/topics/practices.rst Co-authored-by: Andrey Rakhmatullin --- docs/topics/practices.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 710be7aa2..cec098012 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -106,6 +106,7 @@ reactor after ``MySpider`` has finished running. runner = CrawlerRunner() d = runner.crawl(MySpider) + from twisted.internet import reactor d.addBoth(lambda _: reactor.stop()) From 3f66b66e3f645393dbb263a1ec7ab04bdabd74b4 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 22 May 2024 22:01:55 -0300 Subject: [PATCH 184/269] fix: checks --- docs/topics/practices.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index cec098012..aa81ceea5 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -113,8 +113,7 @@ reactor after ``MySpider`` has finished running. reactor.run() # the script will block here until the crawling is finished Same example but using a non-default reactor, it's only necessary call -``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` - already does this automatically. +``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically. .. code-block:: python From e143dc795228424fa98cb40e17b9993617ae61ae Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 22 May 2024 22:26:31 -0300 Subject: [PATCH 185/269] Update tests-macos.yml --- .github/workflows/tests-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 252176464..95016146e 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,4 +1,4 @@ -name: macOS +name: macOS. on: [push, pull_request] concurrency: From 9d5a0d287b69a69fe34cbe3130438fb36f1f3441 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 22 May 2024 22:27:07 -0300 Subject: [PATCH 186/269] Retrigger CI --- .github/workflows/tests-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 95016146e..252176464 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,4 +1,4 @@ -name: macOS. +name: macOS on: [push, pull_request] concurrency: From 17e623cf0cfb5c695c43ceb069026d44cb28ca21 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 23 May 2024 07:00:24 -0300 Subject: [PATCH 187/269] Update docs/topics/practices.rst Co-authored-by: Andrey Rakhmatullin --- docs/topics/practices.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index aa81ceea5..64b3b6e81 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -128,6 +128,7 @@ Same example but using a non-default reactor, it's only necessary call configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + from scrapy.utils.reactor import install_reactor install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") From 8ec67ca230a69effb2e0442fb2a6c06cd6c92adf Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 23 May 2024 07:00:35 -0300 Subject: [PATCH 188/269] Update docs/topics/practices.rst Co-authored-by: Andrey Rakhmatullin --- docs/topics/practices.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 64b3b6e81..ee484e63f 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -134,6 +134,7 @@ Same example but using a non-default reactor, it's only necessary call install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") runner = CrawlerRunner() d = runner.crawl(MySpider) + from twisted.internet import reactor d.addBoth(lambda _: reactor.stop()) From 62c89aaf056687091235bb846ac565f7c801c359 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 23 May 2024 07:00:45 -0300 Subject: [PATCH 189/269] Update docs/topics/practices.rst Co-authored-by: Andrey Rakhmatullin --- docs/topics/practices.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index ee484e63f..1500011e7 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -202,6 +202,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: runner.crawl(MySpider1) runner.crawl(MySpider2) d = runner.join() + from twisted.internet import reactor d.addBoth(lambda _: reactor.stop()) From 2facdd4fb08ec3edaf1752047dd86d5b565621a1 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Sun, 26 May 2024 19:55:54 -0300 Subject: [PATCH 190/269] Add change reactor test to CrawlerRunner --- .flake8 | 1 + scrapy/crawler.py | 2 ++ tests/CrawlerRunner/change_reactor.py | 31 +++++++++++++++++++++++++++ tests/test_crawler.py | 8 +++++++ 4 files changed, 42 insertions(+) create mode 100644 tests/CrawlerRunner/change_reactor.py diff --git a/.flake8 b/.flake8 index 62ccad9cf..0e43b9b56 100644 --- a/.flake8 +++ b/.flake8 @@ -9,6 +9,7 @@ exclude = per-file-ignores = # Exclude files that are meant to provide top-level imports # E402: Module level import not at top of file + tests/CrawlerRunner/change_reactor.py:E402 # F401: Module imported but unused scrapy/__init__.py:E402 scrapy/core/downloader/handlers/http.py:F401 diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ccfe78891..4fe5987a7 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -129,6 +129,8 @@ class Crawler: if is_asyncio_reactor_installed() and event_loop: verify_installed_asyncio_event_loop(event_loop) + log_reactor_info() + self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py new file mode 100644 index 000000000..b20aa0c7c --- /dev/null +++ b/tests/CrawlerRunner/change_reactor.py @@ -0,0 +1,31 @@ +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + def start_requests(self): + return [] + + +configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"}) + + +from scrapy.utils.reactor import install_reactor + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + +runner = CrawlerRunner() + +d = runner.crawl(NoRequestsSpider) + +from twisted.internet import reactor + +d.addBoth(callback=lambda _: reactor.stop()) +reactor.run() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 989208694..791ea1faa 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -926,3 +926,11 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("INFO: Host: not.a.real.domain", log) self.assertIn("INFO: Type: ", log) self.assertIn("INFO: IP address: 127.0.0.1", log) + + def test_change_default_reactor(self): + log = self.run_script("change_reactor.py") + self.assertIn( + "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", + log, + ) + self.assertIn("DEBUG: Using asyncio event loop", log) From 6cd085785028d97393f26e6fee22e6c03e5c90a8 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Sun, 26 May 2024 19:57:16 -0300 Subject: [PATCH 191/269] Move path --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index 0e43b9b56..cf1a96476 100644 --- a/.flake8 +++ b/.flake8 @@ -9,7 +9,6 @@ exclude = per-file-ignores = # Exclude files that are meant to provide top-level imports # E402: Module level import not at top of file - tests/CrawlerRunner/change_reactor.py:E402 # F401: Module imported but unused scrapy/__init__.py:E402 scrapy/core/downloader/handlers/http.py:F401 @@ -17,6 +16,7 @@ per-file-ignores = scrapy/linkextractors/__init__.py:E402,F401 scrapy/selector/__init__.py:F401 scrapy/spiders/__init__.py:E402,F401 + tests/CrawlerRunner/change_reactor.py:E402 # Issues pending a review: scrapy/utils/url.py:F403,F405 From 9ba4dd311dd9d2a5341ee9c0c6d1b50eb44ac406 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 28 May 2024 12:27:49 +0400 Subject: [PATCH 192/269] Install typing stubs for boto3 and botocore. (#6370) --- scrapy/core/downloader/handlers/s3.py | 3 ++- scrapy/extensions/feedexport.py | 11 +++++++---- tox.ini | 13 ++++++++----- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 9a0811a50..1a3d36f45 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -59,7 +59,8 @@ class S3DownloadHandler: assert aws_access_key_id is not None assert aws_secret_access_key is not None SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"] - self._signer = SignerCls( + # botocore.auth.BaseSigner doesn't have an __init__() with args, only subclasses do + self._signer = SignerCls( # type: ignore[call-arg] botocore.credentials.Credentials( aws_access_key_id, aws_secret_access_key, aws_session_token ) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 97f39afe7..3c2bb5593 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -238,13 +238,16 @@ class S3FeedStorage(BlockingFeedStorage): self.acl: Optional[str] = acl self.endpoint_url: Optional[str] = endpoint_url self.region_name: Optional[str] = region_name + # It can be either botocore.client.BaseClient or mypy_boto3_s3.S3Client, + # there seems to be no good way to infer it statically. + self.s3_client: Any if IS_BOTO3_AVAILABLE: import boto3.session - session = boto3.session.Session() + boto3_session = boto3.session.Session() - self.s3_client = session.client( + self.s3_client = boto3_session.client( "s3", aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key, @@ -261,9 +264,9 @@ class S3FeedStorage(BlockingFeedStorage): import botocore.session - session = botocore.session.get_session() + botocore_session = botocore.session.get_session() - self.s3_client = session.create_client( + self.s3_client = botocore_session.create_client( "s3", aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key, diff --git a/tox.ini b/tox.ini index cde4243f3..5a5e80496 100644 --- a/tox.ini +++ b/tox.ini @@ -48,12 +48,15 @@ basepython = python3 deps = mypy==1.10.0 typing-extensions==4.11.0 - types-attrs==19.1.0 types-lxml==2024.4.14 - types-Pillow==10.2.0.20240423 - types-Pygments==2.17.0.20240310 - types-pyOpenSSL==24.0.0.20240417 - types-setuptools==69.5.0.20240423 + types-Pygments==2.18.0.20240506 + types-pyOpenSSL==24.1.0.20240425 + types-setuptools==69.5.0.20240518 + botocore-stubs==1.34.94 + boto3-stubs[s3]==1.34.108 + attrs >= 18.2.0 + Pillow >= 10.3.0 + pytest >= 8.2.0 # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 w3lib >= 2.1.2 commands = From 986d1ee1dd5b2efba0f787af8ee510450b4af3b4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 28 May 2024 12:37:19 +0400 Subject: [PATCH 193/269] Move CI from the decommissioned macos-11 to macos-latest. (#6372) --- .github/workflows/tests-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 252176464..a297f494c 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -7,7 +7,7 @@ concurrency: jobs: tests: - runs-on: macos-11 + runs-on: macos-latest strategy: fail-fast: false matrix: From cadb0dd707fc54670cb0eab06f0af65dcaa99354 Mon Sep 17 00:00:00 2001 From: Sanchay Kumar <51812506+kumar-sanchay@users.noreply.github.com> Date: Tue, 28 May 2024 14:12:58 +0530 Subject: [PATCH 194/269] Fix overridable methods in MediaPipeline (#6368) --- scrapy/pipelines/media.py | 26 +++--- tests/test_pipeline_media.py | 173 ++++++++++++----------------------- 2 files changed, 73 insertions(+), 126 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 5f6c5cb07..25e00b0ea 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -2,6 +2,7 @@ from __future__ import annotations import functools import logging +from abc import ABC, abstractmethod from collections import defaultdict from typing import TYPE_CHECKING @@ -27,7 +28,7 @@ def _DUMMY_CALLBACK(response): return response -class MediaPipeline: +class MediaPipeline(ABC): LOG_FAILED_RESULTS = True class SpiderInfo: @@ -55,14 +56,6 @@ class MediaPipeline: self.handle_httpstatus_list = SequenceExclude(range(300, 400)) def _key_for_pipe(self, key, base_class_name=None, settings=None): - """ - >>> MediaPipeline()._key_for_pipe("IMAGES") - 'IMAGES' - >>> class MyPipe(MediaPipeline): - ... pass - >>> MyPipe()._key_for_pipe("IMAGES", base_class_name="MediaPipeline") - 'MYPIPE_IMAGES' - """ class_name = self.__class__.__name__ formatted_key = f"{class_name.upper()}_{key}" if ( @@ -192,21 +185,25 @@ class MediaPipeline: defer_result(result).chainDeferred(wad) # Overridable Interface + @abstractmethod def media_to_download(self, request, info, *, item=None): """Check request before starting download""" - pass + raise NotImplementedError() + @abstractmethod def get_media_requests(self, item, info): """Returns the media requests to download""" - pass + raise NotImplementedError() + @abstractmethod def media_downloaded(self, response, request, info, *, item=None): """Handler for success downloads""" - return response + raise NotImplementedError() + @abstractmethod def media_failed(self, failure, request, info): """Handler for failed downloads""" - return failure + raise NotImplementedError() def item_completed(self, results, item, info): """Called per item when all media requests has been processed""" @@ -221,6 +218,7 @@ class MediaPipeline: ) return item + @abstractmethod def file_path(self, request, response=None, info=None, *, item=None): """Returns the path where downloaded media should be stored""" - pass + raise NotImplementedError() diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index d4dde4a40..763453551 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,4 +1,3 @@ -import io from typing import Optional from testfixtures import LogCapture @@ -11,7 +10,6 @@ from scrapy import signals from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import FileException -from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.spiders import Spider @@ -35,8 +33,26 @@ def _mocked_download_func(request, info): return response() if callable(response) else response +class UserDefinedPipeline(MediaPipeline): + + def media_to_download(self, request, info, *, item=None): + pass + + def get_media_requests(self, item, info): + pass + + def media_downloaded(self, response, request, info, *, item=None): + return {} + + def media_failed(self, failure, request, info): + return failure + + def file_path(self, request, response=None, info=None, *, item=None): + return "" + + class BaseMediaPipelineTestCase(unittest.TestCase): - pipeline_class = MediaPipeline + pipeline_class = UserDefinedPipeline settings = None def setUp(self): @@ -54,54 +70,6 @@ class BaseMediaPipelineTestCase(unittest.TestCase): if not name.startswith("_"): disconnect_all(signal) - def test_default_media_to_download(self): - request = Request("http://url") - assert self.pipe.media_to_download(request, self.info) is None - - def test_default_get_media_requests(self): - item = {"name": "name"} - assert self.pipe.get_media_requests(item, self.info) is None - - def test_default_media_downloaded(self): - request = Request("http://url") - response = Response("http://url", body=b"") - assert self.pipe.media_downloaded(response, request, self.info) is response - - def test_default_media_failed(self): - request = Request("http://url") - fail = Failure(Exception()) - assert self.pipe.media_failed(fail, request, self.info) is fail - - def test_default_item_completed(self): - item = {"name": "name"} - assert self.pipe.item_completed([], item, self.info) is item - - # Check that failures are logged by default - fail = Failure(Exception()) - results = [(True, 1), (False, fail)] - - with LogCapture() as log: - new_item = self.pipe.item_completed(results, item, self.info) - - assert new_item is item - assert len(log.records) == 1 - record = log.records[0] - assert record.levelname == "ERROR" - self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail)) - - # disable failure logging and check again - self.pipe.LOG_FAILED_RESULTS = False - with LogCapture() as log: - new_item = self.pipe.item_completed(results, item, self.info) - assert new_item is item - assert len(log.records) == 0 - - @inlineCallbacks - def test_default_process_item(self): - item = {"name": "name"} - new_item = yield self.pipe.process_item(item, self.spider) - assert new_item is item - def test_modify_media_request(self): request = Request("http://url") self.pipe._modify_media_request(request) @@ -175,8 +143,38 @@ class BaseMediaPipelineTestCase(unittest.TestCase): context = getattr(info.downloaded[fp].value, "__context__", None) self.assertIsNone(context) + def test_default_item_completed(self): + item = {"name": "name"} + assert self.pipe.item_completed([], item, self.info) is item -class MockedMediaPipeline(MediaPipeline): + # Check that failures are logged by default + fail = Failure(Exception()) + results = [(True, 1), (False, fail)] + + with LogCapture() as log: + new_item = self.pipe.item_completed(results, item, self.info) + + assert new_item is item + assert len(log.records) == 1 + record = log.records[0] + assert record.levelname == "ERROR" + self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail)) + + # disable failure logging and check again + self.pipe.LOG_FAILED_RESULTS = False + with LogCapture() as log: + new_item = self.pipe.item_completed(results, item, self.info) + assert new_item is item + assert len(log.records) == 0 + + @inlineCallbacks + def test_default_process_item(self): + item = {"name": "name"} + new_item = yield self.pipe.process_item(item, self.spider) + assert new_item is item + + +class MockedMediaPipeline(UserDefinedPipeline): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._mockcalled = [] @@ -232,7 +230,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ) item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, rsp)]) + self.assertEqual(new_item["results"], [(True, {})]) self.assertEqual( self.pipe._mockcalled, [ @@ -277,7 +275,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): req2 = Request("http://url2", meta={"response": fail}) item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, rsp1), (False, fail)]) + self.assertEqual(new_item["results"], [(True, {}), (False, fail)]) m = self.pipe._mockcalled # only once self.assertEqual(m[0], "get_media_requests") # first hook called @@ -315,7 +313,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = {"requests": req1} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) - self.assertEqual(new_item["results"], [(True, rsp1)]) + self.assertEqual(new_item["results"], [(True, {})]) # rsp2 is ignored, rsp1 must be in results because request fingerprints are the same req2 = Request( @@ -325,7 +323,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) self.assertEqual(self.fingerprint(req1), self.fingerprint(req2)) - self.assertEqual(new_item["results"], [(True, rsp1)]) + self.assertEqual(new_item["results"], [(True, {})]) @inlineCallbacks def test_results_are_cached_for_requests_of_single_item(self): @@ -337,7 +335,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) - self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) + self.assertEqual(new_item["results"], [(True, {}), (True, {})]) @inlineCallbacks def test_wait_if_request_is_downloading(self): @@ -363,7 +361,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): req2 = Request(req1.url, meta={"response": rsp2_func}) item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) + self.assertEqual(new_item["results"], [(True, {}), (True, {})]) @inlineCallbacks def test_use_media_to_download_result(self): @@ -376,57 +374,15 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ["get_media_requests", "media_to_download", "item_completed"], ) - -class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._mockcalled = [] - - def get_media_requests(self, item, info): - item_url = item["image_urls"][0] - output_img = io.BytesIO() - img = Image.new("RGB", (60, 30), color="red") - img.save(output_img, format="JPEG") - return Request( - item_url, - meta={ - "response": Response(item_url, status=200, body=output_img.getvalue()) - }, + def test_key_for_pipe(self): + self.assertEqual( + self.pipe._key_for_pipe("IMAGES", base_class_name="MediaPipeline"), + "MOCKEDMEDIAPIPELINE_IMAGES", ) - def inc_stats(self, *args, **kwargs): - return True - - def media_to_download(self, request, info): - self._mockcalled.append("media_to_download") - return super().media_to_download(request, info) - - def media_downloaded(self, response, request, info): - self._mockcalled.append("media_downloaded") - return super().media_downloaded(response, request, info) - - def file_downloaded(self, response, request, info): - self._mockcalled.append("file_downloaded") - return super().file_downloaded(response, request, info) - - def file_path(self, request, response=None, info=None): - self._mockcalled.append("file_path") - return super().file_path(request, response, info) - - def thumb_path(self, request, thumb_id, response=None, info=None): - self._mockcalled.append("thumb_path") - return super().thumb_path(request, thumb_id, response, info) - - def get_images(self, response, request, info): - self._mockcalled.append("get_images") - return super().get_images(response, request, info) - - def image_downloaded(self, response, request, info): - self._mockcalled.append("image_downloaded") - return super().image_downloaded(response, request, info) - class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): + def _assert_request_no3xx(self, pipeline_class, settings): pipe = pipeline_class(settings=Settings(settings)) request = Request("http://url") @@ -452,18 +408,11 @@ class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): else: self.assertNotIn(status, request.meta["handle_httpstatus_list"]) - def test_standard_setting(self): - self._assert_request_no3xx(MediaPipeline, {"MEDIA_ALLOW_REDIRECTS": True}) - def test_subclass_standard_setting(self): - class UserDefinedPipeline(MediaPipeline): - pass self._assert_request_no3xx(UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True}) def test_subclass_specific_setting(self): - class UserDefinedPipeline(MediaPipeline): - pass self._assert_request_no3xx( UserDefinedPipeline, {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True} From 0d58af86971ba54bfc1feffa88fc564ffba650f4 Mon Sep 17 00:00:00 2001 From: Fabian Schneebauer Date: Wed, 29 May 2024 10:59:32 +0200 Subject: [PATCH 195/269] Add support for multiple referer policy tokens. --- scrapy/spidermiddlewares/referer.py | 20 ++++++----- tests/test_spidermiddleware_referer.py | 47 ++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a0b6851e5..7706c8c15 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -323,15 +323,17 @@ def _load_policy_class( try: return cast(Type[ReferrerPolicy], load_object(policy)) except ValueError: - try: - return _policy_classes[policy.lower()] - except KeyError: - msg = f"Could not load referrer policy {policy!r}" - if not warning_only: - raise RuntimeError(msg) - else: - warnings.warn(msg, RuntimeWarning) - return None + tokens = [token.strip() for token in policy.lower().split(",")] + for token in tokens[::-1]: + if token in _policy_classes: + return _policy_classes[token] + + msg = f"Could not load referrer policy {policy!r}" + if not warning_only: + raise RuntimeError(msg) + else: + warnings.warn(msg, RuntimeWarning) + return None class RefererMiddleware: diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index afffa87fb..5797edfbd 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -884,6 +884,53 @@ class TestSettingsPolicyByName(TestCase): with self.assertRaises(RuntimeError): RefererMiddleware(settings) + def test_multiple_policy_tokens(self): + # test parsing without space(s) after the comma + settings1 = Settings( + { + "REFERRER_POLICY": ",".join( + [ + "some-custom-unknown-policy", + POLICY_SAME_ORIGIN, + POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, + "another-custom-unknown-policy", + ] + ) + } + ) + mw1 = RefererMiddleware(settings1) + self.assertEqual(mw1.default_policy, StrictOriginWhenCrossOriginPolicy) + + # test parsing with space(s) after the comma + settings2 = Settings( + { + "REFERRER_POLICY": ", ".join( + [ + POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, + "another-custom-unknown-policy", + POLICY_UNSAFE_URL, + ] + ) + } + ) + mw2 = RefererMiddleware(settings2) + self.assertEqual(mw2.default_policy, UnsafeUrlPolicy) + + def test_multiple_policy_tokens_all_invalid(self): + settings = Settings( + { + "REFERRER_POLICY": ",".join( + [ + "some-custom-unknown-policy", + "another-custom-unknown-policy", + "yet-another-custom-unknown-policy", + ] + ) + } + ) + with self.assertRaises(RuntimeError): + RefererMiddleware(settings) + class TestPolicyHeaderPrecedence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"} From 62a028b99dc73b8ddddd37f986780a5a070f2938 Mon Sep 17 00:00:00 2001 From: Fabian Schneebauer <67049088+0xdeb@users.noreply.github.com> Date: Wed, 29 May 2024 13:19:27 +0200 Subject: [PATCH 196/269] Add spec link to scrapy/spidermiddlewares/referer.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/spidermiddlewares/referer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 7706c8c15..8af0bdf5b 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -324,6 +324,7 @@ def _load_policy_class( return cast(Type[ReferrerPolicy], load_object(policy)) except ValueError: tokens = [token.strip() for token in policy.lower().split(",")] + # https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header for token in tokens[::-1]: if token in _policy_classes: return _policy_classes[token] From b4293e8f9efac5046f92e4ebfd744be443b858b0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 May 2024 10:50:36 +0400 Subject: [PATCH 197/269] Misc typing improvements. (#6384) --- scrapy/core/http2/agent.py | 4 +-- scrapy/core/http2/protocol.py | 8 +++-- scrapy/core/http2/stream.py | 4 +-- .../downloadermiddlewares/httpcompression.py | 14 +++++---- scrapy/downloadermiddlewares/offsite.py | 30 ++++++++++++------- scrapy/loader/__init__.py | 12 +++++++- scrapy/utils/benchserver.py | 12 ++++---- scrapy/utils/curl.py | 23 +++++++++----- scrapy/utils/datatypes.py | 2 +- scrapy/utils/request.py | 2 +- scrapy/utils/response.py | 2 +- scrapy/utils/testsite.py | 4 +-- 12 files changed, 78 insertions(+), 39 deletions(-) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 215ea9716..935af2214 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -119,7 +119,7 @@ class H2Agent: self._reactor, self._context_factory, connect_timeout, bind_address ) - def get_endpoint(self, uri: URI): + def get_endpoint(self, uri: URI) -> HostnameEndpoint: return self.endpoint_factory.endpointForURI(uri) def get_key(self, uri: URI) -> Tuple: @@ -161,7 +161,7 @@ class ScrapyProxyH2Agent(H2Agent): ) self._proxy_uri = proxy_uri - def get_endpoint(self, uri: URI): + def get_endpoint(self, uri: URI) -> HostnameEndpoint: return self.endpoint_factory.endpointForURI(self._proxy_uri) def get_key(self, uri: URI) -> Tuple: diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index bc8da50d7..8898b8118 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -22,7 +22,11 @@ from h2.events import ( from h2.exceptions import FrameTooLargeError, H2Error from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError -from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory +from twisted.internet.interfaces import ( + IAddress, + IHandshakeListener, + IProtocolNegotiationFactory, +) from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin @@ -431,7 +435,7 @@ class H2ClientFactory(Factory): self.settings = settings self.conn_lost_deferred = conn_lost_deferred - def buildProtocol(self, addr) -> H2ClientProtocol: + def buildProtocol(self, addr: IAddress) -> H2ClientProtocol: return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred) def acceptableProtocols(self) -> List[bytes]: diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 4132fc385..224691078 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,7 +1,7 @@ import logging from enum import Enum from io import BytesIO -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError @@ -142,7 +142,7 @@ class Stream: "headers": Headers({}), } - def _cancel(_) -> None: + def _cancel(_: Any) -> None: # Close this stream as gracefully as possible # If the associated request is initiated we reset this stream # else we directly call close() method diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 0e5e215ac..8e170a1c7 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -3,7 +3,7 @@ from __future__ import annotations import warnings from itertools import chain from logging import getLogger -from typing import TYPE_CHECKING, List, Optional, Union +from typing import TYPE_CHECKING, List, Optional, Tuple, Union from scrapy import Request, Spider, signals from scrapy.crawler import Crawler @@ -149,20 +149,24 @@ class HttpCompressionMiddleware: return response - def _handle_encoding(self, body, content_encoding, max_size): + def _handle_encoding( + self, body: bytes, content_encoding: List[bytes], max_size: int + ) -> Tuple[bytes, List[bytes]]: to_decode, to_keep = self._split_encodings(content_encoding) for encoding in to_decode: body = self._decode(body, encoding, max_size) return body, to_keep - def _split_encodings(self, content_encoding): - to_keep = [ + def _split_encodings( + self, content_encoding: List[bytes] + ) -> Tuple[List[bytes], List[bytes]]: + to_keep: List[bytes] = [ encoding.strip().lower() for encoding in chain.from_iterable( encodings.split(b",") for encodings in content_encoding ) ] - to_decode = [] + to_decode: List[bytes] = [] while to_keep: encoding = to_keep.pop() if encoding not in ACCEPTED_ENCODINGS: diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index 1e5026925..bd8dbe329 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -1,33 +1,43 @@ +from __future__ import annotations + import logging import re import warnings +from typing import TYPE_CHECKING, Set -from scrapy import signals +from scrapy import Request, Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest +from scrapy.statscollectors import StatsCollector from scrapy.utils.httpobj import urlparse_cached +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) class OffsiteMiddleware: @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: + assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled) return o - def __init__(self, stats): + def __init__(self, stats: StatsCollector): self.stats = stats - self.domains_seen = set() + self.domains_seen: Set[str] = set() - def spider_opened(self, spider): - self.host_regex = self.get_host_regex(spider) + def spider_opened(self, spider: Spider) -> None: + self.host_regex: re.Pattern[str] = self.get_host_regex(spider) - def request_scheduled(self, request, spider): + def request_scheduled(self, request: Request, spider: Spider) -> None: self.process_request(request, spider) - def process_request(self, request, spider): + def process_request(self, request: Request, spider: Spider) -> None: if request.dont_filter or self.should_follow(request, spider): return None domain = urlparse_cached(request).hostname @@ -42,13 +52,13 @@ class OffsiteMiddleware: self.stats.inc_value("offsite/filtered", spider=spider) raise IgnoreRequest - def should_follow(self, request, spider): + def should_follow(self, request: Request, spider: Spider) -> bool: regex = self.host_regex # hostname can be None for wrong urls (like javascript links) host = urlparse_cached(request).hostname or "" return bool(regex.search(host)) - def get_host_regex(self, spider): + def get_host_regex(self, spider: Spider) -> re.Pattern[str]: """Override this method to implement a different offsite policy""" allowed_domains = getattr(spider, "allowed_domains", None) if not allowed_domains: diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 529fa279e..db0b4820f 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -4,8 +4,11 @@ Item Loader See documentation in docs/topics/loaders.rst """ +from typing import Any, Optional + import itemloaders +from scrapy.http import TextResponse from scrapy.item import Item from scrapy.selector import Selector @@ -82,7 +85,14 @@ class ItemLoader(itemloaders.ItemLoader): default_item_class: type = Item default_selector_class = Selector - def __init__(self, item=None, selector=None, response=None, parent=None, **context): + def __init__( + self, + item: Any = None, + selector: Optional[Selector] = None, + response: Optional[TextResponse] = None, + parent: Optional[itemloaders.ItemLoader] = None, + **context: Any + ): if selector is None and response is not None: try: selector = self.default_selector_class(response) diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index f6f704d4b..e9ea51aa1 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -1,21 +1,23 @@ import random +from typing import Any from urllib.parse import urlencode from twisted.web.resource import Resource -from twisted.web.server import Site +from twisted.web.server import Request, Site class Root(Resource): isLeaf = True - def getChild(self, name, request): + def getChild(self, name: str, request: Request) -> Resource: return self - def render(self, request): + def render(self, request: Request) -> bytes: total = _getarg(request, b"total", 100, int) show = _getarg(request, b"show", 10, int) nlist = [random.randint(1, total) for _ in range(show)] # nosec request.write(b"") + assert request.args is not None args = request.args.copy() for nl in nlist: args["n"] = nl @@ -27,7 +29,7 @@ class Root(Resource): return b"" -def _getarg(request, name, default=None, type=str): +def _getarg(request, name: bytes, default: Any = None, type=str): return type(request.args[name][0]) if name in request.args else default @@ -38,7 +40,7 @@ if __name__ == "__main__": factory = Site(root) httpPort = reactor.listenTCP(8998, Site(root)) - def _print_listening(): + def _print_listening() -> None: httpHost = httpPort.getHost() print(f"Bench server at http://{httpHost.host}:{httpHost.port}") diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index f5dbbd64e..c10e48511 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -2,13 +2,20 @@ import argparse import warnings from http.cookies import SimpleCookie from shlex import split +from typing import Any, Dict, List, NoReturn, Optional, Sequence, Tuple, Union from urllib.parse import urlparse from w3lib.http import basic_auth_header class DataAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + values: Union[str, Sequence[Any], None], + option_string: Optional[str] = None, + ) -> None: value = str(values) if value.startswith("$"): value = value[1:] @@ -16,7 +23,7 @@ class DataAction(argparse.Action): class CurlParser(argparse.ArgumentParser): - def error(self, message): + def error(self, message: str) -> NoReturn: error_msg = f"There was an error parsing the curl command: {message}" raise ValueError(error_msg) @@ -42,9 +49,11 @@ for argument in safe_to_ignore_arguments: curl_parser.add_argument(*argument, action="store_true") -def _parse_headers_and_cookies(parsed_args): - headers = [] - cookies = {} +def _parse_headers_and_cookies( + parsed_args: argparse.Namespace, +) -> Tuple[List[Tuple[str, bytes]], Dict[str, str]]: + headers: List[Tuple[str, bytes]] = [] + cookies: Dict[str, str] = {} for header in parsed_args.headers or (): name, val = header.split(":", 1) name = name.strip() @@ -64,7 +73,7 @@ def _parse_headers_and_cookies(parsed_args): def curl_to_request_kwargs( curl_command: str, ignore_unknown_options: bool = True -) -> dict: +) -> Dict[str, Any]: """Convert a cURL command syntax to Request kwargs. :param str curl_command: string containing the curl command @@ -98,7 +107,7 @@ def curl_to_request_kwargs( method = parsed_args.method or "GET" - result = {"method": method.upper(), "url": url} + result: Dict[str, Any] = {"method": method.upper(), "url": url} headers, cookies = _parse_headers_and_cookies(parsed_args) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 0ba2fe4e2..b2118495f 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -110,7 +110,7 @@ class CaseInsensitiveDict(collections.UserDict): as keys and allows case-insensitive lookups. """ - def __init__(self, *args, **kwargs) -> None: + def __init__(self, *args: Any, **kwargs: Any) -> None: self._keys: dict = {} super().__init__(*args, **kwargs) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index c86f9fe39..5be80ec0f 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -138,7 +138,7 @@ class RequestFingerprinter: """ @classmethod - def from_crawler(cls, crawler) -> Self: + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) def __init__(self, crawler: Optional[Crawler] = None): diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index a0b06f75c..320059b3a 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -58,7 +58,7 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str: return f"{status_int} {to_unicode(message)}" -def _remove_html_comments(body): +def _remove_html_comments(body: bytes) -> bytes: start = body.find(b"", start + 1) diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index de9ce992a..ca1f68116 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -15,12 +15,12 @@ class SiteTest: super().tearDown() self.site.stopListening() - def url(self, path): + def url(self, path: str) -> str: return urljoin(self.baseurl, path) class NoMetaRefreshRedirect(util.Redirect): - def render(self, request): + def render(self, request: server.Request) -> bytes: content = util.Redirect.render(self, request) return content.replace( b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"' From da42e8f124362a5087c50bca7f76dcc573e8194a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 May 2024 21:11:50 +0500 Subject: [PATCH 198/269] Add parameters to typing.Dict. --- scrapy/core/downloader/handlers/__init__.py | 5 +++- scrapy/core/http2/protocol.py | 4 +-- scrapy/core/http2/stream.py | 4 +-- scrapy/extensions/feedexport.py | 4 ++- scrapy/http/request/__init__.py | 4 +-- scrapy/http/request/form.py | 30 ++++++++++++++------- scrapy/http/request/json_request.py | 18 ++++++------- scrapy/http/response/__init__.py | 4 +-- scrapy/http/response/text.py | 4 +-- scrapy/item.py | 2 +- scrapy/logformatter.py | 22 ++++++++++----- scrapy/settings/__init__.py | 2 +- scrapy/spiders/__init__.py | 6 ++--- scrapy/utils/conf.py | 3 ++- scrapy/utils/log.py | 13 ++++++--- scrapy/utils/python.py | 10 ++++--- scrapy/utils/request.py | 2 +- 17 files changed, 85 insertions(+), 52 deletions(-) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index ade51ca63..af5282553 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -27,7 +27,10 @@ class DownloadHandlers: self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes self._notconfigured: Dict[str, str] = {} # remembers failed handlers handlers: Dict[str, Union[str, Callable]] = without_none_values( - crawler.settings.getwithbase("DOWNLOAD_HANDLERS") + cast( + Dict[str, Union[str, Callable]], + crawler.settings.getwithbase("DOWNLOAD_HANDLERS"), + ) ) for scheme, clspath in handlers.items(): self._schemes[scheme] = clspath diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 8898b8118..063835b17 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -3,7 +3,7 @@ import itertools import logging from collections import deque from ipaddress import IPv4Address, IPv6Address -from typing import Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -115,7 +115,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # Some meta data of this connection # initialized when connection is successfully made - self.metadata: Dict = { + self.metadata: Dict[str, Any] = { # Peer certificate instance "certificate": None, # Address of the server we are connected to which diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 224691078..7c70e86db 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -110,7 +110,7 @@ class Stream: # Metadata of an HTTP/2 connection stream # initialized when stream is instantiated - self.metadata: Dict = { + self.metadata: Dict[str, Any] = { "request_content_length": ( 0 if self._request.body is None else len(self._request.body) ), @@ -131,7 +131,7 @@ class Stream: # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response: Dict = { + self._response: Dict[str, Any] = { # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. "body": BytesIO(), diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 3c2bb5593..de8a288f6 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -694,7 +694,9 @@ class FeedExporter: self.slots = slots def _load_components(self, setting_prefix: str) -> Dict[str, Any]: - conf = without_none_values(self.settings.getwithbase(setting_prefix)) + conf = without_none_values( + cast(Dict[str, str], self.settings.getwithbase(setting_prefix)) + ) d = {} for k, v in conf.items(): try: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 191b3cef4..dfb1dca89 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -97,7 +97,7 @@ class Request(object_ref): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, + cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, meta: Optional[Dict[str, Any]] = None, encoding: str = "utf-8", priority: int = 0, @@ -123,7 +123,7 @@ class Request(object_ref): self.callback: Optional[Callable] = callback self.errback: Optional[Callable] = errback - self.cookies: Union[dict, List[dict]] = cookies or {} + self.cookies: Union[Dict[str, str], List[Dict[str, str]]] = cookies or {} self.headers: Headers = Headers(headers or {}, encoding=encoding) self.dont_filter: bool = dont_filter diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 3206d79cd..ea98ed795 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -7,7 +7,17 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, + cast, +) from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit from lxml.html import FormElement # nosec @@ -26,8 +36,9 @@ if TYPE_CHECKING: from typing_extensions import Self -FormdataKVType = Tuple[str, Union[str, Iterable[str]]] -FormdataType = Optional[Union[dict, List[FormdataKVType]]] +FormdataVType = Union[str, Iterable[str]] +FormdataKVType = Tuple[str, FormdataVType] +FormdataType = Optional[Union[Dict[str, FormdataVType], List[FormdataKVType]]] class FormRequest(Request): @@ -62,7 +73,7 @@ class FormRequest(Request): formid: Optional[str] = None, formnumber: int = 0, formdata: FormdataType = None, - clickdata: Optional[dict] = None, + clickdata: Optional[Dict[str, Union[str, int]]] = None, dont_click: bool = False, formxpath: Optional[str] = None, formcss: Optional[str] = None, @@ -156,7 +167,7 @@ def _get_inputs( form: FormElement, formdata: FormdataType, dont_click: bool, - clickdata: Optional[dict], + clickdata: Optional[Dict[str, Union[str, int]]], ) -> List[FormdataKVType]: """Return a list of key-value pairs for the inputs found in the given form.""" try: @@ -186,10 +197,8 @@ def _get_inputs( if clickable and clickable[0] not in formdata and not clickable[0] is None: values.append(clickable) - if isinstance(formdata, dict): - formdata = formdata.items() # type: ignore[assignment] - - values.extend((k, v) for k, v in formdata if v is not None) + formdata_items = formdata.items() if isinstance(formdata, dict) else formdata + values.extend((k, v) for k, v in formdata_items if v is not None) return values @@ -216,7 +225,7 @@ def _select_value( def _get_clickable( - clickdata: Optional[dict], form: FormElement + clickdata: Optional[Dict[str, Union[str, int]]], form: FormElement ) -> Optional[Tuple[str, str]]: """ Returns the clickable element specified in clickdata, @@ -243,6 +252,7 @@ def _get_clickable( # because that uniquely identifies the element nr = clickdata.get("nr", None) if nr is not None: + assert isinstance(nr, int) try: el = list(form.inputs)[nr] except IndexError: diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 1dd9e6c87..405c0b9d0 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst import copy import json import warnings -from typing import Any, Optional, Tuple +from typing import Any, Dict, Optional, Tuple from scrapy.http.request import Request @@ -17,15 +17,15 @@ class JsonRequest(Request): attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) def __init__( - self, *args: Any, dumps_kwargs: Optional[dict] = None, **kwargs: Any + self, *args: Any, dumps_kwargs: Optional[Dict[str, Any]] = None, **kwargs: Any ) -> None: dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} dumps_kwargs.setdefault("sort_keys", True) - self._dumps_kwargs = dumps_kwargs + self._dumps_kwargs: Dict[str, Any] = dumps_kwargs body_passed = kwargs.get("body", None) is not None - data = kwargs.pop("data", None) - data_passed = data is not None + data: Any = kwargs.pop("data", None) + data_passed: bool = data is not None if body_passed and data_passed: warnings.warn("Both body and data passed. data will be ignored") @@ -41,13 +41,13 @@ class JsonRequest(Request): ) @property - def dumps_kwargs(self) -> dict: + def dumps_kwargs(self) -> Dict[str, Any]: return self._dumps_kwargs def replace(self, *args: Any, **kwargs: Any) -> Request: body_passed = kwargs.get("body", None) is not None - data = kwargs.pop("data", None) - data_passed = data is not None + data: Any = kwargs.pop("data", None) + data_passed: bool = data is not None if body_passed and data_passed: warnings.warn("Both body and data passed. data will be ignored") @@ -56,6 +56,6 @@ class JsonRequest(Request): return super().replace(*args, **kwargs) - def _dumps(self, data: dict) -> str: + def _dumps(self, data: Any) -> str: """Convert to JSON""" return json.dumps(data, **self._dumps_kwargs) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index d73dfce4b..14618e5e7 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -181,7 +181,7 @@ class Response(object_ref): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, + cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, @@ -234,7 +234,7 @@ class Response(object_ref): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, + cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 522ffc0d5..a83279ac8 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -183,7 +183,7 @@ class TextResponse(Response): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, + cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0, @@ -236,7 +236,7 @@ class TextResponse(Response): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, + cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0, diff --git a/scrapy/item.py b/scrapy/item.py index 2daea64cc..3f93809e7 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -27,7 +27,7 @@ if TYPE_CHECKING: from typing_extensions import Self -class Field(dict): +class Field(Dict[str, Any]): """Container of field metadata""" diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index d720b2f38..42a03b560 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging import os -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, TypedDict, Union from twisted.python.failure import Failure @@ -26,6 +26,12 @@ DOWNLOADERRORMSG_SHORT = "Error downloading %(request)s" DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s" +class LogFormatterResult(TypedDict): + level: int + msg: str + args: Union[Dict[str, Any], Tuple[Any, ...]] + + class LogFormatter: """Class for generating log messages for different actions. @@ -64,7 +70,9 @@ class LogFormatter: } """ - def crawled(self, request: Request, response: Response, spider: Spider) -> dict: + def crawled( + self, request: Request, response: Response, spider: Spider + ) -> LogFormatterResult: """Logs a message when the crawler finds a webpage.""" request_flags = f" {str(request.flags)}" if request.flags else "" response_flags = f" {str(response.flags)}" if response.flags else "" @@ -84,7 +92,7 @@ class LogFormatter: def scraped( self, item: Any, response: Union[Response, Failure], spider: Spider - ) -> dict: + ) -> LogFormatterResult: """Logs a message when an item is scraped by a spider.""" src: Any if isinstance(response, Failure): @@ -102,7 +110,7 @@ class LogFormatter: def dropped( self, item: Any, exception: BaseException, response: Response, spider: Spider - ) -> dict: + ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" return { "level": logging.WARNING, @@ -115,7 +123,7 @@ class LogFormatter: def item_error( self, item: Any, exception: BaseException, response: Response, spider: Spider - ) -> dict: + ) -> LogFormatterResult: """Logs a message when an item causes an error while it is passing through the item pipeline. @@ -135,7 +143,7 @@ class LogFormatter: request: Request, response: Union[Response, Failure], spider: Spider, - ) -> dict: + ) -> LogFormatterResult: """Logs an error message from a spider. .. versionadded:: 2.0 @@ -155,7 +163,7 @@ class LogFormatter: request: Request, spider: Spider, errmsg: Optional[str] = None, - ) -> dict: + ) -> LogFormatterResult: """Logs a download error message from a spider (typically coming from the engine). diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index d270a72f4..4448b6f4b 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -411,7 +411,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ self._assert_mutability() if isinstance(values, str): - values = cast(dict, json.loads(values)) + values = cast(Dict[_SettingsKeyT, Any], json.loads(values)) if values is not None: if isinstance(values, BaseSettings): for name, value in values.items(): diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index bef041325..7b43f04f2 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union, cast from twisted.internet.defer import Deferred @@ -24,7 +24,7 @@ if TYPE_CHECKING: from typing_extensions import Concatenate, Self from scrapy.crawler import Crawler - from scrapy.settings import BaseSettings + from scrapy.settings import BaseSettings, _SettingsKeyT from scrapy.utils.log import SpiderLoggerAdapter CallbackT = Callable[Concatenate[Response, ...], Any] @@ -36,7 +36,7 @@ class Spider(object_ref): """ name: str - custom_settings: Optional[dict] = None + custom_settings: Optional[Dict[_SettingsKeyT, Any]] = None def __init__(self, name: Optional[str] = None, **kwargs: Any): if name is not None: diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 641dfa4a2..c63b69995 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -16,6 +16,7 @@ from typing import ( MutableMapping, Optional, Union, + cast, ) from scrapy.exceptions import ScrapyDeprecationWarning, UsageError @@ -173,7 +174,7 @@ def feed_process_params_from_cli( suitable to be used as the FEEDS setting. """ valid_output_formats: Iterable[str] = without_none_values( - settings.getwithbase("FEED_EXPORTERS") + cast(Dict[str, str], settings.getwithbase("FEED_EXPORTERS")) ).keys() def check_valid_format(output_format: str) -> None: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 430a91e95..cbfd170ed 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -7,6 +7,7 @@ from types import TracebackType from typing import ( TYPE_CHECKING, Any, + Dict, List, MutableMapping, Optional, @@ -20,7 +21,8 @@ from twisted.python import log as twisted_log from twisted.python.failure import Failure import scrapy -from scrapy.settings import Settings +from scrapy.logformatter import LogFormatterResult +from scrapy.settings import Settings, _SettingsKeyT from scrapy.utils.versions import scrapy_components_versions if TYPE_CHECKING: @@ -86,7 +88,8 @@ DEFAULT_LOGGING = { def configure_logging( - settings: Union[Settings, dict, None] = None, install_root_handler: bool = True + settings: Union[Settings, Dict[_SettingsKeyT, Any], None] = None, + install_root_handler: bool = True, ) -> None: """ Initialize logging defaults for Scrapy. @@ -234,7 +237,9 @@ class LogCounterHandler(logging.Handler): self.crawler.stats.inc_value(sname) -def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]: +def logformatter_adapter( + logkws: LogFormatterResult, +) -> Tuple[int, str, Union[Dict[str, Any], Tuple[Any, ...]]]: """ Helper that takes the dictionary output from the methods in LogFormatter and adapts it into a tuple of positional arguments for logger.log calls, @@ -245,7 +250,7 @@ def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]: message = logkws.get("msg") or "" # NOTE: This also handles 'args' being an empty dict, that case doesn't # play well in logger.log calls - args = logkws if not logkws.get("args") else logkws["args"] + args = cast(Dict[str, Any], logkws) if not logkws.get("args") else logkws["args"] return (level, message, args) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 578cde2ac..0a50f4e1e 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -42,6 +42,8 @@ if TYPE_CHECKING: _P = ParamSpec("_P") _T = TypeVar("_T") +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") def flatten(x: Iterable) -> list: @@ -303,14 +305,16 @@ def equal_attributes( @overload -def without_none_values(iterable: Mapping) -> dict: ... +def without_none_values(iterable: Mapping[_KT, _VT]) -> Dict[_KT, _VT]: ... @overload -def without_none_values(iterable: Iterable) -> Iterable: ... +def without_none_values(iterable: Iterable[_KT]) -> Iterable[_KT]: ... -def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]: +def without_none_values( + iterable: Union[Mapping[_KT, _VT], Iterable[_KT]] +) -> Union[Dict[_KT, _VT], Iterable[_KT]]: """Return a copy of ``iterable`` with all ``None`` entries removed. If ``iterable`` is a mapping, return a dictionary where all pairs that have diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 5be80ec0f..42a6537a8 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -197,7 +197,7 @@ def referer_str(request: Request) -> Optional[str]: return to_unicode(referrer, errors="replace") -def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request: +def request_from_dict(d: Dict[str, Any], *, spider: Optional[Spider] = None) -> Request: """Create a :class:`~scrapy.Request` object from a dict. If a spider is given, it will try to resolve the callbacks looking at the From 98c755e5fbc005083a5fde810476f2de610bf912 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 May 2024 21:20:22 +0500 Subject: [PATCH 199/269] Add parameters to typing.List. --- scrapy/core/scheduler.py | 8 ++++---- scrapy/downloadermiddlewares/stats.py | 6 ++++-- scrapy/utils/asyncgen.py | 10 +++++++--- scrapy/utils/python.py | 8 ++++---- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index f30a5d9c9..e3b95e977 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,7 +4,7 @@ import json import logging from abc import abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, Type, cast +from typing import TYPE_CHECKING, Any, List, Optional, Type, cast from twisted.internet.defer import Deferred @@ -362,13 +362,13 @@ class Scheduler(BaseScheduler): return str(dqdir) return None - def _read_dqs_state(self, dqdir: str) -> list: + def _read_dqs_state(self, dqdir: str) -> List[int]: path = Path(dqdir, "active.json") if not path.exists(): return [] with path.open(encoding="utf-8") as f: - return cast(list, json.load(f)) + return cast(List[int], json.load(f)) - def _write_dqs_state(self, dqdir: str, state: list) -> None: + def _write_dqs_state(self, dqdir: str, state: List[int]) -> None: with Path(dqdir, "active.json").open("w", encoding="utf-8") as f: json.dump(state, f) diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index df30e8ca4..444702757 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, Union +from typing import TYPE_CHECKING, Dict, List, Tuple, Union from twisted.web import http @@ -17,7 +17,9 @@ if TYPE_CHECKING: from typing_extensions import Self -def get_header_size(headers: Dict[str, Union[list, tuple]]) -> int: +def get_header_size( + headers: Dict[str, Union[List[Union[str, bytes]], Tuple[Union[str, bytes], ...]]] +) -> int: size = 0 for key, value in headers.items(): if isinstance(value, (list, tuple)): diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 0505db343..67c8e1a01 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,14 +1,18 @@ -from typing import AsyncGenerator, AsyncIterable, Iterable, Union +from typing import AsyncGenerator, AsyncIterable, Iterable, List, TypeVar, Union + +_T = TypeVar("_T") -async def collect_asyncgen(result: AsyncIterable) -> list: +async def collect_asyncgen(result: AsyncIterable[_T]) -> List[_T]: results = [] async for x in result: results.append(x) return results -async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerator: +async def as_async_generator( + it: Union[Iterable[_T], AsyncIterable[_T]] +) -> AsyncGenerator[_T, None]: """Wraps an iterable (sync or async) into an async generator.""" if isinstance(it, AsyncIterable): async for r in it: diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 0a50f4e1e..3db7acf81 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -46,7 +46,7 @@ _KT = TypeVar("_KT") _VT = TypeVar("_VT") -def flatten(x: Iterable) -> list: +def flatten(x: Iterable[Any]) -> List[Any]: """flatten(sequence) -> list Returns a single, flat list which contains all elements retrieved @@ -66,7 +66,7 @@ def flatten(x: Iterable) -> list: return list(iflatten(x)) -def iflatten(x: Iterable) -> Iterable: +def iflatten(x: Iterable[Any]) -> Iterable[Any]: """iflatten(sequence) -> iterator Similar to ``.flatten()``, but returns iterator instead""" @@ -101,10 +101,10 @@ def is_listlike(x: Any) -> bool: return hasattr(x, "__iter__") and not isinstance(x, (str, bytes)) -def unique(list_: Iterable, key: Callable[[Any], Any] = lambda x: x) -> list: +def unique(list_: Iterable[_T], key: Callable[[_T], Any] = lambda x: x) -> List[_T]: """efficient function to uniquify a list preserving item order""" seen = set() - result = [] + result: List[_T] = [] for item in list_: seenkey = key(item) if seenkey in seen: From 4164e63725dc19bc8585abbfb0e5009f8eceefcc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 May 2024 21:23:55 +0500 Subject: [PATCH 200/269] Add parameters to typing.Tuple. --- scrapy/core/http2/agent.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 935af2214..999764a6e 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -20,6 +20,8 @@ from scrapy.http.request import Request from scrapy.settings import Settings from scrapy.spiders import Spider +ConnectionKeyT = Tuple[bytes, bytes, int] + class H2ConnectionPool: def __init__(self, reactor: ReactorBase, settings: Settings) -> None: @@ -28,13 +30,13 @@ class H2ConnectionPool: # Store a dictionary which is used to get the respective # H2ClientProtocolInstance using the key as Tuple(scheme, hostname, port) - self._connections: Dict[Tuple, H2ClientProtocol] = {} + self._connections: Dict[ConnectionKeyT, H2ClientProtocol] = {} # Save all requests that arrive before the connection is established - self._pending_requests: Dict[Tuple, Deque[Deferred]] = {} + self._pending_requests: Dict[ConnectionKeyT, Deque[Deferred]] = {} def get_connection( - self, key: Tuple, uri: URI, endpoint: HostnameEndpoint + self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint ) -> Deferred: if key in self._pending_requests: # Received a request while connecting to remote @@ -54,7 +56,7 @@ class H2ConnectionPool: return self._new_connection(key, uri, endpoint) def _new_connection( - self, key: Tuple, uri: URI, endpoint: HostnameEndpoint + self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint ) -> Deferred: self._pending_requests[key] = deque() @@ -69,7 +71,9 @@ class H2ConnectionPool: self._pending_requests[key].append(d) return d - def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol: + def put_connection( + self, conn: H2ClientProtocol, key: ConnectionKeyT + ) -> H2ClientProtocol: self._connections[key] = conn # Now as we have established a proper HTTP/2 connection @@ -81,7 +85,9 @@ class H2ConnectionPool: return conn - def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None: + def _remove_connection( + self, errors: List[BaseException], key: ConnectionKeyT + ) -> None: self._connections.pop(key) # Call the errback of all the pending requests for this connection @@ -122,7 +128,7 @@ class H2Agent: def get_endpoint(self, uri: URI) -> HostnameEndpoint: return self.endpoint_factory.endpointForURI(uri) - def get_key(self, uri: URI) -> Tuple: + def get_key(self, uri: URI) -> ConnectionKeyT: """ Arguments: uri - URI obtained directly from request URL @@ -164,6 +170,6 @@ class ScrapyProxyH2Agent(H2Agent): def get_endpoint(self, uri: URI) -> HostnameEndpoint: return self.endpoint_factory.endpointForURI(self._proxy_uri) - def get_key(self, uri: URI) -> Tuple: + def get_key(self, uri: URI) -> ConnectionKeyT: """We use the proxy uri instead of uri obtained from request url""" - return "http-proxy", self._proxy_uri.host, self._proxy_uri.port + return b"http-proxy", self._proxy_uri.host, self._proxy_uri.port From 70c56faf4847406de6eb3594758c5531610757e8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 May 2024 21:41:27 +0500 Subject: [PATCH 201/269] Add parameters to typing.IO. --- scrapy/extensions/httpcache.py | 9 ++++++--- scrapy/mail.py | 4 ++-- scrapy/pipelines/files.py | 2 +- scrapy/utils/ftp.py | 2 +- scrapy/utils/misc.py | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index dd5bce24f..3f4af42b7 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -315,7 +315,9 @@ class FilesystemCacheStorage: self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP") # https://github.com/python/mypy/issues/10740 - self._open: Callable[Concatenate[Union[str, os.PathLike], str, ...], IO] = ( + self._open: Callable[ + Concatenate[Union[str, os.PathLike], str, ...], IO[bytes] + ] = ( gzip.open if self.use_gzip else open # type: ignore[assignment] ) @@ -368,11 +370,12 @@ class FilesystemCacheStorage: with self._open(rpath / "pickled_meta", "wb") as f: pickle.dump(metadata, f, protocol=4) with self._open(rpath / "response_headers", "wb") as f: - f.write(headers_dict_to_raw(response.headers)) + # headers_dict_to_raw() needs a better type hint + f.write(cast(bytes, headers_dict_to_raw(response.headers))) with self._open(rpath / "response_body", "wb") as f: f.write(response.body) with self._open(rpath / "request_headers", "wb") as f: - f.write(headers_dict_to_raw(request.headers)) + f.write(cast(bytes, headers_dict_to_raw(request.headers))) with self._open(rpath / "request_body", "wb") as f: f.write(request.body) diff --git a/scrapy/mail.py b/scrapy/mail.py index fd6302550..f4ce2800c 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -97,7 +97,7 @@ class MailSender: subject: str, body: str, cc: Union[str, List[str], None] = None, - attachs: Sequence[Tuple[str, str, IO]] = (), + attachs: Sequence[Tuple[str, str, IO[Any]]] = (), mimetype: str = "text/plain", charset: Optional[str] = None, _callback: Optional[Callable[..., None]] = None, @@ -214,7 +214,7 @@ class MailSender: return d def _create_sender_factory( - self, to_addrs: List[str], msg: IO, d: Deferred + self, to_addrs: List[str], msg: IO[bytes], d: Deferred ) -> ESMTPSenderFactory: from twisted.mail.smtp import ESMTPSenderFactory diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 47457f2a8..c1ce0939c 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -47,7 +47,7 @@ def _to_string(path: Union[str, PathLike]) -> str: return str(path) # convert a Path object to string -def _md5sum(file: IO) -> str: +def _md5sum(file: IO[bytes]) -> str: """Calculate the md5 checksum of a file-like object without reading its whole content in memory. diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index c77681a53..152f3374e 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -21,7 +21,7 @@ def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None: def ftp_store_file( *, path: str, - file: IO, + file: IO[bytes], host: str, port: int, username: str, diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index faf52e44a..b678d1def 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -111,7 +111,7 @@ def walk_modules(path: str) -> List[ModuleType]: return mods -def md5sum(file: IO) -> str: +def md5sum(file: IO[bytes]) -> str: """Calculate the md5 checksum of a file-like object without reading its whole content in memory. From 751c91e614b91827dc68cd462b907c4b9d03f071 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 May 2024 21:57:14 +0500 Subject: [PATCH 202/269] Add parameters to misc generics. --- scrapy/core/engine.py | 20 +++++++++++--------- scrapy/core/http2/protocol.py | 4 ++-- scrapy/utils/datatypes.py | 4 ++-- scrapy/utils/python.py | 2 +- scrapy/utils/test.py | 8 +++++--- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 4eca03800..4cb4454e3 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -5,6 +5,8 @@ For more information see docs/topics/architecture.rst """ +from __future__ import annotations + import logging from time import time from typing import ( @@ -51,15 +53,15 @@ class Slot: self, start_requests: Iterable[Request], close_if_idle: bool, - nextcall: CallLaterOnce, - scheduler: "BaseScheduler", + nextcall: CallLaterOnce[None], + scheduler: BaseScheduler, ) -> None: self.closing: Optional[Deferred] = None self.inprogress: Set[Request] = set() self.start_requests: Optional[Iterator[Request]] = iter(start_requests) self.close_if_idle: bool = close_if_idle - self.nextcall: CallLaterOnce = nextcall - self.scheduler: "BaseScheduler" = scheduler + self.nextcall: CallLaterOnce[None] = nextcall + self.scheduler: BaseScheduler = scheduler self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule) def add_request(self, request: Request) -> None: @@ -84,8 +86,8 @@ class Slot: class ExecutionEngine: - def __init__(self, crawler: "Crawler", spider_closed_callback: Callable) -> None: - self.crawler: "Crawler" = crawler + def __init__(self, crawler: Crawler, spider_closed_callback: Callable) -> None: + self.crawler: Crawler = crawler self.settings: Settings = crawler.settings self.signals: SignalManager = crawler.signals assert crawler.logformatter @@ -94,7 +96,7 @@ class ExecutionEngine: self.spider: Optional[Spider] = None self.running: bool = False self.paused: bool = False - self.scheduler_cls: Type["BaseScheduler"] = self._get_scheduler_class( + self.scheduler_cls: Type[BaseScheduler] = self._get_scheduler_class( crawler.settings ) downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"]) @@ -103,10 +105,10 @@ class ExecutionEngine: self._spider_closed_callback: Callable = spider_closed_callback self.start_time: Optional[float] = None - def _get_scheduler_class(self, settings: BaseSettings) -> Type["BaseScheduler"]: + def _get_scheduler_class(self, settings: BaseSettings) -> Type[BaseScheduler]: from scrapy.core.scheduler import BaseScheduler - scheduler_cls: Type = load_object(settings["SCHEDULER"]) + scheduler_cls: Type[BaseScheduler] = load_object(settings["SCHEDULER"]) if not issubclass(scheduler_cls, BaseScheduler): raise TypeError( f"The provided scheduler class ({settings['SCHEDULER']})" diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 063835b17..f2f1cb0b8 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -3,7 +3,7 @@ import itertools import logging from collections import deque from ipaddress import IPv4Address, IPv6Address -from typing import Any, Dict, List, Optional, Union +from typing import Any, Deque, Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -107,7 +107,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # If requests are received before connection is made we keep # all requests in a pool and send them as the connection is made - self._pending_request_stream_pool: deque = deque() + self._pending_request_stream_pool: Deque[Stream] = deque() # Save an instance of errors raised which lead to losing the connection # We pass these instances to the streams ResponseFailed() failure diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index b2118495f..d06887610 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -196,8 +196,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): class SequenceExclude: """Object to test if an item is NOT within some sequence.""" - def __init__(self, seq: Sequence): - self.seq: Sequence = seq + def __init__(self, seq: Sequence[Any]): + self.seq: Sequence[Any] = seq def __contains__(self, item: Any) -> bool: return item not in self.seq diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 3db7acf81..fc1eb4f69 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -148,7 +148,7 @@ def to_bytes( def re_rsearch( - pattern: Union[str, Pattern], text: str, chunk_size: int = 1024 + pattern: Union[str, Pattern[str]], text: str, chunk_size: int = 1024 ) -> Optional[Tuple[int, int]]: """ This function does a reverse search in a text using a regular expression diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 7a8c5c859..268d8d4be 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -7,7 +7,7 @@ import os from importlib import import_module from pathlib import Path from posixpath import split -from typing import Any, Coroutine, Dict, List, Optional, Tuple, Type +from typing import Any, Awaitable, Dict, List, Optional, Tuple, Type, TypeVar from unittest import TestCase, mock from twisted.internet.defer import Deferred @@ -17,6 +17,8 @@ from scrapy import Spider from scrapy.crawler import Crawler from scrapy.utils.boto import is_botocore_available +_T = TypeVar("_T") + def assert_gcs_environ() -> None: if "GCS_PROJECT_ID" not in os.environ: @@ -118,8 +120,8 @@ def assert_samelines( testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) -def get_from_asyncio_queue(value: Any) -> Coroutine: - q: asyncio.Queue = asyncio.Queue() +def get_from_asyncio_queue(value: _T) -> Awaitable[_T]: + q: asyncio.Queue[_T] = asyncio.Queue() getter = q.get() q.put_nowait(value) return getter From 859a77ee4243f17f338072e45785383f12516308 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 May 2024 22:23:26 +0500 Subject: [PATCH 203/269] Use a TypedDict for the verbose cookie form. --- scrapy/downloadermiddlewares/cookies.py | 25 +++++++++---------------- scrapy/http/request/__init__.py | 20 ++++++++++++++++---- scrapy/http/response/__init__.py | 6 +++--- scrapy/http/response/text.py | 6 +++--- 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 6ada3b474..73c2c57fe 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -3,16 +3,7 @@ from __future__ import annotations import logging from collections import defaultdict from http.cookiejar import Cookie -from typing import ( - TYPE_CHECKING, - Any, - DefaultDict, - Dict, - Iterable, - Optional, - Sequence, - Union, -) +from typing import TYPE_CHECKING, Any, DefaultDict, Iterable, Optional, Sequence, Union from tldextract import TLDExtract @@ -21,6 +12,7 @@ from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar +from scrapy.http.request import VerboseCookie from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode @@ -128,7 +120,7 @@ class CookiesMiddleware: msg = f"Received cookies from: {response}\n{cookies}" logger.debug(msg, extra={"spider": spider}) - def _format_cookie(self, cookie: Dict[str, Any], request: Request) -> Optional[str]: + def _format_cookie(self, cookie: VerboseCookie, request: Request) -> Optional[str]: """ Given a dict consisting of cookie components, return its string representation. Decode from bytes if necessary. @@ -142,18 +134,19 @@ class CookiesMiddleware: logger.warning(msg) return None continue - if isinstance(cookie[key], (bool, float, int, str)): - decoded[key] = str(cookie[key]) + # https://github.com/python/mypy/issues/7178, https://github.com/python/mypy/issues/9168 + if isinstance(cookie[key], (bool, float, int, str)): # type: ignore[literal-required] + decoded[key] = str(cookie[key]) # type: ignore[literal-required] else: try: - decoded[key] = cookie[key].decode("utf8") + decoded[key] = cookie[key].decode("utf8") # type: ignore[literal-required] except UnicodeDecodeError: logger.warning( "Non UTF-8 encoded cookie found in request %s: %s", request, cookie, ) - decoded[key] = cookie[key].decode("latin1", errors="replace") + decoded[key] = cookie[key].decode("latin1", errors="replace") # type: ignore[literal-required] for flag in ("secure",): value = cookie.get(flag, _UNSET) if value is _UNSET or not value: @@ -174,7 +167,7 @@ class CookiesMiddleware: """ if not request.cookies: return [] - cookies: Iterable[Dict[str, Any]] + cookies: Iterable[VerboseCookie] if isinstance(request.cookies, dict): cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items()) else: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index dfb1dca89..96d0dc515 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -20,6 +20,7 @@ from typing import ( NoReturn, Optional, Tuple, + TypedDict, Union, cast, ) @@ -34,8 +35,19 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax if TYPE_CHECKING: - # typing.Self requires Python 3.11 - from typing_extensions import Self + # typing.NotRequired and typing.Self require Python 3.11 + from typing_extensions import NotRequired, Self + + +class VerboseCookie(TypedDict): + name: str + value: str + domain: NotRequired[str] + path: NotRequired[str] + secure: NotRequired[bool] + + +CookiesT = Union[Dict[str, str], List[VerboseCookie]] def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn: @@ -97,7 +109,7 @@ class Request(object_ref): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: str = "utf-8", priority: int = 0, @@ -123,7 +135,7 @@ class Request(object_ref): self.callback: Optional[Callable] = callback self.errback: Optional[Callable] = errback - self.cookies: Union[Dict[str, str], List[Dict[str, str]]] = cookies or {} + self.cookies: CookiesT = cookies or {} self.headers: Headers = Headers(headers or {}, encoding=encoding) self.dont_filter: bool = dont_filter diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 14618e5e7..166c4de97 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -29,7 +29,7 @@ from twisted.internet.ssl import Certificate from scrapy.exceptions import NotSupported from scrapy.http.headers import Headers -from scrapy.http.request import Request +from scrapy.http.request import CookiesT, Request from scrapy.link import Link from scrapy.utils.trackref import object_ref @@ -181,7 +181,7 @@ class Response(object_ref): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, @@ -234,7 +234,7 @@ class Response(object_ref): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index a83279ac8..44c36b682 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -36,7 +36,7 @@ from w3lib.encoding import ( ) from w3lib.html import strip_html5_whitespace -from scrapy.http import Request +from scrapy.http.request import CookiesT, Request from scrapy.http.response import Response from scrapy.link import Link from scrapy.utils.python import memoizemethod_noargs, to_unicode @@ -183,7 +183,7 @@ class TextResponse(Response): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0, @@ -236,7 +236,7 @@ class TextResponse(Response): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0, From 019f23e3b75a0a481a4fcc22dc93c867ce424b18 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 2 Jun 2024 18:42:01 +0500 Subject: [PATCH 204/269] Add parameters to some of typing.Callable. --- scrapy/core/downloader/handlers/__init__.py | 12 +++++++----- scrapy/core/engine.py | 12 +++++++++--- scrapy/http/request/__init__.py | 2 +- scrapy/utils/misc.py | 8 +++++--- scrapy/utils/python.py | 6 +++--- 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index af5282553..5ec5ef6db 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -1,5 +1,7 @@ """Download handlers for different schemes""" +from __future__ import annotations + import logging from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Union, cast @@ -19,16 +21,16 @@ logger = logging.getLogger(__name__) class DownloadHandlers: - def __init__(self, crawler: "Crawler"): - self._crawler: "Crawler" = crawler - self._schemes: Dict[str, Union[str, Callable]] = ( + def __init__(self, crawler: Crawler): + self._crawler: Crawler = crawler + self._schemes: Dict[str, Union[str, Callable[..., Any]]] = ( {} ) # stores acceptable schemes on instancing self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes self._notconfigured: Dict[str, str] = {} # remembers failed handlers - handlers: Dict[str, Union[str, Callable]] = without_none_values( + handlers: Dict[str, Union[str, Callable[..., Any]]] = without_none_values( cast( - Dict[str, Union[str, Callable]], + Dict[str, Union[str, Callable[..., Any]]], crawler.settings.getwithbase("DOWNLOAD_HANDLERS"), ) ) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 4cb4454e3..b342ad7a3 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -86,7 +86,11 @@ class Slot: class ExecutionEngine: - def __init__(self, crawler: Crawler, spider_closed_callback: Callable) -> None: + def __init__( + self, + crawler: Crawler, + spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]], + ) -> None: self.crawler: Crawler = crawler self.settings: Settings = crawler.settings self.signals: SignalManager = crawler.signals @@ -102,7 +106,9 @@ class ExecutionEngine: downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"]) self.downloader: Downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) - self._spider_closed_callback: Callable = spider_closed_callback + self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = ( + spider_closed_callback + ) self.start_time: Optional[float] = None def _get_scheduler_class(self, settings: BaseSettings) -> Type[BaseScheduler]: @@ -427,7 +433,7 @@ class ExecutionEngine: dfd = self.slot.close() - def log_failure(msg: str) -> Callable: + def log_failure(msg: str) -> Callable[[Failure], None]: def errback(failure: Failure) -> None: logger.error( msg, exc_info=failure_to_exc_info(failure), extra={"spider": spider} diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 96d0dc515..77149333c 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -266,7 +266,7 @@ class Request(object_ref): return d -def _find_method(obj: Any, func: Callable) -> str: +def _find_method(obj: Any, func: Callable[..., Any]) -> str: """Helper function for Request.to_dict""" # Only instance methods contain ``__func__`` if obj and hasattr(func, "__func__"): diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index b678d1def..49f36de2d 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -56,7 +56,7 @@ def arg_to_iter(arg: Any) -> Iterable[Any]: return [arg] -def load_object(path: Union[str, Callable]) -> Any: +def load_object(path: Union[str, Callable[..., Any]]) -> Any: """Load an object given its absolute object path, and return it. The object can be the import path of a class, function, variable or an @@ -263,7 +263,7 @@ def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]: _generator_callbacks_cache = LocalWeakReferencedCache(limit=128) -def is_generator_with_return_value(callable: Callable) -> bool: +def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: """ Returns True if a callable is a generator function which includes a 'return' statement with a value different than None, False otherwise @@ -300,7 +300,9 @@ def is_generator_with_return_value(callable: Callable) -> bool: return bool(_generator_callbacks_cache[callable]) -def warn_on_generator_with_return_value(spider: Spider, callable: Callable) -> None: +def warn_on_generator_with_return_value( + spider: Spider, callable: Callable[..., Any] +) -> None: """ Logs a warning if a callable is a generator function and includes a 'return' statement with a value different than None diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index fc1eb4f69..37a84a350 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -217,7 +217,7 @@ def binary_is_text(data: bytes) -> bool: return all(c not in _BINARYCHARS for c in data) -def get_func_args(func: Callable, stripself: bool = False) -> List[str]: +def get_func_args(func: Callable[..., Any], stripself: bool = False) -> List[str]: """Return the argument name list of a callable object""" if not callable(func): raise TypeError(f"func must be callable, got '{type(func).__name__}'") @@ -247,7 +247,7 @@ def get_func_args(func: Callable, stripself: bool = False) -> List[str]: return args -def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]: +def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]: """Returns (args, kwargs) tuple for a function >>> import re >>> get_spec(re.match) @@ -285,7 +285,7 @@ def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]: def equal_attributes( - obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable]]] + obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable[[Any], Any]]]] ) -> bool: """Compare two objects attributes""" # not attributes given return False by default From 492c3bce9dfc6cccdad8fc7002db4bec49cfcb35 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 3 Jun 2024 15:28:20 +0400 Subject: [PATCH 205/269] Don't run callbacks of requests from get_media_requests(). (#6386) --- scrapy/pipelines/media.py | 10 ---------- tests/test_pipeline_media.py | 7 ------- 2 files changed, 17 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 25e00b0ea..0e374265e 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -24,10 +24,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def _DUMMY_CALLBACK(response): - return response - - class MediaPipeline(ABC): LOG_FAILED_RESULTS = True @@ -89,10 +85,6 @@ class MediaPipeline(ABC): def _process_request(self, request, info, item): fp = self._fingerprinter.fingerprint(request) - if not request.callback or request.callback is NO_CALLBACK: - cb = _DUMMY_CALLBACK - else: - cb = request.callback eb = request.errback request.callback = NO_CALLBACK request.errback = None @@ -100,14 +92,12 @@ class MediaPipeline(ABC): # Return cached result if request was already seen if fp in info.downloaded: d = defer_result(info.downloaded[fp]) - d.addCallback(cb) if eb: d.addErrback(eb) return d # Otherwise, wait for result wad = Deferred() - wad.addCallback(cb) if eb: wad.addErrback(eb) info.waiting[fp].append(wad) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 763453551..127775f43 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -211,10 +211,6 @@ class MockedMediaPipeline(UserDefinedPipeline): class MediaPipelineTestCase(BaseMediaPipelineTestCase): pipeline_class = MockedMediaPipeline - def _callback(self, result): - self.pipe._mockcalled.append("request_callback") - return result - def _errback(self, result): self.pipe._mockcalled.append("request_errback") return result @@ -225,7 +221,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): req = Request( "http://url1", meta={"response": rsp}, - callback=self._callback, errback=self._errback, ) item = {"requests": req} @@ -237,7 +232,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): "get_media_requests", "media_to_download", "media_downloaded", - "request_callback", "item_completed", ], ) @@ -249,7 +243,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): req = Request( "http://url1", meta={"response": fail}, - callback=self._callback, errback=self._errback, ) item = {"requests": req} From e56b425198bfe3e86f2c578e7bc1f2988c7d3ec9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 5 Jun 2024 08:33:45 +0400 Subject: [PATCH 206/269] Full typing for scrapy/pipelines. (#6387) --- scrapy/pipelines/__init__.py | 3 +- scrapy/pipelines/files.py | 274 +++++++++++++++++++++++++---------- scrapy/pipelines/images.py | 128 ++++++++++++---- scrapy/pipelines/media.py | 162 ++++++++++++++++----- 4 files changed, 424 insertions(+), 143 deletions(-) diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index f9544d329..0cfbc156f 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -10,6 +10,7 @@ from twisted.internet.defer import Deferred from scrapy import Spider from scrapy.middleware import MiddlewareManager +from scrapy.settings import Settings from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_f_from_coro_f @@ -18,7 +19,7 @@ class ItemPipelineManager(MiddlewareManager): component_name = "item pipeline" @classmethod - def _get_mwlist_from_settings(cls, settings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: return build_component_list(settings.getwithbase("ITEM_PIPELINES")) def _add_middleware(self, pipe: Any) -> None: diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index c1ce0939c..85a8c77da 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -18,16 +18,35 @@ from ftplib import FTP from io import BytesIO from os import PathLike from pathlib import Path -from typing import IO, TYPE_CHECKING, DefaultDict, Optional, Set, Type, Union, cast +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + DefaultDict, + Dict, + List, + NoReturn, + Optional, + Protocol, + Set, + Type, + TypedDict, + Union, + cast, +) from urllib.parse import urlparse from itemadapter import ItemAdapter from twisted.internet import defer, threads +from twisted.internet.defer import Deferred +from twisted.python.failure import Failure +from scrapy import Spider from scrapy.exceptions import IgnoreRequest, NotConfigured -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.media import MediaPipeline +from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline from scrapy.settings import Settings from scrapy.utils.boto import is_botocore_available from scrapy.utils.datatypes import CaseInsensitiveDict @@ -40,10 +59,11 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + logger = logging.getLogger(__name__) -def _to_string(path: Union[str, PathLike]) -> str: +def _to_string(path: Union[str, PathLike[str]]) -> str: return str(path) # convert a Path object to string @@ -68,23 +88,54 @@ class FileException(Exception): """General media error exception""" +class StatInfo(TypedDict, total=False): + checksum: str + last_modified: float + + +class FilesStoreProtocol(Protocol): + def __init__(self, basedir: str): ... + + def persist_file( + self, + path: str, + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + ) -> Optional[Deferred[Any]]: ... + + def stat_file( + self, path: str, info: MediaPipeline.SpiderInfo + ) -> Union[StatInfo, Deferred[StatInfo]]: ... + + class FSFilesStore: - def __init__(self, basedir: Union[str, PathLike]): + def __init__(self, basedir: Union[str, PathLike[str]]): basedir = _to_string(basedir) if "://" in basedir: basedir = basedir.split("://", 1)[1] - self.basedir = basedir + self.basedir: str = basedir self._mkdir(Path(self.basedir)) - self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set) + self.created_directories: DefaultDict[MediaPipeline.SpiderInfo, Set[str]] = ( + defaultdict(set) + ) def persist_file( - self, path: Union[str, PathLike], buf, info, meta=None, headers=None - ): + self, + path: Union[str, PathLike[str]], + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + ) -> None: absolute_path = self._get_filesystem_path(path) self._mkdir(absolute_path.parent, info) absolute_path.write_bytes(buf.getvalue()) - def stat_file(self, path: Union[str, PathLike], info): + def stat_file( + self, path: Union[str, PathLike[str]], info: MediaPipeline.SpiderInfo + ) -> StatInfo: absolute_path = self._get_filesystem_path(path) try: last_modified = absolute_path.stat().st_mtime @@ -96,12 +147,14 @@ class FSFilesStore: return {"last_modified": last_modified, "checksum": checksum} - def _get_filesystem_path(self, path: Union[str, PathLike]) -> Path: + def _get_filesystem_path(self, path: Union[str, PathLike[str]]) -> Path: path_comps = _to_string(path).split("/") return Path(self.basedir, *path_comps) - def _mkdir(self, dirname: Path, domain: Optional[str] = None): - seen = self.created_directories[domain] if domain else set() + def _mkdir( + self, dirname: Path, domain: Optional[MediaPipeline.SpiderInfo] = None + ) -> None: + seen: Set[str] = self.created_directories[domain] if domain else set() if str(dirname) not in seen: if not dirname.exists(): dirname.mkdir(parents=True) @@ -122,7 +175,7 @@ class S3FilesStore: "Cache-Control": "max-age=172800", } - def __init__(self, uri): + def __init__(self, uri: str): if not is_botocore_available(): raise NotConfigured("missing botocore library") import botocore.session @@ -142,8 +195,10 @@ class S3FilesStore: raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'") self.bucket, self.prefix = uri[5:].split("/", 1) - def stat_file(self, path, info): - def _onsuccess(boto_key): + def stat_file( + self, path: str, info: MediaPipeline.SpiderInfo + ) -> Deferred[StatInfo]: + def _onsuccess(boto_key: Dict[str, Any]) -> StatInfo: checksum = boto_key["ETag"].strip('"') last_modified = boto_key["LastModified"] modified_stamp = time.mktime(last_modified.timetuple()) @@ -151,13 +206,23 @@ class S3FilesStore: return self._get_boto_key(path).addCallback(_onsuccess) - def _get_boto_key(self, path): + def _get_boto_key(self, path: str) -> Deferred[Dict[str, Any]]: key_name = f"{self.prefix}{path}" - return threads.deferToThread( - self.s3_client.head_object, Bucket=self.bucket, Key=key_name + return cast( + "Deferred[Dict[str, Any]]", + threads.deferToThread( + self.s3_client.head_object, Bucket=self.bucket, Key=key_name # type: ignore[attr-defined] + ), ) - def persist_file(self, path, buf, info, meta=None, headers=None): + def persist_file( + self, + path: str, + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + ) -> Deferred[Any]: """Upload file to S3 storage""" key_name = f"{self.prefix}{path}" buf.seek(0) @@ -165,7 +230,7 @@ class S3FilesStore: if headers: extra.update(self._headers_to_botocore_kwargs(headers)) return threads.deferToThread( - self.s3_client.put_object, + self.s3_client.put_object, # type: ignore[attr-defined] Bucket=self.bucket, Key=key_name, Body=buf, @@ -174,7 +239,7 @@ class S3FilesStore: **extra, ) - def _headers_to_botocore_kwargs(self, headers): + def _headers_to_botocore_kwargs(self, headers: Dict[str, Any]) -> Dict[str, Any]: """Convert headers to botocore keyword arguments.""" # This is required while we need to support both boto and botocore. mapping = CaseInsensitiveDict( @@ -206,7 +271,7 @@ class S3FilesStore: "X-Amz-Website-Redirect-Location": "WebsiteRedirectLocation", } ) - extra = {} + extra: Dict[str, Any] = {} for key, value in headers.items(): try: kwarg = mapping[key] @@ -226,13 +291,13 @@ class GCSFilesStore: # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. POLICY = None - def __init__(self, uri): + def __init__(self, uri: str): from google.cloud import storage client = storage.Client(project=self.GCS_PROJECT_ID) bucket, prefix = uri[5:].split("/", 1) self.bucket = client.bucket(bucket) - self.prefix = prefix + self.prefix: str = prefix permissions = self.bucket.test_iam_permissions( ["storage.objects.get", "storage.objects.create"] ) @@ -248,8 +313,10 @@ class GCSFilesStore: {"bucket": bucket}, ) - def stat_file(self, path, info): - def _onsuccess(blob): + def stat_file( + self, path: str, info: MediaPipeline.SpiderInfo + ) -> Deferred[StatInfo]: + def _onsuccess(blob) -> StatInfo: if blob: checksum = base64.b64decode(blob.md5_hash).hex() last_modified = time.mktime(blob.updated.timetuple()) @@ -257,19 +324,29 @@ class GCSFilesStore: return {} blob_path = self._get_blob_path(path) - return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback( - _onsuccess + return cast( + Deferred[StatInfo], + threads.deferToThread(self.bucket.get_blob, blob_path).addCallback( + _onsuccess + ), ) - def _get_content_type(self, headers): + def _get_content_type(self, headers: Optional[Dict[str, str]]) -> str: if headers and "Content-Type" in headers: return headers["Content-Type"] return "application/octet-stream" - def _get_blob_path(self, path): + def _get_blob_path(self, path: str) -> str: return self.prefix + path - def persist_file(self, path, buf, info, meta=None, headers=None): + def persist_file( + self, + path: str, + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + ) -> Deferred[Any]: blob_path = self._get_blob_path(path) blob = self.bucket.blob(blob_path) blob.cache_control = self.CACHE_CONTROL @@ -283,22 +360,33 @@ class GCSFilesStore: class FTPFilesStore: - FTP_USERNAME = None - FTP_PASSWORD = None - USE_ACTIVE_MODE = None + FTP_USERNAME: Optional[str] = None + FTP_PASSWORD: Optional[str] = None + USE_ACTIVE_MODE: Optional[bool] = None - def __init__(self, uri): + def __init__(self, uri: str): if not uri.startswith("ftp://"): raise ValueError(f"Incorrect URI scheme in {uri}, expected 'ftp'") u = urlparse(uri) - self.port = u.port - self.host = u.hostname + assert u.port + assert u.hostname + self.port: int = u.port + self.host: str = u.hostname self.port = int(u.port or 21) - self.username = u.username or self.FTP_USERNAME - self.password = u.password or self.FTP_PASSWORD - self.basedir = u.path.rstrip("/") + assert self.FTP_USERNAME + assert self.FTP_PASSWORD + self.username: str = u.username or self.FTP_USERNAME + self.password: str = u.password or self.FTP_PASSWORD + self.basedir: str = u.path.rstrip("/") - def persist_file(self, path, buf, info, meta=None, headers=None): + def persist_file( + self, + path: str, + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + ) -> Deferred[Any]: path = f"{self.basedir}/{path}" return threads.deferToThread( ftp_store_file, @@ -311,8 +399,10 @@ class FTPFilesStore: use_active_mode=self.USE_ACTIVE_MODE, ) - def stat_file(self, path, info): - def _stat_file(path): + def stat_file( + self, path: str, info: MediaPipeline.SpiderInfo + ) -> Deferred[StatInfo]: + def _stat_file(path: str) -> StatInfo: try: ftp = FTP() ftp.connect(self.host, self.port) @@ -328,7 +418,7 @@ class FTPFilesStore: except Exception: return {} - return threads.deferToThread(_stat_file, path) + return cast("Deferred[StatInfo]", threads.deferToThread(_stat_file, path)) class FilesPipeline(MediaPipeline): @@ -350,20 +440,23 @@ class FilesPipeline(MediaPipeline): """ - MEDIA_NAME = "file" - EXPIRES = 90 - STORE_SCHEMES = { + MEDIA_NAME: str = "file" + EXPIRES: int = 90 + STORE_SCHEMES: Dict[str, Type[FilesStoreProtocol]] = { "": FSFilesStore, "file": FSFilesStore, "s3": S3FilesStore, "gs": GCSFilesStore, "ftp": FTPFilesStore, } - DEFAULT_FILES_URLS_FIELD = "file_urls" - DEFAULT_FILES_RESULT_FIELD = "files" + DEFAULT_FILES_URLS_FIELD: str = "file_urls" + DEFAULT_FILES_RESULT_FIELD: str = "files" def __init__( - self, store_uri: Union[str, PathLike], download_func=None, settings=None + self, + store_uri: Union[str, PathLike[str]], + download_func: Optional[Callable[[Request, Spider], Response]] = None, + settings: Union[Settings, Dict[str, Any], None] = None, ): store_uri = _to_string(store_uri) if not store_uri: @@ -372,26 +465,26 @@ class FilesPipeline(MediaPipeline): if isinstance(settings, dict) or settings is None: settings = Settings(settings) cls_name = "FilesPipeline" - self.store = self._get_store(store_uri) + self.store: FilesStoreProtocol = self._get_store(store_uri) resolve = functools.partial( self._key_for_pipe, base_class_name=cls_name, settings=settings ) - self.expires = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES) + self.expires: int = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES) if not hasattr(self, "FILES_URLS_FIELD"): self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELD if not hasattr(self, "FILES_RESULT_FIELD"): self.FILES_RESULT_FIELD = self.DEFAULT_FILES_RESULT_FIELD - self.files_urls_field = settings.get( + self.files_urls_field: str = settings.get( resolve("FILES_URLS_FIELD"), self.FILES_URLS_FIELD ) - self.files_result_field = settings.get( + self.files_result_field: str = settings.get( resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD ) super().__init__(download_func=download_func, settings=settings) @classmethod - def from_settings(cls, settings) -> Self: + def from_settings(cls, settings: Settings) -> Self: s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] @@ -418,7 +511,7 @@ class FilesPipeline(MediaPipeline): store_uri = settings["FILES_STORE"] return cls(store_uri, settings=settings) - def _get_store(self, uri: str): + def _get_store(self, uri: str) -> FilesStoreProtocol: if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir scheme = "file" else: @@ -426,19 +519,21 @@ class FilesPipeline(MediaPipeline): store_cls = self.STORE_SCHEMES[scheme] return store_cls(uri) - def media_to_download(self, request, info, *, item=None): - def _onsuccess(result): + def media_to_download( + self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None + ) -> Deferred[Optional[FileInfo]]: + def _onsuccess(result: StatInfo) -> Optional[FileInfo]: if not result: - return # returning None force download + return None # returning None force download last_modified = result.get("last_modified", None) if not last_modified: - return # returning None force download + return None # returning None force download age_seconds = time.time() - last_modified age_days = age_seconds / 60 / 60 / 24 if age_days > self.expires: - return # returning None force download + return None # returning None force download referer = referer_str(request) logger.debug( @@ -458,19 +553,22 @@ class FilesPipeline(MediaPipeline): } path = self.file_path(request, info=info, item=item) - dfd = defer.maybeDeferred(self.store.stat_file, path, info) - dfd.addCallback(_onsuccess) - dfd.addErrback(lambda _: None) - dfd.addErrback( + # defer.maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type + dfd: Deferred[StatInfo] = defer.maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type] + dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess) + dfd2.addErrback(lambda _: None) + dfd2.addErrback( lambda f: logger.error( self.__class__.__name__ + ".store.stat_file", exc_info=failure_to_exc_info(f), extra={"spider": info.spider}, ) ) - return dfd + return dfd2 - def media_failed(self, failure, request, info): + def media_failed( + self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo + ) -> NoReturn: if not isinstance(failure.value, IgnoreRequest): referer = referer_str(request) logger.warning( @@ -487,7 +585,14 @@ class FilesPipeline(MediaPipeline): raise FileException - def media_downloaded(self, response, request, info, *, item=None): + def media_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> FileInfo: referer = referer_str(request) if response.status != 200: @@ -546,16 +651,26 @@ class FilesPipeline(MediaPipeline): "status": status, } - def inc_stats(self, spider, status): + def inc_stats(self, spider: Spider, status: str) -> None: + assert spider.crawler.stats spider.crawler.stats.inc_value("file_count", spider=spider) spider.crawler.stats.inc_value(f"file_status_count/{status}", spider=spider) # Overridable Interface - def get_media_requests(self, item, info): + def get_media_requests( + self, item: Any, info: MediaPipeline.SpiderInfo + ) -> List[Request]: urls = ItemAdapter(item).get(self.files_urls_field, []) return [Request(u, callback=NO_CALLBACK) for u in urls] - def file_downloaded(self, response, request, info, *, item=None): + def file_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> str: path = self.file_path(request, response=response, info=info, item=item) buf = BytesIO(response.body) checksum = _md5sum(buf) @@ -563,12 +678,21 @@ class FilesPipeline(MediaPipeline): self.store.persist_file(path, buf, info) return checksum - def item_completed(self, results, item, info): + def item_completed( + self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo + ) -> Any: with suppress(KeyError): ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None, *, item=None): + def file_path( + self, + request: Request, + response: Optional[Response] = None, + info: Optional[MediaPipeline.SpiderInfo] = None, + *, + item: Any = None, + ) -> str: media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec media_ext = Path(request.url).suffix # Handles empty and wild extensions by trying to guess the @@ -577,5 +701,5 @@ class FilesPipeline(MediaPipeline): media_ext = "" media_type = mimetypes.guess_type(request.url)[0] if media_type: - media_ext = mimetypes.guess_extension(media_type) + media_ext = cast(str, mimetypes.guess_extension(media_type)) return f"full/{media_guid}{media_ext}" diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index e7ef06fb3..27a57b17c 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -12,12 +12,25 @@ import warnings from contextlib import suppress from io import BytesIO from os import PathLike -from typing import TYPE_CHECKING, Dict, Tuple, Type, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Tuple, + Type, + Union, + cast, +) from itemadapter import ItemAdapter +from scrapy import Spider from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import ( FileException, @@ -27,20 +40,20 @@ from scrapy.pipelines.files import ( S3FilesStore, _md5sum, ) - -# TODO: from scrapy.pipelines.media import MediaPipeline +from scrapy.pipelines.media import FileInfoOrError, MediaPipeline from scrapy.settings import Settings from scrapy.utils.python import get_func_args, to_bytes if TYPE_CHECKING: # typing.Self requires Python 3.11 + from PIL import Image from typing_extensions import Self class NoimagesDrop(DropItem): """Product with no images exception""" - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any): warnings.warn( "The NoimagesDrop class is deprecated", category=ScrapyDeprecationWarning, @@ -56,19 +69,22 @@ class ImageException(FileException): class ImagesPipeline(FilesPipeline): """Abstract pipeline that implement the image thumbnail generation logic""" - MEDIA_NAME = "image" + MEDIA_NAME: str = "image" # Uppercase attributes kept for backward compatibility with code that subclasses # ImagesPipeline. They may be overridden by settings. - MIN_WIDTH = 0 - MIN_HEIGHT = 0 - EXPIRES = 90 + MIN_WIDTH: int = 0 + MIN_HEIGHT: int = 0 + EXPIRES: int = 90 THUMBS: Dict[str, Tuple[int, int]] = {} DEFAULT_IMAGES_URLS_FIELD = "image_urls" DEFAULT_IMAGES_RESULT_FIELD = "images" def __init__( - self, store_uri: Union[str, PathLike], download_func=None, settings=None + self, + store_uri: Union[str, PathLike[str]], + download_func: Optional[Callable[[Request, Spider], Response]] = None, + settings: Union[Settings, Dict[str, Any], None] = None, ): try: from PIL import Image @@ -89,27 +105,33 @@ class ImagesPipeline(FilesPipeline): base_class_name="ImagesPipeline", settings=settings, ) - self.expires = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES) + self.expires: int = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES) if not hasattr(self, "IMAGES_RESULT_FIELD"): - self.IMAGES_RESULT_FIELD = self.DEFAULT_IMAGES_RESULT_FIELD + self.IMAGES_RESULT_FIELD: str = self.DEFAULT_IMAGES_RESULT_FIELD if not hasattr(self, "IMAGES_URLS_FIELD"): - self.IMAGES_URLS_FIELD = self.DEFAULT_IMAGES_URLS_FIELD + self.IMAGES_URLS_FIELD: str = self.DEFAULT_IMAGES_URLS_FIELD - self.images_urls_field = settings.get( + self.images_urls_field: str = settings.get( resolve("IMAGES_URLS_FIELD"), self.IMAGES_URLS_FIELD ) - self.images_result_field = settings.get( + self.images_result_field: str = settings.get( resolve("IMAGES_RESULT_FIELD"), self.IMAGES_RESULT_FIELD ) - self.min_width = settings.getint(resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH) - self.min_height = settings.getint(resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT) - self.thumbs = settings.get(resolve("IMAGES_THUMBS"), self.THUMBS) + self.min_width: int = settings.getint( + resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH + ) + self.min_height: int = settings.getint( + resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT + ) + self.thumbs: Dict[str, Tuple[int, int]] = settings.get( + resolve("IMAGES_THUMBS"), self.THUMBS + ) - self._deprecated_convert_image = None + self._deprecated_convert_image: Optional[bool] = None @classmethod - def from_settings(cls, settings) -> Self: + def from_settings(cls, settings: Settings) -> Self: s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] @@ -136,11 +158,25 @@ class ImagesPipeline(FilesPipeline): store_uri = settings["IMAGES_STORE"] return cls(store_uri, settings=settings) - def file_downloaded(self, response, request, info, *, item=None): + def file_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> str: return self.image_downloaded(response, request, info, item=item) - def image_downloaded(self, response, request, info, *, item=None): - checksum = None + def image_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> str: + checksum: Optional[str] = None for path, image, buf in self.get_images(response, request, info, item=item): if checksum is None: buf.seek(0) @@ -153,9 +189,17 @@ class ImagesPipeline(FilesPipeline): meta={"width": width, "height": height}, headers={"Content-Type": "image/jpeg"}, ) + assert checksum is not None return checksum - def get_images(self, response, request, info, *, item=None): + def get_images( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> Iterable[Tuple[str, Image.Image, BytesIO]]: path = self.file_path(request, response=response, info=info, item=item) orig_image = self._Image.open(BytesIO(response.body)) @@ -196,7 +240,12 @@ class ImagesPipeline(FilesPipeline): thumb_image, thumb_buf = self.convert_image(image, size, buf) yield thumb_path, thumb_image, thumb_buf - def convert_image(self, image, size=None, response_body=None): + def convert_image( + self, + image: Image.Image, + size: Optional[Tuple[int, int]] = None, + response_body: Optional[BytesIO] = None, + ) -> Tuple[Image.Image, BytesIO]: if response_body is None: warnings.warn( f"{self.__class__.__name__}.convert_image() method called in a deprecated way, " @@ -225,7 +274,7 @@ class ImagesPipeline(FilesPipeline): # when updating the minimum requirements for Pillow. resampling_filter = self._Image.Resampling.LANCZOS except AttributeError: - resampling_filter = self._Image.ANTIALIAS + resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined] image.thumbnail(size, resampling_filter) elif response_body is not None and image.format == "JPEG": return image, response_body @@ -234,19 +283,38 @@ class ImagesPipeline(FilesPipeline): image.save(buf, "JPEG") return image, buf - def get_media_requests(self, item, info): + def get_media_requests( + self, item: Any, info: MediaPipeline.SpiderInfo + ) -> List[Request]: urls = ItemAdapter(item).get(self.images_urls_field, []) return [Request(u, callback=NO_CALLBACK) for u in urls] - def item_completed(self, results, item, info): + def item_completed( + self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo + ) -> Any: with suppress(KeyError): ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None, *, item=None): + def file_path( + self, + request: Request, + response: Optional[Response] = None, + info: Optional[MediaPipeline.SpiderInfo] = None, + *, + item: Any = None, + ) -> str: image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec return f"full/{image_guid}.jpg" - def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None): + def thumb_path( + self, + request: Request, + thumb_id: str, + response: Optional[Response] = None, + info: Optional[MediaPipeline.SpiderInfo] = None, + *, + item: Any = None, + ) -> str: thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec return f"thumbs/{thumb_id}/{thumb_guid}.jpg" diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 0e374265e..3e327105e 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -4,54 +4,101 @@ import functools import logging from abc import ABC, abstractmethod from collections import defaultdict -from typing import TYPE_CHECKING +from typing import ( + TYPE_CHECKING, + Any, + Callable, + DefaultDict, + Dict, + List, + Literal, + NoReturn, + Optional, + Set, + Tuple, + TypedDict, + TypeVar, + Union, + cast, +) from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure -from scrapy.http.request import NO_CALLBACK +from scrapy import Spider +from scrapy.crawler import Crawler +from scrapy.http import Response +from scrapy.http.request import NO_CALLBACK, Request from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import defer_result, mustbe_deferred from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter +from scrapy.utils.request import RequestFingerprinter if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self +_T = TypeVar("_T") + + +class FileInfo(TypedDict): + url: str + path: str + checksum: Optional[str] + status: str + + +FileInfoOrError = Union[Tuple[Literal[True], FileInfo], Tuple[Literal[False], Failure]] + logger = logging.getLogger(__name__) class MediaPipeline(ABC): - LOG_FAILED_RESULTS = True + crawler: Crawler + _fingerprinter: RequestFingerprinter + + LOG_FAILED_RESULTS: bool = True class SpiderInfo: - def __init__(self, spider): - self.spider = spider - self.downloading = set() - self.downloaded = {} - self.waiting = defaultdict(list) + def __init__(self, spider: Spider): + self.spider: Spider = spider + self.downloading: Set[bytes] = set() + self.downloaded: Dict[bytes, Union[FileInfo, Failure]] = {} + self.waiting: DefaultDict[bytes, List[Deferred[FileInfo]]] = defaultdict( + list + ) - def __init__(self, download_func=None, settings=None): + def __init__( + self, + download_func: Optional[Callable[[Request, Spider], Response]] = None, + settings: Union[Settings, Dict[str, Any], None] = None, + ): self.download_func = download_func - self._expects_item = {} if isinstance(settings, dict) or settings is None: settings = Settings(settings) resolve = functools.partial( self._key_for_pipe, base_class_name="MediaPipeline", settings=settings ) - self.allow_redirects = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False) + self.allow_redirects: bool = settings.getbool( + resolve("MEDIA_ALLOW_REDIRECTS"), False + ) self._handle_statuses(self.allow_redirects) - def _handle_statuses(self, allow_redirects): + def _handle_statuses(self, allow_redirects: bool) -> None: self.handle_httpstatus_list = None if allow_redirects: self.handle_httpstatus_list = SequenceExclude(range(300, 400)) - def _key_for_pipe(self, key, base_class_name=None, settings=None): + def _key_for_pipe( + self, + key: str, + base_class_name: Optional[str] = None, + settings: Optional[Settings] = None, + ) -> str: class_name = self.__class__.__name__ formatted_key = f"{class_name.upper()}_{key}" if ( @@ -64,26 +111,34 @@ class MediaPipeline(ABC): return formatted_key @classmethod - def from_crawler(cls, crawler) -> Self: + def from_crawler(cls, crawler: Crawler) -> Self: + pipe: Self try: pipe = cls.from_settings(crawler.settings) # type: ignore[attr-defined] except AttributeError: pipe = cls() pipe.crawler = crawler + assert crawler.request_fingerprinter pipe._fingerprinter = crawler.request_fingerprinter return pipe - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> None: self.spiderinfo = self.SpiderInfo(spider) - def process_item(self, item, spider): + def process_item( + self, item: Any, spider: Spider + ) -> Deferred[List[FileInfoOrError]]: info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) dlist = [self._process_request(r, info, item) for r in requests] - dfd = DeferredList(dlist, consumeErrors=True) + dfd = cast( + "Deferred[List[FileInfoOrError]]", DeferredList(dlist, consumeErrors=True) + ) return dfd.addCallback(self.item_completed, item, info) - def _process_request(self, request, info, item): + def _process_request( + self, request: Request, info: SpiderInfo, item: Any + ) -> Deferred[FileInfo]: fp = self._fingerprinter.fingerprint(request) eb = request.errback request.callback = NO_CALLBACK @@ -97,7 +152,7 @@ class MediaPipeline(ABC): return d # Otherwise, wait for result - wad = Deferred() + wad: Deferred[FileInfo] = Deferred() if eb: wad.addErrback(eb) info.waiting[fp].append(wad) @@ -108,36 +163,48 @@ class MediaPipeline(ABC): # Download request checking media_to_download hook output first info.downloading.add(fp) - dfd = mustbe_deferred(self.media_to_download, request, info, item=item) - dfd.addCallback(self._check_media_to_download, request, info, item=item) - dfd.addErrback(self._log_exception) - dfd.addBoth(self._cache_result_and_execute_waiters, fp, info) - return dfd.addBoth(lambda _: wad) # it must return wad at last + dfd: Deferred[Optional[FileInfo]] = mustbe_deferred( + self.media_to_download, request, info, item=item + ) + dfd2: Deferred[FileInfo] = dfd.addCallback( + self._check_media_to_download, request, info, item=item + ) + dfd2.addErrback(self._log_exception) + dfd2.addBoth(self._cache_result_and_execute_waiters, fp, info) + return dfd2.addBoth(lambda _: wad) # it must return wad at last - def _log_exception(self, result): + def _log_exception(self, result: Failure) -> Failure: logger.exception(result) return result - def _modify_media_request(self, request): + def _modify_media_request(self, request: Request) -> None: if self.handle_httpstatus_list: request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list else: request.meta["handle_httpstatus_all"] = True - def _check_media_to_download(self, result, request, info, item): + def _check_media_to_download( + self, result: Optional[FileInfo], request: Request, info: SpiderInfo, item: Any + ) -> Union[FileInfo, Deferred[FileInfo]]: if result is not None: return result + dfd: Deferred[Response] if self.download_func: # this ugly code was left only to support tests. TODO: remove dfd = mustbe_deferred(self.download_func, request, info.spider) else: self._modify_media_request(request) + assert self.crawler.engine dfd = self.crawler.engine.download(request) - dfd.addCallback(self.media_downloaded, request, info, item=item) - dfd.addErrback(self.media_failed, request, info) - return dfd + dfd2: Deferred[FileInfo] = dfd.addCallback( + self.media_downloaded, request, info, item=item + ) + dfd2.addErrback(self.media_failed, request, info) + return dfd2 - def _cache_result_and_execute_waiters(self, result, fp, info): + def _cache_result_and_execute_waiters( + self, result: Union[FileInfo, Failure], fp: bytes, info: SpiderInfo + ) -> None: if isinstance(result, Failure): # minimize cached information for failure result.cleanFailure() @@ -176,30 +243,44 @@ class MediaPipeline(ABC): # Overridable Interface @abstractmethod - def media_to_download(self, request, info, *, item=None): + def media_to_download( + self, request: Request, info: SpiderInfo, *, item: Any = None + ) -> Deferred[Optional[FileInfo]]: """Check request before starting download""" raise NotImplementedError() @abstractmethod - def get_media_requests(self, item, info): + def get_media_requests(self, item: Any, info: SpiderInfo) -> List[Request]: """Returns the media requests to download""" raise NotImplementedError() @abstractmethod - def media_downloaded(self, response, request, info, *, item=None): + def media_downloaded( + self, + response: Response, + request: Request, + info: SpiderInfo, + *, + item: Any = None, + ) -> FileInfo: """Handler for success downloads""" raise NotImplementedError() @abstractmethod - def media_failed(self, failure, request, info): + def media_failed( + self, failure: Failure, request: Request, info: SpiderInfo + ) -> NoReturn: """Handler for failed downloads""" raise NotImplementedError() - def item_completed(self, results, item, info): + def item_completed( + self, results: List[FileInfoOrError], item: Any, info: SpiderInfo + ) -> Any: """Called per item when all media requests has been processed""" if self.LOG_FAILED_RESULTS: for ok, value in results: if not ok: + assert isinstance(value, Failure) logger.error( "%(class)s found errors processing %(item)s", {"class": self.__class__.__name__, "item": item}, @@ -209,6 +290,13 @@ class MediaPipeline(ABC): return item @abstractmethod - def file_path(self, request, response=None, info=None, *, item=None): + def file_path( + self, + request: Request, + response: Optional[Response] = None, + info: Optional[SpiderInfo] = None, + *, + item: Any = None, + ) -> str: """Returns the path where downloaded media should be stored""" raise NotImplementedError() From 3f76853bd27d84f53ebaaa97cb819e8a29195a89 Mon Sep 17 00:00:00 2001 From: Suvan Banerjee Date: Wed, 5 Jun 2024 10:04:46 +0530 Subject: [PATCH 207/269] Handle AttributeError: 'NoneType' in contract parsing (#6388) --- scrapy/contracts/__init__.py | 3 ++- tests/test_contracts.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index b300b8457..27bc2fcba 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -120,7 +120,8 @@ class ContractsManager: if line.startswith("@"): m = re.match(r"@(\w+)\s*(.*)", line) - assert m is not None + if m is None: + continue name, args = m.groups() args = re.split(r"\s+", args) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 1459e0b5f..c9c12f0d8 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -182,6 +182,19 @@ class TestSpider(Spider): """ pass + def invalid_regex(self, response): + """method with invalid regex + @ Scrapy is awsome + """ + pass + + def invalid_regex_with_valid_contract(self, response): + """method with invalid regex + @ scrapy is awsome + @url http://scrapy.org + """ + pass + class CustomContractSuccessSpider(Spider): name = "custom_contract_success_spider" @@ -385,6 +398,21 @@ class ContractsManagerTest(unittest.TestCase): message = "ContractFail: Missing fields: name, url" assert message in self.results.failures[-1][-1] + def test_regex(self): + spider = TestSpider() + response = ResponseMock() + + # invalid regex + request = self.conman.from_method(spider.invalid_regex, self.results) + self.should_succeed() + + # invalid regex with valid contract + request = self.conman.from_method( + spider.invalid_regex_with_valid_contract, self.results + ) + self.should_succeed() + request.callback(response) + def test_custom_contracts(self): self.conman.from_spider(CustomContractSuccessSpider(), self.results) self.should_succeed() From 2e214210f6707181a863dbceabf2d34e767396cb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 2 Jun 2024 01:48:37 +0500 Subject: [PATCH 208/269] Add parameters to iterable generics, replace generators with iterables. --- scrapy/commands/parse.py | 8 ++-- scrapy/core/engine.py | 5 ++- scrapy/core/scraper.py | 6 ++- scrapy/core/spidermw.py | 70 ++++++++++++++++++++------------ scrapy/http/response/__init__.py | 3 +- scrapy/http/response/text.py | 3 +- scrapy/utils/iterators.py | 28 ++++++------- scrapy/utils/misc.py | 6 +-- scrapy/utils/python.py | 32 +++++++-------- scrapy/utils/request.py | 5 +-- scrapy/utils/sitemap.py | 4 +- scrapy/utils/spider.py | 13 +++--- tests/test_commands.py | 4 +- 13 files changed, 103 insertions(+), 84 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 2453c0d39..f916a3e75 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -140,13 +140,13 @@ class Command(BaseRunSpiderCommand): @overload def iterate_spider_output( - self, result: Union[AsyncGenerator, CoroutineType] - ) -> Deferred: ... + self, result: Union[AsyncGenerator[_T, None], CoroutineType[Any, Any, _T]] + ) -> Deferred[_T]: ... @overload - def iterate_spider_output(self, result: _T) -> Iterable: ... + def iterate_spider_output(self, result: _T) -> Iterable[Any]: ... - def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]: + def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred]: if inspect.isasyncgen(result): d = deferred_from_coro( collect_asyncgen(aiter_errback(result, self.handle_exception)) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b342ad7a3..dededf99d 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -372,7 +372,10 @@ class ExecutionEngine: @inlineCallbacks def open_spider( - self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True + self, + spider: Spider, + start_requests: Iterable[Request] = (), + close_if_idle: bool = True, ) -> Generator[Deferred, Any, None]: if self.slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 566e6628b..3b7492838 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -16,6 +16,7 @@ from typing import ( Set, Tuple, Type, + TypeVar, Union, cast, ) @@ -47,6 +48,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler +_T = TypeVar("_T") QueueTuple = Tuple[Union[Response, Failure], Request, Deferred] @@ -256,14 +258,14 @@ class Scraper: def handle_spider_output( self, - result: Union[Iterable, AsyncIterable], + result: Union[Iterable[_T], AsyncIterable[_T]], request: Request, response: Response, spider: Spider, ) -> Deferred: if not result: return defer_succeed(None) - it: Union[Iterable, AsyncIterable] + it: Union[Iterable[_T], AsyncIterable[_T]] if isinstance(result, AsyncIterable): it = aiter_errback( result, self.handle_spider_error, request, response, spider diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 2cef2e1dd..cb1a93a68 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -9,7 +9,6 @@ from inspect import isasyncgenfunction, iscoroutine from itertools import islice from typing import ( Any, - AsyncGenerator, AsyncIterable, Callable, Generator, @@ -17,6 +16,7 @@ from typing import ( List, Optional, Tuple, + TypeVar, Union, cast, ) @@ -42,6 +42,7 @@ from scrapy.utils.python import MutableAsyncChain, MutableChain logger = logging.getLogger(__name__) +_T = TypeVar("_T") ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] @@ -98,31 +99,39 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - iterable: Union[Iterable, AsyncIterable], + iterable: Union[Iterable[_T], AsyncIterable[_T]], exception_processor_index: int, - recover_to: Union[MutableChain, MutableAsyncChain], - ) -> Union[Generator, AsyncGenerator]: - def process_sync(iterable: Iterable) -> Generator: + recover_to: Union[MutableChain[_T], MutableAsyncChain[_T]], + ) -> Union[Iterable[_T], AsyncIterable[_T]]: + def process_sync(iterable: Iterable[_T]) -> Iterable[_T]: try: yield from iterable except Exception as ex: - exception_result = self._process_spider_exception( - response, spider, Failure(ex), exception_processor_index + exception_result = cast( + Union[Failure, MutableChain[_T]], + self._process_spider_exception( + response, spider, Failure(ex), exception_processor_index + ), ) if isinstance(exception_result, Failure): raise + assert isinstance(recover_to, MutableChain) recover_to.extend(exception_result) - async def process_async(iterable: AsyncIterable) -> AsyncGenerator: + async def process_async(iterable: AsyncIterable[_T]) -> AsyncIterable[_T]: try: async for r in iterable: yield r except Exception as ex: - exception_result = self._process_spider_exception( - response, spider, Failure(ex), exception_processor_index + exception_result = cast( + Union[Failure, MutableAsyncChain[_T]], + self._process_spider_exception( + response, spider, Failure(ex), exception_processor_index + ), ) if isinstance(exception_result, Failure): raise + assert isinstance(recover_to, MutableAsyncChain) recover_to.extend(exception_result) if isinstance(iterable, AsyncIterable): @@ -135,7 +144,7 @@ class SpiderMiddlewareManager(MiddlewareManager): spider: Spider, _failure: Failure, start_index: int = 0, - ) -> Union[Failure, MutableChain]: + ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]: exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): @@ -151,14 +160,18 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned - dfd: Deferred = self._process_spider_output( - response, spider, result, method_index + 1 + dfd: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = ( + self._process_spider_output( + response, spider, result, method_index + 1 + ) ) # _process_spider_output() returns a Deferred only because of downgrading so this can be # simplified when downgrading is removed. if dfd.called: # the result is available immediately if _process_spider_output didn't do downgrading - return cast(MutableChain, dfd.result) + return cast( + Union[MutableChain[_T], MutableAsyncChain[_T]], dfd.result + ) # we forbid waiting here because otherwise we would need to return a deferred from # _process_spider_exception too, which complicates the architecture msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" @@ -181,12 +194,12 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - result: Union[Iterable, AsyncIterable], + result: Union[Iterable[_T], AsyncIterable[_T]], start_index: int = 0, - ) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]: + ) -> Generator[Deferred[Any], Any, Union[MutableChain[_T], MutableAsyncChain[_T]]]: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - recovered: Union[MutableChain, MutableAsyncChain] + recovered: Union[MutableChain[_T], MutableAsyncChain[_T]] last_result_is_async = isinstance(result, AsyncIterable) if last_result_is_async: recovered = MutableAsyncChain() @@ -237,7 +250,9 @@ class SpiderMiddlewareManager(MiddlewareManager): # might fail directly if the output value is not a generator result = method(response=response, result=result, spider=spider) except Exception as ex: - exception_result = self._process_spider_exception( + exception_result: Union[ + Failure, MutableChain[_T], MutableAsyncChain[_T] + ] = self._process_spider_exception( response, spider, Failure(ex), method_index + 1 ) if isinstance(exception_result, Failure): @@ -267,9 +282,12 @@ class SpiderMiddlewareManager(MiddlewareManager): return MutableChain(result, recovered) # type: ignore[arg-type] async def _process_callback_output( - self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable] - ) -> Union[MutableChain, MutableAsyncChain]: - recovered: Union[MutableChain, MutableAsyncChain] + self, + response: Response, + spider: Spider, + result: Union[Iterable[_T], AsyncIterable[_T]], + ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]: + recovered: Union[MutableChain[_T], MutableAsyncChain[_T]] if isinstance(result, AsyncIterable): recovered = MutableAsyncChain() else: @@ -293,14 +311,16 @@ class SpiderMiddlewareManager(MiddlewareManager): spider: Spider, ) -> Deferred: async def process_callback_output( - result: Union[Iterable, AsyncIterable] - ) -> Union[MutableChain, MutableAsyncChain]: + result: Union[Iterable[_T], AsyncIterable[_T]] + ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]: return await self._process_callback_output(response, spider, result) - def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]: + def process_spider_exception( + _failure: Failure, + ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]: return self._process_spider_exception(response, spider, _failure) - dfd = mustbe_deferred( + dfd: Deferred = mustbe_deferred( self._process_spider_input, scrape_func, response, request, spider ) dfd.addCallback(deferred_f_from_coro_f(process_callback_output)) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 166c4de97..daf193f59 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -14,7 +14,6 @@ from typing import ( AnyStr, Callable, Dict, - Generator, Iterable, List, Mapping, @@ -242,7 +241,7 @@ class Response(object_ref): errback: Optional[Callable] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, - ) -> Generator[Request, None, None]: + ) -> Iterable[Request]: """ .. versionadded:: 2.0 diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 44c36b682..df4d90829 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -15,7 +15,6 @@ from typing import ( AnyStr, Callable, Dict, - Generator, Iterable, List, Mapping, @@ -246,7 +245,7 @@ class TextResponse(Response): flags: Optional[List[str]] = None, css: Optional[str] = None, xpath: Optional[str] = None, - ) -> Generator[Request, None, None]: + ) -> Iterable[Request]: """ A generator that produces :class:`~.Request` instances to follow all links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index cd6e9d04e..41a842386 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -6,8 +6,7 @@ from typing import ( Any, Callable, Dict, - Generator, - Iterable, + Iterator, List, Literal, Optional, @@ -22,14 +21,12 @@ from lxml import etree # nosec from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse from scrapy.selector import Selector -from scrapy.utils.python import re_rsearch, to_unicode +from scrapy.utils.python import re_rsearch logger = logging.getLogger(__name__) -def xmliter( - obj: Union[Response, str, bytes], nodename: str -) -> Generator[Selector, Any, None]: +def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Iterator[Selector]: """Return a iterator of Selector's over all nodes of a XML document, given the name of the node to iterate. Useful for parsing XML feeds. @@ -90,7 +87,7 @@ def xmliter_lxml( nodename: str, namespace: Optional[str] = None, prefix: str = "x", -) -> Generator[Selector, Any, None]: +) -> Iterator[Selector]: reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( @@ -168,7 +165,7 @@ def csviter( headers: Optional[List[str]] = None, encoding: Optional[str] = None, quotechar: Optional[str] = None, -) -> Generator[Dict[str, str], Any, None]: +) -> Iterator[Dict[str, str]]: """Returns an iterator of dictionaries from the given csv object obj can be: @@ -184,10 +181,13 @@ def csviter( quotechar is the character used to enclosure fields on the given obj. """ - encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8" - - def row_to_unicode(row_: Iterable) -> List[str]: - return [to_unicode(field, encoding) for field in row_] + if encoding is not None: + warn( + "The encoding argument of csviter() is ignored and will be removed" + " in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) lines = StringIO(_body_or_str(obj, unicode=True)) @@ -200,13 +200,11 @@ def csviter( if not headers: try: - row = next(csv_r) + headers = next(csv_r) except StopIteration: return - headers = row_to_unicode(row) for row in csv_r: - row = row_to_unicode(row) if len(row) != len(headers): logger.warning( "ignoring row %(csvlnum)d (length: %(csvrow)d, " diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 49f36de2d..3d11c1035 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -20,8 +20,8 @@ from typing import ( Any, Callable, Deque, - Generator, Iterable, + Iterator, List, Optional, Type, @@ -227,7 +227,7 @@ def build_from_settings( @contextmanager -def set_environ(**kwargs: str) -> Generator[None, Any, None]: +def set_environ(**kwargs: str) -> Iterator[None]: """Temporarily set environment variables inside the context manager and fully restore previous environment afterwards """ @@ -244,7 +244,7 @@ def set_environ(**kwargs: str) -> Generator[None, Any, None]: os.environ[k] = v -def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]: +def walk_callable(node: ast.AST) -> Iterable[ast.AST]: """Similar to ``ast.walk``, but walks only function body and skips nested functions defined within the node. """ diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 37a84a350..059d8e04d 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -15,12 +15,10 @@ from itertools import chain from typing import ( TYPE_CHECKING, Any, - AsyncGenerator, AsyncIterable, AsyncIterator, Callable, Dict, - Generator, Iterable, Iterator, List, @@ -163,7 +161,7 @@ def re_rsearch( the start position of the match, and the ending (regarding the entire text). """ - def _chunk_iter() -> Generator[Tuple[str, int], Any, None]: + def _chunk_iter() -> Iterable[Tuple[str, int]]: offset = len(text) while True: offset -= chunk_size * 1024 @@ -351,43 +349,45 @@ else: gc.collect() -class MutableChain(Iterable): +class MutableChain(Iterable[_T]): """ Thin wrapper around itertools.chain, allowing to add iterables "in-place" """ - def __init__(self, *args: Iterable): - self.data = chain.from_iterable(args) + def __init__(self, *args: Iterable[_T]): + self.data: Iterator[_T] = chain.from_iterable(args) - def extend(self, *iterables: Iterable) -> None: + def extend(self, *iterables: Iterable[_T]) -> None: self.data = chain(self.data, chain.from_iterable(iterables)) - def __iter__(self) -> Iterator: + def __iter__(self) -> Iterator[_T]: return self - def __next__(self) -> Any: + def __next__(self) -> _T: return next(self.data) -async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator: +async def _async_chain( + *iterables: Union[Iterable[_T], AsyncIterable[_T]] +) -> AsyncIterator[_T]: for it in iterables: async for o in as_async_generator(it): yield o -class MutableAsyncChain(AsyncIterable): +class MutableAsyncChain(AsyncIterable[_T]): """ Similar to MutableChain but for async iterables """ - def __init__(self, *args: Union[Iterable, AsyncIterable]): - self.data = _async_chain(*args) + def __init__(self, *args: Union[Iterable[_T], AsyncIterable[_T]]): + self.data: AsyncIterator[_T] = _async_chain(*args) - def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None: + def extend(self, *iterables: Union[Iterable[_T], AsyncIterable[_T]]) -> None: self.data = _async_chain(self.data, _async_chain(*iterables)) - def __aiter__(self) -> AsyncIterator: + def __aiter__(self) -> AsyncIterator[_T]: return self - async def __anext__(self) -> Any: + async def __anext__(self) -> _T: return await self.data.__anext__() diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 42a6537a8..45b8008f4 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -12,7 +12,6 @@ from typing import ( TYPE_CHECKING, Any, Dict, - Generator, Iterable, List, Optional, @@ -40,9 +39,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler -def _serialize_headers( - headers: Iterable[bytes], request: Request -) -> Generator[bytes, Any, None]: +def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[bytes]: for header in headers: if header in request.headers: yield header diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index cf429043d..7a91afe59 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -5,7 +5,7 @@ Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ -from typing import Any, Dict, Generator, Iterator, Optional, Union +from typing import Any, Dict, Iterable, Iterator, Optional, Union from urllib.parse import urljoin import lxml.etree # nosec @@ -42,7 +42,7 @@ class Sitemap: def sitemap_urls_from_robots( robots_text: str, base_url: Optional[str] = None -) -> Generator[str, Any, None]: +) -> Iterable[str]: """Return an iterator over all sitemap urls contained in the given robots.txt file """ diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index cbbb01d85..b05135c04 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -7,7 +7,6 @@ from typing import ( TYPE_CHECKING, Any, AsyncGenerator, - Generator, Iterable, Literal, Optional, @@ -34,18 +33,20 @@ _T = TypeVar("_T") # https://stackoverflow.com/questions/60222982 @overload -def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: ... # type: ignore[overload-overlap] +def iterate_spider_output(result: AsyncGenerator[_T, None]) -> AsyncGenerator[_T, None]: ... # type: ignore[overload-overlap] @overload -def iterate_spider_output(result: CoroutineType) -> Deferred: ... +def iterate_spider_output(result: CoroutineType[Any, Any, _T]) -> Deferred[_T]: ... @overload -def iterate_spider_output(result: _T) -> Iterable: ... +def iterate_spider_output(result: _T) -> Iterable[Any]: ... -def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]: +def iterate_spider_output( + result: Any, +) -> Union[Iterable[Any], AsyncGenerator[_T, None], Deferred[_T]]: if inspect.isasyncgen(result): return result if inspect.iscoroutine(result): @@ -55,7 +56,7 @@ def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferr return arg_to_iter(deferred_from_coro(result)) -def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]: +def iter_spider_classes(module: ModuleType) -> Iterable[Type[Spider]]: """Return an iterator over all spider classes defined in the given module that can be instantiated (i.e. which have name) """ diff --git a/tests/test_commands.py b/tests/test_commands.py index b9d468c66..857a56b73 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -13,7 +13,7 @@ from shutil import copytree, rmtree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from tempfile import TemporaryFile, mkdtemp from threading import Timer -from typing import Dict, Generator, Optional, Union +from typing import Dict, Iterator, Optional, Union from unittest import skipIf from pytest import mark @@ -674,7 +674,7 @@ class BadSpider(scrapy.Spider): """ @contextmanager - def _create_file(self, content, name=None) -> Generator[str, None, None]: + def _create_file(self, content, name=None) -> Iterator[str]: tmpdir = Path(self.mktemp()) tmpdir.mkdir() if name: From de146ad7cef9e3478290be021129979f69fc6d03 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 5 Jun 2024 22:09:19 +0500 Subject: [PATCH 209/269] Bump typing deps. --- scrapy/extensions/httpcache.py | 5 ++--- scrapy/http/headers.py | 3 +-- tox.ini | 9 ++++----- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 3f4af42b7..b7219bf07 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -370,12 +370,11 @@ class FilesystemCacheStorage: with self._open(rpath / "pickled_meta", "wb") as f: pickle.dump(metadata, f, protocol=4) with self._open(rpath / "response_headers", "wb") as f: - # headers_dict_to_raw() needs a better type hint - f.write(cast(bytes, headers_dict_to_raw(response.headers))) + f.write(headers_dict_to_raw(response.headers)) with self._open(rpath / "response_body", "wb") as f: f.write(response.body) with self._open(rpath / "request_headers", "wb") as f: - f.write(cast(bytes, headers_dict_to_raw(request.headers))) + f.write(headers_dict_to_raw(request.headers)) with self._open(rpath / "request_body", "wb") as f: f.write(request.body) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 73aee7178..85b9229d3 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -118,8 +118,7 @@ class Headers(CaselessDict): ] def to_string(self) -> bytes: - # cast() can be removed if the headers_dict_to_raw() hint is improved - return cast(bytes, headers_dict_to_raw(self)) + return headers_dict_to_raw(self) def to_unicode_dict(self) -> CaseInsensitiveDict: """Return headers as a CaseInsensitiveDict with str keys diff --git a/tox.ini b/tox.ini index 5a5e80496..023a86c5a 100644 --- a/tox.ini +++ b/tox.ini @@ -47,18 +47,17 @@ install_command = basepython = python3 deps = mypy==1.10.0 - typing-extensions==4.11.0 + typing-extensions==4.12.1 types-lxml==2024.4.14 types-Pygments==2.18.0.20240506 types-pyOpenSSL==24.1.0.20240425 - types-setuptools==69.5.0.20240518 + types-setuptools==70.0.0.20240524 botocore-stubs==1.34.94 - boto3-stubs[s3]==1.34.108 + boto3-stubs[s3]==1.34.119 attrs >= 18.2.0 Pillow >= 10.3.0 pytest >= 8.2.0 - # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 - w3lib >= 2.1.2 + w3lib >= 2.2.0 commands = mypy {posargs: scrapy tests} From 262c10d85bd34732b0c692bdc8d16375d83a178f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 5 Jun 2024 22:11:34 +0500 Subject: [PATCH 210/269] Use typing.Coroutine instead of types.CoroutineType. --- scrapy/commands/parse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index f916a3e75..ce6f4dc51 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -3,11 +3,11 @@ import functools import inspect import json import logging -from types import CoroutineType from typing import ( Any, AsyncGenerator, Callable, + Coroutine, Dict, Iterable, List, @@ -140,7 +140,7 @@ class Command(BaseRunSpiderCommand): @overload def iterate_spider_output( - self, result: Union[AsyncGenerator[_T, None], CoroutineType[Any, Any, _T]] + self, result: Union[AsyncGenerator[_T, None], Coroutine[Any, Any, _T]] ) -> Deferred[_T]: ... @overload From 480a11b68bee19162cc0da59e9bed42b29bc9cfe Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 5 Jun 2024 22:48:16 +0500 Subject: [PATCH 211/269] Add mssing __future__ imports. --- scrapy/commands/parse.py | 2 ++ scrapy/core/spidermw.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index ce6f4dc51..3320a1ee4 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import functools import inspect diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index cb1a93a68..58873f0d9 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -4,6 +4,8 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ +from __future__ import annotations + import logging from inspect import isasyncgenfunction, iscoroutine from itertools import islice From feb0b8f7dcb78c3df012085f00b992a7fac81f7a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 5 Jun 2024 22:57:18 +0500 Subject: [PATCH 212/269] Add pyupgrade. --- .pre-commit-config.yaml | 6 ++ scrapy/core/downloader/contextfactory.py | 8 +-- scrapy/downloadermiddlewares/retry.py | 4 +- scrapy/extensions/debug.py | 2 +- scrapy/http/request/__init__.py | 6 +- scrapy/http/request/rpc.py | 2 +- scrapy/settings/__init__.py | 6 +- scrapy/utils/benchserver.py | 4 +- scrapy/utils/request.py | 6 +- tests/mockserver.py | 6 +- tests/test_downloadermiddleware_cookies.py | 4 +- tests/test_downloadermiddleware_redirect.py | 2 +- tests/test_downloadermiddleware_robotstxt.py | 4 +- tests/test_feedexport.py | 58 ++++++++++---------- tests/test_http_response.py | 4 +- tests/test_pipeline_crawl.py | 4 +- tests/test_pipeline_images.py | 2 +- tests/test_responsetypes.py | 2 +- tests/test_robotstxt_interface.py | 30 ++++------ 19 files changed, 74 insertions(+), 86 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a911d4cfe..f76a04ca1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,3 +22,9 @@ repos: - id: blacken-docs additional_dependencies: - black==24.2.0 +- repo: https://github.com/asottile/pyupgrade + rev: v3.15.2 + hooks: + - id: pyupgrade + args: [--py38-plus, --keep-runtime-typing] + exclude: scrapy/__init__.py diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 0e77cd2fe..9f6edb630 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -107,7 +107,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT return ctx - def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": + def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: return ScrapyClientTLSOptions( hostname.decode("ascii"), self.getContext(), @@ -134,7 +134,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): ``SSLv23_METHOD``) which allows TLS protocol negotiation. """ - def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": + def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: # trustRoot set to platformTrust() will use the platform's root CAs. # # This means that a website like https://www.cacert.org will be rejected @@ -158,8 +158,8 @@ class AcceptableProtocolsContextFactory: self._wrapped_context_factory: Any = context_factory self._acceptable_protocols: List[bytes] = acceptable_protocols - def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": - options: "ClientTLSOptions" = self._wrapped_context_factory.creatorForNetloc( + def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: + options: ClientTLSOptions = self._wrapped_context_factory.creatorForNetloc( hostname, port ) _setAcceptableProtocols(options._ctx, self._acceptable_protocols) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 46587a898..0637f09d4 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -147,9 +147,7 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): if not settings.getbool("RETRY_ENABLED"): raise NotConfigured self.max_retry_times = settings.getint("RETRY_TIMES") - self.retry_http_codes = set( - int(x) for x in settings.getlist("RETRY_HTTP_CODES") - ) + self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")} self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") try: diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 26726b662..a0fc7b99f 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -55,7 +55,7 @@ class StackTraceDump: ) def _thread_stacks(self) -> str: - id2name = dict((th.ident, th.name) for th in threading.enumerate()) + id2name = {th.ident: th.name for th in threading.enumerate()} dumps = "" for id_, frame in sys._current_frames().items(): name = id2name.get(id_, "") diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 77149333c..3da2e111d 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -189,10 +189,10 @@ class Request(object_ref): def __repr__(self) -> str: return f"<{self.method} {self.url}>" - def copy(self) -> "Request": + def copy(self) -> Request: return self.replace() - def replace(self, *args: Any, **kwargs: Any) -> "Request": + def replace(self, *args: Any, **kwargs: Any) -> Request: """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) @@ -237,7 +237,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]: + def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index e20e7c438..096ecd370 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -21,7 +21,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): def __init__(self, *args: Any, encoding: Optional[str] = None, **kwargs: Any): if "body" not in kwargs and "params" in kwargs: - kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs) + kw = {k: kwargs.pop(k) for k in DUMPS_ARGS if k in kwargs} kwargs["body"] = xmlrpclib.dumps(**kw) # spec defines that requests must use POST method diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 4448b6f4b..ea1db03f1 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -275,7 +275,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): assert isinstance(value, (dict, list)) return copy.deepcopy(value) - def getwithbase(self, name: _SettingsKeyT) -> "BaseSettings": + def getwithbase(self, name: _SettingsKeyT) -> BaseSettings: """Get a composition of a dictionary-like setting and its `_BASE` counterpart. @@ -438,7 +438,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): if self.frozen: raise TypeError("Trying to modify an immutable Settings object") - def copy(self) -> "Self": + def copy(self) -> Self: """ Make a deep copy of current settings. @@ -460,7 +460,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ self.frozen = True - def frozencopy(self) -> "Self": + def frozencopy(self) -> Self: """ Return an immutable copy of the current settings. diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index e9ea51aa1..550516141 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -22,9 +22,7 @@ class Root(Resource): for nl in nlist: args["n"] = nl argstr = urlencode(args, doseq=True) - request.write( - f"follow {nl}
".encode("utf8") - ) + request.write(f"follow {nl}
".encode()) request.write(b"") return b"" diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 42a6537a8..aa0b90ee8 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -49,9 +49,9 @@ def _serialize_headers( yield from request.headers.getlist(header) -_fingerprint_cache: ( - "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" -) +_fingerprint_cache: WeakKeyDictionary[ + Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes] +] _fingerprint_cache = WeakKeyDictionary() diff --git a/tests/mockserver.py b/tests/mockserver.py index 647b0682e..233f6b934 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -189,10 +189,10 @@ class Raw(LeafResource): class Echo(LeafResource): def render_GET(self, request): output = { - "headers": dict( - (to_unicode(k), [to_unicode(v) for v in vs]) + "headers": { + to_unicode(k): [to_unicode(v) for v in vs] for k, vs in request.requestHeaders.getAllRawHeaders() - ), + }, "body": to_unicode(request.content.read()), } return to_bytes(json.dumps(output)) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 5eccd396a..6e343d035 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -362,7 +362,7 @@ class CookiesMiddlewareTest(TestCase): def test_request_cookies_encoding(self): # 1) UTF8-encoded bytes - req1 = Request("http://example.org", cookies={"a": "á".encode("utf8")}) + req1 = Request("http://example.org", cookies={"a": "á".encode()}) assert self.mw.process_request(req1, self.spider) is None self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1") @@ -379,7 +379,7 @@ class CookiesMiddlewareTest(TestCase): @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_request_headers_cookie_encoding(self): # 1) UTF8-encoded bytes - req1 = Request("http://example.org", headers={"Cookie": "a=á".encode("utf8")}) + req1 = Request("http://example.org", headers={"Cookie": "a=á".encode()}) assert self.mw.process_request(req1, self.spider) is None self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1") diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 4bfd34fe2..e37da9715 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1125,7 +1125,7 @@ class RedirectMiddlewareTest(Base.Test): def test_utf8_location(self): req = Request("http://scrapytest.org/first") - utf8_location = "/ação".encode("utf-8") # header using UTF-8 encoding + utf8_location = "/ação".encode() # header using UTF-8 encoding resp = Response( "http://scrapytest.org/first", headers={"Location": utf8_location}, diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 26898a6a1..e166cc000 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -40,9 +40,7 @@ Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: Disallow: /wiki/Käyttäjä: User-Agent: UnicödeBöt Disallow: /some/randome/page.html -""".encode( - "utf-8" - ) +""".encode() response = TextResponse("http://site.local/robots.txt", body=ROBOTS) def return_response(request): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d7560b5ff..3771df8f1 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1359,13 +1359,13 @@ class FeedExportTest(FeedExportTestBase): items = [dict({"foo": "Test\xd6"})] formats = { - "json": '[{"foo": "Test\\u00d6"}]'.encode("utf-8"), - "jsonlines": '{"foo": "Test\\u00d6"}\n'.encode("utf-8"), + "json": b'[{"foo": "Test\\u00d6"}]', + "jsonlines": b'{"foo": "Test\\u00d6"}\n', "xml": ( '\n' "Test\xd6" - ).encode("utf-8"), - "csv": "foo\r\nTest\xd6\r\n".encode("utf-8"), + ).encode(), + "csv": "foo\r\nTest\xd6\r\n".encode(), } for fmt, expected in formats.items(): @@ -1379,13 +1379,13 @@ class FeedExportTest(FeedExportTestBase): self.assertEqual(expected, data[fmt]) formats = { - "json": '[{"foo": "Test\xd6"}]'.encode("latin-1"), - "jsonlines": '{"foo": "Test\xd6"}\n'.encode("latin-1"), + "json": b'[{"foo": "Test\xd6"}]', + "jsonlines": b'{"foo": "Test\xd6"}\n', "xml": ( - '\n' - "Test\xd6" - ).encode("latin-1"), - "csv": "foo\r\nTest\xd6\r\n".encode("latin-1"), + b'\n' + b"Test\xd6" + ), + "csv": b"foo\r\nTest\xd6\r\n", } for fmt, expected in formats.items(): @@ -1404,12 +1404,12 @@ class FeedExportTest(FeedExportTestBase): items = [dict({"foo": "FOO", "bar": "BAR"})] formats = { - "json": '[\n{"bar": "BAR"}\n]'.encode("utf-8"), + "json": b'[\n{"bar": "BAR"}\n]', "xml": ( - '\n' - "\n \n FOO\n \n" - ).encode("latin-1"), - "csv": "bar,foo\r\nBAR,FOO\r\n".encode("utf-8"), + b'\n' + b"\n \n FOO\n \n" + ), + "csv": b"bar,foo\r\nBAR,FOO\r\n", } settings = { @@ -1663,8 +1663,8 @@ class FeedExportTest(FeedExportTestBase): def test_extend_kwargs(self): items = [{"foo": "FOO", "bar": "BAR"}] - expected_with_title_csv = "foo,bar\r\nFOO,BAR\r\n".encode("utf-8") - expected_without_title_csv = "FOO,BAR\r\n".encode("utf-8") + expected_with_title_csv = b"foo,bar\r\nFOO,BAR\r\n" + expected_without_title_csv = b"FOO,BAR\r\n" test_cases = [ # with title { @@ -2519,22 +2519,22 @@ class BatchDeliveriesTest(FeedExportTestBase): formats = { "json": [ - '[\n{"bar": "BAR"}\n]'.encode("utf-8"), - '[\n{"bar": "BAR1"}\n]'.encode("utf-8"), + b'[\n{"bar": "BAR"}\n]', + b'[\n{"bar": "BAR1"}\n]', ], "xml": [ ( - '\n' - "\n \n FOO\n \n" - ).encode("latin-1"), + b'\n' + b"\n \n FOO\n \n" + ), ( - '\n' - "\n \n FOO1\n \n" - ).encode("latin-1"), + b'\n' + b"\n \n FOO1\n \n" + ), ], "csv": [ - "foo,bar\r\nFOO,BAR\r\n".encode("utf-8"), - "foo,bar\r\nFOO1,BAR1\r\n".encode("utf-8"), + b"foo,bar\r\nFOO,BAR\r\n", + b"foo,bar\r\nFOO1,BAR1\r\n", ], } @@ -2577,8 +2577,8 @@ class BatchDeliveriesTest(FeedExportTestBase): items = [dict({"foo": "FOO"}), dict({"foo": "FOO1"})] formats = { "json": [ - '[{"foo": "FOO"}]'.encode("utf-8"), - '[{"foo": "FOO1"}]'.encode("utf-8"), + b'[{"foo": "FOO"}]', + b'[{"foo": "FOO1"}]', ], } settings = { diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 80d46274b..b8a277295 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -728,9 +728,7 @@ class TextResponseTest(BaseResponseTest): resp1 = self.response_class( "http://example.com", encoding="utf8", - body='click me'.encode( - "utf8" - ), + body='click me'.encode(), ) req = self._assert_followed_url( resp1.css("a")[0], diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 5a9a217ce..cd3442dd4 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -107,9 +107,7 @@ class FileDownloadCrawlTestCase(TestCase): # check that the images/files checksums are what we know they should be if self.expected_checksums is not None: - checksums = set( - i["checksum"] for item in items for i in item[self.media_key] - ) + checksums = {i["checksum"] for item in items for i in item[self.media_key]} self.assertEqual(checksums, self.expected_checksums) # check that the image files where actually written to the media store diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 18a2454b3..7d7c78920 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -628,7 +628,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): class NoimagesDropTestCase(unittest.TestCase): def test_deprecation_warning(self): - arg = str() + arg = "" with warnings.catch_warnings(record=True) as w: NoimagesDrop(arg) self.assertEqual(len(w), 1) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 2633cca5b..7be8150fc 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -29,7 +29,7 @@ class ResponseTypesTest(unittest.TestCase): mappings = [ (b'attachment; filename="data.xml"', XmlResponse), (b"attachment; filename=data.xml", XmlResponse), - ("attachment;filename=data£.tar.gz".encode("utf-8"), Response), + ("attachment;filename=data£.tar.gz".encode(), Response), ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response), ("attachment;filename=data高.doc".encode("gbk"), Response), ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse), diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 6ad30deed..28ad910a8 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -36,10 +36,10 @@ class BaseRobotParserTest: def test_allowed(self): robotstxt_robotstxt_body = ( - "User-agent: * \n" - "Disallow: /disallowed \n" - "Allow: /allowed \n" - "Crawl-delay: 10".encode("utf-8") + b"User-agent: * \n" + b"Disallow: /disallowed \n" + b"Allow: /allowed \n" + b"Crawl-delay: 10" ) rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body @@ -48,15 +48,13 @@ class BaseRobotParserTest: self.assertFalse(rp.allowed("https://www.site.local/disallowed", "*")) def test_allowed_wildcards(self): - robotstxt_robotstxt_body = """User-agent: first + robotstxt_robotstxt_body = b"""User-agent: first Disallow: /disallowed/*/end$ User-agent: second Allow: /*allowed Disallow: / - """.encode( - "utf-8" - ) + """ rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) @@ -77,18 +75,14 @@ class BaseRobotParserTest: self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second")) def test_length_based_precedence(self): - robotstxt_robotstxt_body = ( - "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8") - ) + robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) self.assertTrue(rp.allowed("https://www.site.local/page", "*")) def test_order_based_precedence(self): - robotstxt_robotstxt_body = ( - "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8") - ) + robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) @@ -123,9 +117,7 @@ class BaseRobotParserTest: Disallow: /wiki/Käyttäjä: User-Agent: UnicödeBöt - Disallow: /some/randome/page.html""".encode( - "utf-8" - ) + Disallow: /some/randome/page.html""".encode() rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) @@ -145,14 +137,14 @@ class BaseRobotParserTest: class DecodeRobotsTxtTest(unittest.TestCase): def test_native_string_conversion(self): - robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8") + robotstxt_body = b"User-agent: *\nDisallow: /\n" decoded_content = decode_robotstxt( robotstxt_body, spider=None, to_native_str_type=True ) self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") def test_decode_utf8(self): - robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8") + robotstxt_body = b"User-agent: *\nDisallow: /\n" decoded_content = decode_robotstxt(robotstxt_body, spider=None) self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") From 144ff6c756fa58da2bc1a85879aa6f89300030d1 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 5 Jun 2024 21:09:10 -0300 Subject: [PATCH 213/269] Document missing parts of response.json method --- docs/topics/dynamic-content.rst | 7 +++---- docs/topics/selectors.rst | 8 ++++++++ scrapy/selector/unified.py | 1 + 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index a0f4b4411..a99f1e222 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -115,15 +115,14 @@ Handling different response formats Once you have a response with the desired data, how you extract the desired data from it depends on the type of response: -- If the response is HTML or XML, use :ref:`selectors +- If the response is HTML, XML or JSON, use :ref:`selectors ` as usual. -- If the response is JSON, use :func:`json.loads` to load the desired data from - :attr:`response.text `: +- If the response is JSON, use :func:`response.json()` to load the desired data: .. code-block:: python - data = json.loads(response.text) + data = response.json() If the desired data is inside HTML or XML code embedded within JSON data, you can load that HTML or XML code into a diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index e32fc2b70..0aae41cc8 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -1060,6 +1060,12 @@ Selector objects For convenience, this method can be called as ``response.css()`` + .. automethod:: jmespath + + .. note:: + + For convenience, this method can be called as ``response.jmespath()`` + .. automethod:: get See also: :ref:`old-extraction-api` @@ -1092,6 +1098,8 @@ SelectorList objects .. automethod:: css + .. automethod:: jmespath + .. automethod:: getall See also: :ref:`old-extraction-api` diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index e852aadc7..bfddb87cb 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -59,6 +59,7 @@ class Selector(_ParselSelector, object_ref): * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type + * ``"json"`` for :class:`~scrapy.http.TextResponse` type * ``"html"`` for anything else Otherwise, if ``type`` is set, the selector type will be forced and no From 23b1214e901961057bf43a5fb2548b35dfe19b20 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 6 Jun 2024 21:44:07 +0500 Subject: [PATCH 214/269] Add a comment about pyupgrade and scrapy/__init__.py. --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f76a04ca1..505b3c57d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,4 +27,5 @@ repos: hooks: - id: pyupgrade args: [--py38-plus, --keep-runtime-typing] + # scrapy/__init__.py has a sys.version_info check we want to keep exclude: scrapy/__init__.py From ed3a7acaf3169ed6b9f9ffbcffed35db63d840f7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Jun 2024 11:19:37 +0500 Subject: [PATCH 215/269] Remove the Python version check from scrapy/__init__.py. --- .pre-commit-config.yaml | 2 -- scrapy/__init__.py | 6 ------ 2 files changed, 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 505b3c57d..63da5544d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,5 +27,3 @@ repos: hooks: - id: pyupgrade args: [--py38-plus, --keep-runtime-typing] - # scrapy/__init__.py has a sys.version_info check we want to keep - exclude: scrapy/__init__.py diff --git a/scrapy/__init__.py b/scrapy/__init__.py index cc0e539c4..1c1a5c2cc 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -33,12 +33,6 @@ version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("." twisted_version = (_txv.major, _txv.minor, _txv.micro) -# Check minimum required Python version -if sys.version_info < (3, 8): - print(f"Scrapy {__version__} requires Python 3.8+") - sys.exit(1) - - # Ignore noisy twisted deprecation warnings warnings.filterwarnings("ignore", category=DeprecationWarning, module="twisted") From ddc98fe91b454a0944a8558daa2000da08921b62 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 10 Jun 2024 13:16:26 +0500 Subject: [PATCH 216/269] Deprecate scrapy.utils.defer.process_chain_both(). (#6397) --- scrapy/utils/defer.py | 9 ++++++++- tests/test_utils_defer.py | 14 -------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index abb7e1726..f60b7dde8 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio import inspect +import warnings from asyncio import Future from functools import wraps from types import CoroutineType @@ -35,7 +36,7 @@ from twisted.internet.task import Cooperator from twisted.python import failure from twisted.python.failure import Failure -from scrapy.exceptions import IgnoreRequest +from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed if TYPE_CHECKING: @@ -281,6 +282,12 @@ def process_chain_both( **kw: _P.kwargs, ) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" + warnings.warn( + "process_chain_both() is deprecated and will be removed in a future" + " Scrapy version.", + ScrapyDeprecationWarning, + stacklevel=2, + ) d: Deferred = Deferred() for cb, eb in zip(callbacks, errbacks): d.addCallback(cb, *a, **kw) diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index a7d54b565..ec0399865 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -14,7 +14,6 @@ from scrapy.utils.defer import ( mustbe_deferred, parallel_async, process_chain, - process_chain_both, process_parallel, ) @@ -80,19 +79,6 @@ class DeferUtilsTest(unittest.TestCase): gotexc = True self.assertTrue(gotexc) - @defer.inlineCallbacks - def test_process_chain_both(self): - x = yield process_chain_both( - [cb_fail, cb2, cb3], [None, eb1, None], "res", "v1", "v2" - ) - self.assertEqual(x, "(cb3 (eb1 TypeError v1 v2) v1 v2)") - - fail = Failure(ZeroDivisionError()) - x = yield process_chain_both( - [eb1, cb2, cb3], [eb1, None, None], fail, "v1", "v2" - ) - self.assertEqual(x, "(cb3 (cb2 (eb1 ZeroDivisionError v1 v2) v1 v2) v1 v2)") - @defer.inlineCallbacks def test_process_parallel(self): x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2") From 1282ddf8f77299edf613679c2ee0b606e96808ce Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 10 Jun 2024 13:27:50 +0500 Subject: [PATCH 217/269] Add parameters to most Deferred in scrapy/core. (#6395) --- scrapy/core/downloader/__init__.py | 45 ++++++++++---- scrapy/core/downloader/handlers/__init__.py | 38 +++++++++--- scrapy/core/downloader/handlers/ftp.py | 10 ++- scrapy/core/downloader/handlers/http10.py | 3 +- scrapy/core/downloader/handlers/http11.py | 69 ++++++++++++--------- scrapy/core/downloader/handlers/http2.py | 4 +- scrapy/core/downloader/handlers/s3.py | 3 +- scrapy/core/downloader/middleware.py | 23 ++++--- scrapy/core/downloader/webclient.py | 6 +- scrapy/core/engine.py | 55 +++++++++------- scrapy/core/scheduler.py | 8 +-- scrapy/core/scraper.py | 61 +++++++++++------- scrapy/core/spidermw.py | 18 +++--- scrapy/pipelines/__init__.py | 4 +- scrapy/utils/defer.py | 20 +++--- tests/test_downloadermiddleware.py | 2 +- 16 files changed, 237 insertions(+), 132 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 0ab3bdb77..41f729ed9 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,9 +1,22 @@ +from __future__ import annotations + import random import warnings from collections import deque from datetime import datetime from time import time -from typing import TYPE_CHECKING, Any, Deque, Dict, Optional, Set, Tuple, cast +from typing import ( + TYPE_CHECKING, + Any, + Deque, + Dict, + Optional, + Set, + Tuple, + TypeVar, + Union, + cast, +) from twisted.internet import task from twisted.internet.defer import Deferred @@ -22,6 +35,8 @@ from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: from scrapy.crawler import Crawler +_T = TypeVar("_T") + class Slot: """Downloader slot""" @@ -40,7 +55,7 @@ class Slot: self.throttle = throttle self.active: Set[Request] = set() - self.queue: Deque[Tuple[Request, Deferred]] = deque() + self.queue: Deque[Tuple[Request, Deferred[Response]]] = deque() self.transferring: Set[Request] = set() self.lastseen: float = 0 self.latercall = None @@ -93,7 +108,7 @@ def _get_concurrency_delay( class Downloader: DOWNLOAD_SLOT = "download_slot" - def __init__(self, crawler: "Crawler"): + def __init__(self, crawler: Crawler): self.settings: BaseSettings = crawler.settings self.signals: SignalManager = crawler.signals self.slots: Dict[str, Slot] = {} @@ -114,13 +129,17 @@ class Downloader: "DOWNLOAD_SLOTS", {} ) - def fetch(self, request: Request, spider: Spider) -> Deferred: - def _deactivate(response: Response) -> Response: + def fetch( + self, request: Request, spider: Spider + ) -> Deferred[Union[Response, Request]]: + def _deactivate(response: _T) -> _T: self.active.remove(request) return response self.active.add(request) - dfd = self.middleware.download(self._enqueue_request, request, spider) + dfd: Deferred[Union[Response, Request]] = self.middleware.download( + self._enqueue_request, request, spider + ) return dfd.addBoth(_deactivate) def needs_backout(self) -> bool: @@ -163,7 +182,7 @@ class Downloader: ) return self.get_slot_key(request) - def _enqueue_request(self, request: Request, spider: Spider) -> Deferred: + def _enqueue_request(self, request: Request, spider: Spider) -> Deferred[Response]: key, slot = self._get_slot(request, spider) request.meta[self.DOWNLOAD_SLOT] = key @@ -175,7 +194,7 @@ class Downloader: self.signals.send_catch_log( signal=signals.request_reached_downloader, request=request, spider=spider ) - deferred: Deferred = Deferred().addBoth(_deactivate) + deferred: Deferred[Response] = Deferred().addBoth(_deactivate) slot.queue.append((request, deferred)) self._process_queue(spider, slot) return deferred @@ -208,11 +227,15 @@ class Downloader: self._process_queue(spider, slot) break - def _download(self, slot: Slot, request: Request, spider: Spider) -> Deferred: + def _download( + self, slot: Slot, request: Request, spider: Spider + ) -> Deferred[Response]: # The order is very important for the following deferreds. Do not change! # 1. Create the download deferred - dfd = mustbe_deferred(self.handlers.download_request, request, spider) + dfd: Deferred[Response] = mustbe_deferred( + self.handlers.download_request, request, spider + ) # 2. Notify response_downloaded listeners about the recent download # before querying queue for next request @@ -233,7 +256,7 @@ class Downloader: # middleware itself) slot.transferring.add(request) - def finish_transferring(_: Any) -> Any: + def finish_transferring(_: _T) -> _T: slot.transferring.remove(request) self._process_queue(spider, slot) self.signals.send_catch_log( diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 5ec5ef6db..ebc4898b5 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -3,13 +3,25 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + Optional, + Protocol, + Type, + Union, + cast, +) from twisted.internet import defer from twisted.internet.defer import Deferred from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured, NotSupported +from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values @@ -20,13 +32,21 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +class DownloadHandlerProtocol(Protocol): + def download_request( + self, request: Request, spider: Spider + ) -> Deferred[Response]: ... + + class DownloadHandlers: def __init__(self, crawler: Crawler): self._crawler: Crawler = crawler self._schemes: Dict[str, Union[str, Callable[..., Any]]] = ( {} ) # stores acceptable schemes on instancing - self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes + self._handlers: Dict[str, DownloadHandlerProtocol] = ( + {} + ) # stores instanced handlers for schemes self._notconfigured: Dict[str, str] = {} # remembers failed handlers handlers: Dict[str, Union[str, Callable[..., Any]]] = without_none_values( cast( @@ -40,7 +60,7 @@ class DownloadHandlers: crawler.signals.connect(self._close, signals.engine_stopped) - def _get_handler(self, scheme: str) -> Any: + def _get_handler(self, scheme: str) -> Optional[DownloadHandlerProtocol]: """Lazy-load the downloadhandler for a scheme only on the first request for that scheme. """ @@ -54,10 +74,12 @@ class DownloadHandlers: return self._load_handler(scheme) - def _load_handler(self, scheme: str, skip_lazy: bool = False) -> Any: + def _load_handler( + self, scheme: str, skip_lazy: bool = False + ) -> Optional[DownloadHandlerProtocol]: path = self._schemes[scheme] try: - dhcls = load_object(path) + dhcls: Type[DownloadHandlerProtocol] = load_object(path) if skip_lazy and getattr(dhcls, "lazy", True): return None dh = build_from_crawler( @@ -80,17 +102,17 @@ class DownloadHandlers: self._handlers[scheme] = dh return dh - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: scheme = urlparse_cached(request).scheme handler = self._get_handler(scheme) if not handler: raise NotSupported( f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}" ) - return cast(Deferred, handler.download_request(request, spider)) + return handler.download_request(request, spider) @defer.inlineCallbacks - def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred, Any, None]: + def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred[Any], Any, None]: for dh in self._handlers.values(): if hasattr(dh, "close"): yield dh.close() diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 77dcf3c38..724717ffd 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -91,7 +91,7 @@ class FTPDownloadHandler: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: from twisted.internet import reactor parsed_url = urlparse_cached(request) @@ -103,10 +103,14 @@ class FTPDownloadHandler: creator = ClientCreator( reactor, FTPClient, user, password, passive=passive_mode ) - dfd: Deferred = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21) + dfd: Deferred[FTPClient] = creator.connectTCP( + parsed_url.hostname, parsed_url.port or 21 + ) return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path)) - def gotClient(self, client: FTPClient, request: Request, filepath: str) -> Deferred: + def gotClient( + self, client: FTPClient, request: Request, filepath: str + ) -> Deferred[Response]: self.client = client protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename")) d = client.retrieveFile(filepath, protocol) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index da9559525..3c4e48abb 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -9,6 +9,7 @@ from twisted.internet.defer import Deferred from scrapy import Request, Spider from scrapy.crawler import Crawler +from scrapy.http import Response from scrapy.settings import BaseSettings from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode @@ -38,7 +39,7 @@ class HTTP10DownloadHandler: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: """Return a deferred for the HTTP download""" factory = self.HTTPClientFactory(request) self._connect(factory) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 5e84be6ba..e2ad8f59a 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,7 +8,7 @@ import re from contextlib import suppress from io import BytesIO from time import time -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, TypeVar, Union from urllib.parse import urldefrag, urlunparse from twisted.internet import ssl @@ -38,12 +38,22 @@ from scrapy.settings import BaseSettings from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: - # typing.Self requires Python 3.11 - from typing_extensions import Self - + # typing.NotRequired and typing.Self require Python 3.11 + from typing_extensions import NotRequired, Self logger = logging.getLogger(__name__) +_T = TypeVar("_T") + + +class _ResultT(TypedDict): + txresponse: TxResponse + body: bytes + flags: Optional[List[str]] + certificate: Optional[ssl.Certificate] + ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None] + failure: NotRequired[Optional[Failure]] + class HTTP11DownloadHandler: lazy = False @@ -71,7 +81,7 @@ class HTTP11DownloadHandler: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: """Return a deferred for the HTTP download""" agent = ScrapyAgent( contextFactory=self._contextFactory, @@ -83,10 +93,10 @@ class HTTP11DownloadHandler: ) return agent.download_request(request) - def close(self) -> Deferred: + def close(self) -> Deferred[None]: from twisted.internet import reactor - d: Deferred = self._pool.closeCachedConnections() + d: Deferred[None] = self._pool.closeCachedConnections() # closeCachedConnections will hang on network or server issues, so # we'll manually timeout the deferred. # @@ -97,7 +107,7 @@ class HTTP11DownloadHandler: # issue a callback after `_disconnect_timeout` seconds. delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, []) - def cancel_delayed_call(result: Any) -> Any: + def cancel_delayed_call(result: _T) -> _T: if delayed_call.active(): delayed_call.cancel() return result @@ -137,7 +147,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): ): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) - self._tunnelReadyDeferred: Deferred = Deferred() + self._tunnelReadyDeferred: Deferred[Protocol] = Deferred() self._tunneledHost: str = host self._tunneledPort: int = port self._contextFactory: IPolicyForHTTPS = contextFactory @@ -198,7 +208,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): """Propagates the errback to the appropriate deferred.""" self._tunnelReadyDeferred.errback(reason) - def connect(self, protocolFactory: Factory) -> Deferred: + def connect(self, protocolFactory: Factory) -> Deferred[Protocol]: self._protocolFactory = protocolFactory connectDeferred = super().connect(protocolFactory) connectDeferred.addCallback(self.requestTunnel) @@ -271,7 +281,7 @@ class TunnelingAgent(Agent): headers: Optional[TxHeaders], bodyProducer: Optional[IBodyProducer], requestPath: bytes, - ) -> Deferred: + ) -> Deferred[TxResponse]: # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy @@ -310,7 +320,7 @@ class ScrapyProxyAgent(Agent): uri: bytes, headers: Optional[TxHeaders] = None, bodyProducer: Optional[IBodyProducer] = None, - ) -> Deferred: + ) -> Deferred[TxResponse]: """ Issue a new request via the configured proxy. """ @@ -394,7 +404,7 @@ class ScrapyAgent: pool=self._pool, ) - def download_request(self, request: Request) -> Deferred: + def download_request(self, request: Request) -> Deferred[Response]: from twisted.internet import reactor timeout = request.meta.get("download_timeout") or self._connectTimeout @@ -411,22 +421,20 @@ class ScrapyAgent: else: bodyproducer = None start_time = time() - d: Deferred = agent.request( + d: Deferred[TxResponse] = agent.request( method, to_bytes(url, encoding="ascii"), headers, bodyproducer ) # set download latency d.addCallback(self._cb_latency, request, start_time) # response body is ready to be consumed - d.addCallback(self._cb_bodyready, request) - d.addCallback(self._cb_bodydone, request, url) + d2: Deferred[_ResultT] = d.addCallback(self._cb_bodyready, request) + d3: Deferred[Response] = d2.addCallback(self._cb_bodydone, request, url) # check download timeout - self._timeout_cl = reactor.callLater(timeout, d.cancel) - d.addBoth(self._cb_timeout, request, url, timeout) - return d + self._timeout_cl = reactor.callLater(timeout, d3.cancel) + d3.addBoth(self._cb_timeout, request, url, timeout) + return d3 - def _cb_timeout( - self, result: Any, request: Request, url: str, timeout: float - ) -> Any: + def _cb_timeout(self, result: _T, request: Request, url: str, timeout: float) -> _T: if self._timeout_cl.active(): self._timeout_cl.cancel() return result @@ -437,7 +445,7 @@ class ScrapyAgent: raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") - def _cb_latency(self, result: Any, request: Request, start_time: float) -> Any: + def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T: request.meta["download_latency"] = time() - start_time return result @@ -451,7 +459,7 @@ class ScrapyAgent: def _cb_bodyready( self, txresponse: TxResponse, request: Request - ) -> Union[Dict[str, Any], Deferred]: + ) -> Union[_ResultT, Deferred[_ResultT]]: headers_received_result = self._crawler.signals.send_catch_log( signal=signals.headers_received, headers=self._headers_from_twisted_response(txresponse), @@ -520,7 +528,7 @@ class ScrapyAgent: # Abort connection immediately. txresponse._transport._producer.abortConnection() - d: Deferred = Deferred(_cancel) + d: Deferred[_ResultT] = Deferred(_cancel) txresponse.deliverBody( _ResponseReader( finished=d, @@ -539,7 +547,7 @@ class ScrapyAgent: return d def _cb_bodydone( - self, result: Dict[str, Any], request: Request, url: str + self, result: _ResultT, request: Request, url: str ) -> Union[Response, Failure]: headers = self._headers_from_twisted_response(result["txresponse"]) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) @@ -559,8 +567,9 @@ class ScrapyAgent: protocol=protocol, ) if result.get("failure"): + assert result["failure"] result["failure"].value.response = response - return cast(Failure, result["failure"]) + return result["failure"] return response @@ -570,7 +579,7 @@ class _RequestBodyProducer: self.body = body self.length = len(body) - def startProducing(self, consumer: IConsumer) -> Deferred: + def startProducing(self, consumer: IConsumer) -> Deferred[None]: consumer.write(self.body) return succeed(None) @@ -584,7 +593,7 @@ class _RequestBodyProducer: class _ResponseReader(Protocol): def __init__( self, - finished: Deferred, + finished: Deferred[_ResultT], txresponse: TxResponse, request: Request, maxsize: int, @@ -592,7 +601,7 @@ class _ResponseReader(Protocol): fail_on_dataloss: bool, crawler: Crawler, ): - self._finished: Deferred = finished + self._finished: Deferred[_ResultT] = finished self._txresponse: TxResponse = txresponse self._request: Request = request self._bodybuf: BytesIO = BytesIO() diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 16fc1e3ae..2ac4eca86 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -37,7 +37,7 @@ class H2DownloadHandler: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: agent = ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, @@ -98,7 +98,7 @@ class ScrapyH2Agent: pool=self._pool, ) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: from twisted.internet import reactor timeout = request.meta.get("download_timeout") or self._connect_timeout diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 1a3d36f45..0ad340721 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -8,6 +8,7 @@ from scrapy import Request, Spider from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response from scrapy.settings import BaseSettings from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached @@ -76,7 +77,7 @@ class S3DownloadHandler: def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self: return cls(crawler.settings, crawler=crawler, **kwargs) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: p = urlparse_cached(request) scheme = "https" if request.meta.get("is_secure") else "http" bucket = p.hostname diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 52ebe4e22..2d8af114f 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -4,6 +4,8 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ +from __future__ import annotations + from typing import Any, Callable, Generator, List, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks @@ -34,10 +36,15 @@ class DownloaderMiddlewareManager(MiddlewareManager): self.methods["process_exception"].appendleft(mw.process_exception) def download( - self, download_func: Callable, request: Request, spider: Spider - ) -> Deferred: + self, + download_func: Callable[[Request, Spider], Deferred[Response]], + request: Request, + spider: Spider, + ) -> Deferred[Union[Response, Request]]: @inlineCallbacks - def process_request(request: Request) -> Generator[Deferred, Any, Any]: + def process_request( + request: Request, + ) -> Generator[Deferred[Any], Any, Union[Response, Request]]: for method in self.methods["process_request"]: method = cast(Callable, method) response = yield deferred_from_coro( @@ -52,12 +59,12 @@ class DownloaderMiddlewareManager(MiddlewareManager): ) if response: return response - return (yield download_func(request=request, spider=spider)) + return (yield download_func(request, spider)) @inlineCallbacks def process_response( response: Union[Response, Request] - ) -> Generator[Deferred, Any, Union[Response, Request]]: + ) -> Generator[Deferred[Any], Any, Union[Response, Request]]: if response is None: raise TypeError("Received None in process_response") elif isinstance(response, Request): @@ -80,7 +87,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): @inlineCallbacks def process_exception( failure: Failure, - ) -> Generator[Deferred, Any, Union[Failure, Response, Request]]: + ) -> Generator[Deferred[Any], Any, Union[Failure, Response, Request]]: exception = failure.value for method in self.methods["process_exception"]: method = cast(Callable, method) @@ -98,7 +105,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response return failure - deferred = mustbe_deferred(process_request, request) + deferred: Deferred[Union[Response, Request]] = mustbe_deferred( + process_request, request + ) deferred.addErrback(process_exception) deferred.addCallback(process_response) return deferred diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index bb1f73805..08a1d7c71 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -8,7 +8,7 @@ from twisted.internet.protocol import ClientFactory from twisted.web.http import HTTPClient from scrapy import Request -from scrapy.http import Headers +from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode @@ -145,7 +145,7 @@ class ScrapyHTTPClientFactory(ClientFactory): self.response_headers: Optional[Headers] = None self.timeout: float = request.meta.get("download_timeout") or timeout self.start_time: float = time() - self.deferred: defer.Deferred = defer.Deferred().addCallback( + self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback( self._build_response, request ) @@ -155,7 +155,7 @@ class ScrapyHTTPClientFactory(ClientFactory): # needed to add the callback _waitForDisconnect. # Specifically this avoids the AttributeError exception when # clientConnectionFailed method is called. - self._disconnectedDeferred: defer.Deferred = defer.Deferred() + self._disconnectedDeferred: defer.Deferred[None] = defer.Deferred() self._set_connection_attributes(request) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index dededf99d..4ffec78b9 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -19,6 +19,7 @@ from typing import ( Optional, Set, Type, + TypeVar, Union, cast, ) @@ -43,10 +44,13 @@ from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: from scrapy.core.scheduler import BaseScheduler + from scrapy.core.scraper import _HandleOutputDeferred from scrapy.crawler import Crawler logger = logging.getLogger(__name__) +_T = TypeVar("_T") + class Slot: def __init__( @@ -56,7 +60,7 @@ class Slot: nextcall: CallLaterOnce[None], scheduler: BaseScheduler, ) -> None: - self.closing: Optional[Deferred] = None + self.closing: Optional[Deferred[None]] = None self.inprogress: Set[Request] = set() self.start_requests: Optional[Iterator[Request]] = iter(start_requests) self.close_if_idle: bool = close_if_idle @@ -71,7 +75,7 @@ class Slot: self.inprogress.remove(request) self._maybe_fire_closing() - def close(self) -> Deferred: + def close(self) -> Deferred[None]: self.closing = Deferred() self._maybe_fire_closing() return self.closing @@ -123,20 +127,20 @@ class ExecutionEngine: return scheduler_cls @inlineCallbacks - def start(self) -> Generator[Deferred, Any, None]: + def start(self) -> Generator[Deferred[Any], Any, None]: if self.running: raise RuntimeError("Engine already running") self.start_time = time() yield self.signals.send_catch_log_deferred(signal=signals.engine_started) self.running = True - self._closewait: Deferred = Deferred() + self._closewait: Deferred[None] = Deferred() yield self._closewait - def stop(self) -> Deferred: + def stop(self) -> Deferred[None]: """Gracefully stop the execution engine""" @inlineCallbacks - def _finish_stopping_engine(_: Any) -> Generator[Deferred, Any, None]: + def _finish_stopping_engine(_: Any) -> Generator[Deferred[Any], Any, None]: yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) self._closewait.callback(None) @@ -151,7 +155,7 @@ class ExecutionEngine: ) return dfd.addBoth(_finish_stopping_engine) - def close(self) -> Deferred: + def close(self) -> Deferred[None]: """ Gracefully close the execution engine. If it has already been started, stop it. In all cases, close the spider and the downloader. @@ -214,7 +218,7 @@ class ExecutionEngine: or self.scraper.slot.needs_backout() ) - def _next_request_from_scheduler(self) -> Optional[Deferred]: + def _next_request_from_scheduler(self) -> Optional[Deferred[None]]: assert self.slot is not None # typing assert self.spider is not None # typing @@ -222,7 +226,7 @@ class ExecutionEngine: if request is None: return None - d = self._download(request) + d: Deferred[Union[Response, Request]] = self._download(request) d.addBoth(self._handle_downloader_output, request) d.addErrback( lambda f: logger.info( @@ -236,8 +240,8 @@ class ExecutionEngine: assert self.slot self.slot.remove_request(request) - d.addBoth(_remove_request) - d.addErrback( + d2: Deferred[None] = d.addBoth(_remove_request) + d2.addErrback( lambda f: logger.info( "Error while removing request from slot", exc_info=failure_to_exc_info(f), @@ -245,19 +249,19 @@ class ExecutionEngine: ) ) slot = self.slot - d.addBoth(lambda _: slot.nextcall.schedule()) - d.addErrback( + d2.addBoth(lambda _: slot.nextcall.schedule()) + d2.addErrback( lambda f: logger.info( "Error while scheduling new request", exc_info=failure_to_exc_info(f), extra={"spider": self.spider}, ) ) - return d + return d2 def _handle_downloader_output( self, result: Union[Request, Response, Failure], request: Request - ) -> Optional[Deferred]: + ) -> Optional[_HandleOutputDeferred]: assert self.spider is not None # typing if not isinstance(result, (Request, Response, Failure)): @@ -319,20 +323,23 @@ class ExecutionEngine: signals.request_dropped, request=request, spider=spider ) - def download(self, request: Request) -> Deferred: + def download(self, request: Request) -> Deferred[Response]: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - return self._download(request).addBoth(self._downloaded, request) + d: Deferred[Union[Response, Request]] = self._download(request) + # Deferred.addBoth() overloads don't seem to support a Union[_T, Deferred[_T]] return type + d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[arg-type] + return d2 def _downloaded( self, result: Union[Response, Request, Failure], request: Request - ) -> Union[Deferred, Response, Failure]: + ) -> Union[Deferred[Response], Response, Failure]: assert self.slot is not None # typing self.slot.remove_request(request) return self.download(result) if isinstance(result, Request) else result - def _download(self, request: Request) -> Deferred: + def _download(self, request: Request) -> Deferred[Union[Response, Request]]: assert self.slot is not None # typing self.slot.add_request(request) @@ -359,13 +366,15 @@ class ExecutionEngine: ) return result - def _on_complete(_: Any) -> Any: + def _on_complete(_: _T) -> _T: assert self.slot is not None self.slot.nextcall.schedule() return _ assert self.spider is not None - dwld = self.downloader.fetch(request, self.spider) + dwld: Deferred[Union[Response, Request]] = self.downloader.fetch( + request, self.spider + ) dwld.addCallback(_on_success) dwld.addBoth(_on_complete) return dwld @@ -376,7 +385,7 @@ class ExecutionEngine: spider: Spider, start_requests: Iterable[Request] = (), close_if_idle: bool = True, - ) -> Generator[Deferred, Any, None]: + ) -> Generator[Deferred[Any], Any, None]: if self.slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={"spider": spider}) @@ -422,7 +431,7 @@ class ExecutionEngine: assert isinstance(ex, CloseSpider) # typing self.close_spider(self.spider, reason=ex.reason) - def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred: + def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred[None]: """Close (cancel) spider and clear all its outstanding requests""" if self.slot is None: raise RuntimeError("Engine slot not assigned") diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index e3b95e977..1e586c53a 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -71,7 +71,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): """ return cls() - def open(self, spider: Spider) -> Optional[Deferred]: + def open(self, spider: Spider) -> Optional[Deferred[None]]: """ Called when the spider is opened by the engine. It receives the spider instance as argument and it's useful to execute initialization code. @@ -81,7 +81,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): """ pass - def close(self, reason: str) -> Optional[Deferred]: + def close(self, reason: str) -> Optional[Deferred[None]]: """ Called when the spider is closed by the engine. It receives the reason why the crawl finished as argument and it's useful to execute cleaning code. @@ -216,7 +216,7 @@ class Scheduler(BaseScheduler): def has_pending_requests(self) -> bool: return len(self) > 0 - def open(self, spider: Spider) -> Optional[Deferred]: + def open(self, spider: Spider) -> Optional[Deferred[None]]: """ (1) initialize the memory queue (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory @@ -227,7 +227,7 @@ class Scheduler(BaseScheduler): self.dqs: Optional[ScrapyPriorityQueue] = self._dq() if self.dqdir else None return self.df.open() - def close(self, reason: str) -> Optional[Deferred]: + def close(self, reason: str) -> Optional[Deferred[None]]: """ (1) dump pending requests to disk if there is a disk queue (2) return the result of the dupefilter's ``close`` method diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 3b7492838..8a9e8f687 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -12,6 +12,7 @@ from typing import ( Deque, Generator, Iterable, + Iterator, Optional, Set, Tuple, @@ -33,6 +34,7 @@ from scrapy.logformatter import LogFormatter from scrapy.pipelines import ItemPipelineManager from scrapy.signalmanager import SignalManager from scrapy.utils.defer import ( + DeferredListResultListT, aiter_errback, defer_fail, defer_succeed, @@ -48,13 +50,18 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler -_T = TypeVar("_T") -QueueTuple = Tuple[Union[Response, Failure], Request, Deferred] - - logger = logging.getLogger(__name__) +_T = TypeVar("_T") +_ParallelResult = DeferredListResultListT[Iterator[Any]] + +if TYPE_CHECKING: + # parameterized Deferreds require Twisted 21.7.0 + _HandleOutputDeferred = Deferred[Union[_ParallelResult, None]] + QueueTuple = Tuple[Union[Response, Failure], Request, _HandleOutputDeferred] + + class Slot: """Scraper slot (one per running spider)""" @@ -66,12 +73,12 @@ class Slot: self.active: Set[Request] = set() self.active_size: int = 0 self.itemproc_size: int = 0 - self.closing: Optional[Deferred] = None + self.closing: Optional[Deferred[Spider]] = None def add_response_request( self, result: Union[Response, Failure], request: Request - ) -> Deferred: - deferred: Deferred = Deferred() + ) -> _HandleOutputDeferred: + deferred: _HandleOutputDeferred = Deferred() self.queue.append((result, request, deferred)) if isinstance(result, Response): self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE) @@ -117,12 +124,12 @@ class Scraper: self.logformatter: LogFormatter = crawler.logformatter @inlineCallbacks - def open_spider(self, spider: Spider) -> Generator[Deferred, Any, None]: + def open_spider(self, spider: Spider) -> Generator[Deferred[Any], Any, None]: """Open the given spider for scraping and allocate resources for it""" self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE")) yield self.itemproc.open_spider(spider) - def close_spider(self, spider: Spider) -> Deferred: + def close_spider(self, spider: Spider) -> Deferred[Spider]: """Close a spider being scraped and release its resources""" if self.slot is None: raise RuntimeError("Scraper slot not assigned") @@ -142,12 +149,12 @@ class Scraper: def enqueue_scrape( self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred: + ) -> _HandleOutputDeferred: if self.slot is None: raise RuntimeError("Scraper slot not assigned") dfd = self.slot.add_response_request(result, request) - def finish_scraping(_: Any) -> Any: + def finish_scraping(_: _T) -> _T: assert self.slot is not None self.slot.finish_response(result, request) self._check_if_closing(spider) @@ -174,7 +181,7 @@ class Scraper: def _scrape( self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred: + ) -> _HandleOutputDeferred: """ Handle the downloaded response or failure through the spider callback/errback """ @@ -182,32 +189,35 @@ class Scraper: raise TypeError( f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}" ) - dfd = self._scrape2( + dfd: Deferred[Union[Iterable[Any], AsyncIterable[Any]]] = self._scrape2( result, request, spider ) # returns spider's processed output dfd.addErrback(self.handle_spider_error, request, result, spider) - dfd.addCallback( + dfd2: _HandleOutputDeferred = dfd.addCallback( self.handle_spider_output, request, cast(Response, result), spider ) - return dfd + return dfd2 def _scrape2( self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred: + ) -> Deferred[Union[Iterable[Any], AsyncIterable[Any]]]: """ Handle the different cases of request's result been a Response or a Failure """ if isinstance(result, Response): - return self.spidermw.scrape_response( + # Deferreds are invariant so Mutable*Chain isn't matched to *Iterable + return self.spidermw.scrape_response( # type: ignore[return-value] self.call_spider, result, request, spider ) # else result is a Failure dfd = self.call_spider(result, request, spider) - return dfd.addErrback(self._log_download_errors, result, request, spider) + dfd.addErrback(self._log_download_errors, result, request, spider) + return dfd def call_spider( self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred: + ) -> Deferred[Union[Iterable[Any], AsyncIterable[Any]]]: + dfd: Deferred[Any] if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request @@ -225,7 +235,10 @@ class Scraper: if request.errback: warn_on_generator_with_return_value(spider, request.errback) dfd.addErrback(request.errback) - return dfd.addCallback(iterate_spider_output) + dfd2: Deferred[Union[Iterable[Any], AsyncIterable[Any]]] = dfd.addCallback( + iterate_spider_output + ) + return dfd2 def handle_spider_error( self, @@ -262,10 +275,11 @@ class Scraper: request: Request, response: Response, spider: Spider, - ) -> Deferred: + ) -> _HandleOutputDeferred: if not result: return defer_succeed(None) it: Union[Iterable[_T], AsyncIterable[_T]] + dfd: Deferred[_ParallelResult] if isinstance(result, AsyncIterable): it = aiter_errback( result, self.handle_spider_error, request, response, spider @@ -290,11 +304,12 @@ class Scraper: response, spider, ) - return dfd + # returning Deferred[_ParallelResult] instead of Deferred[Union[_ParallelResult, None]] + return dfd # type: ignore[return-value] def _process_spidermw_output( self, output: Any, request: Request, response: Response, spider: Spider - ) -> Optional[Deferred]: + ) -> Optional[Deferred[Any]]: """Process each Request/Item (given in the output parameter) returned from the given spider """ diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 58873f0d9..e792f8ca7 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -45,7 +45,9 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] +ScrapeFunc = Callable[ + [Union[Response, Failure], Request, Spider], Union[Iterable[_T], AsyncIterable[_T]] +] def _isiterable(o: Any) -> bool: @@ -80,7 +82,7 @@ class SpiderMiddlewareManager(MiddlewareManager): response: Response, request: Request, spider: Spider, - ) -> Any: + ) -> Union[Iterable[_T], AsyncIterable[_T]]: for method in self.methods["process_spider_input"]: method = cast(Callable, method) try: @@ -311,7 +313,7 @@ class SpiderMiddlewareManager(MiddlewareManager): response: Response, request: Request, spider: Spider, - ) -> Deferred: + ) -> Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]]: async def process_callback_output( result: Union[Iterable[_T], AsyncIterable[_T]] ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]: @@ -322,12 +324,14 @@ class SpiderMiddlewareManager(MiddlewareManager): ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]: return self._process_spider_exception(response, spider, _failure) - dfd: Deferred = mustbe_deferred( + dfd: Deferred[Union[Iterable[_T], AsyncIterable[_T]]] = mustbe_deferred( self._process_spider_input, scrape_func, response, request, spider ) - dfd.addCallback(deferred_f_from_coro_f(process_callback_output)) - dfd.addErrback(process_spider_exception) - return dfd + dfd2: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = ( + dfd.addCallback(deferred_f_from_coro_f(process_callback_output)) + ) + dfd2.addErrback(process_spider_exception) + return dfd2 def process_start_requests( self, start_requests: Iterable[Request], spider: Spider diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 0cfbc156f..21d649e3c 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -4,6 +4,8 @@ Item pipeline See documentation in docs/item-pipeline.rst """ +from __future__ import annotations + from typing import Any, List from twisted.internet.defer import Deferred @@ -29,5 +31,5 @@ class ItemPipelineManager(MiddlewareManager): deferred_f_from_coro_f(pipe.process_item) ) - def process_item(self, item: Any, spider: Spider) -> Deferred: + def process_item(self, item: Any, spider: Spider) -> Deferred[Any]: return self._process_chain("process_item", item, spider) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index f60b7dde8..ddb68c86b 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -46,6 +46,12 @@ if TYPE_CHECKING: _P = ParamSpec("_P") _T = TypeVar("_T") +_T2 = TypeVar("_T2") + +# copied from twisted.internet.defer +_SelfResultT = TypeVar("_SelfResultT") +_DeferredListResultItemT = Tuple[bool, _SelfResultT] +DeferredListResultListT = List[_DeferredListResultItemT[_SelfResultT]] def defer_fail(_failure: Failure) -> Deferred: @@ -62,7 +68,7 @@ def defer_fail(_failure: Failure) -> Deferred: return d -def defer_succeed(result: Any) -> Deferred: +def defer_succeed(result: _T) -> Deferred[_T]: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop @@ -128,10 +134,10 @@ def mustbe_deferred( def parallel( iterable: Iterable[_T], count: int, - callable: Callable[Concatenate[_T, _P], Any], + callable: Callable[Concatenate[_T, _P], _T2], *args: _P.args, **named: _P.kwargs, -) -> Deferred: +) -> Deferred[DeferredListResultListT[Iterator[_T2]]]: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -191,12 +197,12 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]): def __init__( self, aiterable: AsyncIterable[_T], - callable: Callable[Concatenate[_T, _P], Any], + callable: Callable[Concatenate[_T, _P], _T2], *callable_args: _P.args, **callable_kwargs: _P.kwargs, ): self.aiterator: AsyncIterator[_T] = aiterable.__aiter__() - self.callable: Callable[Concatenate[_T, _P], Any] = callable + self.callable: Callable[Concatenate[_T, _P], _T2] = callable self.callable_args: Tuple[Any, ...] = callable_args self.callable_kwargs: Dict[str, Any] = callable_kwargs self.finished: bool = False @@ -249,10 +255,10 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]): def parallel_async( async_iterable: AsyncIterable[_T], count: int, - callable: Callable[Concatenate[_T, _P], Any], + callable: Callable[Concatenate[_T, _P], _T2], *args: _P.args, **named: _P.kwargs, -) -> Deferred: +) -> Deferred[DeferredListResultListT[Iterator[_T2]]]: """Like parallel but for async iterators""" coop = Cooperator() work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 0155c62eb..dd3f8ceb9 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -36,7 +36,7 @@ class ManagerTestCase(TestCase): if not response: response = Response(request.url) - def download_func(**kwargs): + def download_func(request, spider): return response dfd = self.mwman.download(download_func, request, self.spider) From 365c9e62ad9e99725eb1898cbd2806c63105cd58 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Jun 2024 14:37:11 +0500 Subject: [PATCH 218/269] Removing empty example reference (#6402) Co-authored-by: Michael Duane Mooring --- docs/topics/link-extractors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 1201c926d..f9744ed16 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -85,7 +85,7 @@ LxmlLinkExtractor :param restrict_xpaths: is an XPath (or list of XPath's) which defines regions inside the response where links should be extracted from. If given, only the text selected by those XPath will be scanned for - links. See examples below. + links. :type restrict_xpaths: str or list :param restrict_css: a CSS selector (or list of selectors) which defines From a364560fadbbc0dd7cca78670bbd9d3c00d4d366 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Jun 2024 14:38:10 +0500 Subject: [PATCH 219/269] Unpin markupsafe in extra-deps. (#6403) --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index 023a86c5a..d665fc5a5 100644 --- a/tox.ini +++ b/tox.ini @@ -147,9 +147,6 @@ deps = {[testenv]deps} boto3 google-cloud-storage - # Twisted[http2] currently forces old mitmproxy because of h2 version - # restrictions in their deps, so we need to pin old markupsafe here too. - markupsafe < 2.1.0 robotexclusionrulesparser Pillow Twisted[http2] From d13219062500eae1a6d5330ceea3502590cd89cb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 19 Jun 2024 23:26:25 +0500 Subject: [PATCH 220/269] flake8-debugger --- .pre-commit-config.yaml | 2 ++ scrapy/extensions/debug.py | 2 +- scrapy/utils/console.py | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 63da5544d..eb3404b7f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,6 +8,8 @@ repos: rev: 7.0.0 hooks: - id: flake8 + additional_dependencies: + - flake8-debugger - repo: https://github.com/psf/black.git rev: 24.2.0 hooks: diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index a0fc7b99f..b360ce48d 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -74,4 +74,4 @@ class Debugger: def _enter_debugger(self, signum: int, frame: Optional[FrameType]) -> None: assert frame - Pdb().set_trace(frame.f_back) + Pdb().set_trace(frame.f_back) # noqa: T100 diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index bf1803115..328219831 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -10,10 +10,10 @@ def _embed_ipython_shell( ) -> EmbedFuncT: """Start an IPython Shell""" try: - from IPython.terminal.embed import InteractiveShellEmbed + from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100 from IPython.terminal.ipapp import load_default_config except ImportError: - from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] + from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] # noqa: T100 InteractiveShellEmbed, ) from IPython.frontend.terminal.ipapp import ( # type: ignore[no-redef] From a617e04d2eb89b64f15df7a6a0326bfaf57f8dde Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 19 Jun 2024 23:28:58 +0500 Subject: [PATCH 221/269] flake8-string-format --- .flake8 | 8 ++++++-- .pre-commit-config.yaml | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.flake8 b/.flake8 index cf1a96476..b6048c9ee 100644 --- a/.flake8 +++ b/.flake8 @@ -1,8 +1,12 @@ [flake8] max-line-length = 119 -ignore = E203, E501, E701, E704, W503 - +ignore = + E203, E501, E701, E704, W503 + # docstring does contain unindexed parameters + P102 + # other string does contain unindexed parameters + P103 exclude = docs/conf.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eb3404b7f..47a3df53d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,6 +10,7 @@ repos: - id: flake8 additional_dependencies: - flake8-debugger + - flake8-string-format - repo: https://github.com/psf/black.git rev: 24.2.0 hooks: From 1c70d3e60555084b4bec9dfd794adb93b24b2171 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 19 Jun 2024 23:36:36 +0500 Subject: [PATCH 222/269] flake8-comprehensions --- .pre-commit-config.yaml | 1 + scrapy/extensions/feedexport.py | 2 +- tests/test_commands.py | 2 +- tests/test_feedexport.py | 10 +++++----- tests/test_loader.py | 6 ++---- tests/test_loader_deprecated.py | 4 ++-- tests/test_scheduler.py | 2 +- tests/test_spider.py | 18 +++++++++--------- 8 files changed, 22 insertions(+), 23 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 47a3df53d..974d397c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,7 @@ repos: hooks: - id: flake8 additional_dependencies: + - flake8-comprehensions - flake8-debugger - flake8-string-format - repo: https://github.com/psf/black.git diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index de8a288f6..941bd4b26 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -104,7 +104,7 @@ class ItemFilter: for item_class in feed_options.get("item_classes") or () ) else: - self.item_classes = tuple() + self.item_classes = () def accepts(self, item: Any) -> bool: """ diff --git a/tests/test_commands.py b/tests/test_commands.py index 857a56b73..d829b1701 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -200,7 +200,7 @@ def get_permissions_dict( path_obj = Path(path) - renamings = renamings or tuple() + renamings = renamings or () permissions_dict = { ".": get_permissions(path_obj), } diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 3771df8f1..253987e15 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1356,7 +1356,7 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_encoding(self): - items = [dict({"foo": "Test\xd6"})] + items = [{"foo": "Test\xd6"}] formats = { "json": b'[{"foo": "Test\\u00d6"}]', @@ -1401,7 +1401,7 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_configs(self): - items = [dict({"foo": "FOO", "bar": "BAR"})] + items = [{"foo": "FOO", "bar": "BAR"}] formats = { "json": b'[\n{"bar": "BAR"}\n]', @@ -2513,8 +2513,8 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_configs(self): items = [ - dict({"foo": "FOO", "bar": "BAR"}), - dict({"foo": "FOO1", "bar": "BAR1"}), + {"foo": "FOO", "bar": "BAR"}, + {"foo": "FOO1", "bar": "BAR1"}, ] formats = { @@ -2574,7 +2574,7 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_batch_item_count_feeds_setting(self): - items = [dict({"foo": "FOO"}), dict({"foo": "FOO1"})] + items = [{"foo": "FOO"}, {"foo": "FOO1"}] formats = { "json": [ b'[{"foo": "FOO"}]', diff --git a/tests/test_loader.py b/tests/test_loader.py index b0b7f8723..8db929dcf 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -156,7 +156,7 @@ class InitializationTestMixin: self.assertEqual(il.get_output_value("name"), ["foo"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({"name": ["foo"]})) + self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo"]}) def test_get_output_value_list(self): """Getting output value must not remove value from item""" @@ -165,9 +165,7 @@ class InitializationTestMixin: self.assertEqual(il.get_output_value("name"), ["foo", "bar"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual( - ItemAdapter(loaded_item).asdict(), dict({"name": ["foo", "bar"]}) - ) + self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]}) def test_values_single(self): """Values from initial item must be added to loader._values""" diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 528efa142..0d245bec9 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -526,7 +526,7 @@ class InitializationFromDictTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["foo"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, dict({"name": ["foo"]})) + self.assertEqual(loaded_item, {"name": ["foo"]}) def test_get_output_value_list(self): """Getting output value must not remove value from item""" @@ -535,7 +535,7 @@ class InitializationFromDictTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["foo", "bar"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, dict({"name": ["foo", "bar"]})) + self.assertEqual(loaded_item, {"name": ["foo", "bar"]}) def test_values_single(self): """Values from initial item must be added to loader._values""" diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 02b50baa3..9b7bad4bf 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -284,7 +284,7 @@ class DownloaderAwareSchedulerTestMixin: downloader.decrement(slot) self.assertTrue( - _is_scheduling_fair(list(s for u, s in _URLS_WITH_SLOTS), dequeued_slots) + _is_scheduling_fair([s for u, s in _URLS_WITH_SLOTS], dequeued_slots) ) self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0) diff --git a/tests/test_spider.py b/tests/test_spider.py index d629d33af..18a863350 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -244,7 +244,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -270,7 +270,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -299,7 +299,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 2) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -324,7 +324,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -352,7 +352,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -383,7 +383,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -413,7 +413,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -445,7 +445,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -637,7 +637,7 @@ Sitemap: /sitemap-relative-url.xml class FilteredSitemapSpider(self.spider_class): def sitemap_filter(self, entries): for entry in entries: - alternate_links = entry.get("alternate", tuple()) + alternate_links = entry.get("alternate", ()) for link in alternate_links: if "/deutsch/" in link: entry["loc"] = link From 1ef9c337cad36ac6c80eab86622f8ae9fc8d1075 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 19 Jun 2024 23:57:40 +0500 Subject: [PATCH 223/269] flake8-docstrings --- .flake8 | 33 +++++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 1 + tests/test_dupefilters.py | 2 +- tests/test_linkextractors.py | 2 +- 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/.flake8 b/.flake8 index b6048c9ee..222ba7179 100644 --- a/.flake8 +++ b/.flake8 @@ -2,11 +2,44 @@ max-line-length = 119 ignore = + # black disagrees with flake8 about these E203, E501, E701, E704, W503 # docstring does contain unindexed parameters P102 # other string does contain unindexed parameters P103 + # Missing docstring in public module + D100 + # Missing docstring in public class + D101 + # Missing docstring in public method + D102 + # Missing docstring in public function + D103 + # Missing docstring in public package + D104 + # Missing docstring in magic method + D105 + # Missing docstring in public nested class + D106 + # Missing docstring in __init__ + D107 + # One-line docstring should fit on one line with quotes + D200 + # No blank lines allowed after function docstring + D202 + # 1 blank line required between summary line and description + D205 + # Multi-line docstring closing quotes should be on a separate line + D209 + # First line should end with a period + D400 + # First line should be in imperative mood; try rephrasing + D401 + # First line should not be the function's "signature" + D402 + # First word of the first line should be properly capitalized + D403 exclude = docs/conf.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 974d397c8..6b60eff68 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,6 +11,7 @@ repos: additional_dependencies: - flake8-comprehensions - flake8-debugger + - flake8-docstrings - flake8-string-format - repo: https://github.com/psf/black.git rev: 24.2.0 diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index aa0975555..f617fc027 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -146,7 +146,7 @@ class RFPDupeFilterTest(unittest.TestCase): case_insensitive_dupefilter.close("finished") def test_seenreq_newlines(self): - """Checks against adding duplicate \r to + r"""Checks against adding duplicate \r to line endings on Windows platforms.""" r1 = Request("http://scrapytest.org/1") diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index d9c09a16a..b1043c111 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -186,7 +186,7 @@ class Base: ) def test_nofollow(self): - '''Test the extractor's behaviour for links with rel="nofollow"''' + """Test the extractor's behaviour for links with rel='nofollow'""" html = b"""Page title<title> <body> From 3d8dbd5648406227c9b96736da62046b90c554e5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 20 Jun 2024 00:22:43 +0500 Subject: [PATCH 224/269] flake8-bugbear --- .flake8 | 15 +++++++++++++++ .pre-commit-config.yaml | 1 + scrapy/pipelines/media.py | 2 +- scrapy/utils/defer.py | 2 +- scrapy/utils/python.py | 2 +- scrapy/utils/signal.py | 5 ++++- tests/test_cmdline/__init__.py | 2 +- tests/test_command_version.py | 4 ++-- tests/test_commands.py | 2 +- tests/test_downloader_handlers.py | 2 +- tests/test_engine.py | 2 +- tests/test_request_dict.py | 2 +- 12 files changed, 30 insertions(+), 11 deletions(-) diff --git a/.flake8 b/.flake8 index 222ba7179..57117d2cf 100644 --- a/.flake8 +++ b/.flake8 @@ -4,6 +4,21 @@ max-line-length = 119 ignore = # black disagrees with flake8 about these E203, E501, E701, E704, W503 + # Assigning to `os.environ` doesn't clear the environment. + B003 + # Do not use mutable data structures for argument defaults. + B006 + # Loop control variable not used within the loop body. + B007 + # Do not perform function calls in argument defaults. + B008 + # return/continue/break inside finally blocks cause exceptions to be + # silenced. + B012 + # Star-arg unpacking after a keyword argument is strongly discouraged + B026 + # No explicit stacklevel argument found. + B028 # docstring does contain unindexed parameters P102 # other string does contain unindexed parameters diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6b60eff68..f70effc5d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,7 @@ repos: hooks: - id: flake8 additional_dependencies: + - flake8-bugbear - flake8-comprehensions - flake8-debugger - flake8-docstrings diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 3e327105e..09e95cf5d 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -234,7 +234,7 @@ class MediaPipeline(ABC): # Exception Chaining (https://www.python.org/dev/peps/pep-3134/). context = getattr(result.value, "__context__", None) if isinstance(context, StopIteration): - setattr(result.value, "__context__", None) + result.value.__context__ = None info.downloading.remove(fp) info.downloaded[fp] = result # cache result diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index ddb68c86b..877eb4388 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -407,7 +407,7 @@ def maybeDeferred_coro( """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" try: result = f(*args, **kw) - except: # noqa: E722 + except: # noqa: E722,B001 return defer.fail(failure.Failure(captureVars=Deferred.debug)) if isinstance(result, Deferred): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 059d8e04d..f56950fdd 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -269,7 +269,7 @@ def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]: if inspect.isfunction(func) or inspect.ismethod(func): spec = inspect.getfullargspec(func) - elif hasattr(func, "__call__"): + elif hasattr(func, "__call__"): # noqa: B004 spec = inspect.getfullargspec(func.__call__) else: raise TypeError(f"{type(func)} is not callable") diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 89cfbd2ec..bb6d807ee 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -100,7 +100,10 @@ def send_catch_log_deferred( d.addErrback(logerror, receiver) # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html d.addBoth( - lambda result: (receiver, result) # pylint: disable=cell-var-from-loop + lambda result: ( + receiver, # pylint: disable=cell-var-from-loop # noqa: B023 + result, + ) ) dfds.append(d) d = DeferredList(dfds) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 25ded143c..4835e936b 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -20,7 +20,7 @@ class CmdlineTest(unittest.TestCase): self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings" def _execute(self, *new_args, **kwargs): - encoding = getattr(sys.stdout, "encoding") or "utf-8" + encoding = sys.stdout.encoding or "utf-8" args = (sys.executable, "-m", "scrapy.cmdline") + new_args proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs) comm = proc.communicate()[0].strip() diff --git a/tests/test_command_version.py b/tests/test_command_version.py index a52d0d13c..18c1c531c 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -12,7 +12,7 @@ class VersionTest(ProcessTest, unittest.TestCase): @defer.inlineCallbacks def test_output(self): - encoding = getattr(sys.stdout, "encoding") or "utf-8" + encoding = sys.stdout.encoding or "utf-8" _, out, _ = yield self.execute([]) self.assertEqual( out.strip().decode(encoding), @@ -21,7 +21,7 @@ class VersionTest(ProcessTest, unittest.TestCase): @defer.inlineCallbacks def test_verbose_output(self): - encoding = getattr(sys.stdout, "encoding") or "utf-8" + encoding = sys.stdout.encoding or "utf-8" _, out, _ = yield self.execute(["-v"]) headers = [ line.partition(":")[0].strip() diff --git a/tests/test_commands.py b/tests/test_commands.py index d829b1701..a23b7f4a9 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -101,7 +101,7 @@ class ProjectTest(unittest.TestCase): def kill_proc(): p.kill() p.communicate() - assert False, "Command took too much time to complete" + raise AssertionError("Command took too much time to complete") timer = Timer(15, kill_proc) try: diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index d3fd63847..884491d01 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -892,7 +892,7 @@ class S3TestCase(unittest.TestCase): except Exception as e: self.assertIsInstance(e, (TypeError, NotConfigured)) else: - assert False + raise AssertionError() def test_request_signing1(self): # gets an object from the johnsmith bucket. diff --git a/tests/test_engine.py b/tests/test_engine.py index 33544e8db..86526420f 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -459,7 +459,7 @@ class EngineTest(unittest.TestCase): def kill_proc(): p.kill() p.communicate() - assert False, "Command took too much time to complete" + raise AssertionError("Command took too much time to complete") timer = Timer(15, kill_proc) try: diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 7312eb036..d3f416347 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -147,7 +147,7 @@ class RequestSerializationTest(unittest.TestCase): spider = MySpider() r = Request("http://www.example.com", callback=spider.parse) - setattr(spider, "parse", None) + spider.parse = None self.assertRaises(ValueError, r.to_dict, spider=spider) def test_callback_not_available(self): From 13d3b1af470bbe7e82fda51017f0f72cb8eed9dd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 20 Jun 2024 00:42:43 +0500 Subject: [PATCH 225/269] Split ignores into blocks. --- .flake8 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.flake8 b/.flake8 index 57117d2cf..be9d83eaf 100644 --- a/.flake8 +++ b/.flake8 @@ -4,6 +4,7 @@ max-line-length = 119 ignore = # black disagrees with flake8 about these E203, E501, E701, E704, W503 + # Assigning to `os.environ` doesn't clear the environment. B003 # Do not use mutable data structures for argument defaults. @@ -19,10 +20,12 @@ ignore = B026 # No explicit stacklevel argument found. B028 + # docstring does contain unindexed parameters P102 # other string does contain unindexed parameters P103 + # Missing docstring in public module D100 # Missing docstring in public class From 326e323e11a7f5fc760250be6eae23d0159f6429 Mon Sep 17 00:00:00 2001 From: mlmsmith <mlmsmith@hotmail.co.uk> Date: Fri, 21 Jun 2024 18:24:10 +0800 Subject: [PATCH 226/269] Apply grammar fixes (#6411) --- docs/intro/overview.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 542760b4f..ef1294470 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -44,13 +44,13 @@ https://quotes.toscrape.com, following the pagination: if next_page is not None: yield response.follow(next_page, self.parse) -Put this in a text file, name it to something like ``quotes_spider.py`` +Put this in a text file, name it something like ``quotes_spider.py`` and run the spider using the :command:`runspider` command:: scrapy runspider quotes_spider.py -o quotes.jsonl When this finishes you will have in the ``quotes.jsonl`` file a list of the -quotes in JSON Lines format, containing text and author, looking like this:: +quotes in JSON Lines format, containing the text and author, which will look like this:: {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"} {"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"} @@ -72,11 +72,11 @@ using a CSS Selector, yield a Python dict with the extracted quote text and auth look for a link to the next page and schedule another request using the same ``parse`` method as callback. -Here you notice one of the main advantages about Scrapy: requests are +Here you will notice one of the main advantages of Scrapy: requests are :ref:`scheduled and processed asynchronously <topics-architecture>`. This means that Scrapy doesn't need to wait for a request to be finished and processed, it can send another request or do other things in the meantime. This -also means that other requests can keep going even if some request fails or an +also means that other requests can keep going even if a request fails or an error happens while handling it. While this enables you to do very fast crawls (sending multiple concurrent From d08f559600f0bb45b916be158a06e033753d45f5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 25 Jun 2024 13:20:59 +0500 Subject: [PATCH 227/269] Add flake8-type-checking. (#6413) --- .flake8 | 4 +++ .pre-commit-config.yaml | 1 + scrapy/addons.py | 9 ++++-- scrapy/cmdline.py | 3 +- scrapy/commands/__init__.py | 8 +++-- scrapy/commands/bench.py | 8 +++-- scrapy/commands/crawl.py | 8 +++-- scrapy/commands/fetch.py | 8 +++-- scrapy/commands/list.py | 8 +++-- scrapy/commands/parse.py | 9 ++++-- scrapy/commands/runspider.py | 12 ++++--- scrapy/commands/shell.py | 8 +++-- scrapy/contracts/__init__.py | 11 +++++-- scrapy/core/downloader/__init__.py | 5 +-- scrapy/core/downloader/contextfactory.py | 5 +-- scrapy/core/downloader/handlers/__init__.py | 6 ++-- scrapy/core/downloader/handlers/datauri.py | 8 +++-- scrapy/core/downloader/handlers/file.py | 9 ++++-- scrapy/core/downloader/handlers/ftp.py | 12 ++++--- scrapy/core/downloader/handlers/http10.py | 12 +++---- scrapy/core/downloader/handlers/http11.py | 11 ++++--- scrapy/core/downloader/handlers/http2.py | 16 +++++----- scrapy/core/downloader/handlers/s3.py | 13 ++++---- scrapy/core/downloader/middleware.py | 11 ++++--- scrapy/core/downloader/webclient.py | 8 +++-- scrapy/core/engine.py | 6 ++-- scrapy/core/http2/agent.py | 18 +++++++---- scrapy/core/http2/protocol.py | 21 ++++++++----- scrapy/core/http2/stream.py | 11 ++++--- scrapy/core/scheduler.py | 16 +++++----- scrapy/core/spidermw.py | 6 +++- scrapy/downloadermiddlewares/ajaxcrawl.py | 8 +++-- scrapy/downloadermiddlewares/cookies.py | 10 +++--- .../downloadermiddlewares/defaultheaders.py | 7 +++-- .../downloadermiddlewares/downloadtimeout.py | 5 +-- scrapy/downloadermiddlewares/httpauth.py | 5 +-- scrapy/downloadermiddlewares/httpcache.py | 13 ++++---- .../downloadermiddlewares/httpcompression.py | 6 ++-- scrapy/downloadermiddlewares/httpproxy.py | 7 +++-- scrapy/downloadermiddlewares/offsite.py | 6 ++-- scrapy/downloadermiddlewares/redirect.py | 8 +++-- scrapy/downloadermiddlewares/retry.py | 10 +++--- scrapy/downloadermiddlewares/robotstxt.py | 10 +++--- scrapy/downloadermiddlewares/stats.py | 9 +++--- scrapy/downloadermiddlewares/useragent.py | 5 +-- scrapy/dupefilters.py | 10 +++--- scrapy/extension.py | 8 +++-- scrapy/extensions/closespider.py | 10 +++--- scrapy/extensions/corestats.py | 5 +-- scrapy/extensions/debug.py | 7 +++-- scrapy/extensions/feedexport.py | 17 +++++----- scrapy/extensions/httpcache.py | 9 ++++-- scrapy/extensions/logstats.py | 6 ++-- scrapy/extensions/memdebug.py | 5 +-- scrapy/extensions/memusage.py | 4 ++- scrapy/extensions/periodic_log.py | 6 ++-- scrapy/extensions/spiderstate.py | 3 +- scrapy/extensions/statsmailer.py | 9 +++--- scrapy/extensions/telnet.py | 5 ++- scrapy/extensions/throttle.py | 8 +++-- scrapy/http/cookies.py | 6 ++-- scrapy/http/request/form.py | 3 +- scrapy/http/response/__init__.py | 7 +++-- scrapy/http/response/text.py | 3 +- scrapy/linkextractors/lxmlhtml.py | 13 ++++++-- scrapy/loader/__init__.py | 10 ++++-- scrapy/logformatter.py | 5 +-- scrapy/mail.py | 6 ++-- scrapy/middleware.py | 8 ++--- scrapy/pipelines/__init__.py | 12 ++++--- scrapy/pipelines/files.py | 31 ++++++++++--------- scrapy/pipelines/images.py | 11 ++++--- scrapy/pipelines/media.py | 10 +++--- scrapy/resolver.py | 3 +- scrapy/robotstxt.py | 3 +- scrapy/settings/__init__.py | 3 +- scrapy/signalmanager.py | 8 +++-- scrapy/spiderloader.py | 8 +++-- scrapy/spidermiddlewares/depth.py | 8 +++-- scrapy/spidermiddlewares/httperror.py | 10 +++--- scrapy/spidermiddlewares/offsite.py | 6 ++-- scrapy/spidermiddlewares/referer.py | 6 ++-- scrapy/spidermiddlewares/urllength.py | 6 ++-- scrapy/spiders/init.py | 8 +++-- scrapy/squeues.py | 8 +++-- scrapy/statscollectors.py | 10 +++--- scrapy/utils/decorators.py | 8 ++--- scrapy/utils/defer.py | 3 +- scrapy/utils/engine.py | 5 +-- scrapy/utils/gz.py | 8 +++-- scrapy/utils/httpobj.py | 10 ++++-- scrapy/utils/job.py | 9 ++++-- scrapy/utils/log.py | 3 +- scrapy/utils/misc.py | 4 ++- scrapy/utils/project.py | 5 +-- scrapy/utils/reactor.py | 8 +++-- scrapy/utils/spider.py | 10 +++--- scrapy/utils/ssl.py | 8 +++-- scrapy/utils/template.py | 8 +++-- scrapy/utils/test.py | 19 ++++++++++-- scrapy/utils/testproc.py | 6 ++-- tests/mockserver.py | 8 +++-- tests/test_feedexport.py | 8 +++-- tests/test_http2_client_protocol.py | 8 +++-- 104 files changed, 562 insertions(+), 300 deletions(-) diff --git a/.flake8 b/.flake8 index be9d83eaf..c4814f13a 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,7 @@ [flake8] max-line-length = 119 +extend-select = TC, TC1 ignore = # black disagrees with flake8 about these E203, E501, E701, E704, W503 @@ -58,6 +59,9 @@ ignore = D402 # First word of the first line should be properly capitalized D403 + + # Annotation in typing.cast() should be a string literal + TC006 exclude = docs/conf.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f70effc5d..38526d720 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,7 @@ repos: - flake8-debugger - flake8-docstrings - flake8-string-format + - flake8-type-checking - repo: https://github.com/psf/black.git rev: 24.2.0 hooks: diff --git a/scrapy/addons.py b/scrapy/addons.py index 65d7a0310..f9ec58cea 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,13 +1,16 @@ +from __future__ import annotations + import logging from typing import TYPE_CHECKING, Any, List from scrapy.exceptions import NotConfigured -from scrapy.settings import Settings from scrapy.utils.conf import build_component_list from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler + from scrapy.settings import Settings + logger = logging.getLogger(__name__) @@ -15,8 +18,8 @@ logger = logging.getLogger(__name__) class AddonManager: """This class facilitates loading and storing :ref:`topics-addons`.""" - def __init__(self, crawler: "Crawler") -> None: - self.crawler: "Crawler" = crawler + def __init__(self, crawler: Crawler) -> None: + self.crawler: Crawler = crawler self.addons: List[Any] = [] def load_settings(self, settings: Settings) -> None: diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index da0e51386..e010b159a 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -12,7 +12,6 @@ import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter from scrapy.crawler import CrawlerProcess from scrapy.exceptions import UsageError -from scrapy.settings import BaseSettings, Settings from scrapy.utils.misc import walk_modules from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect @@ -21,6 +20,8 @@ if TYPE_CHECKING: # typing.ParamSpec requires Python 3.10 from typing_extensions import ParamSpec + from scrapy.settings import BaseSettings, Settings + _P = ParamSpec("_P") diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 9fe803d3c..0322390e5 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -2,18 +2,22 @@ Base class for Scrapy commands """ +from __future__ import annotations + import argparse import builtins import os from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional from twisted.python import failure -from scrapy.crawler import Crawler, CrawlerProcess from scrapy.exceptions import UsageError from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli +if TYPE_CHECKING: + from scrapy.crawler import Crawler, CrawlerProcess + class ScrapyCommand: requires_project: bool = False diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 0c4ebcd23..f91fec57e 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,16 +1,20 @@ +from __future__ import annotations + import argparse import subprocess # nosec import sys import time -from typing import Any, Iterable, List +from typing import TYPE_CHECKING, Any, Iterable, List from urllib.parse import urlencode import scrapy -from scrapy import Request from scrapy.commands import ScrapyCommand from scrapy.http import Response, TextResponse from scrapy.linkextractors import LinkExtractor +if TYPE_CHECKING: + from scrapy import Request + class Command(ScrapyCommand): default_settings = { diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 6e023af81..fe1864372 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,11 +1,15 @@ -import argparse -from typing import List, cast +from __future__ import annotations + +from typing import TYPE_CHECKING, List, cast from twisted.python.failure import Failure from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError +if TYPE_CHECKING: + import argparse + class Command(BaseRunSpiderCommand): requires_project = True diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 1acf2d26f..0bdc429da 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,6 +1,7 @@ +from __future__ import annotations + import sys -from argparse import ArgumentParser, Namespace -from typing import Dict, List, Type +from typing import TYPE_CHECKING, Dict, List, Type from w3lib.url import is_url @@ -11,6 +12,9 @@ from scrapy.http import Request, Response from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import DefaultSpider, spidercls_for_request +if TYPE_CHECKING: + from argparse import ArgumentParser, Namespace + class Command(ScrapyCommand): requires_project = False diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index dcc51a694..10330c92a 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,8 +1,12 @@ -import argparse -from typing import List +from __future__ import annotations + +from typing import TYPE_CHECKING, List from scrapy.commands import ScrapyCommand +if TYPE_CHECKING: + import argparse + class Command(ScrapyCommand): requires_project = True diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 3320a1ee4..e6c5e2a47 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -6,6 +6,7 @@ import inspect import json import logging from typing import ( + TYPE_CHECKING, Any, AsyncGenerator, Callable, @@ -22,13 +23,11 @@ from typing import ( from itemadapter import ItemAdapter, is_item from twisted.internet.defer import Deferred, maybeDeferred -from twisted.python.failure import Failure from w3lib.url import is_url from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError from scrapy.http import Request, Response -from scrapy.spiders import Spider from scrapy.utils import display from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.defer import aiter_errback, deferred_from_coro @@ -36,6 +35,12 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter from scrapy.utils.spider import spidercls_for_request +if TYPE_CHECKING: + from twisted.python.failure import Failure + + from scrapy.spiders import Spider + + logger = logging.getLogger(__name__) _T = TypeVar("_T") diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 77850e7b5..87acf9a01 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,17 +1,21 @@ +from __future__ import annotations + import argparse import sys from importlib import import_module -from os import PathLike from pathlib import Path -from types import ModuleType -from typing import List, Union +from typing import TYPE_CHECKING, List, Union from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError from scrapy.utils.spider import iter_spider_classes +if TYPE_CHECKING: + from os import PathLike + from types import ModuleType -def _import_file(filepath: Union[str, PathLike]) -> ModuleType: + +def _import_file(filepath: Union[str, PathLike[str]]) -> ModuleType: abspath = Path(filepath).resolve() if abspath.suffix not in (".py", ".pyw"): raise ValueError(f"Not a Python source file: {abspath}") diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 668c95a7b..f03cf997a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -4,9 +4,10 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ -from argparse import ArgumentParser, Namespace +from __future__ import annotations + from threading import Thread -from typing import Any, Dict, List, Type +from typing import TYPE_CHECKING, Any, Dict, List, Type from scrapy import Spider from scrapy.commands import ScrapyCommand @@ -15,6 +16,9 @@ from scrapy.shell import Shell from scrapy.utils.spider import DefaultSpider, spidercls_for_request from scrapy.utils.url import guess_scheme +if TYPE_CHECKING: + from argparse import ArgumentParser, Namespace + class Command(ScrapyCommand): requires_project = False diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 27bc2fcba..440e0dc44 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -1,9 +1,12 @@ +from __future__ import annotations + import re import sys from functools import wraps from inspect import getmembers from types import CoroutineType from typing import ( + TYPE_CHECKING, Any, AsyncGenerator, Callable, @@ -16,13 +19,15 @@ from typing import ( ) from unittest import TestCase, TestResult -from twisted.python.failure import Failure - -from scrapy import Spider from scrapy.http import Request, Response from scrapy.utils.python import get_spec from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + from twisted.python.failure import Failure + + from scrapy import Spider + class Contract: """Abstract class for contracts""" diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 41f729ed9..6786d7acf 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -25,15 +25,16 @@ from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.http import Response from scrapy.resolver import dnscache -from scrapy.settings import BaseSettings from scrapy.signalmanager import SignalManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings + _T = TypeVar("_T") diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 9f6edb630..2b388a9f5 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -21,8 +21,6 @@ from scrapy.core.downloader.tls import ( ScrapyClientTLSOptions, openssl_methods, ) -from scrapy.crawler import Crawler -from scrapy.settings import BaseSettings from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: @@ -31,6 +29,9 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + @implementer(IPolicyForHTTPS) class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index ebc4898b5..70d356b83 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -17,17 +17,19 @@ from typing import ( ) from twisted.internet import defer -from twisted.internet.defer import Deferred from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured, NotSupported -from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values if TYPE_CHECKING: + from twisted.internet.defer import Deferred + from scrapy.crawler import Crawler + from scrapy.http import Response + logger = logging.getLogger(__name__) diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index a7ae56a85..bf6879521 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -1,12 +1,16 @@ -from typing import Any, Dict +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Dict from w3lib.url import parse_data_uri -from scrapy import Request, Spider from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers +if TYPE_CHECKING: + from scrapy import Request, Spider + class DataURIDownloadHandler: lazy = False diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 17dd7483b..d55c516f0 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -1,12 +1,17 @@ +from __future__ import annotations + from pathlib import Path +from typing import TYPE_CHECKING from w3lib.url import file_uri_to_path -from scrapy import Request, Spider -from scrapy.http import Response from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers +if TYPE_CHECKING: + from scrapy import Request, Spider + from scrapy.http import Response + class FileDownloadHandler: lazy = False diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 724717ffd..69c2d88e1 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -35,23 +35,25 @@ from io import BytesIO from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional from urllib.parse import unquote -from twisted.internet.defer import Deferred from twisted.internet.protocol import ClientCreator, Protocol from twisted.protocols.ftp import CommandFailed, FTPClient -from twisted.python.failure import Failure -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.responsetypes import responsetypes -from scrapy.settings import BaseSettings from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes if TYPE_CHECKING: + from twisted.internet.defer import Deferred + from twisted.python.failure import Failure + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + class ReceivedDataProtocol(Protocol): def __init__(self, filename: Optional[str] = None): diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 3c4e48abb..98f62efcf 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -5,21 +5,21 @@ from __future__ import annotations from typing import TYPE_CHECKING, Type -from twisted.internet.defer import Deferred - -from scrapy import Request, Spider -from scrapy.crawler import Crawler -from scrapy.http import Response -from scrapy.settings import BaseSettings from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings class HTTP10DownloadHandler: diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index e2ad8f59a..c06d90f01 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -12,11 +12,9 @@ from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, TypeVar from urllib.parse import urldefrag, urlunparse from twisted.internet import ssl -from twisted.internet.base import ReactorBase from twisted.internet.defer import CancelledError, Deferred, succeed from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError -from twisted.internet.interfaces import IConsumer from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.python.failure import Failure from twisted.web.client import URI, Agent, HTTPConnectionPool @@ -30,17 +28,22 @@ from zope.interface import implementer from scrapy import Request, Spider, signals from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse -from scrapy.crawler import Crawler from scrapy.exceptions import StopDownload from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes -from scrapy.settings import BaseSettings from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: + from twisted.internet.base import ReactorBase + from twisted.internet.interfaces import IConsumer + # typing.NotRequired and typing.Self require Python 3.11 from typing_extensions import NotRequired, Self + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) _T = TypeVar("_T") diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 2ac4eca86..4722c612d 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -4,25 +4,27 @@ from time import time from typing import TYPE_CHECKING, Optional from urllib.parse import urldefrag -from twisted.internet.base import DelayedCall -from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.web.client import URI -from twisted.web.iweb import IPolicyForHTTPS from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent -from scrapy.crawler import Crawler -from scrapy.http import Request, Response -from scrapy.settings import Settings -from scrapy.spiders import Spider from scrapy.utils.python import to_bytes if TYPE_CHECKING: + from twisted.internet.base import DelayedCall + from twisted.internet.defer import Deferred + from twisted.web.iweb import IPolicyForHTTPS + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Request, Response + from scrapy.settings import Settings + from scrapy.spiders import Spider + class H2DownloadHandler: def __init__(self, settings: Settings, crawler: Crawler): diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 0ad340721..edf370193 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -2,22 +2,23 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, Optional, Type -from twisted.internet.defer import Deferred - -from scrapy import Request, Spider from scrapy.core.downloader.handlers.http import HTTPDownloadHandler -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response -from scrapy.settings import BaseSettings from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings + class S3DownloadHandler: def __init__( diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 2d8af114f..0bdb756c8 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -6,19 +6,22 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations -from typing import Any, Callable, Generator, List, Union, cast +from typing import TYPE_CHECKING, Any, Callable, Generator, List, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks -from twisted.python.failure import Failure -from scrapy import Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.middleware import MiddlewareManager -from scrapy.settings import BaseSettings from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_from_coro, mustbe_deferred +if TYPE_CHECKING: + from twisted.python.failure import Failure + + from scrapy import Spider + from scrapy.settings import BaseSettings + class DownloaderMiddlewareManager(MiddlewareManager): component_name = "downloader middleware" diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 08a1d7c71..99502f0d2 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,18 +1,22 @@ +from __future__ import annotations + import re from time import time -from typing import Optional, Tuple +from typing import TYPE_CHECKING, Optional, Tuple from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse from twisted.internet import defer from twisted.internet.protocol import ClientFactory from twisted.web.http import HTTPClient -from scrapy import Request from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode +if TYPE_CHECKING: + from scrapy import Request + def _parsed_url_args(parsed: ParseResult) -> Tuple[bytes, bytes, bytes, int, bytes]: # Assume parsed is urlparse-d from Request.url, diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 4ffec78b9..5318cbd64 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -34,9 +34,8 @@ from scrapy.core.scraper import Scraper from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response from scrapy.logformatter import LogFormatter -from scrapy.settings import BaseSettings, Settings +from scrapy.settings import Settings from scrapy.signalmanager import SignalManager -from scrapy.spiders import Spider from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import global_object_name @@ -46,6 +45,9 @@ if TYPE_CHECKING: from scrapy.core.scheduler import BaseScheduler from scrapy.core.scraper import _HandleOutputDeferred from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + from scrapy.spiders import Spider + logger = logging.getLogger(__name__) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 999764a6e..d291a5b8a 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,10 +1,10 @@ +from __future__ import annotations + from collections import deque -from typing import Deque, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Deque, Dict, List, Optional, Tuple from twisted.internet import defer -from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred -from twisted.internet.endpoints import HostnameEndpoint from twisted.python.failure import Failure from twisted.web.client import ( URI, @@ -16,9 +16,15 @@ from twisted.web.error import SchemeNotSupported from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol -from scrapy.http.request import Request -from scrapy.settings import Settings -from scrapy.spiders import Spider + +if TYPE_CHECKING: + from twisted.internet.base import ReactorBase + from twisted.internet.endpoints import HostnameEndpoint + + from scrapy.http.request import Request + from scrapy.settings import Settings + from scrapy.spiders import Spider + ConnectionKeyT = Tuple[bytes, bytes, int] diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index f2f1cb0b8..a6809102b 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,9 +1,10 @@ +from __future__ import annotations + import ipaddress import itertools import logging from collections import deque -from ipaddress import IPv4Address, IPv6Address -from typing import Any, Deque, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Deque, Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -20,7 +21,6 @@ from h2.events import ( WindowUpdated, ) from h2.exceptions import FrameTooLargeError, H2Error -from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.internet.interfaces import ( IAddress, @@ -30,14 +30,21 @@ from twisted.internet.interfaces import ( from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin -from twisted.python.failure import Failure -from twisted.web.client import URI from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.http import Request -from scrapy.settings import Settings -from scrapy.spiders import Spider + +if TYPE_CHECKING: + from ipaddress import IPv4Address, IPv6Address + + from twisted.internet.defer import Deferred + from twisted.python.failure import Failure + from twisted.web.client import URI + + from scrapy.settings import Settings + from scrapy.spiders import Spider + logger = logging.getLogger(__name__) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 7c70e86db..a02fbb328 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging from enum import Enum from io import BytesIO @@ -5,19 +7,20 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError -from hpack import HeaderTuple from twisted.internet.defer import CancelledError, Deferred from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: + from hpack import HeaderTuple + from scrapy.core.http2.protocol import H2ClientProtocol + from scrapy.http import Request logger = logging.getLogger(__name__) @@ -87,7 +90,7 @@ class Stream: self, stream_id: int, request: Request, - protocol: "H2ClientProtocol", + protocol: H2ClientProtocol, download_maxsize: int = 0, download_warnsize: int = 0, ) -> None: @@ -99,7 +102,7 @@ class Stream: """ self.stream_id: int = stream_id self._request: Request = request - self._protocol: "H2ClientProtocol" = protocol + self._protocol: H2ClientProtocol = protocol self._download_maxsize = self._request.meta.get( "download_maxsize", download_maxsize diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 1e586c53a..d4286c874 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -6,14 +6,10 @@ from abc import abstractmethod from pathlib import Path from typing import TYPE_CHECKING, Any, List, Optional, Type, cast -from twisted.internet.defer import Deferred +# working around https://github.com/sphinx-doc/sphinx/issues/10400 +from twisted.internet.defer import Deferred # noqa: TC002 -from scrapy.crawler import Crawler -from scrapy.dupefilters import BaseDupeFilter -from scrapy.http.request import Request -from scrapy.pqueues import ScrapyPriorityQueue -from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector +from scrapy.spiders import Spider # noqa: TC001 from scrapy.utils.job import job_dir from scrapy.utils.misc import build_from_crawler, load_object @@ -24,6 +20,12 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.dupefilters import BaseDupeFilter + from scrapy.http.request import Request + from scrapy.pqueues import ScrapyPriorityQueue + from scrapy.statscollectors import StatsCollector + logger = logging.getLogger(__name__) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index e792f8ca7..37a666605 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -10,6 +10,7 @@ import logging from inspect import isasyncgenfunction, iscoroutine from itertools import islice from typing import ( + TYPE_CHECKING, Any, AsyncIterable, Callable, @@ -30,7 +31,6 @@ from scrapy import Request, Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Response from scrapy.middleware import MiddlewareManager -from scrapy.settings import BaseSettings from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.conf import build_component_list from scrapy.utils.defer import ( @@ -41,6 +41,10 @@ from scrapy.utils.defer import ( ) from scrapy.utils.python import MutableAsyncChain, MutableChain +if TYPE_CHECKING: + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 0e757e4be..5fc7f31a3 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -6,16 +6,18 @@ from typing import TYPE_CHECKING, Union from w3lib import html -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import HtmlResponse, Response -from scrapy.settings import BaseSettings if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 73c2c57fe..23140d263 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -2,24 +2,26 @@ from __future__ import annotations import logging from collections import defaultdict -from http.cookiejar import Cookie from typing import TYPE_CHECKING, Any, DefaultDict, Iterable, Optional, Sequence, Union from tldextract import TLDExtract -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar -from scrapy.http.request import VerboseCookie from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from http.cookiejar import Cookie + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http.request import VerboseCookie + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 58fd415b9..49b9fdc05 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -8,15 +8,16 @@ from __future__ import annotations from typing import TYPE_CHECKING, Iterable, Tuple, Union -from scrapy import Request, Spider -from scrapy.crawler import Crawler -from scrapy.http import Response from scrapy.utils.python import without_none_values if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + class DefaultHeadersMiddleware: def __init__(self, headers: Iterable[Tuple[str, str]]): diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index fd7c03a38..ee7a24825 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -9,13 +9,14 @@ from __future__ import annotations from typing import TYPE_CHECKING, Union from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler -from scrapy.http import Response if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + class DownloadTimeoutMiddleware: def __init__(self, timeout: float = 180): diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 63490a37a..39165e155 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -11,14 +11,15 @@ from typing import TYPE_CHECKING, Union from w3lib.http import basic_auth_header from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler -from scrapy.http import Response from scrapy.utils.url import url_is_from_any_domain if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + class HttpAuthMiddleware: """Set Basic HTTP Authorization header diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 971473403..8377a3c1d 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -16,19 +16,20 @@ from twisted.internet.error import ( from twisted.web.client import ResponseFailed from scrapy import signals -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured -from scrapy.http.request import Request -from scrapy.http.response import Response -from scrapy.settings import Settings -from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector from scrapy.utils.misc import load_object if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http.request import Request + from scrapy.http.response import Response + from scrapy.settings import Settings + from scrapy.spiders import Spider + from scrapy.statscollectors import StatsCollector + class HttpCacheMiddleware: DOWNLOAD_EXCEPTIONS = ( diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index f3647e05f..6b0a56f7f 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -6,11 +6,9 @@ from logging import getLogger from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.statscollectors import StatsCollector from scrapy.utils._compression import ( _DecompressionMaxSizeExceeded, _inflate, @@ -24,6 +22,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = getLogger(__name__) ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"] diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 5b56ad449..a7af83f7d 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -9,10 +9,7 @@ from urllib.request import ( # type: ignore[attr-defined] proxy_bypass, ) -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes @@ -20,6 +17,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + class HttpProxyMiddleware: def __init__(self, auth_encoding: Optional[str] = "latin-1"): diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index bd8dbe329..6f67e3975 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -6,15 +6,17 @@ import warnings from typing import TYPE_CHECKING, Set from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest -from scrapy.statscollectors import StatsCollector from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 371e2fd3b..53081237c 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -6,11 +6,8 @@ from urllib.parse import urljoin from w3lib.url import safe_url_string -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import HtmlResponse, Response -from scrapy.settings import BaseSettings from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.response import get_meta_refresh @@ -18,6 +15,11 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 0637f09d4..8d7b7293c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -16,12 +16,8 @@ import warnings from logging import Logger, getLogger from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.http import Response -from scrapy.http.request import Request from scrapy.settings import BaseSettings, Settings -from scrapy.spiders import Spider from scrapy.utils.misc import load_object from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message @@ -30,6 +26,12 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.http.request import Request + from scrapy.spiders import Spider + + retry_logger = getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 6a0ecb7bf..70393576b 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -10,22 +10,24 @@ import logging from typing import TYPE_CHECKING, Any, Dict, Optional, Union from twisted.internet.defer import Deferred, maybeDeferred -from twisted.python.failure import Failure -from scrapy import Spider -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.robotstxt import RobotParser from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import load_object if TYPE_CHECKING: + from twisted.python.failure import Failure + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.robotstxt import RobotParser + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 444702757..0faae7b5a 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -4,11 +4,7 @@ from typing import TYPE_CHECKING, Dict, List, Tuple, Union from twisted.web import http -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response -from scrapy.statscollectors import StatsCollector from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.request import request_httprepr @@ -16,6 +12,11 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.statscollectors import StatsCollector + def get_header_size( headers: Dict[str, Union[List[Union[str, bytes]], Tuple[Union[str, bytes], ...]]] diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index 92f1ec897..109f1a4d9 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -5,13 +5,14 @@ from __future__ import annotations from typing import TYPE_CHECKING, Union from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler -from scrapy.http import Response if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + class UserAgentMiddleware: """This middleware allows spiders to override the user_agent""" diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index dd2420e98..ffaf783a7 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -4,11 +4,6 @@ import logging from pathlib import Path from typing import TYPE_CHECKING, Optional, Set -from twisted.internet.defer import Deferred - -from scrapy.http.request import Request -from scrapy.settings import BaseSettings -from scrapy.spiders import Spider from scrapy.utils.job import job_dir from scrapy.utils.request import ( RequestFingerprinter, @@ -17,10 +12,15 @@ from scrapy.utils.request import ( ) if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import Request + from scrapy.settings import BaseSettings + from scrapy.spiders import Spider class BaseDupeFilter: diff --git a/scrapy/extension.py b/scrapy/extension.py index 8221b675e..8c81ab356 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -4,12 +4,16 @@ The Extension Manager See documentation in docs/topics/extensions.rst """ -from typing import Any, List +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, List from scrapy.middleware import MiddlewareManager -from scrapy.settings import Settings from scrapy.utils.conf import build_component_list +if TYPE_CHECKING: + from scrapy.settings import Settings + class ExtensionManager(MiddlewareManager): component_name = "extension" diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 812b3553c..4627e7f98 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -10,17 +10,19 @@ import logging from collections import defaultdict from typing import TYPE_CHECKING, Any, DefaultDict, Dict -from twisted.python.failure import Failure - from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response if TYPE_CHECKING: + from twisted.python.failure import Failure + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index f3ac19623..6ef2d0382 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -8,13 +8,14 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, Optional from scrapy import Spider, signals -from scrapy.crawler import Crawler -from scrapy.statscollectors import StatsCollector if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + class CoreStats: def __init__(self, stats: StatsCollector): diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index b360ce48d..c54871e02 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -12,17 +12,20 @@ import sys import threading import traceback from pdb import Pdb -from types import FrameType from typing import TYPE_CHECKING, Optional -from scrapy.crawler import Crawler from scrapy.utils.engine import format_engine_status from scrapy.utils.trackref import format_live_refs if TYPE_CHECKING: + from types import FrameType + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 941bd4b26..43c2d2815 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -31,18 +31,15 @@ from typing import ( ) from urllib.parse import unquote, urlparse -from twisted.internet import threads from twisted.internet.defer import Deferred, DeferredList, maybeDeferred -from twisted.python.failure import Failure +from twisted.internet.threads import deferToThread from w3lib.url import file_uri_to_path from zope.interface import Interface, implementer from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.exporters import BaseItemExporter from scrapy.extensions.postprocessing import PostProcessingManager -from scrapy.settings import BaseSettings, Settings +from scrapy.settings import Settings from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.defer import maybe_deferred_to_future @@ -54,11 +51,14 @@ from scrapy.utils.python import without_none_values if TYPE_CHECKING: from _typeshed import OpenBinaryMode + from twisted.python.failure import Failure # typing.Self requires Python 3.11 from typing_extensions import Self -logger = logging.getLogger(__name__) + from scrapy.crawler import Crawler + from scrapy.exporters import BaseItemExporter + from scrapy.settings import BaseSettings try: import boto3 # noqa: F401 @@ -67,6 +67,9 @@ try: except ImportError: IS_BOTO3_AVAILABLE = False + +logger = logging.getLogger(__name__) + UriParamsCallableT = Callable[[Dict[str, Any], Spider], Optional[Dict[str, Any]]] _StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol") @@ -160,7 +163,7 @@ class BlockingFeedStorage: return NamedTemporaryFile(prefix="feed-", dir=path) def store(self, file: IO[bytes]) -> Optional[Deferred]: - return threads.deferToThread(self._store_in_thread, file) + return deferToThread(self._store_in_thread, file) def _store_in_thread(self, file: IO[bytes]) -> None: raise NotImplementedError diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index b7219bf07..448d5f1ab 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import gzip import logging import os @@ -13,10 +15,7 @@ from weakref import WeakKeyDictionary from w3lib.http import headers_dict_to_raw, headers_raw_to_dict from scrapy.http import Headers, Response -from scrapy.http.request import Request from scrapy.responsetypes import responsetypes -from scrapy.settings import BaseSettings -from scrapy.spiders import Spider from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode @@ -26,6 +25,10 @@ if TYPE_CHECKING: # typing.Concatenate requires Python 3.10 from typing_extensions import Concatenate + from scrapy.http.request import Request + from scrapy.settings import BaseSettings + from scrapy.spiders import Spider + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 2388afa75..c4f43482d 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -6,14 +6,16 @@ from typing import TYPE_CHECKING, Optional, Tuple, Union from twisted.internet import task from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.statscollectors import StatsCollector if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index f304e1bf2..3cbbb64e5 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -10,15 +10,16 @@ import gc from typing import TYPE_CHECKING from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.statscollectors import StatsCollector from scrapy.utils.trackref import live_refs if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + class MemoryDebugger: def __init__(self, stats: StatsCollector): diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 9de06b24d..25f63ecc6 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -16,7 +16,6 @@ from typing import TYPE_CHECKING, List from twisted.internet import task from scrapy import signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender from scrapy.utils.engine import get_engine_status @@ -25,6 +24,9 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 9567f948a..80c0a3b26 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -8,15 +8,17 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from twisted.internet import task from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.statscollectors import StatsCollector from scrapy.utils.serialize import ScrapyJSONEncoder if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index c6eb20277..567efd7a1 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -5,7 +5,6 @@ from pathlib import Path from typing import TYPE_CHECKING, Optional from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir @@ -13,6 +12,8 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + class SpiderState: """Store and load spider state during a scraping job""" diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 20b8f910c..e43de6f5c 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -8,18 +8,19 @@ from __future__ import annotations from typing import TYPE_CHECKING, List, Optional -from twisted.internet.defer import Deferred - from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender -from scrapy.statscollectors import StatsCollector if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + class StatsMailer: def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender): diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 00c69434c..c4e01b3d9 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -26,7 +26,6 @@ except (ImportError, SyntaxError): TWISTED_CONCH_AVAILABLE = False from scrapy import signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.utils.decorators import defers from scrapy.utils.engine import print_engine_status @@ -36,6 +35,10 @@ from scrapy.utils.trackref import print_live_refs if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) # signal to update telnet variables diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 217e61a81..6ce9ce63a 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -4,15 +4,17 @@ import logging from typing import TYPE_CHECKING, Optional, Tuple from scrapy import Request, Spider, signals -from scrapy.core.downloader import Slot -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.core.downloader import Slot + from scrapy.crawler import Crawler + from scrapy.http import Response + + logger = logging.getLogger(__name__) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 8af89c74f..cc88a9420 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -17,8 +17,6 @@ from typing import ( cast, ) -from scrapy import Request -from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode @@ -26,6 +24,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request + from scrapy.http import Response + + # Defined in the http.cookiejar module, but undocumented: # https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527 IPV4_RE = re.compile(r"\.\d+$", re.ASCII) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index ea98ed795..a8c242e8b 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -28,13 +28,14 @@ from lxml.html import TextareaElement # nosec from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request -from scrapy.http.response.text import TextResponse from scrapy.utils.python import is_listlike, to_bytes if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.http.response.text import TextResponse + FormdataVType = Union[str, Iterable[str]] FormdataKVType = Tuple[str, FormdataVType] diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 684439097..ff3581abb 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -7,7 +7,6 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from ipaddress import IPv4Address, IPv6Address from typing import ( TYPE_CHECKING, Any, @@ -26,8 +25,6 @@ from typing import ( ) from urllib.parse import urljoin -from twisted.internet.ssl import Certificate - from scrapy.exceptions import NotSupported from scrapy.http.headers import Headers from scrapy.http.request import CookiesT, Request @@ -35,6 +32,10 @@ from scrapy.link import Link from scrapy.utils.trackref import object_ref if TYPE_CHECKING: + from ipaddress import IPv4Address, IPv6Address + + from twisted.internet.ssl import Certificate + # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index df4d90829..0635f744f 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -35,15 +35,16 @@ from w3lib.encoding import ( ) from w3lib.html import strip_html5_whitespace -from scrapy.http.request import CookiesT, Request from scrapy.http.response import Response from scrapy.link import Link from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url if TYPE_CHECKING: + from scrapy.http.request import CookiesT, Request from scrapy.selector import Selector, SelectorList + _NONE = object() diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 33a10cd6c..d27a132b3 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -2,10 +2,13 @@ Link extractor based on lxml.html """ +from __future__ import annotations + import logging import operator from functools import partial from typing import ( + TYPE_CHECKING, Any, Callable, Iterable, @@ -20,13 +23,10 @@ from typing import ( from urllib.parse import urljoin, urlparse from lxml import etree # nosec -from lxml.html import HtmlElement # nosec from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string -from scrapy import Selector -from scrapy.http import TextResponse from scrapy.link import Link from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches, re from scrapy.utils.misc import arg_to_iter, rel_has_nofollow @@ -34,6 +34,13 @@ from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain +if TYPE_CHECKING: + from lxml.html import HtmlElement # nosec + + from scrapy import Selector + from scrapy.http import TextResponse + + logger = logging.getLogger(__name__) # from lxml/src/lxml/html/__init__.py diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index db0b4820f..9644cc093 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -4,14 +4,18 @@ Item Loader See documentation in docs/topics/loaders.rst """ -from typing import Any, Optional +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional import itemloaders -from scrapy.http import TextResponse from scrapy.item import Item from scrapy.selector import Selector +if TYPE_CHECKING: + from scrapy.http import TextResponse + class ItemLoader(itemloaders.ItemLoader): """ @@ -91,7 +95,7 @@ class ItemLoader(itemloaders.ItemLoader): selector: Optional[Selector] = None, response: Optional[TextResponse] = None, parent: Optional[itemloaders.ItemLoader] = None, - **context: Any + **context: Any, ): if selector is None and response is not None: try: diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 42a03b560..601209fb0 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -6,8 +6,9 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, TypedDict, Union from twisted.python.failure import Failure -from scrapy import Request, Spider -from scrapy.http import Response +# working around https://github.com/sphinx-doc/sphinx/issues/10400 +from scrapy import Request, Spider # noqa: TC001 +from scrapy.http import Response # noqa: TC001 from scrapy.utils.request import referer_str if TYPE_CHECKING: diff --git a/scrapy/mail.py b/scrapy/mail.py index f4ce2800c..3ea20e831 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -30,20 +30,22 @@ from typing import ( from twisted import version as twisted_version from twisted.internet import ssl from twisted.internet.defer import Deferred -from twisted.python.failure import Failure from twisted.python.versions import Version -from scrapy.settings import BaseSettings from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes if TYPE_CHECKING: # imports twisted.internet.reactor from twisted.mail.smtp import ESMTPSenderFactory + from twisted.python.failure import Failure # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index f60c726f9..ea5488ba1 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -17,19 +17,19 @@ from typing import ( cast, ) -from twisted.internet.defer import Deferred - -from scrapy import Spider from scrapy.exceptions import NotConfigured -from scrapy.settings import Settings from scrapy.utils.defer import process_chain, process_parallel from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider from scrapy.crawler import Crawler + from scrapy.settings import Settings logger = logging.getLogger(__name__) diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 21d649e3c..480a5a58c 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -6,16 +6,18 @@ See documentation in docs/item-pipeline.rst from __future__ import annotations -from typing import Any, List +from typing import TYPE_CHECKING, Any, List -from twisted.internet.defer import Deferred - -from scrapy import Spider from scrapy.middleware import MiddlewareManager -from scrapy.settings import Settings from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_f_from_coro_f +if TYPE_CHECKING: + from twisted.internet.defer import Deferred + + from scrapy import Spider + from scrapy.settings import Settings + class ItemPipelineManager(MiddlewareManager): component_name = "item pipeline" diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 85a8c77da..1a13aeaf2 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -16,7 +16,6 @@ from collections import defaultdict from contextlib import suppress from ftplib import FTP from io import BytesIO -from os import PathLike from pathlib import Path from typing import ( IO, @@ -38,11 +37,9 @@ from typing import ( from urllib.parse import urlparse from itemadapter import ItemAdapter -from twisted.internet import defer, threads -from twisted.internet.defer import Deferred -from twisted.python.failure import Failure +from twisted.internet.defer import Deferred, maybeDeferred +from twisted.internet.threads import deferToThread -from scrapy import Spider from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK @@ -56,9 +53,15 @@ from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str if TYPE_CHECKING: + from os import PathLike + + from twisted.python.failure import Failure + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + logger = logging.getLogger(__name__) @@ -210,7 +213,7 @@ class S3FilesStore: key_name = f"{self.prefix}{path}" return cast( "Deferred[Dict[str, Any]]", - threads.deferToThread( + deferToThread( self.s3_client.head_object, Bucket=self.bucket, Key=key_name # type: ignore[attr-defined] ), ) @@ -229,7 +232,7 @@ class S3FilesStore: extra = self._headers_to_botocore_kwargs(self.HEADERS) if headers: extra.update(self._headers_to_botocore_kwargs(headers)) - return threads.deferToThread( + return deferToThread( self.s3_client.put_object, # type: ignore[attr-defined] Bucket=self.bucket, Key=key_name, @@ -326,9 +329,7 @@ class GCSFilesStore: blob_path = self._get_blob_path(path) return cast( Deferred[StatInfo], - threads.deferToThread(self.bucket.get_blob, blob_path).addCallback( - _onsuccess - ), + deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess), ) def _get_content_type(self, headers: Optional[Dict[str, str]]) -> str: @@ -351,7 +352,7 @@ class GCSFilesStore: blob = self.bucket.blob(blob_path) blob.cache_control = self.CACHE_CONTROL blob.metadata = {k: str(v) for k, v in (meta or {}).items()} - return threads.deferToThread( + return deferToThread( blob.upload_from_string, data=buf.getvalue(), content_type=self._get_content_type(headers), @@ -388,7 +389,7 @@ class FTPFilesStore: headers: Optional[Dict[str, str]] = None, ) -> Deferred[Any]: path = f"{self.basedir}/{path}" - return threads.deferToThread( + return deferToThread( ftp_store_file, path=path, file=buf, @@ -418,7 +419,7 @@ class FTPFilesStore: except Exception: return {} - return cast("Deferred[StatInfo]", threads.deferToThread(_stat_file, path)) + return cast("Deferred[StatInfo]", deferToThread(_stat_file, path)) class FilesPipeline(MediaPipeline): @@ -553,8 +554,8 @@ class FilesPipeline(MediaPipeline): } path = self.file_path(request, info=info, item=item) - # defer.maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type - dfd: Deferred[StatInfo] = defer.maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type] + # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type + dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type] dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess) dfd2.addErrback(lambda _: None) dfd2.addErrback( diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 27a57b17c..166f81314 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -11,7 +11,6 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO -from os import PathLike from typing import ( TYPE_CHECKING, Any, @@ -28,7 +27,6 @@ from typing import ( from itemadapter import ItemAdapter -from scrapy import Spider from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK @@ -40,15 +38,20 @@ from scrapy.pipelines.files import ( S3FilesStore, _md5sum, ) -from scrapy.pipelines.media import FileInfoOrError, MediaPipeline from scrapy.settings import Settings from scrapy.utils.python import get_func_args, to_bytes if TYPE_CHECKING: - # typing.Self requires Python 3.11 + from os import PathLike + from PIL import Image + + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.pipelines.media import FileInfoOrError, MediaPipeline + class NoimagesDrop(DropItem): """Product with no images exception""" diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 09e95cf5d..ea36a9e8a 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -25,21 +25,23 @@ from typing import ( from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure -from scrapy import Spider -from scrapy.crawler import Crawler -from scrapy.http import Response from scrapy.http.request import NO_CALLBACK, Request from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import defer_result, mustbe_deferred from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter -from scrapy.utils.request import RequestFingerprinter if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.utils.request import RequestFingerprinter + + _T = TypeVar("_T") diff --git a/scrapy/resolver.py b/scrapy/resolver.py index ba7cd716b..d5eedf9b1 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Type from twisted.internet import defer from twisted.internet.base import ReactorBase, ThreadedResolver -from twisted.internet.defer import Deferred from twisted.internet.interfaces import ( IAddress, IHostnameResolver, @@ -17,6 +16,8 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index a33f73306..0d282dc37 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -6,7 +6,6 @@ from abc import ABCMeta, abstractmethod from typing import TYPE_CHECKING, Optional, Union from warnings import warn -from scrapy import Spider from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import to_unicode @@ -14,8 +13,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index ea1db03f1..6703c569f 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -4,7 +4,6 @@ import copy import json from importlib import import_module from pprint import pformat -from types import ModuleType from typing import ( TYPE_CHECKING, Any, @@ -27,6 +26,8 @@ from scrapy.settings import default_settings _SettingsKeyT = Union[bool, float, int, str, None] if TYPE_CHECKING: + from types import ModuleType + # https://github.com/python/typing/issues/445#issuecomment-1131458824 from _typeshed import SupportsItems diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index f6df191d8..3d37b8235 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,10 +1,14 @@ -from typing import Any, List, Tuple +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, List, Tuple from pydispatch import dispatcher -from twisted.internet.defer import Deferred from scrapy.utils import signal as _signal +if TYPE_CHECKING: + from twisted.internet.defer import Deferred + class SignalManager: def __init__(self, sender: Any = dispatcher.Anonymous): diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index d855c962c..b8fe65668 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -3,21 +3,23 @@ from __future__ import annotations import traceback import warnings from collections import defaultdict -from types import ModuleType from typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Type from zope.interface import implementer -from scrapy import Request, Spider from scrapy.interfaces import ISpiderLoader -from scrapy.settings import BaseSettings from scrapy.utils.misc import walk_modules from scrapy.utils.spider import iter_spider_classes if TYPE_CHECKING: + from types import ModuleType + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.settings import BaseSettings + @implementer(ISpiderLoader) class SpiderLoader: diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 1e96654e2..c5b7f0749 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -9,15 +9,17 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable -from scrapy import Spider -from scrapy.crawler import Crawler from scrapy.http import Request, Response -from scrapy.statscollectors import StatsCollector if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 35c869a75..ea1686c25 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -9,16 +9,18 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any, Iterable, List, Optional -from scrapy import Spider -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest -from scrapy.http import Response -from scrapy.settings import BaseSettings if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 50c93ac9f..379c5d0a3 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -12,10 +12,8 @@ import warnings from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable, Set from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response -from scrapy.statscollectors import StatsCollector from scrapy.utils.httpobj import urlparse_cached warnings.warn( @@ -28,6 +26,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 8af0bdf5b..d35cf8f71 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -23,10 +23,8 @@ from urllib.parse import urlparse from w3lib.url import safe_url_string from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response -from scrapy.settings import BaseSettings from scrapy.utils.misc import load_object from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url @@ -35,6 +33,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + LOCAL_SCHEMES: Tuple[str, ...] = ( "about", "blob", diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index e2aa554a7..34df54ca7 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -9,15 +9,17 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable -from scrapy import Spider from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response -from scrapy.settings import BaseSettings if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index a0898a0cf..ce0f1bbaa 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,10 +1,14 @@ -from typing import Any, Iterable, Optional, cast +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Iterable, Optional, cast from scrapy import Request -from scrapy.http import Response from scrapy.spiders import Spider from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + from scrapy.http import Response + class InitSpider(Spider): """Base Spider with initialization facilities""" diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 6f80ee388..d3e7896c5 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -6,20 +6,22 @@ from __future__ import annotations import marshal import pickle # nosec -from os import PathLike from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union from queuelib import queue -from scrapy import Request -from scrapy.crawler import Crawler from scrapy.utils.request import request_from_dict if TYPE_CHECKING: + from os import PathLike + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request + from scrapy.crawler import Crawler + def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]: class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc] diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index ab571a3ab..88e72f366 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -2,15 +2,17 @@ Scrapy extension for collecting scraping stats """ +from __future__ import annotations + import logging import pprint from typing import TYPE_CHECKING, Any, Dict, Optional -from scrapy import Spider - if TYPE_CHECKING: + from scrapy import Spider from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) @@ -18,7 +20,7 @@ StatsT = Dict[str, Any] class StatsCollector: - def __init__(self, crawler: "Crawler"): + def __init__(self, crawler: Crawler): self._dump: bool = crawler.settings.getbool("STATS_DUMP") self._stats: StatsT = {} @@ -67,7 +69,7 @@ class StatsCollector: class MemoryStatsCollector(StatsCollector): - def __init__(self, crawler: "Crawler"): + def __init__(self, crawler: Crawler): super().__init__(crawler) self.spider_stats: Dict[str, StatsT] = {} diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 7e82dd519..2240f0b58 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -4,8 +4,8 @@ import warnings from functools import wraps from typing import TYPE_CHECKING, Any, Callable, TypeVar -from twisted.internet import defer, threads -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, maybeDeferred +from twisted.internet.threads import deferToThread from scrapy.exceptions import ScrapyDeprecationWarning @@ -48,7 +48,7 @@ def defers(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: @wraps(func) def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]: - return defer.maybeDeferred(func, *a, **kw) + return maybeDeferred(func, *a, **kw) return wrapped @@ -60,6 +60,6 @@ def inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: @wraps(func) def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]: - return threads.deferToThread(func, *a, **kw) + return deferToThread(func, *a, **kw) return wrapped diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 877eb4388..1d578e8a3 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -34,12 +34,13 @@ from twisted.internet import defer from twisted.internet.defer import Deferred, DeferredList, ensureDeferred from twisted.internet.task import Cooperator from twisted.python import failure -from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed if TYPE_CHECKING: + from twisted.python.failure import Failure + # typing.Concatenate and typing.ParamSpec require Python 3.10 from typing_extensions import Concatenate, ParamSpec diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index fdcf484d4..770ee0b1b 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -4,9 +4,10 @@ from __future__ import annotations # used in global tests code from time import time # noqa: F401 -from typing import Any, List, Tuple +from typing import TYPE_CHECKING, Any, List, Tuple -from scrapy.core.engine import ExecutionEngine +if TYPE_CHECKING: + from scrapy.core.engine import ExecutionEngine def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]: diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 2e487d88b..85324361c 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,11 +1,15 @@ +from __future__ import annotations + import struct from gzip import GzipFile from io import BytesIO - -from scrapy.http import Response +from typing import TYPE_CHECKING from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded +if TYPE_CHECKING: + from scrapy.http import Response + def gunzip(data: bytes, *, max_size: int = 0) -> bytes: """Gunzip the given data and return as much data as possible. diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index d502e8910..3cf9585ec 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -1,12 +1,16 @@ """Helper functions for scrapy.http objects (Request, Response)""" -from typing import Union +from __future__ import annotations + +from typing import TYPE_CHECKING, Union from urllib.parse import ParseResult, urlparse from weakref import WeakKeyDictionary -from scrapy.http import Request, Response +if TYPE_CHECKING: + from scrapy.http import Request, Response -_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = ( + +_urlparse_cache: WeakKeyDictionary[Union[Request, Response], ParseResult] = ( WeakKeyDictionary() ) diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index e230e4235..488c7994b 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -1,7 +1,10 @@ -from pathlib import Path -from typing import Optional +from __future__ import annotations -from scrapy.settings import BaseSettings +from pathlib import Path +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from scrapy.settings import BaseSettings def job_dir(settings: BaseSettings) -> Optional[str]: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index cbfd170ed..439b065a9 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -21,12 +21,13 @@ from twisted.python import log as twisted_log from twisted.python.failure import Failure import scrapy -from scrapy.logformatter import LogFormatterResult from scrapy.settings import Settings, _SettingsKeyT from scrapy.utils.versions import scrapy_components_versions if TYPE_CHECKING: from scrapy.crawler import Crawler + from scrapy.logformatter import LogFormatterResult + logger = logging.getLogger(__name__) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 3d11c1035..3c787e50f 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -13,7 +13,6 @@ from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules -from types import ModuleType from typing import ( IO, TYPE_CHECKING, @@ -35,10 +34,13 @@ from scrapy.item import Item from scrapy.utils.datatypes import LocalWeakReferencedCache if TYPE_CHECKING: + from types import ModuleType + from scrapy import Spider from scrapy.crawler import Crawler from scrapy.settings import BaseSettings + _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes T = TypeVar("T") diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index de3c8eaf9..efb6af299 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,7 +1,8 @@ +from __future__ import annotations + import os import warnings from importlib import import_module -from os import PathLike from pathlib import Path from typing import Union @@ -46,7 +47,7 @@ def project_data_dir(project: str = "default") -> str: return str(d) -def data_path(path: Union[str, PathLike], createdir: bool = False) -> str: +def data_path(path: Union[str, os.PathLike[str]], createdir: bool = False) -> str: """ Return the given path joined with the .scrapy data directory. If given an absolute path, return it unmodified. diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 5af6d22eb..a627db601 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -2,7 +2,6 @@ from __future__ import annotations import asyncio import sys -from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from contextlib import suppress from typing import ( TYPE_CHECKING, @@ -20,13 +19,16 @@ from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error from twisted.internet.base import DelayedCall -from twisted.internet.protocol import ServerFactory -from twisted.internet.tcp import Port from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import load_object if TYPE_CHECKING: + from asyncio import AbstractEventLoop, AbstractEventLoopPolicy + + from twisted.internet.protocol import ServerFactory + from twisted.internet.tcp import Port + # typing.ParamSpec requires Python 3.10 from typing_extensions import ParamSpec diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index b05135c04..ce754fad3 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -2,7 +2,6 @@ from __future__ import annotations import inspect import logging -from types import CoroutineType, ModuleType from typing import ( TYPE_CHECKING, Any, @@ -16,16 +15,19 @@ from typing import ( overload, ) -from twisted.internet.defer import Deferred - -from scrapy import Request from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter if TYPE_CHECKING: + from types import CoroutineType, ModuleType + + from twisted.internet.defer import Deferred + + from scrapy import Request from scrapy.spiderloader import SpiderLoader + logger = logging.getLogger(__name__) _T = TypeVar("_T") diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index d520ef809..95611ebd9 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,12 +1,16 @@ -from typing import Any, Optional +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL import OpenSSL.version -from OpenSSL.crypto import X509Name from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + from OpenSSL.crypto import X509Name + def ffi_buf_to_string(buf: Any) -> str: return to_unicode(pyOpenSSLutil.ffi.string(buf)) diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 6b22f3bfa..08f3f2dc9 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -1,10 +1,14 @@ """Helper functions for working with templates""" +from __future__ import annotations + import re import string -from os import PathLike from pathlib import Path -from typing import Any, Union +from typing import TYPE_CHECKING, Any, Union + +if TYPE_CHECKING: + from os import PathLike def render_templatefile(path: Union[str, PathLike], **kwargs: Any) -> None: diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 268d8d4be..fe2bfa042 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -2,21 +2,36 @@ This module contains some assorted functions used in tests """ +from __future__ import annotations + import asyncio import os from importlib import import_module from pathlib import Path from posixpath import split -from typing import Any, Awaitable, Dict, List, Optional, Tuple, Type, TypeVar +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Dict, + List, + Optional, + Tuple, + Type, + TypeVar, +) from unittest import TestCase, mock -from twisted.internet.defer import Deferred from twisted.trial.unittest import SkipTest from scrapy import Spider from scrapy.crawler import Crawler from scrapy.utils.boto import is_botocore_available +if TYPE_CHECKING: + from twisted.internet.defer import Deferred + + _T = TypeVar("_T") diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 3bdffcaa7..8882bfc5f 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -2,12 +2,14 @@ from __future__ import annotations import os import sys -from typing import Iterable, List, Optional, Tuple, cast +from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, cast from twisted.internet.defer import Deferred from twisted.internet.error import ProcessTerminated from twisted.internet.protocol import ProcessProtocol -from twisted.python.failure import Failure + +if TYPE_CHECKING: + from twisted.python.failure import Failure class ProcessTest: diff --git a/tests/mockserver.py b/tests/mockserver.py index 233f6b934..6ec46aa3d 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import json import os @@ -7,12 +9,11 @@ from pathlib import Path from shutil import rmtree from subprocess import PIPE, Popen from tempfile import mkdtemp -from typing import Dict +from typing import TYPE_CHECKING, Dict from urllib.parse import urlencode from OpenSSL import SSL from twisted.internet import defer, reactor, ssl -from twisted.internet.protocol import ServerFactory from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory @@ -23,6 +24,9 @@ from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode +if TYPE_CHECKING: + from twisted.internet.protocol import ServerFactory + def getarg(request, name, default=None, type=None): if name in request.args: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 253987e15..ea3ed3b05 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import bz2 import csv import gzip @@ -14,10 +16,9 @@ from collections import defaultdict from contextlib import ExitStack from io import BytesIO from logging import getLogger -from os import PathLike from pathlib import Path from string import ascii_letters, digits -from typing import Union +from typing import TYPE_CHECKING, Union from unittest import mock from urllib.parse import quote, urljoin from urllib.request import pathname2url @@ -53,6 +54,9 @@ from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no from tests.mockserver import MockFTPServer, MockServer from tests.spiders import ItemSpider +if TYPE_CHECKING: + from os import PathLike + def path_to_url(path): return urljoin("file:", pathname2url(str(path))) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 995c02a1a..7ea3fe8c9 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import random import re @@ -6,7 +8,7 @@ import string from ipaddress import IPv4Address from pathlib import Path from tempfile import mkdtemp -from typing import Dict +from typing import TYPE_CHECKING, Dict from unittest import mock, skipIf from urllib.parse import urlencode @@ -20,7 +22,6 @@ from twisted.internet.defer import ( from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint from twisted.internet.error import TimeoutError from twisted.internet.ssl import Certificate, PrivateCertificate, optionsForClientTLS -from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from twisted.web.client import URI, ResponseFailed from twisted.web.http import H2_ENABLED @@ -33,6 +34,9 @@ from scrapy.settings import Settings from scrapy.spiders import Spider from tests.mockserver import LeafResource, Status, ssl_context_factory +if TYPE_CHECKING: + from twisted.python.failure import Failure + def generate_random_string(size): return "".join(random.choices(string.ascii_uppercase + string.digits, k=size)) From e47110f9a5a16f0628e53e16b9cb5f6a4f9721d3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 26 Jun 2024 13:01:43 +0500 Subject: [PATCH 228/269] Add parameteres to most Deferred instances. (#6414) --- scrapy/commands/parse.py | 6 +- scrapy/core/downloader/handlers/http10.py | 3 +- scrapy/core/http2/agent.py | 26 +++++---- scrapy/core/http2/protocol.py | 18 ++++-- scrapy/core/http2/stream.py | 6 +- scrapy/core/scraper.py | 6 +- scrapy/core/spidermw.py | 7 ++- scrapy/crawler.py | 33 +++++++---- scrapy/downloadermiddlewares/robotstxt.py | 26 ++++++--- scrapy/dupefilters.py | 4 +- scrapy/extensions/feedexport.py | 16 ++--- scrapy/extensions/statsmailer.py | 2 +- scrapy/mail.py | 12 ++-- scrapy/middleware.py | 27 ++++++--- scrapy/shell.py | 10 +++- scrapy/signalmanager.py | 4 +- scrapy/spiders/__init__.py | 10 ++-- scrapy/utils/defer.py | 71 ++++++++++++----------- scrapy/utils/signal.py | 24 ++++---- scrapy/utils/test.py | 3 +- scrapy/utils/testproc.py | 4 +- 21 files changed, 190 insertions(+), 128 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index e6c5e2a47..1265aa38e 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -153,7 +153,7 @@ class Command(BaseRunSpiderCommand): @overload def iterate_spider_output(self, result: _T) -> Iterable[Any]: ... - def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred]: + def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred[Any]]: if inspect.isasyncgen(result): d = deferred_from_coro( collect_asyncgen(aiter_errback(result, self.handle_exception)) @@ -233,7 +233,7 @@ class Command(BaseRunSpiderCommand): response: Response, callback: Callable, cb_kwargs: Optional[Dict[str, Any]] = None, - ) -> Deferred: + ) -> Deferred[Any]: cb_kwargs = cb_kwargs or {} d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) return d @@ -345,7 +345,7 @@ class Command(BaseRunSpiderCommand): def prepare_request( self, spider: Spider, request: Request, opts: argparse.Namespace ) -> Request: - def callback(response: Response, **cb_kwargs: Any) -> Deferred: + def callback(response: Response, **cb_kwargs: Any) -> Deferred[List[Any]]: # memorize first request if not self.first_response: self.first_response = response diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 98f62efcf..8d7b0635c 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -10,6 +10,7 @@ from scrapy.utils.python import to_unicode if TYPE_CHECKING: from twisted.internet.defer import Deferred + from twisted.internet.interfaces import IConnector # typing.Self requires Python 3.11 from typing_extensions import Self @@ -45,7 +46,7 @@ class HTTP10DownloadHandler: self._connect(factory) return factory.deferred - def _connect(self, factory: ScrapyHTTPClientFactory) -> Deferred: + def _connect(self, factory: ScrapyHTTPClientFactory) -> IConnector: from twisted.internet import reactor host, port = to_unicode(factory.host), factory.port diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index d291a5b8a..640fb7129 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -21,7 +21,7 @@ if TYPE_CHECKING: from twisted.internet.base import ReactorBase from twisted.internet.endpoints import HostnameEndpoint - from scrapy.http.request import Request + from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider @@ -39,16 +39,18 @@ class H2ConnectionPool: self._connections: Dict[ConnectionKeyT, H2ClientProtocol] = {} # Save all requests that arrive before the connection is established - self._pending_requests: Dict[ConnectionKeyT, Deque[Deferred]] = {} + self._pending_requests: Dict[ + ConnectionKeyT, Deque[Deferred[H2ClientProtocol]] + ] = {} def get_connection( self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint - ) -> Deferred: + ) -> Deferred[H2ClientProtocol]: if key in self._pending_requests: # Received a request while connecting to remote # Create a deferred which will fire with the H2ClientProtocol # instance - d: Deferred = Deferred() + d: Deferred[H2ClientProtocol] = Deferred() self._pending_requests[key].append(d) return d @@ -63,17 +65,17 @@ class H2ConnectionPool: def _new_connection( self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint - ) -> Deferred: + ) -> Deferred[H2ClientProtocol]: self._pending_requests[key] = deque() - conn_lost_deferred: Deferred = Deferred() + conn_lost_deferred: Deferred[List[BaseException]] = Deferred() conn_lost_deferred.addCallback(self._remove_connection, key) factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) conn_d = endpoint.connect(factory) conn_d.addCallback(self.put_connection, key) - d: Deferred = Deferred() + d: Deferred[H2ClientProtocol] = Deferred() self._pending_requests[key].append(d) return d @@ -141,7 +143,7 @@ class H2Agent: """ return uri.scheme, uri.host, uri.port - def request(self, request: Request, spider: Spider) -> Deferred: + def request(self, request: Request, spider: Spider) -> Deferred[Response]: uri = URI.fromBytes(bytes(request.url, encoding="utf-8")) try: endpoint = self.get_endpoint(uri) @@ -149,9 +151,11 @@ class H2Agent: return defer.fail(Failure()) key = self.get_key(uri) - d = self._pool.get_connection(key, uri, endpoint) - d.addCallback(lambda conn: conn.request(request, spider)) - return d + d: Deferred[H2ClientProtocol] = self._pool.get_connection(key, uri, endpoint) + d2: Deferred[Response] = d.addCallback( + lambda conn: conn.request(request, spider) + ) + return d2 class ScrapyProxyH2Agent(H2Agent): diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index a6809102b..8aebbaab4 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -33,7 +33,7 @@ from twisted.protocols.policies import TimeoutMixin from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason -from scrapy.http import Request +from scrapy.http import Request, Response if TYPE_CHECKING: from ipaddress import IPv4Address, IPv6Address @@ -88,7 +88,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin): IDLE_TIMEOUT = 240 def __init__( - self, uri: URI, settings: Settings, conn_lost_deferred: Deferred + self, + uri: URI, + settings: Settings, + conn_lost_deferred: Deferred[List[BaseException]], ) -> None: """ Arguments: @@ -99,7 +102,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): conn_lost_deferred -- Deferred fires with the reason: Failure to notify that connection was lost """ - self._conn_lost_deferred = conn_lost_deferred + self._conn_lost_deferred: Deferred[List[BaseException]] = conn_lost_deferred config = H2Configuration(client_side=True, header_encoding="utf-8") self.conn = H2Connection(config=config) @@ -215,14 +218,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin): data = self.conn.data_to_send() self.transport.write(data) - def request(self, request: Request, spider: Spider) -> Deferred: + def request(self, request: Request, spider: Spider) -> Deferred[Response]: if not isinstance(request, Request): raise TypeError( f"Expected scrapy.http.Request, received {request.__class__.__qualname__}" ) stream = self._new_stream(request, spider) - d = stream.get_response() + d: Deferred[Response] = stream.get_response() # Add the stream to the request pool self._pending_request_stream_pool.append(stream) @@ -436,7 +439,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin): @implementer(IProtocolNegotiationFactory) class H2ClientFactory(Factory): def __init__( - self, uri: URI, settings: Settings, conn_lost_deferred: Deferred + self, + uri: URI, + settings: Settings, + conn_lost_deferred: Deferred[List[BaseException]], ) -> None: self.uri = uri self.settings = settings diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index a02fbb328..d8b5cc8eb 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -20,7 +20,7 @@ if TYPE_CHECKING: from hpack import HeaderTuple from scrapy.core.http2.protocol import H2ClientProtocol - from scrapy.http import Request + from scrapy.http import Request, Response logger = logging.getLogger(__name__) @@ -154,7 +154,7 @@ class Stream: else: self.close(StreamCloseReason.CANCELLED) - self._deferred_response: Deferred = Deferred(_cancel) + self._deferred_response: Deferred[Response] = Deferred(_cancel) def __repr__(self) -> str: return f"Stream(id={self.stream_id!r})" @@ -180,7 +180,7 @@ class Stream: and not self.metadata["reached_warnsize"] ) - def get_response(self) -> Deferred: + def get_response(self) -> Deferred[Response]: """Simply return a Deferred which fires when response from the asynchronous request is available """ diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 8a9e8f687..a7d65e1e3 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -13,6 +13,7 @@ from typing import ( Generator, Iterable, Iterator, + List, Optional, Set, Tuple, @@ -34,7 +35,6 @@ from scrapy.logformatter import LogFormatter from scrapy.pipelines import ItemPipelineManager from scrapy.signalmanager import SignalManager from scrapy.utils.defer import ( - DeferredListResultListT, aiter_errback, defer_fail, defer_succeed, @@ -54,7 +54,7 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -_ParallelResult = DeferredListResultListT[Iterator[Any]] +_ParallelResult = List[Tuple[bool, Iterator[Any]]] if TYPE_CHECKING: # parameterized Deferreds require Twisted 21.7.0 @@ -374,7 +374,7 @@ class Scraper: def _itemproc_finished( self, output: Any, item: Any, response: Response, spider: Spider - ) -> Deferred: + ) -> Deferred[Any]: """ItemProcessor finished for the given ``item`` and returned ``output``""" assert self.slot is not None # typing self.slot.itemproc_size -= 1 diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 37a666605..c9feac29c 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -302,7 +302,10 @@ class SpiderMiddlewareManager(MiddlewareManager): recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) result = await maybe_deferred_to_future( - self._process_spider_output(response, spider, result) + cast( + "Deferred[Union[Iterable[_T], AsyncIterable[_T]]]", + self._process_spider_output(response, spider, result), + ) ) if isinstance(result, AsyncIterable): return MutableAsyncChain(result, recovered) @@ -339,7 +342,7 @@ class SpiderMiddlewareManager(MiddlewareManager): def process_start_requests( self, start_requests: Iterable[Request], spider: Spider - ) -> Deferred: + ) -> Deferred[Iterable[Request]]: return self._process_chain("process_start_requests", start_requests, spider) # This method is only needed until _async compatibility methods are removed. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 4fe5987a7..877ea5928 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,18 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Generator, + Optional, + Set, + Type, + TypeVar, + Union, + cast, +) from twisted.internet.defer import ( Deferred, @@ -54,6 +65,8 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_T = TypeVar("_T") + class Crawler: def __init__( @@ -140,7 +153,7 @@ class Crawler: ) @inlineCallbacks - def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]: + def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred[Any], Any, None]: if self.crawling: raise RuntimeError("Crawling already taking place") if self._started: @@ -172,7 +185,7 @@ class Crawler: return ExecutionEngine(self, lambda _: self.stop()) @inlineCallbacks - def stop(self) -> Generator[Deferred, Any, None]: + def stop(self) -> Generator[Deferred[Any], Any, None]: """Starts a graceful stop of the crawler and returns a deferred that is fired when the crawler is stopped.""" if self.crawling: @@ -256,7 +269,7 @@ class CrawlerRunner: self.settings = settings self.spider_loader = self._get_spider_loader(settings) self._crawlers: Set[Crawler] = set() - self._active: Set[Deferred] = set() + self._active: Set[Deferred[None]] = set() self.bootstrap_failed = False def crawl( @@ -264,7 +277,7 @@ class CrawlerRunner: crawler_or_spidercls: Union[Type[Spider], str, Crawler], *args: Any, **kwargs: Any, - ) -> Deferred: + ) -> Deferred[None]: """ Run a crawler with the provided arguments. @@ -294,12 +307,12 @@ class CrawlerRunner: crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) - def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred: + def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred[None]: self.crawlers.add(crawler) d = crawler.crawl(*args, **kwargs) self._active.add(d) - def _done(result: Any) -> Any: + def _done(result: _T) -> _T: self.crawlers.discard(crawler) self._active.discard(d) self.bootstrap_failed |= not getattr(crawler, "spider", None) @@ -335,7 +348,7 @@ class CrawlerRunner: # temporary cast until self.spider_loader is typed return Crawler(cast(Type[Spider], spidercls), self.settings) - def stop(self) -> Deferred: + def stop(self) -> Deferred[Any]: """ Stops simultaneously all the crawling jobs taking place. @@ -344,7 +357,7 @@ class CrawlerRunner: return DeferredList([c.stop() for c in list(self.crawlers)]) @inlineCallbacks - def join(self) -> Generator[Deferred, Any, None]: + def join(self) -> Generator[Deferred[Any], Any, None]: """ join() @@ -460,7 +473,7 @@ class CrawlerProcess(CrawlerRunner): ) reactor.run(installSignalHandlers=install_signal_handlers) # blocking call - def _graceful_stop_reactor(self) -> Deferred: + def _graceful_stop_reactor(self) -> Deferred[Any]: d = self.stop() d.addBoth(self._stop_reactor) return d diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 70393576b..73757162f 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Dict, Optional, TypeVar, Union from twisted.internet.defer import Deferred, maybeDeferred @@ -31,6 +31,8 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_T = TypeVar("_T") + class RobotsTxtMiddleware: DOWNLOAD_PRIORITY: int = 1000 @@ -43,7 +45,9 @@ class RobotsTxtMiddleware: "ROBOTSTXT_USER_AGENT", None ) self.crawler: Crawler = crawler - self._parsers: Dict[str, Union[RobotParser, Deferred, None]] = {} + self._parsers: Dict[ + str, Union[RobotParser, Deferred[Optional[RobotParser]], None] + ] = {} self._parserimpl: RobotParser = load_object( crawler.settings.get("ROBOTSTXT_PARSER") ) @@ -55,14 +59,18 @@ class RobotsTxtMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]: + def process_request( + self, request: Request, spider: Spider + ) -> Optional[Deferred[None]]: if request.meta.get("dont_obey_robotstxt"): return None if request.url.startswith("data:") or request.url.startswith("file:"): return None - d: Deferred = maybeDeferred(self.robot_parser, request, spider) - d.addCallback(self.process_request_2, request, spider) - return d + d: Deferred[Optional[RobotParser]] = maybeDeferred( + self.robot_parser, request, spider # type: ignore[arg-type] + ) + d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider) + return d2 def process_request_2( self, rp: Optional[RobotParser], request: Request, spider: Spider @@ -86,7 +94,7 @@ class RobotsTxtMiddleware: def robot_parser( self, request: Request, spider: Spider - ) -> Union[RobotParser, Deferred, None]: + ) -> Union[RobotParser, Deferred[Optional[RobotParser]], None]: url = urlparse_cached(request) netloc = url.netloc @@ -109,9 +117,9 @@ class RobotsTxtMiddleware: parser = self._parsers[netloc] if isinstance(parser, Deferred): - d: Deferred = Deferred() + d: Deferred[Optional[RobotParser]] = Deferred() - def cb(result: Any) -> Any: + def cb(result: Optional[RobotParser]) -> Optional[RobotParser]: d.callback(result) return result diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index ffaf783a7..40ea48510 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -31,10 +31,10 @@ class BaseDupeFilter: def request_seen(self, request: Request) -> bool: return False - def open(self) -> Optional[Deferred]: + def open(self) -> Optional[Deferred[None]]: pass - def close(self, reason: str) -> Optional[Deferred]: + def close(self, reason: str) -> Optional[Deferred[None]]: pass def log(self, request: Request, spider: Spider) -> None: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 43c2d2815..0d7f5bfd4 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -149,7 +149,7 @@ class FeedStorageProtocol(Protocol): """Open the storage for the given spider. It must return a file-like object that will be used for the exporters""" - def store(self, file: IO[bytes]) -> Optional[Deferred]: + def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: """Store the given file stream""" @@ -162,7 +162,7 @@ class BlockingFeedStorage: return NamedTemporaryFile(prefix="feed-", dir=path) - def store(self, file: IO[bytes]) -> Optional[Deferred]: + def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: return deferToThread(self._store_in_thread, file) def _store_in_thread(self, file: IO[bytes]) -> None: @@ -192,7 +192,7 @@ class StdoutFeedStorage: def open(self, spider: Spider) -> IO[bytes]: return self._stdout - def store(self, file: IO[bytes]) -> Optional[Deferred]: + def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: pass @@ -211,7 +211,7 @@ class FileFeedStorage: dirname.mkdir(parents=True) return Path(self.path).open(self.write_mode) - def store(self, file: IO[bytes]) -> Optional[Deferred]: + def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: file.close() return None @@ -483,7 +483,7 @@ _FeedSlot = create_deprecated_class( class FeedExporter: - _pending_deferreds: List[Deferred] = [] + _pending_deferreds: List[Deferred[None]] = [] @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -570,7 +570,7 @@ class FeedExporter: self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed) ) - def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred]: + def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred[None]]: def get_file(slot_: FeedSlot) -> IO[bytes]: assert slot_.file if isinstance(slot_.file, PostProcessingManager): @@ -590,7 +590,7 @@ class FeedExporter: return None logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" - d: Deferred = maybeDeferred(slot.storage.store, get_file(slot)) + d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[arg-type] d.addCallback( self._handle_store_success, logmsg, spider, type(slot.storage).__name__ @@ -621,7 +621,7 @@ class FeedExporter: self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") def _handle_store_success( - self, f: Failure, logmsg: str, spider: Spider, slot_type: str + self, result: Any, logmsg: str, spider: Spider, slot_type: str ) -> None: logger.info("Stored %s", logmsg, extra={"spider": spider}) assert self.crawler.stats diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index e43de6f5c..cad607514 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -39,7 +39,7 @@ class StatsMailer: crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_closed(self, spider: Spider) -> Optional[Deferred]: + def spider_closed(self, spider: Spider) -> Optional[Deferred[None]]: spider_stats = self.stats.get_stats(spider) body = "Global stats\n\n" body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items()) diff --git a/scrapy/mail.py b/scrapy/mail.py index 3ea20e831..c020732f9 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -103,7 +103,7 @@ class MailSender: mimetype: str = "text/plain", charset: Optional[str] = None, _callback: Optional[Callable[..., None]] = None, - ) -> Optional[Deferred]: + ) -> Optional[Deferred[None]]: from twisted.internet import reactor msg: MIMEBase @@ -155,7 +155,9 @@ class MailSender: ) return None - dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8")) + dfd: Deferred[Any] = self._sendmail( + rcpts, msg.as_string().encode(charset or "utf-8") + ) dfd.addCallback(self._sent_ok, to, cc, subject, len(attachs)) dfd.addErrback(self._sent_failed, to, cc, subject, len(attachs)) reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd) @@ -198,11 +200,11 @@ class MailSender: ) return failure - def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred: + def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred[Any]: from twisted.internet import reactor msg_io = BytesIO(msg) - d: Deferred = Deferred() + d: Deferred[Any] = Deferred() factory = self._create_sender_factory(to_addrs, msg_io, d) @@ -216,7 +218,7 @@ class MailSender: return d def _create_sender_factory( - self, to_addrs: List[str], msg: IO[bytes], d: Deferred + self, to_addrs: List[str], msg: IO[bytes], d: Deferred[Any] ) -> ESMTPSenderFactory: from twisted.mail.smtp import ESMTPSenderFactory diff --git a/scrapy/middleware.py b/scrapy/middleware.py index ea5488ba1..2296db90e 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -13,6 +13,7 @@ from typing import ( List, Optional, Tuple, + TypeVar, Union, cast, ) @@ -24,16 +25,22 @@ from scrapy.utils.misc import build_from_crawler, build_from_settings, load_obje if TYPE_CHECKING: from twisted.internet.defer import Deferred + # typing.Concatenate and typing.ParamSpec require Python 3.10 # typing.Self requires Python 3.11 - from typing_extensions import Self + from typing_extensions import Concatenate, ParamSpec, Self from scrapy import Spider from scrapy.crawler import Crawler from scrapy.settings import Settings + _P = ParamSpec("_P") + logger = logging.getLogger(__name__) +_T = TypeVar("_T") +_T2 = TypeVar("_T2") + class MiddlewareManager: """Base class for implementing middleware managers""" @@ -98,16 +105,22 @@ class MiddlewareManager: if hasattr(mw, "close_spider"): self.methods["close_spider"].appendleft(mw.close_spider) - def _process_parallel(self, methodname: str, obj: Any, *args: Any) -> Deferred: - methods = cast(Iterable[Callable], self.methods[methodname]) + def _process_parallel( + self, methodname: str, obj: _T, *args: Any + ) -> Deferred[List[_T2]]: + methods = cast( + "Iterable[Callable[Concatenate[_T, _P], _T2]]", self.methods[methodname] + ) return process_parallel(methods, obj, *args) - def _process_chain(self, methodname: str, obj: Any, *args: Any) -> Deferred: - methods = cast(Iterable[Callable], self.methods[methodname]) + def _process_chain(self, methodname: str, obj: _T, *args: Any) -> Deferred[_T]: + methods = cast( + "Iterable[Callable[Concatenate[_T, _P], _T]]", self.methods[methodname] + ) return process_chain(methods, obj, *args) - def open_spider(self, spider: Spider) -> Deferred: + def open_spider(self, spider: Spider) -> Deferred[List[None]]: return self._process_parallel("open_spider", spider) - def close_spider(self, spider: Spider) -> Deferred: + def close_spider(self, spider: Spider) -> Deferred[List[None]]: return self._process_parallel("close_spider", spider) diff --git a/scrapy/shell.py b/scrapy/shell.py index 2c22d3d8f..b7e46274f 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -4,6 +4,8 @@ See documentation in docs/topics/shell.rst """ +from __future__ import annotations + import os import signal from typing import Any, Callable, Dict, Optional, Tuple, Union @@ -92,7 +94,9 @@ class Shell: self.vars, shells=shells, banner=self.vars.pop("banner", "") ) - def _schedule(self, request: Request, spider: Optional[Spider]) -> defer.Deferred: + def _schedule( + self, request: Request, spider: Optional[Spider] + ) -> defer.Deferred[Any]: if is_asyncio_reactor_installed(): # set the asyncio event loop for the current thread event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"] @@ -209,7 +213,7 @@ def inspect_response(response: Response, spider: Spider) -> None: signal.signal(signal.SIGINT, sigint_handler) -def _request_deferred(request: Request) -> defer.Deferred: +def _request_deferred(request: Request) -> defer.Deferred[Any]: """Wrap a request inside a Deferred. This function is harmful, do not use it until you know what you are doing. @@ -228,7 +232,7 @@ def _request_deferred(request: Request) -> defer.Deferred: request.errback = request_errback return result - d: defer.Deferred = defer.Deferred() + d: defer.Deferred[Any] = defer.Deferred() d.addBoth(_restore_callbacks) if request.callback: d.addCallback(request.callback) diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 3d37b8235..b2c6dea5d 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -50,7 +50,9 @@ class SignalManager: kwargs.setdefault("sender", self.sender) return _signal.send_catch_log(signal, **kwargs) - def send_catch_log_deferred(self, signal: Any, **kwargs: Any) -> Deferred: + def send_catch_log_deferred( + self, signal: Any, **kwargs: Any + ) -> Deferred[List[Tuple[Any, Any]]]: """ Like :meth:`send_catch_log` but supports returning :class:`~twisted.internet.defer.Deferred` objects from signal handlers. diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 7b43f04f2..f0b0c0988 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,9 +7,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union, cast - -from twisted.internet.defer import Deferred +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, cast from scrapy import signals from scrapy.http import Request, Response @@ -19,6 +17,8 @@ from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: from collections.abc import Callable + from twisted.internet.defer import Deferred + # typing.Concatenate requires Python 3.10 # typing.Self requires Python 3.11 from typing_extensions import Concatenate, Self @@ -105,10 +105,10 @@ class Spider(object_ref): return url_is_from_spider(request.url, cls) @staticmethod - def close(spider: Spider, reason: str) -> Union[Deferred, None]: + def close(spider: Spider, reason: str) -> Optional[Deferred[None]]: closed = getattr(spider, "closed", None) if callable(closed): - return cast(Union[Deferred, None], closed(reason)) + return cast("Optional[Deferred[None]]", closed(reason)) return None def __repr__(self) -> str: diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 1d578e8a3..c5763a06c 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -49,13 +49,8 @@ if TYPE_CHECKING: _T = TypeVar("_T") _T2 = TypeVar("_T2") -# copied from twisted.internet.defer -_SelfResultT = TypeVar("_SelfResultT") -_DeferredListResultItemT = Tuple[bool, _SelfResultT] -DeferredListResultListT = List[_DeferredListResultItemT[_SelfResultT]] - -def defer_fail(_failure: Failure) -> Deferred: +def defer_fail(_failure: Failure) -> Deferred[Any]: """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -64,7 +59,7 @@ def defer_fail(_failure: Failure) -> Deferred: """ from twisted.internet import reactor - d: Deferred = Deferred() + d: Deferred[Any] = Deferred() reactor.callLater(0.1, d.errback, _failure) return d @@ -78,12 +73,12 @@ def defer_succeed(result: _T) -> Deferred[_T]: """ from twisted.internet import reactor - d: Deferred = Deferred() + d: Deferred[_T] = Deferred() reactor.callLater(0.1, d.callback, result) return d -def defer_result(result: Any) -> Deferred: +def defer_result(result: Any) -> Deferred[Any]: if isinstance(result, Deferred): return result if isinstance(result, failure.Failure): @@ -138,14 +133,14 @@ def parallel( callable: Callable[Concatenate[_T, _P], _T2], *args: _P.args, **named: _P.kwargs, -) -> Deferred[DeferredListResultListT[Iterator[_T2]]]: +) -> Deferred[List[Tuple[bool, Iterator[_T2]]]]: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. Taken from: https://jcalderone.livejournal.com/24285.html """ coop = Cooperator() - work = (callable(elem, *args, **named) for elem in iterable) + work: Iterator[_T2] = (callable(elem, *args, **named) for elem in iterable) return DeferredList([coop.coiterate(work) for _ in range(count)]) @@ -198,16 +193,16 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]): def __init__( self, aiterable: AsyncIterable[_T], - callable: Callable[Concatenate[_T, _P], _T2], + callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]], *callable_args: _P.args, **callable_kwargs: _P.kwargs, ): self.aiterator: AsyncIterator[_T] = aiterable.__aiter__() - self.callable: Callable[Concatenate[_T, _P], _T2] = callable + self.callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]] = callable self.callable_args: Tuple[Any, ...] = callable_args self.callable_kwargs: Dict[str, Any] = callable_kwargs self.finished: bool = False - self.waiting_deferreds: List[Deferred] = [] + self.waiting_deferreds: List[Deferred[Any]] = [] self.anext_deferred: Optional[Deferred[_T]] = None def _callback(self, result: _T) -> None: @@ -241,12 +236,12 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]): self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) self.anext_deferred.addCallbacks(self._callback, self._errback) - def __next__(self) -> Deferred: + def __next__(self) -> Deferred[Any]: # This puts a new Deferred into self.waiting_deferreds and returns it. # It also calls __anext__() if needed. if self.finished: raise StopIteration - d: Deferred = Deferred() + d: Deferred[Any] = Deferred() self.waiting_deferreds.append(d) if not self.anext_deferred: self._call_anext() @@ -256,25 +251,29 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]): def parallel_async( async_iterable: AsyncIterable[_T], count: int, - callable: Callable[Concatenate[_T, _P], _T2], + callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]], *args: _P.args, **named: _P.kwargs, -) -> Deferred[DeferredListResultListT[Iterator[_T2]]]: - """Like parallel but for async iterators""" +) -> Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]]: + """Like ``parallel`` but for async iterators""" coop = Cooperator() - work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named) - dl: Deferred = DeferredList([coop.coiterate(work) for _ in range(count)]) + work: Iterator[Deferred[Any]] = _AsyncCooperatorAdapter( + async_iterable, callable, *args, **named + ) + dl: Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]] = DeferredList( + [coop.coiterate(work) for _ in range(count)] + ) return dl def process_chain( - callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], - input: Any, + callbacks: Iterable[Callable[Concatenate[_T, _P], _T]], + input: _T, *a: _P.args, **kw: _P.kwargs, -) -> Deferred: +) -> Deferred[_T]: """Return a Deferred built by chaining the given callbacks""" - d: Deferred = Deferred() + d: Deferred[_T] = Deferred() for x in callbacks: d.addCallback(x, *a, **kw) d.callback(input) @@ -307,19 +306,21 @@ def process_chain_both( def process_parallel( - callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], - input: Any, + callbacks: Iterable[Callable[Concatenate[_T, _P], _T2]], + input: _T, *a: _P.args, **kw: _P.kwargs, -) -> Deferred: +) -> Deferred[List[_T2]]: """Return a Deferred with the output of all successful calls to the given callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d: Deferred = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) - d.addCallback(lambda r: [x[1] for x in r]) - d.addErrback(lambda f: f.value.subFailure) - return d + d: Deferred[List[Tuple[bool, _T2]]] = DeferredList( + dfds, fireOnOneErrback=True, consumeErrors=True + ) + d2: Deferred[List[_T2]] = d.addCallback(lambda r: [x[1] for x in r]) + d2.addErrback(lambda f: f.value.subFailure) + return d2 def iter_errback( @@ -404,7 +405,7 @@ def deferred_f_from_coro_f( def maybeDeferred_coro( f: Callable[_P, Any], *args: _P.args, **kw: _P.kwargs -) -> Deferred: +) -> Deferred[Any]: """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" try: result = f(*args, **kw) @@ -420,7 +421,7 @@ def maybeDeferred_coro( return defer.succeed(result) -def deferred_to_future(d: Deferred) -> Future: +def deferred_to_future(d: Deferred[_T]) -> Future[_T]: """ .. versionadded:: 2.6.0 @@ -442,7 +443,7 @@ def deferred_to_future(d: Deferred) -> Future: return d.asFuture(_get_asyncio_event_loop()) -def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: +def maybe_deferred_to_future(d: Deferred[_T]) -> Union[Deferred[_T], Future[_T]]: """ .. versionadded:: 2.6.0 diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index bb6d807ee..4310c1d56 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,5 +1,7 @@ """Helper functions for working with signals""" +from __future__ import annotations + import collections.abc import logging from typing import Any as TypingAny @@ -27,7 +29,7 @@ def send_catch_log( signal: TypingAny = Any, sender: TypingAny = Anonymous, *arguments: TypingAny, - **named: TypingAny + **named: TypingAny, ) -> List[Tuple[TypingAny, TypingAny]]: """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. @@ -73,8 +75,8 @@ def send_catch_log_deferred( signal: TypingAny = Any, sender: TypingAny = Anonymous, *arguments: TypingAny, - **named: TypingAny -) -> Deferred: + **named: TypingAny, +) -> Deferred[List[Tuple[TypingAny, TypingAny]]]: """Like send_catch_log but supports returning deferreds on signal handlers. Returns a deferred that gets fired once all signal handlers deferreds were fired. @@ -92,23 +94,25 @@ def send_catch_log_deferred( dont_log = named.pop("dont_log", None) spider = named.get("spider", None) - dfds = [] + dfds: List[Deferred[Tuple[TypingAny, TypingAny]]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): - d = maybeDeferred_coro( + d: Deferred[TypingAny] = maybeDeferred_coro( robustApply, receiver, signal=signal, sender=sender, *arguments, **named ) d.addErrback(logerror, receiver) # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html - d.addBoth( + d2: Deferred[Tuple[TypingAny, TypingAny]] = d.addBoth( lambda result: ( receiver, # pylint: disable=cell-var-from-loop # noqa: B023 result, ) ) - dfds.append(d) - d = DeferredList(dfds) - d.addCallback(lambda out: [x[1] for x in out]) - return d + dfds.append(d2) + dl = DeferredList(dfds) + d3: Deferred[List[Tuple[TypingAny, TypingAny]]] = dl.addCallback( + lambda out: [x[1] for x in out] + ) + return d3 def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None: diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index fe2bfa042..30f235592 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -30,6 +30,7 @@ from scrapy.utils.boto import is_botocore_available if TYPE_CHECKING: from twisted.internet.defer import Deferred + from twisted.web.client import Response as TxResponse _T = TypeVar("_T") @@ -159,7 +160,7 @@ def mock_google_cloud_storage() -> Tuple[Any, Any, Any]: return (client_mock, bucket_mock, blob_mock) -def get_web_client_agent_req(url: str) -> Deferred: +def get_web_client_agent_req(url: str) -> Deferred[TxResponse]: from twisted.internet import reactor from twisted.web.client import Agent # imports twisted.internet.reactor diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 8882bfc5f..bb269a9f5 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -22,7 +22,7 @@ class ProcessTest: args: Iterable[str], check_code: bool = True, settings: Optional[str] = None, - ) -> Deferred: + ) -> Deferred[TestProcessProtocol]: from twisted.internet import reactor env = os.environ.copy() @@ -49,7 +49,7 @@ class ProcessTest: class TestProcessProtocol(ProcessProtocol): def __init__(self) -> None: - self.deferred: Deferred = Deferred() + self.deferred: Deferred[TestProcessProtocol] = Deferred() self.out: bytes = b"" self.err: bytes = b"" self.exitcode: Optional[int] = None From 96d6519b25a3d7b02e8efa1180f6f59e5244f977 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 26 Jun 2024 17:43:59 +0500 Subject: [PATCH 229/269] Bump twine in twinecheck. (#6416) --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d665fc5a5..c325064d9 100644 --- a/tox.ini +++ b/tox.ini @@ -88,7 +88,7 @@ commands = [testenv:twinecheck] basepython = python3 deps = - twine==4.0.2 + twine==5.0.0 build==1.0.3 commands = python -m build --sdist From 41e15e93e7459673e93ff2591462b47b7ae01566 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 26 Jun 2024 17:44:12 +0500 Subject: [PATCH 230/269] Remove an obsolete import. (#6415) --- scrapy/crawler.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 877ea5928..ecb0a8150 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -23,13 +23,6 @@ from twisted.internet.defer import ( inlineCallbacks, maybeDeferred, ) - -try: - # zope >= 5.0 only supports MultipleInvalid - from zope.interface.exceptions import MultipleInvalid -except ImportError: - MultipleInvalid = None - from zope.interface.verify import verifyClass from scrapy import Spider, signals From 558b1d11d2f1e3063aba59d444fdb93d42a9ddb9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 1 Jul 2024 12:30:49 +0500 Subject: [PATCH 231/269] Use CallbackT for Request.callback. (#6422) --- scrapy/commands/parse.py | 16 ++++++++-------- scrapy/contracts/__init__.py | 7 ++++--- scrapy/http/request/__init__.py | 20 ++++++++++++++------ scrapy/http/response/__init__.py | 12 +++++++----- scrapy/http/response/text.py | 12 +++++++----- scrapy/spiders/__init__.py | 8 ++------ scrapy/spiders/crawl.py | 15 +++++++++------ scrapy/spiders/sitemap.py | 8 ++++---- 8 files changed, 55 insertions(+), 43 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 1265aa38e..fbd200d88 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -9,7 +9,6 @@ from typing import ( TYPE_CHECKING, Any, AsyncGenerator, - Callable, Coroutine, Dict, Iterable, @@ -38,6 +37,7 @@ from scrapy.utils.spider import spidercls_for_request if TYPE_CHECKING: from twisted.python.failure import Failure + from scrapy.http.request import CallbackT from scrapy.spiders import Spider @@ -218,8 +218,8 @@ class Command(BaseRunSpiderCommand): opts: argparse.Namespace, depth: int, spider: Spider, - callback: Callable, - ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, Callable]: + callback: CallbackT, + ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT]: items, requests = [], [] for x in spider_output: if is_item(x): @@ -231,7 +231,7 @@ class Command(BaseRunSpiderCommand): def run_callback( self, response: Response, - callback: Callable, + callback: CallbackT, cb_kwargs: Optional[Dict[str, Any]] = None, ) -> Deferred[Any]: cb_kwargs = cb_kwargs or {} @@ -240,7 +240,7 @@ class Command(BaseRunSpiderCommand): def get_callback_from_rules( self, spider: Spider, response: Response - ) -> Union[Callable, str, None]: + ) -> Union[CallbackT, str, None]: if getattr(spider, "rules", None): for rule in spider.rules: # type: ignore[attr-defined] if rule.link_extractor.matches(response.url): @@ -286,7 +286,7 @@ class Command(BaseRunSpiderCommand): def scraped_data( self, args: Tuple[ - List[Any], List[Request], argparse.Namespace, int, Spider, Callable + List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT ], ) -> List[Any]: items, requests, opts, depth, spider, callback = args @@ -313,8 +313,8 @@ class Command(BaseRunSpiderCommand): spider: Spider, opts: argparse.Namespace, response: Optional[Response] = None, - ) -> Callable: - cb: Union[str, Callable, None] = None + ) -> CallbackT: + cb: Union[str, CallbackT, None] = None if response: cb = response.meta["_callback"] if not cb: diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 440e0dc44..a7e129948 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -16,6 +16,7 @@ from typing import ( Optional, Tuple, Type, + cast, ) from unittest import TestCase, TestResult @@ -62,7 +63,7 @@ class Contract: if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") return list( # pylint: disable=return-in-finally - iterate_spider_output(cb_result) + cast(Iterable[Any], iterate_spider_output(cb_result)) ) request.callback = wrapper @@ -79,7 +80,7 @@ class Contract: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") - output = list(iterate_spider_output(cb_result)) + output = list(cast(Iterable[Any], iterate_spider_output(cb_result))) try: results.startTest(self.testcase_post) self.post_process(output) @@ -195,7 +196,7 @@ class ContractsManager: def cb_wrapper(response: Response, **cb_kwargs: Any) -> None: try: output = cb(response, **cb_kwargs) - output = list(iterate_spider_output(output)) + output = list(cast(Iterable[Any], iterate_spider_output(output))) except Exception: case = _create_testcase(method, "callback") results.addError(case, sys.exc_info()) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 04589dd37..9381a6cb3 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -12,7 +12,6 @@ from typing import ( TYPE_CHECKING, Any, AnyStr, - Callable, Dict, Iterable, List, @@ -37,8 +36,17 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax if TYPE_CHECKING: + from collections.abc import Callable + + from twisted.python.failure import Failure + + # typing.Concatenate requires Python 3.10 # typing.NotRequired and typing.Self require Python 3.11 - from typing_extensions import NotRequired, Self + from typing_extensions import Concatenate, NotRequired, Self + + from scrapy.http import Response + + CallbackT = Callable[Concatenate[Response, ...], Any] class VerboseCookie(TypedDict): @@ -110,7 +118,7 @@ class Request(object_ref): def __init__( self, url: str, - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, @@ -119,7 +127,7 @@ class Request(object_ref): encoding: str = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, flags: Optional[List[str]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, ) -> None: @@ -137,8 +145,8 @@ class Request(object_ref): ) if not (callable(errback) or errback is None): raise TypeError(f"errback must be a callable, got {type(errback).__name__}") - self.callback: Optional[Callable] = callback - self.errback: Optional[Callable] = errback + self.callback: Optional[CallbackT] = callback + self.errback: Optional[Callable[[Failure], Any]] = errback self.cookies: CookiesT = cookies or {} self.headers: Headers = Headers(headers or {}, encoding=encoding) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index ff3581abb..92e4852b6 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -27,7 +27,7 @@ from urllib.parse import urljoin from scrapy.exceptions import NotSupported from scrapy.http.headers import Headers -from scrapy.http.request import CookiesT, Request +from scrapy.http.request import Request from scrapy.link import Link from scrapy.utils.trackref import object_ref @@ -35,10 +35,12 @@ if TYPE_CHECKING: from ipaddress import IPv4Address, IPv6Address from twisted.internet.ssl import Certificate + from twisted.python.failure import Failure # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.http.request import CallbackT, CookiesT from scrapy.selector import SelectorList @@ -196,7 +198,7 @@ class Response(object_ref): def follow( self, url: Union[str, Link], - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, @@ -205,7 +207,7 @@ class Response(object_ref): encoding: Optional[str] = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, ) -> Request: @@ -249,7 +251,7 @@ class Response(object_ref): def follow_all( self, urls: Iterable[Union[str, Link]], - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, @@ -258,7 +260,7 @@ class Response(object_ref): encoding: Optional[str] = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, ) -> Iterable[Request]: diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 0635f744f..588695002 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -41,7 +41,9 @@ from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url if TYPE_CHECKING: - from scrapy.http.request import CookiesT, Request + from twisted.python.failure import Failure + + from scrapy.http.request import CallbackT, CookiesT, Request from scrapy.selector import Selector, SelectorList @@ -179,7 +181,7 @@ class TextResponse(Response): def follow( self, url: Union[str, Link, parsel.Selector], - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, @@ -188,7 +190,7 @@ class TextResponse(Response): encoding: Optional[str] = None, priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, ) -> Request: @@ -232,7 +234,7 @@ class TextResponse(Response): def follow_all( self, urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None, - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, @@ -241,7 +243,7 @@ class TextResponse(Response): encoding: Optional[str] = None, priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, css: Optional[str] = None, diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index f0b0c0988..d977acd26 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -15,20 +15,16 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: - from collections.abc import Callable - from twisted.internet.defer import Deferred - # typing.Concatenate requires Python 3.10 # typing.Self requires Python 3.11 - from typing_extensions import Concatenate, Self + from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import CallbackT from scrapy.settings import BaseSettings, _SettingsKeyT from scrapy.utils.log import SpiderLoggerAdapter - CallbackT = Callable[Concatenate[Response, ...], Any] - class Spider(object_ref): """Base class for scrapy spiders. All spiders must inherit from this diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 48c830d2a..2639f14b2 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -39,6 +39,7 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import CallbackT _T = TypeVar("_T") @@ -73,7 +74,7 @@ class Rule: def __init__( self, link_extractor: Optional[LinkExtractor] = None, - callback: Union[Callable, str, None] = None, + callback: Union[CallbackT, str, None] = None, cb_kwargs: Optional[Dict[str, Any]] = None, follow: Optional[bool] = None, process_links: Union[ProcessLinksT, str, None] = None, @@ -81,7 +82,7 @@ class Rule: errback: Union[Callable[[Failure], Any], str, None] = None, ): self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor - self.callback: Union[Callable, str, None] = callback + self.callback: Union[CallbackT, str, None] = callback self.errback: Union[Callable[[Failure], Any], str, None] = errback self.cb_kwargs: Dict[str, Any] = cb_kwargs or {} self.process_links: Union[ProcessLinksT, str] = process_links or _identity @@ -92,7 +93,7 @@ class Rule: def _compile(self, spider: Spider) -> None: # this replaces method names with methods and we can't express this in type hints - self.callback = _get_method(self.callback, spider) + self.callback = cast("CallbackT", _get_method(self.callback, spider)) self.errback = cast(Callable[[Failure], Any], _get_method(self.errback, spider)) self.process_links = cast( ProcessLinksT, _get_method(self.process_links, spider) @@ -122,7 +123,9 @@ class CrawlSpider(Spider): def parse_start_url(self, response: Response, **kwargs: Any) -> Any: return [] - def process_results(self, response: Response, results: Any) -> Any: + def process_results( + self, response: Response, results: Iterable[Any] + ) -> Iterable[Any]: return results def _build_request(self, rule_index: int, link: Link) -> Request: @@ -152,7 +155,7 @@ class CrawlSpider(Spider): rule = self._rules[cast(int, response.meta["rule"])] return self._parse_response( response, - cast(Callable, rule.callback), + cast("CallbackT", rule.callback), {**rule.cb_kwargs, **cb_kwargs}, rule.follow, ) @@ -166,7 +169,7 @@ class CrawlSpider(Spider): async def _parse_response( self, response: Response, - callback: Optional[Callable], + callback: Optional[CallbackT], cb_kwargs: Dict[str, Any], follow: bool = True, ) -> AsyncIterable[Any]: diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index d082fbfdb..1542ef79c 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -5,7 +5,6 @@ import re from typing import ( TYPE_CHECKING, Any, - Callable, Dict, Iterable, List, @@ -27,6 +26,7 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import CallbackT logger = logging.getLogger(__name__) @@ -34,7 +34,7 @@ logger = logging.getLogger(__name__) class SitemapSpider(Spider): sitemap_urls: Sequence[str] = () sitemap_rules: Sequence[ - Tuple[Union[re.Pattern[str], str], Union[str, Callable]] + Tuple[Union[re.Pattern[str], str], Union[str, CallbackT]] ] = [("", "parse")] sitemap_follow: Sequence[Union[re.Pattern[str], str]] = [""] sitemap_alternate_links: bool = False @@ -54,10 +54,10 @@ class SitemapSpider(Spider): def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) - self._cbs: List[Tuple[re.Pattern[str], Callable]] = [] + self._cbs: List[Tuple[re.Pattern[str], CallbackT]] = [] for r, c in self.sitemap_rules: if isinstance(c, str): - c = cast(Callable, getattr(self, c)) + c = cast("CallbackT", getattr(self, c)) self._cbs.append((regex(r), c)) self._follow: List[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] From d8ecd28c5557e27f42e00bd1223b457468ea2ea7 Mon Sep 17 00:00:00 2001 From: mlmsmith <mlmsmith@hotmail.co.uk> Date: Thu, 4 Jul 2024 18:16:26 +0800 Subject: [PATCH 232/269] Documentation improvements (#6429) --- docs/intro/install.rst | 13 ++++--------- docs/intro/overview.rst | 12 ++++++------ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index c90c1d2bf..e6c9a683b 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -37,7 +37,7 @@ Note that sometimes this may require solving compilation issues for some Scrapy dependencies depending on your operating system, so be sure to check the :ref:`intro-install-platform-notes`. -For more detailed and platform specifics instructions, as well as +For more detailed and platform-specific instructions, as well as troubleshooting information, read on. @@ -101,7 +101,7 @@ Windows ------- Though it's possible to install Scrapy on Windows using pip, we recommend you -to install `Anaconda`_ or `Miniconda`_ and use the package from the +install `Anaconda`_ or `Miniconda`_ and use the package from the `conda-forge`_ channel, which will avoid most installation issues. Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: @@ -141,7 +141,7 @@ But it should support older versions of Ubuntu too, like Ubuntu 14.04, albeit with potential issues with TLS connections. **Don't** use the ``python-scrapy`` package provided by Ubuntu, they are -typically too old and slow to catch up with latest Scrapy. +typically too old and slow to catch up with the latest Scrapy release. To install Scrapy on Ubuntu (or Ubuntu-based) systems, you need to install @@ -170,7 +170,7 @@ macOS Building Scrapy's dependencies requires the presence of a C compiler and development headers. On macOS this is typically provided by Apple’s Xcode -development tools. To install the Xcode command line tools open a terminal +development tools. To install the Xcode command-line tools, open a terminal window and run:: xcode-select --install @@ -200,11 +200,6 @@ solutions: brew install python - * Latest versions of python have ``pip`` bundled with them so you won't need - to install it separately. If this is not the case, upgrade python:: - - brew update; brew upgrade python - * *(Optional)* :ref:`Install Scrapy inside a Python virtual environment <intro-using-virtualenv>`. diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index ef1294470..cd17b1968 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -65,7 +65,7 @@ When you ran the command ``scrapy runspider quotes_spider.py``, Scrapy looked fo Spider definition inside it and ran it through its crawler engine. The crawl started by making requests to the URLs defined in the ``start_urls`` -attribute (in this case, only the URL for quotes in *humor* category) +attribute (in this case, only the URL for quotes in the *humor* category) and called the default callback method ``parse``, passing the response object as an argument. In the ``parse`` callback, we loop through the quote elements using a CSS Selector, yield a Python dict with the extracted quote text and author, @@ -83,9 +83,9 @@ While this enables you to do very fast crawls (sending multiple concurrent requests at the same time, in a fault-tolerant way) Scrapy also gives you control over the politeness of the crawl through :ref:`a few settings <topics-settings-ref>`. You can do things like setting a download delay between -each request, limiting amount of concurrent requests per domain or per IP, and +each request, limiting the amount of concurrent requests per domain or per IP, and even :ref:`using an auto-throttling extension <topics-autothrottle>` that tries -to figure out these automatically. +to figure these settings out automatically. .. note:: @@ -106,10 +106,10 @@ scraping easy and efficient, such as: * Built-in support for :ref:`selecting and extracting <topics-selectors>` data from HTML/XML sources using extended CSS selectors and XPath expressions, - with helper methods to extract using regular expressions. + with helper methods for extraction using regular expressions. * An :ref:`interactive shell console <topics-shell>` (IPython aware) for trying - out the CSS and XPath expressions to scrape data, very useful when writing or + out the CSS and XPath expressions to scrape data, which is very useful when writing or debugging your spiders. * Built-in support for :ref:`generating feed exports <topics-feed-exports>` in @@ -124,7 +124,7 @@ scraping easy and efficient, such as: well-defined API (middlewares, :ref:`extensions <topics-extensions>`, and :ref:`pipelines <topics-item-pipeline>`). -* Wide range of built-in extensions and middlewares for handling: +* A wide range of built-in extensions and middlewares for handling: - cookies and session handling - HTTP features like compression, authentication, caching From ceedb026f8c8ccb049187baa14202f98b2a3a60c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 9 Jul 2024 11:34:58 +0500 Subject: [PATCH 233/269] Remove top-level imports that install the reactor from scrapy.extensions.telnet. (#6432) --- scrapy/extensions/telnet.py | 22 +++++++--------------- tests/test_crawler.py | 4 ---- tests/test_utils_log.py | 4 ---- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index c4e01b3d9..c64a0b417 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -10,21 +10,11 @@ import binascii import logging import os import pprint -import traceback from typing import TYPE_CHECKING, Any, Dict, List from twisted.internet import protocol from twisted.internet.tcp import Port -try: - from twisted.conch import manhole, telnet - from twisted.conch.insults import insults - - TWISTED_CONCH_AVAILABLE = True -except (ImportError, SyntaxError): - _TWISTED_CONCH_TRACEBACK = traceback.format_exc() - TWISTED_CONCH_AVAILABLE = False - from scrapy import signals from scrapy.exceptions import NotConfigured from scrapy.utils.decorators import defers @@ -33,6 +23,8 @@ from scrapy.utils.reactor import listen_tcp from scrapy.utils.trackref import print_live_refs if TYPE_CHECKING: + from twisted.conch import telnet + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -50,11 +42,7 @@ class TelnetConsole(protocol.ServerFactory): def __init__(self, crawler: Crawler): if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"): raise NotConfigured - if not TWISTED_CONCH_AVAILABLE: - raise NotConfigured( - "TELNETCONSOLE_ENABLED setting is True but required twisted " - "modules failed to import:\n" + _TWISTED_CONCH_TRACEBACK - ) + self.crawler: Crawler = crawler self.noisy: bool = False self.portrange: List[int] = [ @@ -88,6 +76,10 @@ class TelnetConsole(protocol.ServerFactory): self.port.stopListening() def protocol(self) -> telnet.TelnetTransport: # type: ignore[override] + # these import twisted.internet.reactor + from twisted.conch import manhole, telnet + from twisted.conch.insults import insults + class Portal: """An implementation of IPortal""" diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 791ea1faa..c87e65758 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -21,7 +21,6 @@ import scrapy from scrapy import Spider from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.extensions import telnet from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader @@ -482,7 +481,6 @@ class CrawlerLoggingTestCase(unittest.TestCase): "LOG_FILE": str(log_file), # settings to avoid extra warnings "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", - "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE, } configure_logging() @@ -516,8 +514,6 @@ class CrawlerLoggingTestCase(unittest.TestCase): custom_settings = { "LOG_FILE": str(log_file), "LOG_FILE_APPEND": False, - # disable telnet if not available to avoid an extra warning - "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE, } configure_logging() diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index a8d080822..0f75bdb5c 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -11,7 +11,6 @@ import pytest from testfixtures import LogCapture from twisted.python.failure import Failure -from scrapy.extensions import telnet from scrapy.utils.log import ( LogCounterHandler, SpiderLoggerAdapter, @@ -70,9 +69,6 @@ class TopLevelFormatterTest(unittest.TestCase): class LogCounterHandlerTest(unittest.TestCase): def setUp(self): settings = {"LOG_LEVEL": "WARNING"} - if not telnet.TWISTED_CONCH_AVAILABLE: - # disable it to avoid the extra warning - settings["TELNETCONSOLE_ENABLED"] = False self.logger = logging.getLogger("test") self.logger.setLevel(logging.NOTSET) self.logger.propagate = False From 0b8604bb5d8bffbbd5c78783022965fa2606c131 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <guillermo.bondonno@zyte.com> Date: Tue, 9 Jul 2024 15:52:49 -0300 Subject: [PATCH 234/269] add CLOSESPIDER_PAGECOUNT_NO_ITEM to CloseSpider extension --- scrapy/extensions/closespider.py | 25 +++++++++++++++++++++++++ tests/keys/mitmproxy-dhparam.pem | 14 ++++++++++++++ tests/test_closespider.py | 13 +++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 tests/keys/mitmproxy-dhparam.pem diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 4627e7f98..6ebf98e65 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, DefaultDict, Dict from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured +from scrapy.signalmanager import dispatcher if TYPE_CHECKING: from twisted.python.failure import Failure @@ -36,6 +37,9 @@ class CloseSpider: "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"), "errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"), "timeout_no_item": crawler.settings.getint("CLOSESPIDER_TIMEOUT_NO_ITEM"), + "pagecount_no_item": crawler.settings.getint( + "CLOSESPIDER_PAGECOUNT_NO_ITEM" + ), } if not any(self.close_on.values()): @@ -60,6 +64,19 @@ class CloseSpider: crawler.signals.connect( self.item_scraped_no_item, signal=signals.item_scraped ) + if self.close_on.get("pagecount_no_item"): + if self.page_count not in dispatcher.getReceivers( + signal=signals.response_received + ): + crawler.signals.connect( + self.page_count, signal=signals.response_received + ) + + if self.item_scraped not in dispatcher.getReceivers( + signal=signals.item_scraped + ): + crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) + crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @classmethod @@ -74,9 +91,16 @@ class CloseSpider: def page_count(self, response: Response, request: Request, spider: Spider) -> None: self.counter["pagecount"] += 1 + self.counter["pagecount_since_last_item"] += 1 if self.counter["pagecount"] == self.close_on["pagecount"]: assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_pagecount") + if self.close_on["pagecount_no_item"] and ( + self.counter["pagecount_since_last_item"] + >= self.close_on["pagecount_no_item"] + ): + assert self.crawler.engine + self.crawler.engine.close_spider(spider, "closespider_pagecount_no_item") def spider_opened(self, spider: Spider) -> None: from twisted.internet import reactor @@ -91,6 +115,7 @@ class CloseSpider: def item_scraped(self, item: Any, spider: Spider) -> None: self.counter["itemcount"] += 1 + self.counter["pagecount_since_last_item"] = 0 if self.counter["itemcount"] == self.close_on["itemcount"]: assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_itemcount") diff --git a/tests/keys/mitmproxy-dhparam.pem b/tests/keys/mitmproxy-dhparam.pem new file mode 100644 index 000000000..c10121fbf --- /dev/null +++ b/tests/keys/mitmproxy-dhparam.pem @@ -0,0 +1,14 @@ + +-----BEGIN DH PARAMETERS----- +MIICCAKCAgEAyT6LzpwVFS3gryIo29J5icvgxCnCebcdSe/NHMkD8dKJf8suFCg3 +O2+dguLakSVif/t6dhImxInJk230HmfC8q93hdcg/j8rLGJYDKu3ik6H//BAHKIv +j5O9yjU3rXCfmVJQic2Nne39sg3CreAepEts2TvYHhVv3TEAzEqCtOuTjgDv0ntJ +Gwpj+BJBRQGG9NvprX1YGJ7WOFBP/hWU7d6tgvE6Xa7T/u9QIKpYHMIkcN/l3ZFB +chZEqVlyrcngtSXCROTPcDOQ6Q8QzhaBJS+Z6rcsd7X+haiQqvoFcmaJ08Ks6LQC +ZIL2EtYJw8V8z7C0igVEBIADZBI6OTbuuhDwRw//zU1uq52Oc48CIZlGxTYG/Evq +o9EWAXUYVzWkDSTeBH1r4z/qLPE2cnhtMxbFxuvK53jGB0emy2y1Ei6IhKshJ5qX +IB/aE7SSHyQ3MDHHkCmQJCsOd4Mo26YX61NZ+n501XjqpCBQ2+DfZCBh8Va2wDyv +A2Ryg9SUz8j0AXViRNMJgJrr446yro/FuJZwnQcO3WQnXeqSBnURqKjmqkeFP+d8 +6mk2tqJaY507lRNqtGlLnj7f5RNoBFJDCLBNurVgfvq9TCVWKDIFD4vZRjCrnl6I +rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI= +-----END DH PARAMETERS----- diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 38ede70e4..caaa9f183 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -34,6 +34,19 @@ class TestCloseSpider(TestCase): pagecount = crawler.stats.get_value("response_received_count") self.assertTrue(pagecount >= close_on) + @defer.inlineCallbacks + def test_closespider_pagecount_no_item(self): + close_on = 5 + crawler = get_crawler( + FollowAllSpider, + {"CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on}, + ) + yield crawler.crawl(mockserver=self.mockserver) + reason = crawler.spider.meta["close_reason"] + self.assertEqual(reason, "closespider_pagecount_no_item") + pagecount = crawler.stats.get_value("response_received_count") + self.assertTrue(pagecount >= close_on) + @defer.inlineCallbacks def test_closespider_errorcount(self): close_on = 5 From a44818afeacc25cc5e05705bf8ae5804e0545c89 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <guillermo.bondonno@zyte.com> Date: Tue, 9 Jul 2024 16:07:55 -0300 Subject: [PATCH 235/269] restore mitmproxy-dhparam --- tests/keys/mitmproxy-dhparam.pem | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 tests/keys/mitmproxy-dhparam.pem diff --git a/tests/keys/mitmproxy-dhparam.pem b/tests/keys/mitmproxy-dhparam.pem deleted file mode 100644 index c10121fbf..000000000 --- a/tests/keys/mitmproxy-dhparam.pem +++ /dev/null @@ -1,14 +0,0 @@ - ------BEGIN DH PARAMETERS----- -MIICCAKCAgEAyT6LzpwVFS3gryIo29J5icvgxCnCebcdSe/NHMkD8dKJf8suFCg3 -O2+dguLakSVif/t6dhImxInJk230HmfC8q93hdcg/j8rLGJYDKu3ik6H//BAHKIv -j5O9yjU3rXCfmVJQic2Nne39sg3CreAepEts2TvYHhVv3TEAzEqCtOuTjgDv0ntJ -Gwpj+BJBRQGG9NvprX1YGJ7WOFBP/hWU7d6tgvE6Xa7T/u9QIKpYHMIkcN/l3ZFB -chZEqVlyrcngtSXCROTPcDOQ6Q8QzhaBJS+Z6rcsd7X+haiQqvoFcmaJ08Ks6LQC -ZIL2EtYJw8V8z7C0igVEBIADZBI6OTbuuhDwRw//zU1uq52Oc48CIZlGxTYG/Evq -o9EWAXUYVzWkDSTeBH1r4z/qLPE2cnhtMxbFxuvK53jGB0emy2y1Ei6IhKshJ5qX -IB/aE7SSHyQ3MDHHkCmQJCsOd4Mo26YX61NZ+n501XjqpCBQ2+DfZCBh8Va2wDyv -A2Ryg9SUz8j0AXViRNMJgJrr446yro/FuJZwnQcO3WQnXeqSBnURqKjmqkeFP+d8 -6mk2tqJaY507lRNqtGlLnj7f5RNoBFJDCLBNurVgfvq9TCVWKDIFD4vZRjCrnl6I -rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI= ------END DH PARAMETERS----- From d6352f9f66f655f11332fe6c52ed71ebb2e55bf4 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <guillermo.bondonno@zyte.com> Date: Wed, 10 Jul 2024 11:03:01 -0300 Subject: [PATCH 236/269] refactor changes on closespider.py and improve test --- scrapy/extensions/closespider.py | 18 +++--------------- tests/spiders.py | 17 +++++++++++++++++ tests/test_closespider.py | 24 +++++++++++++++++++----- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 6ebf98e65..cef5527b7 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Any, DefaultDict, Dict from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured -from scrapy.signalmanager import dispatcher if TYPE_CHECKING: from twisted.python.failure import Failure @@ -49,11 +48,11 @@ class CloseSpider: if self.close_on.get("errorcount"): crawler.signals.connect(self.error_count, signal=signals.spider_error) - if self.close_on.get("pagecount"): + if self.close_on.get("pagecount") or self.close_on.get("pagecount_no_item"): crawler.signals.connect(self.page_count, signal=signals.response_received) if self.close_on.get("timeout"): crawler.signals.connect(self.spider_opened, signal=signals.spider_opened) - if self.close_on.get("itemcount"): + if self.close_on.get("itemcount") or self.close_on.get("pagecount_no_item"): crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) if self.close_on.get("timeout_no_item"): self.timeout_no_item: int = self.close_on["timeout_no_item"] @@ -64,18 +63,6 @@ class CloseSpider: crawler.signals.connect( self.item_scraped_no_item, signal=signals.item_scraped ) - if self.close_on.get("pagecount_no_item"): - if self.page_count not in dispatcher.getReceivers( - signal=signals.response_received - ): - crawler.signals.connect( - self.page_count, signal=signals.response_received - ) - - if self.item_scraped not in dispatcher.getReceivers( - signal=signals.item_scraped - ): - crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @@ -95,6 +82,7 @@ class CloseSpider: if self.counter["pagecount"] == self.close_on["pagecount"]: assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_pagecount") + return if self.close_on["pagecount_no_item"] and ( self.counter["pagecount_since_last_item"] >= self.close_on["pagecount_no_item"] diff --git a/tests/spiders.py b/tests/spiders.py index ea419afbd..2bcec5624 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -283,6 +283,23 @@ class ItemSpider(FollowAllSpider): yield {} +class MaxItemsSpider(ItemSpider): + def __init__(self, max_items=10, *args, **kwargs): + super().__init__(*args, **kwargs) + self.max_items = max_items + self.items_scraped = 0 + + def parse(self, response): + for item_or_req in super().parse(response): + if isinstance(item_or_req, Request): + yield item_or_req + else: + if self.items_scraped >= self.max_items: + continue + self.items_scraped += 1 + yield item_or_req + + class DefaultError(Exception): pass diff --git a/tests/test_closespider.py b/tests/test_closespider.py index caaa9f183..0046b4e29 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -3,7 +3,13 @@ from twisted.trial.unittest import TestCase from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider, SlowSpider +from tests.spiders import ( + ErrorSpider, + FollowAllSpider, + ItemSpider, + MaxItemsSpider, + SlowSpider, +) class TestCloseSpider(TestCase): @@ -37,15 +43,23 @@ class TestCloseSpider(TestCase): @defer.inlineCallbacks def test_closespider_pagecount_no_item(self): close_on = 5 + close_on_pagecount = 20 + max_items = 5 crawler = get_crawler( - FollowAllSpider, - {"CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on}, + MaxItemsSpider, + { + "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on, + "CLOSESPIDER_PAGECOUNT": close_on_pagecount, + }, ) - yield crawler.crawl(mockserver=self.mockserver) + yield crawler.crawl(max_items=max_items, mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] self.assertEqual(reason, "closespider_pagecount_no_item") pagecount = crawler.stats.get_value("response_received_count") - self.assertTrue(pagecount >= close_on) + itemcount = crawler.stats.get_value("item_scraped_count") + self.assertEqual(itemcount, max_items) + self.assertLess(pagecount, close_on_pagecount) + self.assertTrue((pagecount - itemcount) >= close_on) @defer.inlineCallbacks def test_closespider_errorcount(self): From 59782d73088e46618d1c042e74ce5197e880536a Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <guillermo.bondonno@zyte.com> Date: Wed, 10 Jul 2024 11:08:22 -0300 Subject: [PATCH 237/269] update docs --- docs/topics/extensions.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index f7b2f3799..a503fd746 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -317,6 +317,18 @@ crawls more than that, the spider will be closed with the reason ``closespider_pagecount``. If zero (or non set), spiders won't be closed by number of crawled responses. +.. setting:: CLOSESPIDER_PAGECOUNT_NO_ITEM + +CLOSESPIDER_PAGECOUNT_NO_ITEM +""""""""""""""""""""" + +Default: ``0`` + +An integer which specifies the maximum number of consecutive responses to crawl without items scraped. If the spider +crawls more consecutive responses than that and no items are scraped in the meantime, the spider will be closed with the reason +``closespider_pagecount_no_item``. If zero (or non set), spiders won't be closed by +number of crawled responses with no items. + .. setting:: CLOSESPIDER_ERRORCOUNT CLOSESPIDER_ERRORCOUNT From 8646d2ec7bc44ef96f5df015e03ff37ceb5554c0 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <guillermo.bondonno@zyte.com> Date: Wed, 10 Jul 2024 11:44:44 -0300 Subject: [PATCH 238/269] fix docs detail --- docs/topics/extensions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index a503fd746..29bcaa0f2 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -320,7 +320,7 @@ number of crawled responses. .. setting:: CLOSESPIDER_PAGECOUNT_NO_ITEM CLOSESPIDER_PAGECOUNT_NO_ITEM -""""""""""""""""""""" +""""""""""""""""""""""""""""" Default: ``0`` From 129dbfa0bf1ad464ab6b50f3dee0da39853de6a1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 11 Jul 2024 12:20:36 +0500 Subject: [PATCH 239/269] Bump tool versions. --- .github/workflows/checks.yml | 4 ++-- .github/workflows/publish.yml | 4 ++-- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 2 +- .github/workflows/tests-windows.yml | 2 +- .pre-commit-config.yaml | 12 ++++++------ tox.ini | 14 +++++++------- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ed1629b67..1841bda1c 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -32,7 +32,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -46,4 +46,4 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index affaa32a5..03e94f761 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,13 +13,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: 3.12 - run: | pip install --upgrade build twine python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.6.4 + uses: pypa/gh-action-pypi-publish@v1.9.0 with: password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index a297f494c..8ebe7f1db 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index f50a4d104..763de9eff 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -62,7 +62,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 757d62285..80d09e7a0 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -35,7 +35,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 38526d720..addad838f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,11 @@ repos: - repo: https://github.com/PyCQA/bandit - rev: 1.7.7 + rev: 1.7.9 hooks: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 - rev: 7.0.0 + rev: 7.1.0 hooks: - id: flake8 additional_dependencies: @@ -16,7 +16,7 @@ repos: - flake8-string-format - flake8-type-checking - repo: https://github.com/psf/black.git - rev: 24.2.0 + rev: 24.4.2 hooks: - id: black - repo: https://github.com/pycqa/isort @@ -24,13 +24,13 @@ repos: hooks: - id: isort - repo: https://github.com/adamchainz/blacken-docs - rev: 1.16.0 + rev: 1.18.0 hooks: - id: blacken-docs additional_dependencies: - - black==24.2.0 + - black==24.4.2 - repo: https://github.com/asottile/pyupgrade - rev: v3.15.2 + rev: v3.16.0 hooks: - id: pyupgrade args: [--py38-plus, --keep-runtime-typing] diff --git a/tox.ini b/tox.ini index c325064d9..29d240031 100644 --- a/tox.ini +++ b/tox.ini @@ -46,14 +46,14 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.10.0 - typing-extensions==4.12.1 + mypy==1.10.1 + typing-extensions==4.12.2 types-lxml==2024.4.14 types-Pygments==2.18.0.20240506 types-pyOpenSSL==24.1.0.20240425 - types-setuptools==70.0.0.20240524 - botocore-stubs==1.34.94 - boto3-stubs[s3]==1.34.119 + types-setuptools==70.3.0.20240710 + botocore-stubs==1.34.143 + boto3-stubs[s3]==1.34.143 attrs >= 18.2.0 Pillow >= 10.3.0 pytest >= 8.2.0 @@ -88,8 +88,8 @@ commands = [testenv:twinecheck] basepython = python3 deps = - twine==5.0.0 - build==1.0.3 + twine==5.1.1 + build==1.2.1 commands = python -m build --sdist twine check dist/* From 435686830cbe86d14aa09c9259157695596be07f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 11 Jul 2024 12:25:13 +0500 Subject: [PATCH 240/269] Bump the Python version for RTD. --- .github/workflows/checks.yml | 2 +- .readthedocs.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 1841bda1c..2be6a9502 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -21,7 +21,7 @@ jobs: - python-version: 3.8 env: TOXENV: typing-tests - - python-version: "3.11" # Keep in sync with .readthedocs.yml + - python-version: "3.12" # Keep in sync with .readthedocs.yml env: TOXENV: docs - python-version: "3.12" diff --git a/.readthedocs.yml b/.readthedocs.yml index e71d34f3a..0c544df7e 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ build: tools: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.11" # Keep in sync with .github/workflows/checks.yml + python: "3.12" # Keep in sync with .github/workflows/checks.yml python: install: From 3c9c1a31bcdcced96e87e299689aaa7be8f5bdee Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 11 Jul 2024 12:30:12 +0500 Subject: [PATCH 241/269] Bump pylint. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 29d240031..e3dd96425 100644 --- a/tox.ini +++ b/tox.ini @@ -81,7 +81,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==3.1.0 + pylint==3.2.5 commands = pylint conftest.py docs extras scrapy setup.py tests From a40d5281cfb8fdaf7d7edce80d3addbddef897a6 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <guillermo.bondonno@zyte.com> Date: Thu, 11 Jul 2024 11:14:30 -0300 Subject: [PATCH 242/269] improve test_closespider_pagecount_no_item and MaxItemsSpider --- tests/spiders.py | 13 +++++-------- tests/test_closespider.py | 10 ++++------ 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/tests/spiders.py b/tests/spiders.py index 2bcec5624..d1998ca69 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -283,21 +283,18 @@ class ItemSpider(FollowAllSpider): yield {} -class MaxItemsSpider(ItemSpider): +class MaxItemsKeepCrawlingSpider(FollowAllSpider): def __init__(self, max_items=10, *args, **kwargs): super().__init__(*args, **kwargs) self.max_items = max_items self.items_scraped = 0 def parse(self, response): - for item_or_req in super().parse(response): - if isinstance(item_or_req, Request): - yield item_or_req - else: - if self.items_scraped >= self.max_items: - continue + for request in super().parse(response): + yield request + if self.items_scraped < self.max_items: + yield Item() self.items_scraped += 1 - yield item_or_req class DefaultError(Exception): diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 0046b4e29..50b483a74 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -7,7 +7,7 @@ from tests.spiders import ( ErrorSpider, FollowAllSpider, ItemSpider, - MaxItemsSpider, + MaxItemsKeepCrawlingSpider, SlowSpider, ) @@ -43,13 +43,11 @@ class TestCloseSpider(TestCase): @defer.inlineCallbacks def test_closespider_pagecount_no_item(self): close_on = 5 - close_on_pagecount = 20 max_items = 5 crawler = get_crawler( - MaxItemsSpider, + MaxItemsKeepCrawlingSpider, { "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on, - "CLOSESPIDER_PAGECOUNT": close_on_pagecount, }, ) yield crawler.crawl(max_items=max_items, mockserver=self.mockserver) @@ -58,8 +56,8 @@ class TestCloseSpider(TestCase): pagecount = crawler.stats.get_value("response_received_count") itemcount = crawler.stats.get_value("item_scraped_count") self.assertEqual(itemcount, max_items) - self.assertLess(pagecount, close_on_pagecount) - self.assertTrue((pagecount - itemcount) >= close_on) + self.assertLessEqual(pagecount, close_on + itemcount) + self.assertGreater(pagecount, itemcount) @defer.inlineCallbacks def test_closespider_errorcount(self): From 5f0fad16f5d86134bcf72964f6e453541031eb06 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <guillermo.bondonno@zyte.com> Date: Thu, 11 Jul 2024 13:26:22 -0300 Subject: [PATCH 243/269] improve test_closespider_pagecount_no_item and corresponding test spider --- tests/spiders.py | 12 ++++++++---- tests/test_closespider.py | 11 ++++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/spiders.py b/tests/spiders.py index d1998ca69..743811893 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -283,15 +283,19 @@ class ItemSpider(FollowAllSpider): yield {} -class MaxItemsKeepCrawlingSpider(FollowAllSpider): - def __init__(self, max_items=10, *args, **kwargs): +class MaxItemsAndRequestsSpider(FollowAllSpider): + def __init__(self, max_items=10, max_requests=10, *args, **kwargs): super().__init__(*args, **kwargs) self.max_items = max_items - self.items_scraped = 0 + self.max_requests = max_requests def parse(self, response): + self.items_scraped = 0 + self.pages_crawled = 1 # account for the start url for request in super().parse(response): - yield request + if self.pages_crawled < self.max_requests: + yield request + self.pages_crawled += 1 if self.items_scraped < self.max_items: yield Item() self.items_scraped += 1 diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 50b483a74..9810d10fb 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -7,7 +7,7 @@ from tests.spiders import ( ErrorSpider, FollowAllSpider, ItemSpider, - MaxItemsKeepCrawlingSpider, + MaxItemsAndRequestsSpider, SlowSpider, ) @@ -44,20 +44,21 @@ class TestCloseSpider(TestCase): def test_closespider_pagecount_no_item(self): close_on = 5 max_items = 5 + max_requests = close_on + max_items crawler = get_crawler( - MaxItemsKeepCrawlingSpider, + MaxItemsAndRequestsSpider, { "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on, }, ) - yield crawler.crawl(max_items=max_items, mockserver=self.mockserver) + yield crawler.crawl( + max_items=max_items, max_requests=max_requests, mockserver=self.mockserver + ) reason = crawler.spider.meta["close_reason"] self.assertEqual(reason, "closespider_pagecount_no_item") pagecount = crawler.stats.get_value("response_received_count") itemcount = crawler.stats.get_value("item_scraped_count") - self.assertEqual(itemcount, max_items) self.assertLessEqual(pagecount, close_on + itemcount) - self.assertGreater(pagecount, itemcount) @defer.inlineCallbacks def test_closespider_errorcount(self): From 9cdbcb4f63922f09194fab7d211ba297319b5135 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <guillermo.bondonno@zyte.com> Date: Thu, 11 Jul 2024 14:02:24 -0300 Subject: [PATCH 244/269] add test_closespider_pagecount_no_item_with_pagecount --- tests/test_closespider.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 9810d10fb..9a837350f 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -60,6 +60,23 @@ class TestCloseSpider(TestCase): itemcount = crawler.stats.get_value("item_scraped_count") self.assertLessEqual(pagecount, close_on + itemcount) + @defer.inlineCallbacks + def test_closespider_pagecount_no_item_with_pagecount(self): + close_on_pagecount_no_item = 5 + close_on_pagecount = 20 + crawler = get_crawler( + FollowAllSpider, + { + "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on_pagecount_no_item, + "CLOSESPIDER_PAGECOUNT": close_on_pagecount, + }, + ) + yield crawler.crawl(mockserver=self.mockserver) + reason = crawler.spider.meta["close_reason"] + self.assertEqual(reason, "closespider_pagecount_no_item") + pagecount = crawler.stats.get_value("response_received_count") + self.assertLess(pagecount, close_on_pagecount) + @defer.inlineCallbacks def test_closespider_errorcount(self): close_on = 5 From 026d6065287e882c244d9b90e0c4fa5e873e29fe Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <95530227+guillermo-bondonno@users.noreply.github.com> Date: Fri, 12 Jul 2024 08:09:03 -0300 Subject: [PATCH 245/269] clean closespider_pagecount_no_item docs section Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name> --- docs/topics/extensions.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 29bcaa0f2..7b34a19d5 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -324,10 +324,11 @@ CLOSESPIDER_PAGECOUNT_NO_ITEM Default: ``0`` -An integer which specifies the maximum number of consecutive responses to crawl without items scraped. If the spider -crawls more consecutive responses than that and no items are scraped in the meantime, the spider will be closed with the reason -``closespider_pagecount_no_item``. If zero (or non set), spiders won't be closed by -number of crawled responses with no items. +An integer which specifies the maximum number of consecutive responses to crawl +without items scraped. If the spider crawls more consecutive responses than that +and no items are scraped in the meantime, the spider will be closed with the +reason ``closespider_pagecount_no_item``. If zero (or not set), spiders won't be +closed by number of crawled responses with no items. .. setting:: CLOSESPIDER_ERRORCOUNT From e376c0b31a01cedd8a8c5c1ccd423d72ae1fb169 Mon Sep 17 00:00:00 2001 From: mlmsmith <mlmsmith@hotmail.co.uk> Date: Wed, 24 Jul 2024 12:40:01 +0800 Subject: [PATCH 246/269] Tutorial edits (#6440) --- docs/intro/tutorial.rst | 53 ++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 8ea98f29b..ee6a1184c 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -18,11 +18,11 @@ This tutorial will walk you through these tasks: 4. Changing spider to recursively follow links 5. Using spider arguments -Scrapy is written in Python_. If you're new to the language you might want to -start by getting an idea of what the language is like, to get the most out of -Scrapy. +Scrapy is written in Python_. The more you learn about Python, the more you +can get out of Scrapy. -If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource. +If you're already familiar with other languages and want to learn Python quickly, the +`Python Tutorial`_ is a good resource. If you're new to programming and want to start with Python, the following books may be useful to you: @@ -76,10 +76,9 @@ This will create a ``tutorial`` directory with the following contents:: Our first Spider ================ -Spiders are classes that you define and that Scrapy uses to scrape information -from a website (or a group of websites). They must subclass -:class:`~scrapy.Spider` and define the initial requests to make, -optionally how to follow links in the pages, and how to parse the downloaded +Spiders are classes that you define and that Scrapy uses to scrape information from a website +(or a group of websites). They must subclass :class:`~scrapy.Spider` and define the initial +requests to be made, and optionally, how to follow links in pages and parse the downloaded page content to extract data. This is the code for our first Spider. Save it in a file named @@ -138,7 +137,7 @@ To put our spider to work, go to the project's top level directory and run:: scrapy crawl quotes -This command runs the spider with name ``quotes`` that we've just added, that +This command runs the spider named ``quotes`` that we've just added, that will send some requests for the ``quotes.toscrape.com`` domain. You will get an output similar to this:: @@ -169,7 +168,7 @@ Scrapy schedules the :class:`scrapy.Request <scrapy.Request>` objects returned by the ``start_requests`` method of the Spider. Upon receiving a response for each one, it instantiates :class:`~scrapy.http.Response` objects and calls the callback method associated with the request (in this case, the -``parse`` method) passing the response as argument. +``parse`` method) passing the response as an argument. A shortcut to the start_requests method @@ -217,7 +216,7 @@ using the :ref:`Scrapy shell <topics-shell>`. Run:: .. note:: - Remember to always enclose urls in quotes when running Scrapy shell from + Remember to always enclose urls in quotes when running Scrapy shell from the command-line, otherwise urls containing arguments (i.e. ``&`` character) will not work. @@ -257,7 +256,7 @@ object: The result of running ``response.css('title')`` is a list-like object called :class:`~scrapy.selector.SelectorList`, which represents a list of :class:`~scrapy.Selector` objects that wrap around XML/HTML elements -and allow you to run further queries to fine-grain the selection or extract the +and allow you to run further queries to refine the selection or extract the data. To extract the text from the title above, you can do: @@ -354,12 +353,12 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions: XPath expressions are very powerful, and are the foundation of Scrapy Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You -can see that if you read closely the text representation of the selector -objects in the shell. +can see that if you read the text representation of the selector +objects in the shell closely. While perhaps not as popular as CSS selectors, XPath expressions offer more power because besides navigating the structure, it can also look at the -content. Using XPath, you're able to select things like: *select the link +content. Using XPath, you're able to select things like: *the link that contains the text "Next Page"*. This makes XPath very fitting to the task of scraping, and we encourage you to learn XPath even if you already know how to construct CSS selectors, it will make scraping much easier. @@ -422,7 +421,7 @@ variable, so that we can run our CSS selectors directly on a particular quote: >>> quote = response.css("div.quote")[0] -Now, let's extract ``text``, ``author`` and the ``tags`` from that quote +Now, let's extract the ``text``, ``author`` and ``tags`` from that quote using the ``quote`` object we just created: .. code-block:: pycon @@ -448,7 +447,7 @@ to get all of them: from sys import version_info Having figured out how to extract each bit, we can now iterate over all the -quotes elements and put them together into a Python dictionary: +quote elements and put them together into a Python dictionary: .. code-block:: pycon @@ -465,8 +464,8 @@ quotes elements and put them together into a Python dictionary: Extracting data in our spider ----------------------------- -Let's get back to our spider. Until now, it doesn't extract any data in -particular, just saves the whole HTML page to a local file. Let's integrate the +Let's get back to our spider. Until now, it hasn't extracted any data in +particular, just saving the whole HTML page to a local file. Let's integrate the extraction logic above into our spider. A Scrapy spider typically generates many dictionaries containing the data @@ -529,8 +528,8 @@ using a different serialization format, such as `JSON Lines`_:: scrapy crawl quotes -o quotes.jsonl -The `JSON Lines`_ format is useful because it's stream-like, you can easily -append new records to it. It doesn't have the same problem of JSON when you run +The `JSON Lines`_ format is useful because it's stream-like, so you can easily +append new records to it. It doesn't have the same problem as JSON when you run twice. Also, as each record is a separate line, you can process big files without having to fit everything in memory, there are tools like `JQ`_ to help do that at the command-line. @@ -555,7 +554,7 @@ from https://quotes.toscrape.com, you want quotes from all the pages in the webs Now that you know how to extract data from pages, let's see how to follow links from them. -First thing is to extract the link to the page we want to follow. Examining +The first thing to do is extract the link to the page we want to follow. Examining our page, we can see there is a link to the next page with the following markup: @@ -589,7 +588,7 @@ There is also an ``attrib`` property available >>> response.css("li.next a").attrib["href"] '/page/2/' -Let's see now our spider modified to recursively follow the link to the next +Now let's see our spider, modified to recursively follow the link to the next page, extracting data from it: .. code-block:: python @@ -756,8 +755,8 @@ Another interesting thing this spider demonstrates is that, even if there are many quotes from the same author, we don't need to worry about visiting the same author page multiple times. By default, Scrapy filters out duplicated requests to URLs already visited, avoiding the problem of hitting servers too -much because of a programming mistake. This can be configured by the setting -:setting:`DUPEFILTER_CLASS`. +much because of a programming mistake. This can be configured in the +:setting:`DUPEFILTER_CLASS` setting. Hopefully by now you have a good understanding of how to use the mechanism of following links and callbacks with Scrapy. @@ -824,12 +823,12 @@ Next steps ========== This tutorial covered only the basics of Scrapy, but there's a lot of other -features not mentioned here. Check the :ref:`topics-whatelse` section in +features not mentioned here. Check the :ref:`topics-whatelse` section in the :ref:`intro-overview` chapter for a quick overview of the most important ones. You can continue from the section :ref:`section-basics` to know more about the command-line tool, spiders, selectors and other things the tutorial hasn't covered like -modeling the scraped data. If you prefer to play with an example project, check +modeling the scraped data. If you'd prefer to play with an example project, check the :ref:`intro-examples` section. .. _JSON: https://en.wikipedia.org/wiki/JSON From 03a15ced4f0a4284c75a917fdfb07c44b21f9ff2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 31 Jul 2024 11:37:19 +0200 Subject: [PATCH 247/269] Do not suggest logging dropped items twice (#6448) Co-authored-by: Kevin Lloyd Bernal <kevinoxy@gmail.com> --- docs/topics/item-pipeline.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index a5f6e07b8..58c922e0d 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -99,7 +99,7 @@ contain a price: adapter["price"] = adapter["price"] * self.vat_factor return item else: - raise DropItem(f"Missing price in {item}") + raise DropItem("Missing price") Write items to a JSON lines file @@ -254,7 +254,7 @@ returns multiples items with the same id: def process_item(self, item, spider): adapter = ItemAdapter(item) if adapter["id"] in self.ids_seen: - raise DropItem(f"Duplicate item found: {item!r}") + raise DropItem(f"Item ID already seen: {adapter['id']}") else: self.ids_seen.add(adapter["id"]) return item From b9ef1326a51140f70325609501265300fdac5e9b Mon Sep 17 00:00:00 2001 From: mlmsmith <mlmsmith@hotmail.co.uk> Date: Thu, 1 Aug 2024 15:29:11 +0800 Subject: [PATCH 248/269] Proofread the commands documentation (#6449) --- docs/intro/tutorial.rst | 4 ++-- docs/topics/commands.rst | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index ee6a1184c..dd1efd3b3 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -216,8 +216,8 @@ using the :ref:`Scrapy shell <topics-shell>`. Run:: .. note:: - Remember to always enclose urls in quotes when running Scrapy shell from the - command-line, otherwise urls containing arguments (i.e. ``&`` character) + Remember to always enclose URLs in quotes when running Scrapy shell from the + command line, otherwise URLs containing arguments (i.e. ``&`` character) will not work. On Windows, use double quotes instead:: diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 1d37895c2..6eb4af9bd 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -6,7 +6,7 @@ Command line tool ================= -Scrapy is controlled through the ``scrapy`` command-line tool, to be referred +Scrapy is controlled through the ``scrapy`` command-line tool, to be referred to here as the "Scrapy tool" to differentiate it from the sub-commands, which we just call "commands" or "Scrapy commands". @@ -185,8 +185,8 @@ And you can see all available commands with:: There are two kinds of commands, those that only work from inside a Scrapy project (Project-specific commands) and those that also work without an active -Scrapy project (Global commands), though they may behave slightly different -when running from inside a project (as they would use the project overridden +Scrapy project (Global commands), though they may behave slightly differently +when run from inside a project (as they would use the project overridden settings). Global commands: @@ -236,7 +236,7 @@ genspider .. versionadded:: 2.6.0 The ability to pass a URL instead of a domain. -Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. +Creates a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. Usage example:: @@ -253,7 +253,7 @@ Usage example:: $ scrapy genspider -t crawl scrapyorg scrapy.org Created spider 'scrapyorg' using template 'crawl' -This is just a convenience shortcut command for creating spiders based on +This is just a convenient shortcut command for creating spiders based on pre-defined templates, but certainly not the only way to create spiders. You can just create the spider source code files yourself, instead of using this command. @@ -274,9 +274,9 @@ Supported options: * ``-a NAME=VALUE``: set a spider argument (may be repeated) -* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout), to define format set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``) +* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout). To define the output format, set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``) -* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file, to define format set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``) +* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file. To define the output format, set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``) * ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O`` @@ -353,7 +353,7 @@ edit Edit the given spider using the editor defined in the ``EDITOR`` environment variable or (if unset) the :setting:`EDITOR` setting. -This command is provided only as a convenience shortcut for the most common +This command is provided only as a convenient shortcut for the most common case, the developer is of course free to choose any tool or IDE to write and debug spiders. @@ -372,7 +372,7 @@ fetch Downloads the given URL using the Scrapy downloader and writes the contents to standard output. -The interesting thing about this command is that it fetches the page how the +The interesting thing about this command is that it fetches the page the way the spider would download it. For example, if the spider has a ``USER_AGENT`` attribute which overrides the User Agent, it will use that one. From af15bd1dadf74b1314b96b1c3b682b41207a1f52 Mon Sep 17 00:00:00 2001 From: mlmsmith <mlmsmith@hotmail.co.uk> Date: Mon, 19 Aug 2024 19:55:09 +0800 Subject: [PATCH 249/269] minor changes to items section of docs (#6462) --- docs/topics/items.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 97ed7a900..f13a7b5b1 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -48,7 +48,7 @@ make it the most feature-complete item type: :class:`Item` objects replicate the standard :class:`dict` API, including its ``__init__`` method. - :class:`Item` allows defining field names, so that: + :class:`Item` allows the defining of field names, so that: - :class:`KeyError` is raised when using undefined field names (i.e. prevents typos going unnoticed) @@ -57,7 +57,7 @@ make it the most feature-complete item type: default even if the first scraped object does not have values for all of them - :class:`Item` also allows defining field metadata, which can be used to + :class:`Item` also allows the defining of field metadata, which can be used to :ref:`customize serialization <topics-exporters-field-serialization>`. :mod:`trackref` tracks :class:`Item` objects to help find memory leaks @@ -94,11 +94,11 @@ Dataclass objects .. versionadded:: 2.2 -:func:`~dataclasses.dataclass` allows defining item classes with field names, +:func:`~dataclasses.dataclass` allows the defining of item classes with field names, so that :ref:`item exporters <topics-exporters>` can export all fields by default even if the first scraped object does not have values for all of them. -Additionally, ``dataclass`` items also allow to: +Additionally, ``dataclass`` items also allow you to: * define the type and default value of each defined field. @@ -126,7 +126,7 @@ attr.s objects .. versionadded:: 2.2 -:func:`attr.s` allows defining item classes with field names, +:func:`attr.s` allows the defining of item classes with field names, so that :ref:`item exporters <topics-exporters>` can export all fields by default even if the first scraped object does not have values for all of them. From 5794071f9679c89ef4ee75e8a627274b2464b65b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 23 Aug 2024 15:48:01 +0500 Subject: [PATCH 250/269] Typing fixes and updates. (#6460) --- scrapy/pipelines/media.py | 2 +- scrapy/utils/defer.py | 3 ++- scrapy/utils/log.py | 2 +- scrapy/utils/ssl.py | 2 +- tox.ini | 12 ++++++------ 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index ea36a9e8a..6bd3ed9b4 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -211,7 +211,7 @@ class MediaPipeline(ABC): # minimize cached information for failure result.cleanFailure() result.frames = [] - result.stack = None + result.stack = [] # This code fixes a memory leak by avoiding to keep references to # the Request and Response objects on the Media Pipeline cache. diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index c5763a06c..33ec23cec 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -19,6 +19,7 @@ from typing import ( Callable, Coroutine, Dict, + Generic, Iterable, Iterator, List, @@ -144,7 +145,7 @@ def parallel( return DeferredList([coop.coiterate(work) for _ in range(count)]) -class _AsyncCooperatorAdapter(Iterator[Deferred]): +class _AsyncCooperatorAdapter(Iterator[Deferred], Generic[_T]): """A class that wraps an async iterable into a normal iterator suitable for using in Cooperator.coiterate(). As it's only needed for parallel_async(), it calls the callable directly in the callback, instead of providing a more diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 439b065a9..4a70de6b4 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -128,7 +128,7 @@ def configure_logging( settings = Settings(settings) if settings.getbool("LOG_STDOUT"): - sys.stdout = StreamLogger(logging.getLogger("stdout")) # type: ignore[assignment] + sys.stdout = StreamLogger(logging.getLogger("stdout")) if install_root_handler: install_scrapy_root_handler(settings) diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 95611ebd9..2c3a259c1 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -20,7 +20,7 @@ def x509name_to_string(x509name: X509Name) -> str: # from OpenSSL.crypto.X509Name.__repr__ result_buffer: Any = pyOpenSSLutil.ffi.new("char[]", 512) pyOpenSSLutil.lib.X509_NAME_oneline( - x509name._name, result_buffer, len(result_buffer) # type: ignore[attr-defined] + x509name._name, result_buffer, len(result_buffer) ) return ffi_buf_to_string(result_buffer) diff --git a/tox.ini b/tox.ini index e3dd96425..2d62f1cb7 100644 --- a/tox.ini +++ b/tox.ini @@ -46,16 +46,16 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.10.1 + mypy==1.11.1 typing-extensions==4.12.2 - types-lxml==2024.4.14 + types-lxml==2024.8.7 types-Pygments==2.18.0.20240506 - types-pyOpenSSL==24.1.0.20240425 - types-setuptools==70.3.0.20240710 - botocore-stubs==1.34.143 - boto3-stubs[s3]==1.34.143 + types-setuptools==71.1.0.20240806 + botocore-stubs==1.34.158 + boto3-stubs[s3]==1.34.158 attrs >= 18.2.0 Pillow >= 10.3.0 + pyOpenSSL >= 24.2.1 pytest >= 8.2.0 w3lib >= 2.2.0 commands = From 6ce0342beb1a5b588f353e52fe03d5e0ec84d938 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi <GeorgeA92@users.noreply.github.com> Date: Mon, 26 Aug 2024 20:53:06 +0200 Subject: [PATCH 251/269] Allow yielding items from start_requests (#6417) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Georgiy Zatserklianyi <george.zatseklyany@gmail.com> Co-authored-by: Adrián Chaves <adrian@chaves.io> Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name> --- docs/topics/signals.rst | 16 ++++++++------ docs/topics/spider-middleware.rst | 2 +- docs/topics/spiders.rst | 3 ++- scrapy/core/engine.py | 14 ++++++++++-- scrapy/core/scraper.py | 21 ++++++++++++------ scrapy/logformatter.py | 19 ++++++++++++---- tests/spiders.py | 13 +++++++++++ tests/test_crawl.py | 36 +++++++++++++++++++++++++++++++ tests/test_spidermiddleware.py | 9 ++++---- 9 files changed, 109 insertions(+), 24 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 13e636055..b45b12540 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -159,8 +159,9 @@ item_scraped :param spider: the spider which scraped the item :type spider: :class:`~scrapy.Spider` object - :param response: the response from where the item was scraped - :type response: :class:`~scrapy.http.Response` object + :param response: the response from where the item was scraped, or ``None`` + if it was yielded from :meth:`~scrapy.Spider.start_requests`. + :type response: :class:`~scrapy.http.Response` | ``None`` item_dropped ~~~~~~~~~~~~ @@ -179,8 +180,9 @@ item_dropped :param spider: the spider which scraped the item :type spider: :class:`~scrapy.Spider` object - :param response: the response from where the item was dropped - :type response: :class:`~scrapy.http.Response` object + :param response: the response from where the item was dropped, or ``None`` + if it was yielded from :meth:`~scrapy.Spider.start_requests`. + :type response: :class:`~scrapy.http.Response` | ``None`` :param exception: the exception (which must be a :exc:`~scrapy.exceptions.DropItem` subclass) which caused the item @@ -201,8 +203,10 @@ item_error :param item: the item that caused the error in the :ref:`topics-item-pipeline` :type item: :ref:`item object <item-types>` - :param response: the response being processed when the exception was raised - :type response: :class:`~scrapy.http.Response` object + :param response: the response being processed when the exception was + raised, or ``None`` if it was yielded from + :meth:`~scrapy.Spider.start_requests`. + :type response: :class:`~scrapy.http.Response` | ``None`` :param spider: the spider which raised the exception :type spider: :class:`~scrapy.Spider` object diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 8ddf17a14..8f39bcd53 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -176,7 +176,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`. items). It receives an iterable (in the ``start_requests`` parameter) and must - return another iterable of :class:`~scrapy.Request` objects. + return another iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects <topics-items>`. .. note:: When implementing this method in your spider middleware, you should always return an iterable (that follows the input one) and diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 8a0102a51..e1b1c5ad6 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -203,7 +203,8 @@ scrapy.Spider .. method:: start_requests() - This method must return an iterable with the first Requests to crawl for + This method must return an iterable with the first Requests to crawl and/or with :ref:`item objects + <topics-items>` for this spider. It is called by Scrapy when the spider is opened for scraping. Scrapy calls it only once, so it is safe to implement :meth:`start_requests` as a generator. diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 5318cbd64..63d84339d 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -24,6 +24,7 @@ from typing import ( cast, ) +from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall from twisted.python.failure import Failure @@ -194,7 +195,7 @@ class ExecutionEngine: if self.slot.start_requests is not None and not self._needs_backout(): try: - request = next(self.slot.start_requests) + request_or_item = next(self.slot.start_requests) except StopIteration: self.slot.start_requests = None except Exception: @@ -205,7 +206,16 @@ class ExecutionEngine: extra={"spider": self.spider}, ) else: - self.crawl(request) + if isinstance(request_or_item, Request): + self.crawl(request_or_item) + elif is_item(request_or_item): + self.scraper.start_itemproc(request_or_item, response=None) + else: + logger.error( + f"Got {request_or_item!r} among start requests. Only " + f"requests and items are supported. It will be " + f"ignored." + ) if self.spider_is_idle() and self.slot.close_if_idle: self._spider_idle() diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index a7d65e1e3..7a51dbeb4 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -313,15 +313,11 @@ class Scraper: """Process each Request/Item (given in the output parameter) returned from the given spider """ - assert self.slot is not None # typing if isinstance(output, Request): assert self.crawler.engine is not None # typing self.crawler.engine.crawl(request=output) elif is_item(output): - self.slot.itemproc_size += 1 - dfd = self.itemproc.process_item(output, spider) - dfd.addBoth(self._itemproc_finished, output, response, spider) - return dfd + return self.start_itemproc(output, response=response) elif output is None: pass else: @@ -333,6 +329,19 @@ class Scraper: ) return None + def start_itemproc(self, item, *, response: Optional[Response]) -> Deferred[Any]: + """Send *item* to the item pipelines for processing. + + *response* is the source of the item data. If the item does not come + from response data, e.g. it was hard-coded, set it to ``None``. + """ + assert self.slot is not None # typing + assert self.crawler.spider is not None # typing + self.slot.itemproc_size += 1 + dfd = self.itemproc.process_item(item, self.crawler.spider) + dfd.addBoth(self._itemproc_finished, item, response, self.crawler.spider) + return dfd + def _log_download_errors( self, spider_failure: Failure, @@ -373,7 +382,7 @@ class Scraper: return None def _itemproc_finished( - self, output: Any, item: Any, response: Response, spider: Spider + self, output: Any, item: Any, response: Optional[Response], spider: Spider ) -> Deferred[Any]: """ItemProcessor finished for the given ``item`` and returned ``output``""" assert self.slot is not None # typing diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 601209fb0..fea7003e5 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -9,6 +9,7 @@ from twisted.python.failure import Failure # working around https://github.com/sphinx-doc/sphinx/issues/10400 from scrapy import Request, Spider # noqa: TC001 from scrapy.http import Response # noqa: TC001 +from scrapy.utils.python import global_object_name from scrapy.utils.request import referer_str if TYPE_CHECKING: @@ -92,11 +93,13 @@ class LogFormatter: } def scraped( - self, item: Any, response: Union[Response, Failure], spider: Spider + self, item: Any, response: Union[Response, Failure, None], spider: Spider ) -> LogFormatterResult: """Logs a message when an item is scraped by a spider.""" src: Any - if isinstance(response, Failure): + if response is None: + src = f"{global_object_name(spider.__class__)}.start_requests" + elif isinstance(response, Failure): src = response.getErrorMessage() else: src = response @@ -110,7 +113,11 @@ class LogFormatter: } def dropped( - self, item: Any, exception: BaseException, response: Response, spider: Spider + self, + item: Any, + exception: BaseException, + response: Optional[Response], + spider: Spider, ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" return { @@ -123,7 +130,11 @@ class LogFormatter: } def item_error( - self, item: Any, exception: BaseException, response: Response, spider: Spider + self, + item: Any, + exception: BaseException, + response: Optional[Response], + spider: Spider, ) -> LogFormatterResult: """Logs a message when an item causes an error while it is passing through the item pipeline. diff --git a/tests/spiders.py b/tests/spiders.py index 743811893..5d5792858 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -346,6 +346,19 @@ class BrokenStartRequestsSpider(FollowAllSpider): yield from super().parse(response) +class StartRequestsItemSpider(FollowAllSpider): + def start_requests(self): + yield {"name": "test item"} + + +class StartRequestsGoodAndBadOutput(FollowAllSpider): + def start_requests(self): + yield {"a": "a"} + yield Request("data:,a") + yield "data:,b" + yield object() + + class SingleRequestSpider(MetaSpider): seed = None callback_func = None diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 6cde4ed8c..125709571 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,5 +1,6 @@ import json import logging +import re import unittest from ipaddress import IPv4Address from socket import gethostbyname @@ -49,6 +50,8 @@ from tests.spiders import ( HeadersReceivedErrbackSpider, SimpleSpider, SingleRequestSpider, + StartRequestsGoodAndBadOutput, + StartRequestsItemSpider, ) @@ -184,6 +187,39 @@ class CrawlTestCase(TestCase): self.assertIsNotNone(record.exc_info) self.assertIs(record.exc_info[0], ZeroDivisionError) + @defer.inlineCallbacks + def test_start_requests_items(self): + with LogCapture("scrapy", level=logging.ERROR) as log: + crawler = get_crawler(StartRequestsItemSpider) + yield crawler.crawl(mockserver=self.mockserver) + + self.assertEqual(len(log.records), 0) + + @defer.inlineCallbacks + def test_start_requests_unsupported_output(self): + with LogCapture("scrapy", level=logging.ERROR) as log: + crawler = get_crawler(StartRequestsGoodAndBadOutput) + yield crawler.crawl(mockserver=self.mockserver) + + self.assertEqual(len(log.records), 2) + self.assertEqual( + log.records[0].msg, + ( + "Got 'data:,b' among start requests. Only requests and items " + "are supported. It will be ignored." + ), + ) + self.assertTrue( + re.match( + ( + r"^Got <object object at 0x[0-9a-fA-F]+> among start " + r"requests\. Only requests and items are supported\. It " + r"will be ignored\.$" + ), + log.records[1].msg, + ) + ) + @defer.inlineCallbacks def test_start_requests_laziness(self): settings = {"CONCURRENT_REQUESTS": 1} diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 38ca8d950..9dbffe353 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,5 +1,5 @@ import collections.abc -from typing import Optional +from typing import Optional, Union from unittest import mock from testfixtures import LogCapture @@ -112,7 +112,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): Should work for process_spider_output and, when it's supported, process_start_requests. """ - ITEM_TYPE: type + ITEM_TYPE: Union[type, tuple] RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects @staticmethod @@ -328,12 +328,13 @@ class ProcessStartRequestsSimpleMiddleware: class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): """process_start_requests tests for simple start_requests""" - ITEM_TYPE = Request + ITEM_TYPE = (Request, dict) MW_SIMPLE = ProcessStartRequestsSimpleMiddleware def _start_requests(self): - for i in range(3): + for i in range(2): yield Request(f"https://example.com/{i}", dont_filter=True) + yield {"name": "test item"} @defer.inlineCallbacks def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): From b85e5a66ede0a2255b335ee4869836e9d30c580a Mon Sep 17 00:00:00 2001 From: Laerte Pereira <laertefbk@gmail.com> Date: Mon, 26 Aug 2024 23:21:09 -0300 Subject: [PATCH 252/269] Add support for meta in Spider Contracts --- docs/topics/contracts.rst | 8 +++ scrapy/contracts/default.py | 14 +++++ scrapy/settings/default_settings.py | 1 + tests/test_contracts.py | 79 +++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 2d61026e9..a912ff986 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -46,6 +46,14 @@ This callback is tested using three built-in contracts: @cb_kwargs {"arg1": "value1", "arg2": "value2", ...} +.. class:: MetadataContract + + This contract (``@meta``) sets the :attr:` meta <scrapy.Request.meta>` + attribute for the sample request. It must be a valid JSON dictionary. + :: + + @meta {"arg1": "value1", "arg2": "value2", ...} + .. class:: ReturnsContract This contract (``@returns``) sets lower and upper bounds for the items and diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 71ca4168a..87099b950 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -35,6 +35,20 @@ class CallbackKeywordArgumentsContract(Contract): return args +class MetadataContract(Contract): + """Contract to key metadata arguments for the request. + The value should be JSON-encoded dictionary, e.g.: + + @meta {"arg1": "some value"} + """ + + name = "meta" + + def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: + args["meta"] = json.loads(" ".join(self.args)) + return args + + class ReturnsContract(Contract): """Contract to check the output of a callback diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 932475fb5..7ba0128a5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -333,6 +333,7 @@ SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { "scrapy.contracts.default.UrlContract": 1, "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1, + "scrapy.contracts.default.MetadataContract": 1, "scrapy.contracts.default.ReturnsContract": 2, "scrapy.contracts.default.ScrapesContract": 3, } diff --git a/tests/test_contracts.py b/tests/test_contracts.py index c9c12f0d8..d578b3af4 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -8,6 +8,7 @@ from scrapy import FormRequest from scrapy.contracts import Contract, ContractsManager from scrapy.contracts.default import ( CallbackKeywordArgumentsContract, + MetadataContract, ReturnsContract, ScrapesContract, UrlContract, @@ -29,6 +30,10 @@ class ResponseMock: url = "http://scrapy.org" +class ResponseMetaMock(ResponseMock): + meta = None + + class CustomSuccessContract(Contract): name = "custom_success_contract" @@ -195,6 +200,33 @@ class TestSpider(Spider): """ pass + def returns_request_meta(self, response): + """method which returns request + @url https://example.org + @meta {"cookiejar": "session1"} + @returns requests 1 + """ + return Request( + "https://example.org", meta=response.meta, callback=self.returns_item_meta + ) + + def returns_item_meta(self, response): + """method which returns item + @url http://scrapy.org + @meta {"key": "example"} + @returns items 1 1 + """ + return TestItem(name="example", url=response.url) + + def returns_error_missing_meta(self, response): + """method which depends of metadata be defined + + @url http://scrapy.org + @returns items 1 + """ + key = response.meta["key"] + yield {key: "value"} + class CustomContractSuccessSpider(Spider): name = "custom_contract_success_spider" @@ -224,6 +256,7 @@ class ContractsManagerTest(unittest.TestCase): contracts = [ UrlContract, CallbackKeywordArgumentsContract, + MetadataContract, ReturnsContract, ScrapesContract, CustomFormContract, @@ -328,6 +361,52 @@ class ContractsManagerTest(unittest.TestCase): request.callback(response, **request.cb_kwargs) self.should_error() + def test_meta(self): + spider = TestSpider() + + # extract contracts correctly + contracts = self.conman.extract_contracts(spider.returns_request_meta) + self.assertEqual(len(contracts), 3) + self.assertEqual( + frozenset(type(x) for x in contracts), + frozenset([UrlContract, MetadataContract, ReturnsContract]), + ) + + contracts = self.conman.extract_contracts(spider.returns_item_meta) + self.assertEqual(len(contracts), 3) + self.assertEqual( + frozenset(type(x) for x in contracts), + frozenset([UrlContract, MetadataContract, ReturnsContract]), + ) + + response = ResponseMetaMock() + + # returns_request + request = self.conman.from_method(spider.returns_request_meta, self.results) + assert request.meta["cookiejar"] == "session1" + response.meta = request.meta + request.callback(response) + assert response.meta["cookiejar"] == "session1" + self.should_succeed() + + response = ResponseMetaMock() + + # returns_item + request = self.conman.from_method(spider.returns_item_meta, self.results) + assert request.meta["key"] == "example" + response.meta = request.meta + request.callback(ResponseMetaMock) + assert response.meta["key"] == "example" + self.should_succeed() + + response = ResponseMetaMock() + + request = self.conman.from_method( + spider.returns_error_missing_meta, self.results + ) + request.callback(response) + self.should_error() + def test_returns(self): spider = TestSpider() response = ResponseMock() From f68f29dd1361f427be151b09c99068b292275923 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Mon, 26 Aug 2024 23:37:57 -0300 Subject: [PATCH 253/269] Update docs/topics/contracts.rst --- docs/topics/contracts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index a912ff986..7557dacc0 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -48,7 +48,7 @@ This callback is tested using three built-in contracts: .. class:: MetadataContract - This contract (``@meta``) sets the :attr:` meta <scrapy.Request.meta>` + This contract (``@meta``) sets the :attr:`meta <scrapy.Request.meta>` attribute for the sample request. It must be a valid JSON dictionary. :: From 3c2a9fa262dd3e63acc58c8f0a2f91cf65c33bc4 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <laertefbk@gmail.com> Date: Tue, 27 Aug 2024 07:16:01 -0300 Subject: [PATCH 254/269] update docs --- docs/topics/contracts.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index a912ff986..82afa0dc1 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -20,13 +20,13 @@ following example: This function parses a sample response. Some contracts are mingled with this docstring. - @url http://www.amazon.com/s?field-keywords=selfish+gene + @url http://www.example.com/s?field-keywords=selfish+gene @returns items 1 16 @returns requests 0 0 @scrapes Title Author Year Price """ -This callback is tested using three built-in contracts: +You can use the following contracts: .. module:: scrapy.contracts.default From ddbdfeb699a2308ca600781b2d1549cbee62725c Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Tue, 27 Aug 2024 07:24:57 -0300 Subject: [PATCH 255/269] Update scrapy/contracts/default.py --- scrapy/contracts/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 87099b950..e7b11d426 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -36,7 +36,7 @@ class CallbackKeywordArgumentsContract(Contract): class MetadataContract(Contract): - """Contract to key metadata arguments for the request. + """Contract to set metadata arguments for the request. The value should be JSON-encoded dictionary, e.g.: @meta {"arg1": "some value"} From 67ab8d4650c1e9212c9508803c7b5265e166cbaa Mon Sep 17 00:00:00 2001 From: Daniel O'Connor <daniel.oconnor@gmail.com> Date: Thu, 29 Aug 2024 04:37:49 +0930 Subject: [PATCH 256/269] Refactor genspider slightly so template variables can be overridden (#6470) --- scrapy/commands/genspider.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 2649fb23d..6c3713f8f 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -116,6 +116,24 @@ class Command(ScrapyCommand): if opts.edit: self.exitcode = os.system(f'scrapy edit "{name}"') # nosec + def _generate_template_variables( + self, + module: str, + name: str, + url: str, + template_name: str, + ): + capitalized_module = "".join(s.capitalize() for s in module.split("_")) + return { + "project_name": self.settings.get("BOT_NAME"), + "ProjectName": string_camelcase(self.settings.get("BOT_NAME")), + "module": module, + "name": name, + "url": url, + "domain": extract_domain(url), + "classname": f"{capitalized_module}Spider", + } + def _genspider( self, module: str, @@ -125,17 +143,7 @@ class Command(ScrapyCommand): template_file: Union[str, os.PathLike], ) -> None: """Generate the spider module, based on the given template""" - capitalized_module = "".join(s.capitalize() for s in module.split("_")) - domain = extract_domain(url) - tvars = { - "project_name": self.settings.get("BOT_NAME"), - "ProjectName": string_camelcase(self.settings.get("BOT_NAME")), - "module": module, - "name": name, - "url": url, - "domain": domain, - "classname": f"{capitalized_module}Spider", - } + tvars = self._generate_template_variables(module, name, url, template_name) if self.settings.get("NEWSPIDER_MODULE"): spiders_module = import_module(self.settings["NEWSPIDER_MODULE"]) assert spiders_module.__file__ From f260f819e0794708868ed447ae154caa74d965f7 Mon Sep 17 00:00:00 2001 From: LucasSD <lucas.stonedrake@gmail.com> Date: Mon, 9 Sep 2024 20:26:02 +0100 Subject: [PATCH 257/269] Remove debug log message from _schedule_request method --- scrapy/core/engine.py | 5 ----- tests/test_engine.py | 1 - 2 files changed, 6 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 63d84339d..fd9a5f781 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -39,7 +39,6 @@ from scrapy.settings import Settings from scrapy.signalmanager import SignalManager from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object -from scrapy.utils.python import global_object_name from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: @@ -325,10 +324,6 @@ class ExecutionEngine: ) for handler, result in request_scheduled_result: if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest): - logger.debug( - f"Signal handler {global_object_name(handler)} dropped " - f"request {request} before it reached the scheduler." - ) return if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log( diff --git a/tests/test_engine.py b/tests/test_engine.py index 86526420f..2ebc0b5e4 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -499,7 +499,6 @@ def test_request_scheduled_signal(caplog): assert scheduler.enqueued == [ keep_request ], f"{scheduler.enqueued!r} != [{keep_request!r}]" - assert "dropped request <GET https://drop.example>" in caplog.text crawler.signals.disconnect(signal_handler, request_scheduled) From 46cddc6ecfbe9a0750676143ef789acf6c2e637d Mon Sep 17 00:00:00 2001 From: mmoriniere <maxime.moriniere@hotmail.fr> Date: Wed, 2 Oct 2024 10:04:03 +0200 Subject: [PATCH 258/269] Ignore SyntaxError as well when SPIDER_LOADER_WARN_ONLY is set to True (#6484) --- docs/news.rst | 6 ++++++ docs/topics/settings.rst | 2 +- scrapy/spiderloader.py | 2 +- tests/test_spiderloader/__init__.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 758b22d80..58b51c9ea 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -8,6 +8,12 @@ Release notes Scrapy VERSION (YYYY-MM-DD) --------------------------- +New features +~~~~~~~~~~~~ + +- If :setting:`SPIDER_LOADER_WARN_ONLY` is set to ``True``, + ``SpiderLoader`` does not raise :exc:`SyntaxError` but emits a warning instead. + Deprecations ~~~~~~~~~~~~ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 904bd7ecc..02fca7ff4 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1580,7 +1580,7 @@ SPIDER_LOADER_WARN_ONLY Default: ``False`` By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`, -it will fail loudly if there is any ``ImportError`` exception. +it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception. But you can choose to silence this exception and turn it into a simple warning by setting ``SPIDER_LOADER_WARN_ONLY = True``. diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index b8fe65668..f5fd899b2 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -64,7 +64,7 @@ class SpiderLoader: try: for module in walk_modules(name): self._load_spiders(module) - except ImportError: + except (ImportError, SyntaxError): if self.warn_only: warnings.warn( f"\n{traceback.format_exc()}Could not load spiders " diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index f950739f2..32699d837 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -4,6 +4,7 @@ import tempfile import warnings from pathlib import Path from tempfile import mkdtemp +from unittest import mock from twisted.trial import unittest from zope.interface.verify import verifyObject @@ -136,6 +137,33 @@ class SpiderLoaderTest(unittest.TestCase): spiders = spider_loader.list() self.assertEqual(spiders, []) + def test_syntax_error_exception(self): + module = "tests.test_spiderloader.test_spiders.spider1" + with mock.patch.object(SpiderLoader, "_load_spiders") as m: + m.side_effect = SyntaxError + settings = Settings({"SPIDER_MODULES": [module]}) + self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings) + + def test_syntax_error_warning(self): + with warnings.catch_warnings(record=True) as w, mock.patch.object( + SpiderLoader, "_load_spiders" + ) as m: + m.side_effect = SyntaxError + module = "tests.test_spiderloader.test_spiders.spider1" + settings = Settings( + {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} + ) + spider_loader = SpiderLoader.from_settings(settings) + if str(w[0].message).startswith("_SixMetaPathImporter"): + # needed on 3.10 because of https://github.com/benjaminp/six/issues/349, + # at least until all six versions we can import (including botocore.vendored.six) + # are updated to 1.16.0+ + w.pop(0) + self.assertIn("Could not load spiders from module", str(w[0].message)) + + spiders = spider_loader.list() + self.assertEqual(spiders, []) + class DuplicateSpiderNameLoaderTest(unittest.TestCase): def setUp(self): From 8c133fcf7e4f19d55d60dc6a090d75dab6db1a72 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 7 Oct 2024 23:04:48 +0500 Subject: [PATCH 259/269] Remove the installation dependency on setuptools. --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 2d6d26b0c..f458a9de3 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,6 @@ install_requires = [ "zope.interface>=5.1.0", "protego>=0.1.15", "itemadapter>=0.1.0", - "setuptools", "packaging", "tldextract", "lxml>=4.4.1", From 87651fdf47403767b5b79f075237e7351ba4853b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 8 Oct 2024 16:04:26 +0500 Subject: [PATCH 260/269] Don't use types-setuptools. --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2d62f1cb7..80ef4a99e 100644 --- a/tox.ini +++ b/tox.ini @@ -50,7 +50,6 @@ deps = typing-extensions==4.12.2 types-lxml==2024.8.7 types-Pygments==2.18.0.20240506 - types-setuptools==71.1.0.20240806 botocore-stubs==1.34.158 boto3-stubs[s3]==1.34.158 attrs >= 18.2.0 From 5ef54741729739e9a161d80e74d0076dfdb973cc Mon Sep 17 00:00:00 2001 From: Klaus Rettinghaus <klaus.rettinghaus@gmail.com> Date: Wed, 9 Oct 2024 20:38:50 +0200 Subject: [PATCH 261/269] update gh-action-pypi-publish --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 03e94f761..5ce48be61 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,6 +20,6 @@ jobs: pip install --upgrade build twine python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.9.0 + uses: pypa/gh-action-pypi-publish@v1.10.3 with: password: ${{ secrets.PYPI_TOKEN }} From 53916630723a86277836053e1c54fe50655f0bd5 Mon Sep 17 00:00:00 2001 From: Vsevolod Breus <vsevolodbreus1@gmail.com> Date: Wed, 16 Oct 2024 08:03:16 +0000 Subject: [PATCH 262/269] Drop Python 3.8 Support (#6472) --- .github/workflows/checks.yml | 4 +- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 12 +-- .github/workflows/tests-windows.yml | 5 +- .pre-commit-config.yaml | 2 +- README.rst | 2 +- docs/intro/install.rst | 2 +- scrapy/addons.py | 4 +- scrapy/cmdline.py | 26 +++--- scrapy/commands/__init__.py | 14 ++-- scrapy/commands/bench.py | 6 +- scrapy/commands/check.py | 3 +- scrapy/commands/crawl.py | 4 +- scrapy/commands/edit.py | 3 +- scrapy/commands/fetch.py | 8 +- scrapy/commands/genspider.py | 4 +- scrapy/commands/list.py | 4 +- scrapy/commands/parse.py | 41 ++++----- scrapy/commands/runspider.py | 4 +- scrapy/commands/settings.py | 3 +- scrapy/commands/shell.py | 8 +- scrapy/commands/startproject.py | 10 +-- scrapy/commands/version.py | 3 +- scrapy/contracts/__init__.py | 41 ++++----- scrapy/contracts/default.py | 14 ++-- scrapy/core/downloader/__init__.py | 29 ++----- scrapy/core/downloader/contextfactory.py | 6 +- scrapy/core/downloader/handlers/__init__.py | 28 +++---- scrapy/core/downloader/handlers/datauri.py | 4 +- scrapy/core/downloader/handlers/ftp.py | 4 +- scrapy/core/downloader/handlers/http10.py | 9 +- scrapy/core/downloader/handlers/http11.py | 14 ++-- scrapy/core/downloader/handlers/s3.py | 4 +- scrapy/core/downloader/middleware.py | 7 +- scrapy/core/downloader/tls.py | 4 +- scrapy/core/downloader/webclient.py | 6 +- scrapy/core/engine.py | 27 ++---- scrapy/core/http2/agent.py | 14 ++-- scrapy/core/http2/protocol.py | 22 ++--- scrapy/core/http2/stream.py | 12 +-- scrapy/core/scheduler.py | 20 ++--- scrapy/core/scraper.py | 31 ++----- scrapy/core/spidermw.py | 22 ++--- scrapy/crawler.py | 43 ++++------ scrapy/downloadermiddlewares/cookies.py | 5 +- .../downloadermiddlewares/defaultheaders.py | 8 +- .../downloadermiddlewares/httpcompression.py | 20 ++--- scrapy/downloadermiddlewares/httpproxy.py | 6 +- scrapy/downloadermiddlewares/offsite.py | 4 +- scrapy/downloadermiddlewares/redirect.py | 4 +- scrapy/downloadermiddlewares/retry.py | 10 +-- scrapy/downloadermiddlewares/robotstxt.py | 4 +- scrapy/downloadermiddlewares/stats.py | 4 +- scrapy/dupefilters.py | 4 +- scrapy/exporters.py | 17 ++-- scrapy/extension.py | 4 +- scrapy/extensions/closespider.py | 6 +- scrapy/extensions/feedexport.py | 84 ++++++++----------- scrapy/extensions/httpcache.py | 28 ++++--- scrapy/extensions/logstats.py | 4 +- scrapy/extensions/memusage.py | 8 +- scrapy/extensions/periodic_log.py | 34 ++++---- scrapy/extensions/postprocessing.py | 12 +-- scrapy/extensions/statsmailer.py | 8 +- scrapy/extensions/telnet.py | 8 +- scrapy/extensions/throttle.py | 4 +- scrapy/http/cookies.py | 22 ++--- scrapy/http/headers.py | 35 +++----- scrapy/http/request/__init__.py | 36 ++++---- scrapy/http/request/form.py | 34 +++----- scrapy/http/request/json_request.py | 14 ++-- scrapy/http/response/__init__.py | 50 ++++------- scrapy/http/response/text.py | 36 +++----- scrapy/item.py | 22 ++--- scrapy/linkextractors/__init__.py | 9 +- scrapy/linkextractors/lxmlhtml.py | 55 +++++------- scrapy/logformatter.py | 6 +- scrapy/mail.py | 33 +++----- scrapy/middleware.py | 29 ++----- scrapy/pipelines/__init__.py | 4 +- scrapy/pipelines/files.py | 61 +++++++------- scrapy/pipelines/images.py | 41 ++++----- scrapy/pipelines/media.py | 26 +++--- scrapy/pqueues.py | 42 ++++------ scrapy/resolver.py | 8 +- scrapy/responsetypes.py | 23 +++-- scrapy/selector/unified.py | 4 +- scrapy/settings/__init__.py | 40 ++++----- scrapy/shell.py | 13 +-- scrapy/signalmanager.py | 6 +- scrapy/spiderloader.py | 14 ++-- scrapy/spidermiddlewares/depth.py | 4 +- scrapy/spidermiddlewares/httperror.py | 6 +- scrapy/spidermiddlewares/offsite.py | 6 +- scrapy/spidermiddlewares/referer.py | 29 +++---- scrapy/spidermiddlewares/urllength.py | 4 +- scrapy/spiders/__init__.py | 8 +- scrapy/spiders/crawl.py | 36 +++----- scrapy/spiders/feed.py | 13 ++- scrapy/spiders/init.py | 3 +- scrapy/spiders/sitemap.py | 27 ++---- scrapy/squeues.py | 17 ++-- scrapy/statscollectors.py | 6 +- scrapy/utils/asyncgen.py | 5 +- scrapy/utils/conf.py | 42 ++++------ scrapy/utils/console.py | 28 ++++--- scrapy/utils/curl.py | 17 ++-- scrapy/utils/datatypes.py | 20 ++--- scrapy/utils/decorators.py | 4 +- scrapy/utils/defer.py | 45 ++++------ scrapy/utils/deprecate.py | 12 +-- scrapy/utils/engine.py | 6 +- scrapy/utils/iterators.py | 26 +++--- scrapy/utils/log.py | 31 +++---- scrapy/utils/misc.py | 28 ++----- scrapy/utils/ossignal.py | 5 +- scrapy/utils/project.py | 2 - scrapy/utils/python.py | 50 ++++------- scrapy/utils/reactor.py | 22 ++--- scrapy/utils/request.py | 25 ++---- scrapy/utils/response.py | 8 +- scrapy/utils/signal.py | 21 ++--- scrapy/utils/sitemap.py | 11 ++- scrapy/utils/spider.py | 28 ++----- scrapy/utils/test.py | 28 +++---- scrapy/utils/testproc.py | 8 +- scrapy/utils/trackref.py | 10 ++- scrapy/utils/url.py | 8 +- scrapy/utils/versions.py | 3 +- setup.py | 11 ++- tests/mocks/dummydbm.py | 6 +- tests/mockserver.py | 4 +- tests/test_addons.py | 4 +- tests/test_commands.py | 9 +- tests/test_crawler.py | 3 +- tests/test_downloader_handlers.py | 18 ++-- tests/test_http2_client_protocol.py | 4 +- tests/test_http_request.py | 6 +- tests/test_pipeline_crawl.py | 4 +- tests/test_pipeline_files.py | 9 +- tests/test_pipeline_images.py | 10 +-- tests/test_request_cb_kwargs.py | 4 +- tests/test_scheduler_base.py | 4 +- tests/test_settings/__init__.py | 7 +- tests/test_spidermiddleware.py | 6 +- tests/test_spidermiddleware_httperror.py | 3 +- tests/test_spidermiddleware_referer.py | 32 +++---- tests/test_utils_datatypes.py | 3 +- tests/test_utils_log.py | 9 +- tests/test_utils_request.py | 8 +- tests_typing/test_http_request.mypy-testing | 2 +- tests_typing/test_http_response.mypy-testing | 2 +- tox.ini | 14 ++-- 153 files changed, 1011 insertions(+), 1307 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 2be6a9502..9240a16f4 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -15,10 +15,10 @@ jobs: - python-version: "3.12" env: TOXENV: pylint - - python-version: 3.8 + - python-version: "3.9" env: TOXENV: typing - - python-version: 3.8 + - python-version: "3.9" env: TOXENV: typing-tests - python-version: "3.12" # Keep in sync with .readthedocs.yml diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 8ebe7f1db..27ea0613d 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 763de9eff..29c870e6a 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.9 + - python-version: "3.9" env: TOXENV: py - python-version: "3.10" @@ -35,19 +35,19 @@ jobs: TOXENV: pypy3 # pinned deps - - python-version: 3.8.17 + - python-version: 3.9.19 env: TOXENV: pinned - - python-version: 3.8.17 + - python-version: 3.9.19 env: TOXENV: asyncio-pinned - - python-version: pypy3.8 + - python-version: pypy3.9 env: TOXENV: pypy3-pinned - - python-version: 3.8.17 + - python-version: 3.9.19 env: TOXENV: extra-deps-pinned - - python-version: 3.8.17 + - python-version: 3.9.19 env: TOXENV: botocore-pinned diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 80d09e7a0..5728c6fd0 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -12,12 +12,9 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.8 + - python-version: "3.9" env: TOXENV: windows-pinned - - python-version: 3.9 - env: - TOXENV: py - python-version: "3.10" env: TOXENV: py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index addad838f..75529be05 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,4 +33,4 @@ repos: rev: v3.16.0 hooks: - id: pyupgrade - args: [--py38-plus, --keep-runtime-typing] + args: [--py39-plus, --keep-runtime-typing] diff --git a/README.rst b/README.rst index 14adff648..e640bce35 100644 --- a/README.rst +++ b/README.rst @@ -59,7 +59,7 @@ including a list of features. Requirements ============ -* Python 3.8+ +* Python 3.9+ * Works on Linux, Windows, macOS, BSD Install diff --git a/docs/intro/install.rst b/docs/intro/install.rst index e6c9a683b..ef541368a 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,7 +9,7 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.8+, either the CPython implementation (default) or +Scrapy requires Python 3.9+, either the CPython implementation (default) or the PyPy implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: diff --git a/scrapy/addons.py b/scrapy/addons.py index f9ec58cea..7a1da3afc 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, List +from typing import TYPE_CHECKING, Any from scrapy.exceptions import NotConfigured from scrapy.utils.conf import build_component_list @@ -20,7 +20,7 @@ class AddonManager: def __init__(self, crawler: Crawler) -> None: self.crawler: Crawler = crawler - self.addons: List[Any] = [] + self.addons: list[Any] = [] def load_settings(self, settings: Settings) -> None: """Load add-ons and configurations from a settings object and apply them. diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index e010b159a..b820eb7f9 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -6,7 +6,7 @@ import inspect import os import sys from importlib.metadata import entry_points -from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple, Type +from typing import TYPE_CHECKING, Optional import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter @@ -17,6 +17,8 @@ from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect if TYPE_CHECKING: + from collections.abc import Callable, Iterable + # typing.ParamSpec requires Python 3.10 from typing_extensions import ParamSpec @@ -28,7 +30,7 @@ if TYPE_CHECKING: class ScrapyArgumentParser(argparse.ArgumentParser): def _parse_optional( self, arg_string: str - ) -> Optional[Tuple[Optional[argparse.Action], str, Optional[str]]]: + ) -> Optional[tuple[Optional[argparse.Action], str, Optional[str]]]: # if starts with -: it means that is a parameter not a argument if arg_string[:2] == "-:": return None @@ -36,7 +38,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser): return super()._parse_optional(arg_string) -def _iter_command_classes(module_name: str) -> Iterable[Type[ScrapyCommand]]: +def _iter_command_classes(module_name: str) -> Iterable[type[ScrapyCommand]]: # TODO: add `name` attribute to commands and merge this function with # scrapy.utils.spider.iter_spider_classes for module in walk_modules(module_name): @@ -50,8 +52,8 @@ def _iter_command_classes(module_name: str) -> Iterable[Type[ScrapyCommand]]: yield obj -def _get_commands_from_module(module: str, inproject: bool) -> Dict[str, ScrapyCommand]: - d: Dict[str, ScrapyCommand] = {} +def _get_commands_from_module(module: str, inproject: bool) -> dict[str, ScrapyCommand]: + d: dict[str, ScrapyCommand] = {} for cmd in _iter_command_classes(module): if inproject or not cmd.requires_project: cmdname = cmd.__module__.split(".")[-1] @@ -61,8 +63,8 @@ def _get_commands_from_module(module: str, inproject: bool) -> Dict[str, ScrapyC def _get_commands_from_entry_points( inproject: bool, group: str = "scrapy.commands" -) -> Dict[str, ScrapyCommand]: - cmds: Dict[str, ScrapyCommand] = {} +) -> dict[str, ScrapyCommand]: + cmds: dict[str, ScrapyCommand] = {} if sys.version_info >= (3, 10): eps = entry_points(group=group) else: @@ -78,7 +80,7 @@ def _get_commands_from_entry_points( def _get_commands_dict( settings: BaseSettings, inproject: bool -) -> Dict[str, ScrapyCommand]: +) -> dict[str, ScrapyCommand]: cmds = _get_commands_from_module("scrapy.commands", inproject) cmds.update(_get_commands_from_entry_points(inproject)) cmds_module = settings["COMMANDS_MODULE"] @@ -87,7 +89,7 @@ def _get_commands_dict( return cmds -def _pop_command_name(argv: List[str]) -> Optional[str]: +def _pop_command_name(argv: list[str]) -> Optional[str]: i = 0 for arg in argv[1:]: if not arg.startswith("-"): @@ -146,7 +148,7 @@ def _run_print_help( def execute( - argv: Optional[List[str]] = None, settings: Optional[Settings] = None + argv: Optional[list[str]] = None, settings: Optional[Settings] = None ) -> None: if argv is None: argv = sys.argv @@ -189,7 +191,7 @@ def execute( sys.exit(cmd.exitcode) -def _run_command(cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace) -> None: +def _run_command(cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace) -> None: if opts.profile: _run_command_profiled(cmd, args, opts) else: @@ -197,7 +199,7 @@ def _run_command(cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace) def _run_command_profiled( - cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace + cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace ) -> None: if opts.profile: sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 0322390e5..a94db90b1 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -8,7 +8,7 @@ import argparse import builtins import os from pathlib import Path -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional +from typing import TYPE_CHECKING, Any, Optional from twisted.python import failure @@ -16,6 +16,8 @@ from scrapy.exceptions import UsageError from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli if TYPE_CHECKING: + from collections.abc import Iterable + from scrapy.crawler import Crawler, CrawlerProcess @@ -24,7 +26,7 @@ class ScrapyCommand: crawler_process: Optional[CrawlerProcess] = None # default settings to be used for this command instead of global defaults - default_settings: Dict[str, Any] = {} + default_settings: dict[str, Any] = {} exitcode: int = 0 @@ -97,7 +99,7 @@ class ScrapyCommand: ) group.add_argument("--pdb", action="store_true", help="enable pdb on failure") - def process_options(self, args: List[str], opts: argparse.Namespace) -> None: + def process_options(self, args: list[str], opts: argparse.Namespace) -> None: try: self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline") except ValueError: @@ -122,7 +124,7 @@ class ScrapyCommand: if opts.pdb: failure.startDebugMode() - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: """ Entry point for running commands """ @@ -167,7 +169,7 @@ class BaseRunSpiderCommand(ScrapyCommand): help="format to use for dumping items", ) - def process_options(self, args: List[str], opts: argparse.Namespace) -> None: + def process_options(self, args: list[str], opts: argparse.Namespace) -> None: super().process_options(args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) @@ -207,7 +209,7 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): parts = self.format_part_strings(builtins.list(part_strings)) return super()._join_parts(parts) - def format_part_strings(self, part_strings: List[str]) -> List[str]: + def format_part_strings(self, part_strings: list[str]) -> list[str]: """ Underline and title case command line help message headers. """ diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index f91fec57e..4f6933006 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -4,7 +4,7 @@ import argparse import subprocess # nosec import sys import time -from typing import TYPE_CHECKING, Any, Iterable, List +from typing import TYPE_CHECKING, Any from urllib.parse import urlencode import scrapy @@ -13,6 +13,8 @@ from scrapy.http import Response, TextResponse from scrapy.linkextractors import LinkExtractor if TYPE_CHECKING: + from collections.abc import Iterable + from scrapy import Request @@ -26,7 +28,7 @@ class Command(ScrapyCommand): def short_desc(self) -> str: return "Run quick benchmark test" - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: with _BenchServer(): assert self.crawler_process self.crawler_process.crawl(_BenchSpider, total=100000) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 22c8abf7a..c7946605b 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -1,7 +1,6 @@ import argparse import time from collections import defaultdict -from typing import List from unittest import TextTestResult as _TextTestResult from unittest import TextTestRunner @@ -69,7 +68,7 @@ class Command(ScrapyCommand): help="print contract tests for all spiders", ) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: # load contracts contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS")) conman = ContractsManager(load_object(c) for c in contracts) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index fe1864372..6b6a80bb5 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, List, cast +from typing import TYPE_CHECKING, cast from twisted.python.failure import Failure @@ -20,7 +20,7 @@ class Command(BaseRunSpiderCommand): def short_desc(self) -> str: return "Run a spider" - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) < 1: raise UsageError() elif len(args) > 1: diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 04012bee8..34313d731 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -1,7 +1,6 @@ import argparse import os import sys -from typing import List from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError @@ -27,7 +26,7 @@ class Command(ScrapyCommand): sys.stderr.write(msg + os.linesep) self.exitcode = 1 - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) != 1: raise UsageError() diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 0bdc429da..a1806f626 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,7 +1,7 @@ from __future__ import annotations import sys -from typing import TYPE_CHECKING, Dict, List, Type +from typing import TYPE_CHECKING from w3lib.url import is_url @@ -48,7 +48,7 @@ class Command(ScrapyCommand): help="do not handle HTTP 3xx status codes and print response as-is", ) - def _print_headers(self, headers: Dict[bytes, List[bytes]], prefix: bytes) -> None: + def _print_headers(self, headers: dict[bytes, list[bytes]], prefix: bytes) -> None: for key, values in headers.items(): for value in values: self._print_bytes(prefix + b" " + key + b": " + value) @@ -65,7 +65,7 @@ class Command(ScrapyCommand): def _print_bytes(self, bytes_: bytes) -> None: sys.stdout.buffer.write(bytes_ + b"\n") - def run(self, args: List[str], opts: Namespace) -> None: + def run(self, args: list[str], opts: Namespace) -> None: if len(args) != 1 or not is_url(args[0]): raise UsageError() request = Request( @@ -81,7 +81,7 @@ class Command(ScrapyCommand): else: request.meta["handle_httpstatus_all"] = True - spidercls: Type[Spider] = DefaultSpider + spidercls: type[Spider] = DefaultSpider assert self.crawler_process spider_loader = self.crawler_process.spider_loader if opts.spider: diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 6c3713f8f..a9b7a6eee 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -4,7 +4,7 @@ import shutil import string from importlib import import_module from pathlib import Path -from typing import List, Optional, Union, cast +from typing import Optional, Union, cast from urllib.parse import urlparse import scrapy @@ -87,7 +87,7 @@ class Command(ScrapyCommand): help="If the spider already exists, overwrite it with the template", ) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if opts.list: self._list_templates() return diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 10330c92a..3b2f127c2 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from scrapy.commands import ScrapyCommand @@ -15,7 +15,7 @@ class Command(ScrapyCommand): def short_desc(self) -> str: return "List available spiders" - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: assert self.crawler_process for s in sorted(self.crawler_process.spider_loader.list()): print(s) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index fbd200d88..bd1fad14b 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -5,20 +5,7 @@ import functools import inspect import json import logging -from typing import ( - TYPE_CHECKING, - Any, - AsyncGenerator, - Coroutine, - Dict, - Iterable, - List, - Optional, - Tuple, - TypeVar, - Union, - overload, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, overload from itemadapter import ItemAdapter, is_item from twisted.internet.defer import Deferred, maybeDeferred @@ -35,6 +22,8 @@ from scrapy.utils.misc import arg_to_iter from scrapy.utils.spider import spidercls_for_request if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Coroutine, Iterable + from twisted.python.failure import Failure from scrapy.http.request import CallbackT @@ -50,8 +39,8 @@ class Command(BaseRunSpiderCommand): requires_project = True spider = None - items: Dict[int, List[Any]] = {} - requests: Dict[int, List[Request]] = {} + items: dict[int, list[Any]] = {} + requests: dict[int, list[Request]] = {} first_response = None @@ -166,11 +155,11 @@ class Command(BaseRunSpiderCommand): return d return arg_to_iter(deferred_from_coro(result)) - def add_items(self, lvl: int, new_items: List[Any]) -> None: + def add_items(self, lvl: int, new_items: list[Any]) -> None: old_items = self.items.get(lvl, []) self.items[lvl] = old_items + new_items - def add_requests(self, lvl: int, new_reqs: List[Request]) -> None: + def add_requests(self, lvl: int, new_reqs: list[Request]) -> None: old_reqs = self.requests.get(lvl, []) self.requests[lvl] = old_reqs + new_reqs @@ -219,7 +208,7 @@ class Command(BaseRunSpiderCommand): depth: int, spider: Spider, callback: CallbackT, - ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT]: + ) -> tuple[list[Any], list[Request], argparse.Namespace, int, Spider, CallbackT]: items, requests = [], [] for x in spider_output: if is_item(x): @@ -232,7 +221,7 @@ class Command(BaseRunSpiderCommand): self, response: Response, callback: CallbackT, - cb_kwargs: Optional[Dict[str, Any]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, ) -> Deferred[Any]: cb_kwargs = cb_kwargs or {} d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) @@ -285,10 +274,10 @@ class Command(BaseRunSpiderCommand): def scraped_data( self, - args: Tuple[ - List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT + args: tuple[ + list[Any], list[Request], argparse.Namespace, int, Spider, CallbackT ], - ) -> List[Any]: + ) -> list[Any]: items, requests, opts, depth, spider, callback = args if opts.pipelines: itemproc = self.pcrawler.engine.scraper.itemproc @@ -345,7 +334,7 @@ class Command(BaseRunSpiderCommand): def prepare_request( self, spider: Spider, request: Request, opts: argparse.Namespace ) -> Request: - def callback(response: Response, **cb_kwargs: Any) -> Deferred[List[Any]]: + def callback(response: Response, **cb_kwargs: Any) -> Deferred[list[Any]]: # memorize first request if not self.first_response: self.first_response = response @@ -376,7 +365,7 @@ class Command(BaseRunSpiderCommand): request.callback = callback return request - def process_options(self, args: List[str], opts: argparse.Namespace) -> None: + def process_options(self, args: list[str], opts: argparse.Namespace) -> None: super().process_options(args, opts) self.process_request_meta(opts) @@ -404,7 +393,7 @@ class Command(BaseRunSpiderCommand): print_help=False, ) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: # parse arguments if not len(args) == 1 or not is_url(args[0]): raise UsageError() diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 87acf9a01..14d58f311 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -4,7 +4,7 @@ import argparse import sys from importlib import import_module from pathlib import Path -from typing import TYPE_CHECKING, List, Union +from typing import TYPE_CHECKING, Union from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError @@ -41,7 +41,7 @@ class Command(BaseRunSpiderCommand): def long_desc(self) -> str: return "Run the spider defined in the given file" - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) != 1: raise UsageError() filename = Path(args[0]) diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index dbda73b44..59f86b9a7 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,6 +1,5 @@ import argparse import json -from typing import List from scrapy.commands import ScrapyCommand from scrapy.settings import BaseSettings @@ -46,7 +45,7 @@ class Command(ScrapyCommand): help="print setting value, interpreted as a list", ) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: assert self.crawler_process settings = self.crawler_process.settings if opts.get: diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index f03cf997a..27e6d68ee 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -7,7 +7,7 @@ See documentation in docs/topics/shell.rst from __future__ import annotations from threading import Thread -from typing import TYPE_CHECKING, Any, Dict, List, Type +from typing import TYPE_CHECKING, Any from scrapy import Spider from scrapy.commands import ScrapyCommand @@ -56,13 +56,13 @@ class Command(ScrapyCommand): help="do not handle HTTP 3xx status codes and print response as-is", ) - def update_vars(self, vars: Dict[str, Any]) -> None: + def update_vars(self, vars: dict[str, Any]) -> None: """You can use this function to update the Scrapy objects that will be available in the shell """ pass - def run(self, args: List[str], opts: Namespace) -> None: + def run(self, args: list[str], opts: Namespace) -> None: url = args[0] if args else None if url: # first argument may be a local file @@ -71,7 +71,7 @@ class Command(ScrapyCommand): assert self.crawler_process spider_loader = self.crawler_process.spider_loader - spidercls: Type[Spider] = DefaultSpider + spidercls: type[Spider] = DefaultSpider if opts.spider: spidercls = spider_loader.load(opts.spider) elif url: diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 58c1aa28f..f7052cd18 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -6,14 +6,14 @@ from importlib.util import find_spec from pathlib import Path from shutil import copy2, copystat, ignore_patterns, move from stat import S_IWUSR as OWNER_WRITE_PERMISSION -from typing import List, Tuple, Union +from typing import Union import scrapy from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.template import render_templatefile, string_camelcase -TEMPLATES_TO_RENDER: Tuple[Tuple[str, ...], ...] = ( +TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = ( ("scrapy.cfg",), ("${project_name}", "settings.py.tmpl"), ("${project_name}", "items.py.tmpl"), @@ -86,7 +86,7 @@ class Command(ScrapyCommand): copystat(src, dst) _make_writable(dst) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) not in (1, 2): raise UsageError() @@ -107,9 +107,7 @@ class Command(ScrapyCommand): return self._copytree(Path(self.templates_dir), project_dir.resolve()) - # On 3.8 shutil.move doesn't fully support Path args, but it supports our use case - # See https://bugs.python.org/issue32689 - move(project_dir / "module", project_dir / project_name) # type: ignore[arg-type] + move(project_dir / "module", project_dir / project_name) for paths in TEMPLATES_TO_RENDER: tplfile = Path( project_dir, diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index f057e8544..571f4fda8 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -1,5 +1,4 @@ import argparse -from typing import List import scrapy from scrapy.commands import ScrapyCommand @@ -25,7 +24,7 @@ class Command(ScrapyCommand): help="also display twisted/python/platform info (useful for bug reports)", ) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if opts.verbose: versions = scrapy_components_versions() width = max(len(n) for (n, _) in versions) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index a7e129948..ffe5053de 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -2,22 +2,11 @@ from __future__ import annotations import re import sys +from collections.abc import AsyncGenerator, Iterable from functools import wraps from inspect import getmembers from types import CoroutineType -from typing import ( - TYPE_CHECKING, - Any, - AsyncGenerator, - Callable, - Dict, - Iterable, - List, - Optional, - Tuple, - Type, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, cast from unittest import TestCase, TestResult from scrapy.http import Request, Response @@ -25,6 +14,8 @@ from scrapy.utils.python import get_spec from scrapy.utils.spider import iterate_spider_output if TYPE_CHECKING: + from collections.abc import Callable + from twisted.python.failure import Failure from scrapy import Spider @@ -33,13 +24,13 @@ if TYPE_CHECKING: class Contract: """Abstract class for contracts""" - request_cls: Optional[Type[Request]] = None + request_cls: Optional[type[Request]] = None name: str def __init__(self, method: Callable, *args: Any): self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook") self.testcase_post = _create_testcase(method, f"@{self.name} post-hook") - self.args: Tuple[Any, ...] = args + self.args: tuple[Any, ...] = args def add_pre_hook(self, request: Request, results: TestResult) -> Request: if hasattr(self, "pre_process"): @@ -47,7 +38,7 @@ class Contract: assert cb is not None @wraps(cb) - def wrapper(response: Response, **cb_kwargs: Any) -> List[Any]: + def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]: try: results.startTest(self.testcase_pre) self.pre_process(response) @@ -76,7 +67,7 @@ class Contract: assert cb is not None @wraps(cb) - def wrapper(response: Response, **cb_kwargs: Any) -> List[Any]: + def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") @@ -98,18 +89,18 @@ class Contract: return request - def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: + def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: return args class ContractsManager: - contracts: Dict[str, Type[Contract]] = {} + contracts: dict[str, type[Contract]] = {} - def __init__(self, contracts: Iterable[Type[Contract]]): + def __init__(self, contracts: Iterable[type[Contract]]): for contract in contracts: self.contracts[contract.name] = contract - def tested_methods_from_spidercls(self, spidercls: Type[Spider]) -> List[str]: + def tested_methods_from_spidercls(self, spidercls: type[Spider]) -> list[str]: is_method = re.compile(r"^\s*@", re.MULTILINE).search methods = [] for key, value in getmembers(spidercls): @@ -118,8 +109,8 @@ class ContractsManager: return methods - def extract_contracts(self, method: Callable) -> List[Contract]: - contracts: List[Contract] = [] + def extract_contracts(self, method: Callable) -> list[Contract]: + contracts: list[Contract] = [] assert method.__doc__ is not None for line in method.__doc__.split("\n"): line = line.strip() @@ -137,8 +128,8 @@ class ContractsManager: def from_spider( self, spider: Spider, results: TestResult - ) -> List[Optional[Request]]: - requests: List[Optional[Request]] = [] + ) -> list[Optional[Request]]: + requests: list[Optional[Request]] = [] for method in self.tested_methods_from_spidercls(type(spider)): bound_method = spider.__getattribute__(method) try: diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index e7b11d426..87170d3c1 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -1,5 +1,5 @@ import json -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Optional from itemadapter import ItemAdapter, is_item @@ -16,7 +16,7 @@ class UrlContract(Contract): name = "url" - def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: + def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: args["url"] = self.args[0] return args @@ -30,7 +30,7 @@ class CallbackKeywordArgumentsContract(Contract): name = "cb_kwargs" - def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: + def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: args["cb_kwargs"] = json.loads(" ".join(self.args)) return args @@ -44,7 +44,7 @@ class MetadataContract(Contract): name = "meta" - def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: + def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: args["meta"] = json.loads(" ".join(self.args)) return args @@ -63,7 +63,7 @@ class ReturnsContract(Contract): """ name = "returns" - object_type_verifiers: Dict[Optional[str], Callable[[Any], bool]] = { + object_type_verifiers: dict[Optional[str], Callable[[Any], bool]] = { "request": lambda x: isinstance(x, Request), "requests": lambda x: isinstance(x, Request), "item": is_item, @@ -90,7 +90,7 @@ class ReturnsContract(Contract): except IndexError: self.max_bound = float("inf") - def post_process(self, output: List[Any]) -> None: + def post_process(self, output: list[Any]) -> None: occurrences = 0 for x in output: if self.obj_type_verifier(x): @@ -116,7 +116,7 @@ class ScrapesContract(Contract): name = "scrapes" - def post_process(self, output: List[Any]) -> None: + def post_process(self, output: list[Any]) -> None: for x in output: if is_item(x): missing = [arg for arg in self.args if arg not in ItemAdapter(x)] diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 6786d7acf..77d57a8d8 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -5,18 +5,7 @@ import warnings from collections import deque from datetime import datetime from time import time -from typing import ( - TYPE_CHECKING, - Any, - Deque, - Dict, - Optional, - Set, - Tuple, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from twisted.internet import task from twisted.internet.defer import Deferred @@ -55,9 +44,9 @@ class Slot: self.randomize_delay: bool = randomize_delay self.throttle = throttle - self.active: Set[Request] = set() - self.queue: Deque[Tuple[Request, Deferred[Response]]] = deque() - self.transferring: Set[Request] = set() + self.active: set[Request] = set() + self.queue: deque[tuple[Request, Deferred[Response]]] = deque() + self.transferring: set[Request] = set() self.lastseen: float = 0 self.latercall = None @@ -95,7 +84,7 @@ class Slot: def _get_concurrency_delay( concurrency: int, spider: Spider, settings: BaseSettings -) -> Tuple[int, float]: +) -> tuple[int, float]: delay: float = settings.getfloat("DOWNLOAD_DELAY") if hasattr(spider, "download_delay"): delay = spider.download_delay @@ -112,8 +101,8 @@ class Downloader: def __init__(self, crawler: Crawler): self.settings: BaseSettings = crawler.settings self.signals: SignalManager = crawler.signals - self.slots: Dict[str, Slot] = {} - self.active: Set[Request] = set() + self.slots: dict[str, Slot] = {} + self.active: set[Request] = set() self.handlers: DownloadHandlers = DownloadHandlers(crawler) self.total_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS") self.domain_concurrency: int = self.settings.getint( @@ -126,7 +115,7 @@ class Downloader: ) self._slot_gc_loop: task.LoopingCall = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) - self.per_slot_settings: Dict[str, Dict[str, Any]] = self.settings.getdict( + self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict( "DOWNLOAD_SLOTS", {} ) @@ -146,7 +135,7 @@ class Downloader: def needs_backout(self) -> bool: return len(self.active) >= self.total_concurrency - def _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]: + def _get_slot(self, request: Request, spider: Spider) -> tuple[str, Slot]: key = self.get_slot_key(request) if key not in self.slots: slot_settings = self.per_slot_settings.get(key, {}) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 2b388a9f5..ba20c3c2c 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any, List, Optional +from typing import TYPE_CHECKING, Any, Optional from OpenSSL import SSL from twisted.internet._sslverify import _setAcceptableProtocols @@ -154,10 +154,10 @@ class AcceptableProtocolsContextFactory: negotiation. """ - def __init__(self, context_factory: Any, acceptable_protocols: List[bytes]): + def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]): verifyObject(IPolicyForHTTPS, context_factory) self._wrapped_context_factory: Any = context_factory - self._acceptable_protocols: List[bytes] = acceptable_protocols + self._acceptable_protocols: list[bytes] = acceptable_protocols def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: options: ClientTLSOptions = self._wrapped_context_factory.creatorForNetloc( diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 70d356b83..c39e480f1 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -3,18 +3,8 @@ from __future__ import annotations import logging -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Generator, - Optional, - Protocol, - Type, - Union, - cast, -) +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, Optional, Protocol, Union, cast from twisted.internet import defer @@ -25,6 +15,8 @@ from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values if TYPE_CHECKING: + from collections.abc import Generator + from twisted.internet.defer import Deferred from scrapy.crawler import Crawler @@ -43,16 +35,16 @@ class DownloadHandlerProtocol(Protocol): class DownloadHandlers: def __init__(self, crawler: Crawler): self._crawler: Crawler = crawler - self._schemes: Dict[str, Union[str, Callable[..., Any]]] = ( + self._schemes: dict[str, Union[str, Callable[..., Any]]] = ( {} ) # stores acceptable schemes on instancing - self._handlers: Dict[str, DownloadHandlerProtocol] = ( + self._handlers: dict[str, DownloadHandlerProtocol] = ( {} ) # stores instanced handlers for schemes - self._notconfigured: Dict[str, str] = {} # remembers failed handlers - handlers: Dict[str, Union[str, Callable[..., Any]]] = without_none_values( + self._notconfigured: dict[str, str] = {} # remembers failed handlers + handlers: dict[str, Union[str, Callable[..., Any]]] = without_none_values( cast( - Dict[str, Union[str, Callable[..., Any]]], + dict[str, Union[str, Callable[..., Any]]], crawler.settings.getwithbase("DOWNLOAD_HANDLERS"), ) ) @@ -81,7 +73,7 @@ class DownloadHandlers: ) -> Optional[DownloadHandlerProtocol]: path = self._schemes[scheme] try: - dhcls: Type[DownloadHandlerProtocol] = load_object(path) + dhcls: type[DownloadHandlerProtocol] = load_object(path) if skip_lazy and getattr(dhcls, "lazy", True): return None dh = build_from_crawler( diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index bf6879521..b3f286d87 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict +from typing import TYPE_CHECKING, Any from w3lib.url import parse_data_uri @@ -20,7 +20,7 @@ class DataURIDownloadHandler: uri = parse_data_uri(request.url) respcls = responsetypes.from_mimetype(uri.media_type) - resp_kwargs: Dict[str, Any] = {} + resp_kwargs: dict[str, Any] = {} if issubclass(respcls, TextResponse) and uri.media_type.split("/")[0] == "text": charset = uri.media_type_parameters.get("charset") resp_kwargs["encoding"] = charset diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 69c2d88e1..bc06c7ef4 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -32,7 +32,7 @@ from __future__ import annotations import re from io import BytesIO -from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional +from typing import TYPE_CHECKING, Any, BinaryIO, Optional from urllib.parse import unquote from twisted.internet.protocol import ClientCreator, Protocol @@ -79,7 +79,7 @@ _CODE_RE = re.compile(r"\d+") class FTPDownloadHandler: lazy = False - CODE_MAPPING: Dict[str, int] = { + CODE_MAPPING: dict[str, int] = { "550": 404, "default": 503, } diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 8d7b0635c..58f7ad577 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -1,9 +1,8 @@ -"""Download handlers for http and https schemes -""" +"""Download handlers for http and https schemes""" from __future__ import annotations -from typing import TYPE_CHECKING, Type +from typing import TYPE_CHECKING from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode @@ -27,10 +26,10 @@ class HTTP10DownloadHandler: lazy = False def __init__(self, settings: BaseSettings, crawler: Crawler): - self.HTTPClientFactory: Type[ScrapyHTTPClientFactory] = load_object( + self.HTTPClientFactory: type[ScrapyHTTPClientFactory] = load_object( settings["DOWNLOADER_HTTPCLIENTFACTORY"] ) - self.ClientContextFactory: Type[ScrapyClientContextFactory] = load_object( + self.ClientContextFactory: type[ScrapyClientContextFactory] = load_object( settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] ) self._settings: BaseSettings = settings diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c06d90f01..f96dc7c98 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,7 +8,7 @@ import re from contextlib import suppress from io import BytesIO from time import time -from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, TypeVar, Union +from typing import TYPE_CHECKING, Any, Optional, TypedDict, TypeVar, Union from urllib.parse import urldefrag, urlunparse from twisted.internet import ssl @@ -52,7 +52,7 @@ _T = TypeVar("_T") class _ResultT(TypedDict): txresponse: TxResponse body: bytes - flags: Optional[List[str]] + flags: Optional[list[str]] certificate: Optional[ssl.Certificate] ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None] failure: NotRequired[Optional[Failure]] @@ -143,10 +143,10 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): reactor: ReactorBase, host: str, port: int, - proxyConf: Tuple[str, int, Optional[bytes]], + proxyConf: tuple[str, int, Optional[bytes]], contextFactory: IPolicyForHTTPS, timeout: float = 30, - bindAddress: Optional[Tuple[str, int]] = None, + bindAddress: Optional[tuple[str, int]] = None, ): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) @@ -254,14 +254,14 @@ class TunnelingAgent(Agent): self, *, reactor: ReactorBase, - proxyConf: Tuple[str, int, Optional[bytes]], + proxyConf: tuple[str, int, Optional[bytes]], contextFactory: IPolicyForHTTPS, connectTimeout: Optional[float] = None, bindAddress: Optional[bytes] = None, pool: Optional[HTTPConnectionPool] = None, ): super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) - self._proxyConf: Tuple[str, int, Optional[bytes]] = proxyConf + self._proxyConf: tuple[str, int, Optional[bytes]] = proxyConf self._contextFactory: IPolicyForHTTPS = contextFactory def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint: @@ -621,7 +621,7 @@ class _ResponseReader(Protocol): self._crawler: Crawler = crawler def _finish_response( - self, flags: Optional[List[str]] = None, failure: Optional[Failure] = None + self, flags: Optional[list[str]] = None, failure: Optional[Failure] = None ) -> None: self._finished.callback( { diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index edf370193..fa660c63c 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, Type +from typing import TYPE_CHECKING, Any, Optional from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.exceptions import NotConfigured @@ -29,7 +29,7 @@ class S3DownloadHandler: aws_access_key_id: Optional[str] = None, aws_secret_access_key: Optional[str] = None, aws_session_token: Optional[str] = None, - httpdownloadhandler: Type[HTTPDownloadHandler] = HTTPDownloadHandler, + httpdownloadhandler: type[HTTPDownloadHandler] = HTTPDownloadHandler, **kw: Any, ): if not is_botocore_available(): diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 0bdb756c8..00d3bd1b0 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -6,7 +6,8 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, Callable, Generator, List, Union, cast +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks @@ -17,6 +18,8 @@ from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_from_coro, mustbe_deferred if TYPE_CHECKING: + from collections.abc import Generator + from twisted.python.failure import Failure from scrapy import Spider @@ -27,7 +30,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): component_name = "downloader middleware" @classmethod - def _get_mwlist_from_settings(cls, settings: BaseSettings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: return build_component_list(settings.getwithbase("DOWNLOADER_MIDDLEWARES")) def _add_middleware(self, mw: Any) -> None: diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 33cea7263..1ae66f614 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Dict +from typing import Any from OpenSSL import SSL from service_identity.exceptions import CertificateError @@ -21,7 +21,7 @@ METHOD_TLSv11 = "TLSv1.1" METHOD_TLSv12 = "TLSv1.2" -openssl_methods: Dict[str, int] = { +openssl_methods: dict[str, int] = { METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only METHOD_TLSv11: SSL.TLSv1_1_METHOD, # TLS 1.1 only diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 99502f0d2..509bda4e4 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -2,7 +2,7 @@ from __future__ import annotations import re from time import time -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Optional from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse from twisted.internet import defer @@ -18,7 +18,7 @@ if TYPE_CHECKING: from scrapy import Request -def _parsed_url_args(parsed: ParseResult) -> Tuple[bytes, bytes, bytes, int, bytes]: +def _parsed_url_args(parsed: ParseResult) -> tuple[bytes, bytes, bytes, int, bytes]: # Assume parsed is urlparse-d from Request.url, # which was passed via safe_url_string and is ascii-only. path_str = urlunparse(("", "", parsed.path or "/", parsed.params, parsed.query, "")) @@ -33,7 +33,7 @@ def _parsed_url_args(parsed: ParseResult) -> Tuple[bytes, bytes, bytes, int, byt return scheme, netloc, host, port, path -def _parse(url: str) -> Tuple[bytes, bytes, bytes, int, bytes]: +def _parse(url: str) -> tuple[bytes, bytes, bytes, int, bytes]: """Return tuple of (scheme, netloc, host, port, path), all in bytes except for port which is int. Assume url is from Request.url, which was passed via safe_url_string diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index fd9a5f781..bb09d066f 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -9,20 +9,7 @@ from __future__ import annotations import logging from time import time -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Generator, - Iterable, - Iterator, - Optional, - Set, - Type, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks, succeed @@ -42,6 +29,8 @@ from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: + from collections.abc import Callable, Generator, Iterable, Iterator + from scrapy.core.scheduler import BaseScheduler from scrapy.core.scraper import _HandleOutputDeferred from scrapy.crawler import Crawler @@ -63,7 +52,7 @@ class Slot: scheduler: BaseScheduler, ) -> None: self.closing: Optional[Deferred[None]] = None - self.inprogress: Set[Request] = set() + self.inprogress: set[Request] = set() self.start_requests: Optional[Iterator[Request]] = iter(start_requests) self.close_if_idle: bool = close_if_idle self.nextcall: CallLaterOnce[None] = nextcall @@ -106,10 +95,10 @@ class ExecutionEngine: self.spider: Optional[Spider] = None self.running: bool = False self.paused: bool = False - self.scheduler_cls: Type[BaseScheduler] = self._get_scheduler_class( + self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( crawler.settings ) - downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"]) + downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) self.downloader: Downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = ( @@ -117,10 +106,10 @@ class ExecutionEngine: ) self.start_time: Optional[float] = None - def _get_scheduler_class(self, settings: BaseSettings) -> Type[BaseScheduler]: + def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: from scrapy.core.scheduler import BaseScheduler - scheduler_cls: Type[BaseScheduler] = load_object(settings["SCHEDULER"]) + scheduler_cls: type[BaseScheduler] = load_object(settings["SCHEDULER"]) if not issubclass(scheduler_cls, BaseScheduler): raise TypeError( f"The provided scheduler class ({settings['SCHEDULER']})" diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 640fb7129..b5ff55eb0 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections import deque -from typing import TYPE_CHECKING, Deque, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Optional from twisted.internet import defer from twisted.internet.defer import Deferred @@ -26,7 +26,7 @@ if TYPE_CHECKING: from scrapy.spiders import Spider -ConnectionKeyT = Tuple[bytes, bytes, int] +ConnectionKeyT = tuple[bytes, bytes, int] class H2ConnectionPool: @@ -36,11 +36,11 @@ class H2ConnectionPool: # Store a dictionary which is used to get the respective # H2ClientProtocolInstance using the key as Tuple(scheme, hostname, port) - self._connections: Dict[ConnectionKeyT, H2ClientProtocol] = {} + self._connections: dict[ConnectionKeyT, H2ClientProtocol] = {} # Save all requests that arrive before the connection is established - self._pending_requests: Dict[ - ConnectionKeyT, Deque[Deferred[H2ClientProtocol]] + self._pending_requests: dict[ + ConnectionKeyT, deque[Deferred[H2ClientProtocol]] ] = {} def get_connection( @@ -68,7 +68,7 @@ class H2ConnectionPool: ) -> Deferred[H2ClientProtocol]: self._pending_requests[key] = deque() - conn_lost_deferred: Deferred[List[BaseException]] = Deferred() + conn_lost_deferred: Deferred[list[BaseException]] = Deferred() conn_lost_deferred.addCallback(self._remove_connection, key) factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) @@ -94,7 +94,7 @@ class H2ConnectionPool: return conn def _remove_connection( - self, errors: List[BaseException], key: ConnectionKeyT + self, errors: list[BaseException], key: ConnectionKeyT ) -> None: self._connections.pop(key) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 8aebbaab4..618423218 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -4,7 +4,7 @@ import ipaddress import itertools import logging from collections import deque -from typing import TYPE_CHECKING, Any, Deque, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -91,7 +91,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self, uri: URI, settings: Settings, - conn_lost_deferred: Deferred[List[BaseException]], + conn_lost_deferred: Deferred[list[BaseException]], ) -> None: """ Arguments: @@ -102,7 +102,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): conn_lost_deferred -- Deferred fires with the reason: Failure to notify that connection was lost """ - self._conn_lost_deferred: Deferred[List[BaseException]] = conn_lost_deferred + self._conn_lost_deferred: Deferred[list[BaseException]] = conn_lost_deferred config = H2Configuration(client_side=True, header_encoding="utf-8") self.conn = H2Connection(config=config) @@ -113,19 +113,19 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._stream_id_generator = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs - self.streams: Dict[int, Stream] = {} + self.streams: dict[int, Stream] = {} # If requests are received before connection is made we keep # all requests in a pool and send them as the connection is made - self._pending_request_stream_pool: Deque[Stream] = deque() + self._pending_request_stream_pool: deque[Stream] = deque() # Save an instance of errors raised which lead to losing the connection # We pass these instances to the streams ResponseFailed() failure - self._conn_lost_errors: List[BaseException] = [] + self._conn_lost_errors: list[BaseException] = [] # Some meta data of this connection # initialized when connection is successfully made - self.metadata: Dict[str, Any] = { + self.metadata: dict[str, Any] = { # Peer certificate instance "certificate": None, # Address of the server we are connected to which @@ -250,7 +250,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.conn.initiate_connection() self._write_to_transport() - def _lose_connection_with_error(self, errors: List[BaseException]) -> None: + def _lose_connection_with_error(self, errors: list[BaseException]) -> None: """Helper function to lose the connection with the error sent as a reason""" self._conn_lost_errors += errors @@ -353,7 +353,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._pending_request_stream_pool.clear() self.conn.close_connection() - def _handle_events(self, events: List[Event]) -> None: + def _handle_events(self, events: list[Event]) -> None: """Private method which acts as a bridge between the events received from the HTTP/2 data and IH2EventsHandler @@ -442,7 +442,7 @@ class H2ClientFactory(Factory): self, uri: URI, settings: Settings, - conn_lost_deferred: Deferred[List[BaseException]], + conn_lost_deferred: Deferred[list[BaseException]], ) -> None: self.uri = uri self.settings = settings @@ -451,5 +451,5 @@ class H2ClientFactory(Factory): def buildProtocol(self, addr: IAddress) -> H2ClientProtocol: return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred) - def acceptableProtocols(self) -> List[bytes]: + def acceptableProtocols(self) -> list[bytes]: return [PROTOCOL_NAME] diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index d8b5cc8eb..51ebdf489 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging from enum import Enum from io import BytesIO -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Optional from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError @@ -113,7 +113,7 @@ class Stream: # Metadata of an HTTP/2 connection stream # initialized when stream is instantiated - self.metadata: Dict[str, Any] = { + self.metadata: dict[str, Any] = { "request_content_length": ( 0 if self._request.body is None else len(self._request.body) ), @@ -134,7 +134,7 @@ class Stream: # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response: Dict[str, Any] = { + self._response: dict[str, Any] = { # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. "body": BytesIO(), @@ -196,7 +196,7 @@ class Stream: == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' ) - def _get_request_headers(self) -> List[Tuple[str, str]]: + def _get_request_headers(self) -> list[tuple[str, str]]: url = urlparse_cached(self._request) path = url.path @@ -349,7 +349,7 @@ class Stream: self._response["flow_controlled_size"], self.stream_id ) - def receive_headers(self, headers: List[HeaderTuple]) -> None: + def receive_headers(self, headers: list[HeaderTuple]) -> None: for name, value in headers: self._response["headers"].appendlist(name, value) @@ -382,7 +382,7 @@ class Stream: def close( self, reason: StreamCloseReason, - errors: Optional[List[BaseException]] = None, + errors: Optional[list[BaseException]] = None, from_protocol: bool = False, ) -> None: """Based on the reason sent we will handle each case.""" diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index d4286c874..ced18fc05 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,7 +4,7 @@ import json import logging from abc import abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any, List, Optional, Type, cast +from typing import TYPE_CHECKING, Any, Optional, cast # working around https://github.com/sphinx-doc/sphinx/issues/10400 from twisted.internet.defer import Deferred # noqa: TC002 @@ -182,18 +182,18 @@ class Scheduler(BaseScheduler): self, dupefilter: BaseDupeFilter, jobdir: Optional[str] = None, - dqclass: Optional[Type[BaseQueue]] = None, - mqclass: Optional[Type[BaseQueue]] = None, + dqclass: Optional[type[BaseQueue]] = None, + mqclass: Optional[type[BaseQueue]] = None, logunser: bool = False, stats: Optional[StatsCollector] = None, - pqclass: Optional[Type[ScrapyPriorityQueue]] = None, + pqclass: Optional[type[ScrapyPriorityQueue]] = None, crawler: Optional[Crawler] = None, ): self.df: BaseDupeFilter = dupefilter self.dqdir: Optional[str] = self._dqdir(jobdir) - self.pqclass: Optional[Type[ScrapyPriorityQueue]] = pqclass - self.dqclass: Optional[Type[BaseQueue]] = dqclass - self.mqclass: Optional[Type[BaseQueue]] = mqclass + self.pqclass: Optional[type[ScrapyPriorityQueue]] = pqclass + self.dqclass: Optional[type[BaseQueue]] = dqclass + self.mqclass: Optional[type[BaseQueue]] = mqclass self.logunser: bool = logunser self.stats: Optional[StatsCollector] = stats self.crawler: Optional[Crawler] = crawler @@ -364,13 +364,13 @@ class Scheduler(BaseScheduler): return str(dqdir) return None - def _read_dqs_state(self, dqdir: str) -> List[int]: + def _read_dqs_state(self, dqdir: str) -> list[int]: path = Path(dqdir, "active.json") if not path.exists(): return [] with path.open(encoding="utf-8") as f: - return cast(List[int], json.load(f)) + return cast(list[int], json.load(f)) - def _write_dqs_state(self, dqdir: str, state: List[int]) -> None: + def _write_dqs_state(self, dqdir: str, state: list[int]) -> None: with Path(dqdir, "active.json").open("w", encoding="utf-8") as f: json.dump(state, f) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 7a51dbeb4..29d7cb0c8 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -5,23 +5,8 @@ from __future__ import annotations import logging from collections import deque -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - Deque, - Generator, - Iterable, - Iterator, - List, - Optional, - Set, - Tuple, - Type, - TypeVar, - Union, - cast, -) +from collections.abc import AsyncIterable, Iterator +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks @@ -47,6 +32,8 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output if TYPE_CHECKING: + from collections.abc import Generator, Iterable + from scrapy.crawler import Crawler @@ -54,12 +41,12 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -_ParallelResult = List[Tuple[bool, Iterator[Any]]] +_ParallelResult = list[tuple[bool, Iterator[Any]]] if TYPE_CHECKING: # parameterized Deferreds require Twisted 21.7.0 _HandleOutputDeferred = Deferred[Union[_ParallelResult, None]] - QueueTuple = Tuple[Union[Response, Failure], Request, _HandleOutputDeferred] + QueueTuple = tuple[Union[Response, Failure], Request, _HandleOutputDeferred] class Slot: @@ -69,8 +56,8 @@ class Slot: def __init__(self, max_active_size: int = 5000000): self.max_active_size = max_active_size - self.queue: Deque[QueueTuple] = deque() - self.active: Set[Request] = set() + self.queue: deque[QueueTuple] = deque() + self.active: set[Request] = set() self.active_size: int = 0 self.itemproc_size: int = 0 self.closing: Optional[Deferred[Spider]] = None @@ -113,7 +100,7 @@ class Scraper: self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler( crawler ) - itemproc_cls: Type[ItemPipelineManager] = load_object( + itemproc_cls: type[ItemPipelineManager] = load_object( crawler.settings["ITEM_PROCESSOR"] ) self.itemproc: ItemPipelineManager = itemproc_cls.from_crawler(crawler) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index c9feac29c..223e4192e 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -7,22 +7,10 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging +from collections.abc import AsyncIterable, Callable, Iterable from inspect import isasyncgenfunction, iscoroutine from itertools import islice -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - Callable, - Generator, - Iterable, - List, - Optional, - Tuple, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure @@ -42,6 +30,8 @@ from scrapy.utils.defer import ( from scrapy.utils.python import MutableAsyncChain, MutableChain if TYPE_CHECKING: + from collections.abc import Generator + from scrapy.settings import BaseSettings @@ -66,7 +56,7 @@ class SpiderMiddlewareManager(MiddlewareManager): self.downgrade_warning_done = False @classmethod - def _get_mwlist_from_settings(cls, settings: BaseSettings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES")) def _add_middleware(self, mw: Any) -> None: @@ -349,7 +339,7 @@ class SpiderMiddlewareManager(MiddlewareManager): @staticmethod def _get_async_method_pair( mw: Any, methodname: str - ) -> Union[None, Callable, Tuple[Callable, Callable]]: + ) -> Union[None, Callable, tuple[Callable, Callable]]: normal_method: Optional[Callable] = getattr(mw, methodname, None) methodname_async = methodname + "_async" async_method: Optional[Callable] = getattr(mw, methodname_async, None) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ecb0a8150..b0a4932e1 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,18 +4,7 @@ import logging import pprint import signal import warnings -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Generator, - Optional, - Set, - Type, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from twisted.internet.defer import ( Deferred, @@ -53,6 +42,8 @@ from scrapy.utils.reactor import ( ) if TYPE_CHECKING: + from collections.abc import Generator + from scrapy.utils.request import RequestFingerprinter @@ -64,8 +55,8 @@ _T = TypeVar("_T") class Crawler: def __init__( self, - spidercls: Type[Spider], - settings: Union[None, Dict[str, Any], Settings] = None, + spidercls: type[Spider], + settings: Union[None, dict[str, Any], Settings] = None, init_reactor: bool = False, ): if isinstance(spidercls, Spider): @@ -74,7 +65,7 @@ class Crawler: if isinstance(settings, dict) or settings is None: settings = Settings(settings) - self.spidercls: Type[Spider] = spidercls + self.spidercls: type[Spider] = spidercls self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) self._update_root_log_handler() @@ -112,7 +103,7 @@ class Crawler: self.__remove_handler = lambda: logging.root.removeHandler(handler) self.signals.connect(self.__remove_handler, signals.engine_stopped) - lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) + lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) self.logformatter = lf_cls.from_crawler(self) self.request_fingerprinter = build_from_crawler( @@ -256,18 +247,18 @@ class CrawlerRunner: verifyClass(ISpiderLoader, loader_cls) return loader_cls.from_settings(settings.frozencopy()) - def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None): + def __init__(self, settings: Union[dict[str, Any], Settings, None] = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings self.spider_loader = self._get_spider_loader(settings) - self._crawlers: Set[Crawler] = set() - self._active: Set[Deferred[None]] = set() + self._crawlers: set[Crawler] = set() + self._active: set[Deferred[None]] = set() self.bootstrap_failed = False def crawl( self, - crawler_or_spidercls: Union[Type[Spider], str, Crawler], + crawler_or_spidercls: Union[type[Spider], str, Crawler], *args: Any, **kwargs: Any, ) -> Deferred[None]: @@ -314,7 +305,7 @@ class CrawlerRunner: return d.addBoth(_done) def create_crawler( - self, crawler_or_spidercls: Union[Type[Spider], str, Crawler] + self, crawler_or_spidercls: Union[type[Spider], str, Crawler] ) -> Crawler: """ Return a :class:`~scrapy.crawler.Crawler` object. @@ -335,11 +326,11 @@ class CrawlerRunner: return crawler_or_spidercls return self._create_crawler(crawler_or_spidercls) - def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler: + def _create_crawler(self, spidercls: Union[str, type[Spider]]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) # temporary cast until self.spider_loader is typed - return Crawler(cast(Type[Spider], spidercls), self.settings) + return Crawler(cast(type[Spider], spidercls), self.settings) def stop(self) -> Deferred[Any]: """ @@ -387,7 +378,7 @@ class CrawlerProcess(CrawlerRunner): def __init__( self, - settings: Union[Dict[str, Any], Settings, None] = None, + settings: Union[dict[str, Any], Settings, None] = None, install_root_handler: bool = True, ): super().__init__(settings) @@ -416,14 +407,14 @@ class CrawlerProcess(CrawlerRunner): ) reactor.callFromThread(self._stop_reactor) - def _create_crawler(self, spidercls: Union[Type[Spider], str]) -> Crawler: + def _create_crawler(self, spidercls: Union[type[Spider], str]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) init_reactor = not self._initialized_reactor self._initialized_reactor = True # temporary cast until self.spider_loader is typed return Crawler( - cast(Type[Spider], spidercls), self.settings, init_reactor=init_reactor + cast(type[Spider], spidercls), self.settings, init_reactor=init_reactor ) def start( diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 23140d263..e384793ee 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging from collections import defaultdict -from typing import TYPE_CHECKING, Any, DefaultDict, Iterable, Optional, Sequence, Union +from typing import TYPE_CHECKING, Any, Optional, Union from tldextract import TLDExtract @@ -13,6 +13,7 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from collections.abc import Iterable, Sequence from http.cookiejar import Cookie # typing.Self requires Python 3.11 @@ -39,7 +40,7 @@ class CookiesMiddleware: """This middleware enables working with sites that need cookies""" def __init__(self, debug: bool = False): - self.jars: DefaultDict[Any, CookieJar] = defaultdict(CookieJar) + self.jars: defaultdict[Any, CookieJar] = defaultdict(CookieJar) self.debug: bool = debug @classmethod diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 49b9fdc05..312c1e026 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -6,11 +6,13 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations -from typing import TYPE_CHECKING, Iterable, Tuple, Union +from typing import TYPE_CHECKING, Union from scrapy.utils.python import without_none_values if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -20,8 +22,8 @@ if TYPE_CHECKING: class DefaultHeadersMiddleware: - def __init__(self, headers: Iterable[Tuple[str, str]]): - self._headers: Iterable[Tuple[str, str]] = headers + def __init__(self, headers: Iterable[tuple[str, str]]): + self._headers: Iterable[tuple[str, str]] = headers @classmethod def from_crawler(cls, crawler: Crawler) -> Self: diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 6b0a56f7f..b0cede97d 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -3,7 +3,7 @@ from __future__ import annotations import warnings from itertools import chain from logging import getLogger -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Optional, Union from scrapy import Request, Spider, signals from scrapy.exceptions import IgnoreRequest, NotConfigured @@ -28,7 +28,7 @@ if TYPE_CHECKING: logger = getLogger(__name__) -ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"] +ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate"] try: try: @@ -50,7 +50,7 @@ else: class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be - sent/received from web sites""" + sent/received from websites""" def __init__( self, @@ -140,7 +140,7 @@ class HttpCompressionMiddleware: respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) - kwargs: Dict[str, Any] = {"body": decoded_body} + kwargs: dict[str, Any] = {"body": decoded_body} if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the # responsetypes guessing is reliable @@ -152,23 +152,23 @@ class HttpCompressionMiddleware: return response def _handle_encoding( - self, body: bytes, content_encoding: List[bytes], max_size: int - ) -> Tuple[bytes, List[bytes]]: + self, body: bytes, content_encoding: list[bytes], max_size: int + ) -> tuple[bytes, list[bytes]]: to_decode, to_keep = self._split_encodings(content_encoding) for encoding in to_decode: body = self._decode(body, encoding, max_size) return body, to_keep def _split_encodings( - self, content_encoding: List[bytes] - ) -> Tuple[List[bytes], List[bytes]]: - to_keep: List[bytes] = [ + self, content_encoding: list[bytes] + ) -> tuple[list[bytes], list[bytes]]: + to_keep: list[bytes] = [ encoding.strip().lower() for encoding in chain.from_iterable( encodings.split(b",") for encodings in content_encoding ) ] - to_decode: List[bytes] = [] + to_decode: list[bytes] = [] while to_keep: encoding = to_keep.pop() if encoding not in ACCEPTED_ENCODINGS: diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index a7af83f7d..b35ecbd54 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,7 +1,7 @@ from __future__ import annotations import base64 -from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union +from typing import TYPE_CHECKING, Optional, Union from urllib.parse import unquote, urlunparse from urllib.request import ( # type: ignore[attr-defined] _parse_proxy, @@ -25,7 +25,7 @@ if TYPE_CHECKING: class HttpProxyMiddleware: def __init__(self, auth_encoding: Optional[str] = "latin-1"): self.auth_encoding: Optional[str] = auth_encoding - self.proxies: Dict[str, Tuple[Optional[bytes], str]] = {} + self.proxies: dict[str, tuple[Optional[bytes], str]] = {} for type_, url in getproxies().items(): try: self.proxies[type_] = self._get_proxy(url, type_) @@ -47,7 +47,7 @@ class HttpProxyMiddleware: ) return base64.b64encode(user_pass) - def _get_proxy(self, url: str, orig_type: str) -> Tuple[Optional[bytes], str]: + def _get_proxy(self, url: str, orig_type: str) -> tuple[Optional[bytes], str]: proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", "")) diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index 6f67e3975..05ec4cad4 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging import re import warnings -from typing import TYPE_CHECKING, Set +from typing import TYPE_CHECKING from scrapy import Request, Spider, signals from scrapy.exceptions import IgnoreRequest @@ -31,7 +31,7 @@ class OffsiteMiddleware: def __init__(self, stats: StatsCollector): self.stats = stats - self.domains_seen: Set[str] = set() + self.domains_seen: set[str] = set() def spider_opened(self, spider: Spider) -> None: self.host_regex: re.Pattern[str] = self.get_host_regex(spider) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 53081237c..6437485cf 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, List, Union, cast +from typing import TYPE_CHECKING, Any, Union, cast from urllib.parse import urljoin from w3lib.url import safe_url_string @@ -180,7 +180,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): def __init__(self, settings: BaseSettings): super().__init__(settings) - self._ignore_tags: List[str] = settings.getlist("METAREFRESH_IGNORE_TAGS") + self._ignore_tags: list[str] = settings.getlist("METAREFRESH_IGNORE_TAGS") self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY") def process_response( diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 8d7b7293c..c32624371 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -7,14 +7,14 @@ RETRY_TIMES - how many times to retry a failed page RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, -once the spider has finished crawling all regular (non failed) pages. +once the spider has finished crawling all regular (non-failed) pages. """ from __future__ import annotations import warnings from logging import Logger, getLogger -from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union +from typing import TYPE_CHECKING, Any, Optional, Union from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.settings import BaseSettings, Settings @@ -35,7 +35,7 @@ if TYPE_CHECKING: retry_logger = getLogger(__name__) -def backwards_compatibility_getattr(self: Any, name: str) -> Tuple[Any, ...]: +def backwards_compatibility_getattr(self: Any, name: str) -> tuple[Any, ...]: if name == "EXCEPTIONS_TO_RETRY": warnings.warn( "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. " @@ -60,7 +60,7 @@ def get_retry_request( request: Request, *, spider: Spider, - reason: Union[str, Exception, Type[Exception]] = "unspecified", + reason: Union[str, Exception, type[Exception]] = "unspecified", max_retry_times: Optional[int] = None, priority_adjust: Optional[int] = None, logger: Logger = retry_logger, @@ -187,7 +187,7 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): def _retry( self, request: Request, - reason: Union[str, Exception, Type[Exception]], + reason: Union[str, Exception, type[Exception]], spider: Spider, ) -> Optional[Request]: max_retry_times = request.meta.get("max_retry_times", self.max_retry_times) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 73757162f..421c58e68 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. from __future__ import annotations import logging -from typing import TYPE_CHECKING, Dict, Optional, TypeVar, Union +from typing import TYPE_CHECKING, Optional, TypeVar, Union from twisted.internet.defer import Deferred, maybeDeferred @@ -45,7 +45,7 @@ class RobotsTxtMiddleware: "ROBOTSTXT_USER_AGENT", None ) self.crawler: Crawler = crawler - self._parsers: Dict[ + self._parsers: dict[ str, Union[RobotParser, Deferred[Optional[RobotParser]], None] ] = {} self._parserimpl: RobotParser = load_object( diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 0faae7b5a..ab5655393 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, List, Tuple, Union +from typing import TYPE_CHECKING, Union from twisted.web import http @@ -19,7 +19,7 @@ if TYPE_CHECKING: def get_header_size( - headers: Dict[str, Union[List[Union[str, bytes]], Tuple[Union[str, bytes], ...]]] + headers: dict[str, Union[list[Union[str, bytes]], tuple[Union[str, bytes], ...]]] ) -> int: size = 0 for key, value in headers.items(): diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 40ea48510..28118977d 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging from pathlib import Path -from typing import TYPE_CHECKING, Optional, Set +from typing import TYPE_CHECKING, Optional from scrapy.utils.job import job_dir from scrapy.utils.request import ( @@ -56,7 +56,7 @@ class RFPDupeFilter(BaseDupeFilter): self.fingerprinter: RequestFingerprinterProtocol = ( fingerprinter or RequestFingerprinter() ) - self.fingerprints: Set[str] = set() + self.fingerprints: set[str] = set() self.logdupes = True self.debug = debug self.logger = logging.getLogger(__name__) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index fb4998099..ee0033dfb 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -6,9 +6,10 @@ import csv import marshal import pickle # nosec import pprint +from collections.abc import Callable, Iterable, Mapping from io import BytesIO, TextIOWrapper from json import JSONEncoder -from typing import Any, Callable, Dict, Iterable, Mapping, Optional, Tuple, Union +from typing import Any, Optional, Union from xml.sax.saxutils import XMLGenerator # nosec from xml.sax.xmlreader import AttributesImpl # nosec @@ -32,10 +33,10 @@ __all__ = [ class BaseItemExporter: def __init__(self, *, dont_fail: bool = False, **kwargs: Any): - self._kwargs: Dict[str, Any] = kwargs + self._kwargs: dict[str, Any] = kwargs self._configure(kwargs, dont_fail=dont_fail) - def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None: + def _configure(self, options: dict[str, Any], dont_fail: bool = False) -> None: """Configure the exporter by popping options from the ``options`` dict. If dont_fail is set, it won't raise an exception on unexpected options (useful for using with keyword arguments in subclasses ``__init__`` methods) @@ -66,7 +67,7 @@ class BaseItemExporter: def _get_serialized_fields( self, item: Any, default_value: Any = None, include_empty: Optional[bool] = None - ) -> Iterable[Tuple[str, Any]]: + ) -> Iterable[tuple[str, Any]]: """Return the fields to export as an iterable of tuples (name, serialized_value) """ @@ -339,7 +340,7 @@ class PythonItemExporter(BaseItemExporter): .. _msgpack: https://pypi.org/project/msgpack/ """ - def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None: + def _configure(self, options: dict[str, Any], dont_fail: bool = False) -> None: super()._configure(options, dont_fail) if not self.encoding: self.encoding = "utf-8" @@ -363,10 +364,10 @@ class PythonItemExporter(BaseItemExporter): return to_unicode(value, encoding=self.encoding) return value - def _serialize_item(self, item: Any) -> Iterable[Tuple[Union[str, bytes], Any]]: + def _serialize_item(self, item: Any) -> Iterable[tuple[Union[str, bytes], Any]]: for key, value in ItemAdapter(item).items(): yield key, self._serialize_value(value) - def export_item(self, item: Any) -> Dict[Union[str, bytes], Any]: # type: ignore[override] - result: Dict[Union[str, bytes], Any] = dict(self._get_serialized_fields(item)) + def export_item(self, item: Any) -> dict[Union[str, bytes], Any]: # type: ignore[override] + result: dict[Union[str, bytes], Any] = dict(self._get_serialized_fields(item)) return result diff --git a/scrapy/extension.py b/scrapy/extension.py index 8c81ab356..9f978fa32 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -6,7 +6,7 @@ See documentation in docs/topics/extensions.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, List +from typing import TYPE_CHECKING, Any from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list @@ -19,5 +19,5 @@ class ExtensionManager(MiddlewareManager): component_name = "extension" @classmethod - def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]: return build_component_list(settings.getwithbase("EXTENSIONS")) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index cef5527b7..dff8bc97e 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -8,7 +8,7 @@ from __future__ import annotations import logging from collections import defaultdict -from typing import TYPE_CHECKING, Any, DefaultDict, Dict +from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured @@ -30,7 +30,7 @@ class CloseSpider: def __init__(self, crawler: Crawler): self.crawler: Crawler = crawler - self.close_on: Dict[str, Any] = { + self.close_on: dict[str, Any] = { "timeout": crawler.settings.getfloat("CLOSESPIDER_TIMEOUT"), "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"), "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"), @@ -44,7 +44,7 @@ class CloseSpider: if not any(self.close_on.values()): raise NotConfigured - self.counter: DefaultDict[str, int] = defaultdict(int) + self.counter: defaultdict[str, int] = defaultdict(int) if self.close_on.get("errorcount"): crawler.signals.connect(self.error_count, signal=signals.spider_error) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 0d7f5bfd4..b1001dabb 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -10,25 +10,11 @@ import logging import re import sys import warnings +from collections.abc import Callable from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - List, - Optional, - Protocol, - Tuple, - Type, - TypeVar, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Any, Optional, Protocol, TypeVar, Union, cast from urllib.parse import unquote, urlparse from twisted.internet.defer import Deferred, DeferredList, maybeDeferred @@ -50,6 +36,8 @@ from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values if TYPE_CHECKING: + from collections.abc import Iterable + from _typeshed import OpenBinaryMode from twisted.python.failure import Failure @@ -70,7 +58,7 @@ except ImportError: logger = logging.getLogger(__name__) -UriParamsCallableT = Callable[[Dict[str, Any], Spider], Optional[Dict[str, Any]]] +UriParamsCallableT = Callable[[dict[str, Any], Spider], Optional[dict[str, Any]]] _StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol") @@ -79,7 +67,7 @@ def build_storage( builder: Callable[..., _StorageT], uri: str, *args: Any, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, preargs: Iterable[Any] = (), **kwargs: Any, ) -> _StorageT: @@ -96,10 +84,10 @@ class ItemFilter: :type feed_options: dict """ - feed_options: Optional[Dict[str, Any]] - item_classes: Tuple[type, ...] + feed_options: Optional[dict[str, Any]] + item_classes: tuple[type, ...] - def __init__(self, feed_options: Optional[Dict[str, Any]]) -> None: + def __init__(self, feed_options: Optional[dict[str, Any]]) -> None: self.feed_options = feed_options if feed_options is not None: self.item_classes = tuple( @@ -141,7 +129,7 @@ class IFeedStorage(Interface): class FeedStorageProtocol(Protocol): """Reimplementation of ``IFeedStorage`` that can be used in type hints.""" - def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None): + def __init__(self, uri: str, *, feed_options: Optional[dict[str, Any]] = None): """Initialize the storage with the parameters given in the URI and the feed-specific options (see :setting:`FEEDS`)""" @@ -176,7 +164,7 @@ class StdoutFeedStorage: uri: str, _stdout: Optional[IO[bytes]] = None, *, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, ): if not _stdout: _stdout = sys.stdout.buffer @@ -198,7 +186,7 @@ class StdoutFeedStorage: @implementer(IFeedStorage) class FileFeedStorage: - def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None): + def __init__(self, uri: str, *, feed_options: Optional[dict[str, Any]] = None): self.path: str = file_uri_to_path(uri) feed_options = feed_options or {} self.write_mode: OpenBinaryMode = ( @@ -225,7 +213,7 @@ class S3FeedStorage(BlockingFeedStorage): acl: Optional[str] = None, endpoint_url: Optional[str] = None, *, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, session_token: Optional[str] = None, region_name: Optional[str] = None, ): @@ -291,7 +279,7 @@ class S3FeedStorage(BlockingFeedStorage): crawler: Crawler, uri: str, *, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, ) -> Self: return build_storage( cls, @@ -307,7 +295,7 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) - kwargs: Dict[str, Any] + kwargs: dict[str, Any] if IS_BOTO3_AVAILABLE: kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} self.s3_client.upload_fileobj( @@ -354,7 +342,7 @@ class FTPFeedStorage(BlockingFeedStorage): uri: str, use_active_mode: bool = False, *, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, ): u = urlparse(uri) if not u.hostname: @@ -373,7 +361,7 @@ class FTPFeedStorage(BlockingFeedStorage): crawler: Crawler, uri: str, *, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, ) -> Self: return build_storage( cls, @@ -405,9 +393,9 @@ class FeedSlot: batch_id: int, uri_template: str, filter: ItemFilter, - feed_options: Dict[str, Any], + feed_options: dict[str, Any], spider: Spider, - exporters: Dict[str, Type[BaseItemExporter]], + exporters: dict[str, type[BaseItemExporter]], settings: BaseSettings, crawler: Crawler, ): @@ -422,9 +410,9 @@ class FeedSlot: self.uri: str = uri self.filter: ItemFilter = filter # exporter params - self.feed_options: Dict[str, Any] = feed_options + self.feed_options: dict[str, Any] = feed_options self.spider: Spider = spider - self.exporters: Dict[str, Type[BaseItemExporter]] = exporters + self.exporters: dict[str, type[BaseItemExporter]] = exporters self.settings: BaseSettings = settings self.crawler: Crawler = crawler # flags @@ -460,7 +448,7 @@ class FeedSlot: self._exporting = True def _get_instance( - self, objcls: Type[BaseItemExporter], *args: Any, **kwargs: Any + self, objcls: type[BaseItemExporter], *args: Any, **kwargs: Any ) -> BaseItemExporter: return build_from_crawler(objcls, self.crawler, *args, **kwargs) @@ -483,7 +471,7 @@ _FeedSlot = create_deprecated_class( class FeedExporter: - _pending_deferreds: List[Deferred[None]] = [] + _pending_deferreds: list[Deferred[None]] = [] @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -497,8 +485,8 @@ class FeedExporter: self.crawler: Crawler = crawler self.settings: Settings = crawler.settings self.feeds = {} - self.slots: List[FeedSlot] = [] - self.filters: Dict[str, ItemFilter] = {} + self.slots: list[FeedSlot] = [] + self.filters: dict[str, ItemFilter] = {} if not self.settings["FEEDS"] and not self.settings["FEED_URI"]: raise NotConfigured @@ -530,10 +518,10 @@ class FeedExporter: ) self.filters[uri] = self._load_filter(feed_options) - self.storages: Dict[str, Type[FeedStorageProtocol]] = self._load_components( + self.storages: dict[str, type[FeedStorageProtocol]] = self._load_components( "FEED_STORAGES" ) - self.exporters: Dict[str, Type[BaseItemExporter]] = self._load_components( + self.exporters: dict[str, type[BaseItemExporter]] = self._load_components( "FEED_EXPORTERS" ) for uri, feed_options in self.feeds.items(): @@ -631,7 +619,7 @@ class FeedExporter: self, batch_id: int, uri: str, - feed_options: Dict[str, Any], + feed_options: dict[str, Any], spider: Spider, uri_template: str, ) -> FeedSlot: @@ -696,9 +684,9 @@ class FeedExporter: slots.append(slot) self.slots = slots - def _load_components(self, setting_prefix: str) -> Dict[str, Any]: + def _load_components(self, setting_prefix: str) -> dict[str, Any]: conf = without_none_values( - cast(Dict[str, str], self.settings.getwithbase(setting_prefix)) + cast(dict[str, str], self.settings.getwithbase(setting_prefix)) ) d = {} for k, v in conf.items(): @@ -732,7 +720,7 @@ class FeedExporter: return False return True - def _storage_supported(self, uri: str, feed_options: Dict[str, Any]) -> bool: + def _storage_supported(self, uri: str, feed_options: dict[str, Any]) -> bool: scheme = urlparse(uri).scheme if scheme in self.storages or PureWindowsPath(uri).drive: try: @@ -748,7 +736,7 @@ class FeedExporter: return False def _get_storage( - self, uri: str, feed_options: Dict[str, Any] + self, uri: str, feed_options: dict[str, Any] ) -> FeedStorageProtocol: """Fork of create_instance specific to feed storage classes @@ -759,7 +747,7 @@ class FeedExporter: crawler = getattr(self, "crawler", None) def build_instance( - builder: Type[FeedStorageProtocol], *preargs: Any + builder: type[FeedStorageProtocol], *preargs: Any ) -> FeedStorageProtocol: return build_storage( builder, uri, feed_options=feed_options, preargs=preargs @@ -784,7 +772,7 @@ class FeedExporter: spider: Spider, uri_params_function: Union[str, UriParamsCallableT, None], slot: Optional[FeedSlot] = None, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: params = {} for k in dir(spider): params[k] = getattr(spider, k) @@ -800,9 +788,9 @@ class FeedExporter: new_params = uripar_function(params, spider) return new_params if new_params is not None else params - def _load_filter(self, feed_options: Dict[str, Any]) -> ItemFilter: + def _load_filter(self, feed_options: dict[str, Any]) -> ItemFilter: # load the item filter if declared else load the default filter class - item_filter_class: Type[ItemFilter] = load_object( + item_filter_class: type[ItemFilter] = load_object( feed_options.get("item_filter", ItemFilter) ) return item_filter_class(feed_options) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 448d5f1ab..a72f9db51 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -9,7 +9,7 @@ from importlib import import_module from pathlib import Path from time import time from types import ModuleType -from typing import IO, TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union, cast +from typing import IO, TYPE_CHECKING, Any, Optional, Union, cast from weakref import WeakKeyDictionary from w3lib.http import headers_dict_to_raw, headers_raw_to_dict @@ -22,6 +22,8 @@ from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.request import RequestFingerprinter if TYPE_CHECKING: + from collections.abc import Callable + # typing.Concatenate requires Python 3.10 from typing_extensions import Concatenate @@ -35,8 +37,8 @@ logger = logging.getLogger(__name__) class DummyPolicy: def __init__(self, settings: BaseSettings): - self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") - self.ignore_http_codes: List[int] = [ + self.ignore_schemes: list[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") + self.ignore_http_codes: list[int] = [ int(x) for x in settings.getlist("HTTPCACHE_IGNORE_HTTP_CODES") ] @@ -62,18 +64,18 @@ class RFC2616Policy: def __init__(self, settings: BaseSettings): self.always_store: bool = settings.getbool("HTTPCACHE_ALWAYS_STORE") - self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") + self.ignore_schemes: list[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") self._cc_parsed: WeakKeyDictionary[ - Union[Request, Response], Dict[bytes, Optional[bytes]] + Union[Request, Response], dict[bytes, Optional[bytes]] ] = WeakKeyDictionary() - self.ignore_response_cache_controls: List[bytes] = [ + self.ignore_response_cache_controls: list[bytes] = [ to_bytes(cc) for cc in settings.getlist("HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS") ] def _parse_cachecontrol( self, r: Union[Request, Response] - ) -> Dict[bytes, Optional[bytes]]: + ) -> dict[bytes, Optional[bytes]]: if r not in self._cc_parsed: cch = r.headers.get(b"Cache-Control", b"") assert cch is not None @@ -189,7 +191,7 @@ class RFC2616Policy: if b"ETag" in cachedresponse.headers: request.headers[b"If-None-Match"] = cachedresponse.headers[b"ETag"] - def _get_max_age(self, cc: Dict[bytes, Optional[bytes]]) -> Optional[int]: + def _get_max_age(self, cc: dict[bytes, Optional[bytes]]) -> Optional[int]: try: return max(0, int(cc[b"max-age"])) # type: ignore[arg-type] except (KeyError, ValueError): @@ -298,7 +300,7 @@ class DbmCacheStorage: self.db[f"{key}_data"] = pickle.dumps(data, protocol=4) self.db[f"{key}_time"] = str(time()) - def _read_data(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]: + def _read_data(self, spider: Spider, request: Request) -> Optional[dict[str, Any]]: key = self._fingerprinter.fingerprint(request).hex() db = self.db tkey = f"{key}_time" @@ -309,7 +311,7 @@ class DbmCacheStorage: if 0 < self.expiration_secs < time() - float(ts): return None # expired - return cast(Dict[str, Any], pickle.loads(db[f"{key}_data"])) # nosec + return cast(dict[str, Any], pickle.loads(db[f"{key}_data"])) # nosec class FilesystemCacheStorage: @@ -385,7 +387,7 @@ class FilesystemCacheStorage: key = self._fingerprinter.fingerprint(request).hex() return str(Path(self.cachedir, spider.name, key[0:2], key)) - def _read_meta(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]: + def _read_meta(self, spider: Spider, request: Request) -> Optional[dict[str, Any]]: rpath = Path(self._get_request_path(spider, request)) metapath = rpath / "pickled_meta" if not metapath.exists(): @@ -394,10 +396,10 @@ class FilesystemCacheStorage: if 0 < self.expiration_secs < time() - mtime: return None # expired with self._open(metapath, "rb") as f: - return cast(Dict[str, Any], pickle.load(f)) # nosec + return cast(dict[str, Any], pickle.load(f)) # nosec -def parse_cachecontrol(header: bytes) -> Dict[bytes, Optional[bytes]]: +def parse_cachecontrol(header: bytes) -> dict[bytes, Optional[bytes]]: """Parse Cache-Control header https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index c4f43482d..01484481b 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Optional, Tuple, Union +from typing import TYPE_CHECKING, Optional, Union from twisted.internet import task @@ -81,7 +81,7 @@ class LogStats: def calculate_final_stats( self, spider: Spider - ) -> Union[Tuple[None, None], Tuple[float, float]]: + ) -> Union[tuple[None, None], tuple[float, float]]: start_time = self.stats.get_value("start_time") finished_time = self.stats.get_value("finished_time") diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 25f63ecc6..73d864d5d 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -11,7 +11,7 @@ import socket import sys from importlib import import_module from pprint import pformat -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from twisted.internet import task @@ -42,7 +42,7 @@ class MemoryUsage: self.crawler: Crawler = crawler self.warned: bool = False - self.notify_mails: List[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") + self.notify_mails: list[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 self.check_interval: float = crawler.settings.getfloat( @@ -66,7 +66,7 @@ class MemoryUsage: def engine_started(self) -> None: assert self.crawler.stats self.crawler.stats.set_value("memusage/startup", self.get_virtual_size()) - self.tasks: List[task.LoopingCall] = [] + self.tasks: list[task.LoopingCall] = [] tsk = task.LoopingCall(self.update) self.tasks.append(tsk) tsk.start(self.check_interval, now=True) @@ -141,7 +141,7 @@ class MemoryUsage: self.crawler.stats.set_value("memusage/warning_notified", 1) self.warned = True - def _send_report(self, rcpts: List[str], subject: str) -> None: + def _send_report(self, rcpts: list[str], subject: str) -> None: """send notification mail with some additional useful info""" assert self.crawler.engine assert self.crawler.stats diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 80c0a3b26..fba12bec7 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging from datetime import datetime, timezone from json import JSONEncoder -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Optional, Union from twisted.internet import task @@ -29,8 +29,8 @@ class PeriodicLog: self, stats: StatsCollector, interval: float = 60.0, - ext_stats: Dict[str, Any] = {}, - ext_delta: Dict[str, Any] = {}, + ext_stats: dict[str, Any] = {}, + ext_delta: dict[str, Any] = {}, ext_timing_enabled: bool = False, ): self.stats: StatsCollector = stats @@ -39,11 +39,11 @@ class PeriodicLog: self.task: Optional[task.LoopingCall] = None self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) self.ext_stats_enabled: bool = bool(ext_stats) - self.ext_stats_include: List[str] = ext_stats.get("include", []) - self.ext_stats_exclude: List[str] = ext_stats.get("exclude", []) + self.ext_stats_include: list[str] = ext_stats.get("include", []) + self.ext_stats_exclude: list[str] = ext_stats.get("exclude", []) self.ext_delta_enabled: bool = bool(ext_delta) - self.ext_delta_include: List[str] = ext_delta.get("include", []) - self.ext_delta_exclude: List[str] = ext_delta.get("exclude", []) + self.ext_delta_include: list[str] = ext_delta.get("include", []) + self.ext_delta_exclude: list[str] = ext_delta.get("exclude", []) self.ext_timing_enabled: bool = ext_timing_enabled @classmethod @@ -52,7 +52,7 @@ class PeriodicLog: if not interval: raise NotConfigured try: - ext_stats: Optional[Dict[str, Any]] = crawler.settings.getdict( + ext_stats: Optional[dict[str, Any]] = crawler.settings.getdict( "PERIODIC_LOG_STATS" ) except (TypeError, ValueError): @@ -62,7 +62,7 @@ class PeriodicLog: else None ) try: - ext_delta: Optional[Dict[str, Any]] = crawler.settings.getdict( + ext_delta: Optional[dict[str, Any]] = crawler.settings.getdict( "PERIODIC_LOG_DELTA" ) except (TypeError, ValueError): @@ -93,14 +93,14 @@ class PeriodicLog: def spider_opened(self, spider: Spider) -> None: self.time_prev: datetime = datetime.now(tz=timezone.utc) - self.delta_prev: Dict[str, Union[int, float]] = {} - self.stats_prev: Dict[str, Union[int, float]] = {} + self.delta_prev: dict[str, Union[int, float]] = {} + self.stats_prev: dict[str, Union[int, float]] = {} self.task = task.LoopingCall(self.log) self.task.start(self.interval) def log(self) -> None: - data: Dict[str, Any] = {} + data: dict[str, Any] = {} if self.ext_timing_enabled: data.update(self.log_timing()) if self.ext_delta_enabled: @@ -109,8 +109,8 @@ class PeriodicLog: data.update(self.log_crawler_stats()) logger.info(self.encoder.encode(data)) - def log_delta(self) -> Dict[str, Any]: - num_stats: Dict[str, Union[int, float]] = { + def log_delta(self) -> dict[str, Any]: + num_stats: dict[str, Union[int, float]] = { k: v for k, v in self.stats._stats.items() if isinstance(v, (int, float)) @@ -120,7 +120,7 @@ class PeriodicLog: self.delta_prev = num_stats return {"delta": delta} - def log_timing(self) -> Dict[str, Any]: + def log_timing(self) -> dict[str, Any]: now = datetime.now(tz=timezone.utc) time = { "log_interval": self.interval, @@ -132,7 +132,7 @@ class PeriodicLog: self.time_prev = now return {"time": time} - def log_crawler_stats(self) -> Dict[str, Any]: + def log_crawler_stats(self) -> dict[str, Any]: stats = { k: v for k, v in self.stats._stats.items() @@ -141,7 +141,7 @@ class PeriodicLog: return {"stats": stats} def param_allowed( - self, stat_name: str, include: List[str], exclude: List[str] + self, stat_name: str, include: list[str], exclude: list[str] ) -> bool: if not include and not exclude: return True diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index ac12ad829..16067f82b 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -6,7 +6,7 @@ from bz2 import BZ2File from gzip import GzipFile from io import IOBase from lzma import LZMAFile -from typing import IO, Any, BinaryIO, Dict, List, cast +from typing import IO, Any, BinaryIO, cast from scrapy.utils.misc import load_object @@ -24,7 +24,7 @@ class GzipPlugin: See :py:class:`gzip.GzipFile` for more info about parameters. """ - def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + def __init__(self, file: BinaryIO, feed_options: dict[str, Any]) -> None: self.file = file self.feed_options = feed_options compress_level = self.feed_options.get("gzip_compresslevel", 9) @@ -56,7 +56,7 @@ class Bz2Plugin: See :py:class:`bz2.BZ2File` for more info about parameters. """ - def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + def __init__(self, file: BinaryIO, feed_options: dict[str, Any]) -> None: self.file = file self.feed_options = feed_options compress_level = self.feed_options.get("bz2_compresslevel", 9) @@ -88,7 +88,7 @@ class LZMAPlugin: See :py:class:`lzma.LZMAFile` for more info about parameters. """ - def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + def __init__(self, file: BinaryIO, feed_options: dict[str, Any]) -> None: self.file = file self.feed_options = feed_options @@ -126,7 +126,7 @@ class PostProcessingManager(IOBase): """ def __init__( - self, plugins: List[Any], file: IO[bytes], feed_options: Dict[str, Any] + self, plugins: list[Any], file: IO[bytes], feed_options: dict[str, Any] ) -> None: self.plugins = self._load_plugins(plugins) self.file = file @@ -156,7 +156,7 @@ class PostProcessingManager(IOBase): def writable(self) -> bool: return True - def _load_plugins(self, plugins: List[Any]) -> List[Any]: + def _load_plugins(self, plugins: list[Any]) -> list[Any]: plugins = [load_object(plugin) for plugin in plugins] return plugins diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index cad607514..c8fefe792 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -6,7 +6,7 @@ Use STATSMAILER_RCPTS setting to enable and give the recipient mail address from __future__ import annotations -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, Optional from scrapy import Spider, signals from scrapy.exceptions import NotConfigured @@ -23,14 +23,14 @@ if TYPE_CHECKING: class StatsMailer: - def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender): + def __init__(self, stats: StatsCollector, recipients: list[str], mail: MailSender): self.stats: StatsCollector = stats - self.recipients: List[str] = recipients + self.recipients: list[str] = recipients self.mail: MailSender = mail @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - recipients: List[str] = crawler.settings.getlist("STATSMAILER_RCPTS") + recipients: list[str] = crawler.settings.getlist("STATSMAILER_RCPTS") if not recipients: raise NotConfigured mail: MailSender = MailSender.from_settings(crawler.settings) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index c64a0b417..07dc5880b 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -10,7 +10,7 @@ import binascii import logging import os import pprint -from typing import TYPE_CHECKING, Any, Dict, List +from typing import TYPE_CHECKING, Any from twisted.internet import protocol from twisted.internet.tcp import Port @@ -45,7 +45,7 @@ class TelnetConsole(protocol.ServerFactory): self.crawler: Crawler = crawler self.noisy: bool = False - self.portrange: List[int] = [ + self.portrange: list[int] = [ int(x) for x in crawler.settings.getlist("TELNETCONSOLE_PORT") ] self.host: str = crawler.settings["TELNETCONSOLE_HOST"] @@ -98,10 +98,10 @@ class TelnetConsole(protocol.ServerFactory): return telnet.TelnetTransport(telnet.AuthenticatingTelnetProtocol, Portal()) - def _get_telnet_vars(self) -> Dict[str, Any]: + def _get_telnet_vars(self) -> dict[str, Any]: # Note: if you add entries here also update topics/telnetconsole.rst assert self.crawler.engine - telnet_vars: Dict[str, Any] = { + telnet_vars: dict[str, Any] = { "engine": self.crawler.engine, "spider": self.crawler.engine.spider, "slot": self.crawler.engine.slot, diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 6ce9ce63a..6b5fd181d 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Optional from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured @@ -90,7 +90,7 @@ class AutoThrottle: def _get_slot( self, request: Request, spider: Spider - ) -> Tuple[Optional[str], Optional[Slot]]: + ) -> tuple[Optional[str], Optional[Slot]]: key: Optional[str] = request.meta.get("download_slot") if key is None: return None, None diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index cc88a9420..b5388a918 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -5,22 +5,14 @@ import time from http.cookiejar import Cookie from http.cookiejar import CookieJar as _CookieJar from http.cookiejar import CookiePolicy, DefaultCookiePolicy -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterator, - List, - Optional, - Sequence, - Tuple, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, cast from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from collections.abc import Iterator, Sequence + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -83,7 +75,7 @@ class CookieJar: self.jar.clear_expired_cookies() @property - def _cookies(self) -> Dict[str, Dict[str, Dict[str, Cookie]]]: + def _cookies(self) -> dict[str, dict[str, dict[str, Cookie]]]: return self.jar._cookies # type: ignore[attr-defined,no-any-return] def clear_session_cookies(self) -> None: @@ -118,7 +110,7 @@ class CookieJar: self.jar.set_cookie_if_ok(cookie, WrappedRequest(request)) # type: ignore[arg-type] -def potential_domain_matches(domain: str) -> List[str]: +def potential_domain_matches(domain: str) -> list[str]: """Potential domain matches for a cookie >>> potential_domain_matches('www.example.com') @@ -200,7 +192,7 @@ class WrappedRequest: value = self.request.headers.get(name, default) return to_unicode(value, errors="replace") if value is not None else None - def header_items(self) -> List[Tuple[str, List[str]]]: + def header_items(self) -> list[tuple[str, list[str]]]: return [ ( to_unicode(k, errors="replace"), @@ -220,7 +212,7 @@ class WrappedResponse: def info(self) -> Self: return self - def get_all(self, name: str, default: Any = None) -> List[str]: + def get_all(self, name: str, default: Any = None) -> list[str]: return [ to_unicode(v, errors="replace") for v in self.response.headers.getlist(name) ] diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 85b9229d3..1dcbcb966 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,18 +1,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Dict, - Iterable, - List, - Optional, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, AnyStr, Optional, Union, cast from w3lib.http import headers_dict_to_raw @@ -20,6 +9,8 @@ from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -34,17 +25,17 @@ class Headers(CaselessDict): def __init__( self, - seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, encoding: str = "utf-8", ): self.encoding: str = encoding super().__init__(seq) def update( # type: ignore[override] - self, seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]]] + self, seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]]] ) -> None: seq = seq.items() if isinstance(seq, Mapping) else seq - iseq: Dict[bytes, List[bytes]] = {} + iseq: dict[bytes, list[bytes]] = {} for k, v in seq: iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v)) super().update(iseq) @@ -53,7 +44,7 @@ class Headers(CaselessDict): """Normalize key to bytes""" return self._tobytes(key.title()) - def normvalue(self, value: Union[_RawValueT, Iterable[_RawValueT]]) -> List[bytes]: + def normvalue(self, value: Union[_RawValueT, Iterable[_RawValueT]]) -> list[bytes]: """Normalize values to bytes""" _value: Iterable[_RawValueT] if value is None: @@ -78,19 +69,19 @@ class Headers(CaselessDict): def __getitem__(self, key: AnyStr) -> Optional[bytes]: try: - return cast(List[bytes], super().__getitem__(key))[-1] + return cast(list[bytes], super().__getitem__(key))[-1] except IndexError: return None def get(self, key: AnyStr, def_val: Any = None) -> Optional[bytes]: try: - return cast(List[bytes], super().get(key, def_val))[-1] + return cast(list[bytes], super().get(key, def_val))[-1] except IndexError: return None - def getlist(self, key: AnyStr, def_val: Any = None) -> List[bytes]: + def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]: try: - return cast(List[bytes], super().__getitem__(key)) + return cast(list[bytes], super().__getitem__(key)) except KeyError: if def_val is not None: return self.normvalue(def_val) @@ -109,10 +100,10 @@ class Headers(CaselessDict): lst.extend(self.normvalue(value)) self[key] = lst - def items(self) -> Iterable[Tuple[bytes, List[bytes]]]: # type: ignore[override] + def items(self) -> Iterable[tuple[bytes, list[bytes]]]: # type: ignore[override] return ((k, self.getlist(k)) for k in self.keys()) - def values(self) -> List[Optional[bytes]]: # type: ignore[override] + def values(self) -> list[Optional[bytes]]: # type: ignore[override] return [ self[k] for k in self.keys() # pylint: disable=consider-using-dict-items ] diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 9381a6cb3..aac8d3e50 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -12,14 +12,8 @@ from typing import ( TYPE_CHECKING, Any, AnyStr, - Dict, - Iterable, - List, - Mapping, NoReturn, Optional, - Tuple, - Type, TypedDict, TypeVar, Union, @@ -36,7 +30,7 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Iterable, Mapping from twisted.python.failure import Failure @@ -57,7 +51,7 @@ class VerboseCookie(TypedDict): secure: NotRequired[bool] -CookiesT = Union[Dict[str, str], List[VerboseCookie]] +CookiesT = Union[dict[str, str], list[VerboseCookie]] RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") @@ -92,7 +86,7 @@ class Request(object_ref): executed by the Downloader, thus generating a :class:`Response`. """ - attributes: Tuple[str, ...] = ( + attributes: tuple[str, ...] = ( "url", "callback", "method", @@ -120,16 +114,16 @@ class Request(object_ref): url: str, callback: Optional[CallbackT] = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[CookiesT] = None, - meta: Optional[Dict[str, Any]] = None, + meta: Optional[dict[str, Any]] = None, encoding: str = "utf-8", priority: int = 0, dont_filter: bool = False, errback: Optional[Callable[[Failure], Any]] = None, - flags: Optional[List[str]] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, + flags: Optional[list[str]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, ) -> None: self._encoding: str = encoding # this one has to be set first self.method: str = str(method).upper() @@ -152,20 +146,20 @@ class Request(object_ref): self.headers: Headers = Headers(headers or {}, encoding=encoding) self.dont_filter: bool = dont_filter - self._meta: Optional[Dict[str, Any]] = dict(meta) if meta else None - self._cb_kwargs: Optional[Dict[str, Any]] = ( + self._meta: Optional[dict[str, Any]] = dict(meta) if meta else None + self._cb_kwargs: Optional[dict[str, Any]] = ( dict(cb_kwargs) if cb_kwargs else None ) - self.flags: List[str] = [] if flags is None else list(flags) + self.flags: list[str] = [] if flags is None else list(flags) @property - def cb_kwargs(self) -> Dict[str, Any]: + def cb_kwargs(self) -> dict[str, Any]: if self._cb_kwargs is None: self._cb_kwargs = {} return self._cb_kwargs @property - def meta(self) -> Dict[str, Any]: + def meta(self) -> dict[str, Any]: if self._meta is None: self._meta = {} return self._meta @@ -207,14 +201,14 @@ class Request(object_ref): @overload def replace( - self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any + self, *args: Any, cls: type[RequestTypeVar], **kwargs: Any ) -> RequestTypeVar: ... @overload def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( - self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any + self, *args: Any, cls: Optional[type[Request]] = None, **kwargs: Any ) -> Request: """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: @@ -261,7 +255,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]: + def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index a8c242e8b..d9c913672 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -7,17 +7,8 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterable, - List, - Optional, - Tuple, - Union, - cast, -) +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, Optional, Union, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit from lxml.html import FormElement # nosec @@ -31,6 +22,7 @@ from scrapy.http.request import Request from scrapy.utils.python import is_listlike, to_bytes if TYPE_CHECKING: + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -38,8 +30,8 @@ if TYPE_CHECKING: FormdataVType = Union[str, Iterable[str]] -FormdataKVType = Tuple[str, FormdataVType] -FormdataType = Optional[Union[Dict[str, FormdataVType], List[FormdataKVType]]] +FormdataKVType = tuple[str, FormdataVType] +FormdataType = Optional[Union[dict[str, FormdataVType], list[FormdataKVType]]] class FormRequest(Request): @@ -74,7 +66,7 @@ class FormRequest(Request): formid: Optional[str] = None, formnumber: int = 0, formdata: FormdataType = None, - clickdata: Optional[Dict[str, Union[str, int]]] = None, + clickdata: Optional[dict[str, Union[str, int]]] = None, dont_click: bool = False, formxpath: Optional[str] = None, formcss: Optional[str] = None, @@ -168,8 +160,8 @@ def _get_inputs( form: FormElement, formdata: FormdataType, dont_click: bool, - clickdata: Optional[Dict[str, Union[str, int]]], -) -> List[FormdataKVType]: + clickdata: Optional[dict[str, Union[str, int]]], +) -> list[FormdataKVType]: """Return a list of key-value pairs for the inputs found in the given form.""" try: formdata_keys = dict(formdata or ()).keys() @@ -187,7 +179,7 @@ def _get_inputs( ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', namespaces={"re": "http://exslt.org/regular-expressions"}, ) - values: List[FormdataKVType] = [ + values: list[FormdataKVType] = [ (k, "" if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata_keys @@ -205,7 +197,7 @@ def _get_inputs( def _value( ele: Union[InputElement, SelectElement, TextareaElement] -) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: +) -> tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: n = ele.name v = ele.value if ele.tag == "select": @@ -215,7 +207,7 @@ def _value( def _select_value( ele: SelectElement, n: Optional[str], v: Union[None, str, MultipleSelectOptions] -) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: +) -> tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected @@ -226,8 +218,8 @@ def _select_value( def _get_clickable( - clickdata: Optional[Dict[str, Union[str, int]]], form: FormElement -) -> Optional[Tuple[str, str]]: + clickdata: Optional[dict[str, Union[str, int]]], form: FormElement +) -> Optional[tuple[str, str]]: """ Returns the clickable element specified in clickdata, if the latter is given. If not, it returns the first diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 057a4f897..48862534e 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -10,7 +10,7 @@ from __future__ import annotations import copy import json import warnings -from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type, overload +from typing import TYPE_CHECKING, Any, Optional, overload from scrapy.http.request import Request, RequestTypeVar @@ -20,14 +20,14 @@ if TYPE_CHECKING: class JsonRequest(Request): - attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) + attributes: tuple[str, ...] = Request.attributes + ("dumps_kwargs",) def __init__( - self, *args: Any, dumps_kwargs: Optional[Dict[str, Any]] = None, **kwargs: Any + self, *args: Any, dumps_kwargs: Optional[dict[str, Any]] = None, **kwargs: Any ) -> None: dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} dumps_kwargs.setdefault("sort_keys", True) - self._dumps_kwargs: Dict[str, Any] = dumps_kwargs + self._dumps_kwargs: dict[str, Any] = dumps_kwargs body_passed = kwargs.get("body", None) is not None data: Any = kwargs.pop("data", None) @@ -47,19 +47,19 @@ class JsonRequest(Request): ) @property - def dumps_kwargs(self) -> Dict[str, Any]: + def dumps_kwargs(self) -> dict[str, Any]: return self._dumps_kwargs @overload def replace( - self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any + self, *args: Any, cls: type[RequestTypeVar], **kwargs: Any ) -> RequestTypeVar: ... @overload def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( - self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any + self, *args: Any, cls: Optional[type[Request]] = None, **kwargs: Any ) -> Request: body_passed = kwargs.get("body", None) is not None data: Any = kwargs.pop("data", None) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 92e4852b6..c69945e2d 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -7,22 +7,7 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Callable, - Dict, - Iterable, - List, - Mapping, - Optional, - Tuple, - Type, - TypeVar, - Union, - overload, -) +from typing import TYPE_CHECKING, Any, AnyStr, Optional, TypeVar, Union, overload from urllib.parse import urljoin from scrapy.exceptions import NotSupported @@ -32,6 +17,7 @@ from scrapy.link import Link from scrapy.utils.trackref import object_ref if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Mapping from ipaddress import IPv4Address, IPv6Address from twisted.internet.ssl import Certificate @@ -52,7 +38,7 @@ class Response(object_ref): downloaded (by the Downloader) and fed to the Spiders for processing. """ - attributes: Tuple[str, ...] = ( + attributes: tuple[str, ...] = ( "url", "status", "headers", @@ -74,9 +60,9 @@ class Response(object_ref): self, url: str, status: int = 200, - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: bytes = b"", - flags: Optional[List[str]] = None, + flags: Optional[list[str]] = None, request: Optional[Request] = None, certificate: Optional[Certificate] = None, ip_address: Union[IPv4Address, IPv6Address, None] = None, @@ -87,13 +73,13 @@ class Response(object_ref): self._set_body(body) self._set_url(url) self.request: Optional[Request] = request - self.flags: List[str] = [] if flags is None else list(flags) + self.flags: list[str] = [] if flags is None else list(flags) self.certificate: Optional[Certificate] = certificate self.ip_address: Union[IPv4Address, IPv6Address, None] = ip_address self.protocol: Optional[str] = protocol @property - def cb_kwargs(self) -> Dict[str, Any]: + def cb_kwargs(self) -> dict[str, Any]: try: return self.request.cb_kwargs # type: ignore[union-attr] except AttributeError: @@ -103,7 +89,7 @@ class Response(object_ref): ) @property - def meta(self) -> Dict[str, Any]: + def meta(self) -> dict[str, Any]: try: return self.request.meta # type: ignore[union-attr] except AttributeError: @@ -149,14 +135,14 @@ class Response(object_ref): @overload def replace( - self, *args: Any, cls: Type[ResponseTypeVar], **kwargs: Any + self, *args: Any, cls: type[ResponseTypeVar], **kwargs: Any ) -> ResponseTypeVar: ... @overload def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( - self, *args: Any, cls: Optional[Type[Response]] = None, **kwargs: Any + self, *args: Any, cls: Optional[type[Response]] = None, **kwargs: Any ) -> Response: """Create a new Response with the same attributes except for those given new values""" for x in self.attributes: @@ -200,16 +186,16 @@ class Response(object_ref): url: Union[str, Link], callback: Optional[CallbackT] = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[CookiesT] = None, - meta: Optional[Dict[str, Any]] = None, + meta: Optional[dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, dont_filter: bool = False, errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, - flags: Optional[List[str]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, + flags: Optional[list[str]] = None, ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -253,16 +239,16 @@ class Response(object_ref): urls: Iterable[Union[str, Link]], callback: Optional[CallbackT] = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[CookiesT] = None, - meta: Optional[Dict[str, Any]] = None, + meta: Optional[dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, dont_filter: bool = False, errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, - flags: Optional[List[str]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, + flags: Optional[list[str]] = None, ) -> Iterable[Request]: """ .. versionadded:: 2.0 diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 588695002..680c1f602 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -8,21 +8,9 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations import json +from collections.abc import Iterable from contextlib import suppress -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Callable, - Dict, - Iterable, - List, - Mapping, - Optional, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, AnyStr, Optional, Union, cast from urllib.parse import urljoin import parsel @@ -41,6 +29,8 @@ from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url if TYPE_CHECKING: + from collections.abc import Callable, Mapping + from twisted.python.failure import Failure from scrapy.http.request import CallbackT, CookiesT, Request @@ -54,7 +44,7 @@ class TextResponse(Response): _DEFAULT_ENCODING = "ascii" _cached_decoded_json = _NONE - attributes: Tuple[str, ...] = Response.attributes + ("encoding",) + attributes: tuple[str, ...] = Response.attributes + ("encoding",) def __init__(self, *args: Any, **kwargs: Any): self._encoding: Optional[str] = kwargs.pop("encoding", None) @@ -183,16 +173,16 @@ class TextResponse(Response): url: Union[str, Link, parsel.Selector], callback: Optional[CallbackT] = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[CookiesT] = None, - meta: Optional[Dict[str, Any]] = None, + meta: Optional[dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0, dont_filter: bool = False, errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, - flags: Optional[List[str]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, + flags: Optional[list[str]] = None, ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -236,16 +226,16 @@ class TextResponse(Response): urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None, callback: Optional[CallbackT] = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[CookiesT] = None, - meta: Optional[Dict[str, Any]] = None, + meta: Optional[dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0, dont_filter: bool = False, errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, - flags: Optional[List[str]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, + flags: Optional[list[str]] = None, css: Optional[str] = None, xpath: Optional[str] = None, ) -> Iterable[Request]: diff --git a/scrapy/item.py b/scrapy/item.py index 3f93809e7..f77002d18 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -7,27 +7,21 @@ See documentation in docs/topics/item.rst from __future__ import annotations from abc import ABCMeta +from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterator, - KeysView, - MutableMapping, - NoReturn, - Tuple, -) +from typing import TYPE_CHECKING, Any, NoReturn from scrapy.utils.trackref import object_ref if TYPE_CHECKING: + from collections.abc import Iterator, KeysView + # typing.Self requires Python 3.11 from typing_extensions import Self -class Field(Dict[str, Any]): +class Field(dict[str, Any]): """Container of field metadata""" @@ -38,7 +32,7 @@ class ItemMeta(ABCMeta): """ def __new__( - mcs, class_name: str, bases: Tuple[type, ...], attrs: Dict[str, Any] + mcs, class_name: str, bases: tuple[type, ...], attrs: dict[str, Any] ) -> ItemMeta: classcell = attrs.pop("__classcell__", None) new_bases = tuple(base._class for base in bases if hasattr(base, "_class")) @@ -83,10 +77,10 @@ class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta): :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks. """ - fields: Dict[str, Field] + fields: dict[str, Field] def __init__(self, *args: Any, **kwargs: Any): - self._values: Dict[str, Any] = {} + self._values: dict[str, Any] = {} if args or kwargs: # avoid creating dict for most common case for k, v in dict(*args, **kwargs).items(): self[k] = v diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index d59005edd..1c7e96ae0 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -6,8 +6,13 @@ This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ -import re -from typing import Iterable, Pattern +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable + from re import Pattern # common file extensions that are not followed if they occur in links IGNORED_EXTENSIONS = [ diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index d27a132b3..73673b1c6 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -6,20 +6,10 @@ from __future__ import annotations import logging import operator +import re +from collections.abc import Callable, Iterable from functools import partial -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Iterable, - List, - Optional, - Pattern, - Set, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from urllib.parse import urljoin, urlparse from lxml import etree # nosec @@ -28,13 +18,14 @@ from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string from scrapy.link import Link -from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches, re +from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain if TYPE_CHECKING: + from lxml.html import HtmlElement # nosec from scrapy import Selector @@ -98,7 +89,7 @@ class LxmlParserLinkExtractor: def _iter_links( self, document: HtmlElement - ) -> Iterable[Tuple[HtmlElement, str, str]]: + ) -> Iterable[tuple[HtmlElement, str, str]]: for el in document.iter(etree.Element): if not self.scan_tag(_nons(el.tag)): continue @@ -114,8 +105,8 @@ class LxmlParserLinkExtractor: response_url: str, response_encoding: str, base_url: str, - ) -> List[Link]: - links: List[Link] = [] + ) -> list[Link]: + links: list[Link] = [] # hacky way to get the underlying lxml parsed document for el, attr, attr_val in self._iter_links(selector.root): # pseudo lxml.html.HtmlElement.make_links_absolute(base_url) @@ -145,26 +136,26 @@ class LxmlParserLinkExtractor: links.append(link) return self._deduplicate_if_needed(links) - def extract_links(self, response: TextResponse) -> List[Link]: + def extract_links(self, response: TextResponse) -> list[Link]: base_url = get_base_url(response) return self._extract_links( response.selector, response.url, response.encoding, base_url ) - def _process_links(self, links: List[Link]) -> List[Link]: + def _process_links(self, links: list[Link]) -> list[Link]: """Normalize and filter extracted links The subclass should override it if necessary """ return self._deduplicate_if_needed(links) - def _deduplicate_if_needed(self, links: List[Link]) -> List[Link]: + def _deduplicate_if_needed(self, links: list[Link]) -> list[Link]: if self.unique: return unique_list(links, key=self.link_key) return links -_RegexT = Union[str, Pattern[str]] +_RegexT = Union[str, re.Pattern[str]] _RegexOrSeveralT = Union[_RegexT, Iterable[_RegexT]] @@ -197,13 +188,13 @@ class LxmlLinkExtractor: strip=strip, canonicalized=not canonicalize, ) - self.allow_res: List[Pattern[str]] = self._compile_regexes(allow) - self.deny_res: List[Pattern[str]] = self._compile_regexes(deny) + self.allow_res: list[re.Pattern[str]] = self._compile_regexes(allow) + self.deny_res: list[re.Pattern[str]] = self._compile_regexes(deny) - self.allow_domains: Set[str] = set(arg_to_iter(allow_domains)) - self.deny_domains: Set[str] = set(arg_to_iter(deny_domains)) + self.allow_domains: set[str] = set(arg_to_iter(allow_domains)) + self.deny_domains: set[str] = set(arg_to_iter(deny_domains)) - self.restrict_xpaths: Tuple[str, ...] = tuple(arg_to_iter(restrict_xpaths)) + self.restrict_xpaths: tuple[str, ...] = tuple(arg_to_iter(restrict_xpaths)) self.restrict_xpaths += tuple( map(self._csstranslator.css_to_xpath, arg_to_iter(restrict_css)) ) @@ -211,11 +202,11 @@ class LxmlLinkExtractor: if deny_extensions is None: deny_extensions = IGNORED_EXTENSIONS self.canonicalize: bool = canonicalize - self.deny_extensions: Set[str] = {"." + e for e in arg_to_iter(deny_extensions)} - self.restrict_text: List[Pattern[str]] = self._compile_regexes(restrict_text) + self.deny_extensions: set[str] = {"." + e for e in arg_to_iter(deny_extensions)} + self.restrict_text: list[re.Pattern[str]] = self._compile_regexes(restrict_text) @staticmethod - def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> List[Pattern[str]]: + def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> list[re.Pattern[str]]: return [ x if isinstance(x, re.Pattern) else re.compile(x) for x in arg_to_iter(value) @@ -257,7 +248,7 @@ class LxmlLinkExtractor: denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] return any(allowed) and not any(denied) - def _process_links(self, links: List[Link]) -> List[Link]: + def _process_links(self, links: list[Link]) -> list[Link]: links = [x for x in links if self._link_allowed(x)] if self.canonicalize: for link in links: @@ -265,10 +256,10 @@ class LxmlLinkExtractor: links = self.link_extractor._process_links(links) return links - def _extract_links(self, *args: Any, **kwargs: Any) -> List[Link]: + def _extract_links(self, *args: Any, **kwargs: Any) -> list[Link]: return self.link_extractor._extract_links(*args, **kwargs) - def extract_links(self, response: TextResponse) -> List[Link]: + def extract_links(self, response: TextResponse) -> list[Link]: """Returns a list of :class:`~scrapy.link.Link` objects from the specified :class:`response <scrapy.http.Response>`. diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index fea7003e5..2b838d8e2 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging import os -from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, TypedDict, Union +from typing import TYPE_CHECKING, Any, Optional, TypedDict, Union from twisted.python.failure import Failure @@ -31,7 +31,7 @@ DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s" class LogFormatterResult(TypedDict): level: int msg: str - args: Union[Dict[str, Any], Tuple[Any, ...]] + args: Union[dict[str, Any], tuple[Any, ...]] class LogFormatter: @@ -181,7 +181,7 @@ class LogFormatter: .. versionadded:: 2.0 """ - args: Dict[str, Any] = {"request": request} + args: dict[str, Any] = {"request": request} if errmsg: msg = DOWNLOADERRORMSG_LONG args["errmsg"] = errmsg diff --git a/scrapy/mail.py b/scrapy/mail.py index c020732f9..f33cf2939 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -14,18 +14,7 @@ from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - List, - Optional, - Sequence, - Tuple, - Union, -) +from typing import IO, TYPE_CHECKING, Any, Optional, Union from twisted import version as twisted_version from twisted.internet import ssl @@ -36,6 +25,8 @@ from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes if TYPE_CHECKING: + from collections.abc import Callable, Sequence + # imports twisted.internet.reactor from twisted.mail.smtp import ESMTPSenderFactory from twisted.python.failure import Failure @@ -95,11 +86,11 @@ class MailSender: def send( self, - to: Union[str, List[str]], + to: Union[str, list[str]], subject: str, body: str, - cc: Union[str, List[str], None] = None, - attachs: Sequence[Tuple[str, str, IO[Any]]] = (), + cc: Union[str, list[str], None] = None, + attachs: Sequence[tuple[str, str, IO[Any]]] = (), mimetype: str = "text/plain", charset: Optional[str] = None, _callback: Optional[Callable[..., None]] = None, @@ -164,7 +155,7 @@ class MailSender: return dfd def _sent_ok( - self, result: Any, to: List[str], cc: List[str], subject: str, nattachs: int + self, result: Any, to: list[str], cc: list[str], subject: str, nattachs: int ) -> None: logger.info( "Mail sent OK: To=%(mailto)s Cc=%(mailcc)s " @@ -180,8 +171,8 @@ class MailSender: def _sent_failed( self, failure: Failure, - to: List[str], - cc: List[str], + to: list[str], + cc: list[str], subject: str, nattachs: int, ) -> Failure: @@ -200,7 +191,7 @@ class MailSender: ) return failure - def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred[Any]: + def _sendmail(self, to_addrs: list[str], msg: bytes) -> Deferred[Any]: from twisted.internet import reactor msg_io = BytesIO(msg) @@ -218,11 +209,11 @@ class MailSender: return d def _create_sender_factory( - self, to_addrs: List[str], msg: IO[bytes], d: Deferred[Any] + self, to_addrs: list[str], msg: IO[bytes], d: Deferred[Any] ) -> ESMTPSenderFactory: from twisted.mail.smtp import ESMTPSenderFactory - factory_keywords: Dict[str, Any] = { + factory_keywords: dict[str, Any] = { "heloFallback": True, "requireAuthentication": False, "requireTransportSecurity": self.smtptls, diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 2296db90e..825d6b4c8 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -3,26 +3,15 @@ from __future__ import annotations import logging import pprint from collections import defaultdict, deque -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Deque, - Dict, - Iterable, - List, - Optional, - Tuple, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from scrapy.exceptions import NotConfigured from scrapy.utils.defer import process_chain, process_parallel from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object if TYPE_CHECKING: + from collections.abc import Callable, Iterable + from twisted.internet.defer import Deferred # typing.Concatenate and typing.ParamSpec require Python 3.10 @@ -51,14 +40,14 @@ class MiddlewareManager: self.middlewares = middlewares # Only process_spider_output and process_spider_exception can be None. # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed. - self.methods: Dict[ - str, Deque[Union[None, Callable, Tuple[Callable, Callable]]] + self.methods: dict[ + str, deque[Union[None, Callable, tuple[Callable, Callable]]] ] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @classmethod - def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]: raise NotImplementedError @classmethod @@ -107,7 +96,7 @@ class MiddlewareManager: def _process_parallel( self, methodname: str, obj: _T, *args: Any - ) -> Deferred[List[_T2]]: + ) -> Deferred[list[_T2]]: methods = cast( "Iterable[Callable[Concatenate[_T, _P], _T2]]", self.methods[methodname] ) @@ -119,8 +108,8 @@ class MiddlewareManager: ) return process_chain(methods, obj, *args) - def open_spider(self, spider: Spider) -> Deferred[List[None]]: + def open_spider(self, spider: Spider) -> Deferred[list[None]]: return self._process_parallel("open_spider", spider) - def close_spider(self, spider: Spider) -> Deferred[List[None]]: + def close_spider(self, spider: Spider) -> Deferred[list[None]]: return self._process_parallel("close_spider", spider) diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 480a5a58c..01f8bd2c8 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -6,7 +6,7 @@ See documentation in docs/item-pipeline.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, List +from typing import TYPE_CHECKING, Any from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list @@ -23,7 +23,7 @@ class ItemPipelineManager(MiddlewareManager): component_name = "item pipeline" @classmethod - def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]: return build_component_list(settings.getwithbase("ITEM_PIPELINES")) def _add_middleware(self, pipe: Any) -> None: diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 1a13aeaf2..9314856c1 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -21,15 +21,9 @@ from typing import ( IO, TYPE_CHECKING, Any, - Callable, - DefaultDict, - Dict, - List, NoReturn, Optional, Protocol, - Set, - Type, TypedDict, Union, cast, @@ -53,6 +47,7 @@ from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str if TYPE_CHECKING: + from collections.abc import Callable from os import PathLike from twisted.python.failure import Failure @@ -104,8 +99,8 @@ class FilesStoreProtocol(Protocol): path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + meta: Optional[dict[str, Any]] = None, + headers: Optional[dict[str, str]] = None, ) -> Optional[Deferred[Any]]: ... def stat_file( @@ -120,7 +115,7 @@ class FSFilesStore: basedir = basedir.split("://", 1)[1] self.basedir: str = basedir self._mkdir(Path(self.basedir)) - self.created_directories: DefaultDict[MediaPipeline.SpiderInfo, Set[str]] = ( + self.created_directories: defaultdict[MediaPipeline.SpiderInfo, set[str]] = ( defaultdict(set) ) @@ -129,8 +124,8 @@ class FSFilesStore: path: Union[str, PathLike[str]], buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + meta: Optional[dict[str, Any]] = None, + headers: Optional[dict[str, str]] = None, ) -> None: absolute_path = self._get_filesystem_path(path) self._mkdir(absolute_path.parent, info) @@ -157,7 +152,7 @@ class FSFilesStore: def _mkdir( self, dirname: Path, domain: Optional[MediaPipeline.SpiderInfo] = None ) -> None: - seen: Set[str] = self.created_directories[domain] if domain else set() + seen: set[str] = self.created_directories[domain] if domain else set() if str(dirname) not in seen: if not dirname.exists(): dirname.mkdir(parents=True) @@ -201,7 +196,7 @@ class S3FilesStore: def stat_file( self, path: str, info: MediaPipeline.SpiderInfo ) -> Deferred[StatInfo]: - def _onsuccess(boto_key: Dict[str, Any]) -> StatInfo: + def _onsuccess(boto_key: dict[str, Any]) -> StatInfo: checksum = boto_key["ETag"].strip('"') last_modified = boto_key["LastModified"] modified_stamp = time.mktime(last_modified.timetuple()) @@ -209,10 +204,10 @@ class S3FilesStore: return self._get_boto_key(path).addCallback(_onsuccess) - def _get_boto_key(self, path: str) -> Deferred[Dict[str, Any]]: + def _get_boto_key(self, path: str) -> Deferred[dict[str, Any]]: key_name = f"{self.prefix}{path}" return cast( - "Deferred[Dict[str, Any]]", + "Deferred[dict[str, Any]]", deferToThread( self.s3_client.head_object, Bucket=self.bucket, Key=key_name # type: ignore[attr-defined] ), @@ -223,8 +218,8 @@ class S3FilesStore: path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + meta: Optional[dict[str, Any]] = None, + headers: Optional[dict[str, str]] = None, ) -> Deferred[Any]: """Upload file to S3 storage""" key_name = f"{self.prefix}{path}" @@ -242,7 +237,7 @@ class S3FilesStore: **extra, ) - def _headers_to_botocore_kwargs(self, headers: Dict[str, Any]) -> Dict[str, Any]: + def _headers_to_botocore_kwargs(self, headers: dict[str, Any]) -> dict[str, Any]: """Convert headers to botocore keyword arguments.""" # This is required while we need to support both boto and botocore. mapping = CaseInsensitiveDict( @@ -274,7 +269,7 @@ class S3FilesStore: "X-Amz-Website-Redirect-Location": "WebsiteRedirectLocation", } ) - extra: Dict[str, Any] = {} + extra: dict[str, Any] = {} for key, value in headers.items(): try: kwarg = mapping[key] @@ -332,7 +327,7 @@ class GCSFilesStore: deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess), ) - def _get_content_type(self, headers: Optional[Dict[str, str]]) -> str: + def _get_content_type(self, headers: Optional[dict[str, str]]) -> str: if headers and "Content-Type" in headers: return headers["Content-Type"] return "application/octet-stream" @@ -345,8 +340,8 @@ class GCSFilesStore: path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + meta: Optional[dict[str, Any]] = None, + headers: Optional[dict[str, str]] = None, ) -> Deferred[Any]: blob_path = self._get_blob_path(path) blob = self.bucket.blob(blob_path) @@ -385,8 +380,8 @@ class FTPFilesStore: path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + meta: Optional[dict[str, Any]] = None, + headers: Optional[dict[str, str]] = None, ) -> Deferred[Any]: path = f"{self.basedir}/{path}" return deferToThread( @@ -443,7 +438,7 @@ class FilesPipeline(MediaPipeline): MEDIA_NAME: str = "file" EXPIRES: int = 90 - STORE_SCHEMES: Dict[str, Type[FilesStoreProtocol]] = { + STORE_SCHEMES: dict[str, type[FilesStoreProtocol]] = { "": FSFilesStore, "file": FSFilesStore, "s3": S3FilesStore, @@ -457,7 +452,7 @@ class FilesPipeline(MediaPipeline): self, store_uri: Union[str, PathLike[str]], download_func: Optional[Callable[[Request, Spider], Response]] = None, - settings: Union[Settings, Dict[str, Any], None] = None, + settings: Union[Settings, dict[str, Any], None] = None, ): store_uri = _to_string(store_uri) if not store_uri: @@ -486,7 +481,7 @@ class FilesPipeline(MediaPipeline): @classmethod def from_settings(cls, settings: Settings) -> Self: - s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) + s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] @@ -496,14 +491,14 @@ class FilesPipeline(MediaPipeline): s3store.AWS_VERIFY = settings["AWS_VERIFY"] s3store.POLICY = settings["FILES_STORE_S3_ACL"] - gcs_store: Type[GCSFilesStore] = cast( - Type[GCSFilesStore], cls.STORE_SCHEMES["gs"] + gcs_store: type[GCSFilesStore] = cast( + type[GCSFilesStore], cls.STORE_SCHEMES["gs"] ) gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] gcs_store.POLICY = settings["FILES_STORE_GCS_ACL"] or None - ftp_store: Type[FTPFilesStore] = cast( - Type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] + ftp_store: type[FTPFilesStore] = cast( + type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] ) ftp_store.FTP_USERNAME = settings["FTP_USER"] ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] @@ -660,7 +655,7 @@ class FilesPipeline(MediaPipeline): # Overridable Interface def get_media_requests( self, item: Any, info: MediaPipeline.SpiderInfo - ) -> List[Request]: + ) -> list[Request]: urls = ItemAdapter(item).get(self.files_urls_field, []) return [Request(u, callback=NO_CALLBACK) for u in urls] @@ -680,7 +675,7 @@ class FilesPipeline(MediaPipeline): return checksum def item_completed( - self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo + self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo ) -> Any: with suppress(KeyError): ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok] diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 166f81314..f2fe4396b 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -11,19 +11,7 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - List, - Optional, - Tuple, - Type, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from itemadapter import ItemAdapter @@ -42,6 +30,7 @@ from scrapy.settings import Settings from scrapy.utils.python import get_func_args, to_bytes if TYPE_CHECKING: + from collections.abc import Callable, Iterable from os import PathLike from PIL import Image @@ -79,7 +68,7 @@ class ImagesPipeline(FilesPipeline): MIN_WIDTH: int = 0 MIN_HEIGHT: int = 0 EXPIRES: int = 90 - THUMBS: Dict[str, Tuple[int, int]] = {} + THUMBS: dict[str, tuple[int, int]] = {} DEFAULT_IMAGES_URLS_FIELD = "image_urls" DEFAULT_IMAGES_RESULT_FIELD = "images" @@ -87,7 +76,7 @@ class ImagesPipeline(FilesPipeline): self, store_uri: Union[str, PathLike[str]], download_func: Optional[Callable[[Request, Spider], Response]] = None, - settings: Union[Settings, Dict[str, Any], None] = None, + settings: Union[Settings, dict[str, Any], None] = None, ): try: from PIL import Image @@ -127,7 +116,7 @@ class ImagesPipeline(FilesPipeline): self.min_height: int = settings.getint( resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT ) - self.thumbs: Dict[str, Tuple[int, int]] = settings.get( + self.thumbs: dict[str, tuple[int, int]] = settings.get( resolve("IMAGES_THUMBS"), self.THUMBS ) @@ -135,7 +124,7 @@ class ImagesPipeline(FilesPipeline): @classmethod def from_settings(cls, settings: Settings) -> Self: - s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) + s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] @@ -145,14 +134,14 @@ class ImagesPipeline(FilesPipeline): s3store.AWS_VERIFY = settings["AWS_VERIFY"] s3store.POLICY = settings["IMAGES_STORE_S3_ACL"] - gcs_store: Type[GCSFilesStore] = cast( - Type[GCSFilesStore], cls.STORE_SCHEMES["gs"] + gcs_store: type[GCSFilesStore] = cast( + type[GCSFilesStore], cls.STORE_SCHEMES["gs"] ) gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None - ftp_store: Type[FTPFilesStore] = cast( - Type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] + ftp_store: type[FTPFilesStore] = cast( + type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] ) ftp_store.FTP_USERNAME = settings["FTP_USER"] ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] @@ -202,7 +191,7 @@ class ImagesPipeline(FilesPipeline): info: MediaPipeline.SpiderInfo, *, item: Any = None, - ) -> Iterable[Tuple[str, Image.Image, BytesIO]]: + ) -> Iterable[tuple[str, Image.Image, BytesIO]]: path = self.file_path(request, response=response, info=info, item=item) orig_image = self._Image.open(BytesIO(response.body)) @@ -246,9 +235,9 @@ class ImagesPipeline(FilesPipeline): def convert_image( self, image: Image.Image, - size: Optional[Tuple[int, int]] = None, + size: Optional[tuple[int, int]] = None, response_body: Optional[BytesIO] = None, - ) -> Tuple[Image.Image, BytesIO]: + ) -> tuple[Image.Image, BytesIO]: if response_body is None: warnings.warn( f"{self.__class__.__name__}.convert_image() method called in a deprecated way, " @@ -288,12 +277,12 @@ class ImagesPipeline(FilesPipeline): def get_media_requests( self, item: Any, info: MediaPipeline.SpiderInfo - ) -> List[Request]: + ) -> list[Request]: urls = ItemAdapter(item).get(self.images_urls_field, []) return [Request(u, callback=NO_CALLBACK) for u in urls] def item_completed( - self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo + self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo ) -> Any: with suppress(KeyError): ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok] diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 6bd3ed9b4..b30cf9264 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -7,15 +7,9 @@ from collections import defaultdict from typing import ( TYPE_CHECKING, Any, - Callable, - DefaultDict, - Dict, - List, Literal, NoReturn, Optional, - Set, - Tuple, TypedDict, TypeVar, Union, @@ -33,6 +27,8 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter if TYPE_CHECKING: + from collections.abc import Callable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -52,7 +48,7 @@ class FileInfo(TypedDict): status: str -FileInfoOrError = Union[Tuple[Literal[True], FileInfo], Tuple[Literal[False], Failure]] +FileInfoOrError = Union[tuple[Literal[True], FileInfo], tuple[Literal[False], Failure]] logger = logging.getLogger(__name__) @@ -67,16 +63,16 @@ class MediaPipeline(ABC): class SpiderInfo: def __init__(self, spider: Spider): self.spider: Spider = spider - self.downloading: Set[bytes] = set() - self.downloaded: Dict[bytes, Union[FileInfo, Failure]] = {} - self.waiting: DefaultDict[bytes, List[Deferred[FileInfo]]] = defaultdict( + self.downloading: set[bytes] = set() + self.downloaded: dict[bytes, Union[FileInfo, Failure]] = {} + self.waiting: defaultdict[bytes, list[Deferred[FileInfo]]] = defaultdict( list ) def __init__( self, download_func: Optional[Callable[[Request, Spider], Response]] = None, - settings: Union[Settings, Dict[str, Any], None] = None, + settings: Union[Settings, dict[str, Any], None] = None, ): self.download_func = download_func @@ -129,12 +125,12 @@ class MediaPipeline(ABC): def process_item( self, item: Any, spider: Spider - ) -> Deferred[List[FileInfoOrError]]: + ) -> Deferred[list[FileInfoOrError]]: info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) dlist = [self._process_request(r, info, item) for r in requests] dfd = cast( - "Deferred[List[FileInfoOrError]]", DeferredList(dlist, consumeErrors=True) + "Deferred[list[FileInfoOrError]]", DeferredList(dlist, consumeErrors=True) ) return dfd.addCallback(self.item_completed, item, info) @@ -252,7 +248,7 @@ class MediaPipeline(ABC): raise NotImplementedError() @abstractmethod - def get_media_requests(self, item: Any, info: SpiderInfo) -> List[Request]: + def get_media_requests(self, item: Any, info: SpiderInfo) -> list[Request]: """Returns the media requests to download""" raise NotImplementedError() @@ -276,7 +272,7 @@ class MediaPipeline(ABC): raise NotImplementedError() def item_completed( - self, results: List[FileInfoOrError], item: Any, info: SpiderInfo + self, results: list[FileInfoOrError], item: Any, info: SpiderInfo ) -> Any: """Called per item when all media requests has been processed""" if self.LOG_FAILED_RESULTS: diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 58a47ef0f..e1bb21fb1 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -2,23 +2,15 @@ from __future__ import annotations import hashlib import logging -from typing import ( - TYPE_CHECKING, - Dict, - Iterable, - List, - Optional, - Protocol, - Tuple, - Type, - cast, -) +from typing import TYPE_CHECKING, Optional, Protocol, cast from scrapy import Request from scrapy.core.downloader import Downloader from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -87,7 +79,7 @@ class ScrapyPriorityQueue: def from_crawler( cls, crawler: Crawler, - downstream_queue_cls: Type[QueueProtocol], + downstream_queue_cls: type[QueueProtocol], key: str, startprios: Iterable[int] = (), ) -> Self: @@ -96,14 +88,14 @@ class ScrapyPriorityQueue: def __init__( self, crawler: Crawler, - downstream_queue_cls: Type[QueueProtocol], + downstream_queue_cls: type[QueueProtocol], key: str, startprios: Iterable[int] = (), ): self.crawler: Crawler = crawler - self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls + self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls self.key: str = key - self.queues: Dict[int, QueueProtocol] = {} + self.queues: dict[int, QueueProtocol] = {} self.curprio: Optional[int] = None self.init_prios(startprios) @@ -160,8 +152,8 @@ class ScrapyPriorityQueue: # Protocols can't declare optional members return cast(Request, queue.peek()) # type: ignore[attr-defined] - def close(self) -> List[int]: - active: List[int] = [] + def close(self) -> list[int]: + active: list[int] = [] for p, q in self.queues.items(): active.append(p) q.close() @@ -176,7 +168,7 @@ class DownloaderInterface: assert crawler.engine self.downloader: Downloader = crawler.engine.downloader - def stats(self, possible_slots: Iterable[str]) -> List[Tuple[int, str]]: + def stats(self, possible_slots: Iterable[str]) -> list[tuple[int, str]]: return [(self._active_downloads(slot), slot) for slot in possible_slots] def get_slot_key(self, request: Request) -> str: @@ -199,18 +191,18 @@ class DownloaderAwarePriorityQueue: def from_crawler( cls, crawler: Crawler, - downstream_queue_cls: Type[QueueProtocol], + downstream_queue_cls: type[QueueProtocol], key: str, - startprios: Optional[Dict[str, Iterable[int]]] = None, + startprios: Optional[dict[str, Iterable[int]]] = None, ) -> Self: return cls(crawler, downstream_queue_cls, key, startprios) def __init__( self, crawler: Crawler, - downstream_queue_cls: Type[QueueProtocol], + downstream_queue_cls: type[QueueProtocol], key: str, - slot_startprios: Optional[Dict[str, Iterable[int]]] = None, + slot_startprios: Optional[dict[str, Iterable[int]]] = None, ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( @@ -229,11 +221,11 @@ class DownloaderAwarePriorityQueue: ) self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler) - self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls + self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls self.key: str = key self.crawler: Crawler = crawler - self.pqueues: Dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue + self.pqueues: dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue for slot, startprios in (slot_startprios or {}).items(): self.pqueues[slot] = self.pqfactory(slot, startprios) @@ -281,7 +273,7 @@ class DownloaderAwarePriorityQueue: queue = self.pqueues[slot] return queue.peek() - def close(self) -> Dict[str, List[int]]: + def close(self) -> dict[str, list[int]]: active = {slot: queue.close() for slot, queue in self.pqueues.items()} self.pqueues.clear() return active diff --git a/scrapy/resolver.py b/scrapy/resolver.py index d5eedf9b1..97fa74bc2 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Type +from typing import TYPE_CHECKING, Any, Optional from twisted.internet import defer from twisted.internet.base import ReactorBase, ThreadedResolver @@ -16,6 +16,8 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache if TYPE_CHECKING: + from collections.abc import Sequence + from twisted.internet.defer import Deferred # typing.Self requires Python 3.11 @@ -82,7 +84,7 @@ class _CachingResolutionReceiver: def __init__(self, resolutionReceiver: IResolutionReceiver, hostName: str): self.resolutionReceiver: IResolutionReceiver = resolutionReceiver self.hostName: str = hostName - self.addresses: List[IAddress] = [] + self.addresses: list[IAddress] = [] def resolutionBegan(self, resolution: IHostResolution) -> None: self.resolutionReceiver.resolutionBegan(resolution) @@ -126,7 +128,7 @@ class CachingHostnameResolver: resolutionReceiver: IResolutionReceiver, hostName: str, portNumber: int = 0, - addressTypes: Optional[Sequence[Type[IAddress]]] = None, + addressTypes: Optional[Sequence[type[IAddress]]] = None, transportSemantics: str = "TCP", ) -> IHostResolution: try: diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 702e50536..7154f2b95 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -3,15 +3,20 @@ This module implements a class which returns the appropriate Response class based on different criteria. """ +from __future__ import annotations + from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data -from typing import Dict, Mapping, Optional, Type, Union +from typing import TYPE_CHECKING, Optional, Union from scrapy.http import Response from scrapy.utils.misc import load_object from scrapy.utils.python import binary_is_text, to_bytes, to_unicode +if TYPE_CHECKING: + from collections.abc import Mapping + class ResponseTypes: CLASSES = { @@ -32,7 +37,7 @@ class ResponseTypes: } def __init__(self) -> None: - self.classes: Dict[str, Type[Response]] = {} + self.classes: dict[str, type[Response]] = {} self.mimetypes: MimeTypes = MimeTypes() mimedata = get_data("scrapy", "mime.types") if not mimedata: @@ -43,7 +48,7 @@ class ResponseTypes: for mimetype, cls in self.CLASSES.items(): self.classes[mimetype] = load_object(cls) - def from_mimetype(self, mimetype: str) -> Type[Response]: + def from_mimetype(self, mimetype: str) -> type[Response]: """Return the most appropriate Response class for the given mimetype""" if mimetype is None: return Response @@ -54,7 +59,7 @@ class ResponseTypes: def from_content_type( self, content_type: Union[str, bytes], content_encoding: Optional[bytes] = None - ) -> Type[Response]: + ) -> type[Response]: """Return the most appropriate Response class from an HTTP Content-Type header""" if content_encoding: @@ -66,7 +71,7 @@ class ResponseTypes: def from_content_disposition( self, content_disposition: Union[str, bytes] - ) -> Type[Response]: + ) -> type[Response]: try: filename = ( to_unicode(content_disposition, encoding="latin-1", errors="replace") @@ -78,7 +83,7 @@ class ResponseTypes: except IndexError: return Response - def from_headers(self, headers: Mapping[bytes, bytes]) -> Type[Response]: + def from_headers(self, headers: Mapping[bytes, bytes]) -> type[Response]: """Return the most appropriate Response class by looking at the HTTP headers""" cls = Response @@ -91,14 +96,14 @@ class ResponseTypes: cls = self.from_content_disposition(headers[b"Content-Disposition"]) return cls - def from_filename(self, filename: str) -> Type[Response]: + def from_filename(self, filename: str) -> type[Response]: """Return the most appropriate Response class from a file name""" mimetype, encoding = self.mimetypes.guess_type(filename) if mimetype and not encoding: return self.from_mimetype(mimetype) return Response - def from_body(self, body: bytes) -> Type[Response]: + def from_body(self, body: bytes) -> type[Response]: """Try to guess the appropriate response based on the body content. This method is a bit magic and could be improved in the future, but it's not meant to be used except for special cases where response types @@ -122,7 +127,7 @@ class ResponseTypes: url: Optional[str] = None, filename: Optional[str] = None, body: Optional[bytes] = None, - ) -> Type[Response]: + ) -> type[Response]: """Guess the most appropriate Response class based on the given arguments.""" cls = Response diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index bfddb87cb..0a3eae409 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -2,7 +2,7 @@ XPath selectors based on lxml """ -from typing import Any, Optional, Type, Union +from typing import Any, Optional, Union from parsel import Selector as _ParselSelector @@ -23,7 +23,7 @@ def _st(response: Optional[TextResponse], st: Optional[str]) -> str: def _response_from_text(text: Union[str, bytes], st: Optional[str]) -> TextResponse: - rt: Type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse + rt: type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8")) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 6703c569f..b7e3763fb 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -2,22 +2,10 @@ from __future__ import annotations import copy import json +from collections.abc import Iterable, Iterator, Mapping, MutableMapping from importlib import import_module from pprint import pformat -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterable, - Iterator, - List, - Mapping, - MutableMapping, - Optional, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from scrapy.settings import default_settings @@ -37,7 +25,7 @@ if TYPE_CHECKING: _SettingsInputT = Union[SupportsItems[_SettingsKeyT, Any], str, None] -SETTINGS_PRIORITIES: Dict[str, int] = { +SETTINGS_PRIORITIES: dict[str, int] = { "default": 0, "command": 10, "addon": 15, @@ -192,8 +180,8 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return float(self.get(name, default)) def getlist( - self, name: _SettingsKeyT, default: Optional[List[Any]] = None - ) -> List[Any]: + self, name: _SettingsKeyT, default: Optional[list[Any]] = None + ) -> list[Any]: """ Get a setting value as a list. If the setting original type is a list, a copy of it will be returned. If it's a string it will be split by ",". @@ -213,8 +201,8 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return list(value) def getdict( - self, name: _SettingsKeyT, default: Optional[Dict[Any, Any]] = None - ) -> Dict[Any, Any]: + self, name: _SettingsKeyT, default: Optional[dict[Any, Any]] = None + ) -> dict[Any, Any]: """ Get a setting value as a dictionary. If the setting original type is a dictionary, a copy of it will be returned. If it is a string it will be @@ -238,8 +226,8 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def getdictorlist( self, name: _SettingsKeyT, - default: Union[Dict[Any, Any], List[Any], Tuple[Any], None] = None, - ) -> Union[Dict[Any, Any], List[Any]]: + default: Union[dict[Any, Any], list[Any], tuple[Any], None] = None, + ) -> Union[dict[Any, Any], list[Any]]: """Get a setting value as either a :class:`dict` or a :class:`list`. If the setting is already a dict or a list, a copy of it will be @@ -412,7 +400,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ self._assert_mutability() if isinstance(values, str): - values = cast(Dict[_SettingsKeyT, Any], json.loads(values)) + values = cast(dict[_SettingsKeyT, Any], json.loads(values)) if values is not None: if isinstance(values, BaseSettings): for name, value in values.items(): @@ -477,7 +465,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def __len__(self) -> int: return len(self.attributes) - def _to_dict(self) -> Dict[_SettingsKeyT, Any]: + def _to_dict(self) -> dict[_SettingsKeyT, Any]: return { self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) for k, v in self.items() @@ -490,7 +478,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): else str(key_value) ) - def copy_to_dict(self) -> Dict[_SettingsKeyT, Any]: + def copy_to_dict(self) -> dict[_SettingsKeyT, Any]: """ Make a copy of current settings and convert to a dict. @@ -553,7 +541,7 @@ class Settings(BaseSettings): self.update(values, priority) -def iter_default_settings() -> Iterable[Tuple[str, Any]]: +def iter_default_settings() -> Iterable[tuple[str, Any]]: """Return the default settings as an iterator of (name, value) tuples""" for name in dir(default_settings): if name.isupper(): @@ -562,7 +550,7 @@ def iter_default_settings() -> Iterable[Tuple[str, Any]]: def overridden_settings( settings: Mapping[_SettingsKeyT, Any] -) -> Iterable[Tuple[str, Any]]: +) -> Iterable[tuple[str, Any]]: """Return an iterable of the settings that have been overridden""" for name, defvalue in iter_default_settings(): value = settings[name] diff --git a/scrapy/shell.py b/scrapy/shell.py index b7e46274f..dc402e678 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -8,7 +8,7 @@ from __future__ import annotations import os import signal -from typing import Any, Callable, Dict, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Optional, Union from itemadapter import is_item from twisted.internet import defer, threads @@ -27,25 +27,28 @@ from scrapy.utils.misc import load_object from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop from scrapy.utils.response import open_in_browser +if TYPE_CHECKING: + from collections.abc import Callable + class Shell: - relevant_classes: Tuple[type, ...] = (Crawler, Spider, Request, Response, Settings) + relevant_classes: tuple[type, ...] = (Crawler, Spider, Request, Response, Settings) def __init__( self, crawler: Crawler, - update_vars: Optional[Callable[[Dict[str, Any]], None]] = None, + update_vars: Optional[Callable[[dict[str, Any]], None]] = None, code: Optional[str] = None, ): self.crawler: Crawler = crawler - self.update_vars: Callable[[Dict[str, Any]], None] = update_vars or ( + self.update_vars: Callable[[dict[str, Any]], None] = update_vars or ( lambda x: None ) self.item_class: type = load_object(crawler.settings["DEFAULT_ITEM_CLASS"]) self.spider: Optional[Spider] = None self.inthread: bool = not threadable.isInIOThread() self.code: Optional[str] = code - self.vars: Dict[str, Any] = {} + self.vars: dict[str, Any] = {} def start( self, diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index b2c6dea5d..e106418d6 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, List, Tuple +from typing import TYPE_CHECKING, Any from pydispatch import dispatcher @@ -40,7 +40,7 @@ class SignalManager: kwargs.setdefault("sender", self.sender) dispatcher.disconnect(receiver, signal, **kwargs) - def send_catch_log(self, signal: Any, **kwargs: Any) -> List[Tuple[Any, Any]]: + def send_catch_log(self, signal: Any, **kwargs: Any) -> list[tuple[Any, Any]]: """ Send a signal, catch exceptions and log them. @@ -52,7 +52,7 @@ class SignalManager: def send_catch_log_deferred( self, signal: Any, **kwargs: Any - ) -> Deferred[List[Tuple[Any, Any]]]: + ) -> Deferred[list[tuple[Any, Any]]]: """ Like :meth:`send_catch_log` but supports returning :class:`~twisted.internet.defer.Deferred` objects from signal handlers. diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index f5fd899b2..210e729a1 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -3,7 +3,7 @@ from __future__ import annotations import traceback import warnings from collections import defaultdict -from typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Type +from typing import TYPE_CHECKING from zope.interface import implementer @@ -29,10 +29,10 @@ class SpiderLoader: """ def __init__(self, settings: BaseSettings): - self.spider_modules: List[str] = settings.getlist("SPIDER_MODULES") + self.spider_modules: list[str] = settings.getlist("SPIDER_MODULES") self.warn_only: bool = settings.getbool("SPIDER_LOADER_WARN_ONLY") - self._spiders: Dict[str, Type[Spider]] = {} - self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list) + self._spiders: dict[str, type[Spider]] = {} + self._found: defaultdict[str, list[tuple[str, str]]] = defaultdict(list) self._load_all_spiders() def _check_name_duplicates(self) -> None: @@ -80,7 +80,7 @@ class SpiderLoader: def from_settings(cls, settings: BaseSettings) -> Self: return cls(settings) - def load(self, spider_name: str) -> Type[Spider]: + def load(self, spider_name: str) -> type[Spider]: """ Return the Spider class for the given spider name. If the spider name is not found, raise a KeyError. @@ -90,7 +90,7 @@ class SpiderLoader: except KeyError: raise KeyError(f"Spider not found: {spider_name}") - def find_by_request(self, request: Request) -> List[str]: + def find_by_request(self, request: Request) -> list[str]: """ Return the list of spider names that can handle the given request. """ @@ -98,7 +98,7 @@ class SpiderLoader: name for name, cls in self._spiders.items() if cls.handles_request(request) ] - def list(self) -> List[str]: + def list(self) -> list[str]: """ Return a list with the names of all spiders available in the project. """ diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index c5b7f0749..3164c1c03 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -7,11 +7,13 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable +from typing import TYPE_CHECKING, Any from scrapy.http import Request, Response if TYPE_CHECKING: + from collections.abc import AsyncIterable, Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index ea1686c25..afab2eac2 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -7,11 +7,13 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Iterable, List, Optional +from typing import TYPE_CHECKING, Any, Optional from scrapy.exceptions import IgnoreRequest if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -39,7 +41,7 @@ class HttpErrorMiddleware: def __init__(self, settings: BaseSettings): self.handle_httpstatus_all: bool = settings.getbool("HTTPERROR_ALLOW_ALL") - self.handle_httpstatus_list: List[int] = settings.getlist( + self.handle_httpstatus_list: list[int] = settings.getlist( "HTTPERROR_ALLOWED_CODES" ) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 379c5d0a3..d3ed64ef5 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -9,7 +9,7 @@ from __future__ import annotations import logging import re import warnings -from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable, Set +from typing import TYPE_CHECKING, Any from scrapy import Spider, signals from scrapy.exceptions import ScrapyDeprecationWarning @@ -23,6 +23,8 @@ warnings.warn( ) if TYPE_CHECKING: + from collections.abc import AsyncIterable, Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -109,7 +111,7 @@ class OffsiteMiddleware: def spider_opened(self, spider: Spider) -> None: self.host_regex: re.Pattern[str] = self.get_host_regex(spider) - self.domains_seen: Set[str] = set() + self.domains_seen: set[str] = set() class URLWarning(Warning): diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index d35cf8f71..8784e4b05 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -6,18 +6,7 @@ originated it. from __future__ import annotations import warnings -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - Dict, - Iterable, - Optional, - Tuple, - Type, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from urllib.parse import urlparse from w3lib.url import safe_url_string @@ -30,6 +19,8 @@ from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url if TYPE_CHECKING: + from collections.abc import AsyncIterable, Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -37,7 +28,7 @@ if TYPE_CHECKING: from scrapy.settings import BaseSettings -LOCAL_SCHEMES: Tuple[str, ...] = ( +LOCAL_SCHEMES: tuple[str, ...] = ( "about", "blob", "data", @@ -56,7 +47,7 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default" class ReferrerPolicy: - NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + NOREFERRER_SCHEMES: tuple[str, ...] = LOCAL_SCHEMES name: str def referrer(self, response_url: str, request_url: str) -> Optional[str]: @@ -291,11 +282,11 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): using ``file://`` or ``s3://`` scheme. """ - NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ("file", "s3") + NOREFERRER_SCHEMES: tuple[str, ...] = LOCAL_SCHEMES + ("file", "s3") name: str = POLICY_SCRAPY_DEFAULT -_policy_classes: Dict[str, Type[ReferrerPolicy]] = { +_policy_classes: dict[str, type[ReferrerPolicy]] = { p.name: p for p in ( NoReferrerPolicy, @@ -316,14 +307,14 @@ _policy_classes[""] = NoReferrerWhenDowngradePolicy def _load_policy_class( policy: str, warning_only: bool = False -) -> Optional[Type[ReferrerPolicy]]: +) -> Optional[type[ReferrerPolicy]]: """ Expect a string for the path to the policy class, otherwise try to interpret the string as a standard value from https://www.w3.org/TR/referrer-policy/#referrer-policies """ try: - return cast(Type[ReferrerPolicy], load_object(policy)) + return cast(type[ReferrerPolicy], load_object(policy)) except ValueError: tokens = [token.strip() for token in policy.lower().split(",")] # https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header @@ -341,7 +332,7 @@ def _load_policy_class( class RefererMiddleware: def __init__(self, settings: Optional[BaseSettings] = None): - self.default_policy: Type[ReferrerPolicy] = DefaultReferrerPolicy + self.default_policy: type[ReferrerPolicy] = DefaultReferrerPolicy if settings is not None: settings_policy = _load_policy_class(settings.get("REFERRER_POLICY")) assert settings_policy diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 34df54ca7..191adb6cd 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -7,12 +7,14 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable +from typing import TYPE_CHECKING, Any from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response if TYPE_CHECKING: + from collections.abc import AsyncIterable, Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index d977acd26..8220aca28 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, cast +from typing import TYPE_CHECKING, Any, Optional, cast from scrapy import signals from scrapy.http import Request, Response @@ -15,6 +15,8 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: + from collections.abc import Iterable + from twisted.internet.defer import Deferred # typing.Self requires Python 3.11 @@ -32,7 +34,7 @@ class Spider(object_ref): """ name: str - custom_settings: Optional[Dict[_SettingsKeyT, Any]] = None + custom_settings: Optional[dict[_SettingsKeyT, Any]] = None def __init__(self, name: Optional[str] = None, **kwargs: Any): if name is not None: @@ -41,7 +43,7 @@ class Spider(object_ref): raise ValueError(f"{type(self).__name__} must have a name") self.__dict__.update(kwargs) if not hasattr(self, "start_urls"): - self.start_urls: List[str] = [] + self.start_urls: list[str] = [] @property def logger(self) -> SpiderLoggerAdapter: diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 2639f14b2..d628f49f6 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -1,6 +1,6 @@ """ This modules implements the CrawlSpider which is the recommended spider to use -for scraping typical web sites that requires crawling pages. +for scraping typical websites that requires crawling pages. See documentation in docs/topics/spiders.rst """ @@ -8,22 +8,8 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import copy -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - Awaitable, - Callable, - Dict, - Iterable, - List, - Optional, - Sequence, - Set, - TypeVar, - Union, - cast, -) +from collections.abc import AsyncIterable, Awaitable, Callable +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from twisted.python.failure import Failure @@ -35,6 +21,8 @@ from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.spider import iterate_spider_output if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -43,7 +31,7 @@ if TYPE_CHECKING: _T = TypeVar("_T") -ProcessLinksT = Callable[[List[Link]], List[Link]] +ProcessLinksT = Callable[[list[Link]], list[Link]] ProcessRequestT = Callable[[Request, Response], Optional[Request]] @@ -75,7 +63,7 @@ class Rule: self, link_extractor: Optional[LinkExtractor] = None, callback: Union[CallbackT, str, None] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, follow: Optional[bool] = None, process_links: Union[ProcessLinksT, str, None] = None, process_request: Union[ProcessRequestT, str, None] = None, @@ -84,7 +72,7 @@ class Rule: self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor self.callback: Union[CallbackT, str, None] = callback self.errback: Union[Callable[[Failure], Any], str, None] = errback - self.cb_kwargs: Dict[str, Any] = cb_kwargs or {} + self.cb_kwargs: dict[str, Any] = cb_kwargs or {} self.process_links: Union[ProcessLinksT, str] = process_links or _identity self.process_request: Union[ProcessRequestT, str] = ( process_request or _identity_process_request @@ -105,7 +93,7 @@ class Rule: class CrawlSpider(Spider): rules: Sequence[Rule] = () - _rules: List[Rule] + _rules: list[Rule] _follow_links: bool def __init__(self, *a: Any, **kw: Any): @@ -139,9 +127,9 @@ class CrawlSpider(Spider): def _requests_to_follow(self, response: Response) -> Iterable[Optional[Request]]: if not isinstance(response, HtmlResponse): return - seen: Set[Link] = set() + seen: set[Link] = set() for rule_index, rule in enumerate(self._rules): - links: List[Link] = [ + links: list[Link] = [ lnk for lnk in rule.link_extractor.extract_links(response) if lnk not in seen @@ -170,7 +158,7 @@ class CrawlSpider(Spider): self, response: Response, callback: Optional[CallbackT], - cb_kwargs: Dict[str, Any], + cb_kwargs: dict[str, Any], follow: bool = True, ) -> AsyncIterable[Any]: if callback: diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 9dd8a5d68..0ddef1f32 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -5,7 +5,9 @@ for scraping from an XML feed. See documentation in docs/topics/spiders.rst """ -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional from scrapy.exceptions import NotConfigured, NotSupported from scrapy.http import Response, TextResponse @@ -14,6 +16,9 @@ from scrapy.spiders import Spider from scrapy.utils.iterators import csviter, xmliter_lxml from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + class XMLFeedSpider(Spider): """ @@ -27,7 +32,7 @@ class XMLFeedSpider(Spider): iterator: str = "iternodes" itertag: str = "item" - namespaces: Sequence[Tuple[str, str]] = () + namespaces: Sequence[tuple[str, str]] = () def process_results( self, response: Response, results: Iterable[Any] @@ -118,7 +123,7 @@ class CSVFeedSpider(Spider): quotechar: Optional[str] = ( None # When this is None, python's csv module's default quotechar is used ) - headers: Optional[List[str]] = None + headers: Optional[list[str]] = None def process_results( self, response: Response, results: Iterable[Any] @@ -130,7 +135,7 @@ class CSVFeedSpider(Spider): """This method has the same purpose as the one in XMLFeedSpider""" return response - def parse_row(self, response: Response, row: Dict[str, str]) -> Any: + def parse_row(self, response: Response, row: dict[str, str]) -> Any: """This method must be overridden with your custom spider functionality""" raise NotImplementedError diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index ce0f1bbaa..ebe288b83 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Iterable, Optional, cast +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, Optional, cast from scrapy import Request from scrapy.spiders import Spider diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 1542ef79c..945539d7b 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -2,18 +2,7 @@ from __future__ import annotations import logging import re -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterable, - List, - Optional, - Sequence, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from scrapy.http import Request, Response, XmlResponse from scrapy.spiders import Spider @@ -22,6 +11,8 @@ from scrapy.utils.gz import gunzip, gzip_magic_number from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -34,7 +25,7 @@ logger = logging.getLogger(__name__) class SitemapSpider(Spider): sitemap_urls: Sequence[str] = () sitemap_rules: Sequence[ - Tuple[Union[re.Pattern[str], str], Union[str, CallbackT]] + tuple[Union[re.Pattern[str], str], Union[str, CallbackT]] ] = [("", "parse")] sitemap_follow: Sequence[Union[re.Pattern[str], str]] = [""] sitemap_alternate_links: bool = False @@ -54,20 +45,20 @@ class SitemapSpider(Spider): def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) - self._cbs: List[Tuple[re.Pattern[str], CallbackT]] = [] + self._cbs: list[tuple[re.Pattern[str], CallbackT]] = [] for r, c in self.sitemap_rules: if isinstance(c, str): c = cast("CallbackT", getattr(self, c)) self._cbs.append((regex(r), c)) - self._follow: List[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] + self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] def start_requests(self) -> Iterable[Request]: for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) def sitemap_filter( - self, entries: Iterable[Dict[str, Any]] - ) -> Iterable[Dict[str, Any]]: + self, entries: Iterable[dict[str, Any]] + ) -> Iterable[dict[str, Any]]: """This method can be used to filter sitemap entries by their attributes, for example, you can filter locs with lastmod greater than a given date (see docs). @@ -142,7 +133,7 @@ def regex(x: Union[re.Pattern[str], str]) -> re.Pattern[str]: return x -def iterloc(it: Iterable[Dict[str, Any]], alt: bool = False) -> Iterable[str]: +def iterloc(it: Iterable[dict[str, Any]], alt: bool = False) -> Iterable[str]: for d in it: yield d["loc"] diff --git a/scrapy/squeues.py b/scrapy/squeues.py index d3e7896c5..767a53db8 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -7,13 +7,14 @@ from __future__ import annotations import marshal import pickle # nosec from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Optional, Union from queuelib import queue from scrapy.utils.request import request_from_dict if TYPE_CHECKING: + from collections.abc import Callable from os import PathLike # typing.Self requires Python 3.11 @@ -23,7 +24,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler -def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]: +def _with_mkdir(queue_class: type[queue.BaseQueue]) -> type[queue.BaseQueue]: class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc] def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any): dirname = Path(path).parent @@ -35,10 +36,10 @@ def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]: def _serializable_queue( - queue_class: Type[queue.BaseQueue], + queue_class: type[queue.BaseQueue], serialize: Callable[[Any], bytes], deserialize: Callable[[bytes], Any], -) -> Type[queue.BaseQueue]: +) -> type[queue.BaseQueue]: class SerializableQueue(queue_class): # type: ignore[valid-type,misc] def push(self, obj: Any) -> None: s = serialize(obj) @@ -71,8 +72,8 @@ def _serializable_queue( def _scrapy_serialization_queue( - queue_class: Type[queue.BaseQueue], -) -> Type[queue.BaseQueue]: + queue_class: type[queue.BaseQueue], +) -> type[queue.BaseQueue]: class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc] def __init__(self, crawler: Crawler, key: str): self.spider = crawler.spider @@ -110,8 +111,8 @@ def _scrapy_serialization_queue( def _scrapy_non_serialization_queue( - queue_class: Type[queue.BaseQueue], -) -> Type[queue.BaseQueue]: + queue_class: type[queue.BaseQueue], +) -> type[queue.BaseQueue]: class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc] @classmethod def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 88e72f366..63c82ec6d 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -6,7 +6,7 @@ from __future__ import annotations import logging import pprint -from typing import TYPE_CHECKING, Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Optional if TYPE_CHECKING: from scrapy import Spider @@ -16,7 +16,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -StatsT = Dict[str, Any] +StatsT = dict[str, Any] class StatsCollector: @@ -71,7 +71,7 @@ class StatsCollector: class MemoryStatsCollector(StatsCollector): def __init__(self, crawler: Crawler): super().__init__(crawler) - self.spider_stats: Dict[str, StatsT] = {} + self.spider_stats: dict[str, StatsT] = {} def _persist_stats(self, stats: StatsT, spider: Spider) -> None: self.spider_stats[spider.name] = stats diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 67c8e1a01..f1505e4bd 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,9 +1,10 @@ -from typing import AsyncGenerator, AsyncIterable, Iterable, List, TypeVar, Union +from collections.abc import AsyncGenerator, AsyncIterable, Iterable +from typing import TypeVar, Union _T = TypeVar("_T") -async def collect_asyncgen(result: AsyncIterable[_T]) -> List[_T]: +async def collect_asyncgen(result: AsyncIterable[_T]) -> list[_T]: results = [] async for x in result: results.append(x) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index c63b69995..463bbb5df 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -1,35 +1,29 @@ +from __future__ import annotations + import numbers import os import sys import warnings +from collections.abc import Iterable from configparser import ConfigParser from operator import itemgetter from pathlib import Path -from typing import ( - Any, - Callable, - Collection, - Dict, - Iterable, - List, - Mapping, - MutableMapping, - Optional, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values +if TYPE_CHECKING: + from collections.abc import Collection, Mapping, MutableMapping + def build_component_list( compdict: MutableMapping[Any, Any], custom: Any = None, convert: Callable[[Any], Any] = update_classpath, -) -> List[Any]: +) -> list[Any]: """Compose a component list from a { class: order } dictionary.""" def _check_components(complist: Collection[Any]) -> None: @@ -39,7 +33,7 @@ def build_component_list( "please update your settings" ) - def _map_keys(compdict: Mapping[Any, Any]) -> Union[BaseSettings, Dict[Any, Any]]: + def _map_keys(compdict: Mapping[Any, Any]) -> Union[BaseSettings, dict[Any, Any]]: if isinstance(compdict, BaseSettings): compbs = BaseSettings() for k, v in compdict.items(): @@ -84,7 +78,7 @@ def build_component_list( return [k for k, v in sorted(compdict.items(), key=itemgetter(1))] -def arglist_to_dict(arglist: List[str]) -> Dict[str, str]: +def arglist_to_dict(arglist: list[str]) -> dict[str, str]: """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a dict """ @@ -130,7 +124,7 @@ def get_config(use_closest: bool = True) -> ConfigParser: return cfg -def get_sources(use_closest: bool = True) -> List[str]: +def get_sources(use_closest: bool = True) -> list[str]: xdg_config_home = ( os.environ.get("XDG_CONFIG_HOME") or Path("~/.config").expanduser() ) @@ -146,8 +140,8 @@ def get_sources(use_closest: bool = True) -> List[str]: def feed_complete_default_values_from_settings( - feed: Dict[str, Any], settings: BaseSettings -) -> Dict[str, Any]: + feed: dict[str, Any], settings: BaseSettings +) -> dict[str, Any]: out = feed.copy() out.setdefault("batch_item_count", settings.getint("FEED_EXPORT_BATCH_ITEM_COUNT")) out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) @@ -164,17 +158,17 @@ def feed_complete_default_values_from_settings( def feed_process_params_from_cli( settings: BaseSettings, - output: List[str], + output: list[str], output_format: Optional[str] = None, - overwrite_output: Optional[List[str]] = None, -) -> Dict[str, Dict[str, Any]]: + overwrite_output: Optional[list[str]] = None, +) -> dict[str, dict[str, Any]]: """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary suitable to be used as the FEEDS setting. """ valid_output_formats: Iterable[str] = without_none_values( - cast(Dict[str, str], settings.getwithbase("FEED_EXPORTERS")) + cast(dict[str, str], settings.getwithbase("FEED_EXPORTERS")) ).keys() def check_valid_format(output_format: str) -> None: @@ -223,7 +217,7 @@ def feed_process_params_from_cli( "URIs are specified" ) - result: Dict[str, Dict[str, Any]] = {} + result: dict[str, dict[str, Any]] = {} for element in output: try: feed_uri, feed_format = element.rsplit(":", 1) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 328219831..3b5596ab7 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,12 +1,18 @@ +from __future__ import annotations + +from collections.abc import Callable from functools import wraps -from typing import Any, Callable, Dict, Iterable, Optional +from typing import TYPE_CHECKING, Any, Optional + +if TYPE_CHECKING: + from collections.abc import Iterable EmbedFuncT = Callable[..., None] -KnownShellsT = Dict[str, Callable[..., EmbedFuncT]] +KnownShellsT = dict[str, Callable[..., EmbedFuncT]] def _embed_ipython_shell( - namespace: Dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start an IPython Shell""" try: @@ -21,7 +27,7 @@ def _embed_ipython_shell( ) @wraps(_embed_ipython_shell) - def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: config = load_default_config() # Always use .instance() to ensure _instance propagation to all parents # this is needed for <TAB> completion works well for new imports @@ -37,26 +43,26 @@ def _embed_ipython_shell( def _embed_bpython_shell( - namespace: Dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start a bpython shell""" import bpython @wraps(_embed_bpython_shell) - def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: bpython.embed(locals_=namespace, banner=banner) return wrapper def _embed_ptpython_shell( - namespace: Dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start a ptpython shell""" import ptpython.repl @wraps(_embed_ptpython_shell) - def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: print(banner) ptpython.repl.embed(locals=namespace) @@ -64,7 +70,7 @@ def _embed_ptpython_shell( def _embed_standard_shell( - namespace: Dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start a standard python shell""" import code @@ -79,7 +85,7 @@ def _embed_standard_shell( readline.parse_and_bind("tab:complete") @wraps(_embed_standard_shell) - def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: code.interact(banner=banner, local=namespace) return wrapper @@ -114,7 +120,7 @@ def get_shell_embed_func( def start_python_console( - namespace: Optional[Dict[str, Any]] = None, + namespace: Optional[dict[str, Any]] = None, banner: str = "", shells: Optional[Iterable[str]] = None, ) -> None: diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index c10e48511..9c7f63848 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -1,12 +1,17 @@ +from __future__ import annotations + import argparse import warnings from http.cookies import SimpleCookie from shlex import split -from typing import Any, Dict, List, NoReturn, Optional, Sequence, Tuple, Union +from typing import TYPE_CHECKING, Any, NoReturn, Optional, Union from urllib.parse import urlparse from w3lib.http import basic_auth_header +if TYPE_CHECKING: + from collections.abc import Sequence + class DataAction(argparse.Action): def __call__( @@ -51,9 +56,9 @@ for argument in safe_to_ignore_arguments: def _parse_headers_and_cookies( parsed_args: argparse.Namespace, -) -> Tuple[List[Tuple[str, bytes]], Dict[str, str]]: - headers: List[Tuple[str, bytes]] = [] - cookies: Dict[str, str] = {} +) -> tuple[list[tuple[str, bytes]], dict[str, str]]: + headers: list[tuple[str, bytes]] = [] + cookies: dict[str, str] = {} for header in parsed_args.headers or (): name, val = header.split(":", 1) name = name.strip() @@ -73,7 +78,7 @@ def _parse_headers_and_cookies( def curl_to_request_kwargs( curl_command: str, ignore_unknown_options: bool = True -) -> Dict[str, Any]: +) -> dict[str, Any]: """Convert a cURL command syntax to Request kwargs. :param str curl_command: string containing the curl command @@ -107,7 +112,7 @@ def curl_to_request_kwargs( method = parsed_args.method or "GET" - result: Dict[str, Any] = {"method": method.upper(), "url": url} + result: dict[str, Any] = {"method": method.upper(), "url": url} headers, cookies = _parse_headers_and_cookies(parsed_args) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index d06887610..c78325676 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -10,23 +10,15 @@ from __future__ import annotations import collections import warnings import weakref +from collections import OrderedDict from collections.abc import Mapping -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Iterable, - Optional, - OrderedDict, - Sequence, - Tuple, - TypeVar, - Union, -) +from typing import TYPE_CHECKING, Any, AnyStr, Optional, TypeVar, Union from scrapy.exceptions import ScrapyDeprecationWarning if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -52,7 +44,7 @@ class CaselessDict(dict): def __init__( self, - seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, ): super().__init__() if seq: @@ -92,7 +84,7 @@ class CaselessDict(dict): return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) # type: ignore[arg-type] # doesn't fully implement MutableMapping.update() - def update(self, seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]]]) -> None: # type: ignore[override] + def update(self, seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]]]) -> None: # type: ignore[override] seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) super().update(iseq) diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 2240f0b58..0f4d0beda 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -2,7 +2,7 @@ from __future__ import annotations import warnings from functools import wraps -from typing import TYPE_CHECKING, Any, Callable, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar from twisted.internet.defer import Deferred, maybeDeferred from twisted.internet.threads import deferToThread @@ -10,6 +10,8 @@ from twisted.internet.threads import deferToThread from scrapy.exceptions import ScrapyDeprecationWarning if TYPE_CHECKING: + from collections.abc import Callable + # typing.ParamSpec requires Python 3.10 from typing_extensions import ParamSpec diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 33ec23cec..3a0dee8f1 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -8,28 +8,10 @@ import asyncio import inspect import warnings from asyncio import Future +from collections.abc import Awaitable, Coroutine, Iterable, Iterator from functools import wraps from types import CoroutineType -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - AsyncIterator, - Awaitable, - Callable, - Coroutine, - Dict, - Generic, - Iterable, - Iterator, - List, - Optional, - Tuple, - TypeVar, - Union, - cast, - overload, -) +from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar, Union, cast, overload from twisted.internet import defer from twisted.internet.defer import Deferred, DeferredList, ensureDeferred @@ -40,6 +22,8 @@ from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed if TYPE_CHECKING: + from collections.abc import AsyncIterable, AsyncIterator, Callable + from twisted.python.failure import Failure # typing.Concatenate and typing.ParamSpec require Python 3.10 @@ -47,6 +31,7 @@ if TYPE_CHECKING: _P = ParamSpec("_P") + _T = TypeVar("_T") _T2 = TypeVar("_T2") @@ -134,7 +119,7 @@ def parallel( callable: Callable[Concatenate[_T, _P], _T2], *args: _P.args, **named: _P.kwargs, -) -> Deferred[List[Tuple[bool, Iterator[_T2]]]]: +) -> Deferred[list[tuple[bool, Iterator[_T2]]]]: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -145,7 +130,7 @@ def parallel( return DeferredList([coop.coiterate(work) for _ in range(count)]) -class _AsyncCooperatorAdapter(Iterator[Deferred], Generic[_T]): +class _AsyncCooperatorAdapter(Iterator, Generic[_T]): """A class that wraps an async iterable into a normal iterator suitable for using in Cooperator.coiterate(). As it's only needed for parallel_async(), it calls the callable directly in the callback, instead of providing a more @@ -200,10 +185,10 @@ class _AsyncCooperatorAdapter(Iterator[Deferred], Generic[_T]): ): self.aiterator: AsyncIterator[_T] = aiterable.__aiter__() self.callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]] = callable - self.callable_args: Tuple[Any, ...] = callable_args - self.callable_kwargs: Dict[str, Any] = callable_kwargs + self.callable_args: tuple[Any, ...] = callable_args + self.callable_kwargs: dict[str, Any] = callable_kwargs self.finished: bool = False - self.waiting_deferreds: List[Deferred[Any]] = [] + self.waiting_deferreds: list[Deferred[Any]] = [] self.anext_deferred: Optional[Deferred[_T]] = None def _callback(self, result: _T) -> None: @@ -255,13 +240,13 @@ def parallel_async( callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]], *args: _P.args, **named: _P.kwargs, -) -> Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]]: +) -> Deferred[list[tuple[bool, Iterator[Deferred[Any]]]]]: """Like ``parallel`` but for async iterators""" coop = Cooperator() work: Iterator[Deferred[Any]] = _AsyncCooperatorAdapter( async_iterable, callable, *args, **named ) - dl: Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]] = DeferredList( + dl: Deferred[list[tuple[bool, Iterator[Deferred[Any]]]]] = DeferredList( [coop.coiterate(work) for _ in range(count)] ) return dl @@ -311,15 +296,15 @@ def process_parallel( input: _T, *a: _P.args, **kw: _P.kwargs, -) -> Deferred[List[_T2]]: +) -> Deferred[list[_T2]]: """Return a Deferred with the output of all successful calls to the given callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d: Deferred[List[Tuple[bool, _T2]]] = DeferredList( + d: Deferred[list[tuple[bool, _T2]]] = DeferredList( dfds, fireOnOneErrback=True, consumeErrors=True ) - d2: Deferred[List[_T2]] = d.addCallback(lambda r: [x[1] for x in r]) + d2: Deferred[list[_T2]] = d.addCallback(lambda r: [x[1] for x in r]) d2.addErrback(lambda f: f.value.subFailure) return d2 diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index e0f2ac763..9b0d476a1 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -2,7 +2,7 @@ import inspect import warnings -from typing import Any, Dict, List, Optional, Tuple, Type, overload +from typing import Any, Optional, overload from scrapy.exceptions import ScrapyDeprecationWarning @@ -20,8 +20,8 @@ def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> No def create_deprecated_class( name: str, new_class: type, - clsdict: Optional[Dict[str, Any]] = None, - warn_category: Type[Warning] = ScrapyDeprecationWarning, + clsdict: Optional[dict[str, Any]] = None, + warn_category: type[Warning] = ScrapyDeprecationWarning, warn_once: bool = True, old_class_path: Optional[str] = None, new_class_path: Optional[str] = None, @@ -59,14 +59,14 @@ def create_deprecated_class( warned_on_subclass: bool = False def __new__( - metacls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any] + metacls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any] ) -> type: cls = super().__new__(metacls, name, bases, clsdict_) if metacls.deprecated_class is None: metacls.deprecated_class = cls return cls - def __init__(cls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any]): + def __init__(cls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any]): meta = cls.__class__ old = meta.deprecated_class if old in bases and not (warn_once and meta.warned_on_subclass): @@ -134,7 +134,7 @@ def _clspath(cls: type, forced: Optional[str] = None) -> str: return f"{cls.__module__}.{cls.__name__}" -DEPRECATION_RULES: List[Tuple[str, str]] = [] +DEPRECATION_RULES: list[tuple[str, str]] = [] @overload diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 770ee0b1b..1430ed8d6 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -4,13 +4,13 @@ from __future__ import annotations # used in global tests code from time import time # noqa: F401 -from typing import TYPE_CHECKING, Any, List, Tuple +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from scrapy.core.engine import ExecutionEngine -def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]: +def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]: """Return a report of the current engine status""" tests = [ "time()-engine.start_time", @@ -29,7 +29,7 @@ def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]: "engine.scraper.slot.needs_backout()", ] - checks: List[Tuple[str, Any]] = [] + checks: list[tuple[str, Any]] = [] for test in tests: try: checks += [(test, eval(test))] # nosec diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 41a842386..a4d339adc 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,19 +1,10 @@ +from __future__ import annotations + import csv import logging import re from io import StringIO -from typing import ( - Any, - Callable, - Dict, - Iterator, - List, - Literal, - Optional, - Union, - cast, - overload, -) +from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast, overload from warnings import warn from lxml import etree # nosec @@ -23,6 +14,9 @@ from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + logger = logging.getLogger(__name__) @@ -59,7 +53,7 @@ def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Iterator[Selecto ) header_end_idx = re_rsearch(HEADER_END_RE, text) header_end = text[header_end_idx[1] :].strip() if header_end_idx else "" - namespaces: Dict[str, str] = {} + namespaces: dict[str, str] = {} if header_end: for tagname in reversed(re.findall(END_TAG_RE, header_end)): assert header_end_idx @@ -162,10 +156,10 @@ class _StreamReader: def csviter( obj: Union[Response, str, bytes], delimiter: Optional[str] = None, - headers: Optional[List[str]] = None, + headers: Optional[list[str]] = None, encoding: Optional[str] = None, quotechar: Optional[str] = None, -) -> Iterator[Dict[str, str]]: +) -> Iterator[dict[str, str]]: """Returns an iterator of dictionaries from the given csv object obj can be: @@ -191,7 +185,7 @@ def csviter( lines = StringIO(_body_or_str(obj, unicode=True)) - kwargs: Dict[str, Any] = {} + kwargs: dict[str, Any] = {} if delimiter: kwargs["delimiter"] = delimiter if quotechar: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 4a70de6b4..2b90c6b36 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -2,20 +2,10 @@ from __future__ import annotations import logging import sys +from collections.abc import MutableMapping from logging.config import dictConfig from types import TracebackType -from typing import ( - TYPE_CHECKING, - Any, - Dict, - List, - MutableMapping, - Optional, - Tuple, - Type, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from twisted.python import log as twisted_log from twisted.python.failure import Failure @@ -25,6 +15,7 @@ from scrapy.settings import Settings, _SettingsKeyT from scrapy.utils.versions import scrapy_components_versions if TYPE_CHECKING: + from scrapy.crawler import Crawler from scrapy.logformatter import LogFormatterResult @@ -34,7 +25,7 @@ logger = logging.getLogger(__name__) def failure_to_exc_info( failure: Failure, -) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]: +) -> Optional[tuple[type[BaseException], BaseException, Optional[TracebackType]]]: """Extract exc_info from Failure instances""" if isinstance(failure, Failure): assert failure.type @@ -48,7 +39,7 @@ def failure_to_exc_info( class TopLevelFormatter(logging.Filter): - """Keep only top level loggers's name (direct children from root) from + """Keep only top level loggers' name (direct children from root) from records. This filter will replace Scrapy loggers' names with 'scrapy'. This mimics @@ -59,8 +50,8 @@ class TopLevelFormatter(logging.Filter): ``loggers`` list where it should act. """ - def __init__(self, loggers: Optional[List[str]] = None): - self.loggers: List[str] = loggers or [] + def __init__(self, loggers: Optional[list[str]] = None): + self.loggers: list[str] = loggers or [] def filter(self, record: logging.LogRecord) -> bool: if any(record.name.startswith(logger + ".") for logger in self.loggers): @@ -89,7 +80,7 @@ DEFAULT_LOGGING = { def configure_logging( - settings: Union[Settings, Dict[_SettingsKeyT, Any], None] = None, + settings: Union[Settings, dict[_SettingsKeyT, Any], None] = None, install_root_handler: bool = True, ) -> None: """ @@ -240,7 +231,7 @@ class LogCounterHandler(logging.Handler): def logformatter_adapter( logkws: LogFormatterResult, -) -> Tuple[int, str, Union[Dict[str, Any], Tuple[Any, ...]]]: +) -> tuple[int, str, Union[dict[str, Any], tuple[Any, ...]]]: """ Helper that takes the dictionary output from the methods in LogFormatter and adapts it into a tuple of positional arguments for logger.log calls, @@ -251,7 +242,7 @@ def logformatter_adapter( message = logkws.get("msg") or "" # NOTE: This also handles 'args' being an empty dict, that case doesn't # play well in logger.log calls - args = cast(Dict[str, Any], logkws) if not logkws.get("args") else logkws["args"] + args = cast(dict[str, Any], logkws) if not logkws.get("args") else logkws["args"] return (level, message, args) @@ -259,7 +250,7 @@ def logformatter_adapter( class SpiderLoggerAdapter(logging.LoggerAdapter): def process( self, msg: str, kwargs: MutableMapping[str, Any] - ) -> Tuple[str, MutableMapping[str, Any]]: + ) -> tuple[str, MutableMapping[str, Any]]: """Method that augments logging with additional 'extra' data""" if isinstance(kwargs.get("extra"), MutableMapping): kwargs["extra"].update(self.extra) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 3c787e50f..e5e00512a 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -9,31 +9,19 @@ import os import re import warnings from collections import deque +from collections.abc import Iterable from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Deque, - Iterable, - Iterator, - List, - Optional, - Type, - TypeVar, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item from scrapy.utils.datatypes import LocalWeakReferencedCache if TYPE_CHECKING: + from collections.abc import Callable, Iterator from types import ModuleType from scrapy import Spider @@ -91,7 +79,7 @@ def load_object(path: Union[str, Callable[..., Any]]) -> Any: return obj -def walk_modules(path: str) -> List[ModuleType]: +def walk_modules(path: str) -> list[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that exception is thrown back. @@ -99,7 +87,7 @@ def walk_modules(path: str) -> List[ModuleType]: For example: walk_modules('scrapy.utils') """ - mods: List[ModuleType] = [] + mods: list[ModuleType] = [] mod = import_module(path) mods.append(mod) if hasattr(mod, "__path__"): @@ -186,7 +174,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): def build_from_crawler( - objcls: Type[T], crawler: Crawler, /, *args: Any, **kwargs: Any + objcls: type[T], crawler: Crawler, /, *args: Any, **kwargs: Any ) -> T: """Construct a class instance using its ``from_crawler`` constructor. @@ -209,7 +197,7 @@ def build_from_crawler( def build_from_settings( - objcls: Type[T], settings: BaseSettings, /, *args: Any, **kwargs: Any + objcls: type[T], settings: BaseSettings, /, *args: Any, **kwargs: Any ) -> T: """Construct a class instance using its ``from_settings`` constructor. @@ -250,7 +238,7 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]: """Similar to ``ast.walk``, but walks only function body and skips nested functions defined within the node. """ - todo: Deque[ast.AST] = deque([node]) + todo: deque[ast.AST] = deque([node]) walked_func_def = False while todo: node = todo.popleft() diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index 5985a847e..cff5eb629 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,13 +1,14 @@ import signal +from collections.abc import Callable from types import FrameType -from typing import Any, Callable, Dict, Optional, Union +from typing import Any, Optional, Union # copy of _HANDLER from typeshed/stdlib/signal.pyi SignalHandlerT = Union[ Callable[[int, Optional[FrameType]], Any], int, signal.Handlers, None ] -signal_names: Dict[int, str] = {} +signal_names: dict[int, str] = {} for signame in dir(signal): if signame.startswith("SIG") and not signame.startswith("SIG_"): signum = getattr(signal, signame) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index efb6af299..c9e5eb857 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import os import warnings from importlib import import_module diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index f56950fdd..91c5d67f5 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -4,36 +4,22 @@ This module contains essential stuff that should've come with Python itself ;) from __future__ import annotations -import collections.abc import gc import inspect import re import sys import weakref +from collections.abc import AsyncIterable, Iterable, Mapping from functools import partial, wraps from itertools import chain -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - AsyncIterator, - Callable, - Dict, - Iterable, - Iterator, - List, - Mapping, - Optional, - Pattern, - Tuple, - TypeVar, - Union, - overload, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, overload from scrapy.utils.asyncgen import as_async_generator if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable, Iterator + from re import Pattern + # typing.Concatenate and typing.ParamSpec require Python 3.10 from typing_extensions import Concatenate, ParamSpec @@ -44,7 +30,7 @@ _KT = TypeVar("_KT") _VT = TypeVar("_VT") -def flatten(x: Iterable[Any]) -> List[Any]: +def flatten(x: Iterable[Any]) -> list[Any]: """flatten(sequence) -> list Returns a single, flat list which contains all elements retrieved @@ -99,10 +85,10 @@ def is_listlike(x: Any) -> bool: return hasattr(x, "__iter__") and not isinstance(x, (str, bytes)) -def unique(list_: Iterable[_T], key: Callable[[_T], Any] = lambda x: x) -> List[_T]: +def unique(list_: Iterable[_T], key: Callable[[_T], Any] = lambda x: x) -> list[_T]: """efficient function to uniquify a list preserving item order""" seen = set() - result: List[_T] = [] + result: list[_T] = [] for item in list_: seenkey = key(item) if seenkey in seen: @@ -147,7 +133,7 @@ def to_bytes( def re_rsearch( pattern: Union[str, Pattern[str]], text: str, chunk_size: int = 1024 -) -> Optional[Tuple[int, int]]: +) -> Optional[tuple[int, int]]: """ This function does a reverse search in a text using a regular expression given in the attribute 'pattern'. @@ -161,7 +147,7 @@ def re_rsearch( the start position of the match, and the ending (regarding the entire text). """ - def _chunk_iter() -> Iterable[Tuple[str, int]]: + def _chunk_iter() -> Iterable[tuple[str, int]]: offset = len(text) while True: offset -= chunk_size * 1024 @@ -215,12 +201,12 @@ def binary_is_text(data: bytes) -> bool: return all(c not in _BINARYCHARS for c in data) -def get_func_args(func: Callable[..., Any], stripself: bool = False) -> List[str]: +def get_func_args(func: Callable[..., Any], stripself: bool = False) -> list[str]: """Return the argument name list of a callable object""" if not callable(func): raise TypeError(f"func must be callable, got '{type(func).__name__}'") - args: List[str] = [] + args: list[str] = [] try: sig = inspect.signature(func) except ValueError: @@ -245,7 +231,7 @@ def get_func_args(func: Callable[..., Any], stripself: bool = False) -> List[str return args -def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]: +def get_spec(func: Callable[..., Any]) -> tuple[list[str], dict[str, Any]]: """Returns (args, kwargs) tuple for a function >>> import re >>> get_spec(re.match) @@ -274,7 +260,7 @@ def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]: else: raise TypeError(f"{type(func)} is not callable") - defaults: Tuple[Any, ...] = spec.defaults or () + defaults: tuple[Any, ...] = spec.defaults or () firstdefault = len(spec.args) - len(defaults) args = spec.args[:firstdefault] @@ -283,7 +269,7 @@ def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]: def equal_attributes( - obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable[[Any], Any]]]] + obj1: Any, obj2: Any, attributes: Optional[list[Union[str, Callable[[Any], Any]]]] ) -> bool: """Compare two objects attributes""" # not attributes given return False by default @@ -303,7 +289,7 @@ def equal_attributes( @overload -def without_none_values(iterable: Mapping[_KT, _VT]) -> Dict[_KT, _VT]: ... +def without_none_values(iterable: Mapping[_KT, _VT]) -> dict[_KT, _VT]: ... @overload @@ -312,13 +298,13 @@ def without_none_values(iterable: Iterable[_KT]) -> Iterable[_KT]: ... def without_none_values( iterable: Union[Mapping[_KT, _VT], Iterable[_KT]] -) -> Union[Dict[_KT, _VT], Iterable[_KT]]: +) -> Union[dict[_KT, _VT], Iterable[_KT]]: """Return a copy of ``iterable`` with all ``None`` entries removed. If ``iterable`` is a mapping, return a dictionary where all pairs that have value ``None`` have been removed. """ - if isinstance(iterable, collections.abc.Mapping): + if isinstance(iterable, Mapping): return {k: v for k, v in iterable.items() if v is not None} else: # the iterable __init__ must take another iterable diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index a627db601..ed2fb5959 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -3,18 +3,7 @@ from __future__ import annotations import asyncio import sys from contextlib import suppress -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Generic, - List, - Optional, - Tuple, - Type, - TypeVar, -) +from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error @@ -25,6 +14,7 @@ from scrapy.utils.misc import load_object if TYPE_CHECKING: from asyncio import AbstractEventLoop, AbstractEventLoopPolicy + from collections.abc import Callable from twisted.internet.protocol import ServerFactory from twisted.internet.tcp import Port @@ -37,7 +27,7 @@ if TYPE_CHECKING: _T = TypeVar("_T") -def listen_tcp(portrange: List[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] +def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor @@ -62,8 +52,8 @@ class CallLaterOnce(Generic[_T]): def __init__(self, func: Callable[_P, _T], *a: _P.args, **kw: _P.kwargs): self._func: Callable[_P, _T] = func - self._a: Tuple[Any, ...] = a - self._kw: Dict[str, Any] = kw + self._a: tuple[Any, ...] = a + self._kw: dict[str, Any] = kw self._call: Optional[DelayedCall] = None def schedule(self, delay: float = 0) -> None: @@ -142,7 +132,7 @@ def _get_asyncio_event_loop() -> AbstractEventLoop: def set_asyncio_event_loop(event_loop_path: Optional[str]) -> AbstractEventLoop: """Sets and returns the event loop with specified import path.""" if event_loop_path is not None: - event_loop_class: Type[AbstractEventLoop] = load_object(event_loop_path) + event_loop_class: type[AbstractEventLoop] = load_object(event_loop_path) event_loop = event_loop_class() asyncio.set_event_loop(event_loop) else: diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 99ca3b7a0..052a3721a 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -8,18 +8,7 @@ from __future__ import annotations import hashlib import json import warnings -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterable, - List, - Optional, - Protocol, - Tuple, - Type, - Union, -) +from typing import TYPE_CHECKING, Any, Optional, Protocol, Union from urllib.parse import urlunparse from weakref import WeakKeyDictionary @@ -33,6 +22,8 @@ from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -47,7 +38,7 @@ def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[b _fingerprint_cache: WeakKeyDictionary[ - Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes] + Request, dict[tuple[Optional[tuple[bytes, ...]], bool], bytes] ] _fingerprint_cache = WeakKeyDictionary() @@ -88,7 +79,7 @@ def fingerprint( If you want to include them, set the keep_fragments argument to True (for instance when handling requests with a headless browser). """ - processed_include_headers: Optional[Tuple[bytes, ...]] = None + processed_include_headers: Optional[tuple[bytes, ...]] = None if include_headers: processed_include_headers = tuple( to_bytes(h.lower()) for h in sorted(include_headers) @@ -98,7 +89,7 @@ def fingerprint( if cache_key not in cache: # To decode bytes reliably (JSON does not support bytes), regardless of # character encoding, we use bytes.hex() - headers: Dict[str, List[str]] = {} + headers: dict[str, list[str]] = {} if processed_include_headers: for header in processed_include_headers: if header in request.headers: @@ -194,13 +185,13 @@ def referer_str(request: Request) -> Optional[str]: return to_unicode(referrer, errors="replace") -def request_from_dict(d: Dict[str, Any], *, spider: Optional[Spider] = None) -> Request: +def request_from_dict(d: dict[str, Any], *, spider: Optional[Spider] = None) -> Request: """Create a :class:`~scrapy.Request` object from a dict. If a spider is given, it will try to resolve the callbacks looking at the spider for methods with the same name. """ - request_cls: Type[Request] = load_object(d["_class"]) if "_class" in d else Request + request_cls: type[Request] = load_object(d["_class"]) if "_class" in d else Request kwargs = {key: value for key, value in d.items() if key in request_cls.attributes} if d.get("callback") and spider: kwargs["callback"] = _get_method(spider, d["callback"]) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 320059b3a..0ca9d07a4 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -9,7 +9,7 @@ import os import re import tempfile import webbrowser -from typing import TYPE_CHECKING, Any, Callable, Iterable, Tuple, Union +from typing import TYPE_CHECKING, Any, Union from weakref import WeakKeyDictionary from twisted.web import http @@ -18,6 +18,8 @@ from w3lib import html from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: + from collections.abc import Callable, Iterable + from scrapy.http import Response, TextResponse _baseurl_cache: WeakKeyDictionary[Response, str] = WeakKeyDictionary() @@ -34,14 +36,14 @@ def get_base_url(response: TextResponse) -> str: _metaref_cache: WeakKeyDictionary[ - Response, Union[Tuple[None, None], Tuple[float, str]] + Response, Union[tuple[None, None], tuple[float, str]] ] = WeakKeyDictionary() def get_meta_refresh( response: TextResponse, ignore_tags: Iterable[str] = ("script", "noscript"), -) -> Union[Tuple[None, None], Tuple[float, str]]: +) -> Union[tuple[None, None], tuple[float, str]]: """Parse the http-equiv refresh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 4310c1d56..c1d3bfffb 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -2,10 +2,9 @@ from __future__ import annotations -import collections.abc import logging +from collections.abc import Sequence from typing import Any as TypingAny -from typing import List, Tuple from pydispatch.dispatcher import ( Anonymous, @@ -30,19 +29,15 @@ def send_catch_log( sender: TypingAny = Anonymous, *arguments: TypingAny, **named: TypingAny, -) -> List[Tuple[TypingAny, TypingAny]]: +) -> list[tuple[TypingAny, TypingAny]]: """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. """ dont_log = named.pop("dont_log", ()) - dont_log = ( - tuple(dont_log) - if isinstance(dont_log, collections.abc.Sequence) - else (dont_log,) - ) + dont_log = tuple(dont_log) if isinstance(dont_log, Sequence) else (dont_log,) dont_log += (StopDownload,) spider = named.get("spider", None) - responses: List[Tuple[TypingAny, TypingAny]] = [] + responses: list[tuple[TypingAny, TypingAny]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): result: TypingAny try: @@ -76,7 +71,7 @@ def send_catch_log_deferred( sender: TypingAny = Anonymous, *arguments: TypingAny, **named: TypingAny, -) -> Deferred[List[Tuple[TypingAny, TypingAny]]]: +) -> Deferred[list[tuple[TypingAny, TypingAny]]]: """Like send_catch_log but supports returning deferreds on signal handlers. Returns a deferred that gets fired once all signal handlers deferreds were fired. @@ -94,14 +89,14 @@ def send_catch_log_deferred( dont_log = named.pop("dont_log", None) spider = named.get("spider", None) - dfds: List[Deferred[Tuple[TypingAny, TypingAny]]] = [] + dfds: list[Deferred[tuple[TypingAny, TypingAny]]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): d: Deferred[TypingAny] = maybeDeferred_coro( robustApply, receiver, signal=signal, sender=sender, *arguments, **named ) d.addErrback(logerror, receiver) # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html - d2: Deferred[Tuple[TypingAny, TypingAny]] = d.addBoth( + d2: Deferred[tuple[TypingAny, TypingAny]] = d.addBoth( lambda result: ( receiver, # pylint: disable=cell-var-from-loop # noqa: B023 result, @@ -109,7 +104,7 @@ def send_catch_log_deferred( ) dfds.append(d2) dl = DeferredList(dfds) - d3: Deferred[List[Tuple[TypingAny, TypingAny]]] = dl.addCallback( + d3: Deferred[list[tuple[TypingAny, TypingAny]]] = dl.addCallback( lambda out: [x[1] for x in out] ) return d3 diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 7a91afe59..1f70fcf69 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -5,11 +5,16 @@ Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ -from typing import Any, Dict, Iterable, Iterator, Optional, Union +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional, Union from urllib.parse import urljoin import lxml.etree # nosec +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + class Sitemap: """Class to parse Sitemap (type=urlset) and Sitemap Index @@ -23,9 +28,9 @@ class Sitemap: rt = self._root.tag self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt - def __iter__(self) -> Iterator[Dict[str, Any]]: + def __iter__(self) -> Iterator[dict[str, Any]]: for elem in self._root.getchildren(): - d: Dict[str, Any] = {} + d: dict[str, Any] = {} for el in elem.getchildren(): tag = el.tag name = tag.split("}", 1)[1] if "}" in tag else tag diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index ce754fad3..02dbb2e90 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -2,24 +2,14 @@ from __future__ import annotations import inspect import logging -from typing import ( - TYPE_CHECKING, - Any, - AsyncGenerator, - Iterable, - Literal, - Optional, - Type, - TypeVar, - Union, - overload, -) +from typing import TYPE_CHECKING, Any, Literal, Optional, TypeVar, Union, overload from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Iterable from types import CoroutineType, ModuleType from twisted.internet.defer import Deferred @@ -58,7 +48,7 @@ def iterate_spider_output( return arg_to_iter(deferred_from_coro(result)) -def iter_spider_classes(module: ModuleType) -> Iterable[Type[Spider]]: +def iter_spider_classes(module: ModuleType) -> Iterable[type[Spider]]: """Return an iterator over all spider classes defined in the given module that can be instantiated (i.e. which have name) """ @@ -80,10 +70,10 @@ def iter_spider_classes(module: ModuleType) -> Iterable[Type[Spider]]: def spidercls_for_request( spider_loader: SpiderLoader, request: Request, - default_spidercls: Type[Spider], + default_spidercls: type[Spider], log_none: bool = ..., log_multiple: bool = ..., -) -> Type[Spider]: ... +) -> type[Spider]: ... @overload @@ -93,7 +83,7 @@ def spidercls_for_request( default_spidercls: Literal[None], log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: ... +) -> Optional[type[Spider]]: ... @overload @@ -103,16 +93,16 @@ def spidercls_for_request( *, log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: ... +) -> Optional[type[Spider]]: ... def spidercls_for_request( spider_loader: SpiderLoader, request: Request, - default_spidercls: Optional[Type[Spider]] = None, + default_spidercls: Optional[type[Spider]] = None, log_none: bool = False, log_multiple: bool = False, -) -> Optional[Type[Spider]]: +) -> Optional[type[Spider]]: """Return a spider class that handles the given Request. This will look for the spiders that can handle the given request (using diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 30f235592..860a2e3dd 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -9,17 +9,7 @@ import os from importlib import import_module from pathlib import Path from posixpath import split -from typing import ( - TYPE_CHECKING, - Any, - Awaitable, - Dict, - List, - Optional, - Tuple, - Type, - TypeVar, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar from unittest import TestCase, mock from twisted.trial.unittest import SkipTest @@ -29,6 +19,8 @@ from scrapy.crawler import Crawler from scrapy.utils.boto import is_botocore_available if TYPE_CHECKING: + from collections.abc import Awaitable + from twisted.internet.defer import Deferred from twisted.web.client import Response as TxResponse @@ -48,7 +40,7 @@ def skip_if_no_boto() -> None: def get_gcs_content_and_delete( bucket: Any, path: str -) -> Tuple[bytes, List[Dict[str, str]], Any]: +) -> tuple[bytes, list[dict[str, str]], Any]: from google.cloud import storage client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) @@ -75,7 +67,7 @@ def get_ftp_content_and_delete( ftp.login(username, password) if use_active_mode: ftp.set_pasv(False) - ftp_data: List[bytes] = [] + ftp_data: list[bytes] = [] def buffer_data(data: bytes) -> None: ftp_data.append(data) @@ -92,8 +84,8 @@ class TestSpider(Spider): def get_crawler( - spidercls: Optional[Type[Spider]] = None, - settings_dict: Optional[Dict[str, Any]] = None, + spidercls: Optional[type[Spider]] = None, + settings_dict: Optional[dict[str, Any]] = None, prevent_warnings: bool = True, ) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it @@ -103,7 +95,7 @@ def get_crawler( from scrapy.crawler import CrawlerRunner # Set by default settings that prevent deprecation warnings. - settings: Dict[str, Any] = {} + settings: dict[str, Any] = {} settings.update(settings_dict or {}) runner = CrawlerRunner(settings) crawler = runner.create_crawler(spidercls or TestSpider) @@ -118,7 +110,7 @@ def get_pythonpath() -> str: return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get("PYTHONPATH", "") -def get_testenv() -> Dict[str, str]: +def get_testenv() -> dict[str, str]: """Return a OS environment dict suitable to fork processes that need to import this installation of Scrapy, instead of a system installed one. """ @@ -143,7 +135,7 @@ def get_from_asyncio_queue(value: _T) -> Awaitable[_T]: return getter -def mock_google_cloud_storage() -> Tuple[Any, Any, Any]: +def mock_google_cloud_storage() -> tuple[Any, Any, Any]: """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob classes and set their proper return values. """ diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index bb269a9f5..dfc823725 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -2,13 +2,15 @@ from __future__ import annotations import os import sys -from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, cast +from typing import TYPE_CHECKING, Optional, cast from twisted.internet.defer import Deferred from twisted.internet.error import ProcessTerminated from twisted.internet.protocol import ProcessProtocol if TYPE_CHECKING: + from collections.abc import Iterable + from twisted.python.failure import Failure @@ -36,8 +38,8 @@ class ProcessTest: return pp.deferred def _process_finished( - self, pp: TestProcessProtocol, cmd: List[str], check_code: bool - ) -> Tuple[int, bytes, bytes]: + self, pp: TestProcessProtocol, cmd: list[str], check_code: bool + ) -> tuple[int, bytes, bytes]: if pp.exitcode and check_code: msg = f"process {cmd} exit with code {pp.exitcode}" msg += f"\n>>> stdout <<<\n{pp.out.decode()}" diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 9ff9a273f..5eec1c10f 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -9,19 +9,23 @@ and no performance penalty at all when disabled (as object_ref becomes just an alias to object in that case). """ +from __future__ import annotations + from collections import defaultdict from operator import itemgetter from time import time -from typing import TYPE_CHECKING, Any, DefaultDict, Iterable +from typing import TYPE_CHECKING, Any from weakref import WeakKeyDictionary if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self NoneType = type(None) -live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) +live_refs: defaultdict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) class object_ref: @@ -29,7 +33,7 @@ class object_ref: __slots__ = () - def __new__(cls, *args: Any, **kwargs: Any) -> "Self": + def __new__(cls, *args: Any, **kwargs: Any) -> Self: obj = object.__new__(cls) live_refs[cls][obj] = time() return obj diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 9d97cb12f..41d268baa 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -6,8 +6,10 @@ Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ +from __future__ import annotations + import re -from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast +from typing import TYPE_CHECKING, Optional, Union, cast from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this @@ -18,6 +20,8 @@ from w3lib.url import _safe_chars, _unquotepath # noqa: F401 from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from collections.abc import Iterable + from scrapy import Spider @@ -33,7 +37,7 @@ def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: return any((host == d) or (host.endswith(f".{d}")) for d in domains) -def url_is_from_spider(url: UrlT, spider: Type["Spider"]) -> bool: +def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool: """Return True if the url belongs to the given spider""" return url_is_from_any_domain( url, [spider.name] + list(getattr(spider, "allowed_domains", [])) diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index 42e5e9be4..4e9e29286 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -1,6 +1,5 @@ import platform import sys -from typing import List, Tuple import cryptography import cssselect @@ -13,7 +12,7 @@ import scrapy from scrapy.utils.ssl import get_openssl_version -def scrapy_components_versions() -> List[Tuple[str, str]]: +def scrapy_components_versions() -> list[tuple[str, str]]: lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) diff --git a/setup.py b/setup.py index f458a9de3..ec9ac6597 100644 --- a/setup.py +++ b/setup.py @@ -6,12 +6,12 @@ version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() install_requires = [ - "Twisted>=18.9.0", - "cryptography>=36.0.0", + "Twisted>=21.7.0", + "cryptography>=37.0.0", "cssselect>=0.9.1", "itemloaders>=1.0.1", "parsel>=1.5.0", - "pyOpenSSL>=21.0.0", + "pyOpenSSL>=22.0.0", "queuelib>=1.4.2", "service_identity>=18.1.0", "w3lib>=1.17.0", @@ -20,7 +20,7 @@ install_requires = [ "itemadapter>=0.1.0", "packaging", "tldextract", - "lxml>=4.4.1", + "lxml>=4.6.0", "defusedxml>=0.7.1", ] extras_require = { @@ -58,7 +58,6 @@ setup( "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -69,7 +68,7 @@ setup( "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules", ], - python_requires=">=3.8", + python_requires=">=3.9", install_requires=install_requires, extras_require=extras_require, ) diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index bde3de228..a7f7f1356 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -1,7 +1,7 @@ """DBM-like dummy module""" -import collections -from typing import Any, DefaultDict +from collections import defaultdict +from typing import Any class DummyDB(dict): @@ -14,7 +14,7 @@ class DummyDB(dict): error = KeyError -_DATABASES: DefaultDict[Any, DummyDB] = collections.defaultdict(DummyDB) +_DATABASES: defaultdict[Any, DummyDB] = defaultdict(DummyDB) def open(file, flag="r", mode=0o666): diff --git a/tests/mockserver.py b/tests/mockserver.py index 6ec46aa3d..f5c12787a 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -9,7 +9,7 @@ from pathlib import Path from shutil import rmtree from subprocess import PIPE, Popen from tempfile import mkdtemp -from typing import TYPE_CHECKING, Dict +from typing import TYPE_CHECKING from urllib.parse import urlencode from OpenSSL import SSL @@ -37,7 +37,7 @@ def getarg(request, name, default=None, type=None): return default -def get_mockserver_env() -> Dict[str, str]: +def get_mockserver_env() -> dict[str, str]: """Return a OS environment dict suitable to run mockserver processes.""" tests_path = Path(__file__).parent.parent diff --git a/tests/test_addons.py b/tests/test_addons.py index f1b01bc5c..775f629b3 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,5 +1,5 @@ import itertools -from typing import Any, Dict +from typing import Any from unittest.mock import patch from twisted.internet.defer import inlineCallbacks @@ -17,7 +17,7 @@ class SimpleAddon: pass -def get_addon_cls(config: Dict[str, Any]) -> type: +def get_addon_cls(config: dict[str, Any]) -> type: class AddonWithConfig: def update_settings(self, settings: BaseSettings): settings.update(config, priority="addon") diff --git a/tests/test_commands.py b/tests/test_commands.py index a23b7f4a9..6ec7c21b0 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import inspect import json @@ -13,7 +15,7 @@ from shutil import copytree, rmtree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from tempfile import TemporaryFile, mkdtemp from threading import Timer -from typing import Dict, Iterator, Optional, Union +from typing import TYPE_CHECKING, Optional, Union from unittest import skipIf from pytest import mark @@ -27,6 +29,9 @@ from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv from tests.test_crawler import ExceptionSpider, NoRequestsSpider +if TYPE_CHECKING: + from collections.abc import Iterator + class CommandSettings(unittest.TestCase): def setUp(self): @@ -194,7 +199,7 @@ class StartprojectTest(ProjectTest): def get_permissions_dict( path: Union[str, os.PathLike], renamings=None, ignore=None -) -> Dict[str, str]: +) -> dict[str, str]: def get_permissions(path: Path) -> str: return oct(path.stat().st_mode) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index c87e65758..69bfb7eb3 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -6,7 +6,6 @@ import subprocess import sys import warnings from pathlib import Path -from typing import List import pytest from packaging.version import parse as parse_version @@ -651,7 +650,7 @@ class ScriptRunnerMixin: script_dir: Path cwd = os.getcwd() - def get_script_args(self, script_name: str, *script_args: str) -> List[str]: + def get_script_args(self, script_name: str, *script_args: str) -> list[str]: script_path = self.script_dir / script_name return [sys.executable, str(script_path)] + list(script_args) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 884491d01..f14a10a32 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -4,7 +4,7 @@ import shutil import sys from pathlib import Path from tempfile import mkdtemp, mkstemp -from typing import Optional, Type +from typing import Optional from unittest import SkipTest, mock from testfixtures import LogCapture @@ -218,7 +218,7 @@ class DuplicateHeaderResource(resource.Resource): class HttpTestCase(unittest.TestCase): scheme = "http" - download_handler_cls: Type = HTTPDownloadHandler + download_handler_cls: type = HTTPDownloadHandler # only used for HTTPS tests keyfile = "keys/localhost.key" @@ -428,7 +428,7 @@ class HttpTestCase(unittest.TestCase): class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" - download_handler_cls: Type = HTTP10DownloadHandler + download_handler_cls: type = HTTP10DownloadHandler def test_protocol(self): request = Request(self.getURL("host"), method="GET") @@ -445,7 +445,7 @@ class Https10TestCase(Http10TestCase): class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" - download_handler_cls: Type = HTTP11DownloadHandler + download_handler_cls: type = HTTP11DownloadHandler def test_download_without_maxsize_limit(self): request = Request(self.getURL("file")) @@ -645,7 +645,7 @@ class Https11InvalidDNSPattern(Https11TestCase): class Https11CustomCiphers(unittest.TestCase): scheme = "https" - download_handler_cls: Type = HTTP11DownloadHandler + download_handler_cls: type = HTTP11DownloadHandler keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" @@ -740,7 +740,7 @@ class UriResource(resource.Resource): class HttpProxyTestCase(unittest.TestCase): - download_handler_cls: Type = HTTPDownloadHandler + download_handler_cls: type = HTTPDownloadHandler expected_http_proxy_request_body = b"http://example.com" def setUp(self): @@ -783,14 +783,14 @@ class HttpProxyTestCase(unittest.TestCase): class Http10ProxyTestCase(HttpProxyTestCase): - download_handler_cls: Type = HTTP10DownloadHandler + download_handler_cls: type = HTTP10DownloadHandler def test_download_with_proxy_https_noconnect(self): raise unittest.SkipTest("noconnect is not supported in HTTP10DownloadHandler") class Http11ProxyTestCase(HttpProxyTestCase): - download_handler_cls: Type = HTTP11DownloadHandler + download_handler_cls: type = HTTP11DownloadHandler @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): @@ -845,7 +845,7 @@ class S3AnonTestCase(unittest.TestCase): class S3TestCase(unittest.TestCase): - download_handler_cls: Type = S3DownloadHandler + download_handler_cls: type = S3DownloadHandler # test use same example keys than amazon developer guide # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 7ea3fe8c9..1f998de1a 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -8,7 +8,7 @@ import string from ipaddress import IPv4Address from pathlib import Path from tempfile import mkdtemp -from typing import TYPE_CHECKING, Dict +from typing import TYPE_CHECKING from unittest import mock, skipIf from urllib.parse import urlencode @@ -152,7 +152,7 @@ class QueryParams(LeafResource): request.setHeader("Content-Type", "application/json; charset=UTF-8") request.setHeader("Content-Encoding", "UTF-8") - query_params: Dict[str, str] = {} + query_params: dict[str, str] = {} assert request.args is not None for k, v in request.args.items(): query_params[str(k, "utf-8")] = str(v[0], "utf-8") diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 7ce73e6ff..d0fb17f1f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -3,7 +3,7 @@ import re import unittest import warnings import xmlrpc.client -from typing import Any, Dict, List +from typing import Any from unittest import mock from urllib.parse import parse_qs, unquote_to_bytes @@ -23,8 +23,8 @@ from scrapy.utils.python import to_bytes, to_unicode class RequestTest(unittest.TestCase): request_class = Request default_method = "GET" - default_headers: Dict[bytes, List[bytes]] = {} - default_meta: Dict[str, Any] = {} + default_headers: dict[bytes, list[bytes]] = {} + default_meta: dict[str, Any] = {} def test_init(self): # Request requires url in the __init__ method diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index cd3442dd4..83e22b070 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,7 +1,7 @@ import shutil from pathlib import Path from tempfile import mkdtemp -from typing import Optional, Set +from typing import Optional from testfixtures import LogCapture from twisted.internet import defer @@ -57,7 +57,7 @@ class FileDownloadCrawlTestCase(TestCase): store_setting_key = "FILES_STORE" media_key = "files" media_urls_key = "file_urls" - expected_checksums: Optional[Set[str]] = { + expected_checksums: Optional[set[str]] = { "5547178b89448faf0015a13f904c936e", "c2281c83670e31d8aaab7cb642b824db", "ed3f6538dc15d4d9179dae57319edc5f", diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 0babde4d9..6ce7fc059 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -7,7 +7,6 @@ from io import BytesIO from pathlib import Path from shutil import rmtree from tempfile import mkdtemp -from typing import Dict, List from unittest import mock from urllib.parse import urlparse @@ -309,11 +308,11 @@ class FilesPipelineTestCaseFieldsDataClass( class FilesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - file_urls: List[str] = attr.ib(default=lambda: []) - files: List[Dict[str, str]] = attr.ib(default=lambda: []) + file_urls: list[str] = attr.ib(default=lambda: []) + files: list[dict[str, str]] = attr.ib(default=lambda: []) # overridden fields - custom_file_urls: List[str] = attr.ib(default=lambda: []) - custom_files: List[Dict[str, str]] = attr.ib(default=lambda: []) + custom_file_urls: list[str] = attr.ib(default=lambda: []) + custom_files: list[dict[str, str]] = attr.ib(default=lambda: []) class FilesPipelineTestCaseFieldsAttrsItem( diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 7d7c78920..296a6fae0 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -5,7 +5,7 @@ import random import warnings from shutil import rmtree from tempfile import mkdtemp -from typing import Dict, List, Optional +from typing import Optional from unittest.mock import patch import attr @@ -406,11 +406,11 @@ class ImagesPipelineTestCaseFieldsDataClass( class ImagesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - image_urls: List[str] = attr.ib(default=lambda: []) - images: List[Dict[str, str]] = attr.ib(default=lambda: []) + image_urls: list[str] = attr.ib(default=lambda: []) + images: list[dict[str, str]] = attr.ib(default=lambda: []) # overridden fields - custom_image_urls: List[str] = attr.ib(default=lambda: []) - custom_images: List[Dict[str, str]] = attr.ib(default=lambda: []) + custom_image_urls: list[str] = attr.ib(default=lambda: []) + custom_images: list[dict[str, str]] = attr.ib(default=lambda: []) class ImagesPipelineTestCaseFieldsAttrsItem( diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 7299972f6..8c0e5764a 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -1,5 +1,3 @@ -from typing import List - from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -64,7 +62,7 @@ class KeywordArgumentsSpider(MockServerSpider): }, } - checks: List[bool] = [] + checks: list[bool] = [] def start_requests(self): data = {"key": "value", "number": 123, "callback": "some_callback"} diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 5db2e4e50..4fd293ec7 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional +from typing import Optional from unittest import TestCase from urllib.parse import urljoin @@ -20,7 +20,7 @@ URLS = [urljoin("https://example.org", p) for p in PATHS] class MinimalScheduler: def __init__(self) -> None: - self.requests: Dict[bytes, Request] = {} + self.requests: dict[bytes, Request] = {} def has_pending_requests(self) -> bool: return bool(self.requests) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 9ee248538..503c29e32 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -105,9 +105,10 @@ class BaseSettingsTest(unittest.TestCase): def test_set_calls_settings_attributes_methods_on_update(self): attr = SettingsAttribute("value", 10) - with mock.patch.object(attr, "__setattr__") as mock_setattr, mock.patch.object( - attr, "set" - ) as mock_set: + with ( + mock.patch.object(attr, "__setattr__") as mock_setattr, + mock.patch.object(attr, "set") as mock_set, + ): self.settings.attributes = {"TEST_OPTION": attr} for priority in (0, 10, 20): diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 9dbffe353..41228b5f2 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,4 +1,4 @@ -import collections.abc +from collections.abc import AsyncIterator, Iterable from typing import Optional, Union from unittest import mock @@ -147,7 +147,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): result = yield self._get_middleware_result( *mw_classes, start_index=start_index ) - self.assertIsInstance(result, collections.abc.Iterable) + self.assertIsInstance(result, Iterable) result_list = list(result) self.assertEqual(len(result_list), self.RESULT_COUNT) self.assertIsInstance(result_list[0], self.ITEM_TYPE) @@ -161,7 +161,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): result = yield self._get_middleware_result( *mw_classes, start_index=start_index ) - self.assertIsInstance(result, collections.abc.AsyncIterator) + self.assertIsInstance(result, AsyncIterator) result_list = yield deferred_from_coro(collect_asyncgen(result)) self.assertEqual(len(result_list), self.RESULT_COUNT) self.assertIsInstance(result_list[0], self.ITEM_TYPE) diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 044455415..01a2b4bb4 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,5 +1,4 @@ import logging -from typing import Set from unittest import TestCase from testfixtures import LogCapture @@ -17,7 +16,7 @@ from tests.spiders import MockServerSpider class _HttpErrorSpider(MockServerSpider): name = "httperror" - bypass_status_codes: Set[int] = set() + bypass_status_codes: set[int] = set() def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 5797edfbd..e73e7ff4c 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,5 +1,5 @@ import warnings -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Optional from unittest import TestCase from urllib.parse import urlparse @@ -32,10 +32,10 @@ from scrapy.spiders import Spider class TestRefererMiddleware(TestCase): - req_meta: Dict[str, Any] = {} - resp_headers: Dict[str, str] = {} - settings: Dict[str, Any] = {} - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + req_meta: dict[str, Any] = {} + resp_headers: dict[str, str] = {} + settings: dict[str, Any] = {} + scenarii: list[tuple[str, str, Optional[bytes]]] = [ ("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"), ] @@ -65,7 +65,7 @@ class MixinDefault: with some additional filtering of s3:// """ - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ ("https://example.com/", "https://scrapy.org/", b"https://example.com/"), ("http://example.com/", "http://scrapy.org/", b"http://example.com/"), ("http://example.com/", "https://scrapy.org/", b"http://example.com/"), @@ -86,7 +86,7 @@ class MixinDefault: class MixinNoReferrer: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ ("https://example.com/page.html", "https://example.com/", None), ("http://www.example.com/", "https://scrapy.org/", None), ("http://www.example.com/", "http://scrapy.org/", None), @@ -96,7 +96,7 @@ class MixinNoReferrer: class MixinNoReferrerWhenDowngrade: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # TLS to TLS: send non-empty referrer ( "https://example.com/page.html", @@ -178,7 +178,7 @@ class MixinNoReferrerWhenDowngrade: class MixinSameOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -247,7 +247,7 @@ class MixinSameOrigin: class MixinOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) ( "https://example.com/page.html", @@ -271,7 +271,7 @@ class MixinOrigin: class MixinStrictOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades ( "https://example.com/page.html", @@ -299,7 +299,7 @@ class MixinStrictOrigin: class MixinOriginWhenCrossOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -406,7 +406,7 @@ class MixinOriginWhenCrossOrigin: class MixinStrictOriginWhenCrossOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -518,7 +518,7 @@ class MixinStrictOriginWhenCrossOrigin: class MixinUnsafeUrl: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # TLS to TLS: send referrer ( "https://example.com/sekrit.html", @@ -968,8 +968,8 @@ class TestPolicyHeaderPrecedence004( class TestReferrerOnRedirect(TestRefererMiddleware): settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"} - scenarii: List[ - Tuple[str, str, Tuple[Tuple[int, str], ...], Optional[bytes], Optional[bytes]] + scenarii: list[ + tuple[str, str, tuple[tuple[int, str], ...], Optional[bytes], Optional[bytes]] ] = [ # type: ignore[assignment] ( "http://scrapytest.org/1", # parent diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index be5c6de81..fb7c90f80 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,8 +1,7 @@ import copy import unittest import warnings -from collections.abc import Mapping, MutableMapping -from typing import Iterator +from collections.abc import Iterator, Mapping, MutableMapping from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 0f75bdb5c..76820eabf 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,10 +1,12 @@ +from __future__ import annotations + import json import logging import re import sys import unittest from io import StringIO -from typing import Any, Dict, Mapping, MutableMapping +from typing import TYPE_CHECKING, Any from unittest import TestCase import pytest @@ -21,6 +23,9 @@ from scrapy.utils.log import ( from scrapy.utils.test import get_crawler from tests.spiders import LogSpider +if TYPE_CHECKING: + from collections.abc import Mapping, MutableMapping + class FailureToExcInfoTest(unittest.TestCase): def test_failure(self): @@ -133,7 +138,7 @@ class StreamLoggerTest(unittest.TestCase): ), ) def test_spider_logger_adapter_process( - base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: Dict + base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict ): logger = logging.getLogger("test") spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 633077eec..ca3bca0b2 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -2,7 +2,7 @@ import json import unittest import warnings from hashlib import sha1 -from typing import Dict, Optional, Tuple, Union +from typing import Optional, Union from weakref import WeakKeyDictionary from scrapy.http import Request @@ -57,11 +57,11 @@ class FingerprintTest(unittest.TestCase): function: staticmethod = staticmethod(fingerprint) cache: Union[ - "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]", - "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]", + "WeakKeyDictionary[Request, dict[tuple[Optional[tuple[bytes, ...]], bool], bytes]]", + "WeakKeyDictionary[Request, dict[tuple[Optional[tuple[bytes, ...]], bool], str]]", ] = _fingerprint_cache default_cache_key = (None, False) - known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = ( + known_hashes: tuple[tuple[Request, Union[bytes, str], dict], ...] = ( ( Request("http://example.org"), b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD", diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing index 665db9088..3926c830f 100644 --- a/tests_typing/test_http_request.mypy-testing +++ b/tests_typing/test_http_request.mypy-testing @@ -16,7 +16,7 @@ class MyRequest2(Request): @pytest.mark.mypy_testing def mypy_test_headers(): - Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]" + Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[tuple[str, Any]], None]" Request("data:,", headers=None) Request("data:,", headers={}) Request("data:,", headers=[]) diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing index d58ac1027..88aedbd3e 100644 --- a/tests_typing/test_http_response.mypy-testing +++ b/tests_typing/test_http_response.mypy-testing @@ -7,7 +7,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse @pytest.mark.mypy_testing def mypy_test_headers(): - Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]" + Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[tuple[str, Any]], None]" Response("data:,", headers=None) Response("data:,", headers={}) Response("data:,", headers=[]) diff --git a/tox.ini b/tox.ini index 80ef4a99e..dad15c6ab 100644 --- a/tox.ini +++ b/tox.ini @@ -61,7 +61,7 @@ commands = mypy {posargs: scrapy tests} [testenv:typing-tests] -basepython = python3.8 +basepython = python3.9 deps = {[test-requirements]deps} {[testenv:typing]deps} @@ -94,21 +94,21 @@ commands = twine check dist/* [pinned] -basepython = python3.8 +basepython = python3.9 deps = - cryptography==36.0.0 + cryptography==37.0.0 cssselect==0.9.1 h2==3.0 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - pyOpenSSL==21.0.0 + pyOpenSSL==22.0.0 queuelib==1.4.2 service_identity==18.1.0 - Twisted[http2]==18.9.0 + Twisted[http2]==21.7.0 w3lib==1.17.0 zope.interface==5.1.0 - lxml==4.4.1 + lxml==4.6.0 {[test-requirements]deps} # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies @@ -194,7 +194,7 @@ commands = pytest {posargs:--durations=10 docs scrapy tests} [testenv:pypy3-pinned] -basepython = pypy3.8 +basepython = pypy3.9 deps = {[pinned]deps} PyPyDispatcher==2.1.0 From f65e64a7243d725d35bbf86ca6f5ae4c350dbcc5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 16 Oct 2024 21:38:43 +0500 Subject: [PATCH 263/269] Misc typing improvements. (#6494) --- scrapy/commands/check.py | 5 ++- scrapy/commands/genspider.py | 4 +-- scrapy/commands/parse.py | 6 ++-- scrapy/core/engine.py | 4 +-- scrapy/core/scraper.py | 6 ++-- scrapy/core/spidermw.py | 4 +-- scrapy/crawler.py | 34 +++++++++---------- .../downloadermiddlewares/httpcompression.py | 2 +- scrapy/downloadermiddlewares/robotstxt.py | 2 +- scrapy/extensions/feedexport.py | 2 +- scrapy/pipelines/files.py | 2 +- scrapy/utils/defer.py | 6 +++- tox.ini | 8 ++--- 13 files changed, 46 insertions(+), 39 deletions(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index c7946605b..1ce155da7 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -13,8 +13,7 @@ from scrapy.utils.misc import load_object, set_environ class TextTestResult(_TextTestResult): def printSummary(self, start: float, stop: float) -> None: write = self.stream.write - # _WritelnDecorator isn't implemented in typeshed yet - writeln = self.stream.writeln # type: ignore[attr-defined] + writeln = self.stream.writeln run = self.testsRun plural = "s" if run != 1 else "" @@ -84,7 +83,7 @@ class Command(ScrapyCommand): with set_environ(SCRAPY_CHECK="true"): for spidername in args or spider_loader.list(): spidercls = spider_loader.load(spidername) - spidercls.start_requests = lambda s: conman.from_spider(s, result) + spidercls.start_requests = lambda s: conman.from_spider(s, result) # type: ignore[assignment,method-assign,return-value] tested_methods = conman.tested_methods_from_spidercls(spidercls) if opts.list: diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index a9b7a6eee..2ac281212 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -4,7 +4,7 @@ import shutil import string from importlib import import_module from pathlib import Path -from typing import Optional, Union, cast +from typing import Any, Optional, Union, cast from urllib.parse import urlparse import scrapy @@ -122,7 +122,7 @@ class Command(ScrapyCommand): name: str, url: str, template_name: str, - ): + ) -> dict[str, Any]: capitalized_module = "".join(s.capitalize() for s in module.split("_")) return { "project_name": self.settings.get("BOT_NAME"), diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index bd1fad14b..ff2bb8ab9 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -38,9 +38,10 @@ _T = TypeVar("_T") class Command(BaseRunSpiderCommand): requires_project = True - spider = None + spider: Optional[Spider] = None items: dict[int, list[Any]] = {} requests: dict[int, list[Request]] = {} + spidercls: Optional[type[Spider]] first_response = None @@ -261,10 +262,11 @@ class Command(BaseRunSpiderCommand): yield self.prepare_request(spider, Request(url), opts) if self.spidercls: - self.spidercls.start_requests = _start_requests + self.spidercls.start_requests = _start_requests # type: ignore[assignment,method-assign] def start_parsing(self, url: str, opts: argparse.Namespace) -> None: assert self.crawler_process + assert self.spidercls self.crawler_process.crawl(self.spidercls, **opts.spargs) self.pcrawler = list(self.crawler_process.crawlers)[0] self.crawler_process.start() diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index bb09d066f..f3d74eccf 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -100,7 +100,7 @@ class ExecutionEngine: ) downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) self.downloader: Downloader = downloader_cls(crawler) - self.scraper = Scraper(crawler) + self.scraper: Scraper = Scraper(crawler) self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = ( spider_closed_callback ) @@ -325,7 +325,7 @@ class ExecutionEngine: raise RuntimeError(f"No open spider to crawl: {request}") d: Deferred[Union[Response, Request]] = self._download(request) # Deferred.addBoth() overloads don't seem to support a Union[_T, Deferred[_T]] return type - d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[arg-type] + d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[call-overload] return d2 def _downloaded( diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 29d7cb0c8..71a0d6aeb 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -55,7 +55,7 @@ class Slot: MIN_RESPONSE_SIZE = 1024 def __init__(self, max_active_size: int = 5000000): - self.max_active_size = max_active_size + self.max_active_size: int = max_active_size self.queue: deque[QueueTuple] = deque() self.active: set[Request] = set() self.active_size: int = 0 @@ -316,7 +316,9 @@ class Scraper: ) return None - def start_itemproc(self, item, *, response: Optional[Response]) -> Deferred[Any]: + def start_itemproc( + self, item: Any, *, response: Optional[Response] + ) -> Deferred[Any]: """Send *item* to the item pipelines for processing. *response* is the source of the item data. If the item does not come diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 223e4192e..3c8513042 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -72,7 +72,7 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_input( self, - scrape_func: ScrapeFunc, + scrape_func: ScrapeFunc[_T], response: Response, request: Request, spider: Spider, @@ -306,7 +306,7 @@ class SpiderMiddlewareManager(MiddlewareManager): def scrape_response( self, - scrape_func: ScrapeFunc, + scrape_func: ScrapeFunc[_T], response: Response, request: Request, spider: Spider, diff --git a/scrapy/crawler.py b/scrapy/crawler.py index b0a4932e1..e75ef52ac 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -42,8 +42,9 @@ from scrapy.utils.reactor import ( ) if TYPE_CHECKING: - from collections.abc import Generator + from collections.abc import Generator, Iterable + from scrapy.spiderloader import SpiderLoader from scrapy.utils.request import RequestFingerprinter @@ -178,16 +179,18 @@ class Crawler: yield maybeDeferred(self.engine.stop) @staticmethod - def _get_component(component_class, components): + def _get_component( + component_class: type[_T], components: Iterable[Any] + ) -> Optional[_T]: for component in components: if isinstance(component, component_class): return component return None - def get_addon(self, cls): + def get_addon(self, cls: type[_T]) -> Optional[_T]: return self._get_component(cls, self.addons.addons) - def get_downloader_middleware(self, cls): + def get_downloader_middleware(self, cls: type[_T]) -> Optional[_T]: if not self.engine: raise RuntimeError( "Crawler.get_downloader_middleware() can only be called after " @@ -195,7 +198,7 @@ class Crawler: ) return self._get_component(cls, self.engine.downloader.middleware.middlewares) - def get_extension(self, cls): + def get_extension(self, cls: type[_T]) -> Optional[_T]: if not self.extensions: raise RuntimeError( "Crawler.get_extension() can only be called after the " @@ -203,7 +206,7 @@ class Crawler: ) return self._get_component(cls, self.extensions.middlewares) - def get_item_pipeline(self, cls): + def get_item_pipeline(self, cls: type[_T]) -> Optional[_T]: if not self.engine: raise RuntimeError( "Crawler.get_item_pipeline() can only be called after the " @@ -211,7 +214,7 @@ class Crawler: ) return self._get_component(cls, self.engine.scraper.itemproc.middlewares) - def get_spider_middleware(self, cls): + def get_spider_middleware(self, cls: type[_T]) -> Optional[_T]: if not self.engine: raise RuntimeError( "Crawler.get_spider_middleware() can only be called after the " @@ -240,18 +243,18 @@ class CrawlerRunner: ) @staticmethod - def _get_spider_loader(settings: BaseSettings): + def _get_spider_loader(settings: BaseSettings) -> SpiderLoader: """Get SpiderLoader instance from settings""" cls_path = settings.get("SPIDER_LOADER_CLASS") loader_cls = load_object(cls_path) verifyClass(ISpiderLoader, loader_cls) - return loader_cls.from_settings(settings.frozencopy()) + return cast("SpiderLoader", loader_cls.from_settings(settings.frozencopy())) def __init__(self, settings: Union[dict[str, Any], Settings, None] = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) - self.settings = settings - self.spider_loader = self._get_spider_loader(settings) + self.settings: Settings = settings + self.spider_loader: SpiderLoader = self._get_spider_loader(settings) self._crawlers: set[Crawler] = set() self._active: set[Deferred[None]] = set() self.bootstrap_failed = False @@ -329,8 +332,7 @@ class CrawlerRunner: def _create_crawler(self, spidercls: Union[str, type[Spider]]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - # temporary cast until self.spider_loader is typed - return Crawler(cast(type[Spider], spidercls), self.settings) + return Crawler(spidercls, self.settings) def stop(self) -> Deferred[Any]: """ @@ -384,7 +386,7 @@ class CrawlerProcess(CrawlerRunner): super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) - self._initialized_reactor = False + self._initialized_reactor: bool = False def _signal_shutdown(self, signum: int, _: Any) -> None: from twisted.internet import reactor @@ -413,9 +415,7 @@ class CrawlerProcess(CrawlerRunner): init_reactor = not self._initialized_reactor self._initialized_reactor = True # temporary cast until self.spider_loader is typed - return Crawler( - cast(type[Spider], spidercls), self.settings, init_reactor=init_reactor - ) + return Crawler(spidercls, self.settings, init_reactor=init_reactor) def start( self, stop_after_crawl: bool = True, install_signal_handlers: bool = True diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index b0cede97d..d913ca25d 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -88,7 +88,7 @@ class HttpCompressionMiddleware: crawler.signals.connect(mw.open_spider, signals.spider_opened) return mw - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> None: if hasattr(spider, "download_maxsize"): self._max_size = spider.download_maxsize if hasattr(spider, "download_warnsize"): diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 421c58e68..81ba009d6 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -67,7 +67,7 @@ class RobotsTxtMiddleware: if request.url.startswith("data:") or request.url.startswith("file:"): return None d: Deferred[Optional[RobotParser]] = maybeDeferred( - self.robot_parser, request, spider # type: ignore[arg-type] + self.robot_parser, request, spider # type: ignore[call-overload] ) d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider) return d2 diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index b1001dabb..7bfcbe6f3 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -578,7 +578,7 @@ class FeedExporter: return None logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" - d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[arg-type] + d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[call-overload] d.addCallback( self._handle_store_success, logmsg, spider, type(slot.storage).__name__ diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 9314856c1..32e9ffe7c 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -550,7 +550,7 @@ class FilesPipeline(MediaPipeline): path = self.file_path(request, info=info, item=item) # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type - dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type] + dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[call-overload] dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess) dfd2.addErrback(lambda _: None) dfd2.addErrback( diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 3a0dee8f1..aeacadb1c 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -305,7 +305,11 @@ def process_parallel( dfds, fireOnOneErrback=True, consumeErrors=True ) d2: Deferred[list[_T2]] = d.addCallback(lambda r: [x[1] for x in r]) - d2.addErrback(lambda f: f.value.subFailure) + + def eb(failure: Failure) -> Failure: + return failure.value.subFailure + + d2.addErrback(eb) return d2 diff --git a/tox.ini b/tox.ini index dad15c6ab..79f72a0f2 100644 --- a/tox.ini +++ b/tox.ini @@ -46,12 +46,12 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.11.1 + mypy==1.12.0 typing-extensions==4.12.2 - types-lxml==2024.8.7 + types-lxml==2024.9.16 types-Pygments==2.18.0.20240506 - botocore-stubs==1.34.158 - boto3-stubs[s3]==1.34.158 + botocore-stubs==1.35.39 + boto3-stubs[s3]==1.35.39 attrs >= 18.2.0 Pillow >= 10.3.0 pyOpenSSL >= 24.2.1 From c8e87ab21a216c546baa797b9a4e6fe27751a4d3 Mon Sep 17 00:00:00 2001 From: Julian Ste <31321934+julian-st@users.noreply.github.com> Date: Thu, 17 Oct 2024 17:03:16 +0200 Subject: [PATCH 264/269] Fixed typos (#6497) --- docs/faq.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index d394406e8..0b650f522 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -269,7 +269,7 @@ To dump into a CSV file:: scrapy crawl myspider -O items.csv -To dump into a XML file:: +To dump into an XML file:: scrapy crawl myspider -O items.xml @@ -417,8 +417,8 @@ How can I make a blank request? blank_request = Request("data:,") -In this case, the URL is set to a data URI scheme. Data URLs allow you to include data -in-line in web pages as if they were external resources. The "data:" scheme with an empty +In this case, the URL is set to a data URI scheme. Data URLs allow you to include data +inline within web pages, similar to external resources. The "data:" scheme with an empty content (",") essentially creates a request to a data URL without any specific content. From c9095ef927bc42e8f23c5d02c05a7b918f7aa5bf Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 17 Oct 2024 21:22:34 +0500 Subject: [PATCH 265/269] Remove --keep-runtime-typing from pyupgrade. --- .pre-commit-config.yaml | 4 +- scrapy/cmdline.py | 10 +-- scrapy/commands/__init__.py | 6 +- scrapy/commands/genspider.py | 8 +- scrapy/commands/parse.py | 22 +++--- scrapy/commands/runspider.py | 4 +- scrapy/commands/startproject.py | 5 +- scrapy/contracts/__init__.py | 12 ++- scrapy/contracts/default.py | 6 +- scrapy/core/downloader/__init__.py | 12 ++- scrapy/core/downloader/contextfactory.py | 6 +- scrapy/core/downloader/handlers/__init__.py | 12 +-- scrapy/core/downloader/handlers/ftp.py | 8 +- scrapy/core/downloader/handlers/http11.py | 62 ++++++++-------- scrapy/core/downloader/handlers/http2.py | 8 +- scrapy/core/downloader/handlers/s3.py | 8 +- scrapy/core/downloader/middleware.py | 14 ++-- scrapy/core/downloader/webclient.py | 6 +- scrapy/core/engine.py | 41 +++++------ scrapy/core/http2/agent.py | 10 +-- scrapy/core/http2/protocol.py | 8 +- scrapy/core/http2/stream.py | 4 +- scrapy/core/scheduler.py | 46 ++++++------ scrapy/core/scraper.py | 51 ++++++------- scrapy/core/spidermw.py | 56 +++++++------- scrapy/crawler.py | 40 +++++----- scrapy/downloadermiddlewares/ajaxcrawl.py | 4 +- scrapy/downloadermiddlewares/cookies.py | 8 +- .../downloadermiddlewares/defaultheaders.py | 4 +- .../downloadermiddlewares/downloadtimeout.py | 4 +- scrapy/downloadermiddlewares/httpauth.py | 4 +- scrapy/downloadermiddlewares/httpcache.py | 16 ++-- .../downloadermiddlewares/httpcompression.py | 10 +-- scrapy/downloadermiddlewares/httpproxy.py | 20 ++--- scrapy/downloadermiddlewares/redirect.py | 6 +- scrapy/downloadermiddlewares/retry.py | 18 ++--- scrapy/downloadermiddlewares/robotstxt.py | 22 +++--- scrapy/downloadermiddlewares/stats.py | 10 +-- scrapy/downloadermiddlewares/useragent.py | 4 +- scrapy/dupefilters.py | 12 +-- scrapy/exporters.py | 28 +++---- scrapy/extensions/corestats.py | 4 +- scrapy/extensions/debug.py | 6 +- scrapy/extensions/feedexport.py | 72 +++++++++--------- scrapy/extensions/httpcache.py | 26 +++---- scrapy/extensions/logstats.py | 6 +- scrapy/extensions/periodic_log.py | 14 ++-- scrapy/extensions/spiderstate.py | 6 +- scrapy/extensions/statsmailer.py | 4 +- scrapy/extensions/throttle.py | 6 +- scrapy/http/cookies.py | 12 +-- scrapy/http/headers.py | 14 ++-- scrapy/http/request/__init__.py | 33 ++++----- scrapy/http/request/form.py | 32 ++++---- scrapy/http/request/json_request.py | 6 +- scrapy/http/request/rpc.py | 6 +- scrapy/http/response/__init__.py | 66 ++++++++--------- scrapy/http/response/text.py | 73 +++++++++---------- scrapy/linkextractors/lxmlhtml.py | 28 +++---- scrapy/loader/__init__.py | 8 +- scrapy/logformatter.py | 14 ++-- scrapy/mail.py | 22 +++--- scrapy/middleware.py | 12 ++- scrapy/pipelines/files.py | 72 ++++++++---------- scrapy/pipelines/images.py | 24 +++--- scrapy/pipelines/media.py | 27 ++++--- scrapy/pqueues.py | 18 ++--- scrapy/resolver.py | 4 +- scrapy/responsetypes.py | 14 ++-- scrapy/robotstxt.py | 30 ++++---- scrapy/selector/unified.py | 16 ++-- scrapy/settings/__init__.py | 38 ++++------ scrapy/shell.py | 34 ++++----- scrapy/spidermiddlewares/httperror.py | 4 +- scrapy/spidermiddlewares/referer.py | 36 +++++---- scrapy/spiders/__init__.py | 10 +-- scrapy/spiders/crawl.py | 38 +++++----- scrapy/spiders/feed.py | 8 +- scrapy/spiders/init.py | 4 +- scrapy/spiders/sitemap.py | 14 ++-- scrapy/squeues.py | 14 ++-- scrapy/statscollectors.py | 30 ++++---- scrapy/utils/asyncgen.py | 8 +- scrapy/utils/conf.py | 12 +-- scrapy/utils/console.py | 8 +- scrapy/utils/curl.py | 6 +- scrapy/utils/datatypes.py | 14 ++-- scrapy/utils/defer.py | 16 ++-- scrapy/utils/deprecate.py | 14 ++-- scrapy/utils/httpobj.py | 6 +- scrapy/utils/iterators.py | 34 ++++----- scrapy/utils/job.py | 6 +- scrapy/utils/log.py | 14 ++-- scrapy/utils/misc.py | 6 +- scrapy/utils/ossignal.py | 2 + scrapy/utils/project.py | 5 +- scrapy/utils/python.py | 22 +++--- scrapy/utils/reactor.py | 8 +- scrapy/utils/request.py | 14 ++-- scrapy/utils/response.py | 12 +-- scrapy/utils/sitemap.py | 6 +- scrapy/utils/spider.py | 14 ++-- scrapy/utils/ssl.py | 4 +- scrapy/utils/template.py | 4 +- scrapy/utils/test.py | 8 +- scrapy/utils/testproc.py | 8 +- scrapy/utils/url.py | 4 +- .../CrawlerProcess/asyncio_deferred_signal.py | 5 +- tests/spiders.py | 13 ++-- tests/test_commands.py | 8 +- tests/test_downloader_handlers.py | 5 +- tests/test_feedexport.py | 4 +- tests/test_linkextractors.py | 5 +- tests/test_loader.py | 5 +- tests/test_pipeline_crawl.py | 7 +- tests/test_pipeline_images.py | 5 +- tests/test_pipeline_media.py | 4 +- tests/test_scheduler.py | 7 +- tests/test_scheduler_base.py | 5 +- tests/test_spidermiddleware.py | 17 +++-- tests/test_spidermiddleware_referer.py | 26 ++++--- tests/test_utils_request.py | 13 ++-- 122 files changed, 947 insertions(+), 981 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 75529be05..fbd710f6f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,7 +30,7 @@ repos: additional_dependencies: - black==24.4.2 - repo: https://github.com/asottile/pyupgrade - rev: v3.16.0 + rev: v3.18.0 hooks: - id: pyupgrade - args: [--py39-plus, --keep-runtime-typing] + args: [--py39-plus] diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index b820eb7f9..b6f19a37f 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -6,7 +6,7 @@ import inspect import os import sys from importlib.metadata import entry_points -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter @@ -30,7 +30,7 @@ if TYPE_CHECKING: class ScrapyArgumentParser(argparse.ArgumentParser): def _parse_optional( self, arg_string: str - ) -> Optional[tuple[Optional[argparse.Action], str, Optional[str]]]: + ) -> tuple[argparse.Action | None, str, str | None] | None: # if starts with -: it means that is a parameter not a argument if arg_string[:2] == "-:": return None @@ -89,7 +89,7 @@ def _get_commands_dict( return cmds -def _pop_command_name(argv: list[str]) -> Optional[str]: +def _pop_command_name(argv: list[str]) -> str | None: i = 0 for arg in argv[1:]: if not arg.startswith("-"): @@ -147,9 +147,7 @@ def _run_print_help( sys.exit(2) -def execute( - argv: Optional[list[str]] = None, settings: Optional[Settings] = None -) -> None: +def execute(argv: list[str] | None = None, settings: Settings | None = None) -> None: if argv is None: argv = sys.argv diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index a94db90b1..eccbef040 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -8,7 +8,7 @@ import argparse import builtins import os from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any from twisted.python import failure @@ -23,7 +23,7 @@ if TYPE_CHECKING: class ScrapyCommand: requires_project: bool = False - crawler_process: Optional[CrawlerProcess] = None + crawler_process: CrawlerProcess | None = None # default settings to be used for this command instead of global defaults default_settings: dict[str, Any] = {} @@ -195,7 +195,7 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): prog: str, indent_increment: int = 2, max_help_position: int = 24, - width: Optional[int] = None, + width: int | None = None, ): super().__init__( prog, diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 2ac281212..b286e703e 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -1,10 +1,12 @@ +from __future__ import annotations + import argparse import os import shutil import string from importlib import import_module from pathlib import Path -from typing import Any, Optional, Union, cast +from typing import Any, cast from urllib.parse import urlparse import scrapy @@ -140,7 +142,7 @@ class Command(ScrapyCommand): name: str, url: str, template_name: str, - template_file: Union[str, os.PathLike], + template_file: str | os.PathLike, ) -> None: """Generate the spider module, based on the given template""" tvars = self._generate_template_variables(module, name, url, template_name) @@ -161,7 +163,7 @@ class Command(ScrapyCommand): if spiders_module: print(f"in module:\n {spiders_module.__name__}.{module}") - def _find_template(self, template: str) -> Optional[Path]: + def _find_template(self, template: str) -> Path | None: template_file = Path(self.templates_dir, f"{template}.tmpl") if template_file.exists(): return template_file diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index ff2bb8ab9..2059dcf75 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -5,7 +5,7 @@ import functools import inspect import json import logging -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, overload +from typing import TYPE_CHECKING, Any, TypeVar, overload from itemadapter import ItemAdapter, is_item from twisted.internet.defer import Deferred, maybeDeferred @@ -38,10 +38,10 @@ _T = TypeVar("_T") class Command(BaseRunSpiderCommand): requires_project = True - spider: Optional[Spider] = None + spider: Spider | None = None items: dict[int, list[Any]] = {} requests: dict[int, list[Request]] = {} - spidercls: Optional[type[Spider]] + spidercls: type[Spider] | None first_response = None @@ -137,13 +137,13 @@ class Command(BaseRunSpiderCommand): @overload def iterate_spider_output( - self, result: Union[AsyncGenerator[_T, None], Coroutine[Any, Any, _T]] + self, result: AsyncGenerator[_T] | Coroutine[Any, Any, _T] ) -> Deferred[_T]: ... @overload def iterate_spider_output(self, result: _T) -> Iterable[Any]: ... - def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred[Any]]: + def iterate_spider_output(self, result: Any) -> Iterable[Any] | Deferred[Any]: if inspect.isasyncgen(result): d = deferred_from_coro( collect_asyncgen(aiter_errback(result, self.handle_exception)) @@ -164,7 +164,7 @@ class Command(BaseRunSpiderCommand): old_reqs = self.requests.get(lvl, []) self.requests[lvl] = old_reqs + new_reqs - def print_items(self, lvl: Optional[int] = None, colour: bool = True) -> None: + def print_items(self, lvl: int | None = None, colour: bool = True) -> None: if lvl is None: items = [item for lst in self.items.values() for item in lst] else: @@ -173,7 +173,7 @@ class Command(BaseRunSpiderCommand): print("# Scraped Items ", "-" * 60) display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour) - def print_requests(self, lvl: Optional[int] = None, colour: bool = True) -> None: + def print_requests(self, lvl: int | None = None, colour: bool = True) -> None: if lvl is None: if self.requests: requests = self.requests[max(self.requests)] @@ -222,7 +222,7 @@ class Command(BaseRunSpiderCommand): self, response: Response, callback: CallbackT, - cb_kwargs: Optional[dict[str, Any]] = None, + cb_kwargs: dict[str, Any] | None = None, ) -> Deferred[Any]: cb_kwargs = cb_kwargs or {} d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) @@ -230,7 +230,7 @@ class Command(BaseRunSpiderCommand): def get_callback_from_rules( self, spider: Spider, response: Response - ) -> Union[CallbackT, str, None]: + ) -> CallbackT | str | None: if getattr(spider, "rules", None): for rule in spider.rules: # type: ignore[attr-defined] if rule.link_extractor.matches(response.url): @@ -303,9 +303,9 @@ class Command(BaseRunSpiderCommand): *, spider: Spider, opts: argparse.Namespace, - response: Optional[Response] = None, + response: Response | None = None, ) -> CallbackT: - cb: Union[str, CallbackT, None] = None + cb: str | CallbackT | None = None if response: cb = response.meta["_callback"] if not cb: diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 14d58f311..7ec56899c 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -4,7 +4,7 @@ import argparse import sys from importlib import import_module from pathlib import Path -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError @@ -15,7 +15,7 @@ if TYPE_CHECKING: from types import ModuleType -def _import_file(filepath: Union[str, PathLike[str]]) -> ModuleType: +def _import_file(filepath: str | PathLike[str]) -> ModuleType: abspath = Path(filepath).resolve() if abspath.suffix not in (".py", ".pyw"): raise ValueError(f"Not a Python source file: {abspath}") diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index f7052cd18..f54c02369 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import os import re @@ -6,7 +8,6 @@ from importlib.util import find_spec from pathlib import Path from shutil import copy2, copystat, ignore_patterns, move from stat import S_IWUSR as OWNER_WRITE_PERMISSION -from typing import Union import scrapy from scrapy.commands import ScrapyCommand @@ -24,7 +25,7 @@ TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = ( IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn") -def _make_writable(path: Union[str, os.PathLike]) -> None: +def _make_writable(path: str | os.PathLike) -> None: current_permissions = os.stat(path).st_mode os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index ffe5053de..c20c02ca6 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -6,7 +6,7 @@ from collections.abc import AsyncGenerator, Iterable from functools import wraps from inspect import getmembers from types import CoroutineType -from typing import TYPE_CHECKING, Any, Optional, cast +from typing import TYPE_CHECKING, Any, cast from unittest import TestCase, TestResult from scrapy.http import Request, Response @@ -24,7 +24,7 @@ if TYPE_CHECKING: class Contract: """Abstract class for contracts""" - request_cls: Optional[type[Request]] = None + request_cls: type[Request] | None = None name: str def __init__(self, method: Callable, *args: Any): @@ -126,10 +126,8 @@ class ContractsManager: return contracts - def from_spider( - self, spider: Spider, results: TestResult - ) -> list[Optional[Request]]: - requests: list[Optional[Request]] = [] + def from_spider(self, spider: Spider, results: TestResult) -> list[Request | None]: + requests: list[Request | None] = [] for method in self.tested_methods_from_spidercls(type(spider)): bound_method = spider.__getattribute__(method) try: @@ -140,7 +138,7 @@ class ContractsManager: return requests - def from_method(self, method: Callable, results: TestResult) -> Optional[Request]: + def from_method(self, method: Callable, results: TestResult) -> Request | None: contracts = self.extract_contracts(method) if contracts: request_cls = Request diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 87170d3c1..6f357ba20 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -1,5 +1,7 @@ +from __future__ import annotations + import json -from typing import Any, Callable, Optional +from typing import Any, Callable from itemadapter import ItemAdapter, is_item @@ -63,7 +65,7 @@ class ReturnsContract(Contract): """ name = "returns" - object_type_verifiers: dict[Optional[str], Callable[[Any], bool]] = { + object_type_verifiers: dict[str | None, Callable[[Any], bool]] = { "request": lambda x: isinstance(x, Request), "requests": lambda x: isinstance(x, Request), "item": is_item, diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 77d57a8d8..1cc0422b7 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -5,7 +5,7 @@ import warnings from collections import deque from datetime import datetime from time import time -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, TypeVar, cast from twisted.internet import task from twisted.internet.defer import Deferred @@ -37,7 +37,7 @@ class Slot: delay: float, randomize_delay: bool, *, - throttle: Optional[bool] = None, + throttle: bool | None = None, ): self.concurrency: int = concurrency self.delay: float = delay @@ -119,15 +119,13 @@ class Downloader: "DOWNLOAD_SLOTS", {} ) - def fetch( - self, request: Request, spider: Spider - ) -> Deferred[Union[Response, Request]]: + def fetch(self, request: Request, spider: Spider) -> Deferred[Response | Request]: def _deactivate(response: _T) -> _T: self.active.remove(request) return response self.active.add(request) - dfd: Deferred[Union[Response, Request]] = self.middleware.download( + dfd: Deferred[Response | Request] = self.middleware.download( self._enqueue_request, request, spider ) return dfd.addBoth(_deactivate) @@ -164,7 +162,7 @@ class Downloader: return key - def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str: + def _get_slot_key(self, request: Request, spider: Spider | None) -> str: warnings.warn( "Use of this protected method is deprecated. Consider using its corresponding public method get_slot_key() instead.", ScrapyDeprecationWarning, diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index ba20c3c2c..f80f832a7 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any from OpenSSL import SSL from twisted.internet._sslverify import _setAcceptableProtocols @@ -49,7 +49,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): self, method: int = SSL.SSLv23_METHOD, tls_verbose_logging: bool = False, - tls_ciphers: Optional[str] = None, + tls_ciphers: str | None = None, *args: Any, **kwargs: Any, ): @@ -73,7 +73,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): tls_verbose_logging: bool = settings.getbool( "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" ) - tls_ciphers: Optional[str] = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] + tls_ciphers: str | None = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] return cls( # type: ignore[misc] method=method, tls_verbose_logging=tls_verbose_logging, diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index c39e480f1..218f44bbb 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -4,7 +4,7 @@ from __future__ import annotations import logging from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Optional, Protocol, Union, cast +from typing import TYPE_CHECKING, Any, Protocol, cast from twisted.internet import defer @@ -35,16 +35,16 @@ class DownloadHandlerProtocol(Protocol): class DownloadHandlers: def __init__(self, crawler: Crawler): self._crawler: Crawler = crawler - self._schemes: dict[str, Union[str, Callable[..., Any]]] = ( + self._schemes: dict[str, str | Callable[..., Any]] = ( {} ) # stores acceptable schemes on instancing self._handlers: dict[str, DownloadHandlerProtocol] = ( {} ) # stores instanced handlers for schemes self._notconfigured: dict[str, str] = {} # remembers failed handlers - handlers: dict[str, Union[str, Callable[..., Any]]] = without_none_values( + handlers: dict[str, str | Callable[..., Any]] = without_none_values( cast( - dict[str, Union[str, Callable[..., Any]]], + "dict[str, str | Callable[..., Any]]", crawler.settings.getwithbase("DOWNLOAD_HANDLERS"), ) ) @@ -54,7 +54,7 @@ class DownloadHandlers: crawler.signals.connect(self._close, signals.engine_stopped) - def _get_handler(self, scheme: str) -> Optional[DownloadHandlerProtocol]: + def _get_handler(self, scheme: str) -> DownloadHandlerProtocol | None: """Lazy-load the downloadhandler for a scheme only on the first request for that scheme. """ @@ -70,7 +70,7 @@ class DownloadHandlers: def _load_handler( self, scheme: str, skip_lazy: bool = False - ) -> Optional[DownloadHandlerProtocol]: + ) -> DownloadHandlerProtocol | None: path = self._schemes[scheme] try: dhcls: type[DownloadHandlerProtocol] = load_object(path) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index bc06c7ef4..70a769771 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -32,7 +32,7 @@ from __future__ import annotations import re from io import BytesIO -from typing import TYPE_CHECKING, Any, BinaryIO, Optional +from typing import TYPE_CHECKING, Any, BinaryIO from urllib.parse import unquote from twisted.internet.protocol import ClientCreator, Protocol @@ -56,8 +56,8 @@ if TYPE_CHECKING: class ReceivedDataProtocol(Protocol): - def __init__(self, filename: Optional[str] = None): - self.__filename: Optional[str] = filename + def __init__(self, filename: str | None = None): + self.__filename: str | None = filename self.body: BinaryIO = open(filename, "wb") if filename else BytesIO() self.size: int = 0 @@ -66,7 +66,7 @@ class ReceivedDataProtocol(Protocol): self.size += len(data) @property - def filename(self) -> Optional[str]: + def filename(self) -> str | None: return self.__filename def close(self) -> None: diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index f96dc7c98..bd3200e9f 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,7 +8,7 @@ import re from contextlib import suppress from io import BytesIO from time import time -from typing import TYPE_CHECKING, Any, Optional, TypedDict, TypeVar, Union +from typing import TYPE_CHECKING, Any, TypedDict, TypeVar from urllib.parse import urldefrag, urlunparse from twisted.internet import ssl @@ -52,10 +52,10 @@ _T = TypeVar("_T") class _ResultT(TypedDict): txresponse: TxResponse body: bytes - flags: Optional[list[str]] - certificate: Optional[ssl.Certificate] - ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None] - failure: NotRequired[Optional[Failure]] + flags: list[str] | None + certificate: ssl.Certificate | None + ip_address: ipaddress.IPv4Address | ipaddress.IPv6Address | None + failure: NotRequired[Failure | None] class HTTP11DownloadHandler: @@ -143,10 +143,10 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): reactor: ReactorBase, host: str, port: int, - proxyConf: tuple[str, int, Optional[bytes]], + proxyConf: tuple[str, int, bytes | None], contextFactory: IPolicyForHTTPS, timeout: float = 30, - bindAddress: Optional[tuple[str, int]] = None, + bindAddress: tuple[str, int] | None = None, ): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) @@ -220,7 +220,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def tunnel_request_data( - host: str, port: int, proxy_auth_header: Optional[bytes] = None + host: str, port: int, proxy_auth_header: bytes | None = None ) -> bytes: r""" Return binary content of a CONNECT request. @@ -254,14 +254,14 @@ class TunnelingAgent(Agent): self, *, reactor: ReactorBase, - proxyConf: tuple[str, int, Optional[bytes]], + proxyConf: tuple[str, int, bytes | None], contextFactory: IPolicyForHTTPS, - connectTimeout: Optional[float] = None, - bindAddress: Optional[bytes] = None, - pool: Optional[HTTPConnectionPool] = None, + connectTimeout: float | None = None, + bindAddress: bytes | None = None, + pool: HTTPConnectionPool | None = None, ): super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) - self._proxyConf: tuple[str, int, Optional[bytes]] = proxyConf + self._proxyConf: tuple[str, int, bytes | None] = proxyConf self._contextFactory: IPolicyForHTTPS = contextFactory def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint: @@ -281,8 +281,8 @@ class TunnelingAgent(Agent): endpoint: TCP4ClientEndpoint, method: bytes, parsedURI: bytes, - headers: Optional[TxHeaders], - bodyProducer: Optional[IBodyProducer], + headers: TxHeaders | None, + bodyProducer: IBodyProducer | None, requestPath: bytes, ) -> Deferred[TxResponse]: # proxy host and port are required for HTTP pool `key` @@ -305,9 +305,9 @@ class ScrapyProxyAgent(Agent): self, reactor: ReactorBase, proxyURI: bytes, - connectTimeout: Optional[float] = None, - bindAddress: Optional[bytes] = None, - pool: Optional[HTTPConnectionPool] = None, + connectTimeout: float | None = None, + bindAddress: bytes | None = None, + pool: HTTPConnectionPool | None = None, ): super().__init__( reactor=reactor, @@ -321,8 +321,8 @@ class ScrapyProxyAgent(Agent): self, method: bytes, uri: bytes, - headers: Optional[TxHeaders] = None, - bodyProducer: Optional[IBodyProducer] = None, + headers: TxHeaders | None = None, + bodyProducer: IBodyProducer | None = None, ) -> Deferred[TxResponse]: """ Issue a new request via the configured proxy. @@ -350,8 +350,8 @@ class ScrapyAgent: *, contextFactory: IPolicyForHTTPS, connectTimeout: float = 10, - bindAddress: Optional[bytes] = None, - pool: Optional[HTTPConnectionPool] = None, + bindAddress: bytes | None = None, + pool: HTTPConnectionPool | None = None, maxsize: int = 0, warnsize: int = 0, fail_on_dataloss: bool = True, @@ -359,12 +359,12 @@ class ScrapyAgent: ): self._contextFactory: IPolicyForHTTPS = contextFactory self._connectTimeout: float = connectTimeout - self._bindAddress: Optional[bytes] = bindAddress - self._pool: Optional[HTTPConnectionPool] = pool + self._bindAddress: bytes | None = bindAddress + self._pool: HTTPConnectionPool | None = pool self._maxsize: int = maxsize self._warnsize: int = warnsize self._fail_on_dataloss: bool = fail_on_dataloss - self._txresponse: Optional[TxResponse] = None + self._txresponse: TxResponse | None = None self._crawler: Crawler = crawler def _get_agent(self, request: Request, timeout: float) -> Agent: @@ -462,7 +462,7 @@ class ScrapyAgent: def _cb_bodyready( self, txresponse: TxResponse, request: Request - ) -> Union[_ResultT, Deferred[_ResultT]]: + ) -> _ResultT | Deferred[_ResultT]: headers_received_result = self._crawler.signals.send_catch_log( signal=signals.headers_received, headers=self._headers_from_twisted_response(txresponse), @@ -551,7 +551,7 @@ class ScrapyAgent: def _cb_bodydone( self, result: _ResultT, request: Request, url: str - ) -> Union[Response, Failure]: + ) -> Response | Failure: headers = self._headers_from_twisted_response(result["txresponse"]) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) try: @@ -614,14 +614,12 @@ class _ResponseReader(Protocol): self._fail_on_dataloss_warned: bool = False self._reached_warnsize: bool = False self._bytes_received: int = 0 - self._certificate: Optional[ssl.Certificate] = None - self._ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None] = ( - None - ) + self._certificate: ssl.Certificate | None = None + self._ip_address: ipaddress.IPv4Address | ipaddress.IPv6Address | None = None self._crawler: Crawler = crawler def _finish_response( - self, flags: Optional[list[str]] = None, failure: Optional[Failure] = None + self, flags: list[str] | None = None, failure: Failure | None = None ) -> None: self._finished.callback( { diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 4722c612d..f0f9ceeb7 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,7 +1,7 @@ from __future__ import annotations from time import time -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from urllib.parse import urldefrag from twisted.internet.error import TimeoutError @@ -60,8 +60,8 @@ class ScrapyH2Agent: context_factory: IPolicyForHTTPS, pool: H2ConnectionPool, connect_timeout: int = 10, - bind_address: Optional[bytes] = None, - crawler: Optional[Crawler] = None, + bind_address: bytes | None = None, + crawler: Crawler | None = None, ) -> None: self._context_factory = context_factory self._connect_timeout = connect_timeout @@ -69,7 +69,7 @@ class ScrapyH2Agent: self._pool = pool self._crawler = crawler - def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent: + def _get_agent(self, request: Request, timeout: float | None) -> H2Agent: from twisted.internet import reactor bind_address = request.meta.get("bindaddress") or self._bind_address diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index fa660c63c..870a26f04 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.exceptions import NotConfigured @@ -26,9 +26,9 @@ class S3DownloadHandler: settings: BaseSettings, *, crawler: Crawler, - aws_access_key_id: Optional[str] = None, - aws_secret_access_key: Optional[str] = None, - aws_session_token: Optional[str] = None, + aws_access_key_id: str | None = None, + aws_secret_access_key: str | None = None, + aws_session_token: str | None = None, httpdownloadhandler: type[HTTPDownloadHandler] = HTTPDownloadHandler, **kw: Any, ): diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 00d3bd1b0..60e7adb2f 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -7,7 +7,7 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Union, cast +from typing import TYPE_CHECKING, Any, cast from twisted.internet.defer import Deferred, inlineCallbacks @@ -46,11 +46,11 @@ class DownloaderMiddlewareManager(MiddlewareManager): download_func: Callable[[Request, Spider], Deferred[Response]], request: Request, spider: Spider, - ) -> Deferred[Union[Response, Request]]: + ) -> Deferred[Response | Request]: @inlineCallbacks def process_request( request: Request, - ) -> Generator[Deferred[Any], Any, Union[Response, Request]]: + ) -> Generator[Deferred[Any], Any, Response | Request]: for method in self.methods["process_request"]: method = cast(Callable, method) response = yield deferred_from_coro( @@ -69,8 +69,8 @@ class DownloaderMiddlewareManager(MiddlewareManager): @inlineCallbacks def process_response( - response: Union[Response, Request] - ) -> Generator[Deferred[Any], Any, Union[Response, Request]]: + response: Response | Request, + ) -> Generator[Deferred[Any], Any, Response | Request]: if response is None: raise TypeError("Received None in process_response") elif isinstance(response, Request): @@ -93,7 +93,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): @inlineCallbacks def process_exception( failure: Failure, - ) -> Generator[Deferred[Any], Any, Union[Failure, Response, Request]]: + ) -> Generator[Deferred[Any], Any, Failure | Response | Request]: exception = failure.value for method in self.methods["process_exception"]: method = cast(Callable, method) @@ -111,7 +111,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response return failure - deferred: Deferred[Union[Response, Request]] = mustbe_deferred( + deferred: Deferred[Response | Request] = mustbe_deferred( process_request, request ) deferred.addErrback(process_exception) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 509bda4e4..ee10ae73b 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -2,7 +2,7 @@ from __future__ import annotations import re from time import time -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse from twisted.internet import defer @@ -144,9 +144,9 @@ class ScrapyHTTPClientFactory(ClientFactory): # converting to bytes to comply to Twisted interface self.url: bytes = to_bytes(self._url, encoding="ascii") self.method: bytes = to_bytes(request.method, encoding="ascii") - self.body: Optional[bytes] = request.body or None + self.body: bytes | None = request.body or None self.headers: Headers = Headers(request.headers) - self.response_headers: Optional[Headers] = None + self.response_headers: Headers | None = None self.timeout: float = request.meta.get("download_timeout") or timeout self.start_time: float = time() self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback( diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index f3d74eccf..d056a00ba 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -9,7 +9,7 @@ from __future__ import annotations import logging from time import time -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, TypeVar, cast from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks, succeed @@ -18,7 +18,7 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.core.downloader import Downloader -from scrapy.core.scraper import Scraper +from scrapy.core.scraper import Scraper, _HandleOutputDeferred from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response from scrapy.logformatter import LogFormatter @@ -32,7 +32,6 @@ if TYPE_CHECKING: from collections.abc import Callable, Generator, Iterable, Iterator from scrapy.core.scheduler import BaseScheduler - from scrapy.core.scraper import _HandleOutputDeferred from scrapy.crawler import Crawler from scrapy.settings import BaseSettings from scrapy.spiders import Spider @@ -51,9 +50,9 @@ class Slot: nextcall: CallLaterOnce[None], scheduler: BaseScheduler, ) -> None: - self.closing: Optional[Deferred[None]] = None + self.closing: Deferred[None] | None = None self.inprogress: set[Request] = set() - self.start_requests: Optional[Iterator[Request]] = iter(start_requests) + self.start_requests: Iterator[Request] | None = iter(start_requests) self.close_if_idle: bool = close_if_idle self.nextcall: CallLaterOnce[None] = nextcall self.scheduler: BaseScheduler = scheduler @@ -84,15 +83,15 @@ class ExecutionEngine: def __init__( self, crawler: Crawler, - spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]], + spider_closed_callback: Callable[[Spider], Deferred[None] | None], ) -> None: self.crawler: Crawler = crawler self.settings: Settings = crawler.settings self.signals: SignalManager = crawler.signals assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter - self.slot: Optional[Slot] = None - self.spider: Optional[Spider] = None + self.slot: Slot | None = None + self.spider: Spider | None = None self.running: bool = False self.paused: bool = False self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( @@ -101,10 +100,10 @@ class ExecutionEngine: downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) self.downloader: Downloader = downloader_cls(crawler) self.scraper: Scraper = Scraper(crawler) - self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = ( + self._spider_closed_callback: Callable[[Spider], Deferred[None] | None] = ( spider_closed_callback ) - self.start_time: Optional[float] = None + self.start_time: float | None = None def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: from scrapy.core.scheduler import BaseScheduler @@ -218,7 +217,7 @@ class ExecutionEngine: or self.scraper.slot.needs_backout() ) - def _next_request_from_scheduler(self) -> Optional[Deferred[None]]: + def _next_request_from_scheduler(self) -> Deferred[None] | None: assert self.slot is not None # typing assert self.spider is not None # typing @@ -226,7 +225,7 @@ class ExecutionEngine: if request is None: return None - d: Deferred[Union[Response, Request]] = self._download(request) + d: Deferred[Response | Request] = self._download(request) d.addBoth(self._handle_downloader_output, request) d.addErrback( lambda f: logger.info( @@ -260,8 +259,8 @@ class ExecutionEngine: return d2 def _handle_downloader_output( - self, result: Union[Request, Response, Failure], request: Request - ) -> Optional[_HandleOutputDeferred]: + self, result: Request | Response | Failure, request: Request + ) -> _HandleOutputDeferred | None: assert self.spider is not None # typing if not isinstance(result, (Request, Response, Failure)): @@ -323,24 +322,24 @@ class ExecutionEngine: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - d: Deferred[Union[Response, Request]] = self._download(request) + d: Deferred[Response | Request] = self._download(request) # Deferred.addBoth() overloads don't seem to support a Union[_T, Deferred[_T]] return type d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[call-overload] return d2 def _downloaded( - self, result: Union[Response, Request, Failure], request: Request - ) -> Union[Deferred[Response], Response, Failure]: + self, result: Response | Request | Failure, request: Request + ) -> Deferred[Response] | Response | Failure: assert self.slot is not None # typing self.slot.remove_request(request) return self.download(result) if isinstance(result, Request) else result - def _download(self, request: Request) -> Deferred[Union[Response, Request]]: + def _download(self, request: Request) -> Deferred[Response | Request]: assert self.slot is not None # typing self.slot.add_request(request) - def _on_success(result: Union[Response, Request]) -> Union[Response, Request]: + def _on_success(result: Response | Request) -> Response | Request: if not isinstance(result, (Response, Request)): raise TypeError( f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}" @@ -368,9 +367,7 @@ class ExecutionEngine: return _ assert self.spider is not None - dwld: Deferred[Union[Response, Request]] = self.downloader.fetch( - request, self.spider - ) + dwld: Deferred[Response | Request] = self.downloader.fetch(request, self.spider) dwld.addCallback(_on_success) dwld.addBoth(_on_complete) return dwld diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index b5ff55eb0..45f32daaa 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections import deque -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from twisted.internet import defer from twisted.internet.defer import Deferred @@ -121,8 +121,8 @@ class H2Agent: reactor: ReactorBase, pool: H2ConnectionPool, context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, - bind_address: Optional[bytes] = None, + connect_timeout: float | None = None, + bind_address: bytes | None = None, ) -> None: self._reactor = reactor self._pool = pool @@ -165,8 +165,8 @@ class ScrapyProxyH2Agent(H2Agent): proxy_uri: URI, pool: H2ConnectionPool, context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, - bind_address: Optional[bytes] = None, + connect_timeout: float | None = None, + bind_address: bytes | None = None, ) -> None: super().__init__( reactor=reactor, diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 618423218..23335b7b2 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -4,7 +4,7 @@ import ipaddress import itertools import logging from collections import deque -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any from h2.config import H2Configuration from h2.connection import H2Connection @@ -63,7 +63,7 @@ class InvalidNegotiatedProtocol(H2Error): class RemoteTerminatedConnection(H2Error): def __init__( self, - remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], + remote_ip_address: IPv4Address | IPv6Address | None, event: ConnectionTerminated, ) -> None: self.remote_ip_address = remote_ip_address @@ -74,9 +74,7 @@ class RemoteTerminatedConnection(H2Error): class MethodNotAllowed405(H2Error): - def __init__( - self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]] - ) -> None: + def __init__(self, remote_ip_address: IPv4Address | IPv6Address | None) -> None: self.remote_ip_address = remote_ip_address def __str__(self) -> str: diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 51ebdf489..a4dc89c18 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging from enum import Enum from io import BytesIO -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError @@ -382,7 +382,7 @@ class Stream: def close( self, reason: StreamCloseReason, - errors: Optional[list[BaseException]] = None, + errors: list[BaseException] | None = None, from_protocol: bool = False, ) -> None: """Based on the reason sent we will handle each case.""" diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index ced18fc05..bebee1236 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,7 +4,7 @@ import json import logging from abc import abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, cast +from typing import TYPE_CHECKING, Any, cast # working around https://github.com/sphinx-doc/sphinx/issues/10400 from twisted.internet.defer import Deferred # noqa: TC002 @@ -73,7 +73,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): """ return cls() - def open(self, spider: Spider) -> Optional[Deferred[None]]: + def open(self, spider: Spider) -> Deferred[None] | None: """ Called when the spider is opened by the engine. It receives the spider instance as argument and it's useful to execute initialization code. @@ -83,7 +83,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): """ pass - def close(self, reason: str) -> Optional[Deferred[None]]: + def close(self, reason: str) -> Deferred[None] | None: """ Called when the spider is closed by the engine. It receives the reason why the crawl finished as argument and it's useful to execute cleaning code. @@ -115,7 +115,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): raise NotImplementedError() @abstractmethod - def next_request(self) -> Optional[Request]: + def next_request(self) -> Request | None: """ Return the next :class:`~scrapy.http.Request` to be processed, or ``None`` to indicate that there are no requests to be considered ready at the moment. @@ -181,22 +181,22 @@ class Scheduler(BaseScheduler): def __init__( self, dupefilter: BaseDupeFilter, - jobdir: Optional[str] = None, - dqclass: Optional[type[BaseQueue]] = None, - mqclass: Optional[type[BaseQueue]] = None, + jobdir: str | None = None, + dqclass: type[BaseQueue] | None = None, + mqclass: type[BaseQueue] | None = None, logunser: bool = False, - stats: Optional[StatsCollector] = None, - pqclass: Optional[type[ScrapyPriorityQueue]] = None, - crawler: Optional[Crawler] = None, + stats: StatsCollector | None = None, + pqclass: type[ScrapyPriorityQueue] | None = None, + crawler: Crawler | None = None, ): self.df: BaseDupeFilter = dupefilter - self.dqdir: Optional[str] = self._dqdir(jobdir) - self.pqclass: Optional[type[ScrapyPriorityQueue]] = pqclass - self.dqclass: Optional[type[BaseQueue]] = dqclass - self.mqclass: Optional[type[BaseQueue]] = mqclass + self.dqdir: str | None = self._dqdir(jobdir) + self.pqclass: type[ScrapyPriorityQueue] | None = pqclass + self.dqclass: type[BaseQueue] | None = dqclass + self.mqclass: type[BaseQueue] | None = mqclass self.logunser: bool = logunser - self.stats: Optional[StatsCollector] = stats - self.crawler: Optional[Crawler] = crawler + self.stats: StatsCollector | None = stats + self.crawler: Crawler | None = crawler @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -218,7 +218,7 @@ class Scheduler(BaseScheduler): def has_pending_requests(self) -> bool: return len(self) > 0 - def open(self, spider: Spider) -> Optional[Deferred[None]]: + def open(self, spider: Spider) -> Deferred[None] | None: """ (1) initialize the memory queue (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory @@ -226,10 +226,10 @@ class Scheduler(BaseScheduler): """ self.spider: Spider = spider self.mqs: ScrapyPriorityQueue = self._mq() - self.dqs: Optional[ScrapyPriorityQueue] = self._dq() if self.dqdir else None + self.dqs: ScrapyPriorityQueue | None = self._dq() if self.dqdir else None return self.df.open() - def close(self, reason: str) -> Optional[Deferred[None]]: + def close(self, reason: str) -> Deferred[None] | None: """ (1) dump pending requests to disk if there is a disk queue (2) return the result of the dupefilter's ``close`` method @@ -263,7 +263,7 @@ class Scheduler(BaseScheduler): self.stats.inc_value("scheduler/enqueued", spider=self.spider) return True - def next_request(self) -> Optional[Request]: + def next_request(self) -> Request | None: """ Return a :class:`~scrapy.http.Request` object from the memory queue, falling back to the disk queue if the memory queue is empty. @@ -272,7 +272,7 @@ class Scheduler(BaseScheduler): Increment the appropriate stats, such as: ``scheduler/dequeued``, ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``. """ - request: Optional[Request] = self.mqs.pop() + request: Request | None = self.mqs.pop() assert self.stats is not None if request is not None: self.stats.inc_value("scheduler/dequeued/memory", spider=self.spider) @@ -318,7 +318,7 @@ class Scheduler(BaseScheduler): def _mqpush(self, request: Request) -> None: self.mqs.push(request) - def _dqpop(self) -> Optional[Request]: + def _dqpop(self) -> Request | None: if self.dqs is not None: return self.dqs.pop() return None @@ -355,7 +355,7 @@ class Scheduler(BaseScheduler): ) return q - def _dqdir(self, jobdir: Optional[str]) -> Optional[str]: + def _dqdir(self, jobdir: str | None) -> str | None: """Return a folder name to keep disk queue state at""" if jobdir: dqdir = Path(jobdir, "requests.queue") diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 71a0d6aeb..83dad0c0b 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -6,7 +6,7 @@ from __future__ import annotations import logging from collections import deque from collections.abc import AsyncIterable, Iterator -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, TypeVar, Union, cast from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks @@ -42,11 +42,8 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") _ParallelResult = list[tuple[bool, Iterator[Any]]] - -if TYPE_CHECKING: - # parameterized Deferreds require Twisted 21.7.0 - _HandleOutputDeferred = Deferred[Union[_ParallelResult, None]] - QueueTuple = tuple[Union[Response, Failure], Request, _HandleOutputDeferred] +_HandleOutputDeferred = Deferred[Union[_ParallelResult, None]] +QueueTuple = tuple[Union[Response, Failure], Request, _HandleOutputDeferred] class Slot: @@ -60,10 +57,10 @@ class Slot: self.active: set[Request] = set() self.active_size: int = 0 self.itemproc_size: int = 0 - self.closing: Optional[Deferred[Spider]] = None + self.closing: Deferred[Spider] | None = None def add_response_request( - self, result: Union[Response, Failure], request: Request + self, result: Response | Failure, request: Request ) -> _HandleOutputDeferred: deferred: _HandleOutputDeferred = Deferred() self.queue.append((result, request, deferred)) @@ -78,9 +75,7 @@ class Slot: self.active.add(request) return response, request, deferred - def finish_response( - self, result: Union[Response, Failure], request: Request - ) -> None: + def finish_response(self, result: Response | Failure, request: Request) -> None: self.active.remove(request) if isinstance(result, Response): self.active_size -= max(len(result.body), self.MIN_RESPONSE_SIZE) @@ -96,7 +91,7 @@ class Slot: class Scraper: def __init__(self, crawler: Crawler) -> None: - self.slot: Optional[Slot] = None + self.slot: Slot | None = None self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler( crawler ) @@ -135,7 +130,7 @@ class Scraper: self.slot.closing.callback(spider) def enqueue_scrape( - self, result: Union[Response, Failure], request: Request, spider: Spider + self, result: Response | Failure, request: Request, spider: Spider ) -> _HandleOutputDeferred: if self.slot is None: raise RuntimeError("Scraper slot not assigned") @@ -167,7 +162,7 @@ class Scraper: self._scrape(response, request, spider).chainDeferred(deferred) def _scrape( - self, result: Union[Response, Failure], request: Request, spider: Spider + self, result: Response | Failure, request: Request, spider: Spider ) -> _HandleOutputDeferred: """ Handle the downloaded response or failure through the spider callback/errback @@ -176,7 +171,7 @@ class Scraper: raise TypeError( f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}" ) - dfd: Deferred[Union[Iterable[Any], AsyncIterable[Any]]] = self._scrape2( + dfd: Deferred[Iterable[Any] | AsyncIterable[Any]] = self._scrape2( result, request, spider ) # returns spider's processed output dfd.addErrback(self.handle_spider_error, request, result, spider) @@ -186,8 +181,8 @@ class Scraper: return dfd2 def _scrape2( - self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred[Union[Iterable[Any], AsyncIterable[Any]]]: + self, result: Response | Failure, request: Request, spider: Spider + ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: """ Handle the different cases of request's result been a Response or a Failure """ @@ -202,8 +197,8 @@ class Scraper: return dfd def call_spider( - self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred[Union[Iterable[Any], AsyncIterable[Any]]]: + self, result: Response | Failure, request: Request, spider: Spider + ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: dfd: Deferred[Any] if isinstance(result, Response): if getattr(result, "request", None) is None: @@ -222,7 +217,7 @@ class Scraper: if request.errback: warn_on_generator_with_return_value(spider, request.errback) dfd.addErrback(request.errback) - dfd2: Deferred[Union[Iterable[Any], AsyncIterable[Any]]] = dfd.addCallback( + dfd2: Deferred[Iterable[Any] | AsyncIterable[Any]] = dfd.addCallback( iterate_spider_output ) return dfd2 @@ -231,7 +226,7 @@ class Scraper: self, _failure: Failure, request: Request, - response: Union[Response, Failure], + response: Response | Failure, spider: Spider, ) -> None: exc = _failure.value @@ -258,14 +253,14 @@ class Scraper: def handle_spider_output( self, - result: Union[Iterable[_T], AsyncIterable[_T]], + result: Iterable[_T] | AsyncIterable[_T], request: Request, response: Response, spider: Spider, ) -> _HandleOutputDeferred: if not result: return defer_succeed(None) - it: Union[Iterable[_T], AsyncIterable[_T]] + it: Iterable[_T] | AsyncIterable[_T] dfd: Deferred[_ParallelResult] if isinstance(result, AsyncIterable): it = aiter_errback( @@ -296,7 +291,7 @@ class Scraper: def _process_spidermw_output( self, output: Any, request: Request, response: Response, spider: Spider - ) -> Optional[Deferred[Any]]: + ) -> Deferred[Any] | None: """Process each Request/Item (given in the output parameter) returned from the given spider """ @@ -316,9 +311,7 @@ class Scraper: ) return None - def start_itemproc( - self, item: Any, *, response: Optional[Response] - ) -> Deferred[Any]: + def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[Any]: """Send *item* to the item pipelines for processing. *response* is the source of the item data. If the item does not come @@ -337,7 +330,7 @@ class Scraper: download_failure: Failure, request: Request, spider: Spider, - ) -> Union[Failure, None]: + ) -> Failure | None: """Log and silence errors that come from the engine (typically download errors that got propagated thru here). @@ -371,7 +364,7 @@ class Scraper: return None def _itemproc_finished( - self, output: Any, item: Any, response: Optional[Response], spider: Spider + self, output: Any, item: Any, response: Response | None, spider: Spider ) -> Deferred[Any]: """ItemProcessor finished for the given ``item`` and returned ``output``""" assert self.slot is not None # typing diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 3c8513042..1edfe1c51 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -10,7 +10,7 @@ import logging from collections.abc import AsyncIterable, Callable, Iterable from inspect import isasyncgenfunction, iscoroutine from itertools import islice -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, TypeVar, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure @@ -76,7 +76,7 @@ class SpiderMiddlewareManager(MiddlewareManager): response: Response, request: Request, spider: Spider, - ) -> Union[Iterable[_T], AsyncIterable[_T]]: + ) -> Iterable[_T] | AsyncIterable[_T]: for method in self.methods["process_spider_input"]: method = cast(Callable, method) try: @@ -97,10 +97,10 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - iterable: Union[Iterable[_T], AsyncIterable[_T]], + iterable: Iterable[_T] | AsyncIterable[_T], exception_processor_index: int, - recover_to: Union[MutableChain[_T], MutableAsyncChain[_T]], - ) -> Union[Iterable[_T], AsyncIterable[_T]]: + recover_to: MutableChain[_T] | MutableAsyncChain[_T], + ) -> Iterable[_T] | AsyncIterable[_T]: def process_sync(iterable: Iterable[_T]) -> Iterable[_T]: try: yield from iterable @@ -142,7 +142,7 @@ class SpiderMiddlewareManager(MiddlewareManager): spider: Spider, _failure: Failure, start_index: int = 0, - ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]: + ) -> Failure | MutableChain[_T] | MutableAsyncChain[_T]: exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): @@ -158,7 +158,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned - dfd: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = ( + dfd: Deferred[MutableChain[_T] | MutableAsyncChain[_T]] = ( self._process_spider_output( response, spider, result, method_index + 1 ) @@ -192,12 +192,12 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - result: Union[Iterable[_T], AsyncIterable[_T]], + result: Iterable[_T] | AsyncIterable[_T], start_index: int = 0, - ) -> Generator[Deferred[Any], Any, Union[MutableChain[_T], MutableAsyncChain[_T]]]: + ) -> Generator[Deferred[Any], Any, MutableChain[_T] | MutableAsyncChain[_T]]: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - recovered: Union[MutableChain[_T], MutableAsyncChain[_T]] + recovered: MutableChain[_T] | MutableAsyncChain[_T] last_result_is_async = isinstance(result, AsyncIterable) if last_result_is_async: recovered = MutableAsyncChain() @@ -248,10 +248,10 @@ class SpiderMiddlewareManager(MiddlewareManager): # might fail directly if the output value is not a generator result = method(response=response, result=result, spider=spider) except Exception as ex: - exception_result: Union[ - Failure, MutableChain[_T], MutableAsyncChain[_T] - ] = self._process_spider_exception( - response, spider, Failure(ex), method_index + 1 + exception_result: Failure | MutableChain[_T] | MutableAsyncChain[_T] = ( + self._process_spider_exception( + response, spider, Failure(ex), method_index + 1 + ) ) if isinstance(exception_result, Failure): raise @@ -283,9 +283,9 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - result: Union[Iterable[_T], AsyncIterable[_T]], - ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]: - recovered: Union[MutableChain[_T], MutableAsyncChain[_T]] + result: Iterable[_T] | AsyncIterable[_T], + ) -> MutableChain[_T] | MutableAsyncChain[_T]: + recovered: MutableChain[_T] | MutableAsyncChain[_T] if isinstance(result, AsyncIterable): recovered = MutableAsyncChain() else: @@ -293,7 +293,7 @@ class SpiderMiddlewareManager(MiddlewareManager): result = self._evaluate_iterable(response, spider, result, 0, recovered) result = await maybe_deferred_to_future( cast( - "Deferred[Union[Iterable[_T], AsyncIterable[_T]]]", + "Deferred[Iterable[_T] | AsyncIterable[_T]]", self._process_spider_output(response, spider, result), ) ) @@ -310,22 +310,22 @@ class SpiderMiddlewareManager(MiddlewareManager): response: Response, request: Request, spider: Spider, - ) -> Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]]: + ) -> Deferred[MutableChain[_T] | MutableAsyncChain[_T]]: async def process_callback_output( - result: Union[Iterable[_T], AsyncIterable[_T]] - ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]: + result: Iterable[_T] | AsyncIterable[_T], + ) -> MutableChain[_T] | MutableAsyncChain[_T]: return await self._process_callback_output(response, spider, result) def process_spider_exception( _failure: Failure, - ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]: + ) -> Failure | MutableChain[_T] | MutableAsyncChain[_T]: return self._process_spider_exception(response, spider, _failure) - dfd: Deferred[Union[Iterable[_T], AsyncIterable[_T]]] = mustbe_deferred( + dfd: Deferred[Iterable[_T] | AsyncIterable[_T]] = mustbe_deferred( self._process_spider_input, scrape_func, response, request, spider ) - dfd2: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = ( - dfd.addCallback(deferred_f_from_coro_f(process_callback_output)) + dfd2: Deferred[MutableChain[_T] | MutableAsyncChain[_T]] = dfd.addCallback( + deferred_f_from_coro_f(process_callback_output) ) dfd2.addErrback(process_spider_exception) return dfd2 @@ -339,10 +339,10 @@ class SpiderMiddlewareManager(MiddlewareManager): @staticmethod def _get_async_method_pair( mw: Any, methodname: str - ) -> Union[None, Callable, tuple[Callable, Callable]]: - normal_method: Optional[Callable] = getattr(mw, methodname, None) + ) -> None | Callable | tuple[Callable, Callable]: + normal_method: Callable | None = getattr(mw, methodname, None) methodname_async = methodname + "_async" - async_method: Optional[Callable] = getattr(mw, methodname_async, None) + async_method: Callable | None = getattr(mw, methodname_async, None) if not async_method: return normal_method if not normal_method: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index e75ef52ac..701dccf57 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,7 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, TypeVar, cast from twisted.internet.defer import ( Deferred, @@ -57,7 +57,7 @@ class Crawler: def __init__( self, spidercls: type[Spider], - settings: Union[None, dict[str, Any], Settings] = None, + settings: None | dict[str, Any] | Settings = None, init_reactor: bool = False, ): if isinstance(spidercls, Spider): @@ -78,12 +78,12 @@ class Crawler: self.crawling: bool = False self._started: bool = False - self.extensions: Optional[ExtensionManager] = None - self.stats: Optional[StatsCollector] = None - self.logformatter: Optional[LogFormatter] = None - self.request_fingerprinter: Optional[RequestFingerprinter] = None - self.spider: Optional[Spider] = None - self.engine: Optional[ExecutionEngine] = None + self.extensions: ExtensionManager | None = None + self.stats: StatsCollector | None = None + self.logformatter: LogFormatter | None = None + self.request_fingerprinter: RequestFingerprinter | None = None + self.spider: Spider | None = None + self.engine: ExecutionEngine | None = None def _update_root_log_handler(self) -> None: if get_scrapy_root_handler() is not None: @@ -181,16 +181,16 @@ class Crawler: @staticmethod def _get_component( component_class: type[_T], components: Iterable[Any] - ) -> Optional[_T]: + ) -> _T | None: for component in components: if isinstance(component, component_class): return component return None - def get_addon(self, cls: type[_T]) -> Optional[_T]: + def get_addon(self, cls: type[_T]) -> _T | None: return self._get_component(cls, self.addons.addons) - def get_downloader_middleware(self, cls: type[_T]) -> Optional[_T]: + def get_downloader_middleware(self, cls: type[_T]) -> _T | None: if not self.engine: raise RuntimeError( "Crawler.get_downloader_middleware() can only be called after " @@ -198,7 +198,7 @@ class Crawler: ) return self._get_component(cls, self.engine.downloader.middleware.middlewares) - def get_extension(self, cls: type[_T]) -> Optional[_T]: + def get_extension(self, cls: type[_T]) -> _T | None: if not self.extensions: raise RuntimeError( "Crawler.get_extension() can only be called after the " @@ -206,7 +206,7 @@ class Crawler: ) return self._get_component(cls, self.extensions.middlewares) - def get_item_pipeline(self, cls: type[_T]) -> Optional[_T]: + def get_item_pipeline(self, cls: type[_T]) -> _T | None: if not self.engine: raise RuntimeError( "Crawler.get_item_pipeline() can only be called after the " @@ -214,7 +214,7 @@ class Crawler: ) return self._get_component(cls, self.engine.scraper.itemproc.middlewares) - def get_spider_middleware(self, cls: type[_T]) -> Optional[_T]: + def get_spider_middleware(self, cls: type[_T]) -> _T | None: if not self.engine: raise RuntimeError( "Crawler.get_spider_middleware() can only be called after the " @@ -250,7 +250,7 @@ class CrawlerRunner: verifyClass(ISpiderLoader, loader_cls) return cast("SpiderLoader", loader_cls.from_settings(settings.frozencopy())) - def __init__(self, settings: Union[dict[str, Any], Settings, None] = None): + def __init__(self, settings: dict[str, Any] | Settings | None = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings: Settings = settings @@ -261,7 +261,7 @@ class CrawlerRunner: def crawl( self, - crawler_or_spidercls: Union[type[Spider], str, Crawler], + crawler_or_spidercls: type[Spider] | str | Crawler, *args: Any, **kwargs: Any, ) -> Deferred[None]: @@ -308,7 +308,7 @@ class CrawlerRunner: return d.addBoth(_done) def create_crawler( - self, crawler_or_spidercls: Union[type[Spider], str, Crawler] + self, crawler_or_spidercls: type[Spider] | str | Crawler ) -> Crawler: """ Return a :class:`~scrapy.crawler.Crawler` object. @@ -329,7 +329,7 @@ class CrawlerRunner: return crawler_or_spidercls return self._create_crawler(crawler_or_spidercls) - def _create_crawler(self, spidercls: Union[str, type[Spider]]) -> Crawler: + def _create_crawler(self, spidercls: str | type[Spider]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) return Crawler(spidercls, self.settings) @@ -380,7 +380,7 @@ class CrawlerProcess(CrawlerRunner): def __init__( self, - settings: Union[dict[str, Any], Settings, None] = None, + settings: dict[str, Any] | Settings | None = None, install_root_handler: bool = True, ): super().__init__(settings) @@ -409,7 +409,7 @@ class CrawlerProcess(CrawlerRunner): ) reactor.callFromThread(self._stop_reactor) - def _create_crawler(self, spidercls: Union[type[Spider], str]) -> Crawler: + def _create_crawler(self, spidercls: type[Spider] | str) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) init_reactor = not self._initialized_reactor diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 5fc7f31a3..b813baf86 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging import re -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from w3lib import html @@ -43,7 +43,7 @@ class AjaxCrawlMiddleware: def process_response( self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + ) -> Request | Response: if not isinstance(response, HtmlResponse) or response.status != 200: return response diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index e384793ee..545dcaac9 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging from collections import defaultdict -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any from tldextract import TLDExtract @@ -70,7 +70,7 @@ class CookiesMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: if request.meta.get("dont_merge_cookies", False): return None @@ -87,7 +87,7 @@ class CookiesMiddleware: def process_response( self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + ) -> Request | Response: if request.meta.get("dont_merge_cookies", False): return response @@ -123,7 +123,7 @@ class CookiesMiddleware: msg = f"Received cookies from: {response}\n{cookies}" logger.debug(msg, extra={"spider": spider}) - def _format_cookie(self, cookie: VerboseCookie, request: Request) -> Optional[str]: + def _format_cookie(self, cookie: VerboseCookie, request: Request) -> str | None: """ Given a dict consisting of cookie components, return its string representation. Decode from bytes if necessary. diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 312c1e026..d58b4490b 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -6,7 +6,7 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from scrapy.utils.python import without_none_values @@ -32,7 +32,7 @@ class DefaultHeadersMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: for k, v in self._headers: request.headers.setdefault(k, v) return None diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index ee7a24825..28456c697 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -6,7 +6,7 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from scrapy import Request, Spider, signals @@ -33,7 +33,7 @@ class DownloadTimeoutMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: if self._timeout: request.meta.setdefault("download_timeout", self._timeout) return None diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 39165e155..b74140ee1 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -6,7 +6,7 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from w3lib.http import basic_auth_header @@ -40,7 +40,7 @@ class HttpAuthMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: auth = getattr(self, "auth", None) if auth and b"Authorization" not in request.headers: if not self.domain or url_is_from_any_domain(request.url, [self.domain]): diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 8377a3c1d..3892dba23 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,7 +1,7 @@ from __future__ import annotations from email.utils import formatdate -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING from twisted.internet import defer from twisted.internet.error import ( @@ -69,7 +69,7 @@ class HttpCacheMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: if request.meta.get("dont_cache", False): return None @@ -79,7 +79,7 @@ class HttpCacheMiddleware: return None # Look for cached response and check if expired - cachedresponse: Optional[Response] = self.storage.retrieve_response( + cachedresponse: Response | None = self.storage.retrieve_response( spider, request ) if cachedresponse is None: @@ -103,7 +103,7 @@ class HttpCacheMiddleware: def process_response( self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + ) -> Request | Response: if request.meta.get("dont_cache", False): return response @@ -118,7 +118,7 @@ class HttpCacheMiddleware: response.headers["Date"] = formatdate(usegmt=True) # Do not validate first-hand responses - cachedresponse: Optional[Response] = request.meta.pop("cached_response", None) + cachedresponse: Response | None = request.meta.pop("cached_response", None) if cachedresponse is None: self.stats.inc_value("httpcache/firsthand", spider=spider) self._cache_response(spider, response, request, cachedresponse) @@ -134,8 +134,8 @@ class HttpCacheMiddleware: def process_exception( self, request: Request, exception: Exception, spider: Spider - ) -> Union[Request, Response, None]: - cachedresponse: Optional[Response] = request.meta.pop("cached_response", None) + ) -> Request | Response | None: + cachedresponse: Response | None = request.meta.pop("cached_response", None) if cachedresponse is not None and isinstance( exception, self.DOWNLOAD_EXCEPTIONS ): @@ -148,7 +148,7 @@ class HttpCacheMiddleware: spider: Spider, response: Response, request: Request, - cachedresponse: Optional[Response], + cachedresponse: Response | None, ) -> None: if self.policy.should_cache_response(response, request): self.stats.inc_value("httpcache/store", spider=spider) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index d913ca25d..84678b8e9 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -3,7 +3,7 @@ from __future__ import annotations import warnings from itertools import chain from logging import getLogger -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals from scrapy.exceptions import IgnoreRequest, NotConfigured @@ -54,9 +54,9 @@ class HttpCompressionMiddleware: def __init__( self, - stats: Optional[StatsCollector] = None, + stats: StatsCollector | None = None, *, - crawler: Optional[Crawler] = None, + crawler: Crawler | None = None, ): if not crawler: self.stats = stats @@ -96,13 +96,13 @@ class HttpCompressionMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS)) return None def process_response( self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + ) -> Request | Response: if request.method == "HEAD": return response if isinstance(response, Response): diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index b35ecbd54..2f3f2db47 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,7 +1,7 @@ from __future__ import annotations import base64 -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING from urllib.parse import unquote, urlunparse from urllib.request import ( # type: ignore[attr-defined] _parse_proxy, @@ -23,9 +23,9 @@ if TYPE_CHECKING: class HttpProxyMiddleware: - def __init__(self, auth_encoding: Optional[str] = "latin-1"): - self.auth_encoding: Optional[str] = auth_encoding - self.proxies: dict[str, tuple[Optional[bytes], str]] = {} + def __init__(self, auth_encoding: str | None = "latin-1"): + self.auth_encoding: str | None = auth_encoding + self.proxies: dict[str, tuple[bytes | None, str]] = {} for type_, url in getproxies().items(): try: self.proxies[type_] = self._get_proxy(url, type_) @@ -38,7 +38,7 @@ class HttpProxyMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("HTTPPROXY_ENABLED"): raise NotConfigured - auth_encoding: Optional[str] = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") + auth_encoding: str | None = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") return cls(auth_encoding) def _basic_auth_header(self, username: str, password: str) -> bytes: @@ -47,7 +47,7 @@ class HttpProxyMiddleware: ) return base64.b64encode(user_pass) - def _get_proxy(self, url: str, orig_type: str) -> tuple[Optional[bytes], str]: + def _get_proxy(self, url: str, orig_type: str) -> tuple[bytes | None, str]: proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", "")) @@ -60,7 +60,7 @@ class HttpProxyMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: creds, proxy_url, scheme = None, None, None if "proxy" in request.meta: if request.meta["proxy"] is not None: @@ -82,9 +82,9 @@ class HttpProxyMiddleware: def _set_proxy_and_creds( self, request: Request, - proxy_url: Optional[str], - creds: Optional[bytes], - scheme: Optional[str], + proxy_url: str | None, + creds: bytes | None, + scheme: str | None, ) -> None: if scheme: request.meta["_scheme_proxy"] = True diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 6437485cf..0b883b43a 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Union, cast +from typing import TYPE_CHECKING, Any, cast from urllib.parse import urljoin from w3lib.url import safe_url_string @@ -144,7 +144,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): def process_response( self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + ) -> Request | Response: if ( request.meta.get("dont_redirect", False) or response.status in getattr(spider, "handle_httpstatus_list", []) @@ -185,7 +185,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): def process_response( self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + ) -> Request | Response: if ( request.meta.get("dont_redirect", False) or request.method == "HEAD" diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index c32624371..7c0e2280c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -14,7 +14,7 @@ from __future__ import annotations import warnings from logging import Logger, getLogger -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.settings import BaseSettings, Settings @@ -60,12 +60,12 @@ def get_retry_request( request: Request, *, spider: Spider, - reason: Union[str, Exception, type[Exception]] = "unspecified", - max_retry_times: Optional[int] = None, - priority_adjust: Optional[int] = None, + reason: str | Exception | type[Exception] = "unspecified", + max_retry_times: int | None = None, + priority_adjust: int | None = None, logger: Logger = retry_logger, stats_base_key: str = "retry", -) -> Optional[Request]: +) -> Request | None: """ Returns a new :class:`~scrapy.Request` object to retry the specified request, or ``None`` if retries of the specified request have been @@ -167,7 +167,7 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): def process_response( self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + ) -> Request | Response: if request.meta.get("dont_retry", False): return response if response.status in self.retry_http_codes: @@ -177,7 +177,7 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): def process_exception( self, request: Request, exception: Exception, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: if isinstance(exception, self.exceptions_to_retry) and not request.meta.get( "dont_retry", False ): @@ -187,9 +187,9 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): def _retry( self, request: Request, - reason: Union[str, Exception, type[Exception]], + reason: str | Exception | type[Exception], spider: Spider, - ) -> Optional[Request]: + ) -> Request | None: max_retry_times = request.meta.get("max_retry_times", self.max_retry_times) priority_adjust = request.meta.get("priority_adjust", self.priority_adjust) return get_retry_request( diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 81ba009d6..ea9f47d69 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. from __future__ import annotations import logging -from typing import TYPE_CHECKING, Optional, TypeVar, Union +from typing import TYPE_CHECKING, TypeVar from twisted.internet.defer import Deferred, maybeDeferred @@ -41,13 +41,11 @@ class RobotsTxtMiddleware: if not crawler.settings.getbool("ROBOTSTXT_OBEY"): raise NotConfigured self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy") - self._robotstxt_useragent: Optional[str] = crawler.settings.get( + self._robotstxt_useragent: str | None = crawler.settings.get( "ROBOTSTXT_USER_AGENT", None ) self.crawler: Crawler = crawler - self._parsers: dict[ - str, Union[RobotParser, Deferred[Optional[RobotParser]], None] - ] = {} + self._parsers: dict[str, RobotParser | Deferred[RobotParser | None] | None] = {} self._parserimpl: RobotParser = load_object( crawler.settings.get("ROBOTSTXT_PARSER") ) @@ -61,24 +59,24 @@ class RobotsTxtMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Optional[Deferred[None]]: + ) -> Deferred[None] | None: if request.meta.get("dont_obey_robotstxt"): return None if request.url.startswith("data:") or request.url.startswith("file:"): return None - d: Deferred[Optional[RobotParser]] = maybeDeferred( + d: Deferred[RobotParser | None] = maybeDeferred( self.robot_parser, request, spider # type: ignore[call-overload] ) d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider) return d2 def process_request_2( - self, rp: Optional[RobotParser], request: Request, spider: Spider + self, rp: RobotParser | None, request: Request, spider: Spider ) -> None: if rp is None: return - useragent: Union[str, bytes, None] = self._robotstxt_useragent + useragent: str | bytes | None = self._robotstxt_useragent if not useragent: useragent = request.headers.get(b"User-Agent", self._default_useragent) assert useragent is not None @@ -94,7 +92,7 @@ class RobotsTxtMiddleware: def robot_parser( self, request: Request, spider: Spider - ) -> Union[RobotParser, Deferred[Optional[RobotParser]], None]: + ) -> RobotParser | Deferred[RobotParser | None] | None: url = urlparse_cached(request) netloc = url.netloc @@ -117,9 +115,9 @@ class RobotsTxtMiddleware: parser = self._parsers[netloc] if isinstance(parser, Deferred): - d: Deferred[Optional[RobotParser]] = Deferred() + d: Deferred[RobotParser | None] = Deferred() - def cb(result: Optional[RobotParser]) -> Optional[RobotParser]: + def cb(result: RobotParser | None) -> RobotParser | None: d.callback(result) return result diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index ab5655393..fb0f30620 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from twisted.web import http @@ -19,7 +19,7 @@ if TYPE_CHECKING: def get_header_size( - headers: dict[str, Union[list[Union[str, bytes]], tuple[Union[str, bytes], ...]]] + headers: dict[str, list[str | bytes] | tuple[str | bytes, ...]] ) -> int: size = 0 for key, value in headers.items(): @@ -47,7 +47,7 @@ class DownloaderStats: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: self.stats.inc_value("downloader/request_count", spider=spider) self.stats.inc_value( f"downloader/request_method_count/{request.method}", spider=spider @@ -58,7 +58,7 @@ class DownloaderStats: def process_response( self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + ) -> Request | Response: self.stats.inc_value("downloader/response_count", spider=spider) self.stats.inc_value( f"downloader/response_status_count/{response.status}", spider=spider @@ -75,7 +75,7 @@ class DownloaderStats: def process_exception( self, request: Request, exception: Exception, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: ex_class = global_object_name(exception.__class__) self.stats.inc_value("downloader/exception_count", spider=spider) self.stats.inc_value( diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index 109f1a4d9..ba379f862 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from scrapy import Request, Spider, signals @@ -31,7 +31,7 @@ class UserAgentMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: if self.user_agent: request.headers.setdefault(b"User-Agent", self.user_agent) return None diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 28118977d..d37d2741a 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging from pathlib import Path -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from scrapy.utils.job import job_dir from scrapy.utils.request import ( @@ -31,10 +31,10 @@ class BaseDupeFilter: def request_seen(self, request: Request) -> bool: return False - def open(self) -> Optional[Deferred[None]]: + def open(self) -> Deferred[None] | None: pass - def close(self, reason: str) -> Optional[Deferred[None]]: + def close(self, reason: str) -> Deferred[None] | None: pass def log(self, request: Request, spider: Spider) -> None: @@ -47,10 +47,10 @@ class RFPDupeFilter(BaseDupeFilter): def __init__( self, - path: Optional[str] = None, + path: str | None = None, debug: bool = False, *, - fingerprinter: Optional[RequestFingerprinterProtocol] = None, + fingerprinter: RequestFingerprinterProtocol | None = None, ) -> None: self.file = None self.fingerprinter: RequestFingerprinterProtocol = ( @@ -70,7 +70,7 @@ class RFPDupeFilter(BaseDupeFilter): cls, settings: BaseSettings, *, - fingerprinter: Optional[RequestFingerprinterProtocol] = None, + fingerprinter: RequestFingerprinterProtocol | None = None, ) -> Self: debug = settings.getbool("DUPEFILTER_DEBUG") return cls(job_dir(settings), debug, fingerprinter=fingerprinter) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index ee0033dfb..c9350a956 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -2,6 +2,8 @@ Item Exporters are used to export/serialize items into different formats. """ +from __future__ import annotations + import csv import marshal import pickle # nosec @@ -9,7 +11,7 @@ import pprint from collections.abc import Callable, Iterable, Mapping from io import BytesIO, TextIOWrapper from json import JSONEncoder -from typing import Any, Optional, Union +from typing import Any from xml.sax.saxutils import XMLGenerator # nosec from xml.sax.xmlreader import AttributesImpl # nosec @@ -41,12 +43,12 @@ class BaseItemExporter: If dont_fail is set, it won't raise an exception on unexpected options (useful for using with keyword arguments in subclasses ``__init__`` methods) """ - self.encoding: Optional[str] = options.pop("encoding", None) - self.fields_to_export: Union[Mapping[str, str], Iterable[str], None] = ( - options.pop("fields_to_export", None) + self.encoding: str | None = options.pop("encoding", None) + self.fields_to_export: Mapping[str, str] | Iterable[str] | None = options.pop( + "fields_to_export", None ) self.export_empty_fields: bool = options.pop("export_empty_fields", False) - self.indent: Optional[int] = options.pop("indent", None) + self.indent: int | None = options.pop("indent", None) if not dont_fail and options: raise TypeError(f"Unexpected options: {', '.join(options.keys())}") @@ -54,7 +56,7 @@ class BaseItemExporter: raise NotImplementedError def serialize_field( - self, field: Union[Mapping[str, Any], Field], name: str, value: Any + self, field: Mapping[str, Any] | Field, name: str, value: Any ) -> Any: serializer: Callable[[Any], Any] = field.get("serializer", lambda x: x) return serializer(value) @@ -66,7 +68,7 @@ class BaseItemExporter: pass def _get_serialized_fields( - self, item: Any, default_value: Any = None, include_empty: Optional[bool] = None + self, item: Any, default_value: Any = None, include_empty: bool | None = None ) -> Iterable[tuple[str, Any]]: """Return the fields to export as an iterable of tuples (name, serialized_value) @@ -225,7 +227,7 @@ class CsvItemExporter(BaseItemExporter): file: BytesIO, include_headers_line: bool = True, join_multivalued: str = ",", - errors: Optional[str] = None, + errors: str | None = None, **kwargs: Any, ): super().__init__(dont_fail=True, **kwargs) @@ -245,7 +247,7 @@ class CsvItemExporter(BaseItemExporter): self._join_multivalued = join_multivalued def serialize_field( - self, field: Union[Mapping[str, Any], Field], name: str, value: Any + self, field: Mapping[str, Any] | Field, name: str, value: Any ) -> Any: serializer: Callable[[Any], Any] = field.get("serializer", self._join_if_needed) return serializer(value) @@ -346,7 +348,7 @@ class PythonItemExporter(BaseItemExporter): self.encoding = "utf-8" def serialize_field( - self, field: Union[Mapping[str, Any], Field], name: str, value: Any + self, field: Mapping[str, Any] | Field, name: str, value: Any ) -> Any: serializer: Callable[[Any], Any] = field.get( "serializer", self._serialize_value @@ -364,10 +366,10 @@ class PythonItemExporter(BaseItemExporter): return to_unicode(value, encoding=self.encoding) return value - def _serialize_item(self, item: Any) -> Iterable[tuple[Union[str, bytes], Any]]: + def _serialize_item(self, item: Any) -> Iterable[tuple[str | bytes, Any]]: for key, value in ItemAdapter(item).items(): yield key, self._serialize_value(value) - def export_item(self, item: Any) -> dict[Union[str, bytes], Any]: # type: ignore[override] - result: dict[Union[str, bytes], Any] = dict(self._get_serialized_fields(item)) + def export_item(self, item: Any) -> dict[str | bytes, Any]: # type: ignore[override] + result: dict[str | bytes, Any] = dict(self._get_serialized_fields(item)) return result diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 6ef2d0382..779cd5d1c 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -5,7 +5,7 @@ Extension for collecting core stats like items scraped and start/finish times from __future__ import annotations from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any from scrapy import Spider, signals @@ -20,7 +20,7 @@ if TYPE_CHECKING: class CoreStats: def __init__(self, stats: StatsCollector): self.stats: StatsCollector = stats - self.start_time: Optional[datetime] = None + self.start_time: datetime | None = None @classmethod def from_crawler(cls, crawler: Crawler) -> Self: diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index c54871e02..d3c225bcd 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -12,7 +12,7 @@ import sys import threading import traceback from pdb import Pdb -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from scrapy.utils.engine import format_engine_status from scrapy.utils.trackref import format_live_refs @@ -43,7 +43,7 @@ class StackTraceDump: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def dump_stacktrace(self, signum: int, frame: Optional[FrameType]) -> None: + def dump_stacktrace(self, signum: int, frame: FrameType | None) -> None: assert self.crawler.engine log_args = { "stackdumps": self._thread_stacks(), @@ -75,6 +75,6 @@ class Debugger: # win32 platforms don't support SIGUSR signals pass - def _enter_debugger(self, signum: int, frame: Optional[FrameType]) -> None: + def _enter_debugger(self, signum: int, frame: FrameType | None) -> None: assert frame Pdb().set_trace(frame.f_back) # noqa: T100 diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 7bfcbe6f3..eb1698ce5 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -14,7 +14,7 @@ from collections.abc import Callable from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import IO, TYPE_CHECKING, Any, Optional, Protocol, TypeVar, Union, cast +from typing import IO, TYPE_CHECKING, Any, Optional, Protocol, TypeVar, cast from urllib.parse import unquote, urlparse from twisted.internet.defer import Deferred, DeferredList, maybeDeferred @@ -67,7 +67,7 @@ def build_storage( builder: Callable[..., _StorageT], uri: str, *args: Any, - feed_options: Optional[dict[str, Any]] = None, + feed_options: dict[str, Any] | None = None, preargs: Iterable[Any] = (), **kwargs: Any, ) -> _StorageT: @@ -84,10 +84,10 @@ class ItemFilter: :type feed_options: dict """ - feed_options: Optional[dict[str, Any]] + feed_options: dict[str, Any] | None item_classes: tuple[type, ...] - def __init__(self, feed_options: Optional[dict[str, Any]]) -> None: + def __init__(self, feed_options: dict[str, Any] | None) -> None: self.feed_options = feed_options if feed_options is not None: self.item_classes = tuple( @@ -129,7 +129,7 @@ class IFeedStorage(Interface): class FeedStorageProtocol(Protocol): """Reimplementation of ``IFeedStorage`` that can be used in type hints.""" - def __init__(self, uri: str, *, feed_options: Optional[dict[str, Any]] = None): + def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None): """Initialize the storage with the parameters given in the URI and the feed-specific options (see :setting:`FEEDS`)""" @@ -137,7 +137,7 @@ class FeedStorageProtocol(Protocol): """Open the storage for the given spider. It must return a file-like object that will be used for the exporters""" - def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: + def store(self, file: IO[bytes]) -> Deferred[None] | None: """Store the given file stream""" @@ -150,7 +150,7 @@ class BlockingFeedStorage: return NamedTemporaryFile(prefix="feed-", dir=path) - def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: + def store(self, file: IO[bytes]) -> Deferred[None] | None: return deferToThread(self._store_in_thread, file) def _store_in_thread(self, file: IO[bytes]) -> None: @@ -162,9 +162,9 @@ class StdoutFeedStorage: def __init__( self, uri: str, - _stdout: Optional[IO[bytes]] = None, + _stdout: IO[bytes] | None = None, *, - feed_options: Optional[dict[str, Any]] = None, + feed_options: dict[str, Any] | None = None, ): if not _stdout: _stdout = sys.stdout.buffer @@ -180,13 +180,13 @@ class StdoutFeedStorage: def open(self, spider: Spider) -> IO[bytes]: return self._stdout - def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: + def store(self, file: IO[bytes]) -> Deferred[None] | None: pass @implementer(IFeedStorage) class FileFeedStorage: - def __init__(self, uri: str, *, feed_options: Optional[dict[str, Any]] = None): + def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None): self.path: str = file_uri_to_path(uri) feed_options = feed_options or {} self.write_mode: OpenBinaryMode = ( @@ -199,7 +199,7 @@ class FileFeedStorage: dirname.mkdir(parents=True) return Path(self.path).open(self.write_mode) - def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: + def store(self, file: IO[bytes]) -> Deferred[None] | None: file.close() return None @@ -208,27 +208,27 @@ class S3FeedStorage(BlockingFeedStorage): def __init__( self, uri: str, - access_key: Optional[str] = None, - secret_key: Optional[str] = None, - acl: Optional[str] = None, - endpoint_url: Optional[str] = None, + access_key: str | None = None, + secret_key: str | None = None, + acl: str | None = None, + endpoint_url: str | None = None, *, - feed_options: Optional[dict[str, Any]] = None, - session_token: Optional[str] = None, - region_name: Optional[str] = None, + feed_options: dict[str, Any] | None = None, + session_token: str | None = None, + region_name: str | None = None, ): if not is_botocore_available(): raise NotConfigured("missing botocore library") u = urlparse(uri) assert u.hostname self.bucketname: str = u.hostname - self.access_key: Optional[str] = u.username or access_key - self.secret_key: Optional[str] = u.password or secret_key - self.session_token: Optional[str] = session_token + self.access_key: str | None = u.username or access_key + self.secret_key: str | None = u.password or secret_key + self.session_token: str | None = session_token self.keyname: str = u.path[1:] # remove first "/" - self.acl: Optional[str] = acl - self.endpoint_url: Optional[str] = endpoint_url - self.region_name: Optional[str] = region_name + self.acl: str | None = acl + self.endpoint_url: str | None = endpoint_url + self.region_name: str | None = region_name # It can be either botocore.client.BaseClient or mypy_boto3_s3.S3Client, # there seems to be no good way to infer it statically. self.s3_client: Any @@ -279,7 +279,7 @@ class S3FeedStorage(BlockingFeedStorage): crawler: Crawler, uri: str, *, - feed_options: Optional[dict[str, Any]] = None, + feed_options: dict[str, Any] | None = None, ) -> Self: return build_storage( cls, @@ -310,9 +310,9 @@ class S3FeedStorage(BlockingFeedStorage): class GCSFeedStorage(BlockingFeedStorage): - def __init__(self, uri: str, project_id: Optional[str], acl: Optional[str]): - self.project_id: Optional[str] = project_id - self.acl: Optional[str] = acl + def __init__(self, uri: str, project_id: str | None, acl: str | None): + self.project_id: str | None = project_id + self.acl: str | None = acl u = urlparse(uri) assert u.hostname self.bucket_name: str = u.hostname @@ -342,7 +342,7 @@ class FTPFeedStorage(BlockingFeedStorage): uri: str, use_active_mode: bool = False, *, - feed_options: Optional[dict[str, Any]] = None, + feed_options: dict[str, Any] | None = None, ): u = urlparse(uri) if not u.hostname: @@ -361,7 +361,7 @@ class FTPFeedStorage(BlockingFeedStorage): crawler: Crawler, uri: str, *, - feed_options: Optional[dict[str, Any]] = None, + feed_options: dict[str, Any] | None = None, ) -> Self: return build_storage( cls, @@ -399,8 +399,8 @@ class FeedSlot: settings: BaseSettings, crawler: Crawler, ): - self.file: Optional[IO[bytes]] = None - self.exporter: Optional[BaseItemExporter] = None + self.file: IO[bytes] | None = None + self.exporter: BaseItemExporter | None = None self.storage: FeedStorageProtocol = storage # feed params self.batch_id: int = batch_id @@ -558,7 +558,7 @@ class FeedExporter: self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed) ) - def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred[None]]: + def _close_slot(self, slot: FeedSlot, spider: Spider) -> Deferred[None] | None: def get_file(slot_: FeedSlot) -> IO[bytes]: assert slot_.file if isinstance(slot_.file, PostProcessingManager): @@ -770,8 +770,8 @@ class FeedExporter: def _get_uri_params( self, spider: Spider, - uri_params_function: Union[str, UriParamsCallableT, None], - slot: Optional[FeedSlot] = None, + uri_params_function: str | UriParamsCallableT | None, + slot: FeedSlot | None = None, ) -> dict[str, Any]: params = {} for k in dir(spider): diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index a72f9db51..0e6120c21 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -9,7 +9,7 @@ from importlib import import_module from pathlib import Path from time import time from types import ModuleType -from typing import IO, TYPE_CHECKING, Any, Optional, Union, cast +from typing import IO, TYPE_CHECKING, Any, cast from weakref import WeakKeyDictionary from w3lib.http import headers_dict_to_raw, headers_raw_to_dict @@ -66,16 +66,14 @@ class RFC2616Policy: self.always_store: bool = settings.getbool("HTTPCACHE_ALWAYS_STORE") self.ignore_schemes: list[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") self._cc_parsed: WeakKeyDictionary[ - Union[Request, Response], dict[bytes, Optional[bytes]] + Request | Response, dict[bytes, bytes | None] ] = WeakKeyDictionary() self.ignore_response_cache_controls: list[bytes] = [ to_bytes(cc) for cc in settings.getlist("HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS") ] - def _parse_cachecontrol( - self, r: Union[Request, Response] - ) -> dict[bytes, Optional[bytes]]: + def _parse_cachecontrol(self, r: Request | Response) -> dict[bytes, bytes | None]: if r not in self._cc_parsed: cch = r.headers.get(b"Cache-Control", b"") assert cch is not None @@ -191,7 +189,7 @@ class RFC2616Policy: if b"ETag" in cachedresponse.headers: request.headers[b"If-None-Match"] = cachedresponse.headers[b"ETag"] - def _get_max_age(self, cc: dict[bytes, Optional[bytes]]) -> Optional[int]: + def _get_max_age(self, cc: dict[bytes, bytes | None]) -> int | None: try: return max(0, int(cc[b"max-age"])) # type: ignore[arg-type] except (KeyError, ValueError): @@ -275,7 +273,7 @@ class DbmCacheStorage: def close_spider(self, spider: Spider) -> None: self.db.close() - def retrieve_response(self, spider: Spider, request: Request) -> Optional[Response]: + def retrieve_response(self, spider: Spider, request: Request) -> Response | None: data = self._read_data(spider, request) if data is None: return None # not cached @@ -300,7 +298,7 @@ class DbmCacheStorage: self.db[f"{key}_data"] = pickle.dumps(data, protocol=4) self.db[f"{key}_time"] = str(time()) - def _read_data(self, spider: Spider, request: Request) -> Optional[dict[str, Any]]: + def _read_data(self, spider: Spider, request: Request) -> dict[str, Any] | None: key = self._fingerprinter.fingerprint(request).hex() db = self.db tkey = f"{key}_time" @@ -320,9 +318,7 @@ class FilesystemCacheStorage: self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP") # https://github.com/python/mypy/issues/10740 - self._open: Callable[ - Concatenate[Union[str, os.PathLike], str, ...], IO[bytes] - ] = ( + self._open: Callable[Concatenate[str | os.PathLike, str, ...], IO[bytes]] = ( gzip.open if self.use_gzip else open # type: ignore[assignment] ) @@ -339,7 +335,7 @@ class FilesystemCacheStorage: def close_spider(self, spider: Spider) -> None: pass - def retrieve_response(self, spider: Spider, request: Request) -> Optional[Response]: + def retrieve_response(self, spider: Spider, request: Request) -> Response | None: """Return response if present in cache, or None otherwise.""" metadata = self._read_meta(spider, request) if metadata is None: @@ -387,7 +383,7 @@ class FilesystemCacheStorage: key = self._fingerprinter.fingerprint(request).hex() return str(Path(self.cachedir, spider.name, key[0:2], key)) - def _read_meta(self, spider: Spider, request: Request) -> Optional[dict[str, Any]]: + def _read_meta(self, spider: Spider, request: Request) -> dict[str, Any] | None: rpath = Path(self._get_request_path(spider, request)) metapath = rpath / "pickled_meta" if not metapath.exists(): @@ -399,7 +395,7 @@ class FilesystemCacheStorage: return cast(dict[str, Any], pickle.load(f)) # nosec -def parse_cachecontrol(header: bytes) -> dict[bytes, Optional[bytes]]: +def parse_cachecontrol(header: bytes) -> dict[bytes, bytes | None]: """Parse Cache-Control header https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 @@ -419,7 +415,7 @@ def parse_cachecontrol(header: bytes) -> dict[bytes, Optional[bytes]]: return directives -def rfc1123_to_epoch(date_str: Union[str, bytes, None]) -> Optional[int]: +def rfc1123_to_epoch(date_str: str | bytes | None) -> int | None: try: date_str = to_unicode(date_str, encoding="ascii") # type: ignore[arg-type] return mktime_tz(parsedate_tz(date_str)) # type: ignore[arg-type] diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 01484481b..e829d8b92 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING from twisted.internet import task @@ -29,7 +29,7 @@ class LogStats: self.stats: StatsCollector = stats self.interval: float = interval self.multiplier: float = 60.0 / self.interval - self.task: Optional[task.LoopingCall] = None + self.task: task.LoopingCall | None = None @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -81,7 +81,7 @@ class LogStats: def calculate_final_stats( self, spider: Spider - ) -> Union[tuple[None, None], tuple[float, float]]: + ) -> tuple[None, None] | tuple[float, float]: start_time = self.stats.get_value("start_time") finished_time = self.stats.get_value("finished_time") diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index fba12bec7..f2e3782a4 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging from datetime import datetime, timezone from json import JSONEncoder -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any from twisted.internet import task @@ -36,7 +36,7 @@ class PeriodicLog: self.stats: StatsCollector = stats self.interval: float = interval self.multiplier: float = 60.0 / self.interval - self.task: Optional[task.LoopingCall] = None + self.task: task.LoopingCall | None = None self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) self.ext_stats_enabled: bool = bool(ext_stats) self.ext_stats_include: list[str] = ext_stats.get("include", []) @@ -52,7 +52,7 @@ class PeriodicLog: if not interval: raise NotConfigured try: - ext_stats: Optional[dict[str, Any]] = crawler.settings.getdict( + ext_stats: dict[str, Any] | None = crawler.settings.getdict( "PERIODIC_LOG_STATS" ) except (TypeError, ValueError): @@ -62,7 +62,7 @@ class PeriodicLog: else None ) try: - ext_delta: Optional[dict[str, Any]] = crawler.settings.getdict( + ext_delta: dict[str, Any] | None = crawler.settings.getdict( "PERIODIC_LOG_DELTA" ) except (TypeError, ValueError): @@ -93,8 +93,8 @@ class PeriodicLog: def spider_opened(self, spider: Spider) -> None: self.time_prev: datetime = datetime.now(tz=timezone.utc) - self.delta_prev: dict[str, Union[int, float]] = {} - self.stats_prev: dict[str, Union[int, float]] = {} + self.delta_prev: dict[str, int | float] = {} + self.stats_prev: dict[str, int | float] = {} self.task = task.LoopingCall(self.log) self.task.start(self.interval) @@ -110,7 +110,7 @@ class PeriodicLog: logger.info(self.encoder.encode(data)) def log_delta(self) -> dict[str, Any]: - num_stats: dict[str, Union[int, float]] = { + num_stats: dict[str, int | float] = { k: v for k, v in self.stats._stats.items() if isinstance(v, (int, float)) diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 567efd7a1..642919be9 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -2,7 +2,7 @@ from __future__ import annotations import pickle # nosec from pathlib import Path -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from scrapy import Spider, signals from scrapy.exceptions import NotConfigured @@ -18,8 +18,8 @@ if TYPE_CHECKING: class SpiderState: """Store and load spider state during a scraping job""" - def __init__(self, jobdir: Optional[str] = None): - self.jobdir: Optional[str] = jobdir + def __init__(self, jobdir: str | None = None): + self.jobdir: str | None = jobdir @classmethod def from_crawler(cls, crawler: Crawler) -> Self: diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index c8fefe792..600eebcf2 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -6,7 +6,7 @@ Use STATSMAILER_RCPTS setting to enable and give the recipient mail address from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from scrapy import Spider, signals from scrapy.exceptions import NotConfigured @@ -39,7 +39,7 @@ class StatsMailer: crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_closed(self, spider: Spider) -> Optional[Deferred[None]]: + def spider_closed(self, spider: Spider) -> Deferred[None] | None: spider_stats = self.stats.get_stats(spider) body = "Global stats\n\n" body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items()) diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 6b5fd181d..d4b4f0e9d 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured @@ -90,8 +90,8 @@ class AutoThrottle: def _get_slot( self, request: Request, spider: Spider - ) -> tuple[Optional[str], Optional[Slot]]: - key: Optional[str] = request.meta.get("download_slot") + ) -> tuple[str | None, Slot | None]: + key: str | None = request.meta.get("download_slot") if key is None: return None, None assert self.crawler.engine diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index b5388a918..56941ad51 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -5,7 +5,7 @@ import time from http.cookiejar import Cookie from http.cookiejar import CookieJar as _CookieJar from http.cookiejar import CookiePolicy, DefaultCookiePolicy -from typing import TYPE_CHECKING, Any, Optional, cast +from typing import TYPE_CHECKING, Any, cast from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode @@ -28,7 +28,7 @@ IPV4_RE = re.compile(r"\.\d+$", re.ASCII) class CookieJar: def __init__( self, - policy: Optional[CookiePolicy] = None, + policy: CookiePolicy | None = None, check_expired_frequency: int = 10000, ): self.policy: CookiePolicy = policy or DefaultCookiePolicy() @@ -83,9 +83,9 @@ class CookieJar: def clear( self, - domain: Optional[str] = None, - path: Optional[str] = None, - name: Optional[str] = None, + domain: str | None = None, + path: str | None = None, + name: str | None = None, ) -> None: self.jar.clear(domain, path, name) @@ -188,7 +188,7 @@ class WrappedRequest: def has_header(self, name: str) -> bool: return name in self.request.headers - def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]: + def get_header(self, name: str, default: str | None = None) -> str | None: value = self.request.headers.get(name, default) return to_unicode(value, errors="replace") if value is not None else None diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 1dcbcb966..29ba9533b 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, AnyStr, Optional, Union, cast +from typing import TYPE_CHECKING, Any, AnyStr, Union, cast from w3lib.http import headers_dict_to_raw @@ -25,14 +25,14 @@ class Headers(CaselessDict): def __init__( self, - seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, + seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, encoding: str = "utf-8", ): self.encoding: str = encoding super().__init__(seq) def update( # type: ignore[override] - self, seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]]] + self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] ) -> None: seq = seq.items() if isinstance(seq, Mapping) else seq iseq: dict[bytes, list[bytes]] = {} @@ -44,7 +44,7 @@ class Headers(CaselessDict): """Normalize key to bytes""" return self._tobytes(key.title()) - def normvalue(self, value: Union[_RawValueT, Iterable[_RawValueT]]) -> list[bytes]: + def normvalue(self, value: _RawValueT | Iterable[_RawValueT]) -> list[bytes]: """Normalize values to bytes""" _value: Iterable[_RawValueT] if value is None: @@ -67,13 +67,13 @@ class Headers(CaselessDict): return str(x).encode(self.encoding) raise TypeError(f"Unsupported value type: {type(x)}") - def __getitem__(self, key: AnyStr) -> Optional[bytes]: + def __getitem__(self, key: AnyStr) -> bytes | None: try: return cast(list[bytes], super().__getitem__(key))[-1] except IndexError: return None - def get(self, key: AnyStr, def_val: Any = None) -> Optional[bytes]: + def get(self, key: AnyStr, def_val: Any = None) -> bytes | None: try: return cast(list[bytes], super().get(key, def_val))[-1] except IndexError: @@ -103,7 +103,7 @@ class Headers(CaselessDict): def items(self) -> Iterable[tuple[bytes, list[bytes]]]: # type: ignore[override] return ((k, self.getlist(k)) for k in self.keys()) - def values(self) -> list[Optional[bytes]]: # type: ignore[override] + def values(self) -> list[bytes | None]: # type: ignore[override] return [ self[k] for k in self.keys() # pylint: disable=consider-using-dict-items ] diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index aac8d3e50..ed225555c 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -13,7 +13,6 @@ from typing import ( Any, AnyStr, NoReturn, - Optional, TypedDict, TypeVar, Union, @@ -112,18 +111,18 @@ class Request(object_ref): def __init__( self, url: str, - callback: Optional[CallbackT] = None, + callback: CallbackT | None = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, - body: Optional[Union[bytes, str]] = None, - cookies: Optional[CookiesT] = None, - meta: Optional[dict[str, Any]] = None, + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + body: bytes | str | None = None, + cookies: CookiesT | None = None, + meta: dict[str, Any] | None = None, encoding: str = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable[[Failure], Any]] = None, - flags: Optional[list[str]] = None, - cb_kwargs: Optional[dict[str, Any]] = None, + errback: Callable[[Failure], Any] | None = None, + flags: list[str] | None = None, + cb_kwargs: dict[str, Any] | None = None, ) -> None: self._encoding: str = encoding # this one has to be set first self.method: str = str(method).upper() @@ -139,17 +138,15 @@ class Request(object_ref): ) if not (callable(errback) or errback is None): raise TypeError(f"errback must be a callable, got {type(errback).__name__}") - self.callback: Optional[CallbackT] = callback - self.errback: Optional[Callable[[Failure], Any]] = errback + self.callback: CallbackT | None = callback + self.errback: Callable[[Failure], Any] | None = errback self.cookies: CookiesT = cookies or {} self.headers: Headers = Headers(headers or {}, encoding=encoding) self.dont_filter: bool = dont_filter - self._meta: Optional[dict[str, Any]] = dict(meta) if meta else None - self._cb_kwargs: Optional[dict[str, Any]] = ( - dict(cb_kwargs) if cb_kwargs else None - ) + self._meta: dict[str, Any] | None = dict(meta) if meta else None + self._cb_kwargs: dict[str, Any] | None = dict(cb_kwargs) if cb_kwargs else None self.flags: list[str] = [] if flags is None else list(flags) @property @@ -186,7 +183,7 @@ class Request(object_ref): def body(self) -> bytes: return self._body - def _set_body(self, body: Optional[Union[str, bytes]]) -> None: + def _set_body(self, body: str | bytes | None) -> None: self._body = b"" if body is None else to_bytes(body, self.encoding) @property @@ -208,7 +205,7 @@ class Request(object_ref): def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( - self, *args: Any, cls: Optional[type[Request]] = None, **kwargs: Any + self, *args: Any, cls: type[Request] | None = None, **kwargs: Any ) -> Request: """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: @@ -255,7 +252,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> dict[str, Any]: + def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index d9c913672..2fabf08d1 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -62,14 +62,14 @@ class FormRequest(Request): def from_response( cls, response: TextResponse, - formname: Optional[str] = None, - formid: Optional[str] = None, + formname: str | None = None, + formid: str | None = None, formnumber: int = 0, formdata: FormdataType = None, - clickdata: Optional[dict[str, Union[str, int]]] = None, + clickdata: dict[str, str | int] | None = None, dont_click: bool = False, - formxpath: Optional[str] = None, - formcss: Optional[str] = None, + formxpath: str | None = None, + formcss: str | None = None, **kwargs: Any, ) -> Self: kwargs.setdefault("encoding", response.encoding) @@ -92,7 +92,7 @@ class FormRequest(Request): return cls(url=url, method=method, formdata=formdata, **kwargs) -def _get_form_url(form: FormElement, url: Optional[str]) -> str: +def _get_form_url(form: FormElement, url: str | None) -> str: assert form.base_url is not None # typing if url is None: action = form.get("action") @@ -113,10 +113,10 @@ def _urlencode(seq: Iterable[FormdataKVType], enc: str) -> str: def _get_form( response: TextResponse, - formname: Optional[str], - formid: Optional[str], + formname: str | None, + formid: str | None, formnumber: int, - formxpath: Optional[str], + formxpath: str | None, ) -> FormElement: """Find the wanted form element within the given response.""" root = response.selector.root @@ -160,7 +160,7 @@ def _get_inputs( form: FormElement, formdata: FormdataType, dont_click: bool, - clickdata: Optional[dict[str, Union[str, int]]], + clickdata: dict[str, str | int] | None, ) -> list[FormdataKVType]: """Return a list of key-value pairs for the inputs found in the given form.""" try: @@ -196,8 +196,8 @@ def _get_inputs( def _value( - ele: Union[InputElement, SelectElement, TextareaElement] -) -> tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: + ele: InputElement | SelectElement | TextareaElement, +) -> tuple[str | None, None | str | MultipleSelectOptions]: n = ele.name v = ele.value if ele.tag == "select": @@ -206,8 +206,8 @@ def _value( def _select_value( - ele: SelectElement, n: Optional[str], v: Union[None, str, MultipleSelectOptions] -) -> tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: + ele: SelectElement, n: str | None, v: None | str | MultipleSelectOptions +) -> tuple[str | None, None | str | MultipleSelectOptions]: multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected @@ -218,8 +218,8 @@ def _select_value( def _get_clickable( - clickdata: Optional[dict[str, Union[str, int]]], form: FormElement -) -> Optional[tuple[str, str]]: + clickdata: dict[str, str | int] | None, form: FormElement +) -> tuple[str, str] | None: """ Returns the clickable element specified in clickdata, if the latter is given. If not, it returns the first diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 48862534e..289c60591 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -10,7 +10,7 @@ from __future__ import annotations import copy import json import warnings -from typing import TYPE_CHECKING, Any, Optional, overload +from typing import TYPE_CHECKING, Any, overload from scrapy.http.request import Request, RequestTypeVar @@ -23,7 +23,7 @@ class JsonRequest(Request): attributes: tuple[str, ...] = Request.attributes + ("dumps_kwargs",) def __init__( - self, *args: Any, dumps_kwargs: Optional[dict[str, Any]] = None, **kwargs: Any + self, *args: Any, dumps_kwargs: dict[str, Any] | None = None, **kwargs: Any ) -> None: dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} dumps_kwargs.setdefault("sort_keys", True) @@ -59,7 +59,7 @@ class JsonRequest(Request): def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( - self, *args: Any, cls: Optional[type[Request]] = None, **kwargs: Any + self, *args: Any, cls: type[Request] | None = None, **kwargs: Any ) -> Request: body_passed = kwargs.get("body", None) is not None data: Any = kwargs.pop("data", None) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 096ecd370..01fe740a8 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -5,8 +5,10 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ +from __future__ import annotations + import xmlrpc.client as xmlrpclib -from typing import Any, Optional +from typing import Any import defusedxml.xmlrpc @@ -19,7 +21,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): - def __init__(self, *args: Any, encoding: Optional[str] = None, **kwargs: Any): + def __init__(self, *args: Any, encoding: str | None = None, **kwargs: Any): if "body" not in kwargs and "params" in kwargs: kw = {k: kwargs.pop(k) for k in DUMPS_ARGS if k in kwargs} kwargs["body"] = xmlrpclib.dumps(**kw) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index c69945e2d..d50388548 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, AnyStr, Optional, TypeVar, Union, overload +from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, overload from urllib.parse import urljoin from scrapy.exceptions import NotSupported @@ -60,23 +60,23 @@ class Response(object_ref): self, url: str, status: int = 200, - headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, body: bytes = b"", - flags: Optional[list[str]] = None, - request: Optional[Request] = None, - certificate: Optional[Certificate] = None, - ip_address: Union[IPv4Address, IPv6Address, None] = None, - protocol: Optional[str] = None, + flags: list[str] | None = None, + request: Request | None = None, + certificate: Certificate | None = None, + ip_address: IPv4Address | IPv6Address | None = None, + protocol: str | None = None, ): self.headers: Headers = Headers(headers or {}) self.status: int = int(status) self._set_body(body) self._set_url(url) - self.request: Optional[Request] = request + self.request: Request | None = request self.flags: list[str] = [] if flags is None else list(flags) - self.certificate: Optional[Certificate] = certificate - self.ip_address: Union[IPv4Address, IPv6Address, None] = ip_address - self.protocol: Optional[str] = protocol + self.certificate: Certificate | None = certificate + self.ip_address: IPv4Address | IPv6Address | None = ip_address + self.protocol: str | None = protocol @property def cb_kwargs(self) -> dict[str, Any]: @@ -114,7 +114,7 @@ class Response(object_ref): def body(self) -> bytes: return self._body - def _set_body(self, body: Optional[bytes]) -> None: + def _set_body(self, body: bytes | None) -> None: if body is None: self._body = b"" elif not isinstance(body, bytes): @@ -142,7 +142,7 @@ class Response(object_ref): def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( - self, *args: Any, cls: Optional[type[Response]] = None, **kwargs: Any + self, *args: Any, cls: type[Response] | None = None, **kwargs: Any ) -> Response: """Create a new Response with the same attributes except for those given new values""" for x in self.attributes: @@ -183,19 +183,19 @@ class Response(object_ref): def follow( self, - url: Union[str, Link], - callback: Optional[CallbackT] = None, + url: str | Link, + callback: CallbackT | None = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, - body: Optional[Union[bytes, str]] = None, - cookies: Optional[CookiesT] = None, - meta: Optional[dict[str, Any]] = None, - encoding: Optional[str] = "utf-8", + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + body: bytes | str | None = None, + cookies: CookiesT | None = None, + meta: dict[str, Any] | None = None, + encoding: str | None = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[dict[str, Any]] = None, - flags: Optional[list[str]] = None, + errback: Callable[[Failure], Any] | None = None, + cb_kwargs: dict[str, Any] | None = None, + flags: list[str] | None = None, ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -236,19 +236,19 @@ class Response(object_ref): def follow_all( self, - urls: Iterable[Union[str, Link]], - callback: Optional[CallbackT] = None, + urls: Iterable[str | Link], + callback: CallbackT | None = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, - body: Optional[Union[bytes, str]] = None, - cookies: Optional[CookiesT] = None, - meta: Optional[dict[str, Any]] = None, - encoding: Optional[str] = "utf-8", + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + body: bytes | str | None = None, + cookies: CookiesT | None = None, + meta: dict[str, Any] | None = None, + encoding: str | None = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[dict[str, Any]] = None, - flags: Optional[list[str]] = None, + errback: Callable[[Failure], Any] | None = None, + cb_kwargs: dict[str, Any] | None = None, + flags: list[str] | None = None, ) -> Iterable[Request]: """ .. versionadded:: 2.0 diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 680c1f602..c713f6188 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -8,9 +8,8 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations import json -from collections.abc import Iterable from contextlib import suppress -from typing import TYPE_CHECKING, Any, AnyStr, Optional, Union, cast +from typing import TYPE_CHECKING, Any, AnyStr, cast from urllib.parse import urljoin import parsel @@ -24,16 +23,16 @@ from w3lib.encoding import ( from w3lib.html import strip_html5_whitespace from scrapy.http.response import Response -from scrapy.link import Link from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url if TYPE_CHECKING: - from collections.abc import Callable, Mapping + from collections.abc import Callable, Iterable, Mapping from twisted.python.failure import Failure from scrapy.http.request import CallbackT, CookiesT, Request + from scrapy.link import Link from scrapy.selector import Selector, SelectorList @@ -47,13 +46,13 @@ class TextResponse(Response): attributes: tuple[str, ...] = Response.attributes + ("encoding",) def __init__(self, *args: Any, **kwargs: Any): - self._encoding: Optional[str] = kwargs.pop("encoding", None) - self._cached_benc: Optional[str] = None - self._cached_ubody: Optional[str] = None - self._cached_selector: Optional[Selector] = None + self._encoding: str | None = kwargs.pop("encoding", None) + self._cached_benc: str | None = None + self._cached_ubody: str | None = None + self._cached_selector: Selector | None = None super().__init__(*args, **kwargs) - def _set_body(self, body: Union[str, bytes, None]) -> None: + def _set_body(self, body: str | bytes | None) -> None: self._body: bytes = b"" # used by encoding detection if isinstance(body, str): if self._encoding is None: @@ -69,7 +68,7 @@ class TextResponse(Response): def encoding(self) -> str: return self._declared_encoding() or self._body_inferred_encoding() - def _declared_encoding(self) -> Optional[str]: + def _declared_encoding(self) -> str | None: return ( self._encoding or self._bom_encoding() @@ -104,7 +103,7 @@ class TextResponse(Response): return urljoin(get_base_url(self), url) @memoizemethod_noargs - def _headers_encoding(self) -> Optional[str]: + def _headers_encoding(self) -> str | None: content_type = cast(bytes, self.headers.get(b"Content-Type", b"")) return http_content_type_encoding(to_unicode(content_type, encoding="latin-1")) @@ -123,7 +122,7 @@ class TextResponse(Response): self._cached_ubody = ubody return self._cached_benc - def _auto_detect_fun(self, text: bytes) -> Optional[str]: + def _auto_detect_fun(self, text: bytes) -> str | None: for enc in (self._DEFAULT_ENCODING, "utf-8", "cp1252"): try: text.decode(enc) @@ -133,11 +132,11 @@ class TextResponse(Response): return None @memoizemethod_noargs - def _body_declared_encoding(self) -> Optional[str]: + def _body_declared_encoding(self) -> str | None: return html_body_declared_encoding(self.body) @memoizemethod_noargs - def _bom_encoding(self) -> Optional[str]: + def _bom_encoding(self) -> str | None: return read_bom(self.body)[0] @property @@ -170,19 +169,19 @@ class TextResponse(Response): def follow( self, - url: Union[str, Link, parsel.Selector], - callback: Optional[CallbackT] = None, + url: str | Link | parsel.Selector, + callback: CallbackT | None = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, - body: Optional[Union[bytes, str]] = None, - cookies: Optional[CookiesT] = None, - meta: Optional[dict[str, Any]] = None, - encoding: Optional[str] = None, + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + body: bytes | str | None = None, + cookies: CookiesT | None = None, + meta: dict[str, Any] | None = None, + encoding: str | None = None, priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[dict[str, Any]] = None, - flags: Optional[list[str]] = None, + errback: Callable[[Failure], Any] | None = None, + cb_kwargs: dict[str, Any] | None = None, + flags: list[str] | None = None, ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -223,21 +222,21 @@ class TextResponse(Response): def follow_all( self, - urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None, - callback: Optional[CallbackT] = None, + urls: Iterable[str | Link] | parsel.SelectorList | None = None, + callback: CallbackT | None = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, - body: Optional[Union[bytes, str]] = None, - cookies: Optional[CookiesT] = None, - meta: Optional[dict[str, Any]] = None, - encoding: Optional[str] = None, + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + body: bytes | str | None = None, + cookies: CookiesT | None = None, + meta: dict[str, Any] | None = None, + encoding: str | None = None, priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[dict[str, Any]] = None, - flags: Optional[list[str]] = None, - css: Optional[str] = None, - xpath: Optional[str] = None, + errback: Callable[[Failure], Any] | None = None, + cb_kwargs: dict[str, Any] | None = None, + flags: list[str] | None = None, + css: str | None = None, + xpath: str | None = None, ) -> Iterable[Request]: """ A generator that produces :class:`~.Request` instances to follow all @@ -279,7 +278,7 @@ class TextResponse(Response): with suppress(_InvalidSelector): urls.append(_url_from_selector(sel)) return super().follow_all( - urls=cast(Iterable[Union[str, Link]], urls), + urls=cast("Iterable[str | Link]", urls), callback=callback, method=method, headers=headers, diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 73673b1c6..192f937ce 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -9,7 +9,7 @@ import operator import re from collections.abc import Callable, Iterable from functools import partial -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Union, cast from urllib.parse import urljoin, urlparse from lxml import etree # nosec @@ -58,9 +58,9 @@ def _canonicalize_link_url(link: Link) -> str: class LxmlParserLinkExtractor: def __init__( self, - tag: Union[str, Callable[[str], bool]] = "a", - attr: Union[str, Callable[[str], bool]] = "href", - process: Optional[Callable[[Any], Any]] = None, + tag: str | Callable[[str], bool] = "a", + attr: str | Callable[[str], bool] = "href", + process: Callable[[Any], Any] | None = None, unique: bool = False, strip: bool = True, canonicalized: bool = False, @@ -166,18 +166,18 @@ class LxmlLinkExtractor: self, allow: _RegexOrSeveralT = (), deny: _RegexOrSeveralT = (), - allow_domains: Union[str, Iterable[str]] = (), - deny_domains: Union[str, Iterable[str]] = (), - restrict_xpaths: Union[str, Iterable[str]] = (), - tags: Union[str, Iterable[str]] = ("a", "area"), - attrs: Union[str, Iterable[str]] = ("href",), + allow_domains: str | Iterable[str] = (), + deny_domains: str | Iterable[str] = (), + restrict_xpaths: str | Iterable[str] = (), + tags: str | Iterable[str] = ("a", "area"), + attrs: str | Iterable[str] = ("href",), canonicalize: bool = False, unique: bool = True, - process_value: Optional[Callable[[Any], Any]] = None, - deny_extensions: Union[str, Iterable[str], None] = None, - restrict_css: Union[str, Iterable[str]] = (), + process_value: Callable[[Any], Any] | None = None, + deny_extensions: str | Iterable[str] | None = None, + restrict_css: str | Iterable[str] = (), strip: bool = True, - restrict_text: Optional[_RegexOrSeveralT] = None, + restrict_text: _RegexOrSeveralT | None = None, ): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) self.link_extractor = LxmlParserLinkExtractor( @@ -206,7 +206,7 @@ class LxmlLinkExtractor: self.restrict_text: list[re.Pattern[str]] = self._compile_regexes(restrict_text) @staticmethod - def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> list[re.Pattern[str]]: + def _compile_regexes(value: _RegexOrSeveralT | None) -> list[re.Pattern[str]]: return [ x if isinstance(x, re.Pattern) else re.compile(x) for x in arg_to_iter(value) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 9644cc093..d35720a45 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -6,7 +6,7 @@ See documentation in docs/topics/loaders.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any import itemloaders @@ -92,9 +92,9 @@ class ItemLoader(itemloaders.ItemLoader): def __init__( self, item: Any = None, - selector: Optional[Selector] = None, - response: Optional[TextResponse] = None, - parent: Optional[itemloaders.ItemLoader] = None, + selector: Selector | None = None, + response: TextResponse | None = None, + parent: itemloaders.ItemLoader | None = None, **context: Any, ): if selector is None and response is not None: diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 2b838d8e2..544f4adfe 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging import os -from typing import TYPE_CHECKING, Any, Optional, TypedDict, Union +from typing import TYPE_CHECKING, Any, TypedDict from twisted.python.failure import Failure @@ -31,7 +31,7 @@ DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s" class LogFormatterResult(TypedDict): level: int msg: str - args: Union[dict[str, Any], tuple[Any, ...]] + args: dict[str, Any] | tuple[Any, ...] class LogFormatter: @@ -93,7 +93,7 @@ class LogFormatter: } def scraped( - self, item: Any, response: Union[Response, Failure, None], spider: Spider + self, item: Any, response: Response | Failure | None, spider: Spider ) -> LogFormatterResult: """Logs a message when an item is scraped by a spider.""" src: Any @@ -116,7 +116,7 @@ class LogFormatter: self, item: Any, exception: BaseException, - response: Optional[Response], + response: Response | None, spider: Spider, ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" @@ -133,7 +133,7 @@ class LogFormatter: self, item: Any, exception: BaseException, - response: Optional[Response], + response: Response | None, spider: Spider, ) -> LogFormatterResult: """Logs a message when an item causes an error while it is passing @@ -153,7 +153,7 @@ class LogFormatter: self, failure: Failure, request: Request, - response: Union[Response, Failure], + response: Response | Failure, spider: Spider, ) -> LogFormatterResult: """Logs an error message from a spider. @@ -174,7 +174,7 @@ class LogFormatter: failure: Failure, request: Request, spider: Spider, - errmsg: Optional[str] = None, + errmsg: str | None = None, ) -> LogFormatterResult: """Logs a download error message from a spider (typically coming from the engine). diff --git a/scrapy/mail.py b/scrapy/mail.py index f33cf2939..1e65b1623 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -14,7 +14,7 @@ from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO -from typing import IO, TYPE_CHECKING, Any, Optional, Union +from typing import IO, TYPE_CHECKING, Any from twisted import version as twisted_version from twisted.internet import ssl @@ -45,7 +45,7 @@ logger = logging.getLogger(__name__) COMMASPACE = ", " -def _to_bytes_or_none(text: Union[str, bytes, None]) -> Optional[bytes]: +def _to_bytes_or_none(text: str | bytes | None) -> bytes | None: if text is None: return None return to_bytes(text) @@ -56,8 +56,8 @@ class MailSender: self, smtphost: str = "localhost", mailfrom: str = "scrapy@localhost", - smtpuser: Optional[str] = None, - smtppass: Optional[str] = None, + smtpuser: str | None = None, + smtppass: str | None = None, smtpport: int = 25, smtptls: bool = False, smtpssl: bool = False, @@ -65,8 +65,8 @@ class MailSender: ): self.smtphost: str = smtphost self.smtpport: int = smtpport - self.smtpuser: Optional[bytes] = _to_bytes_or_none(smtpuser) - self.smtppass: Optional[bytes] = _to_bytes_or_none(smtppass) + self.smtpuser: bytes | None = _to_bytes_or_none(smtpuser) + self.smtppass: bytes | None = _to_bytes_or_none(smtppass) self.smtptls: bool = smtptls self.smtpssl: bool = smtpssl self.mailfrom: str = mailfrom @@ -86,15 +86,15 @@ class MailSender: def send( self, - to: Union[str, list[str]], + to: str | list[str], subject: str, body: str, - cc: Union[str, list[str], None] = None, + cc: str | list[str] | None = None, attachs: Sequence[tuple[str, str, IO[Any]]] = (), mimetype: str = "text/plain", - charset: Optional[str] = None, - _callback: Optional[Callable[..., None]] = None, - ) -> Optional[Deferred[None]]: + charset: str | None = None, + _callback: Callable[..., None] | None = None, + ) -> Deferred[None] | None: from twisted.internet import reactor msg: MIMEBase diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 825d6b4c8..39f26717a 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging import pprint from collections import defaultdict, deque -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, TypeVar, cast from scrapy.exceptions import NotConfigured from scrapy.utils.defer import process_chain, process_parallel @@ -40,9 +40,9 @@ class MiddlewareManager: self.middlewares = middlewares # Only process_spider_output and process_spider_exception can be None. # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed. - self.methods: dict[ - str, deque[Union[None, Callable, tuple[Callable, Callable]]] - ] = defaultdict(deque) + self.methods: dict[str, deque[None | Callable | tuple[Callable, Callable]]] = ( + defaultdict(deque) + ) for mw in middlewares: self._add_middleware(mw) @@ -51,9 +51,7 @@ class MiddlewareManager: raise NotImplementedError @classmethod - def from_settings( - cls, settings: Settings, crawler: Optional[Crawler] = None - ) -> Self: + def from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self: mwlist = cls._get_mwlist_from_settings(settings) middlewares = [] enabled = [] diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 32e9ffe7c..4a8639c22 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -17,17 +17,7 @@ from contextlib import suppress from ftplib import FTP from io import BytesIO from pathlib import Path -from typing import ( - IO, - TYPE_CHECKING, - Any, - NoReturn, - Optional, - Protocol, - TypedDict, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Any, NoReturn, Protocol, TypedDict, cast from urllib.parse import urlparse from itemadapter import ItemAdapter @@ -61,7 +51,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def _to_string(path: Union[str, PathLike[str]]) -> str: +def _to_string(path: str | PathLike[str]) -> str: return str(path) # convert a Path object to string @@ -99,17 +89,17 @@ class FilesStoreProtocol(Protocol): path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[dict[str, Any]] = None, - headers: Optional[dict[str, str]] = None, - ) -> Optional[Deferred[Any]]: ... + meta: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + ) -> Deferred[Any] | None: ... def stat_file( self, path: str, info: MediaPipeline.SpiderInfo - ) -> Union[StatInfo, Deferred[StatInfo]]: ... + ) -> StatInfo | Deferred[StatInfo]: ... class FSFilesStore: - def __init__(self, basedir: Union[str, PathLike[str]]): + def __init__(self, basedir: str | PathLike[str]): basedir = _to_string(basedir) if "://" in basedir: basedir = basedir.split("://", 1)[1] @@ -121,18 +111,18 @@ class FSFilesStore: def persist_file( self, - path: Union[str, PathLike[str]], + path: str | PathLike[str], buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[dict[str, Any]] = None, - headers: Optional[dict[str, str]] = None, + meta: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, ) -> None: absolute_path = self._get_filesystem_path(path) self._mkdir(absolute_path.parent, info) absolute_path.write_bytes(buf.getvalue()) def stat_file( - self, path: Union[str, PathLike[str]], info: MediaPipeline.SpiderInfo + self, path: str | PathLike[str], info: MediaPipeline.SpiderInfo ) -> StatInfo: absolute_path = self._get_filesystem_path(path) try: @@ -145,12 +135,12 @@ class FSFilesStore: return {"last_modified": last_modified, "checksum": checksum} - def _get_filesystem_path(self, path: Union[str, PathLike[str]]) -> Path: + def _get_filesystem_path(self, path: str | PathLike[str]) -> Path: path_comps = _to_string(path).split("/") return Path(self.basedir, *path_comps) def _mkdir( - self, dirname: Path, domain: Optional[MediaPipeline.SpiderInfo] = None + self, dirname: Path, domain: MediaPipeline.SpiderInfo | None = None ) -> None: seen: set[str] = self.created_directories[domain] if domain else set() if str(dirname) not in seen: @@ -218,8 +208,8 @@ class S3FilesStore: path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[dict[str, Any]] = None, - headers: Optional[dict[str, str]] = None, + meta: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, ) -> Deferred[Any]: """Upload file to S3 storage""" key_name = f"{self.prefix}{path}" @@ -327,7 +317,7 @@ class GCSFilesStore: deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess), ) - def _get_content_type(self, headers: Optional[dict[str, str]]) -> str: + def _get_content_type(self, headers: dict[str, str] | None) -> str: if headers and "Content-Type" in headers: return headers["Content-Type"] return "application/octet-stream" @@ -340,8 +330,8 @@ class GCSFilesStore: path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[dict[str, Any]] = None, - headers: Optional[dict[str, str]] = None, + meta: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, ) -> Deferred[Any]: blob_path = self._get_blob_path(path) blob = self.bucket.blob(blob_path) @@ -356,9 +346,9 @@ class GCSFilesStore: class FTPFilesStore: - FTP_USERNAME: Optional[str] = None - FTP_PASSWORD: Optional[str] = None - USE_ACTIVE_MODE: Optional[bool] = None + FTP_USERNAME: str | None = None + FTP_PASSWORD: str | None = None + USE_ACTIVE_MODE: bool | None = None def __init__(self, uri: str): if not uri.startswith("ftp://"): @@ -380,8 +370,8 @@ class FTPFilesStore: path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[dict[str, Any]] = None, - headers: Optional[dict[str, str]] = None, + meta: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, ) -> Deferred[Any]: path = f"{self.basedir}/{path}" return deferToThread( @@ -450,9 +440,9 @@ class FilesPipeline(MediaPipeline): def __init__( self, - store_uri: Union[str, PathLike[str]], - download_func: Optional[Callable[[Request, Spider], Response]] = None, - settings: Union[Settings, dict[str, Any], None] = None, + store_uri: str | PathLike[str], + download_func: Callable[[Request, Spider], Response] | None = None, + settings: Settings | dict[str, Any] | None = None, ): store_uri = _to_string(store_uri) if not store_uri: @@ -517,8 +507,8 @@ class FilesPipeline(MediaPipeline): def media_to_download( self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None - ) -> Deferred[Optional[FileInfo]]: - def _onsuccess(result: StatInfo) -> Optional[FileInfo]: + ) -> Deferred[FileInfo | None]: + def _onsuccess(result: StatInfo) -> FileInfo | None: if not result: return None # returning None force download @@ -551,7 +541,7 @@ class FilesPipeline(MediaPipeline): path = self.file_path(request, info=info, item=item) # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[call-overload] - dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess) + dfd2: Deferred[FileInfo | None] = dfd.addCallback(_onsuccess) dfd2.addErrback(lambda _: None) dfd2.addErrback( lambda f: logger.error( @@ -684,8 +674,8 @@ class FilesPipeline(MediaPipeline): def file_path( self, request: Request, - response: Optional[Response] = None, - info: Optional[MediaPipeline.SpiderInfo] = None, + response: Response | None = None, + info: MediaPipeline.SpiderInfo | None = None, *, item: Any = None, ) -> str: diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index f2fe4396b..bbba7d1e1 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -11,7 +11,7 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, cast from itemadapter import ItemAdapter @@ -74,9 +74,9 @@ class ImagesPipeline(FilesPipeline): def __init__( self, - store_uri: Union[str, PathLike[str]], - download_func: Optional[Callable[[Request, Spider], Response]] = None, - settings: Union[Settings, dict[str, Any], None] = None, + store_uri: str | PathLike[str], + download_func: Callable[[Request, Spider], Response] | None = None, + settings: Settings | dict[str, Any] | None = None, ): try: from PIL import Image @@ -120,7 +120,7 @@ class ImagesPipeline(FilesPipeline): resolve("IMAGES_THUMBS"), self.THUMBS ) - self._deprecated_convert_image: Optional[bool] = None + self._deprecated_convert_image: bool | None = None @classmethod def from_settings(cls, settings: Settings) -> Self: @@ -168,7 +168,7 @@ class ImagesPipeline(FilesPipeline): *, item: Any = None, ) -> str: - checksum: Optional[str] = None + checksum: str | None = None for path, image, buf in self.get_images(response, request, info, item=item): if checksum is None: buf.seek(0) @@ -235,8 +235,8 @@ class ImagesPipeline(FilesPipeline): def convert_image( self, image: Image.Image, - size: Optional[tuple[int, int]] = None, - response_body: Optional[BytesIO] = None, + size: tuple[int, int] | None = None, + response_body: BytesIO | None = None, ) -> tuple[Image.Image, BytesIO]: if response_body is None: warnings.warn( @@ -291,8 +291,8 @@ class ImagesPipeline(FilesPipeline): def file_path( self, request: Request, - response: Optional[Response] = None, - info: Optional[MediaPipeline.SpiderInfo] = None, + response: Response | None = None, + info: MediaPipeline.SpiderInfo | None = None, *, item: Any = None, ) -> str: @@ -303,8 +303,8 @@ class ImagesPipeline(FilesPipeline): self, request: Request, thumb_id: str, - response: Optional[Response] = None, - info: Optional[MediaPipeline.SpiderInfo] = None, + response: Response | None = None, + info: MediaPipeline.SpiderInfo | None = None, *, item: Any = None, ) -> str: diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index b30cf9264..61eddffa7 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -9,7 +9,6 @@ from typing import ( Any, Literal, NoReturn, - Optional, TypedDict, TypeVar, Union, @@ -44,7 +43,7 @@ _T = TypeVar("_T") class FileInfo(TypedDict): url: str path: str - checksum: Optional[str] + checksum: str | None status: str @@ -64,15 +63,15 @@ class MediaPipeline(ABC): def __init__(self, spider: Spider): self.spider: Spider = spider self.downloading: set[bytes] = set() - self.downloaded: dict[bytes, Union[FileInfo, Failure]] = {} + self.downloaded: dict[bytes, FileInfo | Failure] = {} self.waiting: defaultdict[bytes, list[Deferred[FileInfo]]] = defaultdict( list ) def __init__( self, - download_func: Optional[Callable[[Request, Spider], Response]] = None, - settings: Union[Settings, dict[str, Any], None] = None, + download_func: Callable[[Request, Spider], Response] | None = None, + settings: Settings | dict[str, Any] | None = None, ): self.download_func = download_func @@ -94,8 +93,8 @@ class MediaPipeline(ABC): def _key_for_pipe( self, key: str, - base_class_name: Optional[str] = None, - settings: Optional[Settings] = None, + base_class_name: str | None = None, + settings: Settings | None = None, ) -> str: class_name = self.__class__.__name__ formatted_key = f"{class_name.upper()}_{key}" @@ -161,7 +160,7 @@ class MediaPipeline(ABC): # Download request checking media_to_download hook output first info.downloading.add(fp) - dfd: Deferred[Optional[FileInfo]] = mustbe_deferred( + dfd: Deferred[FileInfo | None] = mustbe_deferred( self.media_to_download, request, info, item=item ) dfd2: Deferred[FileInfo] = dfd.addCallback( @@ -182,8 +181,8 @@ class MediaPipeline(ABC): request.meta["handle_httpstatus_all"] = True def _check_media_to_download( - self, result: Optional[FileInfo], request: Request, info: SpiderInfo, item: Any - ) -> Union[FileInfo, Deferred[FileInfo]]: + self, result: FileInfo | None, request: Request, info: SpiderInfo, item: Any + ) -> FileInfo | Deferred[FileInfo]: if result is not None: return result dfd: Deferred[Response] @@ -201,7 +200,7 @@ class MediaPipeline(ABC): return dfd2 def _cache_result_and_execute_waiters( - self, result: Union[FileInfo, Failure], fp: bytes, info: SpiderInfo + self, result: FileInfo | Failure, fp: bytes, info: SpiderInfo ) -> None: if isinstance(result, Failure): # minimize cached information for failure @@ -243,7 +242,7 @@ class MediaPipeline(ABC): @abstractmethod def media_to_download( self, request: Request, info: SpiderInfo, *, item: Any = None - ) -> Deferred[Optional[FileInfo]]: + ) -> Deferred[FileInfo | None]: """Check request before starting download""" raise NotImplementedError() @@ -291,8 +290,8 @@ class MediaPipeline(ABC): def file_path( self, request: Request, - response: Optional[Response] = None, - info: Optional[SpiderInfo] = None, + response: Response | None = None, + info: SpiderInfo | None = None, *, item: Any = None, ) -> str: diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index e1bb21fb1..28e2073a2 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -2,7 +2,7 @@ from __future__ import annotations import hashlib import logging -from typing import TYPE_CHECKING, Optional, Protocol, cast +from typing import TYPE_CHECKING, Protocol, cast from scrapy import Request from scrapy.core.downloader import Downloader @@ -42,7 +42,7 @@ class QueueProtocol(Protocol): def push(self, request: Request) -> None: ... - def pop(self) -> Optional[Request]: ... + def pop(self) -> Request | None: ... def close(self) -> None: ... @@ -96,7 +96,7 @@ class ScrapyPriorityQueue: self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls self.key: str = key self.queues: dict[int, QueueProtocol] = {} - self.curprio: Optional[int] = None + self.curprio: int | None = None self.init_prios(startprios) def init_prios(self, startprios: Iterable[int]) -> None: @@ -127,7 +127,7 @@ class ScrapyPriorityQueue: if self.curprio is None or priority < self.curprio: self.curprio = priority - def pop(self) -> Optional[Request]: + def pop(self) -> Request | None: if self.curprio is None: return None q = self.queues[self.curprio] @@ -139,7 +139,7 @@ class ScrapyPriorityQueue: self.curprio = min(prios) if prios else None return m - def peek(self) -> Optional[Request]: + def peek(self) -> Request | None: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -193,7 +193,7 @@ class DownloaderAwarePriorityQueue: crawler: Crawler, downstream_queue_cls: type[QueueProtocol], key: str, - startprios: Optional[dict[str, Iterable[int]]] = None, + startprios: dict[str, Iterable[int]] | None = None, ) -> Self: return cls(crawler, downstream_queue_cls, key, startprios) @@ -202,7 +202,7 @@ class DownloaderAwarePriorityQueue: crawler: Crawler, downstream_queue_cls: type[QueueProtocol], key: str, - slot_startprios: Optional[dict[str, Iterable[int]]] = None, + slot_startprios: dict[str, Iterable[int]] | None = None, ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( @@ -239,7 +239,7 @@ class DownloaderAwarePriorityQueue: startprios, ) - def pop(self) -> Optional[Request]: + def pop(self) -> Request | None: stats = self._downloader_interface.stats(self.pqueues) if not stats: @@ -259,7 +259,7 @@ class DownloaderAwarePriorityQueue: queue = self.pqueues[slot] queue.push(request) - def peek(self) -> Optional[Request]: + def peek(self) -> Request | None: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 97fa74bc2..99a6cc5f6 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any from twisted.internet import defer from twisted.internet.base import ReactorBase, ThreadedResolver @@ -128,7 +128,7 @@ class CachingHostnameResolver: resolutionReceiver: IResolutionReceiver, hostName: str, portNumber: int = 0, - addressTypes: Optional[Sequence[type[IAddress]]] = None, + addressTypes: Sequence[type[IAddress]] | None = None, transportSemantics: str = "TCP", ) -> IHostResolution: try: diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 7154f2b95..3f6f030a5 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -8,7 +8,7 @@ from __future__ import annotations from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING from scrapy.http import Response from scrapy.utils.misc import load_object @@ -58,7 +58,7 @@ class ResponseTypes: return self.classes.get(basetype, Response) def from_content_type( - self, content_type: Union[str, bytes], content_encoding: Optional[bytes] = None + self, content_type: str | bytes, content_encoding: bytes | None = None ) -> type[Response]: """Return the most appropriate Response class from an HTTP Content-Type header""" @@ -70,7 +70,7 @@ class ResponseTypes: return self.from_mimetype(mimetype) def from_content_disposition( - self, content_disposition: Union[str, bytes] + self, content_disposition: str | bytes ) -> type[Response]: try: filename = ( @@ -123,10 +123,10 @@ class ResponseTypes: def from_args( self, - headers: Optional[Mapping[bytes, bytes]] = None, - url: Optional[str] = None, - filename: Optional[str] = None, - body: Optional[bytes] = None, + headers: Mapping[bytes, bytes] | None = None, + url: str | None = None, + filename: str | None = None, + body: bytes | None = None, ) -> type[Response]: """Guess the most appropriate Response class based on the given arguments.""" diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 0d282dc37..a0e5fc671 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging import sys from abc import ABCMeta, abstractmethod -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING from warnings import warn from scrapy.exceptions import ScrapyDeprecationWarning @@ -21,7 +21,7 @@ logger = logging.getLogger(__name__) def decode_robotstxt( - robotstxt_body: bytes, spider: Optional[Spider], to_native_str_type: bool = False + robotstxt_body: bytes, spider: Spider | None, to_native_str_type: bool = False ) -> str: try: if to_native_str_type: @@ -57,7 +57,7 @@ class RobotParser(metaclass=ABCMeta): pass @abstractmethod - def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: """Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``. :param url: Absolute URL @@ -70,10 +70,10 @@ class RobotParser(metaclass=ABCMeta): class PythonRobotParser(RobotParser): - def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): + def __init__(self, robotstxt_body: bytes, spider: Spider | None): from urllib.robotparser import RobotFileParser - self.spider: Optional[Spider] = spider + self.spider: Spider | None = spider body_decoded = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True) self.rp: RobotFileParser = RobotFileParser() self.rp.parse(body_decoded.splitlines()) @@ -84,18 +84,18 @@ class PythonRobotParser(RobotParser): o = cls(robotstxt_body, spider) return o - def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(user_agent, url) class ReppyRobotParser(RobotParser): - def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): + def __init__(self, robotstxt_body: bytes, spider: Spider | None): warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2) from reppy.robots import Robots - self.spider: Optional[Spider] = spider + self.spider: Spider | None = spider self.rp = Robots.parse("", robotstxt_body) @classmethod @@ -104,15 +104,15 @@ class ReppyRobotParser(RobotParser): o = cls(robotstxt_body, spider) return o - def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: return self.rp.allowed(url, user_agent) class RerpRobotParser(RobotParser): - def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): + def __init__(self, robotstxt_body: bytes, spider: Spider | None): from robotexclusionrulesparser import RobotExclusionRulesParser - self.spider: Optional[Spider] = spider + self.spider: Spider | None = spider self.rp: RobotExclusionRulesParser = RobotExclusionRulesParser() body_decoded = decode_robotstxt(robotstxt_body, spider) self.rp.parse(body_decoded) @@ -123,17 +123,17 @@ class RerpRobotParser(RobotParser): o = cls(robotstxt_body, spider) return o - def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.is_allowed(user_agent, url) class ProtegoRobotParser(RobotParser): - def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): + def __init__(self, robotstxt_body: bytes, spider: Spider | None): from protego import Protego - self.spider: Optional[Spider] = spider + self.spider: Spider | None = spider body_decoded = decode_robotstxt(robotstxt_body, spider) self.rp = Protego.parse(body_decoded) @@ -143,7 +143,7 @@ class ProtegoRobotParser(RobotParser): o = cls(robotstxt_body, spider) return o - def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(url, user_agent) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 0a3eae409..db9014b41 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -2,7 +2,9 @@ XPath selectors based on lxml """ -from typing import Any, Optional, Union +from __future__ import annotations + +from typing import Any from parsel import Selector as _ParselSelector @@ -16,13 +18,13 @@ __all__ = ["Selector", "SelectorList"] _NOT_SET = object() -def _st(response: Optional[TextResponse], st: Optional[str]) -> str: +def _st(response: TextResponse | None, st: str | None) -> str: if st is None: return "xml" if isinstance(response, XmlResponse) else "html" return st -def _response_from_text(text: Union[str, bytes], st: Optional[str]) -> TextResponse: +def _response_from_text(text: str | bytes, st: str | None) -> TextResponse: rt: type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8")) @@ -71,10 +73,10 @@ class Selector(_ParselSelector, object_ref): def __init__( self, - response: Optional[TextResponse] = None, - text: Optional[str] = None, - type: Optional[str] = None, - root: Optional[Any] = _NOT_SET, + response: TextResponse | None = None, + text: str | None = None, + type: str | None = None, + root: Any | None = _NOT_SET, **kwargs: Any, ): if response is not None and text is not None: diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b7e3763fb..274ced3e3 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -5,7 +5,7 @@ import json from collections.abc import Iterable, Iterator, Mapping, MutableMapping from importlib import import_module from pprint import pformat -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Union, cast from scrapy.settings import default_settings @@ -35,7 +35,7 @@ SETTINGS_PRIORITIES: dict[str, int] = { } -def get_settings_priority(priority: Union[int, str]) -> int: +def get_settings_priority(priority: int | str) -> int: """ Small helper function that looks up a given string priority in the :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its @@ -97,9 +97,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): __default = object() - def __init__( - self, values: _SettingsInputT = None, priority: Union[int, str] = "project" - ): + def __init__(self, values: _SettingsInputT = None, priority: int | str = "project"): self.frozen: bool = False self.attributes: dict[_SettingsKeyT, SettingsAttribute] = {} if values: @@ -180,7 +178,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return float(self.get(name, default)) def getlist( - self, name: _SettingsKeyT, default: Optional[list[Any]] = None + self, name: _SettingsKeyT, default: list[Any] | None = None ) -> list[Any]: """ Get a setting value as a list. If the setting original type is a list, a @@ -201,7 +199,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return list(value) def getdict( - self, name: _SettingsKeyT, default: Optional[dict[Any, Any]] = None + self, name: _SettingsKeyT, default: dict[Any, Any] | None = None ) -> dict[Any, Any]: """ Get a setting value as a dictionary. If the setting original type is a @@ -226,8 +224,8 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def getdictorlist( self, name: _SettingsKeyT, - default: Union[dict[Any, Any], list[Any], tuple[Any], None] = None, - ) -> Union[dict[Any, Any], list[Any]]: + default: dict[Any, Any] | list[Any] | tuple[Any] | None = None, + ) -> dict[Any, Any] | list[Any]: """Get a setting value as either a :class:`dict` or a :class:`list`. If the setting is already a dict or a list, a copy of it will be @@ -278,7 +276,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): compbs.update(self[name]) return compbs - def getpriority(self, name: _SettingsKeyT) -> Optional[int]: + def getpriority(self, name: _SettingsKeyT) -> int | None: """ Return the current numerical priority value of a setting, or ``None`` if the given ``name`` does not exist. @@ -305,7 +303,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): self.set(name, value) def set( - self, name: _SettingsKeyT, value: Any, priority: Union[int, str] = "project" + self, name: _SettingsKeyT, value: Any, priority: int | str = "project" ) -> None: """ Store a key/value attribute with a given priority. @@ -338,7 +336,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): self, name: _SettingsKeyT, default: Any = None, - priority: Union[int, str] = "project", + priority: int | str = "project", ) -> Any: if name not in self: self.set(name, default, priority) @@ -346,13 +344,11 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return self.attributes[name].value - def setdict( - self, values: _SettingsInputT, priority: Union[int, str] = "project" - ) -> None: + def setdict(self, values: _SettingsInputT, priority: int | str = "project") -> None: self.update(values, priority) def setmodule( - self, module: Union[ModuleType, str], priority: Union[int, str] = "project" + self, module: ModuleType | str, priority: int | str = "project" ) -> None: """ Store settings from a module with a given priority. @@ -376,7 +372,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): self.set(key, getattr(module, key), priority) # BaseSettings.update() doesn't support all inputs that MutableMapping.update() supports - def update(self, values: _SettingsInputT, priority: Union[int, str] = "project") -> None: # type: ignore[override] + def update(self, values: _SettingsInputT, priority: int | str = "project") -> None: # type: ignore[override] """ Store key/value pairs with a given priority. @@ -409,9 +405,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): for name, value in values.items(): self.set(name, value, priority) - def delete( - self, name: _SettingsKeyT, priority: Union[int, str] = "project" - ) -> None: + def delete(self, name: _SettingsKeyT, priority: int | str = "project") -> None: if name not in self: raise KeyError(name) self._assert_mutability() @@ -525,9 +519,7 @@ class Settings(BaseSettings): described on :ref:`topics-settings-ref` already populated. """ - def __init__( - self, values: _SettingsInputT = None, priority: Union[int, str] = "project" - ): + def __init__(self, values: _SettingsInputT = None, priority: int | str = "project"): # Do not pass kwarg values here. We don't want to promote user-defined # dicts, and we want to update, not replace, default dicts with the # values given by the user diff --git a/scrapy/shell.py b/scrapy/shell.py index dc402e678..31349c4ff 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -8,7 +8,7 @@ from __future__ import annotations import os import signal -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any from itemadapter import is_item from twisted.internet import defer, threads @@ -37,25 +37,25 @@ class Shell: def __init__( self, crawler: Crawler, - update_vars: Optional[Callable[[dict[str, Any]], None]] = None, - code: Optional[str] = None, + update_vars: Callable[[dict[str, Any]], None] | None = None, + code: str | None = None, ): self.crawler: Crawler = crawler self.update_vars: Callable[[dict[str, Any]], None] = update_vars or ( lambda x: None ) self.item_class: type = load_object(crawler.settings["DEFAULT_ITEM_CLASS"]) - self.spider: Optional[Spider] = None + self.spider: Spider | None = None self.inthread: bool = not threadable.isInIOThread() - self.code: Optional[str] = code + self.code: str | None = code self.vars: dict[str, Any] = {} def start( self, - url: Optional[str] = None, - request: Optional[Request] = None, - response: Optional[Response] = None, - spider: Optional[Spider] = None, + url: str | None = None, + request: Request | None = None, + response: Response | None = None, + spider: Spider | None = None, redirect: bool = True, ) -> None: # disable accidental Ctrl-C key press from shutting down the engine @@ -97,9 +97,7 @@ class Shell: self.vars, shells=shells, banner=self.vars.pop("banner", "") ) - def _schedule( - self, request: Request, spider: Optional[Spider] - ) -> defer.Deferred[Any]: + def _schedule(self, request: Request, spider: Spider | None) -> defer.Deferred[Any]: if is_asyncio_reactor_installed(): # set the asyncio event loop for the current thread event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"] @@ -111,7 +109,7 @@ class Shell: self.crawler.engine.crawl(request) return d - def _open_spider(self, request: Request, spider: Optional[Spider]) -> Spider: + def _open_spider(self, request: Request, spider: Spider | None) -> Spider: if self.spider: return self.spider @@ -126,8 +124,8 @@ class Shell: def fetch( self, - request_or_url: Union[Request, str], - spider: Optional[Spider] = None, + request_or_url: Request | str, + spider: Spider | None = None, redirect: bool = True, **kwargs: Any, ) -> None: @@ -155,9 +153,9 @@ class Shell: def populate_vars( self, - response: Optional[Response] = None, - request: Optional[Request] = None, - spider: Optional[Spider] = None, + response: Response | None = None, + request: Request | None = None, + spider: Spider | None = None, ) -> None: import scrapy diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index afab2eac2..42619ec7f 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any from scrapy.exceptions import IgnoreRequest @@ -65,7 +65,7 @@ class HttpErrorMiddleware: def process_spider_exception( self, response: Response, exception: Exception, spider: Spider - ) -> Optional[Iterable[Any]]: + ) -> Iterable[Any] | None: if isinstance(exception, HttpError): assert spider.crawler.stats spider.crawler.stats.inc_value("httperror/response_ignored_count") diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 8784e4b05..bdf1f168a 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -6,7 +6,7 @@ originated it. from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, cast from urllib.parse import urlparse from w3lib.url import safe_url_string @@ -50,20 +50,20 @@ class ReferrerPolicy: NOREFERRER_SCHEMES: tuple[str, ...] = LOCAL_SCHEMES name: str - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: raise NotImplementedError() - def stripped_referrer(self, url: str) -> Optional[str]: + def stripped_referrer(self, url: str) -> str | None: if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: return self.strip_url(url) return None - def origin_referrer(self, url: str) -> Optional[str]: + def origin_referrer(self, url: str) -> str | None: if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: return self.origin(url) return None - def strip_url(self, url: str, origin_only: bool = False) -> Optional[str]: + def strip_url(self, url: str, origin_only: bool = False) -> str | None: """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -87,7 +87,7 @@ class ReferrerPolicy: origin_only=origin_only, ) - def origin(self, url: str) -> Optional[str]: + def origin(self, url: str) -> str | None: """Return serialized origin (scheme, host, path) for a request or response URL.""" return self.strip_url(url, origin_only=True) @@ -113,7 +113,7 @@ class NoReferrerPolicy(ReferrerPolicy): name: str = POLICY_NO_REFERRER - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: return None @@ -134,7 +134,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: if not self.tls_protected(response_url) or self.tls_protected(request_url): return self.stripped_referrer(response_url) return None @@ -153,7 +153,7 @@ class SameOriginPolicy(ReferrerPolicy): name: str = POLICY_SAME_ORIGIN - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: if self.origin(response_url) == self.origin(request_url): return self.stripped_referrer(response_url) return None @@ -171,7 +171,7 @@ class OriginPolicy(ReferrerPolicy): name: str = POLICY_ORIGIN - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: return self.origin_referrer(response_url) @@ -191,7 +191,7 @@ class StrictOriginPolicy(ReferrerPolicy): name: str = POLICY_STRICT_ORIGIN - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: if ( self.tls_protected(response_url) and self.potentially_trustworthy(request_url) @@ -215,7 +215,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) @@ -242,7 +242,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) @@ -271,7 +271,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): name: str = POLICY_UNSAFE_URL - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: return self.stripped_referrer(response_url) @@ -307,7 +307,7 @@ _policy_classes[""] = NoReferrerWhenDowngradePolicy def _load_policy_class( policy: str, warning_only: bool = False -) -> Optional[type[ReferrerPolicy]]: +) -> type[ReferrerPolicy] | None: """ Expect a string for the path to the policy class, otherwise try to interpret the string as a standard value @@ -331,7 +331,7 @@ def _load_policy_class( class RefererMiddleware: - def __init__(self, settings: Optional[BaseSettings] = None): + def __init__(self, settings: BaseSettings | None = None): self.default_policy: type[ReferrerPolicy] = DefaultReferrerPolicy if settings is not None: settings_policy = _load_policy_class(settings.get("REFERRER_POLICY")) @@ -349,9 +349,7 @@ class RefererMiddleware: return mw - def policy( - self, resp_or_url: Union[Response, str], request: Request - ) -> ReferrerPolicy: + def policy(self, resp_or_url: Response | str, request: Request) -> ReferrerPolicy: """ Determine Referrer-Policy to use from a parent Response (or URL), and a Request to be sent. diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 8220aca28..6136dabc7 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Optional, cast +from typing import TYPE_CHECKING, Any, cast from scrapy import signals from scrapy.http import Request, Response @@ -34,9 +34,9 @@ class Spider(object_ref): """ name: str - custom_settings: Optional[dict[_SettingsKeyT, Any]] = None + custom_settings: dict[_SettingsKeyT, Any] | None = None - def __init__(self, name: Optional[str] = None, **kwargs: Any): + def __init__(self, name: str | None = None, **kwargs: Any): if name is not None: self.name: str = name elif not getattr(self, "name", None): @@ -103,10 +103,10 @@ class Spider(object_ref): return url_is_from_spider(request.url, cls) @staticmethod - def close(spider: Spider, reason: str) -> Optional[Deferred[None]]: + def close(spider: Spider, reason: str) -> Deferred[None] | None: closed = getattr(spider, "closed", None) if callable(closed): - return cast("Optional[Deferred[None]]", closed(reason)) + return cast("Deferred[None] | None", closed(reason)) return None def __repr__(self) -> str: diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index d628f49f6..087049425 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -9,7 +9,7 @@ from __future__ import annotations import copy from collections.abc import AsyncIterable, Awaitable, Callable -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast from twisted.python.failure import Failure @@ -39,15 +39,11 @@ def _identity(x: _T) -> _T: return x -def _identity_process_request( - request: Request, response: Response -) -> Optional[Request]: +def _identity_process_request(request: Request, response: Response) -> Request | None: return request -def _get_method( - method: Union[Callable, str, None], spider: Spider -) -> Optional[Callable]: +def _get_method(method: Callable | str | None, spider: Spider) -> Callable | None: if callable(method): return method if isinstance(method, str): @@ -61,20 +57,20 @@ _default_link_extractor = LinkExtractor() class Rule: def __init__( self, - link_extractor: Optional[LinkExtractor] = None, - callback: Union[CallbackT, str, None] = None, - cb_kwargs: Optional[dict[str, Any]] = None, - follow: Optional[bool] = None, - process_links: Union[ProcessLinksT, str, None] = None, - process_request: Union[ProcessRequestT, str, None] = None, - errback: Union[Callable[[Failure], Any], str, None] = None, + link_extractor: LinkExtractor | None = None, + callback: CallbackT | str | None = None, + cb_kwargs: dict[str, Any] | None = None, + follow: bool | None = None, + process_links: ProcessLinksT | str | None = None, + process_request: ProcessRequestT | str | None = None, + errback: Callable[[Failure], Any] | str | None = None, ): self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor - self.callback: Union[CallbackT, str, None] = callback - self.errback: Union[Callable[[Failure], Any], str, None] = errback + self.callback: CallbackT | str | None = callback + self.errback: Callable[[Failure], Any] | str | None = errback self.cb_kwargs: dict[str, Any] = cb_kwargs or {} - self.process_links: Union[ProcessLinksT, str] = process_links or _identity - self.process_request: Union[ProcessRequestT, str] = ( + self.process_links: ProcessLinksT | str = process_links or _identity + self.process_request: ProcessRequestT | str = ( process_request or _identity_process_request ) self.follow: bool = follow if follow is not None else not callback @@ -124,7 +120,7 @@ class CrawlSpider(Spider): meta={"rule": rule_index, "link_text": link.text}, ) - def _requests_to_follow(self, response: Response) -> Iterable[Optional[Request]]: + def _requests_to_follow(self, response: Response) -> Iterable[Request | None]: if not isinstance(response, HtmlResponse): return seen: set[Link] = set() @@ -157,7 +153,7 @@ class CrawlSpider(Spider): async def _parse_response( self, response: Response, - callback: Optional[CallbackT], + callback: CallbackT | None, cb_kwargs: dict[str, Any], follow: bool = True, ) -> AsyncIterable[Any]: @@ -176,7 +172,7 @@ class CrawlSpider(Spider): yield request_or_item def _handle_failure( - self, failure: Failure, errback: Optional[Callable[[Failure], Any]] + self, failure: Failure, errback: Callable[[Failure], Any] | None ) -> Iterable[Any]: if errback: results = errback(failure) or () diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 0ddef1f32..395183613 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any from scrapy.exceptions import NotConfigured, NotSupported from scrapy.http import Response, TextResponse @@ -117,13 +117,13 @@ class CSVFeedSpider(Spider): and the file's headers. """ - delimiter: Optional[str] = ( + delimiter: str | None = ( None # When this is None, python's csv module's default delimiter is used ) - quotechar: Optional[str] = ( + quotechar: str | None = ( None # When this is None, python's csv module's default quotechar is used ) - headers: Optional[list[str]] = None + headers: list[str] | None = None def process_results( self, response: Response, results: Iterable[Any] diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index ebe288b83..4ec2919f7 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Iterable -from typing import TYPE_CHECKING, Any, Optional, cast +from typing import TYPE_CHECKING, Any, cast from scrapy import Request from scrapy.spiders import Spider @@ -18,7 +18,7 @@ class InitSpider(Spider): self._postinit_reqs: Iterable[Request] = super().start_requests() return cast(Iterable[Request], iterate_spider_output(self.init_request())) - def initialized(self, response: Optional[Response] = None) -> Any: + def initialized(self, response: Response | None = None) -> Any: """This method must be set as the callback of your last initialization request. See self.init_request() docstring for more info. """ diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 945539d7b..91c7e3be9 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging import re -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, cast from scrapy.http import Request, Response, XmlResponse from scrapy.spiders import Spider @@ -24,10 +24,10 @@ logger = logging.getLogger(__name__) class SitemapSpider(Spider): sitemap_urls: Sequence[str] = () - sitemap_rules: Sequence[ - tuple[Union[re.Pattern[str], str], Union[str, CallbackT]] - ] = [("", "parse")] - sitemap_follow: Sequence[Union[re.Pattern[str], str]] = [""] + sitemap_rules: Sequence[tuple[re.Pattern[str] | str, str | CallbackT]] = [ + ("", "parse") + ] + sitemap_follow: Sequence[re.Pattern[str] | str] = [""] sitemap_alternate_links: bool = False _max_size: int _warn_size: int @@ -93,7 +93,7 @@ class SitemapSpider(Spider): yield Request(loc, callback=c) break - def _get_sitemap_body(self, response: Response) -> Optional[bytes]: + def _get_sitemap_body(self, response: Response) -> bytes | None: """Return the sitemap body contained in the given response, or None if the response is not a sitemap. """ @@ -127,7 +127,7 @@ class SitemapSpider(Spider): return None -def regex(x: Union[re.Pattern[str], str]) -> re.Pattern[str]: +def regex(x: re.Pattern[str] | str) -> re.Pattern[str]: if isinstance(x, str): return re.compile(x) return x diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 767a53db8..7732187fd 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -7,7 +7,7 @@ from __future__ import annotations import marshal import pickle # nosec from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any from queuelib import queue @@ -26,7 +26,7 @@ if TYPE_CHECKING: def _with_mkdir(queue_class: type[queue.BaseQueue]) -> type[queue.BaseQueue]: class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc] - def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any): + def __init__(self, path: str | PathLike, *args: Any, **kwargs: Any): dirname = Path(path).parent if not dirname.exists(): dirname.mkdir(parents=True, exist_ok=True) @@ -45,13 +45,13 @@ def _serializable_queue( s = serialize(obj) super().push(s) - def pop(self) -> Optional[Any]: + def pop(self) -> Any | None: s = super().pop() if s: return deserialize(s) return None - def peek(self) -> Optional[Any]: + def peek(self) -> Any | None: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -89,13 +89,13 @@ def _scrapy_serialization_queue( request_dict = request.to_dict(spider=self.spider) super().push(request_dict) - def pop(self) -> Optional[Request]: + def pop(self) -> Request | None: request = super().pop() if not request: return None return request_from_dict(request, spider=self.spider) - def peek(self) -> Optional[Request]: + def peek(self) -> Request | None: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -118,7 +118,7 @@ def _scrapy_non_serialization_queue( def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: return cls() - def peek(self) -> Optional[Any]: + def peek(self) -> Any | None: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 63c82ec6d..f3dd0f8e7 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -6,7 +6,7 @@ from __future__ import annotations import logging import pprint -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from scrapy import Spider @@ -25,32 +25,32 @@ class StatsCollector: self._stats: StatsT = {} def get_value( - self, key: str, default: Any = None, spider: Optional[Spider] = None + self, key: str, default: Any = None, spider: Spider | None = None ) -> Any: return self._stats.get(key, default) - def get_stats(self, spider: Optional[Spider] = None) -> StatsT: + def get_stats(self, spider: Spider | None = None) -> StatsT: return self._stats - def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def set_value(self, key: str, value: Any, spider: Spider | None = None) -> None: self._stats[key] = value - def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None: + def set_stats(self, stats: StatsT, spider: Spider | None = None) -> None: self._stats = stats def inc_value( - self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None + self, key: str, count: int = 1, start: int = 0, spider: Spider | None = None ) -> None: d = self._stats d[key] = d.setdefault(key, start) + count - def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def max_value(self, key: str, value: Any, spider: Spider | None = None) -> None: self._stats[key] = max(self._stats.setdefault(key, value), value) - def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def min_value(self, key: str, value: Any, spider: Spider | None = None) -> None: self._stats[key] = min(self._stats.setdefault(key, value), value) - def clear_stats(self, spider: Optional[Spider] = None) -> None: + def clear_stats(self, spider: Spider | None = None) -> None: self._stats.clear() def open_spider(self, spider: Spider) -> None: @@ -79,23 +79,23 @@ class MemoryStatsCollector(StatsCollector): class DummyStatsCollector(StatsCollector): def get_value( - self, key: str, default: Any = None, spider: Optional[Spider] = None + self, key: str, default: Any = None, spider: Spider | None = None ) -> Any: return default - def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def set_value(self, key: str, value: Any, spider: Spider | None = None) -> None: pass - def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None: + def set_stats(self, stats: StatsT, spider: Spider | None = None) -> None: pass def inc_value( - self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None + self, key: str, count: int = 1, start: int = 0, spider: Spider | None = None ) -> None: pass - def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def max_value(self, key: str, value: Any, spider: Spider | None = None) -> None: pass - def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def min_value(self, key: str, value: Any, spider: Spider | None = None) -> None: pass diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index f1505e4bd..905959c25 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,5 +1,7 @@ +from __future__ import annotations + from collections.abc import AsyncGenerator, AsyncIterable, Iterable -from typing import TypeVar, Union +from typing import TypeVar _T = TypeVar("_T") @@ -12,8 +14,8 @@ async def collect_asyncgen(result: AsyncIterable[_T]) -> list[_T]: async def as_async_generator( - it: Union[Iterable[_T], AsyncIterable[_T]] -) -> AsyncGenerator[_T, None]: + it: Iterable[_T] | AsyncIterable[_T], +) -> AsyncGenerator[_T]: """Wraps an iterable (sync or async) into an async generator.""" if isinstance(it, AsyncIterable): async for r in it: diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 463bbb5df..64cd31c4b 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -8,7 +8,7 @@ from collections.abc import Iterable from configparser import ConfigParser from operator import itemgetter from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Callable, cast from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings @@ -33,7 +33,7 @@ def build_component_list( "please update your settings" ) - def _map_keys(compdict: Mapping[Any, Any]) -> Union[BaseSettings, dict[Any, Any]]: + def _map_keys(compdict: Mapping[Any, Any]) -> BaseSettings | dict[Any, Any]: if isinstance(compdict, BaseSettings): compbs = BaseSettings() for k, v in compdict.items(): @@ -86,8 +86,8 @@ def arglist_to_dict(arglist: list[str]) -> dict[str, str]: def closest_scrapy_cfg( - path: Union[str, os.PathLike] = ".", - prevpath: Optional[Union[str, os.PathLike]] = None, + path: str | os.PathLike = ".", + prevpath: str | os.PathLike | None = None, ) -> str: """Return the path to the closest scrapy.cfg file by traversing the current directory and its parents @@ -159,8 +159,8 @@ def feed_complete_default_values_from_settings( def feed_process_params_from_cli( settings: BaseSettings, output: list[str], - output_format: Optional[str] = None, - overwrite_output: Optional[list[str]] = None, + output_format: str | None = None, + overwrite_output: list[str] | None = None, ) -> dict[str, dict[str, Any]]: """ Receives feed export params (from the 'crawl' or 'runspider' commands), diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 3b5596ab7..aecd3fdb7 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -2,7 +2,7 @@ from __future__ import annotations from collections.abc import Callable from functools import wraps -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from collections.abc import Iterable @@ -100,7 +100,7 @@ DEFAULT_PYTHON_SHELLS: KnownShellsT = { def get_shell_embed_func( - shells: Optional[Iterable[str]] = None, known_shells: Optional[KnownShellsT] = None + shells: Iterable[str] | None = None, known_shells: KnownShellsT | None = None ) -> Any: """Return the first acceptable shell-embed function from a given list of shell names. @@ -120,9 +120,9 @@ def get_shell_embed_func( def start_python_console( - namespace: Optional[dict[str, Any]] = None, + namespace: dict[str, Any] | None = None, banner: str = "", - shells: Optional[Iterable[str]] = None, + shells: Iterable[str] | None = None, ) -> None: """Start Python console bound to the given namespace. Readline support and tab completion will be used on Unix, if available. diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 9c7f63848..bfdd4dc8a 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -4,7 +4,7 @@ import argparse import warnings from http.cookies import SimpleCookie from shlex import split -from typing import TYPE_CHECKING, Any, NoReturn, Optional, Union +from typing import TYPE_CHECKING, Any, NoReturn from urllib.parse import urlparse from w3lib.http import basic_auth_header @@ -18,8 +18,8 @@ class DataAction(argparse.Action): self, parser: argparse.ArgumentParser, namespace: argparse.Namespace, - values: Union[str, Sequence[Any], None], - option_string: Optional[str] = None, + values: str | Sequence[Any] | None, + option_string: str | None = None, ) -> None: value = str(values) if value.startswith("$"): diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index c78325676..98ecb2f02 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -12,7 +12,7 @@ import warnings import weakref from collections import OrderedDict from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, AnyStr, Optional, TypeVar, Union +from typing import TYPE_CHECKING, Any, AnyStr, TypeVar from scrapy.exceptions import ScrapyDeprecationWarning @@ -44,7 +44,7 @@ class CaselessDict(dict): def __init__( self, - seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, + seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, ): super().__init__() if seq: @@ -84,7 +84,7 @@ class CaselessDict(dict): return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) # type: ignore[arg-type] # doesn't fully implement MutableMapping.update() - def update(self, seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]]]) -> None: # type: ignore[override] + def update(self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]) -> None: # type: ignore[override] seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) super().update(iseq) @@ -145,9 +145,9 @@ class LocalCache(OrderedDict[_KT, _VT]): Older items expires first. """ - def __init__(self, limit: Optional[int] = None): + def __init__(self, limit: int | None = None): super().__init__() - self.limit: Optional[int] = limit + self.limit: int | None = limit def __setitem__(self, key: _KT, value: _VT) -> None: if self.limit: @@ -168,7 +168,7 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): it cannot be instantiated with an initial dictionary. """ - def __init__(self, limit: Optional[int] = None): + def __init__(self, limit: int | None = None): super().__init__() self.data: LocalCache = LocalCache(limit=limit) @@ -178,7 +178,7 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): except TypeError: pass # key is not weak-referenceable, skip caching - def __getitem__(self, key: _KT) -> Optional[_VT]: # type: ignore[override] + def __getitem__(self, key: _KT) -> _VT | None: # type: ignore[override] try: return super().__getitem__(key) except (TypeError, KeyError): diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index aeacadb1c..9ca6c6a24 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -11,7 +11,7 @@ from asyncio import Future from collections.abc import Awaitable, Coroutine, Iterable, Iterator from functools import wraps from types import CoroutineType -from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar, Union, cast, overload +from typing import TYPE_CHECKING, Any, Generic, TypeVar, Union, cast, overload from twisted.internet import defer from twisted.internet.defer import Deferred, DeferredList, ensureDeferred @@ -93,7 +93,7 @@ def mustbe_deferred( def mustbe_deferred( - f: Callable[_P, Union[Deferred[_T], Coroutine[Deferred[Any], Any, _T], _T]], + f: Callable[_P, Deferred[_T] | Coroutine[Deferred[Any], Any, _T] | _T], *args: _P.args, **kw: _P.kwargs, ) -> Deferred[_T]: @@ -179,17 +179,17 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): def __init__( self, aiterable: AsyncIterable[_T], - callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]], + callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], *callable_args: _P.args, **callable_kwargs: _P.kwargs, ): self.aiterator: AsyncIterator[_T] = aiterable.__aiter__() - self.callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]] = callable + self.callable: Callable[Concatenate[_T, _P], Deferred[Any] | None] = callable self.callable_args: tuple[Any, ...] = callable_args self.callable_kwargs: dict[str, Any] = callable_kwargs self.finished: bool = False self.waiting_deferreds: list[Deferred[Any]] = [] - self.anext_deferred: Optional[Deferred[_T]] = None + self.anext_deferred: Deferred[_T] | None = None def _callback(self, result: _T) -> None: # This gets called when the result from aiterator.__anext__() is available. @@ -237,7 +237,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): def parallel_async( async_iterable: AsyncIterable[_T], count: int, - callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]], + callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], *args: _P.args, **named: _P.kwargs, ) -> Deferred[list[tuple[bool, Iterator[Deferred[Any]]]]]: @@ -362,7 +362,7 @@ def deferred_from_coro(o: _CT) -> Deferred: ... def deferred_from_coro(o: _T) -> _T: ... -def deferred_from_coro(o: _T) -> Union[Deferred, _T]: +def deferred_from_coro(o: _T) -> Deferred | _T: """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, Deferred): return o @@ -433,7 +433,7 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]: return d.asFuture(_get_asyncio_event_loop()) -def maybe_deferred_to_future(d: Deferred[_T]) -> Union[Deferred[_T], Future[_T]]: +def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: """ .. versionadded:: 2.6.0 diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 9b0d476a1..32430cd6c 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -1,8 +1,10 @@ """Some helpers for deprecation messages""" +from __future__ import annotations + import inspect import warnings -from typing import Any, Optional, overload +from typing import Any, overload from scrapy.exceptions import ScrapyDeprecationWarning @@ -20,11 +22,11 @@ def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> No def create_deprecated_class( name: str, new_class: type, - clsdict: Optional[dict[str, Any]] = None, + clsdict: dict[str, Any] | None = None, warn_category: type[Warning] = ScrapyDeprecationWarning, warn_once: bool = True, - old_class_path: Optional[str] = None, - new_class_path: Optional[str] = None, + old_class_path: str | None = None, + new_class_path: str | None = None, subclass_warn_message: str = "{cls} inherits from deprecated class {old}, please inherit from {new}.", instance_warn_message: str = "{cls} is deprecated, instantiate {new} instead.", ) -> type: @@ -55,7 +57,7 @@ def create_deprecated_class( # https://github.com/python/mypy/issues/4177 class DeprecatedClass(new_class.__class__): # type: ignore[misc, name-defined] - deprecated_class: Optional[type] = None + deprecated_class: type | None = None warned_on_subclass: bool = False def __new__( @@ -128,7 +130,7 @@ def create_deprecated_class( return deprecated_cls -def _clspath(cls: type, forced: Optional[str] = None) -> str: +def _clspath(cls: type, forced: str | None = None) -> str: if forced is not None: return forced return f"{cls.__module__}.{cls.__name__}" diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index 3cf9585ec..58b4539bf 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from urllib.parse import ParseResult, urlparse from weakref import WeakKeyDictionary @@ -10,12 +10,12 @@ if TYPE_CHECKING: from scrapy.http import Request, Response -_urlparse_cache: WeakKeyDictionary[Union[Request, Response], ParseResult] = ( +_urlparse_cache: WeakKeyDictionary[Request | Response, ParseResult] = ( WeakKeyDictionary() ) -def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult: +def urlparse_cached(request_or_response: Request | Response) -> ParseResult: """Return urlparse.urlparse caching the result, where the argument can be a Request or Response object """ diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index a4d339adc..ba58d939c 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -4,7 +4,7 @@ import csv import logging import re from io import StringIO -from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast, overload +from typing import TYPE_CHECKING, Any, Literal, cast, overload from warnings import warn from lxml import etree # nosec @@ -20,7 +20,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Iterator[Selector]: +def xmliter(obj: Response | str | bytes, nodename: str) -> Iterator[Selector]: """Return a iterator of Selector's over all nodes of a XML document, given the name of the node to iterate. Useful for parsing XML feeds. @@ -77,9 +77,9 @@ def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Iterator[Selecto def xmliter_lxml( - obj: Union[Response, str, bytes], + obj: Response | str | bytes, nodename: str, - namespace: Optional[str] = None, + namespace: str | None = None, prefix: str = "x", ) -> Iterator[Selector]: reader = _StreamReader(obj) @@ -120,9 +120,9 @@ def xmliter_lxml( class _StreamReader: - def __init__(self, obj: Union[Response, str, bytes]): + def __init__(self, obj: Response | str | bytes): self._ptr: int = 0 - self._text: Union[str, bytes] + self._text: str | bytes if isinstance(obj, TextResponse): self._text, self.encoding = obj.body, obj.encoding elif isinstance(obj, Response): @@ -154,11 +154,11 @@ class _StreamReader: def csviter( - obj: Union[Response, str, bytes], - delimiter: Optional[str] = None, - headers: Optional[list[str]] = None, - encoding: Optional[str] = None, - quotechar: Optional[str] = None, + obj: Response | str | bytes, + delimiter: str | None = None, + headers: list[str] | None = None, + encoding: str | None = None, + quotechar: str | None = None, ) -> Iterator[dict[str, str]]: """Returns an iterator of dictionaries from the given csv object @@ -214,22 +214,18 @@ def csviter( @overload -def _body_or_str(obj: Union[Response, str, bytes]) -> str: ... +def _body_or_str(obj: Response | str | bytes) -> str: ... @overload -def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: ... +def _body_or_str(obj: Response | str | bytes, unicode: Literal[True]) -> str: ... @overload -def _body_or_str( - obj: Union[Response, str, bytes], unicode: Literal[False] -) -> bytes: ... +def _body_or_str(obj: Response | str | bytes, unicode: Literal[False]) -> bytes: ... -def _body_or_str( - obj: Union[Response, str, bytes], unicode: bool = True -) -> Union[str, bytes]: +def _body_or_str(obj: Response | str | bytes, unicode: bool = True) -> str | bytes: expected_types = (Response, str, bytes) if not isinstance(obj, expected_types): expected_types_str = " or ".join(t.__name__ for t in expected_types) diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index 488c7994b..37e6aeb51 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -1,14 +1,14 @@ from __future__ import annotations from pathlib import Path -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING if TYPE_CHECKING: from scrapy.settings import BaseSettings -def job_dir(settings: BaseSettings) -> Optional[str]: - path: Optional[str] = settings["JOBDIR"] +def job_dir(settings: BaseSettings) -> str | None: + path: str | None = settings["JOBDIR"] if not path: return None if not Path(path).exists(): diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 2b90c6b36..c3808426a 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -5,7 +5,7 @@ import sys from collections.abc import MutableMapping from logging.config import dictConfig from types import TracebackType -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Optional, cast from twisted.python import log as twisted_log from twisted.python.failure import Failure @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) def failure_to_exc_info( failure: Failure, -) -> Optional[tuple[type[BaseException], BaseException, Optional[TracebackType]]]: +) -> tuple[type[BaseException], BaseException, TracebackType | None] | None: """Extract exc_info from Failure instances""" if isinstance(failure, Failure): assert failure.type @@ -50,7 +50,7 @@ class TopLevelFormatter(logging.Filter): ``loggers`` list where it should act. """ - def __init__(self, loggers: Optional[list[str]] = None): + def __init__(self, loggers: list[str] | None = None): self.loggers: list[str] = loggers or [] def filter(self, record: logging.LogRecord) -> bool: @@ -80,7 +80,7 @@ DEFAULT_LOGGING = { def configure_logging( - settings: Union[Settings, dict[_SettingsKeyT, Any], None] = None, + settings: Settings | dict[_SettingsKeyT, Any] | None = None, install_root_handler: bool = True, ) -> None: """ @@ -125,7 +125,7 @@ def configure_logging( install_scrapy_root_handler(settings) -_scrapy_root_handler: Optional[logging.Handler] = None +_scrapy_root_handler: logging.Handler | None = None def install_scrapy_root_handler(settings: Settings) -> None: @@ -141,7 +141,7 @@ def install_scrapy_root_handler(settings: Settings) -> None: logging.root.addHandler(_scrapy_root_handler) -def get_scrapy_root_handler() -> Optional[logging.Handler]: +def get_scrapy_root_handler() -> logging.Handler | None: return _scrapy_root_handler @@ -231,7 +231,7 @@ class LogCounterHandler(logging.Handler): def logformatter_adapter( logkws: LogFormatterResult, -) -> tuple[int, str, Union[dict[str, Any], tuple[Any, ...]]]: +) -> tuple[int, str, dict[str, Any] | tuple[Any, ...]]: """ Helper that takes the dictionary output from the methods in LogFormatter and adapts it into a tuple of positional arguments for logger.log calls, diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index e5e00512a..1ab30f097 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -14,7 +14,7 @@ from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules -from typing import IO, TYPE_CHECKING, Any, Optional, TypeVar, Union, cast +from typing import IO, TYPE_CHECKING, Any, TypeVar, cast from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item @@ -46,7 +46,7 @@ def arg_to_iter(arg: Any) -> Iterable[Any]: return [arg] -def load_object(path: Union[str, Callable[..., Any]]) -> Any: +def load_object(path: str | Callable[..., Any]) -> Any: """Load an object given its absolute object path, and return it. The object can be the import path of a class, function, variable or an @@ -126,7 +126,7 @@ def md5sum(file: IO[bytes]) -> str: return m.hexdigest() -def rel_has_nofollow(rel: Optional[str]) -> bool: +def rel_has_nofollow(rel: str | None) -> bool: """Return True if link rel attribute has nofollow type""" return rel is not None and "nofollow" in rel.replace(",", " ").split() diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index cff5eb629..ad758b783 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import signal from collections.abc import Callable from types import FrameType diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index c9e5eb857..0139720b7 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,8 +1,9 @@ +from __future__ import annotations + import os import warnings from importlib import import_module from pathlib import Path -from typing import Union from scrapy.exceptions import NotConfigured from scrapy.settings import Settings @@ -45,7 +46,7 @@ def project_data_dir(project: str = "default") -> str: return str(d) -def data_path(path: Union[str, os.PathLike[str]], createdir: bool = False) -> str: +def data_path(path: str | os.PathLike[str], createdir: bool = False) -> str: """ Return the given path joined with the .scrapy data directory. If given an absolute path, return it unmodified. diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 91c5d67f5..6268af728 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -12,7 +12,7 @@ import weakref from collections.abc import AsyncIterable, Iterable, Mapping from functools import partial, wraps from itertools import chain -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, overload +from typing import TYPE_CHECKING, Any, TypeVar, overload from scrapy.utils.asyncgen import as_async_generator @@ -99,7 +99,7 @@ def unique(list_: Iterable[_T], key: Callable[[_T], Any] = lambda x: x) -> list[ def to_unicode( - text: Union[str, bytes], encoding: Optional[str] = None, errors: str = "strict" + text: str | bytes, encoding: str | None = None, errors: str = "strict" ) -> str: """Return the unicode representation of a bytes object ``text``. If ``text`` is already an unicode object, return it as-is.""" @@ -116,7 +116,7 @@ def to_unicode( def to_bytes( - text: Union[str, bytes], encoding: Optional[str] = None, errors: str = "strict" + text: str | bytes, encoding: str | None = None, errors: str = "strict" ) -> bytes: """Return the binary representation of ``text``. If ``text`` is already a bytes object, return it as-is.""" @@ -132,8 +132,8 @@ def to_bytes( def re_rsearch( - pattern: Union[str, Pattern[str]], text: str, chunk_size: int = 1024 -) -> Optional[tuple[int, int]]: + pattern: str | Pattern[str], text: str, chunk_size: int = 1024 +) -> tuple[int, int] | None: """ This function does a reverse search in a text using a regular expression given in the attribute 'pattern'. @@ -269,7 +269,7 @@ def get_spec(func: Callable[..., Any]) -> tuple[list[str], dict[str, Any]]: def equal_attributes( - obj1: Any, obj2: Any, attributes: Optional[list[Union[str, Callable[[Any], Any]]]] + obj1: Any, obj2: Any, attributes: list[str | Callable[[Any], Any]] | None ) -> bool: """Compare two objects attributes""" # not attributes given return False by default @@ -297,8 +297,8 @@ def without_none_values(iterable: Iterable[_KT]) -> Iterable[_KT]: ... def without_none_values( - iterable: Union[Mapping[_KT, _VT], Iterable[_KT]] -) -> Union[dict[_KT, _VT], Iterable[_KT]]: + iterable: Mapping[_KT, _VT] | Iterable[_KT] +) -> dict[_KT, _VT] | Iterable[_KT]: """Return a copy of ``iterable`` with all ``None`` entries removed. If ``iterable`` is a mapping, return a dictionary where all pairs that have @@ -354,7 +354,7 @@ class MutableChain(Iterable[_T]): async def _async_chain( - *iterables: Union[Iterable[_T], AsyncIterable[_T]] + *iterables: Iterable[_T] | AsyncIterable[_T], ) -> AsyncIterator[_T]: for it in iterables: async for o in as_async_generator(it): @@ -366,10 +366,10 @@ class MutableAsyncChain(AsyncIterable[_T]): Similar to MutableChain but for async iterables """ - def __init__(self, *args: Union[Iterable[_T], AsyncIterable[_T]]): + def __init__(self, *args: Iterable[_T] | AsyncIterable[_T]): self.data: AsyncIterator[_T] = _async_chain(*args) - def extend(self, *iterables: Union[Iterable[_T], AsyncIterable[_T]]) -> None: + def extend(self, *iterables: Iterable[_T] | AsyncIterable[_T]) -> None: self.data = _async_chain(self.data, _async_chain(*iterables)) def __aiter__(self) -> AsyncIterator[_T]: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index ed2fb5959..18bb583b8 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -3,7 +3,7 @@ from __future__ import annotations import asyncio import sys from contextlib import suppress -from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar +from typing import TYPE_CHECKING, Any, Generic, TypeVar from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error @@ -54,7 +54,7 @@ class CallLaterOnce(Generic[_T]): self._func: Callable[_P, _T] = func self._a: tuple[Any, ...] = a self._kw: dict[str, Any] = kw - self._call: Optional[DelayedCall] = None + self._call: DelayedCall | None = None def schedule(self, delay: float = 0) -> None: from twisted.internet import reactor @@ -107,7 +107,7 @@ def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: return policy -def install_reactor(reactor_path: str, event_loop_path: Optional[str] = None) -> None: +def install_reactor(reactor_path: str, event_loop_path: str | None = None) -> None: """Installs the :mod:`~twisted.internet.reactor` with the specified import path. Also installs the asyncio event loop with the specified import path if the asyncio reactor is enabled""" @@ -129,7 +129,7 @@ def _get_asyncio_event_loop() -> AbstractEventLoop: return set_asyncio_event_loop(None) -def set_asyncio_event_loop(event_loop_path: Optional[str]) -> AbstractEventLoop: +def set_asyncio_event_loop(event_loop_path: str | None) -> AbstractEventLoop: """Sets and returns the event loop with specified import path.""" if event_loop_path is not None: event_loop_class: type[AbstractEventLoop] = load_object(event_loop_path) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 052a3721a..82bdcb0f9 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -8,7 +8,7 @@ from __future__ import annotations import hashlib import json import warnings -from typing import TYPE_CHECKING, Any, Optional, Protocol, Union +from typing import TYPE_CHECKING, Any, Protocol from urllib.parse import urlunparse from weakref import WeakKeyDictionary @@ -38,7 +38,7 @@ def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[b _fingerprint_cache: WeakKeyDictionary[ - Request, dict[tuple[Optional[tuple[bytes, ...]], bool], bytes] + Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes] ] _fingerprint_cache = WeakKeyDictionary() @@ -46,7 +46,7 @@ _fingerprint_cache = WeakKeyDictionary() def fingerprint( request: Request, *, - include_headers: Optional[Iterable[Union[bytes, str]]] = None, + include_headers: Iterable[bytes | str] | None = None, keep_fragments: bool = False, ) -> bytes: """ @@ -79,7 +79,7 @@ def fingerprint( If you want to include them, set the keep_fragments argument to True (for instance when handling requests with a headless browser). """ - processed_include_headers: Optional[tuple[bytes, ...]] = None + processed_include_headers: tuple[bytes, ...] | None = None if include_headers: processed_include_headers = tuple( to_bytes(h.lower()) for h in sorted(include_headers) @@ -129,7 +129,7 @@ class RequestFingerprinter: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def __init__(self, crawler: Optional[Crawler] = None): + def __init__(self, crawler: Crawler | None = None): if crawler: implementation = crawler.settings.get( "REQUEST_FINGERPRINTER_IMPLEMENTATION" @@ -177,7 +177,7 @@ def request_httprepr(request: Request) -> bytes: return s -def referer_str(request: Request) -> Optional[str]: +def referer_str(request: Request) -> str | None: """Return Referer HTTP header suitable for logging.""" referrer = request.headers.get("Referer") if referrer is None: @@ -185,7 +185,7 @@ def referer_str(request: Request) -> Optional[str]: return to_unicode(referrer, errors="replace") -def request_from_dict(d: dict[str, Any], *, spider: Optional[Spider] = None) -> Request: +def request_from_dict(d: dict[str, Any], *, spider: Spider | None = None) -> Request: """Create a :class:`~scrapy.Request` object from a dict. If a spider is given, it will try to resolve the callbacks looking at the diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 0ca9d07a4..ecc83d1c8 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -9,7 +9,7 @@ import os import re import tempfile import webbrowser -from typing import TYPE_CHECKING, Any, Union +from typing import TYPE_CHECKING, Any from weakref import WeakKeyDictionary from twisted.web import http @@ -35,15 +35,15 @@ def get_base_url(response: TextResponse) -> str: return _baseurl_cache[response] -_metaref_cache: WeakKeyDictionary[ - Response, Union[tuple[None, None], tuple[float, str]] -] = WeakKeyDictionary() +_metaref_cache: WeakKeyDictionary[Response, tuple[None, None] | tuple[float, str]] = ( + WeakKeyDictionary() +) def get_meta_refresh( response: TextResponse, ignore_tags: Iterable[str] = ("script", "noscript"), -) -> Union[tuple[None, None], tuple[float, str]]: +) -> tuple[None, None] | tuple[float, str]: """Parse the http-equiv refresh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] @@ -53,7 +53,7 @@ def get_meta_refresh( return _metaref_cache[response] -def response_status_message(status: Union[bytes, float, int, str]) -> str: +def response_status_message(status: bytes | float | int | str) -> str: """Return status code plus status text descriptive message""" status_int = int(status) message = http.RESPONSES.get(status_int, "Unknown Status") diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 1f70fcf69..c572580ae 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -7,7 +7,7 @@ SitemapSpider, its API is subject to change without notice. from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any from urllib.parse import urljoin import lxml.etree # nosec @@ -20,7 +20,7 @@ class Sitemap: """Class to parse Sitemap (type=urlset) and Sitemap Index (type=sitemapindex) files""" - def __init__(self, xmltext: Union[str, bytes]): + def __init__(self, xmltext: str | bytes): xmlp = lxml.etree.XMLParser( recover=True, remove_comments=True, resolve_entities=False ) @@ -46,7 +46,7 @@ class Sitemap: def sitemap_urls_from_robots( - robots_text: str, base_url: Optional[str] = None + robots_text: str, base_url: str | None = None ) -> Iterable[str]: """Return an iterator over all sitemap urls contained in the given robots.txt file diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 02dbb2e90..e58eb8134 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -2,7 +2,7 @@ from __future__ import annotations import inspect import logging -from typing import TYPE_CHECKING, Any, Literal, Optional, TypeVar, Union, overload +from typing import TYPE_CHECKING, Any, Literal, TypeVar, overload from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro @@ -25,7 +25,7 @@ _T = TypeVar("_T") # https://stackoverflow.com/questions/60222982 @overload -def iterate_spider_output(result: AsyncGenerator[_T, None]) -> AsyncGenerator[_T, None]: ... # type: ignore[overload-overlap] +def iterate_spider_output(result: AsyncGenerator[_T]) -> AsyncGenerator[_T]: ... # type: ignore[overload-overlap] @overload @@ -38,7 +38,7 @@ def iterate_spider_output(result: _T) -> Iterable[Any]: ... def iterate_spider_output( result: Any, -) -> Union[Iterable[Any], AsyncGenerator[_T, None], Deferred[_T]]: +) -> Iterable[Any] | AsyncGenerator[_T] | Deferred[_T]: if inspect.isasyncgen(result): return result if inspect.iscoroutine(result): @@ -83,7 +83,7 @@ def spidercls_for_request( default_spidercls: Literal[None], log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[type[Spider]]: ... +) -> type[Spider] | None: ... @overload @@ -93,16 +93,16 @@ def spidercls_for_request( *, log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[type[Spider]]: ... +) -> type[Spider] | None: ... def spidercls_for_request( spider_loader: SpiderLoader, request: Request, - default_spidercls: Optional[type[Spider]] = None, + default_spidercls: type[Spider] | None = None, log_none: bool = False, log_multiple: bool = False, -) -> Optional[type[Spider]]: +) -> type[Spider] | None: """Return a spider class that handles the given Request. This will look for the spiders that can handle the given request (using diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 2c3a259c1..7d46cbd4f 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL @@ -26,7 +26,7 @@ def x509name_to_string(x509name: X509Name) -> str: return ffi_buf_to_string(result_buffer) -def get_temp_key_info(ssl_object: Any) -> Optional[str]: +def get_temp_key_info(ssl_object: Any) -> str | None: # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key() if not hasattr(pyOpenSSLutil.lib, "SSL_get_server_tmp_key"): # removed in cryptography 40.0.0 diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 08f3f2dc9..3e4dae5c8 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -5,13 +5,13 @@ from __future__ import annotations import re import string from pathlib import Path -from typing import TYPE_CHECKING, Any, Union +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from os import PathLike -def render_templatefile(path: Union[str, PathLike], **kwargs: Any) -> None: +def render_templatefile(path: str | PathLike, **kwargs: Any) -> None: path_obj = Path(path) raw = path_obj.read_text("utf8") diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 860a2e3dd..d65f2a76d 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -9,7 +9,7 @@ import os from importlib import import_module from pathlib import Path from posixpath import split -from typing import TYPE_CHECKING, Any, Optional, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar from unittest import TestCase, mock from twisted.trial.unittest import SkipTest @@ -84,8 +84,8 @@ class TestSpider(Spider): def get_crawler( - spidercls: Optional[type[Spider]] = None, - settings_dict: Optional[dict[str, Any]] = None, + spidercls: type[Spider] | None = None, + settings_dict: dict[str, Any] | None = None, prevent_warnings: bool = True, ) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it @@ -120,7 +120,7 @@ def get_testenv() -> dict[str, str]: def assert_samelines( - testcase: TestCase, text1: str, text2: str, msg: Optional[str] = None + testcase: TestCase, text1: str, text2: str, msg: str | None = None ) -> None: """Asserts text1 and text2 have the same lines, ignoring differences in line endings between platforms diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index dfc823725..05e04e2d1 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -2,7 +2,7 @@ from __future__ import annotations import os import sys -from typing import TYPE_CHECKING, Optional, cast +from typing import TYPE_CHECKING, cast from twisted.internet.defer import Deferred from twisted.internet.error import ProcessTerminated @@ -15,7 +15,7 @@ if TYPE_CHECKING: class ProcessTest: - command: Optional[str] = None + command: str | None = None prefix = [sys.executable, "-m", "scrapy.cmdline"] cwd = os.getcwd() # trial chdirs to temp dir @@ -23,7 +23,7 @@ class ProcessTest: self, args: Iterable[str], check_code: bool = True, - settings: Optional[str] = None, + settings: str | None = None, ) -> Deferred[TestProcessProtocol]: from twisted.internet import reactor @@ -54,7 +54,7 @@ class TestProcessProtocol(ProcessProtocol): self.deferred: Deferred[TestProcessProtocol] = Deferred() self.out: bytes = b"" self.err: bytes = b"" - self.exitcode: Optional[int] = None + self.exitcode: int | None = None def outReceived(self, data: bytes) -> None: self.out += data diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 41d268baa..e0a2973f7 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -9,7 +9,7 @@ to the w3lib.url module. Always import those from there instead. from __future__ import annotations import re -from typing import TYPE_CHECKING, Optional, Union, cast +from typing import TYPE_CHECKING, Union, cast from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this @@ -50,7 +50,7 @@ def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool: return any(lowercase_path.endswith(ext) for ext in extensions) -def parse_url(url: UrlT, encoding: Optional[str] = None) -> ParseResult: +def parse_url(url: UrlT, encoding: str | None = None) -> ParseResult: """Return urlparsed url from the given argument (which could be an already parsed url) """ diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index 1afef4d24..028e3a08a 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -1,6 +1,7 @@ +from __future__ import annotations + import asyncio import sys -from typing import Optional from scrapy import Spider from scrapy.crawler import CrawlerProcess @@ -31,7 +32,7 @@ class UrlSpider(Spider): if __name__ == "__main__": - ASYNCIO_EVENT_LOOP: Optional[str] + ASYNCIO_EVENT_LOOP: str | None try: ASYNCIO_EVENT_LOOP = sys.argv[1] except IndexError: diff --git a/tests/spiders.py b/tests/spiders.py index 5d5792858..cc54240ef 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -2,9 +2,10 @@ Some spiders used for testing and benchmarking """ +from __future__ import annotations + import asyncio import time -from typing import Optional from urllib.parse import urlencode from twisted.internet import defer @@ -82,19 +83,19 @@ class DelaySpider(MetaSpider): class LogSpider(MetaSpider): name = "log_spider" - def log_debug(self, message: str, extra: Optional[dict] = None): + def log_debug(self, message: str, extra: dict | None = None): self.logger.debug(message, extra=extra) - def log_info(self, message: str, extra: Optional[dict] = None): + def log_info(self, message: str, extra: dict | None = None): self.logger.info(message, extra=extra) - def log_warning(self, message: str, extra: Optional[dict] = None): + def log_warning(self, message: str, extra: dict | None = None): self.logger.warning(message, extra=extra) - def log_error(self, message: str, extra: Optional[dict] = None): + def log_error(self, message: str, extra: dict | None = None): self.logger.error(message, extra=extra) - def log_critical(self, message: str, extra: Optional[dict] = None): + def log_critical(self, message: str, extra: dict | None = None): self.logger.critical(message, extra=extra) def parse(self, response): diff --git a/tests/test_commands.py b/tests/test_commands.py index 6ec7c21b0..e7df7b6e8 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -15,7 +15,7 @@ from shutil import copytree, rmtree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from tempfile import TemporaryFile, mkdtemp from threading import Timer -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING from unittest import skipIf from pytest import mark @@ -117,9 +117,7 @@ class ProjectTest(unittest.TestCase): return p, to_unicode(stdout), to_unicode(stderr) - def find_in_file( - self, filename: Union[str, os.PathLike], regex - ) -> Optional[re.Match]: + def find_in_file(self, filename: str | os.PathLike, regex) -> re.Match | None: """Find first pattern occurrence in file""" pattern = re.compile(regex) with Path(filename).open("r", encoding="utf-8") as f: @@ -198,7 +196,7 @@ class StartprojectTest(ProjectTest): def get_permissions_dict( - path: Union[str, os.PathLike], renamings=None, ignore=None + path: str | os.PathLike, renamings=None, ignore=None ) -> dict[str, str]: def get_permissions(path: Path) -> str: return oct(path.stat().st_mode) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index f14a10a32..19cea97ec 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,10 +1,11 @@ +from __future__ import annotations + import contextlib import os import shutil import sys from pathlib import Path from tempfile import mkdtemp, mkstemp -from typing import Optional from unittest import SkipTest, mock from testfixtures import LogCapture @@ -692,7 +693,7 @@ class Https11CustomCiphers(unittest.TestCase): class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" - settings_dict: Optional[dict] = None + settings_dict: dict | None = None def setUp(self): self.mockserver = MockServer() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ea3ed3b05..f59412ab4 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -18,7 +18,7 @@ from io import BytesIO from logging import getLogger from pathlib import Path from string import ascii_letters, digits -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from unittest import mock from urllib.parse import quote, urljoin from urllib.request import pathname2url @@ -66,7 +66,7 @@ def printf_escape(string): return string.replace("%", "%%") -def build_url(path: Union[str, PathLike]) -> str: +def build_url(path: str | PathLike) -> str: path_str = str(path) if path_str[0] != "/": path_str = "/" + path_str diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index b1043c111..ed3394b01 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,7 +1,8 @@ +from __future__ import annotations + import pickle import re import unittest -from typing import Optional from packaging.version import Version from pytest import mark @@ -16,7 +17,7 @@ from tests import get_testdata # a hack to skip base class tests in pytest class Base: class LinkExtractorTestCase(unittest.TestCase): - extractor_cls: Optional[type] = None + extractor_cls: type | None = None def setUp(self): body = get_testdata("link_extractor", "linkextractor.html") diff --git a/tests/test_loader.py b/tests/test_loader.py index 8db929dcf..aca428bbe 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,6 +1,7 @@ +from __future__ import annotations + import dataclasses import unittest -from typing import Optional import attr from itemadapter import ItemAdapter @@ -88,7 +89,7 @@ class BasicItemLoaderTest(unittest.TestCase): class InitializationTestMixin: - item_class: Optional[type] = None + item_class: type | None = None def test_keep_single_value(self): """Loaded item should contain values from the initial item""" diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 83e22b070..5cf4a63aa 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,7 +1,8 @@ +from __future__ import annotations + import shutil from pathlib import Path from tempfile import mkdtemp -from typing import Optional from testfixtures import LogCapture from twisted.internet import defer @@ -57,7 +58,7 @@ class FileDownloadCrawlTestCase(TestCase): store_setting_key = "FILES_STORE" media_key = "files" media_urls_key = "file_urls" - expected_checksums: Optional[set[str]] = { + expected_checksums: set[str] | None = { "5547178b89448faf0015a13f904c936e", "c2281c83670e31d8aaab7cb642b824db", "ed3f6538dc15d4d9179dae57319edc5f", @@ -216,7 +217,7 @@ class FileDownloadCrawlTestCase(TestCase): self.assertIn("ZeroDivisionError", str(log)) -skip_pillow: Optional[str] +skip_pillow: str | None try: from PIL import Image # noqa: imported just to check for the import error except ImportError: diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 296a6fae0..2c3b191fe 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import dataclasses import hashlib import io @@ -5,7 +7,6 @@ import random import warnings from shutil import rmtree from tempfile import mkdtemp -from typing import Optional from unittest.mock import patch import attr @@ -19,7 +20,7 @@ from scrapy.pipelines.images import ImageException, ImagesPipeline, NoimagesDrop from scrapy.settings import Settings from scrapy.utils.python import to_bytes -skip_pillow: Optional[str] +skip_pillow: str | None try: from PIL import Image except ImportError: diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 127775f43..0faf6d015 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,4 +1,4 @@ -from typing import Optional +from __future__ import annotations from testfixtures import LogCapture from twisted.internet import reactor @@ -20,7 +20,7 @@ from scrapy.utils.test import get_crawler try: from PIL import Image # noqa: imported just to check for the import error except ImportError: - skip_pillow: Optional[str] = ( + skip_pillow: str | None = ( "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" ) else: diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 9b7bad4bf..6b7cd5dac 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,8 +1,9 @@ +from __future__ import annotations + import collections import shutil import tempfile import unittest -from typing import Optional from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -60,7 +61,7 @@ class MockCrawler(Crawler): class SchedulerHandler: - priority_queue_cls: Optional[str] = None + priority_queue_cls: str | None = None jobdir = None def create_scheduler(self): @@ -254,7 +255,7 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots): class DownloaderAwareSchedulerTestMixin: - priority_queue_cls: Optional[str] = "scrapy.pqueues.DownloaderAwarePriorityQueue" + priority_queue_cls: str | None = "scrapy.pqueues.DownloaderAwarePriorityQueue" reopen = False def test_logic(self): diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 4fd293ec7..b48a65e67 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -1,4 +1,5 @@ -from typing import Optional +from __future__ import annotations + from unittest import TestCase from urllib.parse import urljoin @@ -32,7 +33,7 @@ class MinimalScheduler: return True return False - def next_request(self) -> Optional[Request]: + def next_request(self) -> Request | None: if self.has_pending_requests(): fp, request = self.requests.popitem() return request diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 41228b5f2..1a80eb7be 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,5 +1,6 @@ +from __future__ import annotations + from collections.abc import AsyncIterator, Iterable -from typing import Optional, Union from unittest import mock from testfixtures import LogCapture @@ -112,11 +113,11 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): Should work for process_spider_output and, when it's supported, process_start_requests. """ - ITEM_TYPE: Union[type, tuple] + ITEM_TYPE: type | tuple RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects @staticmethod - def _construct_mw_setting(*mw_classes, start_index: Optional[int] = None): + def _construct_mw_setting(*mw_classes, start_index: int | None = None): if start_index is None: start_index = 10 return {i: c for c, i in enumerate(mw_classes, start=start_index)} @@ -127,7 +128,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): yield {"foo": 3} @defer.inlineCallbacks - def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): + def _get_middleware_result(self, *mw_classes, start_index: int | None = None): setting = self._construct_mw_setting(*mw_classes, start_index=start_index) self.crawler = get_crawler( Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} @@ -141,7 +142,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): @defer.inlineCallbacks def _test_simple_base( - self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None + self, *mw_classes, downgrade: bool = False, start_index: int | None = None ): with LogCapture() as log: result = yield self._get_middleware_result( @@ -155,7 +156,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): @defer.inlineCallbacks def _test_asyncgen_base( - self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None + self, *mw_classes, downgrade: bool = False, start_index: int | None = None ): with LogCapture() as log: result = yield self._get_middleware_result( @@ -337,7 +338,7 @@ class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): yield {"name": "test item"} @defer.inlineCallbacks - def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): + def _get_middleware_result(self, *mw_classes, start_index: int | None = None): setting = self._construct_mw_setting(*mw_classes, start_index=start_index) self.crawler = get_crawler( Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} @@ -441,7 +442,7 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware @defer.inlineCallbacks - def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): + def _get_middleware_result(self, *mw_classes, start_index: int | None = None): setting = self._construct_mw_setting(*mw_classes, start_index=start_index) self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES": setting}) self.spider = self.crawler._create_spider("foo") diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index e73e7ff4c..facbaa60d 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,5 +1,7 @@ +from __future__ import annotations + import warnings -from typing import Any, Optional +from typing import Any from unittest import TestCase from urllib.parse import urlparse @@ -35,7 +37,7 @@ class TestRefererMiddleware(TestCase): req_meta: dict[str, Any] = {} resp_headers: dict[str, str] = {} settings: dict[str, Any] = {} - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ ("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"), ] @@ -65,7 +67,7 @@ class MixinDefault: with some additional filtering of s3:// """ - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ ("https://example.com/", "https://scrapy.org/", b"https://example.com/"), ("http://example.com/", "http://scrapy.org/", b"http://example.com/"), ("http://example.com/", "https://scrapy.org/", b"http://example.com/"), @@ -86,7 +88,7 @@ class MixinDefault: class MixinNoReferrer: - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ ("https://example.com/page.html", "https://example.com/", None), ("http://www.example.com/", "https://scrapy.org/", None), ("http://www.example.com/", "http://scrapy.org/", None), @@ -96,7 +98,7 @@ class MixinNoReferrer: class MixinNoReferrerWhenDowngrade: - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # TLS to TLS: send non-empty referrer ( "https://example.com/page.html", @@ -178,7 +180,7 @@ class MixinNoReferrerWhenDowngrade: class MixinSameOrigin: - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -247,7 +249,7 @@ class MixinSameOrigin: class MixinOrigin: - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) ( "https://example.com/page.html", @@ -271,7 +273,7 @@ class MixinOrigin: class MixinStrictOrigin: - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades ( "https://example.com/page.html", @@ -299,7 +301,7 @@ class MixinStrictOrigin: class MixinOriginWhenCrossOrigin: - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -406,7 +408,7 @@ class MixinOriginWhenCrossOrigin: class MixinStrictOriginWhenCrossOrigin: - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -518,7 +520,7 @@ class MixinStrictOriginWhenCrossOrigin: class MixinUnsafeUrl: - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # TLS to TLS: send referrer ( "https://example.com/sekrit.html", @@ -969,7 +971,7 @@ class TestPolicyHeaderPrecedence004( class TestReferrerOnRedirect(TestRefererMiddleware): settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"} scenarii: list[ - tuple[str, str, tuple[tuple[int, str], ...], Optional[bytes], Optional[bytes]] + tuple[str, str, tuple[tuple[int, str], ...], bytes | None, bytes | None] ] = [ # type: ignore[assignment] ( "http://scrapytest.org/1", # parent diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index ca3bca0b2..7156b13d0 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,8 +1,9 @@ +from __future__ import annotations + import json import unittest import warnings from hashlib import sha1 -from typing import Optional, Union from weakref import WeakKeyDictionary from scrapy.http import Request @@ -56,12 +57,12 @@ class FingerprintTest(unittest.TestCase): maxDiff = None function: staticmethod = staticmethod(fingerprint) - cache: Union[ - "WeakKeyDictionary[Request, dict[tuple[Optional[tuple[bytes, ...]], bool], bytes]]", - "WeakKeyDictionary[Request, dict[tuple[Optional[tuple[bytes, ...]], bool], str]]", - ] = _fingerprint_cache + cache: ( + WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]] + | WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], str]] + ) = _fingerprint_cache default_cache_key = (None, False) - known_hashes: tuple[tuple[Request, Union[bytes, str], dict], ...] = ( + known_hashes: tuple[tuple[Request, bytes | str, dict], ...] = ( ( Request("http://example.org"), b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD", From 7196a11f5321d05b79c9dedc29398a200d00c911 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 17 Oct 2024 21:51:13 +0500 Subject: [PATCH 266/269] Reorder unions with None. --- scrapy/core/spidermw.py | 2 +- scrapy/crawler.py | 2 +- scrapy/http/request/form.py | 6 +++--- scrapy/middleware.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 1edfe1c51..f7947d35d 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -339,7 +339,7 @@ class SpiderMiddlewareManager(MiddlewareManager): @staticmethod def _get_async_method_pair( mw: Any, methodname: str - ) -> None | Callable | tuple[Callable, Callable]: + ) -> Callable | tuple[Callable, Callable] | None: normal_method: Callable | None = getattr(mw, methodname, None) methodname_async = methodname + "_async" async_method: Callable | None = getattr(mw, methodname_async, None) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 701dccf57..3e5657d22 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -57,7 +57,7 @@ class Crawler: def __init__( self, spidercls: type[Spider], - settings: None | dict[str, Any] | Settings = None, + settings: dict[str, Any] | Settings | None = None, init_reactor: bool = False, ): if isinstance(spidercls, Spider): diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 2fabf08d1..29743565d 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -197,7 +197,7 @@ def _get_inputs( def _value( ele: InputElement | SelectElement | TextareaElement, -) -> tuple[str | None, None | str | MultipleSelectOptions]: +) -> tuple[str | None, str | MultipleSelectOptions | None]: n = ele.name v = ele.value if ele.tag == "select": @@ -206,8 +206,8 @@ def _value( def _select_value( - ele: SelectElement, n: str | None, v: None | str | MultipleSelectOptions -) -> tuple[str | None, None | str | MultipleSelectOptions]: + ele: SelectElement, n: str | None, v: str | MultipleSelectOptions | None +) -> tuple[str | None, str | MultipleSelectOptions | None]: multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 39f26717a..b6a427895 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -40,7 +40,7 @@ class MiddlewareManager: self.middlewares = middlewares # Only process_spider_output and process_spider_exception can be None. # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed. - self.methods: dict[str, deque[None | Callable | tuple[Callable, Callable]]] = ( + self.methods: dict[str, deque[Callable | tuple[Callable, Callable] | None]] = ( defaultdict(deque) ) for mw in middlewares: From 7e07d48cc5bfb4e07e1319334884ab420a2616c0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 17 Oct 2024 23:22:37 +0500 Subject: [PATCH 267/269] Small 3.7 and 3.8 cleanup. --- scrapy/utils/reactor.py | 6 ++---- tests/CrawlerProcess/asyncio_enabled_reactor.py | 2 +- .../asyncio_enabled_reactor_different_loop.py | 2 +- tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py | 2 +- tox.ini | 3 --- 5 files changed, 5 insertions(+), 10 deletions(-) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 18bb583b8..f8904a9aa 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -97,10 +97,8 @@ def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: policy = asyncio.get_event_loop_policy() - if ( - sys.version_info >= (3, 8) - and sys.platform == "win32" - and not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy) + if sys.platform == "win32" and not isinstance( + policy, asyncio.WindowsSelectorEventLoopPolicy ): policy = asyncio.WindowsSelectorEventLoopPolicy() asyncio.set_event_loop_policy(policy) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index 01d23c963..f013eed27 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -3,7 +3,7 @@ import sys from twisted.internet import asyncioreactor -if sys.version_info >= (3, 8) and sys.platform == "win32": +if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index 9dc8ce46b..e9d6d8875 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -4,7 +4,7 @@ import sys from twisted.internet import asyncioreactor from twisted.python import log -if sys.version_info >= (3, 8) and sys.platform == "win32": +if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index be9c83b95..c72a0a17c 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -4,7 +4,7 @@ import sys from twisted.internet import asyncioreactor from uvloop import Loop -if sys.version_info >= (3, 8) and sys.platform == "win32": +if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncio.set_event_loop(Loop()) asyncioreactor.install(asyncio.get_event_loop()) diff --git a/tox.ini b/tox.ini index 79f72a0f2..fbbce48d4 100644 --- a/tox.ini +++ b/tox.ini @@ -26,9 +26,6 @@ deps = # mitmproxy does not support PyPy mitmproxy; implementation_name != 'pypy' - # https://github.com/pallets/werkzeug/pull/2768 breaks flask, required by - # mitmproxy. - werkzeug < 3; python_version < '3.9' and implementation_name != 'pypy' passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From 5759b3f0f2b0a45588e7ae7cd455ee5e7d4f531c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 17 Oct 2024 23:41:23 +0500 Subject: [PATCH 268/269] Drop Reppy. --- docs/topics/downloader-middleware.rst | 32 -------------------- scrapy/robotstxt.py | 20 ------------ tests/test_downloadermiddleware_robotstxt.py | 13 +------- tests/test_robotstxt_interface.py | 24 --------------- 4 files changed, 1 insertion(+), 88 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index c31f7fe43..13064ccdd 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1086,7 +1086,6 @@ RobotsTxtMiddleware * :ref:`Protego <protego-parser>` (default) * :ref:`RobotFileParser <python-robotfileparser>` * :ref:`Robotexclusionrulesparser <rerp-parser>` - * :ref:`Reppy <reppy-parser>` (deprecated) You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`. @@ -1154,37 +1153,6 @@ In order to use this parser, set: * :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser`` -.. _reppy-parser: - -Reppy parser -~~~~~~~~~~~~ - -Based on `Reppy <https://github.com/seomoz/reppy/>`_: - -* is a Python wrapper around `Robots Exclusion Protocol Parser for C++ - <https://github.com/seomoz/rep-cpp>`_ - -* is compliant with `Martijn Koster's 1996 draft specification - <https://www.robotstxt.org/norobots-rfc.txt>`_ - -* supports wildcard matching - -* uses the length based rule - -Native implementation, provides better speed than Protego. - -In order to use this parser: - -* Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy`` - - .. warning:: `Upstream issue #122 - <https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in Python 3.9+. - Because of this the Reppy parser is deprecated. - -* Set :setting:`ROBOTSTXT_PARSER` setting to - ``scrapy.robotstxt.ReppyRobotParser`` - - .. _rerp-parser: Robotexclusionrulesparser diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index a0e5fc671..f0a6e7467 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -4,9 +4,7 @@ import logging import sys from abc import ABCMeta, abstractmethod from typing import TYPE_CHECKING -from warnings import warn -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import to_unicode if TYPE_CHECKING: @@ -90,24 +88,6 @@ class PythonRobotParser(RobotParser): return self.rp.can_fetch(user_agent, url) -class ReppyRobotParser(RobotParser): - def __init__(self, robotstxt_body: bytes, spider: Spider | None): - warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2) - from reppy.robots import Robots - - self.spider: Spider | None = spider - self.rp = Robots.parse("", robotstxt_body) - - @classmethod - def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: - spider = None if not crawler else crawler.spider - o = cls(robotstxt_body, spider) - return o - - def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: - return self.rp.allowed(url, user_agent) - - class RerpRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): from robotexclusionrulesparser import RobotExclusionRulesParser diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index e166cc000..12b541456 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -11,7 +11,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings -from tests.test_robotstxt_interface import reppy_available, rerp_available +from tests.test_robotstxt_interface import rerp_available class RobotsTxtMiddlewareTest(unittest.TestCase): @@ -254,14 +254,3 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): self.crawler.settings.set( "ROBOTSTXT_PARSER", "scrapy.robotstxt.RerpRobotParser" ) - - -class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest): - if not reppy_available(): - skip = "Reppy parser is not installed" - - def setUp(self): - super().setUp() - self.crawler.settings.set( - "ROBOTSTXT_PARSER", "scrapy.robotstxt.ReppyRobotParser" - ) diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 28ad910a8..541979dcc 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -3,15 +3,6 @@ from twisted.trial import unittest from scrapy.robotstxt import decode_robotstxt -def reppy_available(): - # check if reppy parser is installed - try: - from reppy.robots import Robots # noqa: F401 - except ImportError: - return False - return True - - def rerp_available(): # check if robotexclusionrulesparser is installed try: @@ -169,21 +160,6 @@ class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): raise unittest.SkipTest("RobotFileParser does not support wildcards.") -class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase): - if not reppy_available(): - skip = "Reppy parser is not installed" - - def setUp(self): - from scrapy.robotstxt import ReppyRobotParser - - super()._setUp(ReppyRobotParser) - - def test_order_based_precedence(self): - raise unittest.SkipTest( - "Reppy does not support order based directives precedence." - ) - - class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase): if not rerp_available(): skip = "Rerp parser is not installed" From 677e9772070ec8a92033f66dff45d7c421763203 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 18 Oct 2024 00:03:32 +0500 Subject: [PATCH 269/269] Remove dead links to the Reppy doc from the release notes. --- docs/news.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 58b51c9ea..2bbca77cc 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1530,7 +1530,7 @@ Documentation - Provided better context and instructions to disable the :setting:`URLLENGTH_LIMIT` setting. (:issue:`5135`, :issue:`5250`) -- Documented that :ref:`reppy-parser` does not support Python 3.9+. +- Documented that Reppy parser does not support Python 3.9+. (:issue:`5226`, :issue:`5231`) - Documented :ref:`the scheduler component <topics-scheduler>`. @@ -3344,7 +3344,7 @@ New features * A new :setting:`ROBOTSTXT_PARSER` setting allows choosing which robots.txt_ parser to use. It includes built-in support for :ref:`RobotFileParser <python-robotfileparser>`, - :ref:`Protego <protego-parser>` (default), :ref:`Reppy <reppy-parser>`, and + :ref:`Protego <protego-parser>` (default), Reppy, and :ref:`Robotexclusionrulesparser <rerp-parser>`, and allows you to :ref:`implement support for additional parsers <support-for-new-robots-parser>` (:issue:`754`, :issue:`2669`,