diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a911d4cfe..f76a04ca1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,3 +22,9 @@ repos: - id: blacken-docs additional_dependencies: - black==24.2.0 +- repo: https://github.com/asottile/pyupgrade + rev: v3.15.2 + hooks: + - id: pyupgrade + args: [--py38-plus, --keep-runtime-typing] + exclude: scrapy/__init__.py diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 0e77cd2fe..9f6edb630 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -107,7 +107,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT return ctx - def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": + def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: return ScrapyClientTLSOptions( hostname.decode("ascii"), self.getContext(), @@ -134,7 +134,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): ``SSLv23_METHOD``) which allows TLS protocol negotiation. """ - def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": + def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: # trustRoot set to platformTrust() will use the platform's root CAs. # # This means that a website like https://www.cacert.org will be rejected @@ -158,8 +158,8 @@ class AcceptableProtocolsContextFactory: self._wrapped_context_factory: Any = context_factory self._acceptable_protocols: List[bytes] = acceptable_protocols - def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": - options: "ClientTLSOptions" = self._wrapped_context_factory.creatorForNetloc( + def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: + options: ClientTLSOptions = self._wrapped_context_factory.creatorForNetloc( hostname, port ) _setAcceptableProtocols(options._ctx, self._acceptable_protocols) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 46587a898..0637f09d4 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -147,9 +147,7 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): if not settings.getbool("RETRY_ENABLED"): raise NotConfigured self.max_retry_times = settings.getint("RETRY_TIMES") - self.retry_http_codes = set( - int(x) for x in settings.getlist("RETRY_HTTP_CODES") - ) + self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")} self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") try: diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 26726b662..a0fc7b99f 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -55,7 +55,7 @@ class StackTraceDump: ) def _thread_stacks(self) -> str: - id2name = dict((th.ident, th.name) for th in threading.enumerate()) + id2name = {th.ident: th.name for th in threading.enumerate()} dumps = "" for id_, frame in sys._current_frames().items(): name = id2name.get(id_, "") diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 77149333c..3da2e111d 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -189,10 +189,10 @@ class Request(object_ref): def __repr__(self) -> str: return f"<{self.method} {self.url}>" - def copy(self) -> "Request": + def copy(self) -> Request: return self.replace() - def replace(self, *args: Any, **kwargs: Any) -> "Request": + def replace(self, *args: Any, **kwargs: Any) -> Request: """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) @@ -237,7 +237,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]: + def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index e20e7c438..096ecd370 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -21,7 +21,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): def __init__(self, *args: Any, encoding: Optional[str] = None, **kwargs: Any): if "body" not in kwargs and "params" in kwargs: - kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs) + kw = {k: kwargs.pop(k) for k in DUMPS_ARGS if k in kwargs} kwargs["body"] = xmlrpclib.dumps(**kw) # spec defines that requests must use POST method diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 4448b6f4b..ea1db03f1 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -275,7 +275,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): assert isinstance(value, (dict, list)) return copy.deepcopy(value) - def getwithbase(self, name: _SettingsKeyT) -> "BaseSettings": + def getwithbase(self, name: _SettingsKeyT) -> BaseSettings: """Get a composition of a dictionary-like setting and its `_BASE` counterpart. @@ -438,7 +438,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): if self.frozen: raise TypeError("Trying to modify an immutable Settings object") - def copy(self) -> "Self": + def copy(self) -> Self: """ Make a deep copy of current settings. @@ -460,7 +460,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ self.frozen = True - def frozencopy(self) -> "Self": + def frozencopy(self) -> Self: """ Return an immutable copy of the current settings. diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index e9ea51aa1..550516141 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -22,9 +22,7 @@ class Root(Resource): for nl in nlist: args["n"] = nl argstr = urlencode(args, doseq=True) - request.write( - f"follow {nl}
".encode("utf8") - ) + request.write(f"follow {nl}
".encode()) request.write(b"") return b"" diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 42a6537a8..aa0b90ee8 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -49,9 +49,9 @@ def _serialize_headers( yield from request.headers.getlist(header) -_fingerprint_cache: ( - "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" -) +_fingerprint_cache: WeakKeyDictionary[ + Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes] +] _fingerprint_cache = WeakKeyDictionary() diff --git a/tests/mockserver.py b/tests/mockserver.py index 647b0682e..233f6b934 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -189,10 +189,10 @@ class Raw(LeafResource): class Echo(LeafResource): def render_GET(self, request): output = { - "headers": dict( - (to_unicode(k), [to_unicode(v) for v in vs]) + "headers": { + to_unicode(k): [to_unicode(v) for v in vs] for k, vs in request.requestHeaders.getAllRawHeaders() - ), + }, "body": to_unicode(request.content.read()), } return to_bytes(json.dumps(output)) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 5eccd396a..6e343d035 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -362,7 +362,7 @@ class CookiesMiddlewareTest(TestCase): def test_request_cookies_encoding(self): # 1) UTF8-encoded bytes - req1 = Request("http://example.org", cookies={"a": "á".encode("utf8")}) + req1 = Request("http://example.org", cookies={"a": "á".encode()}) assert self.mw.process_request(req1, self.spider) is None self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1") @@ -379,7 +379,7 @@ class CookiesMiddlewareTest(TestCase): @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_request_headers_cookie_encoding(self): # 1) UTF8-encoded bytes - req1 = Request("http://example.org", headers={"Cookie": "a=á".encode("utf8")}) + req1 = Request("http://example.org", headers={"Cookie": "a=á".encode()}) assert self.mw.process_request(req1, self.spider) is None self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1") diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 4bfd34fe2..e37da9715 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1125,7 +1125,7 @@ class RedirectMiddlewareTest(Base.Test): def test_utf8_location(self): req = Request("http://scrapytest.org/first") - utf8_location = "/ação".encode("utf-8") # header using UTF-8 encoding + utf8_location = "/ação".encode() # header using UTF-8 encoding resp = Response( "http://scrapytest.org/first", headers={"Location": utf8_location}, diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 26898a6a1..e166cc000 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -40,9 +40,7 @@ Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: Disallow: /wiki/Käyttäjä: User-Agent: UnicödeBöt Disallow: /some/randome/page.html -""".encode( - "utf-8" - ) +""".encode() response = TextResponse("http://site.local/robots.txt", body=ROBOTS) def return_response(request): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d7560b5ff..3771df8f1 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1359,13 +1359,13 @@ class FeedExportTest(FeedExportTestBase): items = [dict({"foo": "Test\xd6"})] formats = { - "json": '[{"foo": "Test\\u00d6"}]'.encode("utf-8"), - "jsonlines": '{"foo": "Test\\u00d6"}\n'.encode("utf-8"), + "json": b'[{"foo": "Test\\u00d6"}]', + "jsonlines": b'{"foo": "Test\\u00d6"}\n', "xml": ( '\n' "Test\xd6" - ).encode("utf-8"), - "csv": "foo\r\nTest\xd6\r\n".encode("utf-8"), + ).encode(), + "csv": "foo\r\nTest\xd6\r\n".encode(), } for fmt, expected in formats.items(): @@ -1379,13 +1379,13 @@ class FeedExportTest(FeedExportTestBase): self.assertEqual(expected, data[fmt]) formats = { - "json": '[{"foo": "Test\xd6"}]'.encode("latin-1"), - "jsonlines": '{"foo": "Test\xd6"}\n'.encode("latin-1"), + "json": b'[{"foo": "Test\xd6"}]', + "jsonlines": b'{"foo": "Test\xd6"}\n', "xml": ( - '\n' - "Test\xd6" - ).encode("latin-1"), - "csv": "foo\r\nTest\xd6\r\n".encode("latin-1"), + b'\n' + b"Test\xd6" + ), + "csv": b"foo\r\nTest\xd6\r\n", } for fmt, expected in formats.items(): @@ -1404,12 +1404,12 @@ class FeedExportTest(FeedExportTestBase): items = [dict({"foo": "FOO", "bar": "BAR"})] formats = { - "json": '[\n{"bar": "BAR"}\n]'.encode("utf-8"), + "json": b'[\n{"bar": "BAR"}\n]', "xml": ( - '\n' - "\n \n FOO\n \n" - ).encode("latin-1"), - "csv": "bar,foo\r\nBAR,FOO\r\n".encode("utf-8"), + b'\n' + b"\n \n FOO\n \n" + ), + "csv": b"bar,foo\r\nBAR,FOO\r\n", } settings = { @@ -1663,8 +1663,8 @@ class FeedExportTest(FeedExportTestBase): def test_extend_kwargs(self): items = [{"foo": "FOO", "bar": "BAR"}] - expected_with_title_csv = "foo,bar\r\nFOO,BAR\r\n".encode("utf-8") - expected_without_title_csv = "FOO,BAR\r\n".encode("utf-8") + expected_with_title_csv = b"foo,bar\r\nFOO,BAR\r\n" + expected_without_title_csv = b"FOO,BAR\r\n" test_cases = [ # with title { @@ -2519,22 +2519,22 @@ class BatchDeliveriesTest(FeedExportTestBase): formats = { "json": [ - '[\n{"bar": "BAR"}\n]'.encode("utf-8"), - '[\n{"bar": "BAR1"}\n]'.encode("utf-8"), + b'[\n{"bar": "BAR"}\n]', + b'[\n{"bar": "BAR1"}\n]', ], "xml": [ ( - '\n' - "\n \n FOO\n \n" - ).encode("latin-1"), + b'\n' + b"\n \n FOO\n \n" + ), ( - '\n' - "\n \n FOO1\n \n" - ).encode("latin-1"), + b'\n' + b"\n \n FOO1\n \n" + ), ], "csv": [ - "foo,bar\r\nFOO,BAR\r\n".encode("utf-8"), - "foo,bar\r\nFOO1,BAR1\r\n".encode("utf-8"), + b"foo,bar\r\nFOO,BAR\r\n", + b"foo,bar\r\nFOO1,BAR1\r\n", ], } @@ -2577,8 +2577,8 @@ class BatchDeliveriesTest(FeedExportTestBase): items = [dict({"foo": "FOO"}), dict({"foo": "FOO1"})] formats = { "json": [ - '[{"foo": "FOO"}]'.encode("utf-8"), - '[{"foo": "FOO1"}]'.encode("utf-8"), + b'[{"foo": "FOO"}]', + b'[{"foo": "FOO1"}]', ], } settings = { diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 80d46274b..b8a277295 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -728,9 +728,7 @@ class TextResponseTest(BaseResponseTest): resp1 = self.response_class( "http://example.com", encoding="utf8", - body='click me'.encode( - "utf8" - ), + body='click me'.encode(), ) req = self._assert_followed_url( resp1.css("a")[0], diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 5a9a217ce..cd3442dd4 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -107,9 +107,7 @@ class FileDownloadCrawlTestCase(TestCase): # check that the images/files checksums are what we know they should be if self.expected_checksums is not None: - checksums = set( - i["checksum"] for item in items for i in item[self.media_key] - ) + checksums = {i["checksum"] for item in items for i in item[self.media_key]} self.assertEqual(checksums, self.expected_checksums) # check that the image files where actually written to the media store diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 18a2454b3..7d7c78920 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -628,7 +628,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): class NoimagesDropTestCase(unittest.TestCase): def test_deprecation_warning(self): - arg = str() + arg = "" with warnings.catch_warnings(record=True) as w: NoimagesDrop(arg) self.assertEqual(len(w), 1) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 2633cca5b..7be8150fc 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -29,7 +29,7 @@ class ResponseTypesTest(unittest.TestCase): mappings = [ (b'attachment; filename="data.xml"', XmlResponse), (b"attachment; filename=data.xml", XmlResponse), - ("attachment;filename=data£.tar.gz".encode("utf-8"), Response), + ("attachment;filename=data£.tar.gz".encode(), Response), ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response), ("attachment;filename=data高.doc".encode("gbk"), Response), ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse), diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 6ad30deed..28ad910a8 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -36,10 +36,10 @@ class BaseRobotParserTest: def test_allowed(self): robotstxt_robotstxt_body = ( - "User-agent: * \n" - "Disallow: /disallowed \n" - "Allow: /allowed \n" - "Crawl-delay: 10".encode("utf-8") + b"User-agent: * \n" + b"Disallow: /disallowed \n" + b"Allow: /allowed \n" + b"Crawl-delay: 10" ) rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body @@ -48,15 +48,13 @@ class BaseRobotParserTest: self.assertFalse(rp.allowed("https://www.site.local/disallowed", "*")) def test_allowed_wildcards(self): - robotstxt_robotstxt_body = """User-agent: first + robotstxt_robotstxt_body = b"""User-agent: first Disallow: /disallowed/*/end$ User-agent: second Allow: /*allowed Disallow: / - """.encode( - "utf-8" - ) + """ rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) @@ -77,18 +75,14 @@ class BaseRobotParserTest: self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second")) def test_length_based_precedence(self): - robotstxt_robotstxt_body = ( - "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8") - ) + robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) self.assertTrue(rp.allowed("https://www.site.local/page", "*")) def test_order_based_precedence(self): - robotstxt_robotstxt_body = ( - "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8") - ) + robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) @@ -123,9 +117,7 @@ class BaseRobotParserTest: Disallow: /wiki/Käyttäjä: User-Agent: UnicödeBöt - Disallow: /some/randome/page.html""".encode( - "utf-8" - ) + Disallow: /some/randome/page.html""".encode() rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) @@ -145,14 +137,14 @@ class BaseRobotParserTest: class DecodeRobotsTxtTest(unittest.TestCase): def test_native_string_conversion(self): - robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8") + robotstxt_body = b"User-agent: *\nDisallow: /\n" decoded_content = decode_robotstxt( robotstxt_body, spider=None, to_native_str_type=True ) self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") def test_decode_utf8(self): - robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8") + robotstxt_body = b"User-agent: *\nDisallow: /\n" decoded_content = decode_robotstxt(robotstxt_body, spider=None) self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")