Add pyupgrade.

.pre-commit-config.yaml

@@ -22,3 +22,9 @@ repos:
   - id: blacken-docs
     additional_dependencies:
     - black==24.2.0
+- repo: https://github.com/asottile/pyupgrade
+  rev: v3.15.2
+  hooks:
+  - id: pyupgrade
+    args: [--py38-plus, --keep-runtime-typing]
+    exclude: scrapy/__init__.py

scrapy/core/downloader/contextfactory.py

@@ -107,7 +107,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         ctx.set_options(0x4)  # OP_LEGACY_SERVER_CONNECT
         return ctx
 
-    def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
+    def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
         return ScrapyClientTLSOptions(
             hostname.decode("ascii"),
             self.getContext(),
@@ -134,7 +134,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
     ``SSLv23_METHOD``) which allows TLS protocol negotiation.
     """
 
-    def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
+    def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
         # trustRoot set to platformTrust() will use the platform's root CAs.
         #
         # This means that a website like https://www.cacert.org will be rejected
@@ -158,8 +158,8 @@ class AcceptableProtocolsContextFactory:
         self._wrapped_context_factory: Any = context_factory
         self._acceptable_protocols: List[bytes] = acceptable_protocols
 
-    def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
-        options: "ClientTLSOptions" = self._wrapped_context_factory.creatorForNetloc(
+    def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
+        options: ClientTLSOptions = self._wrapped_context_factory.creatorForNetloc(
             hostname, port
         )
         _setAcceptableProtocols(options._ctx, self._acceptable_protocols)

scrapy/downloadermiddlewares/retry.py

@@ -147,9 +147,7 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
         if not settings.getbool("RETRY_ENABLED"):
             raise NotConfigured
         self.max_retry_times = settings.getint("RETRY_TIMES")
-        self.retry_http_codes = set(
-            int(x) for x in settings.getlist("RETRY_HTTP_CODES")
-        )
+        self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")}
         self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
 
         try:

scrapy/extensions/debug.py

@@ -55,7 +55,7 @@ class StackTraceDump:
         )
 
     def _thread_stacks(self) -> str:
-        id2name = dict((th.ident, th.name) for th in threading.enumerate())
+        id2name = {th.ident: th.name for th in threading.enumerate()}
         dumps = ""
         for id_, frame in sys._current_frames().items():
             name = id2name.get(id_, "")

scrapy/http/request/__init__.py

@@ -189,10 +189,10 @@ class Request(object_ref):
     def __repr__(self) -> str:
         return f"<{self.method} {self.url}>"
 
-    def copy(self) -> "Request":
+    def copy(self) -> Request:
         return self.replace()
 
-    def replace(self, *args: Any, **kwargs: Any) -> "Request":
+    def replace(self, *args: Any, **kwargs: Any) -> Request:
         """Create a new Request with the same attributes except for those given new values"""
         for x in self.attributes:
             kwargs.setdefault(x, getattr(self, x))
@@ -237,7 +237,7 @@ class Request(object_ref):
         request_kwargs.update(kwargs)
         return cls(**request_kwargs)
 
-    def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]:
+    def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]:
         """Return a dictionary containing the Request's data.
 
         Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.

scrapy/http/request/rpc.py

@@ -21,7 +21,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps)
 class XmlRpcRequest(Request):
     def __init__(self, *args: Any, encoding: Optional[str] = None, **kwargs: Any):
         if "body" not in kwargs and "params" in kwargs:
-            kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs)
+            kw = {k: kwargs.pop(k) for k in DUMPS_ARGS if k in kwargs}
             kwargs["body"] = xmlrpclib.dumps(**kw)
 
         # spec defines that requests must use POST method

scrapy/settings/__init__.py

@@ -275,7 +275,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         assert isinstance(value, (dict, list))
         return copy.deepcopy(value)
 
-    def getwithbase(self, name: _SettingsKeyT) -> "BaseSettings":
+    def getwithbase(self, name: _SettingsKeyT) -> BaseSettings:
         """Get a composition of a dictionary-like setting and its `_BASE`
         counterpart.
 
@@ -438,7 +438,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         if self.frozen:
             raise TypeError("Trying to modify an immutable Settings object")
 
-    def copy(self) -> "Self":
+    def copy(self) -> Self:
         """
         Make a deep copy of current settings.
 
@@ -460,7 +460,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         """
         self.frozen = True
 
-    def frozencopy(self) -> "Self":
+    def frozencopy(self) -> Self:
         """
         Return an immutable copy of the current settings.
 

scrapy/utils/benchserver.py

@@ -22,9 +22,7 @@ class Root(Resource):
         for nl in nlist:
             args["n"] = nl
             argstr = urlencode(args, doseq=True)
-            request.write(
-                f"<a href='/follow?{argstr}'>follow {nl}</a><br>".encode("utf8")
-            )
+            request.write(f"<a href='/follow?{argstr}'>follow {nl}</a><br>".encode())
         request.write(b"</body></html>")
         return b""
 

scrapy/utils/request.py

@@ -49,9 +49,9 @@ def _serialize_headers(
             yield from request.headers.getlist(header)
 
 
-_fingerprint_cache: (
-    "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
-)
+_fingerprint_cache: WeakKeyDictionary[
+    Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]
+]
 _fingerprint_cache = WeakKeyDictionary()
 
 

tests/mockserver.py

@@ -189,10 +189,10 @@ class Raw(LeafResource):
 class Echo(LeafResource):
     def render_GET(self, request):
         output = {
-            "headers": dict(
-                (to_unicode(k), [to_unicode(v) for v in vs])
+            "headers": {
+                to_unicode(k): [to_unicode(v) for v in vs]
                 for k, vs in request.requestHeaders.getAllRawHeaders()
-            ),
+            },
             "body": to_unicode(request.content.read()),
         }
         return to_bytes(json.dumps(output))

tests/test_downloadermiddleware_cookies.py

@@ -362,7 +362,7 @@ class CookiesMiddlewareTest(TestCase):
 
     def test_request_cookies_encoding(self):
         # 1) UTF8-encoded bytes
-        req1 = Request("http://example.org", cookies={"a": "á".encode("utf8")})
+        req1 = Request("http://example.org", cookies={"a": "á".encode()})
         assert self.mw.process_request(req1, self.spider) is None
         self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1")
 
@@ -379,7 +379,7 @@ class CookiesMiddlewareTest(TestCase):
     @pytest.mark.xfail(reason="Cookie header is not currently being processed")
     def test_request_headers_cookie_encoding(self):
         # 1) UTF8-encoded bytes
-        req1 = Request("http://example.org", headers={"Cookie": "a=á".encode("utf8")})
+        req1 = Request("http://example.org", headers={"Cookie": "a=á".encode()})
         assert self.mw.process_request(req1, self.spider) is None
         self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1")
 

tests/test_downloadermiddleware_redirect.py

@@ -1125,7 +1125,7 @@ class RedirectMiddlewareTest(Base.Test):
 
     def test_utf8_location(self):
         req = Request("http://scrapytest.org/first")
-        utf8_location = "/ação".encode("utf-8")  # header using UTF-8 encoding
+        utf8_location = "/ação".encode()  # header using UTF-8 encoding
         resp = Response(
             "http://scrapytest.org/first",
             headers={"Location": utf8_location},

tests/test_downloadermiddleware_robotstxt.py

@@ -40,9 +40,7 @@ Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
 Disallow: /wiki/Käyttäjä:
 User-Agent: UnicödeBöt
 Disallow: /some/randome/page.html
-""".encode(
-            "utf-8"
-        )
+""".encode()
         response = TextResponse("http://site.local/robots.txt", body=ROBOTS)
 
         def return_response(request):

tests/test_feedexport.py

@@ -1359,13 +1359,13 @@ class FeedExportTest(FeedExportTestBase):
         items = [dict({"foo": "Test\xd6"})]
 
         formats = {
-            "json": '[{"foo": "Test\\u00d6"}]'.encode("utf-8"),
-            "jsonlines": '{"foo": "Test\\u00d6"}\n'.encode("utf-8"),
+            "json": b'[{"foo": "Test\\u00d6"}]',
+            "jsonlines": b'{"foo": "Test\\u00d6"}\n',
             "xml": (
                 '<?xml version="1.0" encoding="utf-8"?>\n'
                 "<items><item><foo>Test\xd6</foo></item></items>"
-            ).encode("utf-8"),
-            "csv": "foo\r\nTest\xd6\r\n".encode("utf-8"),
+            ).encode(),
+            "csv": "foo\r\nTest\xd6\r\n".encode(),
         }
 
         for fmt, expected in formats.items():
@@ -1379,13 +1379,13 @@ class FeedExportTest(FeedExportTestBase):
             self.assertEqual(expected, data[fmt])
 
         formats = {
-            "json": '[{"foo": "Test\xd6"}]'.encode("latin-1"),
-            "jsonlines": '{"foo": "Test\xd6"}\n'.encode("latin-1"),
+            "json": b'[{"foo": "Test\xd6"}]',
+            "jsonlines": b'{"foo": "Test\xd6"}\n',
             "xml": (
-                '<?xml version="1.0" encoding="latin-1"?>\n'
-                "<items><item><foo>Test\xd6</foo></item></items>"
-            ).encode("latin-1"),
-            "csv": "foo\r\nTest\xd6\r\n".encode("latin-1"),
+                b'<?xml version="1.0" encoding="latin-1"?>\n'
+                b"<items><item><foo>Test\xd6</foo></item></items>"
+            ),
+            "csv": b"foo\r\nTest\xd6\r\n",
         }
 
         for fmt, expected in formats.items():
@@ -1404,12 +1404,12 @@ class FeedExportTest(FeedExportTestBase):
         items = [dict({"foo": "FOO", "bar": "BAR"})]
 
         formats = {
-            "json": '[\n{"bar": "BAR"}\n]'.encode("utf-8"),
+            "json": b'[\n{"bar": "BAR"}\n]',
             "xml": (
-                '<?xml version="1.0" encoding="latin-1"?>\n'
-                "<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>"
-            ).encode("latin-1"),
-            "csv": "bar,foo\r\nBAR,FOO\r\n".encode("utf-8"),
+                b'<?xml version="1.0" encoding="latin-1"?>\n'
+                b"<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>"
+            ),
+            "csv": b"bar,foo\r\nBAR,FOO\r\n",
         }
 
         settings = {
@@ -1663,8 +1663,8 @@ class FeedExportTest(FeedExportTestBase):
     def test_extend_kwargs(self):
         items = [{"foo": "FOO", "bar": "BAR"}]
 
-        expected_with_title_csv = "foo,bar\r\nFOO,BAR\r\n".encode("utf-8")
-        expected_without_title_csv = "FOO,BAR\r\n".encode("utf-8")
+        expected_with_title_csv = b"foo,bar\r\nFOO,BAR\r\n"
+        expected_without_title_csv = b"FOO,BAR\r\n"
         test_cases = [
             # with title
             {
@@ -2519,22 +2519,22 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
         formats = {
             "json": [
-                '[\n{"bar": "BAR"}\n]'.encode("utf-8"),
-                '[\n{"bar": "BAR1"}\n]'.encode("utf-8"),
+                b'[\n{"bar": "BAR"}\n]',
+                b'[\n{"bar": "BAR1"}\n]',
             ],
             "xml": [
                 (
-                    '<?xml version="1.0" encoding="latin-1"?>\n'
-                    "<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>"
-                ).encode("latin-1"),
+                    b'<?xml version="1.0" encoding="latin-1"?>\n'
+                    b"<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>"
+                ),
                 (
-                    '<?xml version="1.0" encoding="latin-1"?>\n'
-                    "<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>"
-                ).encode("latin-1"),
+                    b'<?xml version="1.0" encoding="latin-1"?>\n'
+                    b"<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>"
+                ),
             ],
             "csv": [
-                "foo,bar\r\nFOO,BAR\r\n".encode("utf-8"),
-                "foo,bar\r\nFOO1,BAR1\r\n".encode("utf-8"),
+                b"foo,bar\r\nFOO,BAR\r\n",
+                b"foo,bar\r\nFOO1,BAR1\r\n",
             ],
         }
 
@@ -2577,8 +2577,8 @@ class BatchDeliveriesTest(FeedExportTestBase):
         items = [dict({"foo": "FOO"}), dict({"foo": "FOO1"})]
         formats = {
             "json": [
-                '[{"foo": "FOO"}]'.encode("utf-8"),
-                '[{"foo": "FOO1"}]'.encode("utf-8"),
+                b'[{"foo": "FOO"}]',
+                b'[{"foo": "FOO1"}]',
             ],
         }
         settings = {

tests/test_http_response.py

@@ -728,9 +728,7 @@ class TextResponseTest(BaseResponseTest):
         resp1 = self.response_class(
             "http://example.com",
             encoding="utf8",
-            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode(
-                "utf8"
-            ),
+            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode(),
         )
         req = self._assert_followed_url(
             resp1.css("a")[0],

tests/test_pipeline_crawl.py

@@ -107,9 +107,7 @@ class FileDownloadCrawlTestCase(TestCase):
 
         # check that the images/files checksums are what we know they should be
         if self.expected_checksums is not None:
-            checksums = set(
-                i["checksum"] for item in items for i in item[self.media_key]
-            )
+            checksums = {i["checksum"] for item in items for i in item[self.media_key]}
             self.assertEqual(checksums, self.expected_checksums)
 
         # check that the image files where actually written to the media store

tests/test_pipeline_images.py

@@ -628,7 +628,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
 
 class NoimagesDropTestCase(unittest.TestCase):
     def test_deprecation_warning(self):
-        arg = str()
+        arg = ""
         with warnings.catch_warnings(record=True) as w:
             NoimagesDrop(arg)
             self.assertEqual(len(w), 1)

tests/test_responsetypes.py

@@ -29,7 +29,7 @@ class ResponseTypesTest(unittest.TestCase):
         mappings = [
             (b'attachment; filename="data.xml"', XmlResponse),
             (b"attachment; filename=data.xml", XmlResponse),
-            ("attachment;filename=data£.tar.gz".encode("utf-8"), Response),
+            ("attachment;filename=data£.tar.gz".encode(), Response),
             ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response),
             ("attachment;filename=data高.doc".encode("gbk"), Response),
             ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse),

tests/test_robotstxt_interface.py

@@ -36,10 +36,10 @@ class BaseRobotParserTest:
 
     def test_allowed(self):
         robotstxt_robotstxt_body = (
-            "User-agent: * \n"
-            "Disallow: /disallowed \n"
-            "Allow: /allowed \n"
-            "Crawl-delay: 10".encode("utf-8")
+            b"User-agent: * \n"
+            b"Disallow: /disallowed \n"
+            b"Allow: /allowed \n"
+            b"Crawl-delay: 10"
         )
         rp = self.parser_cls.from_crawler(
             crawler=None, robotstxt_body=robotstxt_robotstxt_body
@@ -48,15 +48,13 @@ class BaseRobotParserTest:
         self.assertFalse(rp.allowed("https://www.site.local/disallowed", "*"))
 
     def test_allowed_wildcards(self):
-        robotstxt_robotstxt_body = """User-agent: first
+        robotstxt_robotstxt_body = b"""User-agent: first
                                 Disallow: /disallowed/*/end$
 
                                 User-agent: second
                                 Allow: /*allowed
                                 Disallow: /
-                                """.encode(
-            "utf-8"
-        )
+                                """
         rp = self.parser_cls.from_crawler(
             crawler=None, robotstxt_body=robotstxt_robotstxt_body
         )
@@ -77,18 +75,14 @@ class BaseRobotParserTest:
         self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second"))
 
     def test_length_based_precedence(self):
-        robotstxt_robotstxt_body = (
-            "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8")
-        )
+        robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page"
         rp = self.parser_cls.from_crawler(
             crawler=None, robotstxt_body=robotstxt_robotstxt_body
         )
         self.assertTrue(rp.allowed("https://www.site.local/page", "*"))
 
     def test_order_based_precedence(self):
-        robotstxt_robotstxt_body = (
-            "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8")
-        )
+        robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page"
         rp = self.parser_cls.from_crawler(
             crawler=None, robotstxt_body=robotstxt_robotstxt_body
         )
@@ -123,9 +117,7 @@ class BaseRobotParserTest:
         Disallow: /wiki/Käyttäjä:
 
         User-Agent: UnicödeBöt
-        Disallow: /some/randome/page.html""".encode(
-            "utf-8"
-        )
+        Disallow: /some/randome/page.html""".encode()
         rp = self.parser_cls.from_crawler(
             crawler=None, robotstxt_body=robotstxt_robotstxt_body
         )
@@ -145,14 +137,14 @@ class BaseRobotParserTest:
 
 class DecodeRobotsTxtTest(unittest.TestCase):
     def test_native_string_conversion(self):
-        robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")
+        robotstxt_body = b"User-agent: *\nDisallow: /\n"
         decoded_content = decode_robotstxt(
             robotstxt_body, spider=None, to_native_str_type=True
         )
         self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")
 
     def test_decode_utf8(self):
-        robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")
+        robotstxt_body = b"User-agent: *\nDisallow: /\n"
         decoded_content = decode_robotstxt(robotstxt_body, spider=None)
         self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")