Merge branch 'master' into master

.bandit.yml

@@ -1,21 +1,7 @@
 skips:
-- B101
-- B113  # https://github.com/PyCQA/bandit/issues/1010
-- B105
-- B301
-- B303
-- B306
-- B307
-- B311
-- B320
-- B321
-- B324
-- B402  # https://github.com/scrapy/scrapy/issues/4180
-- B403
-- B404
-- B406
-- B410
-- B503
-- B603
-- B605
+- B101  # assert_used, needed for mypy
+- B321  # ftplib, https://github.com/scrapy/scrapy/issues/4180
+- B402  # import_ftplib, https://github.com/scrapy/scrapy/issues/4180
+- B411  # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082
+- B503  # ssl_with_bad_defaults
 exclude_dirs: ['tests']

.flake8

@@ -1,7 +1,7 @@
 [flake8]
 
 max-line-length = 119
-ignore = W503, E203
+ignore = E203, E501, E701, E704, W503
 
 exclude =
     docs/conf.py

.pre-commit-config.yaml

@@ -1,19 +1,19 @@
 repos:
 - repo: https://github.com/PyCQA/bandit
-  rev: 1.7.5
+  rev: 1.7.7
   hooks:
   - id: bandit
     args: [-r, -c, .bandit.yml]
 - repo: https://github.com/PyCQA/flake8
-  rev: 6.1.0
+  rev: 7.0.0
   hooks:
   - id: flake8
 - repo: https://github.com/psf/black.git
-  rev: 23.9.1
+  rev: 24.2.0
   hooks:
   - id: black
 - repo: https://github.com/pycqa/isort
-  rev: 5.12.0
+  rev: 5.13.2
   hooks:
   - id: isort
 - repo: https://github.com/adamchainz/blacken-docs
@@ -21,4 +21,4 @@ repos:
   hooks:
   - id: blacken-docs
     additional_dependencies:
-    - black==23.9.1
+    - black==24.2.0

conftest.py

@@ -1,10 +1,6 @@
-import platform
-import sys
 from pathlib import Path
 
 import pytest
-from twisted import version as twisted_version
-from twisted.python.versions import Version
 from twisted.web.http import H2_ENABLED
 
 from scrapy.utils.reactor import install_reactor
@@ -85,14 +81,12 @@ def only_not_asyncio(request, reactor_pytest):
 def requires_uvloop(request):
     if not request.node.get_closest_marker("requires_uvloop"):
         return
-    if sys.implementation.name == "pypy":
-        pytest.skip("uvloop does not support pypy properly")
-    if platform.system() == "Windows":
-        pytest.skip("uvloop does not support Windows")
-    if twisted_version == Version("twisted", 21, 2, 0):
-        pytest.skip("https://twistedmatrix.com/trac/ticket/10106")
-    if sys.version_info >= (3, 12):
-        pytest.skip("uvloop doesn't support Python 3.12 yet")
+    try:
+        import uvloop
+
+        del uvloop
+    except ImportError:
+        pytest.skip("uvloop is not installed")
 
 
 def pytest_configure(config):

docs/conf.py

@@ -227,7 +227,7 @@ latex_documents = [
 # A list of regular expressions that match URIs that should not be checked when
 # doing a linkcheck build.
 linkcheck_ignore = [
-    "http://localhost:\d+",
+    r"http://localhost:\d+",
     "http://hg.scrapy.org",
     "http://directory.google.com/",
 ]

docs/topics/addons.rst

@@ -150,8 +150,7 @@ Access the crawler instance:
         def from_crawler(cls, crawler):
             return cls(crawler)
 
-        def update_settings(self, settings):
-            ...
+        def update_settings(self, settings): ...
 
 Use a fallback component:
 

docs/topics/autothrottle.rst

@@ -47,6 +47,18 @@ effect, but there are some important differences:
 
 AutoThrottle doesn't have these issues.
 
+Disabling throttling on a downloader slot
+=========================================
+
+It is possible to disable AutoThrottle for a specific download slot at run time
+by setting its ``throttle`` attribute to ``False``, e.g. using
+:setting:`DOWNLOAD_SLOTS`.
+
+Note, however, that AutoThrottle still determines the starting delay of every
+slot by setting the ``download_delay`` attribute on the running spider. You
+might want to set a custom value for the ``delay`` attribute of the slot, e.g.
+using :setting:`DOWNLOAD_SLOTS`.
+
 Throttling algorithm
 ====================
 
@@ -131,7 +143,7 @@ AUTOTHROTTLE_TARGET_CONCURRENCY
 Default: ``1.0``
 
 Average number of requests Scrapy should be sending in parallel to remote
-websites.
+websites. It must be higher than ``0.0``.
 
 By default, AutoThrottle adjusts the delay to send a single
 concurrent request to each of the remote websites. Set this option to

docs/topics/settings.rst

@@ -835,7 +835,7 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
 .. setting:: DOWNLOAD_SLOTS
 
 DOWNLOAD_SLOTS
-----------------
+--------------
 
 Default: ``{}``
 
@@ -844,7 +844,12 @@ Allows to define concurrency/delay parameters on per slot (domain) basis:
     .. code-block:: python
 
         DOWNLOAD_SLOTS = {
-            "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False},
+            "quotes.toscrape.com": {
+                "concurrency": 1,
+                "delay": 2,
+                "randomize_delay": False,
+                "throttle": False,
+            },
             "books.toscrape.com": {"delay": 3, "randomize_delay": False},
         }
 
@@ -856,6 +861,9 @@ Allows to define concurrency/delay parameters on per slot (domain) basis:
     -   :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency``
     -   :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay``
 
+    There is no global setting for ``throttle``, whose default value is
+    ``None``.
+
 
 .. setting:: DOWNLOAD_TIMEOUT
 

docs/topics/telnetconsole.rst

@@ -172,8 +172,8 @@ TELNETCONSOLE_PORT
 
 Default: ``[6023, 6073]``
 
-The port range to use for the telnet console. If set to ``None`` or ``0``, a
-dynamically assigned port is used.
+The port range to use for the telnet console. If set to ``None``, a dynamically
+assigned port is used.
 
 
 .. setting:: TELNETCONSOLE_HOST

pylintrc

@@ -4,21 +4,14 @@ jobs=1  # >1 hides results
 
 [MESSAGES CONTROL]
 disable=abstract-method,
-        anomalous-backslash-in-string,
         arguments-differ,
         arguments-renamed,
         attribute-defined-outside-init,
         bad-classmethod-argument,
-        bad-mcs-classmethod-argument,
         bare-except,
         broad-except,
         broad-exception-raised,
         c-extension-no-member,
-        catching-non-exception,
-        cell-var-from-loop,
-        comparison-with-callable,
-        consider-using-dict-items,
-        consider-using-in,
         consider-using-with,
         cyclic-import,
         dangerous-default-value,
@@ -32,7 +25,6 @@ disable=abstract-method,
         implicit-str-concat,
         import-error,
         import-outside-toplevel,
-        import-self,
         inconsistent-return-statements,
         inherit-non-class,
         invalid-name,
@@ -44,7 +36,6 @@ disable=abstract-method,
         logging-fstring-interpolation,
         logging-not-lazy,
         lost-exception,
-        method-hidden,
         missing-docstring,
         no-else-raise,
         no-else-return,
@@ -52,7 +43,7 @@ disable=abstract-method,
         no-method-argument,
         no-name-in-module,
         no-self-argument,
-        no-value-for-parameter,
+        no-value-for-parameter,  # https://github.com/pylint-dev/pylint/issues/3268
         not-callable,
         pointless-exception-statement,
         pointless-statement,
@@ -77,23 +68,15 @@ disable=abstract-method,
         too-many-public-methods,
         too-many-return-statements,
         unbalanced-tuple-unpacking,
-        undefined-variable,
-        undefined-loop-variable,
-        unexpected-special-method-signature,
-        unnecessary-comprehension,
         unnecessary-dunder-call,
         unnecessary-pass,
         unreachable,
-        unsubscriptable-object,
         unused-argument,
         unused-import,
         unused-private-member,
         unused-variable,
         unused-wildcard-import,
-        use-dict-literal,
         used-before-assignment,
-        useless-object-inheritance,  # Required for Python 2 support
         useless-return,
-        useless-super-delegation,
         wildcard-import,
         wrong-import-position

scrapy/commands/__init__.py

@@ -1,62 +1,64 @@
 """
 Base class for Scrapy commands
 """
+
 import argparse
+import builtins
 import os
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Iterable, List, Optional
 
 from twisted.python import failure
 
-from scrapy.crawler import CrawlerProcess
+from scrapy.crawler import Crawler, CrawlerProcess
 from scrapy.exceptions import UsageError
 from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
 
 
 class ScrapyCommand:
-    requires_project = False
+    requires_project: bool = False
     crawler_process: Optional[CrawlerProcess] = None
 
     # default settings to be used for this command instead of global defaults
     default_settings: Dict[str, Any] = {}
 
-    exitcode = 0
+    exitcode: int = 0
 
     def __init__(self) -> None:
         self.settings: Any = None  # set in scrapy.cmdline
 
-    def set_crawler(self, crawler):
+    def set_crawler(self, crawler: Crawler) -> None:
         if hasattr(self, "_crawler"):
             raise RuntimeError("crawler already set")
-        self._crawler = crawler
+        self._crawler: Crawler = crawler
 
-    def syntax(self):
+    def syntax(self) -> str:
         """
         Command syntax (preferably one-line). Do not include command name.
         """
         return ""
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         """
         A short description of the command
         """
         return ""
 
-    def long_desc(self):
+    def long_desc(self) -> str:
         """A long description of the command. Return short description when not
         available. It cannot contain newlines since contents will be formatted
         by optparser which removes newlines and wraps text.
         """
         return self.short_desc()
 
-    def help(self):
+    def help(self) -> str:
         """An extensive help for the command. It will be shown when using the
         "help" command. It can contain newlines since no post-formatting will
         be applied to its contents.
         """
         return self.long_desc()
 
-    def add_options(self, parser):
+    def add_options(self, parser: argparse.ArgumentParser) -> None:
         """
         Populate option parse with options available for this command
         """
@@ -91,7 +93,7 @@ class ScrapyCommand:
         )
         group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
 
-    def process_options(self, args, opts):
+    def process_options(self, args: List[str], opts: argparse.Namespace) -> None:
         try:
             self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline")
         except ValueError:
@@ -128,8 +130,8 @@ class BaseRunSpiderCommand(ScrapyCommand):
     Common class used to share functionality between the crawl, parse and runspider commands
     """
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+    def add_options(self, parser: argparse.ArgumentParser) -> None:
+        super().add_options(parser)
         parser.add_argument(
             "-a",
             dest="spargs",
@@ -161,8 +163,8 @@ class BaseRunSpiderCommand(ScrapyCommand):
             help="format to use for dumping items",
         )
 
-    def process_options(self, args, opts):
-        ScrapyCommand.process_options(self, args, opts)
+    def process_options(self, args: List[str], opts: argparse.Namespace) -> None:
+        super().process_options(args, opts)
         try:
             opts.spargs = arglist_to_dict(opts.spargs)
         except ValueError:
@@ -182,7 +184,13 @@ class ScrapyHelpFormatter(argparse.HelpFormatter):
     Help Formatter for scrapy command line help messages.
     """
 
-    def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
+    def __init__(
+        self,
+        prog: str,
+        indent_increment: int = 2,
+        max_help_position: int = 24,
+        width: Optional[int] = None,
+    ):
         super().__init__(
             prog,
             indent_increment=indent_increment,
@@ -190,11 +198,12 @@ class ScrapyHelpFormatter(argparse.HelpFormatter):
             width=width,
         )
 
-    def _join_parts(self, part_strings):
-        parts = self.format_part_strings(part_strings)
+    def _join_parts(self, part_strings: Iterable[str]) -> str:
+        # scrapy.commands.list shadows builtins.list
+        parts = self.format_part_strings(builtins.list(part_strings))
         return super()._join_parts(parts)
 
-    def format_part_strings(self, part_strings):
+    def format_part_strings(self, part_strings: List[str]) -> List[str]:
         """
         Underline and title case command line help message headers.
         """

scrapy/commands/bench.py

@@ -1,10 +1,14 @@
-import subprocess
+import argparse
+import subprocess  # nosec
 import sys
 import time
+from typing import Any, Iterable, List
 from urllib.parse import urlencode
 
 import scrapy
+from scrapy import Request
 from scrapy.commands import ScrapyCommand
+from scrapy.http import Response
 from scrapy.linkextractors import LinkExtractor
 
 
@@ -15,24 +19,28 @@ class Command(ScrapyCommand):
         "CLOSESPIDER_TIMEOUT": 10,
     }
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Run quick benchmark test"
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         with _BenchServer():
+            assert self.crawler_process
             self.crawler_process.crawl(_BenchSpider, total=100000)
             self.crawler_process.start()
 
 
 class _BenchServer:
-    def __enter__(self):
+    def __enter__(self) -> None:
         from scrapy.utils.test import get_testenv
 
         pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]
-        self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE, env=get_testenv())
+        self.proc = subprocess.Popen(
+            pargs, stdout=subprocess.PIPE, env=get_testenv()
+        )  # nosec
+        assert self.proc.stdout
         self.proc.stdout.readline()
 
-    def __exit__(self, exc_type, exc_value, traceback):
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
         self.proc.kill()
         self.proc.wait()
         time.sleep(0.2)
@@ -47,11 +55,11 @@ class _BenchSpider(scrapy.Spider):
     baseurl = "http://localhost:8998"
     link_extractor = LinkExtractor()
 
-    def start_requests(self):
+    def start_requests(self) -> Iterable[Request]:
         qargs = {"total": self.total, "show": self.show}
         url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}"
         return [scrapy.Request(url, dont_filter=True)]
 
-    def parse(self, response):
+    def parse(self, response: Response) -> Any:  # type: ignore[override]
         for link in self.link_extractor.extract_links(response):
             yield scrapy.Request(link.url, callback=self.parse)

scrapy/commands/check.py

@@ -1,5 +1,7 @@
+import argparse
 import time
 from collections import defaultdict
+from typing import List
 from unittest import TextTestResult as _TextTestResult
 from unittest import TextTestRunner
 
@@ -10,9 +12,10 @@ from scrapy.utils.misc import load_object, set_environ
 
 
 class TextTestResult(_TextTestResult):
-    def printSummary(self, start, stop):
+    def printSummary(self, start: float, stop: float) -> None:
         write = self.stream.write
-        writeln = self.stream.writeln
+        # _WritelnDecorator isn't implemented in typeshed yet
+        writeln = self.stream.writeln  # type: ignore[attr-defined]
 
         run = self.testsRun
         plural = "s" if run != 1 else ""
@@ -42,14 +45,14 @@ class Command(ScrapyCommand):
     requires_project = True
     default_settings = {"LOG_ENABLED": False}
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[options] <spider>"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Check spider contracts"
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+    def add_options(self, parser: argparse.ArgumentParser) -> None:
+        super().add_options(parser)
         parser.add_argument(
             "-l",
             "--list",
@@ -66,7 +69,7 @@ class Command(ScrapyCommand):
             help="print contract tests for all spiders",
         )
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         # load contracts
         contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS"))
         conman = ContractsManager(load_object(c) for c in contracts)
@@ -76,6 +79,7 @@ class Command(ScrapyCommand):
         # contract requests
         contract_reqs = defaultdict(list)
 
+        assert self.crawler_process
         spider_loader = self.crawler_process.spider_loader
 
         with set_environ(SCRAPY_CHECK="true"):

scrapy/commands/crawl.py

@@ -1,3 +1,8 @@
+import argparse
+from typing import List, cast
+
+from twisted.python.failure import Failure
+
 from scrapy.commands import BaseRunSpiderCommand
 from scrapy.exceptions import UsageError
 
@@ -5,13 +10,13 @@ from scrapy.exceptions import UsageError
 class Command(BaseRunSpiderCommand):
     requires_project = True
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[options] <spider>"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Run a spider"
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         if len(args) < 1:
             raise UsageError()
         elif len(args) > 1:
@@ -20,10 +25,11 @@ class Command(BaseRunSpiderCommand):
             )
         spname = args[0]
 
+        assert self.crawler_process
         crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)
 
         if getattr(crawl_defer, "result", None) is not None and issubclass(
-            crawl_defer.result.type, Exception
+            cast(Failure, crawl_defer.result).type, Exception
         ):
             self.exitcode = 1
         else:

scrapy/commands/edit.py

@@ -1,5 +1,7 @@
+import argparse
 import os
 import sys
+from typing import List
 
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
@@ -9,32 +11,34 @@ class Command(ScrapyCommand):
     requires_project = True
     default_settings = {"LOG_ENABLED": False}
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "<spider>"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Edit spider"
 
-    def long_desc(self):
+    def long_desc(self) -> str:
         return (
             "Edit a spider using the editor defined in the EDITOR environment"
             " variable or else the EDITOR setting"
         )
 
-    def _err(self, msg):
+    def _err(self, msg: str) -> None:
         sys.stderr.write(msg + os.linesep)
         self.exitcode = 1
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         if len(args) != 1:
             raise UsageError()
 
         editor = self.settings["EDITOR"]
+        assert self.crawler_process
         try:
             spidercls = self.crawler_process.spider_loader.load(args[0])
         except KeyError:
             return self._err(f"Spider not found: {args[0]}")
 
         sfile = sys.modules[spidercls.__module__].__file__
+        assert sfile
         sfile = sfile.replace(".pyc", ".py")
-        self.exitcode = os.system(f'{editor} "{sfile}"')
+        self.exitcode = os.system(f'{editor} "{sfile}"')  # nosec

scrapy/commands/fetch.py

@@ -1,13 +1,13 @@
 import sys
-from argparse import Namespace
-from typing import List, Type
+from argparse import ArgumentParser, Namespace
+from typing import Dict, List, Type
 
 from w3lib.url import is_url
 
 from scrapy import Spider
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
-from scrapy.http import Request
+from scrapy.http import Request, Response
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.spider import DefaultSpider, spidercls_for_request
 
@@ -15,20 +15,20 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request
 class Command(ScrapyCommand):
     requires_project = False
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[options] <url>"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Fetch a URL using the Scrapy downloader"
 
-    def long_desc(self):
+    def long_desc(self) -> str:
         return (
             "Fetch a URL using the Scrapy downloader and print its content"
             " to stdout. You may want to use --nolog to disable logging"
         )
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+    def add_options(self, parser: ArgumentParser) -> None:
+        super().add_options(parser)
         parser.add_argument("--spider", dest="spider", help="use this spider")
         parser.add_argument(
             "--headers",
@@ -44,20 +44,21 @@ class Command(ScrapyCommand):
             help="do not handle HTTP 3xx status codes and print response as-is",
         )
 
-    def _print_headers(self, headers, prefix):
+    def _print_headers(self, headers: Dict[bytes, List[bytes]], prefix: bytes) -> None:
         for key, values in headers.items():
             for value in values:
                 self._print_bytes(prefix + b" " + key + b": " + value)
 
-    def _print_response(self, response, opts):
+    def _print_response(self, response: Response, opts: Namespace) -> None:
         if opts.headers:
+            assert response.request
             self._print_headers(response.request.headers, b">")
             print(">")
             self._print_headers(response.headers, b"<")
         else:
             self._print_bytes(response.body)
 
-    def _print_bytes(self, bytes_):
+    def _print_bytes(self, bytes_: bytes) -> None:
         sys.stdout.buffer.write(bytes_ + b"\n")
 
     def run(self, args: List[str], opts: Namespace) -> None:

scrapy/commands/genspider.py

@@ -1,9 +1,10 @@
+import argparse
 import os
 import shutil
 import string
 from importlib import import_module
 from pathlib import Path
-from typing import Optional, cast
+from typing import List, Optional, Union, cast
 from urllib.parse import urlparse
 
 import scrapy
@@ -12,7 +13,7 @@ from scrapy.exceptions import UsageError
 from scrapy.utils.template import render_templatefile, string_camelcase
 
 
-def sanitize_module_name(module_name):
+def sanitize_module_name(module_name: str) -> str:
     """Sanitize the given module name, by replacing dashes and points
     with underscores and prefixing it with a letter if it doesn't start
     with one
@@ -23,7 +24,7 @@ def sanitize_module_name(module_name):
     return module_name
 
 
-def extract_domain(url):
+def extract_domain(url: str) -> str:
     """Extract domain name from URL string"""
     o = urlparse(url)
     if o.scheme == "" and o.netloc == "":
@@ -31,7 +32,7 @@ def extract_domain(url):
     return o.netloc
 
 
-def verify_url_scheme(url):
+def verify_url_scheme(url: str) -> str:
     """Check url for scheme and insert https if none found."""
     parsed = urlparse(url)
     if parsed.scheme == "" and parsed.netloc == "":
@@ -43,14 +44,14 @@ class Command(ScrapyCommand):
     requires_project = False
     default_settings = {"LOG_ENABLED": False}
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[options] <name> <domain>"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Generate new spider using pre-defined templates"
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+    def add_options(self, parser: argparse.ArgumentParser) -> None:
+        super().add_options(parser)
         parser.add_argument(
             "-l",
             "--list",
@@ -86,7 +87,7 @@ class Command(ScrapyCommand):
             help="If the spider already exists, overwrite it with the template",
         )
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         if opts.list:
             self._list_templates()
             return
@@ -113,9 +114,16 @@ class Command(ScrapyCommand):
         if template_file:
             self._genspider(module, name, url, opts.template, template_file)
             if opts.edit:
-                self.exitcode = os.system(f'scrapy edit "{name}"')
+                self.exitcode = os.system(f'scrapy edit "{name}"')  # nosec
 
-    def _genspider(self, module, name, url, template_name, template_file):
+    def _genspider(
+        self,
+        module: str,
+        name: str,
+        url: str,
+        template_name: str,
+        template_file: Union[str, os.PathLike],
+    ) -> None:
         """Generate the spider module, based on the given template"""
         capitalized_module = "".join(s.capitalize() for s in module.split("_"))
         domain = extract_domain(url)
@@ -130,6 +138,7 @@ class Command(ScrapyCommand):
         }
         if self.settings.get("NEWSPIDER_MODULE"):
             spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
+            assert spiders_module.__file__
             spiders_dir = Path(spiders_module.__file__).parent.resolve()
         else:
             spiders_module = None
@@ -152,7 +161,7 @@ class Command(ScrapyCommand):
         print('Use "scrapy genspider --list" to see all available templates.')
         return None
 
-    def _list_templates(self):
+    def _list_templates(self) -> None:
         print("Available templates:")
         for file in sorted(Path(self.templates_dir).iterdir()):
             if file.suffix == ".tmpl":

scrapy/commands/list.py

@@ -1,3 +1,6 @@
+import argparse
+from typing import List
+
 from scrapy.commands import ScrapyCommand
 
 
@@ -5,9 +8,10 @@ class Command(ScrapyCommand):
     requires_project = True
     default_settings = {"LOG_ENABLED": False}
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "List available spiders"
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
+        assert self.crawler_process
         for s in sorted(self.crawler_process.spider_loader.list()):
             print(s)

scrapy/commands/parse.py

@@ -1,16 +1,32 @@
+import argparse
 import functools
 import inspect
 import json
 import logging
-from typing import Dict
+from types import CoroutineType
+from typing import (
+    Any,
+    AsyncGenerator,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+    TypeVar,
+    Union,
+    overload,
+)
 
 from itemadapter import ItemAdapter, is_item
-from twisted.internet.defer import maybeDeferred
+from twisted.internet.defer import Deferred, maybeDeferred
+from twisted.python.failure import Failure
 from w3lib.url import is_url
 
 from scrapy.commands import BaseRunSpiderCommand
 from scrapy.exceptions import UsageError
-from scrapy.http import Request
+from scrapy.http import Request, Response
+from scrapy.spiders import Spider
 from scrapy.utils import display
 from scrapy.utils.asyncgen import collect_asyncgen
 from scrapy.utils.defer import aiter_errback, deferred_from_coro
@@ -20,24 +36,26 @@ from scrapy.utils.spider import spidercls_for_request
 
 logger = logging.getLogger(__name__)
 
+_T = TypeVar("_T")
+
 
 class Command(BaseRunSpiderCommand):
     requires_project = True
 
     spider = None
-    items: Dict[int, list] = {}
-    requests: Dict[int, list] = {}
+    items: Dict[int, List[Any]] = {}
+    requests: Dict[int, List[Request]] = {}
 
     first_response = None
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[options] <url>"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Parse URL (using its spider) and print the results"
 
-    def add_options(self, parser):
-        BaseRunSpiderCommand.add_options(self, parser)
+    def add_options(self, parser: argparse.ArgumentParser) -> None:
+        super().add_options(parser)
         parser.add_argument(
             "--spider",
             dest="spider",
@@ -106,7 +124,7 @@ class Command(BaseRunSpiderCommand):
         )
 
     @property
-    def max_level(self):
+    def max_level(self) -> int:
         max_items, max_requests = 0, 0
         if self.items:
             max_items = max(self.items)
@@ -114,13 +132,21 @@ class Command(BaseRunSpiderCommand):
             max_requests = max(self.requests)
         return max(max_items, max_requests)
 
-    def handle_exception(self, _failure):
+    def handle_exception(self, _failure: Failure) -> None:
         logger.error(
             "An error is caught while iterating the async iterable",
             exc_info=failure_to_exc_info(_failure),
         )
 
-    def iterate_spider_output(self, result):
+    @overload
+    def iterate_spider_output(
+        self, result: Union[AsyncGenerator, CoroutineType]
+    ) -> Deferred: ...
+
+    @overload
+    def iterate_spider_output(self, result: _T) -> Iterable: ...
+
+    def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]:
         if inspect.isasyncgen(result):
             d = deferred_from_coro(
                 collect_asyncgen(aiter_errback(result, self.handle_exception))
@@ -133,15 +159,15 @@ class Command(BaseRunSpiderCommand):
             return d
         return arg_to_iter(deferred_from_coro(result))
 
-    def add_items(self, lvl, new_items):
+    def add_items(self, lvl: int, new_items: List[Any]) -> None:
         old_items = self.items.get(lvl, [])
         self.items[lvl] = old_items + new_items
 
-    def add_requests(self, lvl, new_reqs):
+    def add_requests(self, lvl: int, new_reqs: List[Request]) -> None:
         old_reqs = self.requests.get(lvl, [])
         self.requests[lvl] = old_reqs + new_reqs
 
-    def print_items(self, lvl=None, colour=True):
+    def print_items(self, lvl: Optional[int] = None, colour: bool = True) -> None:
         if lvl is None:
             items = [item for lst in self.items.values() for item in lst]
         else:
@@ -150,7 +176,7 @@ class Command(BaseRunSpiderCommand):
         print("# Scraped Items ", "-" * 60)
         display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)
 
-    def print_requests(self, lvl=None, colour=True):
+    def print_requests(self, lvl: Optional[int] = None, colour: bool = True) -> None:
         if lvl is None:
             if self.requests:
                 requests = self.requests[max(self.requests)]
@@ -162,7 +188,7 @@ class Command(BaseRunSpiderCommand):
         print("# Requests ", "-" * 65)
         display.pprint(requests, colorize=colour)
 
-    def print_results(self, opts):
+    def print_results(self, opts: argparse.Namespace) -> None:
         colour = not opts.nocolour
 
         if opts.verbose:
@@ -179,7 +205,14 @@ class Command(BaseRunSpiderCommand):
             if not opts.nolinks:
                 self.print_requests(colour=colour)
 
-    def _get_items_and_requests(self, spider_output, opts, depth, spider, callback):
+    def _get_items_and_requests(
+        self,
+        spider_output: Iterable[Any],
+        opts: argparse.Namespace,
+        depth: int,
+        spider: Spider,
+        callback: Callable,
+    ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, Callable]:
         items, requests = [], []
         for x in spider_output:
             if is_item(x):
@@ -188,14 +221,21 @@ class Command(BaseRunSpiderCommand):
                 requests.append(x)
         return items, requests, opts, depth, spider, callback
 
-    def run_callback(self, response, callback, cb_kwargs=None):
+    def run_callback(
+        self,
+        response: Response,
+        callback: Callable,
+        cb_kwargs: Optional[Dict[str, Any]] = None,
+    ) -> Deferred:
         cb_kwargs = cb_kwargs or {}
         d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs))
         return d
 
-    def get_callback_from_rules(self, spider, response):
+    def get_callback_from_rules(
+        self, spider: Spider, response: Response
+    ) -> Union[Callable, str, None]:
         if getattr(spider, "rules", None):
-            for rule in spider.rules:
+            for rule in spider.rules:  # type: ignore[attr-defined]
                 if rule.link_extractor.matches(response.url):
                     return rule.callback or "parse"
         else:
@@ -204,8 +244,10 @@ class Command(BaseRunSpiderCommand):
                 "please specify a callback to use for parsing",
                 {"spider": spider.name},
             )
+        return None
 
-    def set_spidercls(self, url, opts):
+    def set_spidercls(self, url: str, opts: argparse.Namespace) -> None:
+        assert self.crawler_process
         spider_loader = self.crawler_process.spider_loader
         if opts.spider:
             try:
@@ -219,13 +261,14 @@ class Command(BaseRunSpiderCommand):
             if not self.spidercls:
                 logger.error("Unable to find spider for: %(url)s", {"url": url})
 
-        def _start_requests(spider):
+        def _start_requests(spider: Spider) -> Iterable[Request]:
             yield self.prepare_request(spider, Request(url), opts)
 
         if self.spidercls:
             self.spidercls.start_requests = _start_requests
 
-    def start_parsing(self, url, opts):
+    def start_parsing(self, url: str, opts: argparse.Namespace) -> None:
+        assert self.crawler_process
         self.crawler_process.crawl(self.spidercls, **opts.spargs)
         self.pcrawler = list(self.crawler_process.crawlers)[0]
         self.crawler_process.start()
@@ -233,7 +276,12 @@ class Command(BaseRunSpiderCommand):
         if not self.first_response:
             logger.error("No response downloaded for: %(url)s", {"url": url})
 
-    def scraped_data(self, args):
+    def scraped_data(
+        self,
+        args: Tuple[
+            List[Any], List[Request], argparse.Namespace, int, Spider, Callable
+        ],
+    ) -> List[Any]:
         items, requests, opts, depth, spider, callback = args
         if opts.pipelines:
             itemproc = self.pcrawler.engine.scraper.itemproc
@@ -252,8 +300,14 @@ class Command(BaseRunSpiderCommand):
 
         return scraped_data
 
-    def _get_callback(self, *, spider, opts, response=None):
-        cb = None
+    def _get_callback(
+        self,
+        *,
+        spider: Spider,
+        opts: argparse.Namespace,
+        response: Optional[Response] = None,
+    ) -> Callable:
+        cb: Union[str, Callable, None] = None
         if response:
             cb = response.meta["_callback"]
         if not cb:
@@ -270,6 +324,7 @@ class Command(BaseRunSpiderCommand):
                 cb = "parse"
 
         if not callable(cb):
+            assert cb is not None
             cb_method = getattr(spider, cb, None)
             if callable(cb_method):
                 cb = cb_method
@@ -277,10 +332,13 @@ class Command(BaseRunSpiderCommand):
                 raise ValueError(
                     f"Cannot find callback {cb!r} in spider: {spider.name}"
                 )
+        assert callable(cb)
         return cb
 
-    def prepare_request(self, spider, request, opts):
-        def callback(response, **cb_kwargs):
+    def prepare_request(
+        self, spider: Spider, request: Request, opts: argparse.Namespace
+    ) -> Request:
+        def callback(response: Response, **cb_kwargs: Any) -> Deferred:
             # memorize first request
             if not self.first_response:
                 self.first_response = response
@@ -288,7 +346,7 @@ class Command(BaseRunSpiderCommand):
             cb = self._get_callback(spider=spider, opts=opts, response=response)
 
             # parse items and requests
-            depth = response.meta["_depth"]
+            depth: int = response.meta["_depth"]
 
             d = self.run_callback(response, cb, cb_kwargs)
             d.addCallback(self._get_items_and_requests, opts, depth, spider, callback)
@@ -311,13 +369,13 @@ class Command(BaseRunSpiderCommand):
         request.callback = callback
         return request
 
-    def process_options(self, args, opts):
-        BaseRunSpiderCommand.process_options(self, args, opts)
+    def process_options(self, args: List[str], opts: argparse.Namespace) -> None:
+        super().process_options(args, opts)
 
         self.process_request_meta(opts)
         self.process_request_cb_kwargs(opts)
 
-    def process_request_meta(self, opts):
+    def process_request_meta(self, opts: argparse.Namespace) -> None:
         if opts.meta:
             try:
                 opts.meta = json.loads(opts.meta)
@@ -328,7 +386,7 @@ class Command(BaseRunSpiderCommand):
                     print_help=False,
                 )
 
-    def process_request_cb_kwargs(self, opts):
+    def process_request_cb_kwargs(self, opts: argparse.Namespace) -> None:
         if opts.cbkwargs:
             try:
                 opts.cbkwargs = json.loads(opts.cbkwargs)
@@ -339,7 +397,7 @@ class Command(BaseRunSpiderCommand):
                     print_help=False,
                 )
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         # parse arguments
         if not len(args) == 1 or not is_url(args[0]):
             raise UsageError()

scrapy/commands/runspider.py

@@ -1,9 +1,10 @@
+import argparse
 import sys
 from importlib import import_module
 from os import PathLike
 from pathlib import Path
 from types import ModuleType
-from typing import Union
+from typing import List, Union
 
 from scrapy.commands import BaseRunSpiderCommand
 from scrapy.exceptions import UsageError
@@ -27,16 +28,16 @@ class Command(BaseRunSpiderCommand):
     requires_project = False
     default_settings = {"SPIDER_LOADER_WARN_ONLY": True}
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[options] <spider_file>"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Run a self-contained spider (without creating a project)"
 
-    def long_desc(self):
+    def long_desc(self) -> str:
         return "Run the spider defined in the given file"
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         if len(args) != 1:
             raise UsageError()
         filename = Path(args[0])
@@ -51,6 +52,7 @@ class Command(BaseRunSpiderCommand):
             raise UsageError(f"No spider found in file: {filename}\n")
         spidercls = spclasses.pop()
 
+        assert self.crawler_process
         self.crawler_process.crawl(spidercls, **opts.spargs)
         self.crawler_process.start()
 

scrapy/commands/settings.py

@@ -1,4 +1,6 @@
+import argparse
 import json
+from typing import List
 
 from scrapy.commands import ScrapyCommand
 from scrapy.settings import BaseSettings
@@ -8,14 +10,14 @@ class Command(ScrapyCommand):
     requires_project = False
     default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[options]"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Get settings values"
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+    def add_options(self, parser: argparse.ArgumentParser) -> None:
+        super().add_options(parser)
         parser.add_argument(
             "--get", dest="get", metavar="SETTING", help="print raw setting value"
         )
@@ -44,7 +46,8 @@ class Command(ScrapyCommand):
             help="print setting value, interpreted as a list",
         )
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
+        assert self.crawler_process
         settings = self.crawler_process.settings
         if opts.get:
             s = settings.get(opts.get)

scrapy/commands/shell.py

@@ -3,9 +3,10 @@ Scrapy Shell
 
 See documentation in docs/topics/shell.rst
 """
-from argparse import Namespace
+
+from argparse import ArgumentParser, Namespace
 from threading import Thread
-from typing import List, Type
+from typing import Any, Dict, List, Type
 
 from scrapy import Spider
 from scrapy.commands import ScrapyCommand
@@ -23,20 +24,20 @@ class Command(ScrapyCommand):
         "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
     }
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[url|file]"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Interactive scraping console"
 
-    def long_desc(self):
+    def long_desc(self) -> str:
         return (
             "Interactive console for scraping the given url or file. "
             "Use ./file.html syntax or full path for local file."
         )
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+    def add_options(self, parser: ArgumentParser) -> None:
+        super().add_options(parser)
         parser.add_argument(
             "-c",
             dest="code",
@@ -51,7 +52,7 @@ class Command(ScrapyCommand):
             help="do not handle HTTP 3xx status codes and print response as-is",
         )
 
-    def update_vars(self, vars):
+    def update_vars(self, vars: Dict[str, Any]) -> None:
         """You can use this function to update the Scrapy objects that will be
         available in the shell
         """
@@ -87,7 +88,8 @@ class Command(ScrapyCommand):
         shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
         shell.start(url=url, redirect=not opts.no_redirect)
 
-    def _start_crawler_thread(self):
+    def _start_crawler_thread(self) -> None:
+        assert self.crawler_process
         t = Thread(
             target=self.crawler_process.start,
             kwargs={"stop_after_crawl": False, "install_signal_handlers": False},

scrapy/commands/startproject.py

@@ -1,3 +1,4 @@
+import argparse
 import os
 import re
 import string
@@ -5,13 +6,14 @@ from importlib.util import find_spec
 from pathlib import Path
 from shutil import copy2, copystat, ignore_patterns, move
 from stat import S_IWUSR as OWNER_WRITE_PERMISSION
+from typing import List, Tuple, Union
 
 import scrapy
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.template import render_templatefile, string_camelcase
 
-TEMPLATES_TO_RENDER = (
+TEMPLATES_TO_RENDER: Tuple[Tuple[str, ...], ...] = (
     ("scrapy.cfg",),
     ("${project_name}", "settings.py.tmpl"),
     ("${project_name}", "items.py.tmpl"),
@@ -22,7 +24,7 @@ TEMPLATES_TO_RENDER = (
 IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn")
 
 
-def _make_writable(path):
+def _make_writable(path: Union[str, os.PathLike]) -> None:
     current_permissions = os.stat(path).st_mode
     os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION)
 
@@ -31,14 +33,14 @@ class Command(ScrapyCommand):
     requires_project = False
     default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "<project_name> [project_dir]"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Create new project"
 
-    def _is_valid_name(self, project_name):
-        def _module_exists(module_name):
+    def _is_valid_name(self, project_name: str) -> bool:
+        def _module_exists(module_name: str) -> bool:
             spec = find_spec(module_name)
             return spec is not None and spec.loader is not None
 
@@ -53,7 +55,7 @@ class Command(ScrapyCommand):
             return True
         return False
 
-    def _copytree(self, src: Path, dst: Path):
+    def _copytree(self, src: Path, dst: Path) -> None:
         """
         Since the original function always creates the directory, to resolve
         the issue a new function had to be created. It's a simple copy and
@@ -84,7 +86,7 @@ class Command(ScrapyCommand):
         copystat(src, dst)
         _make_writable(dst)
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         if len(args) not in (1, 2):
             raise UsageError()
 
@@ -105,7 +107,9 @@ class Command(ScrapyCommand):
             return
 
         self._copytree(Path(self.templates_dir), project_dir.resolve())
-        move(project_dir / "module", project_dir / project_name)
+        # On 3.8 shutil.move doesn't fully support Path args, but it supports our use case
+        # See https://bugs.python.org/issue32689
+        move(project_dir / "module", project_dir / project_name)  # type: ignore[arg-type]
         for paths in TEMPLATES_TO_RENDER:
             tplfile = Path(
                 project_dir,

scrapy/commands/version.py

@@ -1,3 +1,6 @@
+import argparse
+from typing import List
+
 import scrapy
 from scrapy.commands import ScrapyCommand
 from scrapy.utils.versions import scrapy_components_versions
@@ -6,14 +9,14 @@ from scrapy.utils.versions import scrapy_components_versions
 class Command(ScrapyCommand):
     default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[-v]"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Print Scrapy version"
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+    def add_options(self, parser: argparse.ArgumentParser) -> None:
+        super().add_options(parser)
         parser.add_argument(
             "--verbose",
             "-v",
@@ -22,7 +25,7 @@ class Command(ScrapyCommand):
             help="also display twisted/python/platform info (useful for bug reports)",
         )
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         if opts.verbose:
             versions = scrapy_components_versions()
             width = max(len(n) for (n, _) in versions)

scrapy/commands/view.py

@@ -1,21 +1,28 @@
 import argparse
+import logging
 
 from scrapy.commands import fetch
+from scrapy.http import Response, TextResponse
 from scrapy.utils.response import open_in_browser
 
+logger = logging.getLogger(__name__)
+
 
 class Command(fetch.Command):
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Open URL in browser, as seen by Scrapy"
 
-    def long_desc(self):
+    def long_desc(self) -> str:
         return (
             "Fetch a URL using the Scrapy downloader and show its contents in a browser"
         )
 
-    def add_options(self, parser):
+    def add_options(self, parser: argparse.ArgumentParser) -> None:
         super().add_options(parser)
         parser.add_argument("--headers", help=argparse.SUPPRESS)
 
-    def _print_response(self, response, opts):
+    def _print_response(self, response: Response, opts: argparse.Namespace) -> None:
+        if not isinstance(response, TextResponse):
+            logger.error("Cannot view a non-text response.")
+            return
         open_in_browser(response)

scrapy/core/downloader/__init__.py

@@ -2,7 +2,7 @@ import random
 from collections import deque
 from datetime import datetime
 from time import time
-from typing import TYPE_CHECKING, Any, Deque, Dict, Set, Tuple, cast
+from typing import TYPE_CHECKING, Any, Deque, Dict, Optional, Set, Tuple, cast
 
 from twisted.internet import task
 from twisted.internet.defer import Deferred
@@ -24,10 +24,18 @@ if TYPE_CHECKING:
 class Slot:
     """Downloader slot"""
 
-    def __init__(self, concurrency: int, delay: float, randomize_delay: bool):
+    def __init__(
+        self,
+        concurrency: int,
+        delay: float,
+        randomize_delay: bool,
+        *,
+        throttle: Optional[bool] = None,
+    ):
         self.concurrency: int = concurrency
         self.delay: float = delay
         self.randomize_delay: bool = randomize_delay
+        self.throttle = throttle
 
         self.active: Set[Request] = set()
         self.queue: Deque[Tuple[Request, Deferred]] = deque()
@@ -40,7 +48,7 @@ class Slot:
 
     def download_delay(self) -> float:
         if self.randomize_delay:
-            return random.uniform(0.5 * self.delay, 1.5 * self.delay)
+            return random.uniform(0.5 * self.delay, 1.5 * self.delay)  # nosec
         return self.delay
 
     def close(self) -> None:
@@ -52,13 +60,15 @@ class Slot:
         return (
             f"{cls_name}(concurrency={self.concurrency!r}, "
             f"delay={self.delay:.2f}, "
-            f"randomize_delay={self.randomize_delay!r})"
+            f"randomize_delay={self.randomize_delay!r}, "
+            f"throttle={self.throttle!r})"
         )
 
     def __str__(self) -> str:
         return (
             f"<downloader.Slot concurrency={self.concurrency!r} "
             f"delay={self.delay:.2f} randomize_delay={self.randomize_delay!r} "
+            f"throttle={self.throttle!r} "
             f"len(active)={len(self.active)} len(queue)={len(self.queue)} "
             f"len(transferring)={len(self.transferring)} "
             f"lastseen={datetime.fromtimestamp(self.lastseen).isoformat()}>"
@@ -127,7 +137,8 @@ class Downloader:
                 slot_settings.get("delay", delay),
             )
             randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
-            new_slot = Slot(conc, delay, randomize_delay)
+            throttle = slot_settings.get("throttle", None)
+            new_slot = Slot(conc, delay, randomize_delay, throttle=throttle)
             self.slots[key] = new_slot
 
         return key, self.slots[key]

scrapy/core/downloader/handlers/__init__.py

@@ -21,9 +21,9 @@ logger = logging.getLogger(__name__)
 class DownloadHandlers:
     def __init__(self, crawler: "Crawler"):
         self._crawler: "Crawler" = crawler
-        self._schemes: Dict[
-            str, Union[str, Callable]
-        ] = {}  # stores acceptable schemes on instancing
+        self._schemes: Dict[str, Union[str, Callable]] = (
+            {}
+        )  # stores acceptable schemes on instancing
         self._handlers: Dict[str, Any] = {}  # stores instanced handlers for schemes
         self._notconfigured: Dict[str, str] = {}  # remembers failed handlers
         handlers: Dict[str, Union[str, Callable]] = without_none_values(

scrapy/core/downloader/handlers/http10.py

@@ -1,5 +1,6 @@
 """Download handlers for http and https schemes
 """
+
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.python import to_unicode
 

scrapy/core/downloader/middleware.py

@@ -3,6 +3,7 @@ Downloader Middleware manager
 
 See documentation in docs/topics/downloader-middleware.rst
 """
+
 from typing import Any, Callable, Generator, List, Union, cast
 
 from twisted.internet.defer import Deferred, inlineCallbacks

scrapy/core/engine.py

@@ -4,6 +4,7 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spider.
 For more information see docs/topics/architecture.rst
 
 """
+
 import logging
 from time import time
 from typing import (

scrapy/core/http2/stream.py

@@ -111,17 +111,17 @@ class Stream:
         # Metadata of an HTTP/2 connection stream
         # initialized when stream is instantiated
         self.metadata: Dict = {
-            "request_content_length": 0
-            if self._request.body is None
-            else len(self._request.body),
+            "request_content_length": (
+                0 if self._request.body is None else len(self._request.body)
+            ),
             # Flag to keep track whether the stream has initiated the request
             "request_sent": False,
             # Flag to track whether we have logged about exceeding download warnsize
             "reached_warnsize": False,
             # Each time we send a data frame, we will decrease value by the amount send.
-            "remaining_content_length": 0
-            if self._request.body is None
-            else len(self._request.body),
+            "remaining_content_length": (
+                0 if self._request.body is None else len(self._request.body)
+            ),
             # Flag to keep track whether client (self) have closed this stream
             "stream_closed_local": False,
             # Flag to keep track whether the server has closed the stream

scrapy/core/scraper.py

@@ -1,5 +1,6 @@
 """This module implements the Scraper component which parses responses and
 extracts information from them"""
+
 from __future__ import annotations
 
 import logging

scrapy/core/spidermw.py

@@ -3,6 +3,7 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
+
 import logging
 from inspect import isasyncgenfunction, iscoroutine
 from itertools import islice

scrapy/downloadermiddlewares/defaultheaders.py

@@ -3,6 +3,7 @@ DefaultHeaders downloader middleware
 
 See documentation in docs/topics/downloader-middleware.rst
 """
+
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, Iterable, Tuple, Union

scrapy/downloadermiddlewares/downloadtimeout.py

@@ -3,6 +3,7 @@ Download timeout middleware
 
 See documentation in docs/topics/downloader-middleware.rst
 """
+
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, Union

scrapy/downloadermiddlewares/httpcompression.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import warnings
+from itertools import chain
 from logging import getLogger
 from typing import TYPE_CHECKING, List, Optional, Union
 
@@ -28,7 +29,10 @@ logger = getLogger(__name__)
 ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]
 
 try:
-    import brotli  # noqa: F401
+    try:
+        import brotli  # noqa: F401
+    except ImportError:
+        import brotlicffi  # noqa: F401
 except ImportError:
     pass
 else:
@@ -102,18 +106,18 @@ class HttpCompressionMiddleware:
         if isinstance(response, Response):
             content_encoding = response.headers.getlist("Content-Encoding")
             if content_encoding:
-                encoding = content_encoding.pop()
                 max_size = request.meta.get("download_maxsize", self._max_size)
                 warn_size = request.meta.get("download_warnsize", self._warn_size)
                 try:
-                    decoded_body = self._decode(
-                        response.body, encoding.lower(), max_size
+                    decoded_body, content_encoding = self._handle_encoding(
+                        response.body, content_encoding, max_size
                     )
                 except _DecompressionMaxSizeExceeded:
                     raise IgnoreRequest(
                         f"Ignored response {response} because its body "
-                        f"({len(response.body)} B) exceeded DOWNLOAD_MAXSIZE "
-                        f"({max_size} B) during decompression."
+                        f"({len(response.body)} B compressed) exceeded "
+                        f"DOWNLOAD_MAXSIZE ({max_size} B) during "
+                        f"decompression."
                     )
                 if len(response.body) < warn_size <= len(decoded_body):
                     logger.warning(
@@ -121,6 +125,7 @@ class HttpCompressionMiddleware:
                         f"({len(decoded_body)} B) is larger than the "
                         f"download warning size ({warn_size} B)."
                     )
+                response.headers["Content-Encoding"] = content_encoding
                 if self.stats:
                     self.stats.inc_value(
                         "httpcompression/response_bytes",
@@ -133,7 +138,7 @@ class HttpCompressionMiddleware:
                 respcls = responsetypes.from_args(
                     headers=response.headers, url=response.url, body=decoded_body
                 )
-                kwargs = dict(cls=respcls, body=decoded_body)
+                kwargs = {"cls": respcls, "body": decoded_body}
                 if issubclass(respcls, TextResponse):
                     # force recalculating the encoding until we make sure the
                     # responsetypes guessing is reliable
@@ -144,8 +149,30 @@ class HttpCompressionMiddleware:
 
         return response
 
+    def _handle_encoding(self, body, content_encoding, max_size):
+        to_decode, to_keep = self._split_encodings(content_encoding)
+        for encoding in to_decode:
+            body = self._decode(body, encoding, max_size)
+        return body, to_keep
+
+    def _split_encodings(self, content_encoding):
+        to_keep = [
+            encoding.strip().lower()
+            for encoding in chain.from_iterable(
+                encodings.split(b",") for encodings in content_encoding
+            )
+        ]
+        to_decode = []
+        while to_keep:
+            encoding = to_keep.pop()
+            if encoding not in ACCEPTED_ENCODINGS:
+                to_keep.append(encoding)
+                return to_decode, to_keep
+            to_decode.append(encoding)
+        return to_decode, to_keep
+
     def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes:
-        if encoding == b"gzip" or encoding == b"x-gzip":
+        if encoding in {b"gzip", b"x-gzip"}:
             return gunzip(body, max_size=max_size)
         if encoding == b"deflate":
             return _inflate(body, max_size=max_size)

scrapy/downloadermiddlewares/retry.py

@@ -9,6 +9,7 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
 Failed pages are collected on the scraping process and rescheduled at the end,
 once the spider has finished crawling all regular (non failed) pages.
 """
+
 from __future__ import annotations
 
 import warnings

scrapy/exceptions.py

@@ -4,6 +4,7 @@ Scrapy core exceptions
 These exceptions are documented in docs/topics/exceptions.rst. Please don't add
 new exceptions here without documenting them there.
 """
+
 from typing import Any
 
 # Internal

scrapy/exporters.py

@@ -3,16 +3,18 @@ Item Exporters are used to export/serialize items into different formats.
 """
 
 import csv
-import io
 import marshal
-import pickle
+import pickle  # nosec
 import pprint
-from collections.abc import Mapping
-from xml.sax.saxutils import XMLGenerator
+from io import BytesIO, TextIOWrapper
+from json import JSONEncoder
+from typing import Any, Callable, Dict, Iterable, Mapping, Optional, Tuple, Union
+from xml.sax.saxutils import XMLGenerator  # nosec
+from xml.sax.xmlreader import AttributesImpl  # nosec
 
 from itemadapter import ItemAdapter, is_item
 
-from scrapy.item import Item
+from scrapy.item import Field, Item
 from scrapy.utils.python import is_listlike, to_bytes, to_unicode
 from scrapy.utils.serialize import ScrapyJSONEncoder
 
@@ -29,36 +31,42 @@ __all__ = [
 
 
 class BaseItemExporter:
-    def __init__(self, *, dont_fail=False, **kwargs):
-        self._kwargs = kwargs
+    def __init__(self, *, dont_fail: bool = False, **kwargs: Any):
+        self._kwargs: Dict[str, Any] = kwargs
         self._configure(kwargs, dont_fail=dont_fail)
 
-    def _configure(self, options, dont_fail=False):
+    def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None:
         """Configure the exporter by popping options from the ``options`` dict.
         If dont_fail is set, it won't raise an exception on unexpected options
         (useful for using with keyword arguments in subclasses ``__init__`` methods)
         """
-        self.encoding = options.pop("encoding", None)
-        self.fields_to_export = options.pop("fields_to_export", None)
-        self.export_empty_fields = options.pop("export_empty_fields", False)
-        self.indent = options.pop("indent", None)
+        self.encoding: Optional[str] = options.pop("encoding", None)
+        self.fields_to_export: Union[Mapping[str, str], Iterable[str], None] = (
+            options.pop("fields_to_export", None)
+        )
+        self.export_empty_fields: bool = options.pop("export_empty_fields", False)
+        self.indent: Optional[int] = options.pop("indent", None)
         if not dont_fail and options:
             raise TypeError(f"Unexpected options: {', '.join(options.keys())}")
 
-    def export_item(self, item):
+    def export_item(self, item: Any) -> None:
         raise NotImplementedError
 
-    def serialize_field(self, field, name, value):
-        serializer = field.get("serializer", lambda x: x)
+    def serialize_field(
+        self, field: Union[Mapping[str, Any], Field], name: str, value: Any
+    ) -> Any:
+        serializer: Callable[[Any], Any] = field.get("serializer", lambda x: x)
         return serializer(value)
 
-    def start_exporting(self):
+    def start_exporting(self) -> None:
         pass
 
-    def finish_exporting(self):
+    def finish_exporting(self) -> None:
         pass
 
-    def _get_serialized_fields(self, item, default_value=None, include_empty=None):
+    def _get_serialized_fields(
+        self, item: Any, default_value: Any = None, include_empty: Optional[bool] = None
+    ) -> Iterable[Tuple[str, Any]]:
         """Return the fields to export as an iterable of tuples
         (name, serialized_value)
         """
@@ -100,22 +108,22 @@ class BaseItemExporter:
 
 
 class JsonLinesItemExporter(BaseItemExporter):
-    def __init__(self, file, **kwargs):
+    def __init__(self, file: BytesIO, **kwargs: Any):
         super().__init__(dont_fail=True, **kwargs)
-        self.file = file
+        self.file: BytesIO = file
         self._kwargs.setdefault("ensure_ascii", not self.encoding)
-        self.encoder = ScrapyJSONEncoder(**self._kwargs)
+        self.encoder: JSONEncoder = ScrapyJSONEncoder(**self._kwargs)
 
-    def export_item(self, item):
+    def export_item(self, item: Any) -> None:
         itemdict = dict(self._get_serialized_fields(item))
         data = self.encoder.encode(itemdict) + "\n"
         self.file.write(to_bytes(data, self.encoding))
 
 
 class JsonItemExporter(BaseItemExporter):
-    def __init__(self, file, **kwargs):
+    def __init__(self, file: BytesIO, **kwargs: Any):
         super().__init__(dont_fail=True, **kwargs)
-        self.file = file
+        self.file: BytesIO = file
         # there is a small difference between the behaviour or JsonItemExporter.indent
         # and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent
         # the addition of newlines everywhere
@@ -127,26 +135,26 @@ class JsonItemExporter(BaseItemExporter):
         self.encoder = ScrapyJSONEncoder(**self._kwargs)
         self.first_item = True
 
-    def _beautify_newline(self):
+    def _beautify_newline(self) -> None:
         if self.indent is not None:
             self.file.write(b"\n")
 
-    def _add_comma_after_first(self):
+    def _add_comma_after_first(self) -> None:
         if self.first_item:
             self.first_item = False
         else:
             self.file.write(b",")
             self._beautify_newline()
 
-    def start_exporting(self):
+    def start_exporting(self) -> None:
         self.file.write(b"[")
         self._beautify_newline()
 
-    def finish_exporting(self):
+    def finish_exporting(self) -> None:
         self._beautify_newline()
         self.file.write(b"]")
 
-    def export_item(self, item):
+    def export_item(self, item: Any) -> None:
         itemdict = dict(self._get_serialized_fields(item))
         data = to_bytes(self.encoder.encode(itemdict), self.encoding)
         self._add_comma_after_first()
@@ -154,7 +162,7 @@ class JsonItemExporter(BaseItemExporter):
 
 
 class XmlItemExporter(BaseItemExporter):
-    def __init__(self, file, **kwargs):
+    def __init__(self, file: BytesIO, **kwargs: Any):
         self.item_element = kwargs.pop("item_element", "item")
         self.root_element = kwargs.pop("root_element", "items")
         super().__init__(**kwargs)
@@ -162,22 +170,22 @@ class XmlItemExporter(BaseItemExporter):
             self.encoding = "utf-8"
         self.xg = XMLGenerator(file, encoding=self.encoding)
 
-    def _beautify_newline(self, new_item=False):
+    def _beautify_newline(self, new_item: bool = False) -> None:
         if self.indent is not None and (self.indent > 0 or new_item):
             self.xg.characters("\n")
 
-    def _beautify_indent(self, depth=1):
+    def _beautify_indent(self, depth: int = 1) -> None:
         if self.indent:
             self.xg.characters(" " * self.indent * depth)
 
-    def start_exporting(self):
+    def start_exporting(self) -> None:
         self.xg.startDocument()
-        self.xg.startElement(self.root_element, {})
+        self.xg.startElement(self.root_element, AttributesImpl({}))
         self._beautify_newline(new_item=True)
 
-    def export_item(self, item):
+    def export_item(self, item: Any) -> None:
         self._beautify_indent(depth=1)
-        self.xg.startElement(self.item_element, {})
+        self.xg.startElement(self.item_element, AttributesImpl({}))
         self._beautify_newline()
         for name, value in self._get_serialized_fields(item, default_value=""):
             self._export_xml_field(name, value, depth=2)
@@ -185,13 +193,13 @@ class XmlItemExporter(BaseItemExporter):
         self.xg.endElement(self.item_element)
         self._beautify_newline(new_item=True)
 
-    def finish_exporting(self):
+    def finish_exporting(self) -> None:
         self.xg.endElement(self.root_element)
         self.xg.endDocument()
 
-    def _export_xml_field(self, name, serialized_value, depth):
+    def _export_xml_field(self, name: str, serialized_value: Any, depth: int) -> None:
         self._beautify_indent(depth=depth)
-        self.xg.startElement(name, {})
+        self.xg.startElement(name, AttributesImpl({}))
         if hasattr(serialized_value, "items"):
             self._beautify_newline()
             for subname, value in serialized_value.items():
@@ -213,17 +221,17 @@ class XmlItemExporter(BaseItemExporter):
 class CsvItemExporter(BaseItemExporter):
     def __init__(
         self,
-        file,
-        include_headers_line=True,
-        join_multivalued=",",
-        errors=None,
-        **kwargs,
+        file: BytesIO,
+        include_headers_line: bool = True,
+        join_multivalued: str = ",",
+        errors: Optional[str] = None,
+        **kwargs: Any,
     ):
         super().__init__(dont_fail=True, **kwargs)
         if not self.encoding:
             self.encoding = "utf-8"
         self.include_headers_line = include_headers_line
-        self.stream = io.TextIOWrapper(
+        self.stream = TextIOWrapper(
             file,
             line_buffering=False,
             write_through=True,
@@ -235,11 +243,13 @@ class CsvItemExporter(BaseItemExporter):
         self._headers_not_written = True
         self._join_multivalued = join_multivalued
 
-    def serialize_field(self, field, name, value):
-        serializer = field.get("serializer", self._join_if_needed)
+    def serialize_field(
+        self, field: Union[Mapping[str, Any], Field], name: str, value: Any
+    ) -> Any:
+        serializer: Callable[[Any], Any] = field.get("serializer", self._join_if_needed)
         return serializer(value)
 
-    def _join_if_needed(self, value):
+    def _join_if_needed(self, value: Any) -> Any:
         if isinstance(value, (list, tuple)):
             try:
                 return self._join_multivalued.join(value)
@@ -247,7 +257,7 @@ class CsvItemExporter(BaseItemExporter):
                 pass
         return value
 
-    def export_item(self, item):
+    def export_item(self, item: Any) -> None:
         if self._headers_not_written:
             self._headers_not_written = False
             self._write_headers_and_set_fields_to_export(item)
@@ -256,36 +266,38 @@ class CsvItemExporter(BaseItemExporter):
         values = list(self._build_row(x for _, x in fields))
         self.csv_writer.writerow(values)
 
-    def finish_exporting(self):
+    def finish_exporting(self) -> None:
         self.stream.detach()  # Avoid closing the wrapped file.
 
-    def _build_row(self, values):
+    def _build_row(self, values: Iterable[Any]) -> Iterable[Any]:
         for s in values:
             try:
                 yield to_unicode(s, self.encoding)
             except TypeError:
                 yield s
 
-    def _write_headers_and_set_fields_to_export(self, item):
+    def _write_headers_and_set_fields_to_export(self, item: Any) -> None:
         if self.include_headers_line:
             if not self.fields_to_export:
                 # use declared field names, or keys if the item is a dict
                 self.fields_to_export = ItemAdapter(item).field_names()
+            fields: Iterable[str]
             if isinstance(self.fields_to_export, Mapping):
                 fields = self.fields_to_export.values()
             else:
+                assert self.fields_to_export
                 fields = self.fields_to_export
             row = list(self._build_row(fields))
             self.csv_writer.writerow(row)
 
 
 class PickleItemExporter(BaseItemExporter):
-    def __init__(self, file, protocol=4, **kwargs):
+    def __init__(self, file: BytesIO, protocol: int = 4, **kwargs: Any):
         super().__init__(**kwargs)
-        self.file = file
-        self.protocol = protocol
+        self.file: BytesIO = file
+        self.protocol: int = protocol
 
-    def export_item(self, item):
+    def export_item(self, item: Any) -> None:
         d = dict(self._get_serialized_fields(item))
         pickle.dump(d, self.file, self.protocol)
 
@@ -299,20 +311,20 @@ class MarshalItemExporter(BaseItemExporter):
                  opened in binary mode, a :class:`~io.BytesIO` object, etc)
     """
 
-    def __init__(self, file, **kwargs):
+    def __init__(self, file: BytesIO, **kwargs: Any):
         super().__init__(**kwargs)
-        self.file = file
+        self.file: BytesIO = file
 
-    def export_item(self, item):
+    def export_item(self, item: Any) -> None:
         marshal.dump(dict(self._get_serialized_fields(item)), self.file)
 
 
 class PprintItemExporter(BaseItemExporter):
-    def __init__(self, file, **kwargs):
+    def __init__(self, file: BytesIO, **kwargs: Any):
         super().__init__(**kwargs)
-        self.file = file
+        self.file: BytesIO = file
 
-    def export_item(self, item):
+    def export_item(self, item: Any) -> None:
         itemdict = dict(self._get_serialized_fields(item))
         self.file.write(to_bytes(pprint.pformat(itemdict) + "\n"))
 
@@ -327,16 +339,20 @@ class PythonItemExporter(BaseItemExporter):
     .. _msgpack: https://pypi.org/project/msgpack/
     """
 
-    def _configure(self, options, dont_fail=False):
+    def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None:
         super()._configure(options, dont_fail)
         if not self.encoding:
             self.encoding = "utf-8"
 
-    def serialize_field(self, field, name, value):
-        serializer = field.get("serializer", self._serialize_value)
+    def serialize_field(
+        self, field: Union[Mapping[str, Any], Field], name: str, value: Any
+    ) -> Any:
+        serializer: Callable[[Any], Any] = field.get(
+            "serializer", self._serialize_value
+        )
         return serializer(value)
 
-    def _serialize_value(self, value):
+    def _serialize_value(self, value: Any) -> Any:
         if isinstance(value, Item):
             return self.export_item(value)
         if is_item(value):
@@ -347,10 +363,10 @@ class PythonItemExporter(BaseItemExporter):
             return to_unicode(value, encoding=self.encoding)
         return value
 
-    def _serialize_item(self, item):
+    def _serialize_item(self, item: Any) -> Iterable[Tuple[Union[str, bytes], Any]]:
         for key, value in ItemAdapter(item).items():
             yield key, self._serialize_value(value)
 
-    def export_item(self, item):
-        result = dict(self._get_serialized_fields(item))
+    def export_item(self, item: Any) -> Dict[Union[str, bytes], Any]:  # type: ignore[override]
+        result: Dict[Union[str, bytes], Any] = dict(self._get_serialized_fields(item))
         return result

scrapy/extension.py

@@ -3,6 +3,7 @@ The Extension Manager
 
 See documentation in docs/topics/extensions.rst
 """
+
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.conf import build_component_list
 

scrapy/extensions/closespider.py

@@ -4,20 +4,31 @@ conditions are met.
 See documentation in docs/topics/extensions.rst
 """
 
+from __future__ import annotations
+
 import logging
 from collections import defaultdict
+from typing import TYPE_CHECKING, Any, DefaultDict, Dict
 
-from scrapy import signals
+from twisted.python.failure import Failure
+
+from scrapy import Request, Spider, signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
+from scrapy.http import Response
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 
 logger = logging.getLogger(__name__)
 
 
 class CloseSpider:
-    def __init__(self, crawler):
-        self.crawler = crawler
+    def __init__(self, crawler: Crawler):
+        self.crawler: Crawler = crawler
 
-        self.close_on = {
+        self.close_on: Dict[str, Any] = {
             "timeout": crawler.settings.getfloat("CLOSESPIDER_TIMEOUT"),
             "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"),
             "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"),
@@ -28,7 +39,7 @@ class CloseSpider:
         if not any(self.close_on.values()):
             raise NotConfigured
 
-        self.counter = defaultdict(int)
+        self.counter: DefaultDict[str, int] = defaultdict(int)
 
         if self.close_on.get("errorcount"):
             crawler.signals.connect(self.error_count, signal=signals.spider_error)
@@ -39,8 +50,8 @@ class CloseSpider:
         if self.close_on.get("itemcount"):
             crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
         if self.close_on.get("timeout_no_item"):
-            self.timeout_no_item = self.close_on["timeout_no_item"]
-            self.items_in_period = 0
+            self.timeout_no_item: int = self.close_on["timeout_no_item"]
+            self.items_in_period: int = 0
             crawler.signals.connect(
                 self.spider_opened_no_item, signal=signals.spider_opened
             )
@@ -50,22 +61,25 @@ class CloseSpider:
         crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler)
 
-    def error_count(self, failure, response, spider):
+    def error_count(self, failure: Failure, response: Response, spider: Spider) -> None:
         self.counter["errorcount"] += 1
         if self.counter["errorcount"] == self.close_on["errorcount"]:
+            assert self.crawler.engine
             self.crawler.engine.close_spider(spider, "closespider_errorcount")
 
-    def page_count(self, response, request, spider):
+    def page_count(self, response: Response, request: Request, spider: Spider) -> None:
         self.counter["pagecount"] += 1
         if self.counter["pagecount"] == self.close_on["pagecount"]:
+            assert self.crawler.engine
             self.crawler.engine.close_spider(spider, "closespider_pagecount")
 
-    def spider_opened(self, spider):
+    def spider_opened(self, spider: Spider) -> None:
         from twisted.internet import reactor
 
+        assert self.crawler.engine
         self.task = reactor.callLater(
             self.close_on["timeout"],
             self.crawler.engine.close_spider,
@@ -73,21 +87,22 @@ class CloseSpider:
             reason="closespider_timeout",
         )
 
-    def item_scraped(self, item, spider):
+    def item_scraped(self, item: Any, spider: Spider) -> None:
         self.counter["itemcount"] += 1
         if self.counter["itemcount"] == self.close_on["itemcount"]:
+            assert self.crawler.engine
             self.crawler.engine.close_spider(spider, "closespider_itemcount")
 
-    def spider_closed(self, spider):
-        task = getattr(self, "task", False)
+    def spider_closed(self, spider: Spider) -> None:
+        task = getattr(self, "task", None)
         if task and task.active():
             task.cancel()
 
-        task_no_item = getattr(self, "task_no_item", False)
+        task_no_item = getattr(self, "task_no_item", None)
         if task_no_item and task_no_item.running:
             task_no_item.stop()
 
-    def spider_opened_no_item(self, spider):
+    def spider_opened_no_item(self, spider: Spider) -> None:
         from twisted.internet import task
 
         self.task_no_item = task.LoopingCall(self._count_items_produced, spider)
@@ -98,10 +113,10 @@ class CloseSpider:
             f"{self.timeout_no_item} seconds."
         )
 
-    def item_scraped_no_item(self, item, spider):
+    def item_scraped_no_item(self, item: Any, spider: Spider) -> None:
         self.items_in_period += 1
 
-    def _count_items_produced(self, spider):
+    def _count_items_produced(self, spider: Spider) -> None:
         if self.items_in_period >= 1:
             self.items_in_period = 0
         else:
@@ -109,4 +124,5 @@ class CloseSpider:
                 f"Closing spider since no items were produced in the last "
                 f"{self.timeout_no_item} seconds."
             )
+            assert self.crawler.engine
             self.crawler.engine.close_spider(spider, "closespider_timeout_no_item")

scrapy/extensions/corestats.py

@@ -1,18 +1,29 @@
 """
 Extension for collecting core stats like items scraped and start/finish times
 """
-from datetime import datetime, timezone
 
-from scrapy import signals
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any, Optional
+
+from scrapy import Spider, signals
+from scrapy.crawler import Crawler
+from scrapy.statscollectors import StatsCollector
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 
 
 class CoreStats:
-    def __init__(self, stats):
-        self.stats = stats
-        self.start_time = None
+    def __init__(self, stats: StatsCollector):
+        self.stats: StatsCollector = stats
+        self.start_time: Optional[datetime] = None
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
+        assert crawler.stats
         o = cls(crawler.stats)
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
@@ -21,11 +32,12 @@ class CoreStats:
         crawler.signals.connect(o.response_received, signal=signals.response_received)
         return o
 
-    def spider_opened(self, spider):
+    def spider_opened(self, spider: Spider) -> None:
         self.start_time = datetime.now(tz=timezone.utc)
         self.stats.set_value("start_time", self.start_time, spider=spider)
 
-    def spider_closed(self, spider, reason):
+    def spider_closed(self, spider: Spider, reason: str) -> None:
+        assert self.start_time is not None
         finish_time = datetime.now(tz=timezone.utc)
         elapsed_time = finish_time - self.start_time
         elapsed_time_seconds = elapsed_time.total_seconds()
@@ -35,13 +47,13 @@ class CoreStats:
         self.stats.set_value("finish_time", finish_time, spider=spider)
         self.stats.set_value("finish_reason", reason, spider=spider)
 
-    def item_scraped(self, item, spider):
+    def item_scraped(self, item: Any, spider: Spider) -> None:
         self.stats.inc_value("item_scraped_count", spider=spider)
 
-    def response_received(self, spider):
+    def response_received(self, spider: Spider) -> None:
         self.stats.inc_value("response_received_count", spider=spider)
 
-    def item_dropped(self, item, spider, exception):
+    def item_dropped(self, item: Any, spider: Spider, exception: BaseException) -> None:
         reason = exception.__class__.__name__
         self.stats.inc_value("item_dropped_count", spider=spider)
         self.stats.inc_value(f"item_dropped_reasons_count/{reason}", spider=spider)

scrapy/extensions/debug.py

@@ -4,22 +4,31 @@ Extensions for debugging Scrapy
 See documentation in docs/topics/extensions.rst
 """
 
+from __future__ import annotations
+
 import logging
 import signal
 import sys
 import threading
 import traceback
 from pdb import Pdb
+from types import FrameType
+from typing import TYPE_CHECKING, Optional
 
+from scrapy.crawler import Crawler
 from scrapy.utils.engine import format_engine_status
 from scrapy.utils.trackref import format_live_refs
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 logger = logging.getLogger(__name__)
 
 
 class StackTraceDump:
-    def __init__(self, crawler=None):
-        self.crawler = crawler
+    def __init__(self, crawler: Crawler):
+        self.crawler: Crawler = crawler
         try:
             signal.signal(signal.SIGUSR2, self.dump_stacktrace)
             signal.signal(signal.SIGQUIT, self.dump_stacktrace)
@@ -28,10 +37,11 @@ class StackTraceDump:
             pass
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler)
 
-    def dump_stacktrace(self, signum, frame):
+    def dump_stacktrace(self, signum: int, frame: Optional[FrameType]) -> None:
+        assert self.crawler.engine
         log_args = {
             "stackdumps": self._thread_stacks(),
             "enginestatus": format_engine_status(self.crawler.engine),
@@ -44,7 +54,7 @@ class StackTraceDump:
             extra={"crawler": self.crawler},
         )
 
-    def _thread_stacks(self):
+    def _thread_stacks(self) -> str:
         id2name = dict((th.ident, th.name) for th in threading.enumerate())
         dumps = ""
         for id_, frame in sys._current_frames().items():
@@ -55,12 +65,13 @@ class StackTraceDump:
 
 
 class Debugger:
-    def __init__(self):
+    def __init__(self) -> None:
         try:
             signal.signal(signal.SIGUSR2, self._enter_debugger)
         except AttributeError:
             # win32 platforms don't support SIGUSR signals
             pass
 
-    def _enter_debugger(self, signum, frame):
+    def _enter_debugger(self, signum: int, frame: Optional[FrameType]) -> None:
+        assert frame
         Pdb().set_trace(frame.f_back)

scrapy/extensions/feedexport.py

@@ -11,7 +11,7 @@ import warnings
 from datetime import datetime, timezone
 from pathlib import Path, PureWindowsPath
 from tempfile import NamedTemporaryFile
-from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union
+from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Type, Union
 from urllib.parse import unquote, urlparse
 
 from twisted.internet import defer, threads
@@ -21,6 +21,7 @@ from zope.interface import Interface, implementer
 
 from scrapy import Spider, signals
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
+from scrapy.exporters import BaseItemExporter
 from scrapy.extensions.postprocessing import PostProcessingManager
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.conf import feed_complete_default_values_from_settings
@@ -324,12 +325,12 @@ class FeedSlot:
         filter,
         feed_options,
         spider,
-        exporters,
+        exporters: Dict[str, Type[BaseItemExporter]],
         settings,
         crawler,
     ):
         self.file = None
-        self.exporter = None
+        self.exporter: Optional[BaseItemExporter] = None
         self.storage = storage
         # feed params
         self.batch_id = batch_id
@@ -341,7 +342,7 @@ class FeedSlot:
         # exporter params
         self.feed_options = feed_options
         self.spider = spider
-        self.exporters = exporters
+        self.exporters: Dict[str, Type[BaseItemExporter]] = exporters
         self.settings = settings
         self.crawler = crawler
         # flags
@@ -373,7 +374,7 @@ class FeedSlot:
     def _get_instance(self, objcls, *args, **kwargs):
         return build_from_crawler(objcls, self.crawler, *args, **kwargs)
 
-    def _get_exporter(self, file, format, *args, **kwargs):
+    def _get_exporter(self, file, format, *args, **kwargs) -> BaseItemExporter:
         return self._get_instance(self.exporters[format], file, *args, **kwargs)
 
     def finish_exporting(self):

scrapy/extensions/httpcache.py

@@ -1,6 +1,6 @@
 import gzip
 import logging
-import pickle
+import pickle  # nosec
 from email.utils import mktime_tz, parsedate_tz
 from importlib import import_module
 from pathlib import Path
@@ -274,7 +274,7 @@ class DbmCacheStorage:
         if 0 < self.expiration_secs < time() - float(ts):
             return  # expired
 
-        return pickle.loads(db[f"{key}_data"])
+        return pickle.loads(db[f"{key}_data"])  # nosec
 
 
 class FilesystemCacheStorage:
@@ -352,7 +352,7 @@ class FilesystemCacheStorage:
         if 0 < self.expiration_secs < time() - mtime:
             return  # expired
         with self._open(metapath, "rb") as f:
-            return pickle.load(f)
+            return pickle.load(f)  # nosec
 
 
 def parse_cachecontrol(header):

scrapy/extensions/logstats.py

@@ -1,9 +1,18 @@
+from __future__ import annotations
+
 import logging
+from typing import TYPE_CHECKING, Optional, Tuple, Union
 
 from twisted.internet import task
 
-from scrapy import signals
+from scrapy import Spider, signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
+from scrapy.statscollectors import StatsCollector
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 
 logger = logging.getLogger(__name__)
 
@@ -14,30 +23,31 @@ class LogStats:
     * IPM - Items per Minute
     """
 
-    def __init__(self, stats, interval=60.0):
-        self.stats = stats
-        self.interval = interval
-        self.multiplier = 60.0 / self.interval
-        self.task = None
+    def __init__(self, stats: StatsCollector, interval: float = 60.0):
+        self.stats: StatsCollector = stats
+        self.interval: float = interval
+        self.multiplier: float = 60.0 / self.interval
+        self.task: Optional[task.LoopingCall] = None
 
     @classmethod
-    def from_crawler(cls, crawler):
-        interval = crawler.settings.getfloat("LOGSTATS_INTERVAL")
+    def from_crawler(cls, crawler: Crawler) -> Self:
+        interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL")
         if not interval:
             raise NotConfigured
+        assert crawler.stats
         o = cls(crawler.stats, interval)
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
         return o
 
-    def spider_opened(self, spider):
-        self.pagesprev = 0
-        self.itemsprev = 0
+    def spider_opened(self, spider: Spider) -> None:
+        self.pagesprev: int = 0
+        self.itemsprev: int = 0
 
         self.task = task.LoopingCall(self.log, spider)
         self.task.start(self.interval)
 
-    def log(self, spider):
+    def log(self, spider: Spider) -> None:
         self.calculate_stats()
 
         msg = (
@@ -52,14 +62,14 @@ class LogStats:
         }
         logger.info(msg, log_args, extra={"spider": spider})
 
-    def calculate_stats(self):
-        self.items = self.stats.get_value("item_scraped_count", 0)
-        self.pages = self.stats.get_value("response_received_count", 0)
-        self.irate = (self.items - self.itemsprev) * self.multiplier
-        self.prate = (self.pages - self.pagesprev) * self.multiplier
+    def calculate_stats(self) -> None:
+        self.items: int = self.stats.get_value("item_scraped_count", 0)
+        self.pages: int = self.stats.get_value("response_received_count", 0)
+        self.irate: float = (self.items - self.itemsprev) * self.multiplier
+        self.prate: float = (self.pages - self.pagesprev) * self.multiplier
         self.pagesprev, self.itemsprev = self.pages, self.items
 
-    def spider_closed(self, spider, reason):
+    def spider_closed(self, spider: Spider, reason: str) -> None:
         if self.task and self.task.running:
             self.task.stop()
 
@@ -67,7 +77,9 @@ class LogStats:
         self.stats.set_value("responses_per_minute", rpm_final)
         self.stats.set_value("items_per_minute", ipm_final)
 
-    def calculate_final_stats(self, spider):
+    def calculate_final_stats(
+        self, spider: Spider
+    ) -> Union[Tuple[None, None], Tuple[float, float]]:
         start_time = self.stats.get_value("start_time")
         finished_time = self.stats.get_value("finished_time")
 

scrapy/extensions/memdebug.py

@@ -4,26 +4,36 @@ MemoryDebugger extension
 See documentation in docs/topics/extensions.rst
 """
 
-import gc
+from __future__ import annotations
 
-from scrapy import signals
+import gc
+from typing import TYPE_CHECKING
+
+from scrapy import Spider, signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
+from scrapy.statscollectors import StatsCollector
 from scrapy.utils.trackref import live_refs
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 
 class MemoryDebugger:
-    def __init__(self, stats):
-        self.stats = stats
+    def __init__(self, stats: StatsCollector):
+        self.stats: StatsCollector = stats
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
         if not crawler.settings.getbool("MEMDEBUG_ENABLED"):
             raise NotConfigured
+        assert crawler.stats
         o = cls(crawler.stats)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
         return o
 
-    def spider_closed(self, spider, reason):
+    def spider_closed(self, spider: Spider, reason: str) -> None:
         gc.collect()
         self.stats.set_value(
             "memdebug/gc_garbage_count", len(gc.garbage), spider=spider

scrapy/extensions/memusage.py

@@ -3,24 +3,33 @@ MemoryUsage extension
 
 See documentation in docs/topics/extensions.rst
 """
+
+from __future__ import annotations
+
 import logging
 import socket
 import sys
 from importlib import import_module
 from pprint import pformat
+from typing import TYPE_CHECKING, List
 
 from twisted.internet import task
 
 from scrapy import signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.mail import MailSender
 from scrapy.utils.engine import get_engine_status
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 logger = logging.getLogger(__name__)
 
 
 class MemoryUsage:
-    def __init__(self, crawler):
+    def __init__(self, crawler: Crawler):
         if not crawler.settings.getbool("MEMUSAGE_ENABLED"):
             raise NotConfigured
         try:
@@ -29,32 +38,33 @@ class MemoryUsage:
         except ImportError:
             raise NotConfigured
 
-        self.crawler = crawler
-        self.warned = False
-        self.notify_mails = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL")
-        self.limit = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024
-        self.warning = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024
-        self.check_interval = crawler.settings.getfloat(
+        self.crawler: Crawler = crawler
+        self.warned: bool = False
+        self.notify_mails: List[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL")
+        self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024
+        self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024
+        self.check_interval: float = crawler.settings.getfloat(
             "MEMUSAGE_CHECK_INTERVAL_SECONDS"
         )
-        self.mail = MailSender.from_settings(crawler.settings)
+        self.mail: MailSender = MailSender.from_settings(crawler.settings)
         crawler.signals.connect(self.engine_started, signal=signals.engine_started)
         crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped)
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler)
 
-    def get_virtual_size(self):
-        size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss
+    def get_virtual_size(self) -> int:
+        size: int = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss
         if sys.platform != "darwin":
             # on macOS ru_maxrss is in bytes, on Linux it is in KB
             size *= 1024
         return size
 
-    def engine_started(self):
+    def engine_started(self) -> None:
+        assert self.crawler.stats
         self.crawler.stats.set_value("memusage/startup", self.get_virtual_size())
-        self.tasks = []
+        self.tasks: List[task.LoopingCall] = []
         tsk = task.LoopingCall(self.update)
         self.tasks.append(tsk)
         tsk.start(self.check_interval, now=True)
@@ -67,15 +77,18 @@ class MemoryUsage:
             self.tasks.append(tsk)
             tsk.start(self.check_interval, now=True)
 
-    def engine_stopped(self):
+    def engine_stopped(self) -> None:
         for tsk in self.tasks:
             if tsk.running:
                 tsk.stop()
 
-    def update(self):
+    def update(self) -> None:
+        assert self.crawler.stats
         self.crawler.stats.max_value("memusage/max", self.get_virtual_size())
 
-    def _check_limit(self):
+    def _check_limit(self) -> None:
+        assert self.crawler.engine
+        assert self.crawler.stats
         peak_mem_usage = self.get_virtual_size()
         if peak_mem_usage > self.limit:
             self.crawler.stats.set_value("memusage/limit_reached", 1)
@@ -105,9 +118,10 @@ class MemoryUsage:
                 {"virtualsize": peak_mem_usage / 1024 / 1024},
             )
 
-    def _check_warning(self):
+    def _check_warning(self) -> None:
         if self.warned:  # warn only once
             return
+        assert self.crawler.stats
         if self.get_virtual_size() > self.warning:
             self.crawler.stats.set_value("memusage/warning_reached", 1)
             mem = self.warning / 1024 / 1024
@@ -125,8 +139,10 @@ class MemoryUsage:
                 self.crawler.stats.set_value("memusage/warning_notified", 1)
             self.warned = True
 
-    def _send_report(self, rcpts, subject):
+    def _send_report(self, rcpts: List[str], subject: str) -> None:
         """send notification mail with some additional useful info"""
+        assert self.crawler.engine
+        assert self.crawler.stats
         stats = self.crawler.stats
         s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"
         s += f"Maximum memory usage          : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n"

scrapy/extensions/periodic_log.py

@@ -1,12 +1,22 @@
+from __future__ import annotations
+
 import logging
 from datetime import datetime, timezone
+from json import JSONEncoder
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 from twisted.internet import task
 
-from scrapy import signals
+from scrapy import Spider, signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
+from scrapy.statscollectors import StatsCollector
 from scrapy.utils.serialize import ScrapyJSONEncoder
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 logger = logging.getLogger(__name__)
 
 
@@ -15,32 +25,34 @@ class PeriodicLog:
 
     def __init__(
         self,
-        stats,
-        interval=60.0,
-        ext_stats={},
-        ext_delta={},
-        ext_timing_enabled=False,
+        stats: StatsCollector,
+        interval: float = 60.0,
+        ext_stats: Dict[str, Any] = {},
+        ext_delta: Dict[str, Any] = {},
+        ext_timing_enabled: bool = False,
     ):
-        self.stats = stats
-        self.interval = interval
-        self.multiplier = 60.0 / self.interval
-        self.task = None
-        self.encoder = ScrapyJSONEncoder(sort_keys=True, indent=4)
-        self.ext_stats_enabled = bool(ext_stats)
-        self.ext_stats_include = ext_stats.get("include", [])
-        self.ext_stats_exclude = ext_stats.get("exclude", [])
-        self.ext_delta_enabled = bool(ext_delta)
-        self.ext_delta_include = ext_delta.get("include", [])
-        self.ext_delta_exclude = ext_delta.get("exclude", [])
-        self.ext_timing_enabled = ext_timing_enabled
+        self.stats: StatsCollector = stats
+        self.interval: float = interval
+        self.multiplier: float = 60.0 / self.interval
+        self.task: Optional[task.LoopingCall] = None
+        self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4)
+        self.ext_stats_enabled: bool = bool(ext_stats)
+        self.ext_stats_include: List[str] = ext_stats.get("include", [])
+        self.ext_stats_exclude: List[str] = ext_stats.get("exclude", [])
+        self.ext_delta_enabled: bool = bool(ext_delta)
+        self.ext_delta_include: List[str] = ext_delta.get("include", [])
+        self.ext_delta_exclude: List[str] = ext_delta.get("exclude", [])
+        self.ext_timing_enabled: bool = ext_timing_enabled
 
     @classmethod
-    def from_crawler(cls, crawler):
-        interval = crawler.settings.getfloat("LOGSTATS_INTERVAL")
+    def from_crawler(cls, crawler: Crawler) -> Self:
+        interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL")
         if not interval:
             raise NotConfigured
         try:
-            ext_stats = crawler.settings.getdict("PERIODIC_LOG_STATS")
+            ext_stats: Optional[Dict[str, Any]] = crawler.settings.getdict(
+                "PERIODIC_LOG_STATS"
+            )
         except (TypeError, ValueError):
             ext_stats = (
                 {"enabled": True}
@@ -48,7 +60,9 @@ class PeriodicLog:
                 else None
             )
         try:
-            ext_delta = crawler.settings.getdict("PERIODIC_LOG_DELTA")
+            ext_delta: Optional[Dict[str, Any]] = crawler.settings.getdict(
+                "PERIODIC_LOG_DELTA"
+            )
         except (TypeError, ValueError):
             ext_delta = (
                 {"enabled": True}
@@ -56,11 +70,14 @@ class PeriodicLog:
                 else None
             )
 
-        ext_timing_enabled = crawler.settings.getbool(
+        ext_timing_enabled: bool = crawler.settings.getbool(
             "PERIODIC_LOG_TIMING_ENABLED", False
         )
         if not (ext_stats or ext_delta or ext_timing_enabled):
             raise NotConfigured
+        assert crawler.stats
+        assert ext_stats is not None
+        assert ext_delta is not None
         o = cls(
             crawler.stats,
             interval,
@@ -72,16 +89,16 @@ class PeriodicLog:
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
         return o
 
-    def spider_opened(self, spider):
-        self.time_prev = datetime.now(tz=timezone.utc)
-        self.delta_prev = {}
-        self.stats_prev = {}
+    def spider_opened(self, spider: Spider) -> None:
+        self.time_prev: datetime = datetime.now(tz=timezone.utc)
+        self.delta_prev: Dict[str, Union[int, float]] = {}
+        self.stats_prev: Dict[str, Union[int, float]] = {}
 
         self.task = task.LoopingCall(self.log)
         self.task.start(self.interval)
 
-    def log(self):
-        data = {}
+    def log(self) -> None:
+        data: Dict[str, Any] = {}
         if self.ext_timing_enabled:
             data.update(self.log_timing())
         if self.ext_delta_enabled:
@@ -90,8 +107,8 @@ class PeriodicLog:
             data.update(self.log_crawler_stats())
         logger.info(self.encoder.encode(data))
 
-    def log_delta(self):
-        num_stats = {
+    def log_delta(self) -> Dict[str, Any]:
+        num_stats: Dict[str, Union[int, float]] = {
             k: v
             for k, v in self.stats._stats.items()
             if isinstance(v, (int, float))
@@ -101,7 +118,7 @@ class PeriodicLog:
         self.delta_prev = num_stats
         return {"delta": delta}
 
-    def log_timing(self):
+    def log_timing(self) -> Dict[str, Any]:
         now = datetime.now(tz=timezone.utc)
         time = {
             "log_interval": self.interval,
@@ -113,7 +130,7 @@ class PeriodicLog:
         self.time_prev = now
         return {"time": time}
 
-    def log_crawler_stats(self):
+    def log_crawler_stats(self) -> Dict[str, Any]:
         stats = {
             k: v
             for k, v in self.stats._stats.items()
@@ -121,7 +138,9 @@ class PeriodicLog:
         }
         return {"stats": stats}
 
-    def param_allowed(self, stat_name, include, exclude):
+    def param_allowed(
+        self, stat_name: str, include: List[str], exclude: List[str]
+    ) -> bool:
         if not include and not exclude:
             return True
         for p in exclude:
@@ -134,7 +153,7 @@ class PeriodicLog:
                 return True
         return False
 
-    def spider_closed(self, spider, reason):
+    def spider_closed(self, spider: Spider, reason: str) -> None:
         self.log()
         if self.task and self.task.running:
             self.task.stop()

scrapy/extensions/postprocessing.py

@@ -1,11 +1,12 @@
 """
 Extension for processing data before they are exported to feeds.
 """
+
 from bz2 import BZ2File
 from gzip import GzipFile
 from io import IOBase
 from lzma import LZMAFile
-from typing import Any, BinaryIO, Dict, List
+from typing import Any, BinaryIO, Dict, List, cast
 
 from scrapy.utils.misc import load_object
 
@@ -141,7 +142,7 @@ class PostProcessingManager(IOBase):
         :return: returns number of bytes written
         :rtype: int
         """
-        return self.head_plugin.write(data)
+        return cast(int, self.head_plugin.write(data))
 
     def tell(self) -> int:
         return self.file.tell()

scrapy/extensions/spiderstate.py

@@ -1,19 +1,27 @@
-import pickle
-from pathlib import Path
+from __future__ import annotations
 
-from scrapy import signals
+import pickle  # nosec
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional
+
+from scrapy import Spider, signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.job import job_dir
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 
 class SpiderState:
     """Store and load spider state during a scraping job"""
 
-    def __init__(self, jobdir=None):
-        self.jobdir = jobdir
+    def __init__(self, jobdir: Optional[str] = None):
+        self.jobdir: Optional[str] = jobdir
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
         jobdir = job_dir(crawler.settings)
         if not jobdir:
             raise NotConfigured
@@ -23,18 +31,20 @@ class SpiderState:
         crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened)
         return obj
 
-    def spider_closed(self, spider):
+    def spider_closed(self, spider: Spider) -> None:
         if self.jobdir:
             with Path(self.statefn).open("wb") as f:
+                assert hasattr(spider, "state")  # set in spider_opened
                 pickle.dump(spider.state, f, protocol=4)
 
-    def spider_opened(self, spider):
+    def spider_opened(self, spider: Spider) -> None:
         if self.jobdir and Path(self.statefn).exists():
             with Path(self.statefn).open("rb") as f:
-                spider.state = pickle.load(f)
+                spider.state = pickle.load(f)  # type: ignore[attr-defined]  # nosec
         else:
-            spider.state = {}
+            spider.state = {}  # type: ignore[attr-defined]
 
     @property
     def statefn(self) -> str:
+        assert self.jobdir
         return str(Path(self.jobdir, "spider.state"))

scrapy/extensions/statsmailer.py

@@ -4,28 +4,41 @@ StatsMailer extension sends an email when a spider finishes scraping.
 Use STATSMAILER_RCPTS setting to enable and give the recipient mail address
 """
 
-from scrapy import signals
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, List, Optional
+
+from twisted.internet.defer import Deferred
+
+from scrapy import Spider, signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.mail import MailSender
+from scrapy.statscollectors import StatsCollector
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 
 
 class StatsMailer:
-    def __init__(self, stats, recipients, mail):
-        self.stats = stats
-        self.recipients = recipients
-        self.mail = mail
+    def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender):
+        self.stats: StatsCollector = stats
+        self.recipients: List[str] = recipients
+        self.mail: MailSender = mail
 
     @classmethod
-    def from_crawler(cls, crawler):
-        recipients = crawler.settings.getlist("STATSMAILER_RCPTS")
+    def from_crawler(cls, crawler: Crawler) -> Self:
+        recipients: List[str] = crawler.settings.getlist("STATSMAILER_RCPTS")
         if not recipients:
             raise NotConfigured
-        mail = MailSender.from_settings(crawler.settings)
+        mail: MailSender = MailSender.from_settings(crawler.settings)
+        assert crawler.stats
         o = cls(crawler.stats, recipients, mail)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
         return o
 
-    def spider_closed(self, spider):
+    def spider_closed(self, spider: Spider) -> Optional[Deferred]:
         spider_stats = self.stats.get_stats(spider)
         body = "Global stats\n\n"
         body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items())

scrapy/extensions/telnet.py

@@ -4,13 +4,17 @@ Scrapy Telnet Console extension
 See documentation in docs/topics/telnetconsole.rst
 """
 
+from __future__ import annotations
+
 import binascii
 import logging
 import os
 import pprint
 import traceback
+from typing import TYPE_CHECKING, Any, Dict, List
 
 from twisted.internet import protocol
+from twisted.internet.tcp import Port
 
 try:
     from twisted.conch import manhole, telnet
@@ -22,12 +26,16 @@ except (ImportError, SyntaxError):
     TWISTED_CONCH_AVAILABLE = False
 
 from scrapy import signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.decorators import defers
 from scrapy.utils.engine import print_engine_status
 from scrapy.utils.reactor import listen_tcp
 from scrapy.utils.trackref import print_live_refs
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 logger = logging.getLogger(__name__)
 
 # signal to update telnet variables
@@ -36,7 +44,7 @@ update_telnet_vars = object()
 
 
 class TelnetConsole(protocol.ServerFactory):
-    def __init__(self, crawler):
+    def __init__(self, crawler: Crawler):
         if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"):
             raise NotConfigured
         if not TWISTED_CONCH_AVAILABLE:
@@ -44,14 +52,14 @@ class TelnetConsole(protocol.ServerFactory):
                 "TELNETCONSOLE_ENABLED setting is True but required twisted "
                 "modules failed to import:\n" + _TWISTED_CONCH_TRACEBACK
             )
-        self.crawler = crawler
-        self.noisy = False
-        self.portrange = [
+        self.crawler: Crawler = crawler
+        self.noisy: bool = False
+        self.portrange: List[int] = [
             int(x) for x in crawler.settings.getlist("TELNETCONSOLE_PORT")
         ]
-        self.host = crawler.settings["TELNETCONSOLE_HOST"]
-        self.username = crawler.settings["TELNETCONSOLE_USERNAME"]
-        self.password = crawler.settings["TELNETCONSOLE_PASSWORD"]
+        self.host: str = crawler.settings["TELNETCONSOLE_HOST"]
+        self.username: str = crawler.settings["TELNETCONSOLE_USERNAME"]
+        self.password: str = crawler.settings["TELNETCONSOLE_PASSWORD"]
 
         if not self.password:
             self.password = binascii.hexlify(os.urandom(8)).decode("utf8")
@@ -61,11 +69,11 @@ class TelnetConsole(protocol.ServerFactory):
         self.crawler.signals.connect(self.stop_listening, signals.engine_stopped)
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler)
 
-    def start_listening(self):
-        self.port = listen_tcp(self.portrange, self.host, self)
+    def start_listening(self) -> None:
+        self.port: Port = listen_tcp(self.portrange, self.host, self)
         h = self.port.getHost()
         logger.info(
             "Telnet console listening on %(host)s:%(port)d",
@@ -73,10 +81,10 @@ class TelnetConsole(protocol.ServerFactory):
             extra={"crawler": self.crawler},
         )
 
-    def stop_listening(self):
+    def stop_listening(self) -> None:
         self.port.stopListening()
 
-    def protocol(self):
+    def protocol(self) -> telnet.TelnetTransport:  # type: ignore[override]
         class Portal:
             """An implementation of IPortal"""
 
@@ -95,9 +103,10 @@ class TelnetConsole(protocol.ServerFactory):
 
         return telnet.TelnetTransport(telnet.AuthenticatingTelnetProtocol, Portal())
 
-    def _get_telnet_vars(self):
+    def _get_telnet_vars(self) -> Dict[str, Any]:
         # Note: if you add entries here also update topics/telnetconsole.rst
-        telnet_vars = {
+        assert self.crawler.engine
+        telnet_vars: Dict[str, Any] = {
             "engine": self.crawler.engine,
             "spider": self.crawler.engine.spider,
             "slot": self.crawler.engine.slot,

scrapy/extensions/throttle.py

@@ -1,51 +1,68 @@
-import logging
+from __future__ import annotations
 
-from scrapy import signals
+import logging
+from typing import TYPE_CHECKING, Optional, Tuple
+
+from scrapy import Request, Spider, signals
+from scrapy.core.downloader import Slot
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
+from scrapy.http import Response
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 
 logger = logging.getLogger(__name__)
 
 
 class AutoThrottle:
-    def __init__(self, crawler):
-        self.crawler = crawler
+    def __init__(self, crawler: Crawler):
+        self.crawler: Crawler = crawler
         if not crawler.settings.getbool("AUTOTHROTTLE_ENABLED"):
             raise NotConfigured
 
-        self.debug = crawler.settings.getbool("AUTOTHROTTLE_DEBUG")
-        self.target_concurrency = crawler.settings.getfloat(
+        self.debug: bool = crawler.settings.getbool("AUTOTHROTTLE_DEBUG")
+        self.target_concurrency: float = crawler.settings.getfloat(
             "AUTOTHROTTLE_TARGET_CONCURRENCY"
         )
+        if self.target_concurrency <= 0.0:
+            raise NotConfigured(
+                f"AUTOTHROTTLE_TARGET_CONCURRENCY "
+                f"({self.target_concurrency!r}) must be higher than 0."
+            )
         crawler.signals.connect(self._spider_opened, signal=signals.spider_opened)
         crawler.signals.connect(
             self._response_downloaded, signal=signals.response_downloaded
         )
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler)
 
-    def _spider_opened(self, spider):
+    def _spider_opened(self, spider: Spider) -> None:
         self.mindelay = self._min_delay(spider)
         self.maxdelay = self._max_delay(spider)
-        spider.download_delay = self._start_delay(spider)
+        spider.download_delay = self._start_delay(spider)  # type: ignore[attr-defined]
 
-    def _min_delay(self, spider):
+    def _min_delay(self, spider: Spider) -> float:
         s = self.crawler.settings
         return getattr(spider, "download_delay", s.getfloat("DOWNLOAD_DELAY"))
 
-    def _max_delay(self, spider):
+    def _max_delay(self, spider: Spider) -> float:
         return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY")
 
-    def _start_delay(self, spider):
+    def _start_delay(self, spider: Spider) -> float:
         return max(
             self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY")
         )
 
-    def _response_downloaded(self, response, request, spider):
+    def _response_downloaded(
+        self, response: Response, request: Request, spider: Spider
+    ) -> None:
         key, slot = self._get_slot(request, spider)
         latency = request.meta.get("download_latency")
-        if latency is None or slot is None:
+        if latency is None or slot is None or slot.throttle is False:
             return
 
         olddelay = slot.delay
@@ -69,11 +86,16 @@ class AutoThrottle:
                 extra={"spider": spider},
             )
 
-    def _get_slot(self, request, spider):
-        key = request.meta.get("download_slot")
+    def _get_slot(
+        self, request: Request, spider: Spider
+    ) -> Tuple[Optional[str], Optional[Slot]]:
+        key: Optional[str] = request.meta.get("download_slot")
+        if key is None:
+            return None, None
+        assert self.crawler.engine
         return key, self.crawler.engine.downloader.slots.get(key)
 
-    def _adjust_delay(self, slot, latency, response):
+    def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None:
         """Define delay adjustment policy"""
 
         # If a server needs `latency` seconds to respond then

scrapy/http/headers.py

@@ -113,7 +113,9 @@ class Headers(CaselessDict):
         return ((k, self.getlist(k)) for k in self.keys())
 
     def values(self) -> List[Optional[bytes]]:  # type: ignore[override]
-        return [self[k] for k in self.keys()]
+        return [
+            self[k] for k in self.keys()  # pylint: disable=consider-using-dict-items
+        ]
 
     def to_string(self) -> bytes:
         # cast() can be removed if the headers_dict_to_raw() hint is improved

scrapy/http/request/__init__.py

@@ -4,6 +4,7 @@ requests in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
+
 import inspect
 from typing import (
     Any,
@@ -231,12 +232,16 @@ class Request(object_ref):
         """
         d = {
             "url": self.url,  # urls are safe (safe_string_url)
-            "callback": _find_method(spider, self.callback)
-            if callable(self.callback)
-            else self.callback,
-            "errback": _find_method(spider, self.errback)
-            if callable(self.errback)
-            else self.errback,
+            "callback": (
+                _find_method(spider, self.callback)
+                if callable(self.callback)
+                else self.callback
+            ),
+            "errback": (
+                _find_method(spider, self.errback)
+                if callable(self.errback)
+                else self.errback
+            ),
             "headers": dict(self.headers),
         }
         for attr in self.attributes:

scrapy/http/request/form.py

@@ -10,21 +10,16 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast
 from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
 
-from lxml.html import (
-    FormElement,
-    HTMLParser,
-    InputElement,
-    MultipleSelectOptions,
-    SelectElement,
-    TextareaElement,
-)
-from parsel.selector import create_root_node
+from lxml.html import FormElement  # nosec
+from lxml.html import InputElement  # nosec
+from lxml.html import MultipleSelectOptions  # nosec
+from lxml.html import SelectElement  # nosec
+from lxml.html import TextareaElement  # nosec
 from w3lib.html import strip_html5_whitespace
 
 from scrapy.http.request import Request
 from scrapy.http.response.text import TextResponse
 from scrapy.utils.python import is_listlike, to_bytes
-from scrapy.utils.response import get_base_url
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
@@ -120,7 +115,7 @@ def _get_form(
     formxpath: Optional[str],
 ) -> FormElement:
     """Find the wanted form element within the given response."""
-    root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response))
+    root = response.selector.root
     forms = root.xpath("//form")
     if not forms:
         raise ValueError(f"No <form> element found in {response}")

scrapy/http/request/rpc.py

@@ -4,12 +4,17 @@ This module implements the XmlRpcRequest class which is a more convenient class
 
 See documentation in docs/topics/request-response.rst
 """
+
 import xmlrpc.client as xmlrpclib
 from typing import Any, Optional
 
+import defusedxml.xmlrpc
+
 from scrapy.http.request import Request
 from scrapy.utils.python import get_func_args
 
+defusedxml.xmlrpc.monkey_patch()
+
 DUMPS_ARGS = get_func_args(xmlrpclib.dumps)
 
 

scrapy/http/response/__init__.py

@@ -4,6 +4,7 @@ responses in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
+
 from __future__ import annotations
 
 from ipaddress import IPv4Address, IPv6Address

scrapy/http/response/text.py

@@ -4,6 +4,7 @@ discovering (through HTTP headers) to base Response class.
 
 See documentation in docs/topics/request-response.rst
 """
+
 from __future__ import annotations
 
 import json

scrapy/link.py

@@ -4,6 +4,7 @@ This module defines the Link object used in Link extractors.
 For actual link extractors implementation see scrapy.linkextractors, or
 its documentation in: docs/topics/link-extractors.rst
 """
+
 from typing import Any
 
 

scrapy/linkextractors/__init__.py

@@ -5,6 +5,7 @@ This package contains a collection of Link Extractors.
 
 For more info see docs/topics/link-extractors.rst
 """
+
 import re
 
 # common file extensions that are not followed if they occur in links

scrapy/linkextractors/lxmlhtml.py

@@ -1,12 +1,13 @@
 """
 Link extractor based on lxml.html
 """
+
 import logging
 import operator
 from functools import partial
 from urllib.parse import urljoin, urlparse
 
-from lxml import etree
+from lxml import etree  # nosec
 from parsel.csstranslator import HTMLTranslator
 from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url, safe_url_string

scrapy/loader/__init__.py

@@ -3,6 +3,7 @@ Item Loader
 
 See documentation in docs/topics/loaders.rst
 """
+
 import itemloaders
 
 from scrapy.item import Item

scrapy/mail.py

@@ -3,6 +3,9 @@ Mail sending helpers
 
 See documentation in docs/topics/email.rst
 """
+
+from __future__ import annotations
+
 import logging
 from email import encoders as Encoders
 from email.mime.base import MIMEBase
@@ -11,14 +14,21 @@ from email.mime.nonmultipart import MIMENonMultipart
 from email.mime.text import MIMEText
 from email.utils import formatdate
 from io import BytesIO
+from typing import TYPE_CHECKING, Optional
 
 from twisted import version as twisted_version
-from twisted.internet import defer, ssl
+from twisted.internet import ssl
+from twisted.internet.defer import Deferred
 from twisted.python.versions import Version
 
+from scrapy.settings import BaseSettings
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import to_bytes
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 logger = logging.getLogger(__name__)
 
 
@@ -55,7 +65,7 @@ class MailSender:
         self.debug = debug
 
     @classmethod
-    def from_settings(cls, settings):
+    def from_settings(cls, settings: BaseSettings) -> Self:
         return cls(
             smtphost=settings["MAIL_HOST"],
             mailfrom=settings["MAIL_FROM"],
@@ -76,9 +86,10 @@ class MailSender:
         mimetype="text/plain",
         charset=None,
         _callback=None,
-    ):
+    ) -> Optional[Deferred]:
         from twisted.internet import reactor
 
+        msg: MIMEBase
         if attachs:
             msg = MIMEMultipart()
         else:
@@ -125,7 +136,7 @@ class MailSender:
                     "mailattachs": len(attachs),
                 },
             )
-            return
+            return None
 
         dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8"))
         dfd.addCallbacks(
@@ -169,7 +180,7 @@ class MailSender:
         from twisted.internet import reactor
 
         msg = BytesIO(msg)
-        d = defer.Deferred()
+        d = Deferred()
 
         factory = self._create_sender_factory(to_addrs, msg, d)
 
@@ -202,7 +213,7 @@ class MailSender:
             to_addrs,
             msg,
             d,
-            **factory_keywords
+            **factory_keywords,
         )
         factory.noisy = False
         return factory

scrapy/pipelines/__init__.py

@@ -3,6 +3,7 @@ Item pipeline
 
 See documentation in docs/item-pipeline.rst
 """
+
 from typing import Any, List
 
 from twisted.internet.defer import Deferred

scrapy/pipelines/files.py

@@ -3,6 +3,7 @@ Files Pipeline
 
 See documentation in topics/media-pipeline.rst
 """
+
 import base64
 import functools
 import hashlib
@@ -15,7 +16,7 @@ from ftplib import FTP
 from io import BytesIO
 from os import PathLike
 from pathlib import Path
-from typing import DefaultDict, Optional, Set, Union
+from typing import IO, DefaultDict, Optional, Set, Union
 from urllib.parse import urlparse
 
 from itemadapter import ItemAdapter
@@ -30,7 +31,6 @@ from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.datatypes import CaseInsensitiveDict
 from scrapy.utils.ftp import ftp_store_file
 from scrapy.utils.log import failure_to_exc_info
-from scrapy.utils.misc import md5sum
 from scrapy.utils.python import to_bytes
 from scrapy.utils.request import referer_str
 
@@ -41,6 +41,23 @@ def _to_string(path: Union[str, PathLike]) -> str:
     return str(path)  # convert a Path object to string
 
 
+def _md5sum(file: IO) -> str:
+    """Calculate the md5 checksum of a file-like object without reading its
+    whole content in memory.
+
+    >>> from io import BytesIO
+    >>> _md5sum(BytesIO(b'file content to hash'))
+    '784406af91dd5a54fbb9c84c2236595a'
+    """
+    m = hashlib.md5()  # nosec
+    while True:
+        d = file.read(8096)
+        if not d:
+            break
+        m.update(d)
+    return m.hexdigest()
+
+
 class FileException(Exception):
     """General media error exception"""
 
@@ -69,7 +86,7 @@ class FSFilesStore:
             return {}
 
         with absolute_path.open("rb") as f:
-            checksum = md5sum(f)
+            checksum = _md5sum(f)
 
         return {"last_modified": last_modified, "checksum": checksum}
 
@@ -298,7 +315,7 @@ class FTPFilesStore:
                     ftp.set_pasv(False)
                 file_path = f"{self.basedir}/{path}"
                 last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip())
-                m = hashlib.md5()
+                m = hashlib.md5()  # nosec
                 ftp.retrbinary(f"RETR {file_path}", m.update)
                 return {"last_modified": last_modified, "checksum": m.hexdigest()}
             # The file doesn't exist
@@ -530,7 +547,7 @@ class FilesPipeline(MediaPipeline):
     def file_downloaded(self, response, request, info, *, item=None):
         path = self.file_path(request, response=response, info=info, item=item)
         buf = BytesIO(response.body)
-        checksum = md5sum(buf)
+        checksum = _md5sum(buf)
         buf.seek(0)
         self.store.persist_file(path, buf, info)
         return checksum
@@ -541,7 +558,7 @@ class FilesPipeline(MediaPipeline):
         return item
 
     def file_path(self, request, response=None, info=None, *, item=None):
-        media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
+        media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()  # nosec
         media_ext = Path(request.url).suffix
         # Handles empty and wild extensions by trying to guess the
         # mime type then extension or default to empty string otherwise

scrapy/pipelines/images.py

@@ -3,6 +3,7 @@ Images Pipeline
 
 See documentation in topics/media-pipeline.rst
 """
+
 import functools
 import hashlib
 import warnings
@@ -16,11 +17,10 @@ from itemadapter import ItemAdapter
 from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.http.request import NO_CALLBACK
-from scrapy.pipelines.files import FileException, FilesPipeline
+from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum
 
 # TODO: from scrapy.pipelines.media import MediaPipeline
 from scrapy.settings import Settings
-from scrapy.utils.misc import md5sum
 from scrapy.utils.python import get_func_args, to_bytes
 
 
@@ -127,7 +127,7 @@ class ImagesPipeline(FilesPipeline):
         for path, image, buf in self.get_images(response, request, info, item=item):
             if checksum is None:
                 buf.seek(0)
-                checksum = md5sum(buf)
+                checksum = _md5sum(buf)
             width, height = image.size
             self.store.persist_file(
                 path,
@@ -227,9 +227,9 @@ class ImagesPipeline(FilesPipeline):
         return item
 
     def file_path(self, request, response=None, info=None, *, item=None):
-        image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
+        image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()  # nosec
         return f"full/{image_guid}.jpg"
 
     def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
-        thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
+        thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()  # nosec
         return f"thumbs/{thumb_id}/{thumb_guid}.jpg"

scrapy/pqueues.py

@@ -20,7 +20,7 @@ def _path_safe(text):
     pathable_slot = "".join([c if c.isalnum() or c in "-._" else "_" for c in text])
     # as we replace some letters we can get collision for different slots
     # add we add unique part
-    unique_slot = hashlib.md5(text.encode("utf8")).hexdigest()
+    unique_slot = hashlib.md5(text.encode("utf8")).hexdigest()  # nosec
     return "-".join([pathable_slot, unique_slot])
 
 

scrapy/responsetypes.py

@@ -2,6 +2,7 @@
 This module implements a class which returns the appropriate Response class
 based on different criteria.
 """
+
 from io import StringIO
 from mimetypes import MimeTypes
 from pkgutil import get_data

scrapy/selector/unified.py

@@ -1,12 +1,14 @@
 """
 XPath selectors based on lxml
 """
+
 from typing import Any, Optional, Type, Union
 
 from parsel import Selector as _ParselSelector
 
 from scrapy.http import HtmlResponse, TextResponse, XmlResponse
 from scrapy.utils.python import to_bytes
+from scrapy.utils.response import get_base_url
 from scrapy.utils.trackref import object_ref
 
 __all__ = ["Selector", "SelectorList"]
@@ -87,7 +89,7 @@ class Selector(_ParselSelector, object_ref):
 
         if response is not None:
             text = response.text
-            kwargs.setdefault("base_url", response.url)
+            kwargs.setdefault("base_url", get_base_url(response))
 
         self.response = response
 

scrapy/settings/__init__.py

@@ -58,7 +58,6 @@ def get_settings_priority(priority: Union[int, str]) -> int:
 
 
 class SettingsAttribute:
-
     """Class for storing data related to settings attributes.
 
     This class is intended for internal usage, you should try Settings class

scrapy/settings/default_settings.py

@@ -177,7 +177,7 @@ FILES_STORE_S3_ACL = "private"
 FILES_STORE_GCS_ACL = ""
 
 FTP_USER = "anonymous"
-FTP_PASSWORD = "guest"
+FTP_PASSWORD = "guest"  # nosec
 FTP_PASSIVE_MODE = True
 
 GCS_PROJECT_ID = None

scrapy/shell.py

@@ -3,8 +3,10 @@
 See documentation in docs/topics/shell.rst
 
 """
+
 import os
 import signal
+from typing import Any, Callable, Dict, Optional, Tuple, Union
 
 from itemadapter import is_item
 from twisted.internet import defer, threads
@@ -25,18 +27,32 @@ from scrapy.utils.response import open_in_browser
 
 
 class Shell:
-    relevant_classes = (Crawler, Spider, Request, Response, Settings)
+    relevant_classes: Tuple[type, ...] = (Crawler, Spider, Request, Response, Settings)
 
-    def __init__(self, crawler, update_vars=None, code=None):
-        self.crawler = crawler
-        self.update_vars = update_vars or (lambda x: None)
-        self.item_class = load_object(crawler.settings["DEFAULT_ITEM_CLASS"])
-        self.spider = None
-        self.inthread = not threadable.isInIOThread()
-        self.code = code
-        self.vars = {}
+    def __init__(
+        self,
+        crawler: Crawler,
+        update_vars: Optional[Callable[[Dict[str, Any]], None]] = None,
+        code: Optional[str] = None,
+    ):
+        self.crawler: Crawler = crawler
+        self.update_vars: Callable[[Dict[str, Any]], None] = update_vars or (
+            lambda x: None
+        )
+        self.item_class: type = load_object(crawler.settings["DEFAULT_ITEM_CLASS"])
+        self.spider: Optional[Spider] = None
+        self.inthread: bool = not threadable.isInIOThread()
+        self.code: Optional[str] = code
+        self.vars: Dict[str, Any] = {}
 
-    def start(self, url=None, request=None, response=None, spider=None, redirect=True):
+    def start(
+        self,
+        url: Optional[str] = None,
+        request: Optional[Request] = None,
+        response: Optional[Response] = None,
+        spider: Optional[Spider] = None,
+        redirect: bool = True,
+    ) -> None:
         # disable accidental Ctrl-C key press from shutting down the engine
         signal.signal(signal.SIGINT, signal.SIG_IGN)
         if url:
@@ -49,7 +65,7 @@ class Shell:
         else:
             self.populate_vars()
         if self.code:
-            print(eval(self.code, globals(), self.vars))
+            print(eval(self.code, globals(), self.vars))  # nosec
         else:
             """
             Detect interactive shell setting in scrapy.cfg
@@ -76,7 +92,7 @@ class Shell:
                 self.vars, shells=shells, banner=self.vars.pop("banner", "")
             )
 
-    def _schedule(self, request, spider):
+    def _schedule(self, request: Request, spider: Optional[Spider]) -> defer.Deferred:
         if is_asyncio_reactor_installed():
             # set the asyncio event loop for the current thread
             event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"]
@@ -84,10 +100,11 @@ class Shell:
         spider = self._open_spider(request, spider)
         d = _request_deferred(request)
         d.addCallback(lambda x: (x, spider))
+        assert self.crawler.engine
         self.crawler.engine.crawl(request)
         return d
 
-    def _open_spider(self, request, spider):
+    def _open_spider(self, request: Request, spider: Optional[Spider]) -> Spider:
         if self.spider:
             return self.spider
 
@@ -95,11 +112,18 @@ class Shell:
             spider = self.crawler.spider or self.crawler._create_spider()
 
         self.crawler.spider = spider
+        assert self.crawler.engine
         self.crawler.engine.open_spider(spider, close_if_idle=False)
         self.spider = spider
         return spider
 
-    def fetch(self, request_or_url, spider=None, redirect=True, **kwargs):
+    def fetch(
+        self,
+        request_or_url: Union[Request, str],
+        spider: Optional[Spider] = None,
+        redirect: bool = True,
+        **kwargs: Any,
+    ) -> None:
         from twisted.internet import reactor
 
         if isinstance(request_or_url, Request):
@@ -122,7 +146,12 @@ class Shell:
             pass
         self.populate_vars(response, request, spider)
 
-    def populate_vars(self, response=None, request=None, spider=None):
+    def populate_vars(
+        self,
+        response: Optional[Response] = None,
+        request: Optional[Request] = None,
+        spider: Optional[Spider] = None,
+    ) -> None:
         import scrapy
 
         self.vars["scrapy"] = scrapy
@@ -140,10 +169,10 @@ class Shell:
         if not self.code:
             self.vars["banner"] = self.get_help()
 
-    def print_help(self):
+    def print_help(self) -> None:
         print(self.get_help())
 
-    def get_help(self):
+    def get_help(self) -> str:
         b = []
         b.append("Available Scrapy objects:")
         b.append(
@@ -167,11 +196,11 @@ class Shell:
 
         return "\n".join(f"[s] {line}" for line in b)
 
-    def _is_relevant(self, value):
+    def _is_relevant(self, value: Any) -> bool:
         return isinstance(value, self.relevant_classes) or is_item(value)
 
 
-def inspect_response(response, spider):
+def inspect_response(response: Response, spider: Spider) -> None:
     """Open a shell to inspect the given response"""
     # Shell.start removes the SIGINT handler, so save it and re-add it after
     # the shell has closed
@@ -180,7 +209,7 @@ def inspect_response(response, spider):
     signal.signal(signal.SIGINT, sigint_handler)
 
 
-def _request_deferred(request):
+def _request_deferred(request: Request) -> defer.Deferred:
     """Wrap a request inside a Deferred.
 
     This function is harmful, do not use it until you know what you are doing.
@@ -194,12 +223,12 @@ def _request_deferred(request):
     request_callback = request.callback
     request_errback = request.errback
 
-    def _restore_callbacks(result):
+    def _restore_callbacks(result: Any) -> Any:
         request.callback = request_callback
         request.errback = request_errback
         return result
 
-    d = defer.Deferred()
+    d: defer.Deferred = defer.Deferred()
     d.addBoth(_restore_callbacks)
     if request.callback:
         d.addCallbacks(request.callback, request.errback)

scrapy/spidermiddlewares/httperror.py

@@ -3,6 +3,7 @@ HttpError Spider Middleware
 
 See documentation in docs/topics/spider-middleware.rst
 """
+
 from __future__ import annotations
 
 import logging

scrapy/spidermiddlewares/offsite.py

@@ -3,6 +3,7 @@ Offsite Spider Middleware
 
 See documentation in docs/topics/spider-middleware.rst
 """
+
 from __future__ import annotations
 
 import logging

scrapy/spidermiddlewares/referer.py

@@ -2,6 +2,7 @@
 RefererMiddleware: populates Request referer field, based on the Response which
 originated it.
 """
+
 from __future__ import annotations
 
 import warnings

scrapy/spiders/__init__.py

@@ -3,6 +3,7 @@ Base class for Scrapy spiders
 
 See documentation in docs/topics/spiders.rst
 """
+
 from __future__ import annotations
 
 import logging

scrapy/spiders/crawl.py

@@ -85,7 +85,7 @@ class CrawlSpider(Spider):
             url=link.url,
             callback=self._callback,
             errback=self._errback,
-            meta=dict(rule=rule_index, link_text=link.text),
+            meta={"rule": rule_index, "link_text": link.text},
         )
 
     def _requests_to_follow(self, response):

scrapy/spiders/feed.py

@@ -4,6 +4,7 @@ for scraping from an XML feed.
 
 See documentation in docs/topics/spiders.rst
 """
+
 from scrapy.exceptions import NotConfigured, NotSupported
 from scrapy.selector import Selector
 from scrapy.spiders import Spider

scrapy/squeues.py

@@ -3,7 +3,7 @@ Scheduler queues
 """
 
 import marshal
-import pickle
+import pickle  # nosec
 from os import PathLike
 from pathlib import Path
 from typing import Union

scrapy/statscollectors.py

@@ -1,6 +1,7 @@
 """
 Scrapy extension for collecting scraping stats
 """
+
 import logging
 import pprint
 from typing import TYPE_CHECKING, Any, Dict, Optional

scrapy/utils/_compression.py

@@ -1,10 +1,42 @@
 import zlib
 from io import BytesIO
+from warnings import warn
+
+from scrapy.exceptions import ScrapyDeprecationWarning
 
 try:
-    import brotli
+    try:
+        import brotli
+    except ImportError:
+        import brotlicffi as brotli
 except ImportError:
     pass
+else:
+    try:
+        brotli.Decompressor.process
+    except AttributeError:
+        warn(
+            (
+                "You have brotlipy installed, and Scrapy will use it, but "
+                "Scrapy support for brotlipy is deprecated and will stop "
+                "working in a future version of Scrapy. brotlipy itself is "
+                "deprecated, it has been superseded by brotlicffi. "
+                "Please, uninstall brotlipy "
+                "and install brotli or brotlicffi instead. brotlipy has the same import "
+                "name as brotli, so keeping both installed is strongly "
+                "discouraged."
+            ),
+            ScrapyDeprecationWarning,
+        )
+
+        def _brotli_decompress(decompressor, data):
+            return decompressor.decompress(data)
+
+    else:
+
+        def _brotli_decompress(decompressor, data):
+            return decompressor.process(data)
+
 
 try:
     import zstandard
@@ -61,7 +93,7 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:
     decompressed_size = 0
     while output_chunk:
         input_chunk = input_stream.read(_CHUNK_SIZE)
-        output_chunk = decompressor.process(input_chunk)
+        output_chunk = _brotli_decompress(decompressor, input_chunk)
         decompressed_size += len(output_chunk)
         if max_size and decompressed_size > max_size:
             raise _DecompressionMaxSizeExceeded(

scrapy/utils/benchserver.py

@@ -14,7 +14,7 @@ class Root(Resource):
     def render(self, request):
         total = _getarg(request, b"total", 100, int)
         show = _getarg(request, b"show", 10, int)
-        nlist = [random.randint(1, total) for _ in range(show)]
+        nlist = [random.randint(1, total) for _ in range(show)]  # nosec
         request.write(b"<html><head></head><body>")
         args = request.args.copy()
         for nl in nlist:

scrapy/utils/console.py

@@ -1,17 +1,27 @@
 from functools import wraps
+from typing import Any, Callable, Dict, Iterable, Optional
+
+EmbedFuncT = Callable[..., None]
+KnownShellsT = Dict[str, Callable[..., EmbedFuncT]]
 
 
-def _embed_ipython_shell(namespace={}, banner=""):
+def _embed_ipython_shell(
+    namespace: Dict[str, Any] = {}, banner: str = ""
+) -> EmbedFuncT:
     """Start an IPython Shell"""
     try:
         from IPython.terminal.embed import InteractiveShellEmbed
         from IPython.terminal.ipapp import load_default_config
     except ImportError:
-        from IPython.frontend.terminal.embed import InteractiveShellEmbed
-        from IPython.frontend.terminal.ipapp import load_default_config
+        from IPython.frontend.terminal.embed import (  # type: ignore[no-redef]
+            InteractiveShellEmbed,
+        )
+        from IPython.frontend.terminal.ipapp import (  # type: ignore[no-redef]
+            load_default_config,
+        )
 
     @wraps(_embed_ipython_shell)
-    def wrapper(namespace=namespace, banner=""):
+    def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None:
         config = load_default_config()
         # Always use .instance() to ensure _instance propagation to all parents
         # this is needed for <TAB> completion works well for new imports
@@ -26,30 +36,36 @@ def _embed_ipython_shell(namespace={}, banner=""):
     return wrapper
 
 
-def _embed_bpython_shell(namespace={}, banner=""):
+def _embed_bpython_shell(
+    namespace: Dict[str, Any] = {}, banner: str = ""
+) -> EmbedFuncT:
     """Start a bpython shell"""
     import bpython
 
     @wraps(_embed_bpython_shell)
-    def wrapper(namespace=namespace, banner=""):
+    def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None:
         bpython.embed(locals_=namespace, banner=banner)
 
     return wrapper
 
 
-def _embed_ptpython_shell(namespace={}, banner=""):
+def _embed_ptpython_shell(
+    namespace: Dict[str, Any] = {}, banner: str = ""
+) -> EmbedFuncT:
     """Start a ptpython shell"""
     import ptpython.repl
 
     @wraps(_embed_ptpython_shell)
-    def wrapper(namespace=namespace, banner=""):
+    def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None:
         print(banner)
         ptpython.repl.embed(locals=namespace)
 
     return wrapper
 
 
-def _embed_standard_shell(namespace={}, banner=""):
+def _embed_standard_shell(
+    namespace: Dict[str, Any] = {}, banner: str = ""
+) -> EmbedFuncT:
     """Start a standard python shell"""
     import code
 
@@ -63,13 +79,13 @@ def _embed_standard_shell(namespace={}, banner=""):
         readline.parse_and_bind("tab:complete")
 
     @wraps(_embed_standard_shell)
-    def wrapper(namespace=namespace, banner=""):
+    def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None:
         code.interact(banner=banner, local=namespace)
 
     return wrapper
 
 
-DEFAULT_PYTHON_SHELLS = {
+DEFAULT_PYTHON_SHELLS: KnownShellsT = {
     "ptpython": _embed_ptpython_shell,
     "ipython": _embed_ipython_shell,
     "bpython": _embed_bpython_shell,
@@ -77,7 +93,9 @@ DEFAULT_PYTHON_SHELLS = {
 }
 
 
-def get_shell_embed_func(shells=None, known_shells=None):
+def get_shell_embed_func(
+    shells: Optional[Iterable[str]] = None, known_shells: Optional[KnownShellsT] = None
+) -> Any:
     """Return the first acceptable shell-embed function
     from a given list of shell names.
     """
@@ -95,7 +113,11 @@ def get_shell_embed_func(shells=None, known_shells=None):
                 continue
 
 
-def start_python_console(namespace=None, banner="", shells=None):
+def start_python_console(
+    namespace: Optional[Dict[str, Any]] = None,
+    banner: str = "",
+    shells: Optional[Iterable[str]] = None,
+) -> None:
     """Start Python console bound to the given namespace.
     Readline support and tab completion will be used on Unix, if available.
     """

scrapy/utils/defer.py

@@ -1,6 +1,7 @@
 """
 Helper functions for dealing with Twisted deferreds
 """
+
 import asyncio
 import inspect
 from asyncio import Future
@@ -304,13 +305,11 @@ _T = TypeVar("_T")
 
 
 @overload
-def deferred_from_coro(o: _CT) -> Deferred:
-    ...
+def deferred_from_coro(o: _CT) -> Deferred: ...
 
 
 @overload
-def deferred_from_coro(o: _T) -> _T:
-    ...
+def deferred_from_coro(o: _T) -> _T: ...
 
 
 def deferred_from_coro(o: _T) -> Union[Deferred, _T]:

scrapy/utils/deprecate.py

@@ -138,13 +138,11 @@ DEPRECATION_RULES: List[Tuple[str, str]] = []
 
 
 @overload
-def update_classpath(path: str) -> str:
-    ...
+def update_classpath(path: str) -> str: ...
 
 
 @overload
-def update_classpath(path: Any) -> Any:
-    ...
+def update_classpath(path: Any) -> Any: ...
 
 
 def update_classpath(path: Any) -> Any:

scrapy/utils/engine.py

@@ -1,14 +1,15 @@
 """Some debugging functions for working with the Scrapy engine"""
 
+from __future__ import annotations
+
 # used in global tests code
 from time import time  # noqa: F401
-from typing import TYPE_CHECKING, Any, List, Tuple
+from typing import Any, List, Tuple
 
-if TYPE_CHECKING:
-    from scrapy.core.engine import ExecutionEngine
+from scrapy.core.engine import ExecutionEngine
 
 
-def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]:
+def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]:
     """Return a report of the current engine status"""
     tests = [
         "time()-engine.start_time",
@@ -30,14 +31,14 @@ def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]:
     checks: List[Tuple[str, Any]] = []
     for test in tests:
         try:
-            checks += [(test, eval(test))]
+            checks += [(test, eval(test))]  # nosec
         except Exception as e:
             checks += [(test, f"{type(e).__name__} (exception)")]
 
     return checks
 
 
-def format_engine_status(engine: "ExecutionEngine") -> str:
+def format_engine_status(engine: ExecutionEngine) -> str:
     checks = get_engine_status(engine)
     s = "Execution engine status\n\n"
     for test, result in checks:
@@ -47,5 +48,5 @@ def format_engine_status(engine: "ExecutionEngine") -> str:
     return s
 
 
-def print_engine_status(engine: "ExecutionEngine") -> None:
+def print_engine_status(engine: ExecutionEngine) -> None:
     print(format_engine_status(engine))

scrapy/utils/iterators.py

@@ -18,7 +18,7 @@ from typing import (
 )
 from warnings import warn
 
-from lxml import etree
+from lxml import etree  # nosec
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse
@@ -26,7 +26,7 @@ from scrapy.selector import Selector
 from scrapy.utils.python import re_rsearch, to_unicode
 
 if TYPE_CHECKING:
-    from lxml._types import SupportsReadClose
+    from lxml._types import SupportsReadClose  # nosec
 
 logger = logging.getLogger(__name__)
 
@@ -101,6 +101,7 @@ def xmliter_lxml(
         cast("SupportsReadClose[bytes]", reader),
         encoding=reader.encoding,
         events=("end", "start-ns"),
+        resolve_entities=False,
         huge_tree=True,
     )
     selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename)
@@ -225,18 +226,17 @@ def csviter(
 
 
 @overload
-def _body_or_str(obj: Union[Response, str, bytes]) -> str:
-    ...
+def _body_or_str(obj: Union[Response, str, bytes]) -> str: ...
 
 
 @overload
-def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str:
-    ...
+def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: ...
 
 
 @overload
-def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[False]) -> bytes:
-    ...
+def _body_or_str(
+    obj: Union[Response, str, bytes], unicode: Literal[False]
+) -> bytes: ...
 
 
 def _body_or_str(

scrapy/utils/misc.py

@@ -1,4 +1,5 @@
 """Helper functions which don't fit anywhere else"""
+
 import ast
 import hashlib
 import inspect
@@ -112,7 +113,15 @@ def md5sum(file: IO) -> str:
     >>> md5sum(BytesIO(b'file content to hash'))
     '784406af91dd5a54fbb9c84c2236595a'
     """
-    m = hashlib.md5()
+    warnings.warn(
+        (
+            "The scrapy.utils.misc.md5sum function is deprecated, and will be "
+            "removed in a future version of Scrapy."
+        ),
+        ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
+    m = hashlib.md5()  # nosec
     while True:
         d = file.read(8096)
         if not d:

scrapy/utils/ossignal.py

@@ -24,7 +24,11 @@ def install_shutdown_handlers(
     (e.g. Pdb)
     """
     signal.signal(signal.SIGTERM, function)
-    if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint:
+    if (
+        signal.getsignal(signal.SIGINT)  # pylint: disable=comparison-with-callable
+        == signal.default_int_handler
+        or override_sigint
+    ):
         signal.signal(signal.SIGINT, function)
     # Catch Ctrl-Break in windows
     if hasattr(signal, "SIGBREAK"):

scrapy/utils/python.py

@@ -1,6 +1,7 @@
 """
 This module contains essential stuff that should've come with Python itself ;)
 """
+
 import collections.abc
 import gc
 import inspect
@@ -161,7 +162,7 @@ def re_rsearch(
         pattern = re.compile(pattern)
 
     for chunk, offset in _chunk_iter():
-        matches = [match for match in pattern.finditer(chunk)]
+        matches = list(pattern.finditer(chunk))
         if matches:
             start, end = matches[-1].span()
             return offset + start, offset + end
@@ -285,13 +286,11 @@ def equal_attributes(
 
 
 @overload
-def without_none_values(iterable: Mapping) -> dict:
-    ...
+def without_none_values(iterable: Mapping) -> dict: ...
 
 
 @overload
-def without_none_values(iterable: Iterable) -> Iterable:
-    ...
+def without_none_values(iterable: Iterable) -> Iterable: ...
 
 
 def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]:

scrapy/utils/reactor.py

@@ -2,17 +2,19 @@ import asyncio
 import sys
 from asyncio import AbstractEventLoop, AbstractEventLoopPolicy
 from contextlib import suppress
-from typing import Any, Callable, Dict, Optional, Sequence, Type
+from typing import Any, Callable, Dict, List, Optional, Sequence, Type
 from warnings import catch_warnings, filterwarnings, warn
 
 from twisted.internet import asyncioreactor, error
 from twisted.internet.base import DelayedCall
+from twisted.internet.protocol import ServerFactory
+from twisted.internet.tcp import Port
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.misc import load_object
 
 
-def listen_tcp(portrange, host, factory):
+def listen_tcp(portrange: List[int], host: str, factory: ServerFactory) -> Port:  # type: ignore[return]
     """Like reactor.listenTCP but tries different ports in a range."""
     from twisted.internet import reactor
 
@@ -20,8 +22,6 @@ def listen_tcp(portrange, host, factory):
         raise ValueError(f"invalid portrange: {portrange}")
     if not portrange:
         return reactor.listenTCP(0, factory, interface=host)
-    if not hasattr(portrange, "__iter__"):
-        return reactor.listenTCP(portrange, factory, interface=host)
     if len(portrange) == 1:
         return reactor.listenTCP(portrange[0], factory, interface=host)
     for x in range(portrange[0], portrange[1] + 1):

scrapy/utils/request.py

@@ -44,7 +44,9 @@ def _serialize_headers(
             yield from request.headers.getlist(header)
 
 
-_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
+_fingerprint_cache: (
+    "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
+)
 _fingerprint_cache = WeakKeyDictionary()
 
 
@@ -109,13 +111,12 @@ def fingerprint(
             "headers": headers,
         }
         fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)
-        cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest()
+        cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest()  # nosec
     return cache[cache_key]
 
 
 class RequestFingerprinterProtocol(Protocol):
-    def fingerprint(self, request: Request) -> bytes:
-        ...
+    def fingerprint(self, request: Request) -> bytes: ...
 
 
 class RequestFingerprinter:

scrapy/utils/response.py

@@ -2,24 +2,28 @@
 This module provides some useful functions for working with
 scrapy.http.Response objects
 """
+
+from __future__ import annotations
+
 import os
 import re
 import tempfile
 import webbrowser
-from typing import Any, Callable, Iterable, Tuple, Union
+from typing import TYPE_CHECKING, Any, Callable, Iterable, Tuple, Union
 from weakref import WeakKeyDictionary
 
 from twisted.web import http
 from w3lib import html
 
-import scrapy
-from scrapy.http.response import Response
 from scrapy.utils.python import to_bytes, to_unicode
 
-_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary()
+if TYPE_CHECKING:
+    from scrapy.http import Response, TextResponse
+
+_baseurl_cache: WeakKeyDictionary[Response, str] = WeakKeyDictionary()
 
 
-def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str:
+def get_base_url(response: TextResponse) -> str:
     """Return the base url of the given response, joined with the response url"""
     if response not in _baseurl_cache:
         text = response.text[0:4096]
@@ -29,13 +33,13 @@ def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str:
     return _baseurl_cache[response]
 
 
-_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = (
-    WeakKeyDictionary()
-)
+_metaref_cache: WeakKeyDictionary[
+    Response, Union[Tuple[None, None], Tuple[float, str]]
+] = WeakKeyDictionary()
 
 
 def get_meta_refresh(
-    response: "scrapy.http.response.text.TextResponse",
+    response: TextResponse,
     ignore_tags: Iterable[str] = ("script", "noscript"),
 ) -> Union[Tuple[None, None], Tuple[float, str]]:
     """Parse the http-equiv refresh parameter from the given response"""
@@ -67,10 +71,7 @@ def _remove_html_comments(body):
 
 
 def open_in_browser(
-    response: Union[
-        "scrapy.http.response.html.HtmlResponse",
-        "scrapy.http.response.text.TextResponse",
-    ],
+    response: TextResponse,
     _openfunc: Callable[[str], Any] = webbrowser.open,
 ) -> Any:
     """Open *response* in a local web browser, adjusting the `base tag`_ for

scrapy/utils/signal.py

@@ -1,4 +1,5 @@
 """Helper functions for working with signals"""
+
 import collections.abc
 import logging
 from typing import Any as TypingAny
@@ -97,7 +98,10 @@ def send_catch_log_deferred(
             robustApply, receiver, signal=signal, sender=sender, *arguments, **named
         )
         d.addErrback(logerror, receiver)
-        d.addBoth(lambda result: (receiver, result))
+        # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html
+        d.addBoth(
+            lambda result: (receiver, result)  # pylint: disable=cell-var-from-loop
+        )
         dfds.append(d)
     d = DeferredList(dfds)
     d.addCallback(lambda out: [x[1] for x in out])

scrapy/utils/sitemap.py

@@ -4,10 +4,11 @@ Module for processing Sitemaps.
 Note: The main purpose of this module is to provide support for the
 SitemapSpider, its API is subject to change without notice.
 """
+
 from typing import Any, Dict, Generator, Iterator, Optional
 from urllib.parse import urljoin
 
-import lxml.etree
+import lxml.etree  # nosec
 
 
 class Sitemap:
@@ -18,7 +19,7 @@ class Sitemap:
         xmlp = lxml.etree.XMLParser(
             recover=True, remove_comments=True, resolve_entities=False
         )
-        self._root = lxml.etree.fromstring(xmltext, parser=xmlp)
+        self._root = lxml.etree.fromstring(xmltext, parser=xmlp)  # nosec
         rt = self._root.tag
         self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt
 

scrapy/utils/spider.py

@@ -34,18 +34,15 @@ _T = TypeVar("_T")
 
 # https://stackoverflow.com/questions/60222982
 @overload
-def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator:  # type: ignore[misc]
-    ...
+def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: ...  # type: ignore[overload-overlap]
 
 
 @overload
-def iterate_spider_output(result: CoroutineType) -> Deferred:
-    ...
+def iterate_spider_output(result: CoroutineType) -> Deferred: ...
 
 
 @overload
-def iterate_spider_output(result: _T) -> Iterable:
-    ...
+def iterate_spider_output(result: _T) -> Iterable: ...
 
 
 def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]:
@@ -83,8 +80,7 @@ def spidercls_for_request(
     default_spidercls: Type[Spider],
     log_none: bool = ...,
     log_multiple: bool = ...,
-) -> Type[Spider]:
-    ...
+) -> Type[Spider]: ...
 
 
 @overload
@@ -94,8 +90,7 @@ def spidercls_for_request(
     default_spidercls: Literal[None],
     log_none: bool = ...,
     log_multiple: bool = ...,
-) -> Optional[Type[Spider]]:
-    ...
+) -> Optional[Type[Spider]]: ...
 
 
 @overload
@@ -105,8 +100,7 @@ def spidercls_for_request(
     *,
     log_none: bool = ...,
     log_multiple: bool = ...,
-) -> Optional[Type[Spider]]:
-    ...
+) -> Optional[Type[Spider]]: ...
 
 
 def spidercls_for_request(

scrapy/utils/ssl.py

@@ -1,6 +1,6 @@
 from typing import Any, Optional
 
-import OpenSSL._util as pyOpenSSLutil  # type: ignore[import-untyped]
+import OpenSSL._util as pyOpenSSLutil
 import OpenSSL.SSL
 import OpenSSL.version
 from OpenSSL.crypto import X509Name

scrapy/utils/url.py

@@ -5,6 +5,7 @@ library.
 Some of the functions that used to be imported from this module have been moved
 to the w3lib.url module. Always import those from there instead.
 """
+
 import re
 from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast
 from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse