diff --git a/.flake8 b/.flake8 index 62ccad9cf..c4814f13a 100644 --- a/.flake8 +++ b/.flake8 @@ -1,8 +1,67 @@ [flake8] max-line-length = 119 -ignore = E203, E501, E701, E704, W503 +extend-select = TC, TC1 +ignore = + # black disagrees with flake8 about these + E203, E501, E701, E704, W503 + # Assigning to `os.environ` doesn't clear the environment. + B003 + # Do not use mutable data structures for argument defaults. + B006 + # Loop control variable not used within the loop body. + B007 + # Do not perform function calls in argument defaults. + B008 + # return/continue/break inside finally blocks cause exceptions to be + # silenced. + B012 + # Star-arg unpacking after a keyword argument is strongly discouraged + B026 + # No explicit stacklevel argument found. + B028 + + # docstring does contain unindexed parameters + P102 + # other string does contain unindexed parameters + P103 + + # Missing docstring in public module + D100 + # Missing docstring in public class + D101 + # Missing docstring in public method + D102 + # Missing docstring in public function + D103 + # Missing docstring in public package + D104 + # Missing docstring in magic method + D105 + # Missing docstring in public nested class + D106 + # Missing docstring in __init__ + D107 + # One-line docstring should fit on one line with quotes + D200 + # No blank lines allowed after function docstring + D202 + # 1 blank line required between summary line and description + D205 + # Multi-line docstring closing quotes should be on a separate line + D209 + # First line should end with a period + D400 + # First line should be in imperative mood; try rephrasing + D401 + # First line should not be the function's "signature" + D402 + # First word of the first line should be properly capitalized + D403 + + # Annotation in typing.cast() should be a string literal + TC006 exclude = docs/conf.py @@ -16,6 +75,7 @@ per-file-ignores = scrapy/linkextractors/__init__.py:E402,F401 scrapy/selector/__init__.py:F401 scrapy/spiders/__init__.py:E402,F401 + tests/CrawlerRunner/change_reactor.py:E402 # Issues pending a review: scrapy/utils/url.py:F403,F405 diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d60e259ba..224d5cbbe 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -21,7 +21,7 @@ jobs: - python-version: 3.8 env: TOXENV: typing-tests - - python-version: "3.11" # Keep in sync with .readthedocs.yml + - python-version: "3.12" # Keep in sync with .readthedocs.yml env: TOXENV: docs - python-version: "3.13.0-beta.1" @@ -32,7 +32,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -52,4 +52,4 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ad94ae9cd..75a7479eb 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,13 +13,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.13.0-beta.1" - run: | pip install --upgrade build twine python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.6.4 + uses: pypa/gh-action-pypi-publish@v1.9.0 with: password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 18890239c..2e6e4265d 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -7,7 +7,7 @@ concurrency: jobs: tests: - runs-on: macos-11 + runs-on: macos-latest strategy: fail-fast: false matrix: @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 7ea58b7df..39af0c79f 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -65,7 +65,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index e23c3e67d..d32d19958 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -38,7 +38,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a911d4cfe..addad838f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,15 +1,22 @@ repos: - repo: https://github.com/PyCQA/bandit - rev: 1.7.7 + rev: 1.7.9 hooks: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 - rev: 7.0.0 + rev: 7.1.0 hooks: - id: flake8 + additional_dependencies: + - flake8-bugbear + - flake8-comprehensions + - flake8-debugger + - flake8-docstrings + - flake8-string-format + - flake8-type-checking - repo: https://github.com/psf/black.git - rev: 24.2.0 + rev: 24.4.2 hooks: - id: black - repo: https://github.com/pycqa/isort @@ -17,8 +24,13 @@ repos: hooks: - id: isort - repo: https://github.com/adamchainz/blacken-docs - rev: 1.16.0 + rev: 1.18.0 hooks: - id: blacken-docs additional_dependencies: - - black==24.2.0 + - black==24.4.2 +- repo: https://github.com/asottile/pyupgrade + rev: v3.16.0 + hooks: + - id: pyupgrade + args: [--py38-plus, --keep-runtime-typing] diff --git a/.readthedocs.yml b/.readthedocs.yml index e71d34f3a..0c544df7e 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ build: tools: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.11" # Keep in sync with .github/workflows/checks.yml + python: "3.12" # Keep in sync with .github/workflows/checks.yml python: install: diff --git a/docs/intro/install.rst b/docs/intro/install.rst index c90c1d2bf..e6c9a683b 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -37,7 +37,7 @@ Note that sometimes this may require solving compilation issues for some Scrapy dependencies depending on your operating system, so be sure to check the :ref:`intro-install-platform-notes`. -For more detailed and platform specifics instructions, as well as +For more detailed and platform-specific instructions, as well as troubleshooting information, read on. @@ -101,7 +101,7 @@ Windows ------- Though it's possible to install Scrapy on Windows using pip, we recommend you -to install `Anaconda`_ or `Miniconda`_ and use the package from the +install `Anaconda`_ or `Miniconda`_ and use the package from the `conda-forge`_ channel, which will avoid most installation issues. Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: @@ -141,7 +141,7 @@ But it should support older versions of Ubuntu too, like Ubuntu 14.04, albeit with potential issues with TLS connections. **Don't** use the ``python-scrapy`` package provided by Ubuntu, they are -typically too old and slow to catch up with latest Scrapy. +typically too old and slow to catch up with the latest Scrapy release. To install Scrapy on Ubuntu (or Ubuntu-based) systems, you need to install @@ -170,7 +170,7 @@ macOS Building Scrapy's dependencies requires the presence of a C compiler and development headers. On macOS this is typically provided by Apple’s Xcode -development tools. To install the Xcode command line tools open a terminal +development tools. To install the Xcode command-line tools, open a terminal window and run:: xcode-select --install @@ -200,11 +200,6 @@ solutions: brew install python - * Latest versions of python have ``pip`` bundled with them so you won't need - to install it separately. If this is not the case, upgrade python:: - - brew update; brew upgrade python - * *(Optional)* :ref:`Install Scrapy inside a Python virtual environment `. diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 542760b4f..cd17b1968 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -44,13 +44,13 @@ https://quotes.toscrape.com, following the pagination: if next_page is not None: yield response.follow(next_page, self.parse) -Put this in a text file, name it to something like ``quotes_spider.py`` +Put this in a text file, name it something like ``quotes_spider.py`` and run the spider using the :command:`runspider` command:: scrapy runspider quotes_spider.py -o quotes.jsonl When this finishes you will have in the ``quotes.jsonl`` file a list of the -quotes in JSON Lines format, containing text and author, looking like this:: +quotes in JSON Lines format, containing the text and author, which will look like this:: {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"} {"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"} @@ -65,27 +65,27 @@ When you ran the command ``scrapy runspider quotes_spider.py``, Scrapy looked fo Spider definition inside it and ran it through its crawler engine. The crawl started by making requests to the URLs defined in the ``start_urls`` -attribute (in this case, only the URL for quotes in *humor* category) +attribute (in this case, only the URL for quotes in the *humor* category) and called the default callback method ``parse``, passing the response object as an argument. In the ``parse`` callback, we loop through the quote elements using a CSS Selector, yield a Python dict with the extracted quote text and author, look for a link to the next page and schedule another request using the same ``parse`` method as callback. -Here you notice one of the main advantages about Scrapy: requests are +Here you will notice one of the main advantages of Scrapy: requests are :ref:`scheduled and processed asynchronously `. This means that Scrapy doesn't need to wait for a request to be finished and processed, it can send another request or do other things in the meantime. This -also means that other requests can keep going even if some request fails or an +also means that other requests can keep going even if a request fails or an error happens while handling it. While this enables you to do very fast crawls (sending multiple concurrent requests at the same time, in a fault-tolerant way) Scrapy also gives you control over the politeness of the crawl through :ref:`a few settings `. You can do things like setting a download delay between -each request, limiting amount of concurrent requests per domain or per IP, and +each request, limiting the amount of concurrent requests per domain or per IP, and even :ref:`using an auto-throttling extension ` that tries -to figure out these automatically. +to figure these settings out automatically. .. note:: @@ -106,10 +106,10 @@ scraping easy and efficient, such as: * Built-in support for :ref:`selecting and extracting ` data from HTML/XML sources using extended CSS selectors and XPath expressions, - with helper methods to extract using regular expressions. + with helper methods for extraction using regular expressions. * An :ref:`interactive shell console ` (IPython aware) for trying - out the CSS and XPath expressions to scrape data, very useful when writing or + out the CSS and XPath expressions to scrape data, which is very useful when writing or debugging your spiders. * Built-in support for :ref:`generating feed exports ` in @@ -124,7 +124,7 @@ scraping easy and efficient, such as: well-defined API (middlewares, :ref:`extensions `, and :ref:`pipelines `). -* Wide range of built-in extensions and middlewares for handling: +* A wide range of built-in extensions and middlewares for handling: - cookies and session handling - HTTP features like compression, authentication, caching diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 8ea98f29b..dd1efd3b3 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -18,11 +18,11 @@ This tutorial will walk you through these tasks: 4. Changing spider to recursively follow links 5. Using spider arguments -Scrapy is written in Python_. If you're new to the language you might want to -start by getting an idea of what the language is like, to get the most out of -Scrapy. +Scrapy is written in Python_. The more you learn about Python, the more you +can get out of Scrapy. -If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource. +If you're already familiar with other languages and want to learn Python quickly, the +`Python Tutorial`_ is a good resource. If you're new to programming and want to start with Python, the following books may be useful to you: @@ -76,10 +76,9 @@ This will create a ``tutorial`` directory with the following contents:: Our first Spider ================ -Spiders are classes that you define and that Scrapy uses to scrape information -from a website (or a group of websites). They must subclass -:class:`~scrapy.Spider` and define the initial requests to make, -optionally how to follow links in the pages, and how to parse the downloaded +Spiders are classes that you define and that Scrapy uses to scrape information from a website +(or a group of websites). They must subclass :class:`~scrapy.Spider` and define the initial +requests to be made, and optionally, how to follow links in pages and parse the downloaded page content to extract data. This is the code for our first Spider. Save it in a file named @@ -138,7 +137,7 @@ To put our spider to work, go to the project's top level directory and run:: scrapy crawl quotes -This command runs the spider with name ``quotes`` that we've just added, that +This command runs the spider named ``quotes`` that we've just added, that will send some requests for the ``quotes.toscrape.com`` domain. You will get an output similar to this:: @@ -169,7 +168,7 @@ Scrapy schedules the :class:`scrapy.Request ` objects returned by the ``start_requests`` method of the Spider. Upon receiving a response for each one, it instantiates :class:`~scrapy.http.Response` objects and calls the callback method associated with the request (in this case, the -``parse`` method) passing the response as argument. +``parse`` method) passing the response as an argument. A shortcut to the start_requests method @@ -217,8 +216,8 @@ using the :ref:`Scrapy shell `. Run:: .. note:: - Remember to always enclose urls in quotes when running Scrapy shell from - command-line, otherwise urls containing arguments (i.e. ``&`` character) + Remember to always enclose URLs in quotes when running Scrapy shell from the + command line, otherwise URLs containing arguments (i.e. ``&`` character) will not work. On Windows, use double quotes instead:: @@ -257,7 +256,7 @@ object: The result of running ``response.css('title')`` is a list-like object called :class:`~scrapy.selector.SelectorList`, which represents a list of :class:`~scrapy.Selector` objects that wrap around XML/HTML elements -and allow you to run further queries to fine-grain the selection or extract the +and allow you to run further queries to refine the selection or extract the data. To extract the text from the title above, you can do: @@ -354,12 +353,12 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions: XPath expressions are very powerful, and are the foundation of Scrapy Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You -can see that if you read closely the text representation of the selector -objects in the shell. +can see that if you read the text representation of the selector +objects in the shell closely. While perhaps not as popular as CSS selectors, XPath expressions offer more power because besides navigating the structure, it can also look at the -content. Using XPath, you're able to select things like: *select the link +content. Using XPath, you're able to select things like: *the link that contains the text "Next Page"*. This makes XPath very fitting to the task of scraping, and we encourage you to learn XPath even if you already know how to construct CSS selectors, it will make scraping much easier. @@ -422,7 +421,7 @@ variable, so that we can run our CSS selectors directly on a particular quote: >>> quote = response.css("div.quote")[0] -Now, let's extract ``text``, ``author`` and the ``tags`` from that quote +Now, let's extract the ``text``, ``author`` and ``tags`` from that quote using the ``quote`` object we just created: .. code-block:: pycon @@ -448,7 +447,7 @@ to get all of them: from sys import version_info Having figured out how to extract each bit, we can now iterate over all the -quotes elements and put them together into a Python dictionary: +quote elements and put them together into a Python dictionary: .. code-block:: pycon @@ -465,8 +464,8 @@ quotes elements and put them together into a Python dictionary: Extracting data in our spider ----------------------------- -Let's get back to our spider. Until now, it doesn't extract any data in -particular, just saves the whole HTML page to a local file. Let's integrate the +Let's get back to our spider. Until now, it hasn't extracted any data in +particular, just saving the whole HTML page to a local file. Let's integrate the extraction logic above into our spider. A Scrapy spider typically generates many dictionaries containing the data @@ -529,8 +528,8 @@ using a different serialization format, such as `JSON Lines`_:: scrapy crawl quotes -o quotes.jsonl -The `JSON Lines`_ format is useful because it's stream-like, you can easily -append new records to it. It doesn't have the same problem of JSON when you run +The `JSON Lines`_ format is useful because it's stream-like, so you can easily +append new records to it. It doesn't have the same problem as JSON when you run twice. Also, as each record is a separate line, you can process big files without having to fit everything in memory, there are tools like `JQ`_ to help do that at the command-line. @@ -555,7 +554,7 @@ from https://quotes.toscrape.com, you want quotes from all the pages in the webs Now that you know how to extract data from pages, let's see how to follow links from them. -First thing is to extract the link to the page we want to follow. Examining +The first thing to do is extract the link to the page we want to follow. Examining our page, we can see there is a link to the next page with the following markup: @@ -589,7 +588,7 @@ There is also an ``attrib`` property available >>> response.css("li.next a").attrib["href"] '/page/2/' -Let's see now our spider modified to recursively follow the link to the next +Now let's see our spider, modified to recursively follow the link to the next page, extracting data from it: .. code-block:: python @@ -756,8 +755,8 @@ Another interesting thing this spider demonstrates is that, even if there are many quotes from the same author, we don't need to worry about visiting the same author page multiple times. By default, Scrapy filters out duplicated requests to URLs already visited, avoiding the problem of hitting servers too -much because of a programming mistake. This can be configured by the setting -:setting:`DUPEFILTER_CLASS`. +much because of a programming mistake. This can be configured in the +:setting:`DUPEFILTER_CLASS` setting. Hopefully by now you have a good understanding of how to use the mechanism of following links and callbacks with Scrapy. @@ -824,12 +823,12 @@ Next steps ========== This tutorial covered only the basics of Scrapy, but there's a lot of other -features not mentioned here. Check the :ref:`topics-whatelse` section in +features not mentioned here. Check the :ref:`topics-whatelse` section in the :ref:`intro-overview` chapter for a quick overview of the most important ones. You can continue from the section :ref:`section-basics` to know more about the command-line tool, spiders, selectors and other things the tutorial hasn't covered like -modeling the scraped data. If you prefer to play with an example project, check +modeling the scraped data. If you'd prefer to play with an example project, check the :ref:`intro-examples` section. .. _JSON: https://en.wikipedia.org/wiki/JSON diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 1d37895c2..6eb4af9bd 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -6,7 +6,7 @@ Command line tool ================= -Scrapy is controlled through the ``scrapy`` command-line tool, to be referred +Scrapy is controlled through the ``scrapy`` command-line tool, to be referred to here as the "Scrapy tool" to differentiate it from the sub-commands, which we just call "commands" or "Scrapy commands". @@ -185,8 +185,8 @@ And you can see all available commands with:: There are two kinds of commands, those that only work from inside a Scrapy project (Project-specific commands) and those that also work without an active -Scrapy project (Global commands), though they may behave slightly different -when running from inside a project (as they would use the project overridden +Scrapy project (Global commands), though they may behave slightly differently +when run from inside a project (as they would use the project overridden settings). Global commands: @@ -236,7 +236,7 @@ genspider .. versionadded:: 2.6.0 The ability to pass a URL instead of a domain. -Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. +Creates a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. Usage example:: @@ -253,7 +253,7 @@ Usage example:: $ scrapy genspider -t crawl scrapyorg scrapy.org Created spider 'scrapyorg' using template 'crawl' -This is just a convenience shortcut command for creating spiders based on +This is just a convenient shortcut command for creating spiders based on pre-defined templates, but certainly not the only way to create spiders. You can just create the spider source code files yourself, instead of using this command. @@ -274,9 +274,9 @@ Supported options: * ``-a NAME=VALUE``: set a spider argument (may be repeated) -* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout), to define format set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``) +* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout). To define the output format, set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``) -* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file, to define format set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``) +* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file. To define the output format, set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``) * ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O`` @@ -353,7 +353,7 @@ edit Edit the given spider using the editor defined in the ``EDITOR`` environment variable or (if unset) the :setting:`EDITOR` setting. -This command is provided only as a convenience shortcut for the most common +This command is provided only as a convenient shortcut for the most common case, the developer is of course free to choose any tool or IDE to write and debug spiders. @@ -372,7 +372,7 @@ fetch Downloads the given URL using the Scrapy downloader and writes the contents to standard output. -The interesting thing about this command is that it fetches the page how the +The interesting thing about this command is that it fetches the page the way the spider would download it. For example, if the spider has a ``USER_AGENT`` attribute which overrides the User Agent, it will use that one. diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index a0f4b4411..a99f1e222 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -115,15 +115,14 @@ Handling different response formats Once you have a response with the desired data, how you extract the desired data from it depends on the type of response: -- If the response is HTML or XML, use :ref:`selectors +- If the response is HTML, XML or JSON, use :ref:`selectors ` as usual. -- If the response is JSON, use :func:`json.loads` to load the desired data from - :attr:`response.text `: +- If the response is JSON, use :func:`response.json()` to load the desired data: .. code-block:: python - data = json.loads(response.text) + data = response.json() If the desired data is inside HTML or XML code embedded within JSON data, you can load that HTML or XML code into a diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index f7b2f3799..7b34a19d5 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -317,6 +317,19 @@ crawls more than that, the spider will be closed with the reason ``closespider_pagecount``. If zero (or non set), spiders won't be closed by number of crawled responses. +.. setting:: CLOSESPIDER_PAGECOUNT_NO_ITEM + +CLOSESPIDER_PAGECOUNT_NO_ITEM +""""""""""""""""""""""""""""" + +Default: ``0`` + +An integer which specifies the maximum number of consecutive responses to crawl +without items scraped. If the spider crawls more consecutive responses than that +and no items are scraped in the meantime, the spider will be closed with the +reason ``closespider_pagecount_no_item``. If zero (or not set), spiders won't be +closed by number of crawled responses with no items. + .. setting:: CLOSESPIDER_ERRORCOUNT CLOSESPIDER_ERRORCOUNT diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index a5f6e07b8..58c922e0d 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -99,7 +99,7 @@ contain a price: adapter["price"] = adapter["price"] * self.vat_factor return item else: - raise DropItem(f"Missing price in {item}") + raise DropItem("Missing price") Write items to a JSON lines file @@ -254,7 +254,7 @@ returns multiples items with the same id: def process_item(self, item, spider): adapter = ItemAdapter(item) if adapter["id"] in self.ids_seen: - raise DropItem(f"Duplicate item found: {item!r}") + raise DropItem(f"Item ID already seen: {adapter['id']}") else: self.ids_seen.add(adapter["id"]) return item diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 1201c926d..f9744ed16 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -85,7 +85,7 @@ LxmlLinkExtractor :param restrict_xpaths: is an XPath (or list of XPath's) which defines regions inside the response where links should be extracted from. If given, only the text selected by those XPath will be scanned for - links. See examples below. + links. :type restrict_xpaths: str or list :param restrict_css: a CSS selector (or list of selectors) which defines diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index cd359b147..1500011e7 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -92,7 +92,6 @@ reactor after ``MySpider`` has finished running. .. code-block:: python - from twisted.internet import reactor import scrapy from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging @@ -107,6 +106,37 @@ reactor after ``MySpider`` has finished running. runner = CrawlerRunner() d = runner.crawl(MySpider) + + from twisted.internet import reactor + + d.addBoth(lambda _: reactor.stop()) + reactor.run() # the script will block here until the crawling is finished + +Same example but using a non-default reactor, it's only necessary call +``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically. + +.. code-block:: python + + import scrapy + from scrapy.crawler import CrawlerRunner + from scrapy.utils.log import configure_logging + + + class MySpider(scrapy.Spider): + # Your spider definition + ... + + + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + + from scrapy.utils.reactor import install_reactor + + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + runner = CrawlerRunner() + d = runner.crawl(MySpider) + + from twisted.internet import reactor + d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished @@ -151,7 +181,6 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: .. code-block:: python import scrapy - from twisted.internet import reactor from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging from scrapy.utils.project import get_project_settings @@ -173,6 +202,9 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: runner.crawl(MySpider1) runner.crawl(MySpider2) d = runner.join() + + from twisted.internet import reactor + d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until all crawling jobs are finished @@ -181,7 +213,7 @@ Same example but running the spiders sequentially by chaining the deferreds: .. code-block:: python - from twisted.internet import reactor, defer + from twisted.internet import defer from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging from scrapy.utils.project import get_project_settings @@ -209,6 +241,8 @@ Same example but running the spiders sequentially by chaining the deferreds: reactor.stop() + from twisted.internet import reactor + crawl() reactor.run() # the script will block here until the last crawl call is finished diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index e32fc2b70..0aae41cc8 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -1060,6 +1060,12 @@ Selector objects For convenience, this method can be called as ``response.css()`` + .. automethod:: jmespath + + .. note:: + + For convenience, this method can be called as ``response.jmespath()`` + .. automethod:: get See also: :ref:`old-extraction-api` @@ -1092,6 +1098,8 @@ SelectorList objects .. automethod:: css + .. automethod:: jmespath + .. automethod:: getall See also: :ref:`old-extraction-api` diff --git a/scrapy/__init__.py b/scrapy/__init__.py index cc0e539c4..1c1a5c2cc 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -33,12 +33,6 @@ version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("." twisted_version = (_txv.major, _txv.minor, _txv.micro) -# Check minimum required Python version -if sys.version_info < (3, 8): - print(f"Scrapy {__version__} requires Python 3.8+") - sys.exit(1) - - # Ignore noisy twisted deprecation warnings warnings.filterwarnings("ignore", category=DeprecationWarning, module="twisted") diff --git a/scrapy/addons.py b/scrapy/addons.py index 65d7a0310..f9ec58cea 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,13 +1,16 @@ +from __future__ import annotations + import logging from typing import TYPE_CHECKING, Any, List from scrapy.exceptions import NotConfigured -from scrapy.settings import Settings from scrapy.utils.conf import build_component_list from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler + from scrapy.settings import Settings + logger = logging.getLogger(__name__) @@ -15,8 +18,8 @@ logger = logging.getLogger(__name__) class AddonManager: """This class facilitates loading and storing :ref:`topics-addons`.""" - def __init__(self, crawler: "Crawler") -> None: - self.crawler: "Crawler" = crawler + def __init__(self, crawler: Crawler) -> None: + self.crawler: Crawler = crawler self.addons: List[Any] = [] def load_settings(self, settings: Settings) -> None: diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index da0e51386..e010b159a 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -12,7 +12,6 @@ import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter from scrapy.crawler import CrawlerProcess from scrapy.exceptions import UsageError -from scrapy.settings import BaseSettings, Settings from scrapy.utils.misc import walk_modules from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect @@ -21,6 +20,8 @@ if TYPE_CHECKING: # typing.ParamSpec requires Python 3.10 from typing_extensions import ParamSpec + from scrapy.settings import BaseSettings, Settings + _P = ParamSpec("_P") diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 9fe803d3c..0322390e5 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -2,18 +2,22 @@ Base class for Scrapy commands """ +from __future__ import annotations + import argparse import builtins import os from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional from twisted.python import failure -from scrapy.crawler import Crawler, CrawlerProcess from scrapy.exceptions import UsageError from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli +if TYPE_CHECKING: + from scrapy.crawler import Crawler, CrawlerProcess + class ScrapyCommand: requires_project: bool = False diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 0c4ebcd23..f91fec57e 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,16 +1,20 @@ +from __future__ import annotations + import argparse import subprocess # nosec import sys import time -from typing import Any, Iterable, List +from typing import TYPE_CHECKING, Any, Iterable, List from urllib.parse import urlencode import scrapy -from scrapy import Request from scrapy.commands import ScrapyCommand from scrapy.http import Response, TextResponse from scrapy.linkextractors import LinkExtractor +if TYPE_CHECKING: + from scrapy import Request + class Command(ScrapyCommand): default_settings = { diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 6e023af81..fe1864372 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,11 +1,15 @@ -import argparse -from typing import List, cast +from __future__ import annotations + +from typing import TYPE_CHECKING, List, cast from twisted.python.failure import Failure from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError +if TYPE_CHECKING: + import argparse + class Command(BaseRunSpiderCommand): requires_project = True diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 1acf2d26f..0bdc429da 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,6 +1,7 @@ +from __future__ import annotations + import sys -from argparse import ArgumentParser, Namespace -from typing import Dict, List, Type +from typing import TYPE_CHECKING, Dict, List, Type from w3lib.url import is_url @@ -11,6 +12,9 @@ from scrapy.http import Request, Response from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import DefaultSpider, spidercls_for_request +if TYPE_CHECKING: + from argparse import ArgumentParser, Namespace + class Command(ScrapyCommand): requires_project = False diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index dcc51a694..10330c92a 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,8 +1,12 @@ -import argparse -from typing import List +from __future__ import annotations + +from typing import TYPE_CHECKING, List from scrapy.commands import ScrapyCommand +if TYPE_CHECKING: + import argparse + class Command(ScrapyCommand): requires_project = True diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 2453c0d39..fbd200d88 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,13 +1,15 @@ +from __future__ import annotations + import argparse import functools import inspect import json import logging -from types import CoroutineType from typing import ( + TYPE_CHECKING, Any, AsyncGenerator, - Callable, + Coroutine, Dict, Iterable, List, @@ -20,13 +22,11 @@ from typing import ( from itemadapter import ItemAdapter, is_item from twisted.internet.defer import Deferred, maybeDeferred -from twisted.python.failure import Failure from w3lib.url import is_url from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError from scrapy.http import Request, Response -from scrapy.spiders import Spider from scrapy.utils import display from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.defer import aiter_errback, deferred_from_coro @@ -34,6 +34,13 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter from scrapy.utils.spider import spidercls_for_request +if TYPE_CHECKING: + from twisted.python.failure import Failure + + from scrapy.http.request import CallbackT + from scrapy.spiders import Spider + + logger = logging.getLogger(__name__) _T = TypeVar("_T") @@ -140,13 +147,13 @@ class Command(BaseRunSpiderCommand): @overload def iterate_spider_output( - self, result: Union[AsyncGenerator, CoroutineType] - ) -> Deferred: ... + self, result: Union[AsyncGenerator[_T, None], Coroutine[Any, Any, _T]] + ) -> Deferred[_T]: ... @overload - def iterate_spider_output(self, result: _T) -> Iterable: ... + def iterate_spider_output(self, result: _T) -> Iterable[Any]: ... - def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]: + def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred[Any]]: if inspect.isasyncgen(result): d = deferred_from_coro( collect_asyncgen(aiter_errback(result, self.handle_exception)) @@ -211,8 +218,8 @@ class Command(BaseRunSpiderCommand): opts: argparse.Namespace, depth: int, spider: Spider, - callback: Callable, - ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, Callable]: + callback: CallbackT, + ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT]: items, requests = [], [] for x in spider_output: if is_item(x): @@ -224,16 +231,16 @@ class Command(BaseRunSpiderCommand): def run_callback( self, response: Response, - callback: Callable, + callback: CallbackT, cb_kwargs: Optional[Dict[str, Any]] = None, - ) -> Deferred: + ) -> Deferred[Any]: cb_kwargs = cb_kwargs or {} d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) return d def get_callback_from_rules( self, spider: Spider, response: Response - ) -> Union[Callable, str, None]: + ) -> Union[CallbackT, str, None]: if getattr(spider, "rules", None): for rule in spider.rules: # type: ignore[attr-defined] if rule.link_extractor.matches(response.url): @@ -279,7 +286,7 @@ class Command(BaseRunSpiderCommand): def scraped_data( self, args: Tuple[ - List[Any], List[Request], argparse.Namespace, int, Spider, Callable + List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT ], ) -> List[Any]: items, requests, opts, depth, spider, callback = args @@ -306,8 +313,8 @@ class Command(BaseRunSpiderCommand): spider: Spider, opts: argparse.Namespace, response: Optional[Response] = None, - ) -> Callable: - cb: Union[str, Callable, None] = None + ) -> CallbackT: + cb: Union[str, CallbackT, None] = None if response: cb = response.meta["_callback"] if not cb: @@ -338,7 +345,7 @@ class Command(BaseRunSpiderCommand): def prepare_request( self, spider: Spider, request: Request, opts: argparse.Namespace ) -> Request: - def callback(response: Response, **cb_kwargs: Any) -> Deferred: + def callback(response: Response, **cb_kwargs: Any) -> Deferred[List[Any]]: # memorize first request if not self.first_response: self.first_response = response diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 77850e7b5..87acf9a01 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,17 +1,21 @@ +from __future__ import annotations + import argparse import sys from importlib import import_module -from os import PathLike from pathlib import Path -from types import ModuleType -from typing import List, Union +from typing import TYPE_CHECKING, List, Union from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError from scrapy.utils.spider import iter_spider_classes +if TYPE_CHECKING: + from os import PathLike + from types import ModuleType -def _import_file(filepath: Union[str, PathLike]) -> ModuleType: + +def _import_file(filepath: Union[str, PathLike[str]]) -> ModuleType: abspath = Path(filepath).resolve() if abspath.suffix not in (".py", ".pyw"): raise ValueError(f"Not a Python source file: {abspath}") diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 668c95a7b..f03cf997a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -4,9 +4,10 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ -from argparse import ArgumentParser, Namespace +from __future__ import annotations + from threading import Thread -from typing import Any, Dict, List, Type +from typing import TYPE_CHECKING, Any, Dict, List, Type from scrapy import Spider from scrapy.commands import ScrapyCommand @@ -15,6 +16,9 @@ from scrapy.shell import Shell from scrapy.utils.spider import DefaultSpider, spidercls_for_request from scrapy.utils.url import guess_scheme +if TYPE_CHECKING: + from argparse import ArgumentParser, Namespace + class Command(ScrapyCommand): requires_project = False diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index d46eb7c51..a7e129948 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -1,32 +1,53 @@ +from __future__ import annotations + import re import sys from functools import wraps from inspect import getmembers from types import CoroutineType -from typing import AsyncGenerator, Dict, Optional, Type -from unittest import TestCase +from typing import ( + TYPE_CHECKING, + Any, + AsyncGenerator, + Callable, + Dict, + Iterable, + List, + Optional, + Tuple, + Type, + cast, +) +from unittest import TestCase, TestResult -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.utils.python import get_spec from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + from twisted.python.failure import Failure + + from scrapy import Spider + class Contract: """Abstract class for contracts""" request_cls: Optional[Type[Request]] = None + name: str - def __init__(self, method, *args): + def __init__(self, method: Callable, *args: Any): self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook") self.testcase_post = _create_testcase(method, f"@{self.name} post-hook") - self.args = args + self.args: Tuple[Any, ...] = args - def add_pre_hook(self, request, results): + def add_pre_hook(self, request: Request, results: TestResult) -> Request: if hasattr(self, "pre_process"): cb = request.callback + assert cb is not None @wraps(cb) - def wrapper(response, **cb_kwargs): + def wrapper(response: Response, **cb_kwargs: Any) -> List[Any]: try: results.startTest(self.testcase_pre) self.pre_process(response) @@ -42,23 +63,24 @@ class Contract: if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") return list( # pylint: disable=return-in-finally - iterate_spider_output(cb_result) + cast(Iterable[Any], iterate_spider_output(cb_result)) ) request.callback = wrapper return request - def add_post_hook(self, request, results): + def add_post_hook(self, request: Request, results: TestResult) -> Request: if hasattr(self, "post_process"): cb = request.callback + assert cb is not None @wraps(cb) - def wrapper(response, **cb_kwargs): + def wrapper(response: Response, **cb_kwargs: Any) -> List[Any]: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") - output = list(iterate_spider_output(cb_result)) + output = list(cast(Iterable[Any], iterate_spider_output(cb_result))) try: results.startTest(self.testcase_post) self.post_process(output) @@ -76,18 +98,18 @@ class Contract: return request - def adjust_request_args(self, args): + def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: return args class ContractsManager: - contracts: Dict[str, Contract] = {} + contracts: Dict[str, Type[Contract]] = {} - def __init__(self, contracts): + def __init__(self, contracts: Iterable[Type[Contract]]): for contract in contracts: self.contracts[contract.name] = contract - def tested_methods_from_spidercls(self, spidercls): + def tested_methods_from_spidercls(self, spidercls: Type[Spider]) -> List[str]: is_method = re.compile(r"^\s*@", re.MULTILINE).search methods = [] for key, value in getmembers(spidercls): @@ -96,21 +118,27 @@ class ContractsManager: return methods - def extract_contracts(self, method): - contracts = [] + def extract_contracts(self, method: Callable) -> List[Contract]: + contracts: List[Contract] = [] + assert method.__doc__ is not None for line in method.__doc__.split("\n"): line = line.strip() if line.startswith("@"): - name, args = re.match(r"@(\w+)\s*(.*)", line).groups() + m = re.match(r"@(\w+)\s*(.*)", line) + if m is None: + continue + name, args = m.groups() args = re.split(r"\s+", args) contracts.append(self.contracts[name](method, *args)) return contracts - def from_spider(self, spider, results): - requests = [] + def from_spider( + self, spider: Spider, results: TestResult + ) -> List[Optional[Request]]: + requests: List[Optional[Request]] = [] for method in self.tested_methods_from_spidercls(type(spider)): bound_method = spider.__getattribute__(method) try: @@ -121,7 +149,7 @@ class ContractsManager: return requests - def from_method(self, method, results): + def from_method(self, method: Callable, results: TestResult) -> Optional[Request]: contracts = self.extract_contracts(method) if contracts: request_cls = Request @@ -154,22 +182,26 @@ class ContractsManager: self._clean_req(request, method, results) return request + return None - def _clean_req(self, request, method, results): + def _clean_req( + self, request: Request, method: Callable, results: TestResult + ) -> None: """stop the request from returning objects and records any errors""" cb = request.callback + assert cb is not None @wraps(cb) - def cb_wrapper(response, **cb_kwargs): + def cb_wrapper(response: Response, **cb_kwargs: Any) -> None: try: output = cb(response, **cb_kwargs) - output = list(iterate_spider_output(output)) + output = list(cast(Iterable[Any], iterate_spider_output(output))) except Exception: case = _create_testcase(method, "callback") results.addError(case, sys.exc_info()) - def eb_wrapper(failure): + def eb_wrapper(failure: Failure) -> None: case = _create_testcase(method, "errback") exc_info = failure.type, failure.value, failure.getTracebackObject() results.addError(case, exc_info) @@ -178,11 +210,11 @@ class ContractsManager: request.errback = eb_wrapper -def _create_testcase(method, desc): - spider = method.__self__.name +def _create_testcase(method: Callable, desc: str) -> TestCase: + spider = method.__self__.name # type: ignore[attr-defined] class ContractTestCase(TestCase): - def __str__(_self): + def __str__(_self) -> str: return f"[{spider}] {method.__name__} ({desc})" name = f"{spider}_{method.__name__}" diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index eac702cef..71ca4168a 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -1,4 +1,5 @@ import json +from typing import Any, Callable, Dict, List, Optional from itemadapter import ItemAdapter, is_item @@ -15,7 +16,7 @@ class UrlContract(Contract): name = "url" - def adjust_request_args(self, args): + def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: args["url"] = self.args[0] return args @@ -29,7 +30,7 @@ class CallbackKeywordArgumentsContract(Contract): name = "cb_kwargs" - def adjust_request_args(self, args): + def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: args["cb_kwargs"] = json.loads(" ".join(self.args)) return args @@ -48,14 +49,14 @@ class ReturnsContract(Contract): """ name = "returns" - object_type_verifiers = { + object_type_verifiers: Dict[Optional[str], Callable[[Any], bool]] = { "request": lambda x: isinstance(x, Request), "requests": lambda x: isinstance(x, Request), "item": is_item, "items": is_item, } - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) if len(self.args) not in [1, 2, 3]: @@ -66,16 +67,16 @@ class ReturnsContract(Contract): self.obj_type_verifier = self.object_type_verifiers[self.obj_name] try: - self.min_bound = int(self.args[1]) + self.min_bound: float = int(self.args[1]) except IndexError: self.min_bound = 1 try: - self.max_bound = int(self.args[2]) + self.max_bound: float = int(self.args[2]) except IndexError: self.max_bound = float("inf") - def post_process(self, output): + def post_process(self, output: List[Any]) -> None: occurrences = 0 for x in output: if self.obj_type_verifier(x): @@ -85,7 +86,7 @@ class ReturnsContract(Contract): if not assertion: if self.min_bound == self.max_bound: - expected = self.min_bound + expected = str(self.min_bound) else: expected = f"{self.min_bound}..{self.max_bound}" @@ -101,7 +102,7 @@ class ScrapesContract(Contract): name = "scrapes" - def post_process(self, output): + def post_process(self, output: List[Any]) -> None: for x in output: if is_item(x): missing = [arg for arg in self.args if arg not in ItemAdapter(x)] diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 0ab3bdb77..6786d7acf 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,9 +1,22 @@ +from __future__ import annotations + import random import warnings from collections import deque from datetime import datetime from time import time -from typing import TYPE_CHECKING, Any, Deque, Dict, Optional, Set, Tuple, cast +from typing import ( + TYPE_CHECKING, + Any, + Deque, + Dict, + Optional, + Set, + Tuple, + TypeVar, + Union, + cast, +) from twisted.internet import task from twisted.internet.defer import Deferred @@ -12,15 +25,18 @@ from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.http import Response from scrapy.resolver import dnscache -from scrapy.settings import BaseSettings from scrapy.signalmanager import SignalManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings + + +_T = TypeVar("_T") class Slot: @@ -40,7 +56,7 @@ class Slot: self.throttle = throttle self.active: Set[Request] = set() - self.queue: Deque[Tuple[Request, Deferred]] = deque() + self.queue: Deque[Tuple[Request, Deferred[Response]]] = deque() self.transferring: Set[Request] = set() self.lastseen: float = 0 self.latercall = None @@ -93,7 +109,7 @@ def _get_concurrency_delay( class Downloader: DOWNLOAD_SLOT = "download_slot" - def __init__(self, crawler: "Crawler"): + def __init__(self, crawler: Crawler): self.settings: BaseSettings = crawler.settings self.signals: SignalManager = crawler.signals self.slots: Dict[str, Slot] = {} @@ -114,13 +130,17 @@ class Downloader: "DOWNLOAD_SLOTS", {} ) - def fetch(self, request: Request, spider: Spider) -> Deferred: - def _deactivate(response: Response) -> Response: + def fetch( + self, request: Request, spider: Spider + ) -> Deferred[Union[Response, Request]]: + def _deactivate(response: _T) -> _T: self.active.remove(request) return response self.active.add(request) - dfd = self.middleware.download(self._enqueue_request, request, spider) + dfd: Deferred[Union[Response, Request]] = self.middleware.download( + self._enqueue_request, request, spider + ) return dfd.addBoth(_deactivate) def needs_backout(self) -> bool: @@ -163,7 +183,7 @@ class Downloader: ) return self.get_slot_key(request) - def _enqueue_request(self, request: Request, spider: Spider) -> Deferred: + def _enqueue_request(self, request: Request, spider: Spider) -> Deferred[Response]: key, slot = self._get_slot(request, spider) request.meta[self.DOWNLOAD_SLOT] = key @@ -175,7 +195,7 @@ class Downloader: self.signals.send_catch_log( signal=signals.request_reached_downloader, request=request, spider=spider ) - deferred: Deferred = Deferred().addBoth(_deactivate) + deferred: Deferred[Response] = Deferred().addBoth(_deactivate) slot.queue.append((request, deferred)) self._process_queue(spider, slot) return deferred @@ -208,11 +228,15 @@ class Downloader: self._process_queue(spider, slot) break - def _download(self, slot: Slot, request: Request, spider: Spider) -> Deferred: + def _download( + self, slot: Slot, request: Request, spider: Spider + ) -> Deferred[Response]: # The order is very important for the following deferreds. Do not change! # 1. Create the download deferred - dfd = mustbe_deferred(self.handlers.download_request, request, spider) + dfd: Deferred[Response] = mustbe_deferred( + self.handlers.download_request, request, spider + ) # 2. Notify response_downloaded listeners about the recent download # before querying queue for next request @@ -233,7 +257,7 @@ class Downloader: # middleware itself) slot.transferring.add(request) - def finish_transferring(_: Any) -> Any: + def finish_transferring(_: _T) -> _T: slot.transferring.remove(request) self._process_queue(spider, slot) self.signals.send_catch_log( diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 0e77cd2fe..2b388a9f5 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -21,8 +21,6 @@ from scrapy.core.downloader.tls import ( ScrapyClientTLSOptions, openssl_methods, ) -from scrapy.crawler import Crawler -from scrapy.settings import BaseSettings from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: @@ -31,6 +29,9 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + @implementer(IPolicyForHTTPS) class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): @@ -107,7 +108,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT return ctx - def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": + def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: return ScrapyClientTLSOptions( hostname.decode("ascii"), self.getContext(), @@ -134,7 +135,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): ``SSLv23_METHOD``) which allows TLS protocol negotiation. """ - def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": + def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: # trustRoot set to platformTrust() will use the platform's root CAs. # # This means that a website like https://www.cacert.org will be rejected @@ -158,8 +159,8 @@ class AcceptableProtocolsContextFactory: self._wrapped_context_factory: Any = context_factory self._acceptable_protocols: List[bytes] = acceptable_protocols - def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": - options: "ClientTLSOptions" = self._wrapped_context_factory.creatorForNetloc( + def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: + options: ClientTLSOptions = self._wrapped_context_factory.creatorForNetloc( hostname, port ) _setAcceptableProtocols(options._ctx, self._acceptable_protocols) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index ade51ca63..70d356b83 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -1,10 +1,22 @@ """Download handlers for different schemes""" +from __future__ import annotations + import logging -from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + Optional, + Protocol, + Type, + Union, + cast, +) from twisted.internet import defer -from twisted.internet.defer import Deferred from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured, NotSupported @@ -13,21 +25,36 @@ from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values if TYPE_CHECKING: + from twisted.internet.defer import Deferred + from scrapy.crawler import Crawler + from scrapy.http import Response + logger = logging.getLogger(__name__) +class DownloadHandlerProtocol(Protocol): + def download_request( + self, request: Request, spider: Spider + ) -> Deferred[Response]: ... + + class DownloadHandlers: - def __init__(self, crawler: "Crawler"): - self._crawler: "Crawler" = crawler - self._schemes: Dict[str, Union[str, Callable]] = ( + def __init__(self, crawler: Crawler): + self._crawler: Crawler = crawler + self._schemes: Dict[str, Union[str, Callable[..., Any]]] = ( {} ) # stores acceptable schemes on instancing - self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes + self._handlers: Dict[str, DownloadHandlerProtocol] = ( + {} + ) # stores instanced handlers for schemes self._notconfigured: Dict[str, str] = {} # remembers failed handlers - handlers: Dict[str, Union[str, Callable]] = without_none_values( - crawler.settings.getwithbase("DOWNLOAD_HANDLERS") + handlers: Dict[str, Union[str, Callable[..., Any]]] = without_none_values( + cast( + Dict[str, Union[str, Callable[..., Any]]], + crawler.settings.getwithbase("DOWNLOAD_HANDLERS"), + ) ) for scheme, clspath in handlers.items(): self._schemes[scheme] = clspath @@ -35,7 +62,7 @@ class DownloadHandlers: crawler.signals.connect(self._close, signals.engine_stopped) - def _get_handler(self, scheme: str) -> Any: + def _get_handler(self, scheme: str) -> Optional[DownloadHandlerProtocol]: """Lazy-load the downloadhandler for a scheme only on the first request for that scheme. """ @@ -49,10 +76,12 @@ class DownloadHandlers: return self._load_handler(scheme) - def _load_handler(self, scheme: str, skip_lazy: bool = False) -> Any: + def _load_handler( + self, scheme: str, skip_lazy: bool = False + ) -> Optional[DownloadHandlerProtocol]: path = self._schemes[scheme] try: - dhcls = load_object(path) + dhcls: Type[DownloadHandlerProtocol] = load_object(path) if skip_lazy and getattr(dhcls, "lazy", True): return None dh = build_from_crawler( @@ -75,17 +104,17 @@ class DownloadHandlers: self._handlers[scheme] = dh return dh - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: scheme = urlparse_cached(request).scheme handler = self._get_handler(scheme) if not handler: raise NotSupported( f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}" ) - return cast(Deferred, handler.download_request(request, spider)) + return handler.download_request(request, spider) @defer.inlineCallbacks - def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred, Any, None]: + def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred[Any], Any, None]: for dh in self._handlers.values(): if hasattr(dh, "close"): yield dh.close() diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index a7ae56a85..bf6879521 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -1,12 +1,16 @@ -from typing import Any, Dict +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Dict from w3lib.url import parse_data_uri -from scrapy import Request, Spider from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers +if TYPE_CHECKING: + from scrapy import Request, Spider + class DataURIDownloadHandler: lazy = False diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 17dd7483b..d55c516f0 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -1,12 +1,17 @@ +from __future__ import annotations + from pathlib import Path +from typing import TYPE_CHECKING from w3lib.url import file_uri_to_path -from scrapy import Request, Spider -from scrapy.http import Response from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers +if TYPE_CHECKING: + from scrapy import Request, Spider + from scrapy.http import Response + class FileDownloadHandler: lazy = False diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 77dcf3c38..69c2d88e1 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -35,23 +35,25 @@ from io import BytesIO from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional from urllib.parse import unquote -from twisted.internet.defer import Deferred from twisted.internet.protocol import ClientCreator, Protocol from twisted.protocols.ftp import CommandFailed, FTPClient -from twisted.python.failure import Failure -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.responsetypes import responsetypes -from scrapy.settings import BaseSettings from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes if TYPE_CHECKING: + from twisted.internet.defer import Deferred + from twisted.python.failure import Failure + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + class ReceivedDataProtocol(Protocol): def __init__(self, filename: Optional[str] = None): @@ -91,7 +93,7 @@ class FTPDownloadHandler: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: from twisted.internet import reactor parsed_url = urlparse_cached(request) @@ -103,10 +105,14 @@ class FTPDownloadHandler: creator = ClientCreator( reactor, FTPClient, user, password, passive=passive_mode ) - dfd: Deferred = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21) + dfd: Deferred[FTPClient] = creator.connectTCP( + parsed_url.hostname, parsed_url.port or 21 + ) return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path)) - def gotClient(self, client: FTPClient, request: Request, filepath: str) -> Deferred: + def gotClient( + self, client: FTPClient, request: Request, filepath: str + ) -> Deferred[Response]: self.client = client protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename")) d = client.retrieveFile(filepath, protocol) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index da9559525..8d7b0635c 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -5,20 +5,22 @@ from __future__ import annotations from typing import TYPE_CHECKING, Type -from twisted.internet.defer import Deferred - -from scrapy import Request, Spider -from scrapy.crawler import Crawler -from scrapy.settings import BaseSettings from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from twisted.internet.defer import Deferred + from twisted.internet.interfaces import IConnector + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings class HTTP10DownloadHandler: @@ -38,13 +40,13 @@ class HTTP10DownloadHandler: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: """Return a deferred for the HTTP download""" factory = self.HTTPClientFactory(request) self._connect(factory) return factory.deferred - def _connect(self, factory: ScrapyHTTPClientFactory) -> Deferred: + def _connect(self, factory: ScrapyHTTPClientFactory) -> IConnector: from twisted.internet import reactor host, port = to_unicode(factory.host), factory.port diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 5e84be6ba..c06d90f01 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,15 +8,13 @@ import re from contextlib import suppress from io import BytesIO from time import time -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, TypeVar, Union from urllib.parse import urldefrag, urlunparse from twisted.internet import ssl -from twisted.internet.base import ReactorBase from twisted.internet.defer import CancelledError, Deferred, succeed from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError -from twisted.internet.interfaces import IConsumer from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.python.failure import Failure from twisted.web.client import URI, Agent, HTTPConnectionPool @@ -30,20 +28,35 @@ from zope.interface import implementer from scrapy import Request, Spider, signals from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse -from scrapy.crawler import Crawler from scrapy.exceptions import StopDownload from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes -from scrapy.settings import BaseSettings from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: - # typing.Self requires Python 3.11 - from typing_extensions import Self + from twisted.internet.base import ReactorBase + from twisted.internet.interfaces import IConsumer + + # typing.NotRequired and typing.Self require Python 3.11 + from typing_extensions import NotRequired, Self + + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings logger = logging.getLogger(__name__) +_T = TypeVar("_T") + + +class _ResultT(TypedDict): + txresponse: TxResponse + body: bytes + flags: Optional[List[str]] + certificate: Optional[ssl.Certificate] + ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None] + failure: NotRequired[Optional[Failure]] + class HTTP11DownloadHandler: lazy = False @@ -71,7 +84,7 @@ class HTTP11DownloadHandler: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: """Return a deferred for the HTTP download""" agent = ScrapyAgent( contextFactory=self._contextFactory, @@ -83,10 +96,10 @@ class HTTP11DownloadHandler: ) return agent.download_request(request) - def close(self) -> Deferred: + def close(self) -> Deferred[None]: from twisted.internet import reactor - d: Deferred = self._pool.closeCachedConnections() + d: Deferred[None] = self._pool.closeCachedConnections() # closeCachedConnections will hang on network or server issues, so # we'll manually timeout the deferred. # @@ -97,7 +110,7 @@ class HTTP11DownloadHandler: # issue a callback after `_disconnect_timeout` seconds. delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, []) - def cancel_delayed_call(result: Any) -> Any: + def cancel_delayed_call(result: _T) -> _T: if delayed_call.active(): delayed_call.cancel() return result @@ -137,7 +150,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): ): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) - self._tunnelReadyDeferred: Deferred = Deferred() + self._tunnelReadyDeferred: Deferred[Protocol] = Deferred() self._tunneledHost: str = host self._tunneledPort: int = port self._contextFactory: IPolicyForHTTPS = contextFactory @@ -198,7 +211,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): """Propagates the errback to the appropriate deferred.""" self._tunnelReadyDeferred.errback(reason) - def connect(self, protocolFactory: Factory) -> Deferred: + def connect(self, protocolFactory: Factory) -> Deferred[Protocol]: self._protocolFactory = protocolFactory connectDeferred = super().connect(protocolFactory) connectDeferred.addCallback(self.requestTunnel) @@ -271,7 +284,7 @@ class TunnelingAgent(Agent): headers: Optional[TxHeaders], bodyProducer: Optional[IBodyProducer], requestPath: bytes, - ) -> Deferred: + ) -> Deferred[TxResponse]: # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy @@ -310,7 +323,7 @@ class ScrapyProxyAgent(Agent): uri: bytes, headers: Optional[TxHeaders] = None, bodyProducer: Optional[IBodyProducer] = None, - ) -> Deferred: + ) -> Deferred[TxResponse]: """ Issue a new request via the configured proxy. """ @@ -394,7 +407,7 @@ class ScrapyAgent: pool=self._pool, ) - def download_request(self, request: Request) -> Deferred: + def download_request(self, request: Request) -> Deferred[Response]: from twisted.internet import reactor timeout = request.meta.get("download_timeout") or self._connectTimeout @@ -411,22 +424,20 @@ class ScrapyAgent: else: bodyproducer = None start_time = time() - d: Deferred = agent.request( + d: Deferred[TxResponse] = agent.request( method, to_bytes(url, encoding="ascii"), headers, bodyproducer ) # set download latency d.addCallback(self._cb_latency, request, start_time) # response body is ready to be consumed - d.addCallback(self._cb_bodyready, request) - d.addCallback(self._cb_bodydone, request, url) + d2: Deferred[_ResultT] = d.addCallback(self._cb_bodyready, request) + d3: Deferred[Response] = d2.addCallback(self._cb_bodydone, request, url) # check download timeout - self._timeout_cl = reactor.callLater(timeout, d.cancel) - d.addBoth(self._cb_timeout, request, url, timeout) - return d + self._timeout_cl = reactor.callLater(timeout, d3.cancel) + d3.addBoth(self._cb_timeout, request, url, timeout) + return d3 - def _cb_timeout( - self, result: Any, request: Request, url: str, timeout: float - ) -> Any: + def _cb_timeout(self, result: _T, request: Request, url: str, timeout: float) -> _T: if self._timeout_cl.active(): self._timeout_cl.cancel() return result @@ -437,7 +448,7 @@ class ScrapyAgent: raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") - def _cb_latency(self, result: Any, request: Request, start_time: float) -> Any: + def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T: request.meta["download_latency"] = time() - start_time return result @@ -451,7 +462,7 @@ class ScrapyAgent: def _cb_bodyready( self, txresponse: TxResponse, request: Request - ) -> Union[Dict[str, Any], Deferred]: + ) -> Union[_ResultT, Deferred[_ResultT]]: headers_received_result = self._crawler.signals.send_catch_log( signal=signals.headers_received, headers=self._headers_from_twisted_response(txresponse), @@ -520,7 +531,7 @@ class ScrapyAgent: # Abort connection immediately. txresponse._transport._producer.abortConnection() - d: Deferred = Deferred(_cancel) + d: Deferred[_ResultT] = Deferred(_cancel) txresponse.deliverBody( _ResponseReader( finished=d, @@ -539,7 +550,7 @@ class ScrapyAgent: return d def _cb_bodydone( - self, result: Dict[str, Any], request: Request, url: str + self, result: _ResultT, request: Request, url: str ) -> Union[Response, Failure]: headers = self._headers_from_twisted_response(result["txresponse"]) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) @@ -559,8 +570,9 @@ class ScrapyAgent: protocol=protocol, ) if result.get("failure"): + assert result["failure"] result["failure"].value.response = response - return cast(Failure, result["failure"]) + return result["failure"] return response @@ -570,7 +582,7 @@ class _RequestBodyProducer: self.body = body self.length = len(body) - def startProducing(self, consumer: IConsumer) -> Deferred: + def startProducing(self, consumer: IConsumer) -> Deferred[None]: consumer.write(self.body) return succeed(None) @@ -584,7 +596,7 @@ class _RequestBodyProducer: class _ResponseReader(Protocol): def __init__( self, - finished: Deferred, + finished: Deferred[_ResultT], txresponse: TxResponse, request: Request, maxsize: int, @@ -592,7 +604,7 @@ class _ResponseReader(Protocol): fail_on_dataloss: bool, crawler: Crawler, ): - self._finished: Deferred = finished + self._finished: Deferred[_ResultT] = finished self._txresponse: TxResponse = txresponse self._request: Request = request self._bodybuf: BytesIO = BytesIO() diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 16fc1e3ae..4722c612d 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -4,25 +4,27 @@ from time import time from typing import TYPE_CHECKING, Optional from urllib.parse import urldefrag -from twisted.internet.base import DelayedCall -from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.web.client import URI -from twisted.web.iweb import IPolicyForHTTPS from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent -from scrapy.crawler import Crawler -from scrapy.http import Request, Response -from scrapy.settings import Settings -from scrapy.spiders import Spider from scrapy.utils.python import to_bytes if TYPE_CHECKING: + from twisted.internet.base import DelayedCall + from twisted.internet.defer import Deferred + from twisted.web.iweb import IPolicyForHTTPS + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Request, Response + from scrapy.settings import Settings + from scrapy.spiders import Spider + class H2DownloadHandler: def __init__(self, settings: Settings, crawler: Crawler): @@ -37,7 +39,7 @@ class H2DownloadHandler: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: agent = ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, @@ -98,7 +100,7 @@ class ScrapyH2Agent: pool=self._pool, ) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: from twisted.internet import reactor timeout = request.meta.get("download_timeout") or self._connect_timeout diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 9a0811a50..edf370193 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -2,21 +2,23 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, Optional, Type -from twisted.internet.defer import Deferred - -from scrapy import Request, Spider from scrapy.core.downloader.handlers.http import HTTPDownloadHandler -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.settings import BaseSettings from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings + class S3DownloadHandler: def __init__( @@ -59,7 +61,8 @@ class S3DownloadHandler: assert aws_access_key_id is not None assert aws_secret_access_key is not None SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"] - self._signer = SignerCls( + # botocore.auth.BaseSigner doesn't have an __init__() with args, only subclasses do + self._signer = SignerCls( # type: ignore[call-arg] botocore.credentials.Credentials( aws_access_key_id, aws_secret_access_key, aws_session_token ) @@ -75,7 +78,7 @@ class S3DownloadHandler: def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self: return cls(crawler.settings, crawler=crawler, **kwargs) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: p = urlparse_cached(request) scheme = "https" if request.meta.get("is_secure") else "http" bucket = p.hostname diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 52ebe4e22..0bdb756c8 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -4,19 +4,24 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -from typing import Any, Callable, Generator, List, Union, cast +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Callable, Generator, List, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks -from twisted.python.failure import Failure -from scrapy import Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.middleware import MiddlewareManager -from scrapy.settings import BaseSettings from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_from_coro, mustbe_deferred +if TYPE_CHECKING: + from twisted.python.failure import Failure + + from scrapy import Spider + from scrapy.settings import BaseSettings + class DownloaderMiddlewareManager(MiddlewareManager): component_name = "downloader middleware" @@ -34,10 +39,15 @@ class DownloaderMiddlewareManager(MiddlewareManager): self.methods["process_exception"].appendleft(mw.process_exception) def download( - self, download_func: Callable, request: Request, spider: Spider - ) -> Deferred: + self, + download_func: Callable[[Request, Spider], Deferred[Response]], + request: Request, + spider: Spider, + ) -> Deferred[Union[Response, Request]]: @inlineCallbacks - def process_request(request: Request) -> Generator[Deferred, Any, Any]: + def process_request( + request: Request, + ) -> Generator[Deferred[Any], Any, Union[Response, Request]]: for method in self.methods["process_request"]: method = cast(Callable, method) response = yield deferred_from_coro( @@ -52,12 +62,12 @@ class DownloaderMiddlewareManager(MiddlewareManager): ) if response: return response - return (yield download_func(request=request, spider=spider)) + return (yield download_func(request, spider)) @inlineCallbacks def process_response( response: Union[Response, Request] - ) -> Generator[Deferred, Any, Union[Response, Request]]: + ) -> Generator[Deferred[Any], Any, Union[Response, Request]]: if response is None: raise TypeError("Received None in process_response") elif isinstance(response, Request): @@ -80,7 +90,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): @inlineCallbacks def process_exception( failure: Failure, - ) -> Generator[Deferred, Any, Union[Failure, Response, Request]]: + ) -> Generator[Deferred[Any], Any, Union[Failure, Response, Request]]: exception = failure.value for method in self.methods["process_exception"]: method = cast(Callable, method) @@ -98,7 +108,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response return failure - deferred = mustbe_deferred(process_request, request) + deferred: Deferred[Union[Response, Request]] = mustbe_deferred( + process_request, request + ) deferred.addErrback(process_exception) deferred.addCallback(process_response) return deferred diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index bb1f73805..99502f0d2 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,18 +1,22 @@ +from __future__ import annotations + import re from time import time -from typing import Optional, Tuple +from typing import TYPE_CHECKING, Optional, Tuple from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse from twisted.internet import defer from twisted.internet.protocol import ClientFactory from twisted.web.http import HTTPClient -from scrapy import Request -from scrapy.http import Headers +from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode +if TYPE_CHECKING: + from scrapy import Request + def _parsed_url_args(parsed: ParseResult) -> Tuple[bytes, bytes, bytes, int, bytes]: # Assume parsed is urlparse-d from Request.url, @@ -145,7 +149,7 @@ class ScrapyHTTPClientFactory(ClientFactory): self.response_headers: Optional[Headers] = None self.timeout: float = request.meta.get("download_timeout") or timeout self.start_time: float = time() - self.deferred: defer.Deferred = defer.Deferred().addCallback( + self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback( self._build_response, request ) @@ -155,7 +159,7 @@ class ScrapyHTTPClientFactory(ClientFactory): # needed to add the callback _waitForDisconnect. # Specifically this avoids the AttributeError exception when # clientConnectionFailed method is called. - self._disconnectedDeferred: defer.Deferred = defer.Deferred() + self._disconnectedDeferred: defer.Deferred[None] = defer.Deferred() self._set_connection_attributes(request) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 4eca03800..5318cbd64 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -5,6 +5,8 @@ For more information see docs/topics/architecture.rst """ +from __future__ import annotations + import logging from time import time from typing import ( @@ -17,6 +19,7 @@ from typing import ( Optional, Set, Type, + TypeVar, Union, cast, ) @@ -31,9 +34,8 @@ from scrapy.core.scraper import Scraper from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response from scrapy.logformatter import LogFormatter -from scrapy.settings import BaseSettings, Settings +from scrapy.settings import Settings from scrapy.signalmanager import SignalManager -from scrapy.spiders import Spider from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import global_object_name @@ -41,25 +43,31 @@ from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: from scrapy.core.scheduler import BaseScheduler + from scrapy.core.scraper import _HandleOutputDeferred from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + from scrapy.spiders import Spider + logger = logging.getLogger(__name__) +_T = TypeVar("_T") + class Slot: def __init__( self, start_requests: Iterable[Request], close_if_idle: bool, - nextcall: CallLaterOnce, - scheduler: "BaseScheduler", + nextcall: CallLaterOnce[None], + scheduler: BaseScheduler, ) -> None: - self.closing: Optional[Deferred] = None + self.closing: Optional[Deferred[None]] = None self.inprogress: Set[Request] = set() self.start_requests: Optional[Iterator[Request]] = iter(start_requests) self.close_if_idle: bool = close_if_idle - self.nextcall: CallLaterOnce = nextcall - self.scheduler: "BaseScheduler" = scheduler + self.nextcall: CallLaterOnce[None] = nextcall + self.scheduler: BaseScheduler = scheduler self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule) def add_request(self, request: Request) -> None: @@ -69,7 +77,7 @@ class Slot: self.inprogress.remove(request) self._maybe_fire_closing() - def close(self) -> Deferred: + def close(self) -> Deferred[None]: self.closing = Deferred() self._maybe_fire_closing() return self.closing @@ -84,8 +92,12 @@ class Slot: class ExecutionEngine: - def __init__(self, crawler: "Crawler", spider_closed_callback: Callable) -> None: - self.crawler: "Crawler" = crawler + def __init__( + self, + crawler: Crawler, + spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]], + ) -> None: + self.crawler: Crawler = crawler self.settings: Settings = crawler.settings self.signals: SignalManager = crawler.signals assert crawler.logformatter @@ -94,19 +106,21 @@ class ExecutionEngine: self.spider: Optional[Spider] = None self.running: bool = False self.paused: bool = False - self.scheduler_cls: Type["BaseScheduler"] = self._get_scheduler_class( + self.scheduler_cls: Type[BaseScheduler] = self._get_scheduler_class( crawler.settings ) downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"]) self.downloader: Downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) - self._spider_closed_callback: Callable = spider_closed_callback + self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = ( + spider_closed_callback + ) self.start_time: Optional[float] = None - def _get_scheduler_class(self, settings: BaseSettings) -> Type["BaseScheduler"]: + def _get_scheduler_class(self, settings: BaseSettings) -> Type[BaseScheduler]: from scrapy.core.scheduler import BaseScheduler - scheduler_cls: Type = load_object(settings["SCHEDULER"]) + scheduler_cls: Type[BaseScheduler] = load_object(settings["SCHEDULER"]) if not issubclass(scheduler_cls, BaseScheduler): raise TypeError( f"The provided scheduler class ({settings['SCHEDULER']})" @@ -115,20 +129,20 @@ class ExecutionEngine: return scheduler_cls @inlineCallbacks - def start(self) -> Generator[Deferred, Any, None]: + def start(self) -> Generator[Deferred[Any], Any, None]: if self.running: raise RuntimeError("Engine already running") self.start_time = time() yield self.signals.send_catch_log_deferred(signal=signals.engine_started) self.running = True - self._closewait: Deferred = Deferred() + self._closewait: Deferred[None] = Deferred() yield self._closewait - def stop(self) -> Deferred: + def stop(self) -> Deferred[None]: """Gracefully stop the execution engine""" @inlineCallbacks - def _finish_stopping_engine(_: Any) -> Generator[Deferred, Any, None]: + def _finish_stopping_engine(_: Any) -> Generator[Deferred[Any], Any, None]: yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) self._closewait.callback(None) @@ -143,7 +157,7 @@ class ExecutionEngine: ) return dfd.addBoth(_finish_stopping_engine) - def close(self) -> Deferred: + def close(self) -> Deferred[None]: """ Gracefully close the execution engine. If it has already been started, stop it. In all cases, close the spider and the downloader. @@ -206,7 +220,7 @@ class ExecutionEngine: or self.scraper.slot.needs_backout() ) - def _next_request_from_scheduler(self) -> Optional[Deferred]: + def _next_request_from_scheduler(self) -> Optional[Deferred[None]]: assert self.slot is not None # typing assert self.spider is not None # typing @@ -214,7 +228,7 @@ class ExecutionEngine: if request is None: return None - d = self._download(request) + d: Deferred[Union[Response, Request]] = self._download(request) d.addBoth(self._handle_downloader_output, request) d.addErrback( lambda f: logger.info( @@ -228,8 +242,8 @@ class ExecutionEngine: assert self.slot self.slot.remove_request(request) - d.addBoth(_remove_request) - d.addErrback( + d2: Deferred[None] = d.addBoth(_remove_request) + d2.addErrback( lambda f: logger.info( "Error while removing request from slot", exc_info=failure_to_exc_info(f), @@ -237,19 +251,19 @@ class ExecutionEngine: ) ) slot = self.slot - d.addBoth(lambda _: slot.nextcall.schedule()) - d.addErrback( + d2.addBoth(lambda _: slot.nextcall.schedule()) + d2.addErrback( lambda f: logger.info( "Error while scheduling new request", exc_info=failure_to_exc_info(f), extra={"spider": self.spider}, ) ) - return d + return d2 def _handle_downloader_output( self, result: Union[Request, Response, Failure], request: Request - ) -> Optional[Deferred]: + ) -> Optional[_HandleOutputDeferred]: assert self.spider is not None # typing if not isinstance(result, (Request, Response, Failure)): @@ -311,20 +325,23 @@ class ExecutionEngine: signals.request_dropped, request=request, spider=spider ) - def download(self, request: Request) -> Deferred: + def download(self, request: Request) -> Deferred[Response]: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - return self._download(request).addBoth(self._downloaded, request) + d: Deferred[Union[Response, Request]] = self._download(request) + # Deferred.addBoth() overloads don't seem to support a Union[_T, Deferred[_T]] return type + d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[arg-type] + return d2 def _downloaded( self, result: Union[Response, Request, Failure], request: Request - ) -> Union[Deferred, Response, Failure]: + ) -> Union[Deferred[Response], Response, Failure]: assert self.slot is not None # typing self.slot.remove_request(request) return self.download(result) if isinstance(result, Request) else result - def _download(self, request: Request) -> Deferred: + def _download(self, request: Request) -> Deferred[Union[Response, Request]]: assert self.slot is not None # typing self.slot.add_request(request) @@ -351,21 +368,26 @@ class ExecutionEngine: ) return result - def _on_complete(_: Any) -> Any: + def _on_complete(_: _T) -> _T: assert self.slot is not None self.slot.nextcall.schedule() return _ assert self.spider is not None - dwld = self.downloader.fetch(request, self.spider) + dwld: Deferred[Union[Response, Request]] = self.downloader.fetch( + request, self.spider + ) dwld.addCallback(_on_success) dwld.addBoth(_on_complete) return dwld @inlineCallbacks def open_spider( - self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True - ) -> Generator[Deferred, Any, None]: + self, + spider: Spider, + start_requests: Iterable[Request] = (), + close_if_idle: bool = True, + ) -> Generator[Deferred[Any], Any, None]: if self.slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={"spider": spider}) @@ -411,7 +433,7 @@ class ExecutionEngine: assert isinstance(ex, CloseSpider) # typing self.close_spider(self.spider, reason=ex.reason) - def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred: + def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred[None]: """Close (cancel) spider and clear all its outstanding requests""" if self.slot is None: raise RuntimeError("Engine slot not assigned") @@ -425,7 +447,7 @@ class ExecutionEngine: dfd = self.slot.close() - def log_failure(msg: str) -> Callable: + def log_failure(msg: str) -> Callable[[Failure], None]: def errback(failure: Failure) -> None: logger.error( msg, exc_info=failure_to_exc_info(failure), extra={"spider": spider} diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 215ea9716..640fb7129 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,10 +1,10 @@ +from __future__ import annotations + from collections import deque -from typing import Deque, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Deque, Dict, List, Optional, Tuple from twisted.internet import defer -from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred -from twisted.internet.endpoints import HostnameEndpoint from twisted.python.failure import Failure from twisted.web.client import ( URI, @@ -16,9 +16,17 @@ from twisted.web.error import SchemeNotSupported from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol -from scrapy.http.request import Request -from scrapy.settings import Settings -from scrapy.spiders import Spider + +if TYPE_CHECKING: + from twisted.internet.base import ReactorBase + from twisted.internet.endpoints import HostnameEndpoint + + from scrapy.http import Request, Response + from scrapy.settings import Settings + from scrapy.spiders import Spider + + +ConnectionKeyT = Tuple[bytes, bytes, int] class H2ConnectionPool: @@ -28,19 +36,21 @@ class H2ConnectionPool: # Store a dictionary which is used to get the respective # H2ClientProtocolInstance using the key as Tuple(scheme, hostname, port) - self._connections: Dict[Tuple, H2ClientProtocol] = {} + self._connections: Dict[ConnectionKeyT, H2ClientProtocol] = {} # Save all requests that arrive before the connection is established - self._pending_requests: Dict[Tuple, Deque[Deferred]] = {} + self._pending_requests: Dict[ + ConnectionKeyT, Deque[Deferred[H2ClientProtocol]] + ] = {} def get_connection( - self, key: Tuple, uri: URI, endpoint: HostnameEndpoint - ) -> Deferred: + self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint + ) -> Deferred[H2ClientProtocol]: if key in self._pending_requests: # Received a request while connecting to remote # Create a deferred which will fire with the H2ClientProtocol # instance - d: Deferred = Deferred() + d: Deferred[H2ClientProtocol] = Deferred() self._pending_requests[key].append(d) return d @@ -54,22 +64,24 @@ class H2ConnectionPool: return self._new_connection(key, uri, endpoint) def _new_connection( - self, key: Tuple, uri: URI, endpoint: HostnameEndpoint - ) -> Deferred: + self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint + ) -> Deferred[H2ClientProtocol]: self._pending_requests[key] = deque() - conn_lost_deferred: Deferred = Deferred() + conn_lost_deferred: Deferred[List[BaseException]] = Deferred() conn_lost_deferred.addCallback(self._remove_connection, key) factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) conn_d = endpoint.connect(factory) conn_d.addCallback(self.put_connection, key) - d: Deferred = Deferred() + d: Deferred[H2ClientProtocol] = Deferred() self._pending_requests[key].append(d) return d - def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol: + def put_connection( + self, conn: H2ClientProtocol, key: ConnectionKeyT + ) -> H2ClientProtocol: self._connections[key] = conn # Now as we have established a proper HTTP/2 connection @@ -81,7 +93,9 @@ class H2ConnectionPool: return conn - def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None: + def _remove_connection( + self, errors: List[BaseException], key: ConnectionKeyT + ) -> None: self._connections.pop(key) # Call the errback of all the pending requests for this connection @@ -119,17 +133,17 @@ class H2Agent: self._reactor, self._context_factory, connect_timeout, bind_address ) - def get_endpoint(self, uri: URI): + def get_endpoint(self, uri: URI) -> HostnameEndpoint: return self.endpoint_factory.endpointForURI(uri) - def get_key(self, uri: URI) -> Tuple: + def get_key(self, uri: URI) -> ConnectionKeyT: """ Arguments: uri - URI obtained directly from request URL """ return uri.scheme, uri.host, uri.port - def request(self, request: Request, spider: Spider) -> Deferred: + def request(self, request: Request, spider: Spider) -> Deferred[Response]: uri = URI.fromBytes(bytes(request.url, encoding="utf-8")) try: endpoint = self.get_endpoint(uri) @@ -137,9 +151,11 @@ class H2Agent: return defer.fail(Failure()) key = self.get_key(uri) - d = self._pool.get_connection(key, uri, endpoint) - d.addCallback(lambda conn: conn.request(request, spider)) - return d + d: Deferred[H2ClientProtocol] = self._pool.get_connection(key, uri, endpoint) + d2: Deferred[Response] = d.addCallback( + lambda conn: conn.request(request, spider) + ) + return d2 class ScrapyProxyH2Agent(H2Agent): @@ -161,9 +177,9 @@ class ScrapyProxyH2Agent(H2Agent): ) self._proxy_uri = proxy_uri - def get_endpoint(self, uri: URI): + def get_endpoint(self, uri: URI) -> HostnameEndpoint: return self.endpoint_factory.endpointForURI(self._proxy_uri) - def get_key(self, uri: URI) -> Tuple: + def get_key(self, uri: URI) -> ConnectionKeyT: """We use the proxy uri instead of uri obtained from request url""" - return "http-proxy", self._proxy_uri.host, self._proxy_uri.port + return b"http-proxy", self._proxy_uri.host, self._proxy_uri.port diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index bc8da50d7..8aebbaab4 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,9 +1,10 @@ +from __future__ import annotations + import ipaddress import itertools import logging from collections import deque -from ipaddress import IPv4Address, IPv6Address -from typing import Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Deque, Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -20,20 +21,30 @@ from h2.events import ( WindowUpdated, ) from h2.exceptions import FrameTooLargeError, H2Error -from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError -from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory +from twisted.internet.interfaces import ( + IAddress, + IHandshakeListener, + IProtocolNegotiationFactory, +) from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin -from twisted.python.failure import Failure -from twisted.web.client import URI from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason -from scrapy.http import Request -from scrapy.settings import Settings -from scrapy.spiders import Spider +from scrapy.http import Request, Response + +if TYPE_CHECKING: + from ipaddress import IPv4Address, IPv6Address + + from twisted.internet.defer import Deferred + from twisted.python.failure import Failure + from twisted.web.client import URI + + from scrapy.settings import Settings + from scrapy.spiders import Spider + logger = logging.getLogger(__name__) @@ -77,7 +88,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin): IDLE_TIMEOUT = 240 def __init__( - self, uri: URI, settings: Settings, conn_lost_deferred: Deferred + self, + uri: URI, + settings: Settings, + conn_lost_deferred: Deferred[List[BaseException]], ) -> None: """ Arguments: @@ -88,7 +102,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): conn_lost_deferred -- Deferred fires with the reason: Failure to notify that connection was lost """ - self._conn_lost_deferred = conn_lost_deferred + self._conn_lost_deferred: Deferred[List[BaseException]] = conn_lost_deferred config = H2Configuration(client_side=True, header_encoding="utf-8") self.conn = H2Connection(config=config) @@ -103,7 +117,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # If requests are received before connection is made we keep # all requests in a pool and send them as the connection is made - self._pending_request_stream_pool: deque = deque() + self._pending_request_stream_pool: Deque[Stream] = deque() # Save an instance of errors raised which lead to losing the connection # We pass these instances to the streams ResponseFailed() failure @@ -111,7 +125,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # Some meta data of this connection # initialized when connection is successfully made - self.metadata: Dict = { + self.metadata: Dict[str, Any] = { # Peer certificate instance "certificate": None, # Address of the server we are connected to which @@ -204,14 +218,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin): data = self.conn.data_to_send() self.transport.write(data) - def request(self, request: Request, spider: Spider) -> Deferred: + def request(self, request: Request, spider: Spider) -> Deferred[Response]: if not isinstance(request, Request): raise TypeError( f"Expected scrapy.http.Request, received {request.__class__.__qualname__}" ) stream = self._new_stream(request, spider) - d = stream.get_response() + d: Deferred[Response] = stream.get_response() # Add the stream to the request pool self._pending_request_stream_pool.append(stream) @@ -425,13 +439,16 @@ class H2ClientProtocol(Protocol, TimeoutMixin): @implementer(IProtocolNegotiationFactory) class H2ClientFactory(Factory): def __init__( - self, uri: URI, settings: Settings, conn_lost_deferred: Deferred + self, + uri: URI, + settings: Settings, + conn_lost_deferred: Deferred[List[BaseException]], ) -> None: self.uri = uri self.settings = settings self.conn_lost_deferred = conn_lost_deferred - def buildProtocol(self, addr) -> H2ClientProtocol: + def buildProtocol(self, addr: IAddress) -> H2ClientProtocol: return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred) def acceptableProtocols(self) -> List[bytes]: diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 4132fc385..d8b5cc8eb 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,23 +1,26 @@ +from __future__ import annotations + import logging from enum import Enum from io import BytesIO -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError -from hpack import HeaderTuple from twisted.internet.defer import CancelledError, Deferred from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: + from hpack import HeaderTuple + from scrapy.core.http2.protocol import H2ClientProtocol + from scrapy.http import Request, Response logger = logging.getLogger(__name__) @@ -87,7 +90,7 @@ class Stream: self, stream_id: int, request: Request, - protocol: "H2ClientProtocol", + protocol: H2ClientProtocol, download_maxsize: int = 0, download_warnsize: int = 0, ) -> None: @@ -99,7 +102,7 @@ class Stream: """ self.stream_id: int = stream_id self._request: Request = request - self._protocol: "H2ClientProtocol" = protocol + self._protocol: H2ClientProtocol = protocol self._download_maxsize = self._request.meta.get( "download_maxsize", download_maxsize @@ -110,7 +113,7 @@ class Stream: # Metadata of an HTTP/2 connection stream # initialized when stream is instantiated - self.metadata: Dict = { + self.metadata: Dict[str, Any] = { "request_content_length": ( 0 if self._request.body is None else len(self._request.body) ), @@ -131,7 +134,7 @@ class Stream: # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response: Dict = { + self._response: Dict[str, Any] = { # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. "body": BytesIO(), @@ -142,7 +145,7 @@ class Stream: "headers": Headers({}), } - def _cancel(_) -> None: + def _cancel(_: Any) -> None: # Close this stream as gracefully as possible # If the associated request is initiated we reset this stream # else we directly call close() method @@ -151,7 +154,7 @@ class Stream: else: self.close(StreamCloseReason.CANCELLED) - self._deferred_response: Deferred = Deferred(_cancel) + self._deferred_response: Deferred[Response] = Deferred(_cancel) def __repr__(self) -> str: return f"Stream(id={self.stream_id!r})" @@ -177,7 +180,7 @@ class Stream: and not self.metadata["reached_warnsize"] ) - def get_response(self) -> Deferred: + def get_response(self) -> Deferred[Response]: """Simply return a Deferred which fires when response from the asynchronous request is available """ diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index f30a5d9c9..d4286c874 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,16 +4,12 @@ import json import logging from abc import abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, Type, cast +from typing import TYPE_CHECKING, Any, List, Optional, Type, cast -from twisted.internet.defer import Deferred +# working around https://github.com/sphinx-doc/sphinx/issues/10400 +from twisted.internet.defer import Deferred # noqa: TC002 -from scrapy.crawler import Crawler -from scrapy.dupefilters import BaseDupeFilter -from scrapy.http.request import Request -from scrapy.pqueues import ScrapyPriorityQueue -from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector +from scrapy.spiders import Spider # noqa: TC001 from scrapy.utils.job import job_dir from scrapy.utils.misc import build_from_crawler, load_object @@ -24,6 +20,12 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.dupefilters import BaseDupeFilter + from scrapy.http.request import Request + from scrapy.pqueues import ScrapyPriorityQueue + from scrapy.statscollectors import StatsCollector + logger = logging.getLogger(__name__) @@ -71,7 +73,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): """ return cls() - def open(self, spider: Spider) -> Optional[Deferred]: + def open(self, spider: Spider) -> Optional[Deferred[None]]: """ Called when the spider is opened by the engine. It receives the spider instance as argument and it's useful to execute initialization code. @@ -81,7 +83,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): """ pass - def close(self, reason: str) -> Optional[Deferred]: + def close(self, reason: str) -> Optional[Deferred[None]]: """ Called when the spider is closed by the engine. It receives the reason why the crawl finished as argument and it's useful to execute cleaning code. @@ -216,7 +218,7 @@ class Scheduler(BaseScheduler): def has_pending_requests(self) -> bool: return len(self) > 0 - def open(self, spider: Spider) -> Optional[Deferred]: + def open(self, spider: Spider) -> Optional[Deferred[None]]: """ (1) initialize the memory queue (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory @@ -227,7 +229,7 @@ class Scheduler(BaseScheduler): self.dqs: Optional[ScrapyPriorityQueue] = self._dq() if self.dqdir else None return self.df.open() - def close(self, reason: str) -> Optional[Deferred]: + def close(self, reason: str) -> Optional[Deferred[None]]: """ (1) dump pending requests to disk if there is a disk queue (2) return the result of the dupefilter's ``close`` method @@ -362,13 +364,13 @@ class Scheduler(BaseScheduler): return str(dqdir) return None - def _read_dqs_state(self, dqdir: str) -> list: + def _read_dqs_state(self, dqdir: str) -> List[int]: path = Path(dqdir, "active.json") if not path.exists(): return [] with path.open(encoding="utf-8") as f: - return cast(list, json.load(f)) + return cast(List[int], json.load(f)) - def _write_dqs_state(self, dqdir: str, state: list) -> None: + def _write_dqs_state(self, dqdir: str, state: List[int]) -> None: with Path(dqdir, "active.json").open("w", encoding="utf-8") as f: json.dump(state, f) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 566e6628b..a7d65e1e3 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -12,10 +12,13 @@ from typing import ( Deque, Generator, Iterable, + Iterator, + List, Optional, Set, Tuple, Type, + TypeVar, Union, cast, ) @@ -47,12 +50,18 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler -QueueTuple = Tuple[Union[Response, Failure], Request, Deferred] - - logger = logging.getLogger(__name__) +_T = TypeVar("_T") +_ParallelResult = List[Tuple[bool, Iterator[Any]]] + +if TYPE_CHECKING: + # parameterized Deferreds require Twisted 21.7.0 + _HandleOutputDeferred = Deferred[Union[_ParallelResult, None]] + QueueTuple = Tuple[Union[Response, Failure], Request, _HandleOutputDeferred] + + class Slot: """Scraper slot (one per running spider)""" @@ -64,12 +73,12 @@ class Slot: self.active: Set[Request] = set() self.active_size: int = 0 self.itemproc_size: int = 0 - self.closing: Optional[Deferred] = None + self.closing: Optional[Deferred[Spider]] = None def add_response_request( self, result: Union[Response, Failure], request: Request - ) -> Deferred: - deferred: Deferred = Deferred() + ) -> _HandleOutputDeferred: + deferred: _HandleOutputDeferred = Deferred() self.queue.append((result, request, deferred)) if isinstance(result, Response): self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE) @@ -115,12 +124,12 @@ class Scraper: self.logformatter: LogFormatter = crawler.logformatter @inlineCallbacks - def open_spider(self, spider: Spider) -> Generator[Deferred, Any, None]: + def open_spider(self, spider: Spider) -> Generator[Deferred[Any], Any, None]: """Open the given spider for scraping and allocate resources for it""" self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE")) yield self.itemproc.open_spider(spider) - def close_spider(self, spider: Spider) -> Deferred: + def close_spider(self, spider: Spider) -> Deferred[Spider]: """Close a spider being scraped and release its resources""" if self.slot is None: raise RuntimeError("Scraper slot not assigned") @@ -140,12 +149,12 @@ class Scraper: def enqueue_scrape( self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred: + ) -> _HandleOutputDeferred: if self.slot is None: raise RuntimeError("Scraper slot not assigned") dfd = self.slot.add_response_request(result, request) - def finish_scraping(_: Any) -> Any: + def finish_scraping(_: _T) -> _T: assert self.slot is not None self.slot.finish_response(result, request) self._check_if_closing(spider) @@ -172,7 +181,7 @@ class Scraper: def _scrape( self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred: + ) -> _HandleOutputDeferred: """ Handle the downloaded response or failure through the spider callback/errback """ @@ -180,32 +189,35 @@ class Scraper: raise TypeError( f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}" ) - dfd = self._scrape2( + dfd: Deferred[Union[Iterable[Any], AsyncIterable[Any]]] = self._scrape2( result, request, spider ) # returns spider's processed output dfd.addErrback(self.handle_spider_error, request, result, spider) - dfd.addCallback( + dfd2: _HandleOutputDeferred = dfd.addCallback( self.handle_spider_output, request, cast(Response, result), spider ) - return dfd + return dfd2 def _scrape2( self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred: + ) -> Deferred[Union[Iterable[Any], AsyncIterable[Any]]]: """ Handle the different cases of request's result been a Response or a Failure """ if isinstance(result, Response): - return self.spidermw.scrape_response( + # Deferreds are invariant so Mutable*Chain isn't matched to *Iterable + return self.spidermw.scrape_response( # type: ignore[return-value] self.call_spider, result, request, spider ) # else result is a Failure dfd = self.call_spider(result, request, spider) - return dfd.addErrback(self._log_download_errors, result, request, spider) + dfd.addErrback(self._log_download_errors, result, request, spider) + return dfd def call_spider( self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred: + ) -> Deferred[Union[Iterable[Any], AsyncIterable[Any]]]: + dfd: Deferred[Any] if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request @@ -223,7 +235,10 @@ class Scraper: if request.errback: warn_on_generator_with_return_value(spider, request.errback) dfd.addErrback(request.errback) - return dfd.addCallback(iterate_spider_output) + dfd2: Deferred[Union[Iterable[Any], AsyncIterable[Any]]] = dfd.addCallback( + iterate_spider_output + ) + return dfd2 def handle_spider_error( self, @@ -256,14 +271,15 @@ class Scraper: def handle_spider_output( self, - result: Union[Iterable, AsyncIterable], + result: Union[Iterable[_T], AsyncIterable[_T]], request: Request, response: Response, spider: Spider, - ) -> Deferred: + ) -> _HandleOutputDeferred: if not result: return defer_succeed(None) - it: Union[Iterable, AsyncIterable] + it: Union[Iterable[_T], AsyncIterable[_T]] + dfd: Deferred[_ParallelResult] if isinstance(result, AsyncIterable): it = aiter_errback( result, self.handle_spider_error, request, response, spider @@ -288,11 +304,12 @@ class Scraper: response, spider, ) - return dfd + # returning Deferred[_ParallelResult] instead of Deferred[Union[_ParallelResult, None]] + return dfd # type: ignore[return-value] def _process_spidermw_output( self, output: Any, request: Request, response: Response, spider: Spider - ) -> Optional[Deferred]: + ) -> Optional[Deferred[Any]]: """Process each Request/Item (given in the output parameter) returned from the given spider """ @@ -357,7 +374,7 @@ class Scraper: def _itemproc_finished( self, output: Any, item: Any, response: Response, spider: Spider - ) -> Deferred: + ) -> Deferred[Any]: """ItemProcessor finished for the given ``item`` and returned ``output``""" assert self.slot is not None # typing self.slot.itemproc_size -= 1 diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 2cef2e1dd..c9feac29c 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -4,12 +4,14 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ +from __future__ import annotations + import logging from inspect import isasyncgenfunction, iscoroutine from itertools import islice from typing import ( + TYPE_CHECKING, Any, - AsyncGenerator, AsyncIterable, Callable, Generator, @@ -17,6 +19,7 @@ from typing import ( List, Optional, Tuple, + TypeVar, Union, cast, ) @@ -28,7 +31,6 @@ from scrapy import Request, Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Response from scrapy.middleware import MiddlewareManager -from scrapy.settings import BaseSettings from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.conf import build_component_list from scrapy.utils.defer import ( @@ -39,10 +41,17 @@ from scrapy.utils.defer import ( ) from scrapy.utils.python import MutableAsyncChain, MutableChain +if TYPE_CHECKING: + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) -ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] +_T = TypeVar("_T") +ScrapeFunc = Callable[ + [Union[Response, Failure], Request, Spider], Union[Iterable[_T], AsyncIterable[_T]] +] def _isiterable(o: Any) -> bool: @@ -77,7 +86,7 @@ class SpiderMiddlewareManager(MiddlewareManager): response: Response, request: Request, spider: Spider, - ) -> Any: + ) -> Union[Iterable[_T], AsyncIterable[_T]]: for method in self.methods["process_spider_input"]: method = cast(Callable, method) try: @@ -98,31 +107,39 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - iterable: Union[Iterable, AsyncIterable], + iterable: Union[Iterable[_T], AsyncIterable[_T]], exception_processor_index: int, - recover_to: Union[MutableChain, MutableAsyncChain], - ) -> Union[Generator, AsyncGenerator]: - def process_sync(iterable: Iterable) -> Generator: + recover_to: Union[MutableChain[_T], MutableAsyncChain[_T]], + ) -> Union[Iterable[_T], AsyncIterable[_T]]: + def process_sync(iterable: Iterable[_T]) -> Iterable[_T]: try: yield from iterable except Exception as ex: - exception_result = self._process_spider_exception( - response, spider, Failure(ex), exception_processor_index + exception_result = cast( + Union[Failure, MutableChain[_T]], + self._process_spider_exception( + response, spider, Failure(ex), exception_processor_index + ), ) if isinstance(exception_result, Failure): raise + assert isinstance(recover_to, MutableChain) recover_to.extend(exception_result) - async def process_async(iterable: AsyncIterable) -> AsyncGenerator: + async def process_async(iterable: AsyncIterable[_T]) -> AsyncIterable[_T]: try: async for r in iterable: yield r except Exception as ex: - exception_result = self._process_spider_exception( - response, spider, Failure(ex), exception_processor_index + exception_result = cast( + Union[Failure, MutableAsyncChain[_T]], + self._process_spider_exception( + response, spider, Failure(ex), exception_processor_index + ), ) if isinstance(exception_result, Failure): raise + assert isinstance(recover_to, MutableAsyncChain) recover_to.extend(exception_result) if isinstance(iterable, AsyncIterable): @@ -135,7 +152,7 @@ class SpiderMiddlewareManager(MiddlewareManager): spider: Spider, _failure: Failure, start_index: int = 0, - ) -> Union[Failure, MutableChain]: + ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]: exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): @@ -151,14 +168,18 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned - dfd: Deferred = self._process_spider_output( - response, spider, result, method_index + 1 + dfd: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = ( + self._process_spider_output( + response, spider, result, method_index + 1 + ) ) # _process_spider_output() returns a Deferred only because of downgrading so this can be # simplified when downgrading is removed. if dfd.called: # the result is available immediately if _process_spider_output didn't do downgrading - return cast(MutableChain, dfd.result) + return cast( + Union[MutableChain[_T], MutableAsyncChain[_T]], dfd.result + ) # we forbid waiting here because otherwise we would need to return a deferred from # _process_spider_exception too, which complicates the architecture msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" @@ -181,12 +202,12 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - result: Union[Iterable, AsyncIterable], + result: Union[Iterable[_T], AsyncIterable[_T]], start_index: int = 0, - ) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]: + ) -> Generator[Deferred[Any], Any, Union[MutableChain[_T], MutableAsyncChain[_T]]]: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - recovered: Union[MutableChain, MutableAsyncChain] + recovered: Union[MutableChain[_T], MutableAsyncChain[_T]] last_result_is_async = isinstance(result, AsyncIterable) if last_result_is_async: recovered = MutableAsyncChain() @@ -237,7 +258,9 @@ class SpiderMiddlewareManager(MiddlewareManager): # might fail directly if the output value is not a generator result = method(response=response, result=result, spider=spider) except Exception as ex: - exception_result = self._process_spider_exception( + exception_result: Union[ + Failure, MutableChain[_T], MutableAsyncChain[_T] + ] = self._process_spider_exception( response, spider, Failure(ex), method_index + 1 ) if isinstance(exception_result, Failure): @@ -267,16 +290,22 @@ class SpiderMiddlewareManager(MiddlewareManager): return MutableChain(result, recovered) # type: ignore[arg-type] async def _process_callback_output( - self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable] - ) -> Union[MutableChain, MutableAsyncChain]: - recovered: Union[MutableChain, MutableAsyncChain] + self, + response: Response, + spider: Spider, + result: Union[Iterable[_T], AsyncIterable[_T]], + ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]: + recovered: Union[MutableChain[_T], MutableAsyncChain[_T]] if isinstance(result, AsyncIterable): recovered = MutableAsyncChain() else: recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) result = await maybe_deferred_to_future( - self._process_spider_output(response, spider, result) + cast( + "Deferred[Union[Iterable[_T], AsyncIterable[_T]]]", + self._process_spider_output(response, spider, result), + ) ) if isinstance(result, AsyncIterable): return MutableAsyncChain(result, recovered) @@ -291,25 +320,29 @@ class SpiderMiddlewareManager(MiddlewareManager): response: Response, request: Request, spider: Spider, - ) -> Deferred: + ) -> Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]]: async def process_callback_output( - result: Union[Iterable, AsyncIterable] - ) -> Union[MutableChain, MutableAsyncChain]: + result: Union[Iterable[_T], AsyncIterable[_T]] + ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]: return await self._process_callback_output(response, spider, result) - def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]: + def process_spider_exception( + _failure: Failure, + ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]: return self._process_spider_exception(response, spider, _failure) - dfd = mustbe_deferred( + dfd: Deferred[Union[Iterable[_T], AsyncIterable[_T]]] = mustbe_deferred( self._process_spider_input, scrape_func, response, request, spider ) - dfd.addCallback(deferred_f_from_coro_f(process_callback_output)) - dfd.addErrback(process_spider_exception) - return dfd + dfd2: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = ( + dfd.addCallback(deferred_f_from_coro_f(process_callback_output)) + ) + dfd2.addErrback(process_spider_exception) + return dfd2 def process_start_requests( self, start_requests: Iterable[Request], spider: Spider - ) -> Deferred: + ) -> Deferred[Iterable[Request]]: return self._process_chain("process_start_requests", start_requests, spider) # This method is only needed until _async compatibility methods are removed. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ccfe78891..ecb0a8150 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,18 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Generator, + Optional, + Set, + Type, + TypeVar, + Union, + cast, +) from twisted.internet.defer import ( Deferred, @@ -12,13 +23,6 @@ from twisted.internet.defer import ( inlineCallbacks, maybeDeferred, ) - -try: - # zope >= 5.0 only supports MultipleInvalid - from zope.interface.exceptions import MultipleInvalid -except ImportError: - MultipleInvalid = None - from zope.interface.verify import verifyClass from scrapy import Spider, signals @@ -54,6 +58,8 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_T = TypeVar("_T") + class Crawler: def __init__( @@ -129,6 +135,8 @@ class Crawler: if is_asyncio_reactor_installed() and event_loop: verify_installed_asyncio_event_loop(event_loop) + log_reactor_info() + self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() @@ -138,7 +146,7 @@ class Crawler: ) @inlineCallbacks - def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]: + def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred[Any], Any, None]: if self.crawling: raise RuntimeError("Crawling already taking place") if self._started: @@ -170,7 +178,7 @@ class Crawler: return ExecutionEngine(self, lambda _: self.stop()) @inlineCallbacks - def stop(self) -> Generator[Deferred, Any, None]: + def stop(self) -> Generator[Deferred[Any], Any, None]: """Starts a graceful stop of the crawler and returns a deferred that is fired when the crawler is stopped.""" if self.crawling: @@ -254,7 +262,7 @@ class CrawlerRunner: self.settings = settings self.spider_loader = self._get_spider_loader(settings) self._crawlers: Set[Crawler] = set() - self._active: Set[Deferred] = set() + self._active: Set[Deferred[None]] = set() self.bootstrap_failed = False def crawl( @@ -262,7 +270,7 @@ class CrawlerRunner: crawler_or_spidercls: Union[Type[Spider], str, Crawler], *args: Any, **kwargs: Any, - ) -> Deferred: + ) -> Deferred[None]: """ Run a crawler with the provided arguments. @@ -292,12 +300,12 @@ class CrawlerRunner: crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) - def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred: + def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred[None]: self.crawlers.add(crawler) d = crawler.crawl(*args, **kwargs) self._active.add(d) - def _done(result: Any) -> Any: + def _done(result: _T) -> _T: self.crawlers.discard(crawler) self._active.discard(d) self.bootstrap_failed |= not getattr(crawler, "spider", None) @@ -333,7 +341,7 @@ class CrawlerRunner: # temporary cast until self.spider_loader is typed return Crawler(cast(Type[Spider], spidercls), self.settings) - def stop(self) -> Deferred: + def stop(self) -> Deferred[Any]: """ Stops simultaneously all the crawling jobs taking place. @@ -342,7 +350,7 @@ class CrawlerRunner: return DeferredList([c.stop() for c in list(self.crawlers)]) @inlineCallbacks - def join(self) -> Generator[Deferred, Any, None]: + def join(self) -> Generator[Deferred[Any], Any, None]: """ join() @@ -458,7 +466,7 @@ class CrawlerProcess(CrawlerRunner): ) reactor.run(installSignalHandlers=install_signal_handlers) # blocking call - def _graceful_stop_reactor(self) -> Deferred: + def _graceful_stop_reactor(self) -> Deferred[Any]: d = self.stop() d.addBoth(self._stop_reactor) return d diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 0e757e4be..5fc7f31a3 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -6,16 +6,18 @@ from typing import TYPE_CHECKING, Union from w3lib import html -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import HtmlResponse, Response -from scrapy.settings import BaseSettings if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 6ada3b474..23140d263 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -2,22 +2,10 @@ from __future__ import annotations import logging from collections import defaultdict -from http.cookiejar import Cookie -from typing import ( - TYPE_CHECKING, - Any, - DefaultDict, - Dict, - Iterable, - Optional, - Sequence, - Union, -) +from typing import TYPE_CHECKING, Any, DefaultDict, Iterable, Optional, Sequence, Union from tldextract import TLDExtract -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar @@ -25,9 +13,15 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from http.cookiejar import Cookie + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http.request import VerboseCookie + logger = logging.getLogger(__name__) @@ -128,7 +122,7 @@ class CookiesMiddleware: msg = f"Received cookies from: {response}\n{cookies}" logger.debug(msg, extra={"spider": spider}) - def _format_cookie(self, cookie: Dict[str, Any], request: Request) -> Optional[str]: + def _format_cookie(self, cookie: VerboseCookie, request: Request) -> Optional[str]: """ Given a dict consisting of cookie components, return its string representation. Decode from bytes if necessary. @@ -142,18 +136,19 @@ class CookiesMiddleware: logger.warning(msg) return None continue - if isinstance(cookie[key], (bool, float, int, str)): - decoded[key] = str(cookie[key]) + # https://github.com/python/mypy/issues/7178, https://github.com/python/mypy/issues/9168 + if isinstance(cookie[key], (bool, float, int, str)): # type: ignore[literal-required] + decoded[key] = str(cookie[key]) # type: ignore[literal-required] else: try: - decoded[key] = cookie[key].decode("utf8") + decoded[key] = cookie[key].decode("utf8") # type: ignore[literal-required] except UnicodeDecodeError: logger.warning( "Non UTF-8 encoded cookie found in request %s: %s", request, cookie, ) - decoded[key] = cookie[key].decode("latin1", errors="replace") + decoded[key] = cookie[key].decode("latin1", errors="replace") # type: ignore[literal-required] for flag in ("secure",): value = cookie.get(flag, _UNSET) if value is _UNSET or not value: @@ -174,7 +169,7 @@ class CookiesMiddleware: """ if not request.cookies: return [] - cookies: Iterable[Dict[str, Any]] + cookies: Iterable[VerboseCookie] if isinstance(request.cookies, dict): cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items()) else: diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 58fd415b9..49b9fdc05 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -8,15 +8,16 @@ from __future__ import annotations from typing import TYPE_CHECKING, Iterable, Tuple, Union -from scrapy import Request, Spider -from scrapy.crawler import Crawler -from scrapy.http import Response from scrapy.utils.python import without_none_values if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + class DefaultHeadersMiddleware: def __init__(self, headers: Iterable[Tuple[str, str]]): diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index fd7c03a38..ee7a24825 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -9,13 +9,14 @@ from __future__ import annotations from typing import TYPE_CHECKING, Union from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler -from scrapy.http import Response if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + class DownloadTimeoutMiddleware: def __init__(self, timeout: float = 180): diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 63490a37a..39165e155 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -11,14 +11,15 @@ from typing import TYPE_CHECKING, Union from w3lib.http import basic_auth_header from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler -from scrapy.http import Response from scrapy.utils.url import url_is_from_any_domain if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + class HttpAuthMiddleware: """Set Basic HTTP Authorization header diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 971473403..8377a3c1d 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -16,19 +16,20 @@ from twisted.internet.error import ( from twisted.web.client import ResponseFailed from scrapy import signals -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured -from scrapy.http.request import Request -from scrapy.http.response import Response -from scrapy.settings import Settings -from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector from scrapy.utils.misc import load_object if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http.request import Request + from scrapy.http.response import Response + from scrapy.settings import Settings + from scrapy.spiders import Spider + from scrapy.statscollectors import StatsCollector + class HttpCacheMiddleware: DOWNLOAD_EXCEPTIONS = ( diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 0e5e215ac..6b0a56f7f 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -3,14 +3,12 @@ from __future__ import annotations import warnings from itertools import chain from logging import getLogger -from typing import TYPE_CHECKING, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.statscollectors import StatsCollector from scrapy.utils._compression import ( _DecompressionMaxSizeExceeded, _inflate, @@ -24,6 +22,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = getLogger(__name__) ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"] @@ -138,31 +140,35 @@ class HttpCompressionMiddleware: respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) - kwargs = {"cls": respcls, "body": decoded_body} + kwargs: Dict[str, Any] = {"body": decoded_body} if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the # responsetypes guessing is reliable kwargs["encoding"] = None - response = response.replace(**kwargs) + response = response.replace(cls=respcls, **kwargs) if not content_encoding: del response.headers["Content-Encoding"] return response - def _handle_encoding(self, body, content_encoding, max_size): + def _handle_encoding( + self, body: bytes, content_encoding: List[bytes], max_size: int + ) -> Tuple[bytes, List[bytes]]: to_decode, to_keep = self._split_encodings(content_encoding) for encoding in to_decode: body = self._decode(body, encoding, max_size) return body, to_keep - def _split_encodings(self, content_encoding): - to_keep = [ + def _split_encodings( + self, content_encoding: List[bytes] + ) -> Tuple[List[bytes], List[bytes]]: + to_keep: List[bytes] = [ encoding.strip().lower() for encoding in chain.from_iterable( encodings.split(b",") for encodings in content_encoding ) ] - to_decode = [] + to_decode: List[bytes] = [] while to_keep: encoding = to_keep.pop() if encoding not in ACCEPTED_ENCODINGS: diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 5b56ad449..a7af83f7d 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -9,10 +9,7 @@ from urllib.request import ( # type: ignore[attr-defined] proxy_bypass, ) -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes @@ -20,6 +17,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + class HttpProxyMiddleware: def __init__(self, auth_encoding: Optional[str] = "latin-1"): diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index 1e5026925..6f67e3975 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -1,33 +1,45 @@ +from __future__ import annotations + import logging import re import warnings +from typing import TYPE_CHECKING, Set -from scrapy import signals +from scrapy import Request, Spider, signals from scrapy.exceptions import IgnoreRequest from scrapy.utils.httpobj import urlparse_cached +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) class OffsiteMiddleware: @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: + assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled) return o - def __init__(self, stats): + def __init__(self, stats: StatsCollector): self.stats = stats - self.domains_seen = set() + self.domains_seen: Set[str] = set() - def spider_opened(self, spider): - self.host_regex = self.get_host_regex(spider) + def spider_opened(self, spider: Spider) -> None: + self.host_regex: re.Pattern[str] = self.get_host_regex(spider) - def request_scheduled(self, request, spider): + def request_scheduled(self, request: Request, spider: Spider) -> None: self.process_request(request, spider) - def process_request(self, request, spider): + def process_request(self, request: Request, spider: Spider) -> None: if request.dont_filter or self.should_follow(request, spider): return None domain = urlparse_cached(request).hostname @@ -42,13 +54,13 @@ class OffsiteMiddleware: self.stats.inc_value("offsite/filtered", spider=spider) raise IgnoreRequest - def should_follow(self, request, spider): + def should_follow(self, request: Request, spider: Spider) -> bool: regex = self.host_regex # hostname can be None for wrong urls (like javascript links) host = urlparse_cached(request).hostname or "" return bool(regex.search(host)) - def get_host_regex(self, spider): + def get_host_regex(self, spider: Spider) -> re.Pattern[str]: """Override this method to implement a different offsite policy""" allowed_domains = getattr(spider, "allowed_domains", None) if not allowed_domains: diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index aa08827c4..53081237c 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -6,11 +6,8 @@ from urllib.parse import urljoin from w3lib.url import safe_url_string -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import HtmlResponse, Response -from scrapy.settings import BaseSettings from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.response import get_meta_refresh @@ -18,6 +15,11 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) @@ -27,6 +29,7 @@ def _build_redirect_request( redirect_request = source_request.replace( url=url, **kwargs, + cls=None, cookies=None, ) if "_scheme_proxy" in redirect_request.meta: diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 46587a898..8d7b7293c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -16,12 +16,8 @@ import warnings from logging import Logger, getLogger from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.http import Response -from scrapy.http.request import Request from scrapy.settings import BaseSettings, Settings -from scrapy.spiders import Spider from scrapy.utils.misc import load_object from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message @@ -30,6 +26,12 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.http.request import Request + from scrapy.spiders import Spider + + retry_logger = getLogger(__name__) @@ -147,9 +149,7 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): if not settings.getbool("RETRY_ENABLED"): raise NotConfigured self.max_retry_times = settings.getint("RETRY_TIMES") - self.retry_http_codes = set( - int(x) for x in settings.getlist("RETRY_HTTP_CODES") - ) + self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")} self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") try: diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 6a0ecb7bf..73757162f 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -7,28 +7,32 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Dict, Optional, TypeVar, Union from twisted.internet.defer import Deferred, maybeDeferred -from twisted.python.failure import Failure -from scrapy import Spider -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.robotstxt import RobotParser from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import load_object if TYPE_CHECKING: + from twisted.python.failure import Failure + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.robotstxt import RobotParser + logger = logging.getLogger(__name__) +_T = TypeVar("_T") + class RobotsTxtMiddleware: DOWNLOAD_PRIORITY: int = 1000 @@ -41,7 +45,9 @@ class RobotsTxtMiddleware: "ROBOTSTXT_USER_AGENT", None ) self.crawler: Crawler = crawler - self._parsers: Dict[str, Union[RobotParser, Deferred, None]] = {} + self._parsers: Dict[ + str, Union[RobotParser, Deferred[Optional[RobotParser]], None] + ] = {} self._parserimpl: RobotParser = load_object( crawler.settings.get("ROBOTSTXT_PARSER") ) @@ -53,14 +59,18 @@ class RobotsTxtMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]: + def process_request( + self, request: Request, spider: Spider + ) -> Optional[Deferred[None]]: if request.meta.get("dont_obey_robotstxt"): return None if request.url.startswith("data:") or request.url.startswith("file:"): return None - d: Deferred = maybeDeferred(self.robot_parser, request, spider) - d.addCallback(self.process_request_2, request, spider) - return d + d: Deferred[Optional[RobotParser]] = maybeDeferred( + self.robot_parser, request, spider # type: ignore[arg-type] + ) + d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider) + return d2 def process_request_2( self, rp: Optional[RobotParser], request: Request, spider: Spider @@ -84,7 +94,7 @@ class RobotsTxtMiddleware: def robot_parser( self, request: Request, spider: Spider - ) -> Union[RobotParser, Deferred, None]: + ) -> Union[RobotParser, Deferred[Optional[RobotParser]], None]: url = urlparse_cached(request) netloc = url.netloc @@ -107,9 +117,9 @@ class RobotsTxtMiddleware: parser = self._parsers[netloc] if isinstance(parser, Deferred): - d: Deferred = Deferred() + d: Deferred[Optional[RobotParser]] = Deferred() - def cb(result: Any) -> Any: + def cb(result: Optional[RobotParser]) -> Optional[RobotParser]: d.callback(result) return result diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index df30e8ca4..0faae7b5a 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,14 +1,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, Union +from typing import TYPE_CHECKING, Dict, List, Tuple, Union from twisted.web import http -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response -from scrapy.statscollectors import StatsCollector from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.request import request_httprepr @@ -16,8 +12,15 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.statscollectors import StatsCollector -def get_header_size(headers: Dict[str, Union[list, tuple]]) -> int: + +def get_header_size( + headers: Dict[str, Union[List[Union[str, bytes]], Tuple[Union[str, bytes], ...]]] +) -> int: size = 0 for key, value in headers.items(): if isinstance(value, (list, tuple)): diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index 92f1ec897..109f1a4d9 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -5,13 +5,14 @@ from __future__ import annotations from typing import TYPE_CHECKING, Union from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler -from scrapy.http import Response if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + class UserAgentMiddleware: """This middleware allows spiders to override the user_agent""" diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index dd2420e98..40ea48510 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -4,11 +4,6 @@ import logging from pathlib import Path from typing import TYPE_CHECKING, Optional, Set -from twisted.internet.defer import Deferred - -from scrapy.http.request import Request -from scrapy.settings import BaseSettings -from scrapy.spiders import Spider from scrapy.utils.job import job_dir from scrapy.utils.request import ( RequestFingerprinter, @@ -17,10 +12,15 @@ from scrapy.utils.request import ( ) if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import Request + from scrapy.settings import BaseSettings + from scrapy.spiders import Spider class BaseDupeFilter: @@ -31,10 +31,10 @@ class BaseDupeFilter: def request_seen(self, request: Request) -> bool: return False - def open(self) -> Optional[Deferred]: + def open(self) -> Optional[Deferred[None]]: pass - def close(self, reason: str) -> Optional[Deferred]: + def close(self, reason: str) -> Optional[Deferred[None]]: pass def log(self, request: Request, spider: Spider) -> None: diff --git a/scrapy/extension.py b/scrapy/extension.py index 8221b675e..8c81ab356 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -4,12 +4,16 @@ The Extension Manager See documentation in docs/topics/extensions.rst """ -from typing import Any, List +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, List from scrapy.middleware import MiddlewareManager -from scrapy.settings import Settings from scrapy.utils.conf import build_component_list +if TYPE_CHECKING: + from scrapy.settings import Settings + class ExtensionManager(MiddlewareManager): component_name = "extension" diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 812b3553c..cef5527b7 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -10,17 +10,19 @@ import logging from collections import defaultdict from typing import TYPE_CHECKING, Any, DefaultDict, Dict -from twisted.python.failure import Failure - from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response if TYPE_CHECKING: + from twisted.python.failure import Failure + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + + logger = logging.getLogger(__name__) @@ -34,6 +36,9 @@ class CloseSpider: "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"), "errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"), "timeout_no_item": crawler.settings.getint("CLOSESPIDER_TIMEOUT_NO_ITEM"), + "pagecount_no_item": crawler.settings.getint( + "CLOSESPIDER_PAGECOUNT_NO_ITEM" + ), } if not any(self.close_on.values()): @@ -43,11 +48,11 @@ class CloseSpider: if self.close_on.get("errorcount"): crawler.signals.connect(self.error_count, signal=signals.spider_error) - if self.close_on.get("pagecount"): + if self.close_on.get("pagecount") or self.close_on.get("pagecount_no_item"): crawler.signals.connect(self.page_count, signal=signals.response_received) if self.close_on.get("timeout"): crawler.signals.connect(self.spider_opened, signal=signals.spider_opened) - if self.close_on.get("itemcount"): + if self.close_on.get("itemcount") or self.close_on.get("pagecount_no_item"): crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) if self.close_on.get("timeout_no_item"): self.timeout_no_item: int = self.close_on["timeout_no_item"] @@ -58,6 +63,7 @@ class CloseSpider: crawler.signals.connect( self.item_scraped_no_item, signal=signals.item_scraped ) + crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @classmethod @@ -72,9 +78,17 @@ class CloseSpider: def page_count(self, response: Response, request: Request, spider: Spider) -> None: self.counter["pagecount"] += 1 + self.counter["pagecount_since_last_item"] += 1 if self.counter["pagecount"] == self.close_on["pagecount"]: assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_pagecount") + return + if self.close_on["pagecount_no_item"] and ( + self.counter["pagecount_since_last_item"] + >= self.close_on["pagecount_no_item"] + ): + assert self.crawler.engine + self.crawler.engine.close_spider(spider, "closespider_pagecount_no_item") def spider_opened(self, spider: Spider) -> None: from twisted.internet import reactor @@ -89,6 +103,7 @@ class CloseSpider: def item_scraped(self, item: Any, spider: Spider) -> None: self.counter["itemcount"] += 1 + self.counter["pagecount_since_last_item"] = 0 if self.counter["itemcount"] == self.close_on["itemcount"]: assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_itemcount") diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index f3ac19623..6ef2d0382 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -8,13 +8,14 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, Optional from scrapy import Spider, signals -from scrapy.crawler import Crawler -from scrapy.statscollectors import StatsCollector if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + class CoreStats: def __init__(self, stats: StatsCollector): diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 26726b662..c54871e02 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -12,17 +12,20 @@ import sys import threading import traceback from pdb import Pdb -from types import FrameType from typing import TYPE_CHECKING, Optional -from scrapy.crawler import Crawler from scrapy.utils.engine import format_engine_status from scrapy.utils.trackref import format_live_refs if TYPE_CHECKING: + from types import FrameType + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) @@ -55,7 +58,7 @@ class StackTraceDump: ) def _thread_stacks(self) -> str: - id2name = dict((th.ident, th.name) for th in threading.enumerate()) + id2name = {th.ident: th.name for th in threading.enumerate()} dumps = "" for id_, frame in sys._current_frames().items(): name = id2name.get(id_, "") @@ -74,4 +77,4 @@ class Debugger: def _enter_debugger(self, signum: int, frame: Optional[FrameType]) -> None: assert frame - Pdb().set_trace(frame.f_back) + Pdb().set_trace(frame.f_back) # noqa: T100 diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 97f39afe7..0d7f5bfd4 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -31,18 +31,15 @@ from typing import ( ) from urllib.parse import unquote, urlparse -from twisted.internet import threads from twisted.internet.defer import Deferred, DeferredList, maybeDeferred -from twisted.python.failure import Failure +from twisted.internet.threads import deferToThread from w3lib.url import file_uri_to_path from zope.interface import Interface, implementer from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.exporters import BaseItemExporter from scrapy.extensions.postprocessing import PostProcessingManager -from scrapy.settings import BaseSettings, Settings +from scrapy.settings import Settings from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.defer import maybe_deferred_to_future @@ -54,11 +51,14 @@ from scrapy.utils.python import without_none_values if TYPE_CHECKING: from _typeshed import OpenBinaryMode + from twisted.python.failure import Failure # typing.Self requires Python 3.11 from typing_extensions import Self -logger = logging.getLogger(__name__) + from scrapy.crawler import Crawler + from scrapy.exporters import BaseItemExporter + from scrapy.settings import BaseSettings try: import boto3 # noqa: F401 @@ -67,6 +67,9 @@ try: except ImportError: IS_BOTO3_AVAILABLE = False + +logger = logging.getLogger(__name__) + UriParamsCallableT = Callable[[Dict[str, Any], Spider], Optional[Dict[str, Any]]] _StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol") @@ -104,7 +107,7 @@ class ItemFilter: for item_class in feed_options.get("item_classes") or () ) else: - self.item_classes = tuple() + self.item_classes = () def accepts(self, item: Any) -> bool: """ @@ -146,7 +149,7 @@ class FeedStorageProtocol(Protocol): """Open the storage for the given spider. It must return a file-like object that will be used for the exporters""" - def store(self, file: IO[bytes]) -> Optional[Deferred]: + def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: """Store the given file stream""" @@ -159,8 +162,8 @@ class BlockingFeedStorage: return NamedTemporaryFile(prefix="feed-", dir=path) - def store(self, file: IO[bytes]) -> Optional[Deferred]: - return threads.deferToThread(self._store_in_thread, file) + def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: + return deferToThread(self._store_in_thread, file) def _store_in_thread(self, file: IO[bytes]) -> None: raise NotImplementedError @@ -189,7 +192,7 @@ class StdoutFeedStorage: def open(self, spider: Spider) -> IO[bytes]: return self._stdout - def store(self, file: IO[bytes]) -> Optional[Deferred]: + def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: pass @@ -208,7 +211,7 @@ class FileFeedStorage: dirname.mkdir(parents=True) return Path(self.path).open(self.write_mode) - def store(self, file: IO[bytes]) -> Optional[Deferred]: + def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: file.close() return None @@ -238,13 +241,16 @@ class S3FeedStorage(BlockingFeedStorage): self.acl: Optional[str] = acl self.endpoint_url: Optional[str] = endpoint_url self.region_name: Optional[str] = region_name + # It can be either botocore.client.BaseClient or mypy_boto3_s3.S3Client, + # there seems to be no good way to infer it statically. + self.s3_client: Any if IS_BOTO3_AVAILABLE: import boto3.session - session = boto3.session.Session() + boto3_session = boto3.session.Session() - self.s3_client = session.client( + self.s3_client = boto3_session.client( "s3", aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key, @@ -261,9 +267,9 @@ class S3FeedStorage(BlockingFeedStorage): import botocore.session - session = botocore.session.get_session() + botocore_session = botocore.session.get_session() - self.s3_client = session.create_client( + self.s3_client = botocore_session.create_client( "s3", aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key, @@ -477,7 +483,7 @@ _FeedSlot = create_deprecated_class( class FeedExporter: - _pending_deferreds: List[Deferred] = [] + _pending_deferreds: List[Deferred[None]] = [] @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -564,7 +570,7 @@ class FeedExporter: self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed) ) - def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred]: + def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred[None]]: def get_file(slot_: FeedSlot) -> IO[bytes]: assert slot_.file if isinstance(slot_.file, PostProcessingManager): @@ -584,7 +590,7 @@ class FeedExporter: return None logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" - d: Deferred = maybeDeferred(slot.storage.store, get_file(slot)) + d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[arg-type] d.addCallback( self._handle_store_success, logmsg, spider, type(slot.storage).__name__ @@ -615,7 +621,7 @@ class FeedExporter: self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") def _handle_store_success( - self, f: Failure, logmsg: str, spider: Spider, slot_type: str + self, result: Any, logmsg: str, spider: Spider, slot_type: str ) -> None: logger.info("Stored %s", logmsg, extra={"spider": spider}) assert self.crawler.stats @@ -691,7 +697,9 @@ class FeedExporter: self.slots = slots def _load_components(self, setting_prefix: str) -> Dict[str, Any]: - conf = without_none_values(self.settings.getwithbase(setting_prefix)) + conf = without_none_values( + cast(Dict[str, str], self.settings.getwithbase(setting_prefix)) + ) d = {} for k, v in conf.items(): try: diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index dd5bce24f..448d5f1ab 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import gzip import logging import os @@ -13,10 +15,7 @@ from weakref import WeakKeyDictionary from w3lib.http import headers_dict_to_raw, headers_raw_to_dict from scrapy.http import Headers, Response -from scrapy.http.request import Request from scrapy.responsetypes import responsetypes -from scrapy.settings import BaseSettings -from scrapy.spiders import Spider from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode @@ -26,6 +25,10 @@ if TYPE_CHECKING: # typing.Concatenate requires Python 3.10 from typing_extensions import Concatenate + from scrapy.http.request import Request + from scrapy.settings import BaseSettings + from scrapy.spiders import Spider + logger = logging.getLogger(__name__) @@ -315,7 +318,9 @@ class FilesystemCacheStorage: self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP") # https://github.com/python/mypy/issues/10740 - self._open: Callable[Concatenate[Union[str, os.PathLike], str, ...], IO] = ( + self._open: Callable[ + Concatenate[Union[str, os.PathLike], str, ...], IO[bytes] + ] = ( gzip.open if self.use_gzip else open # type: ignore[assignment] ) diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 2388afa75..c4f43482d 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -6,14 +6,16 @@ from typing import TYPE_CHECKING, Optional, Tuple, Union from twisted.internet import task from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.statscollectors import StatsCollector if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index f304e1bf2..3cbbb64e5 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -10,15 +10,16 @@ import gc from typing import TYPE_CHECKING from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.statscollectors import StatsCollector from scrapy.utils.trackref import live_refs if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + class MemoryDebugger: def __init__(self, stats: StatsCollector): diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 9de06b24d..25f63ecc6 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -16,7 +16,6 @@ from typing import TYPE_CHECKING, List from twisted.internet import task from scrapy import signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender from scrapy.utils.engine import get_engine_status @@ -25,6 +24,9 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 9567f948a..80c0a3b26 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -8,15 +8,17 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from twisted.internet import task from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.statscollectors import StatsCollector from scrapy.utils.serialize import ScrapyJSONEncoder if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index c6eb20277..567efd7a1 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -5,7 +5,6 @@ from pathlib import Path from typing import TYPE_CHECKING, Optional from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir @@ -13,6 +12,8 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + class SpiderState: """Store and load spider state during a scraping job""" diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 20b8f910c..cad607514 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -8,18 +8,19 @@ from __future__ import annotations from typing import TYPE_CHECKING, List, Optional -from twisted.internet.defer import Deferred - from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender -from scrapy.statscollectors import StatsCollector if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + class StatsMailer: def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender): @@ -38,7 +39,7 @@ class StatsMailer: crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_closed(self, spider: Spider) -> Optional[Deferred]: + def spider_closed(self, spider: Spider) -> Optional[Deferred[None]]: spider_stats = self.stats.get_stats(spider) body = "Global stats\n\n" body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items()) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 00c69434c..c64a0b417 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -10,23 +10,12 @@ import binascii import logging import os import pprint -import traceback from typing import TYPE_CHECKING, Any, Dict, List from twisted.internet import protocol from twisted.internet.tcp import Port -try: - from twisted.conch import manhole, telnet - from twisted.conch.insults import insults - - TWISTED_CONCH_AVAILABLE = True -except (ImportError, SyntaxError): - _TWISTED_CONCH_TRACEBACK = traceback.format_exc() - TWISTED_CONCH_AVAILABLE = False - from scrapy import signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.utils.decorators import defers from scrapy.utils.engine import print_engine_status @@ -34,8 +23,14 @@ from scrapy.utils.reactor import listen_tcp from scrapy.utils.trackref import print_live_refs if TYPE_CHECKING: + from twisted.conch import telnet + # typing.Self requires Python 3.11 from typing_extensions import Self + + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) # signal to update telnet variables @@ -47,11 +42,7 @@ class TelnetConsole(protocol.ServerFactory): def __init__(self, crawler: Crawler): if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"): raise NotConfigured - if not TWISTED_CONCH_AVAILABLE: - raise NotConfigured( - "TELNETCONSOLE_ENABLED setting is True but required twisted " - "modules failed to import:\n" + _TWISTED_CONCH_TRACEBACK - ) + self.crawler: Crawler = crawler self.noisy: bool = False self.portrange: List[int] = [ @@ -85,6 +76,10 @@ class TelnetConsole(protocol.ServerFactory): self.port.stopListening() def protocol(self) -> telnet.TelnetTransport: # type: ignore[override] + # these import twisted.internet.reactor + from twisted.conch import manhole, telnet + from twisted.conch.insults import insults + class Portal: """An implementation of IPortal""" diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 217e61a81..6ce9ce63a 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -4,15 +4,17 @@ import logging from typing import TYPE_CHECKING, Optional, Tuple from scrapy import Request, Spider, signals -from scrapy.core.downloader import Slot -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.core.downloader import Slot + from scrapy.crawler import Crawler + from scrapy.http import Response + + logger = logging.getLogger(__name__) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 8af89c74f..cc88a9420 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -17,8 +17,6 @@ from typing import ( cast, ) -from scrapy import Request -from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode @@ -26,6 +24,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request + from scrapy.http import Response + + # Defined in the http.cookiejar module, but undocumented: # https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527 IPV4_RE = re.compile(r"\.\d+$", re.ASCII) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 73aee7178..85b9229d3 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -118,8 +118,7 @@ class Headers(CaselessDict): ] def to_string(self) -> bytes: - # cast() can be removed if the headers_dict_to_raw() hint is improved - return cast(bytes, headers_dict_to_raw(self)) + return headers_dict_to_raw(self) def to_unicode_dict(self) -> CaseInsensitiveDict: """Return headers as a CaseInsensitiveDict with str keys diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 191b3cef4..9381a6cb3 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -12,7 +12,6 @@ from typing import ( TYPE_CHECKING, Any, AnyStr, - Callable, Dict, Iterable, List, @@ -20,8 +19,11 @@ from typing import ( NoReturn, Optional, Tuple, + Type, + TypedDict, + TypeVar, Union, - cast, + overload, ) from w3lib.url import safe_url_string @@ -34,8 +36,31 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax if TYPE_CHECKING: - # typing.Self requires Python 3.11 - from typing_extensions import Self + from collections.abc import Callable + + from twisted.python.failure import Failure + + # typing.Concatenate requires Python 3.10 + # typing.NotRequired and typing.Self require Python 3.11 + from typing_extensions import Concatenate, NotRequired, Self + + from scrapy.http import Response + + CallbackT = Callable[Concatenate[Response, ...], Any] + + +class VerboseCookie(TypedDict): + name: str + value: str + domain: NotRequired[str] + path: NotRequired[str] + secure: NotRequired[bool] + + +CookiesT = Union[Dict[str, str], List[VerboseCookie]] + + +RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn: @@ -93,16 +118,16 @@ class Request(object_ref): def __init__( self, url: str, - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: str = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, flags: Optional[List[str]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, ) -> None: @@ -120,10 +145,10 @@ class Request(object_ref): ) if not (callable(errback) or errback is None): raise TypeError(f"errback must be a callable, got {type(errback).__name__}") - self.callback: Optional[Callable] = callback - self.errback: Optional[Callable] = errback + self.callback: Optional[CallbackT] = callback + self.errback: Optional[Callable[[Failure], Any]] = errback - self.cookies: Union[dict, List[dict]] = cookies or {} + self.cookies: CookiesT = cookies or {} self.headers: Headers = Headers(headers or {}, encoding=encoding) self.dont_filter: bool = dont_filter @@ -177,15 +202,26 @@ class Request(object_ref): def __repr__(self) -> str: return f"<{self.method} {self.url}>" - def copy(self) -> "Request": + def copy(self) -> Self: return self.replace() - def replace(self, *args: Any, **kwargs: Any) -> "Request": + @overload + def replace( + self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any + ) -> RequestTypeVar: ... + + @overload + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... + + def replace( + self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any + ) -> Request: """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) - cls = kwargs.pop("cls", self.__class__) - return cast(Request, cls(*args, **kwargs)) + if cls is None: + cls = self.__class__ + return cls(*args, **kwargs) @classmethod def from_curl( @@ -225,7 +261,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]: + def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. @@ -254,7 +290,7 @@ class Request(object_ref): return d -def _find_method(obj: Any, func: Callable) -> str: +def _find_method(obj: Any, func: Callable[..., Any]) -> str: """Helper function for Request.to_dict""" # Only instance methods contain ``__func__`` if obj and hasattr(func, "__func__"): diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 3206d79cd..a8c242e8b 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -7,7 +7,17 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, + cast, +) from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit from lxml.html import FormElement # nosec @@ -18,16 +28,18 @@ from lxml.html import TextareaElement # nosec from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request -from scrapy.http.response.text import TextResponse from scrapy.utils.python import is_listlike, to_bytes if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.http.response.text import TextResponse -FormdataKVType = Tuple[str, Union[str, Iterable[str]]] -FormdataType = Optional[Union[dict, List[FormdataKVType]]] + +FormdataVType = Union[str, Iterable[str]] +FormdataKVType = Tuple[str, FormdataVType] +FormdataType = Optional[Union[Dict[str, FormdataVType], List[FormdataKVType]]] class FormRequest(Request): @@ -62,7 +74,7 @@ class FormRequest(Request): formid: Optional[str] = None, formnumber: int = 0, formdata: FormdataType = None, - clickdata: Optional[dict] = None, + clickdata: Optional[Dict[str, Union[str, int]]] = None, dont_click: bool = False, formxpath: Optional[str] = None, formcss: Optional[str] = None, @@ -156,7 +168,7 @@ def _get_inputs( form: FormElement, formdata: FormdataType, dont_click: bool, - clickdata: Optional[dict], + clickdata: Optional[Dict[str, Union[str, int]]], ) -> List[FormdataKVType]: """Return a list of key-value pairs for the inputs found in the given form.""" try: @@ -186,10 +198,8 @@ def _get_inputs( if clickable and clickable[0] not in formdata and not clickable[0] is None: values.append(clickable) - if isinstance(formdata, dict): - formdata = formdata.items() # type: ignore[assignment] - - values.extend((k, v) for k, v in formdata if v is not None) + formdata_items = formdata.items() if isinstance(formdata, dict) else formdata + values.extend((k, v) for k, v in formdata_items if v is not None) return values @@ -216,7 +226,7 @@ def _select_value( def _get_clickable( - clickdata: Optional[dict], form: FormElement + clickdata: Optional[Dict[str, Union[str, int]]], form: FormElement ) -> Optional[Tuple[str, str]]: """ Returns the clickable element specified in clickdata, @@ -243,6 +253,7 @@ def _get_clickable( # because that uniquely identifies the element nr = clickdata.get("nr", None) if nr is not None: + assert isinstance(nr, int) try: el = list(form.inputs)[nr] except IndexError: diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 1dd9e6c87..057a4f897 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -5,27 +5,33 @@ This module implements the JsonRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ +from __future__ import annotations + import copy import json import warnings -from typing import Any, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type, overload -from scrapy.http.request import Request +from scrapy.http.request import Request, RequestTypeVar + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class JsonRequest(Request): attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) def __init__( - self, *args: Any, dumps_kwargs: Optional[dict] = None, **kwargs: Any + self, *args: Any, dumps_kwargs: Optional[Dict[str, Any]] = None, **kwargs: Any ) -> None: dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} dumps_kwargs.setdefault("sort_keys", True) - self._dumps_kwargs = dumps_kwargs + self._dumps_kwargs: Dict[str, Any] = dumps_kwargs body_passed = kwargs.get("body", None) is not None - data = kwargs.pop("data", None) - data_passed = data is not None + data: Any = kwargs.pop("data", None) + data_passed: bool = data is not None if body_passed and data_passed: warnings.warn("Both body and data passed. data will be ignored") @@ -41,21 +47,31 @@ class JsonRequest(Request): ) @property - def dumps_kwargs(self) -> dict: + def dumps_kwargs(self) -> Dict[str, Any]: return self._dumps_kwargs - def replace(self, *args: Any, **kwargs: Any) -> Request: + @overload + def replace( + self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any + ) -> RequestTypeVar: ... + + @overload + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... + + def replace( + self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any + ) -> Request: body_passed = kwargs.get("body", None) is not None - data = kwargs.pop("data", None) - data_passed = data is not None + data: Any = kwargs.pop("data", None) + data_passed: bool = data is not None if body_passed and data_passed: warnings.warn("Both body and data passed. data will be ignored") elif not body_passed and data_passed: kwargs["body"] = self._dumps(data) - return super().replace(*args, **kwargs) + return super().replace(*args, cls=cls, **kwargs) - def _dumps(self, data: dict) -> str: + def _dumps(self, data: Any) -> str: """Convert to JSON""" return json.dumps(data, **self._dumps_kwargs) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index e20e7c438..096ecd370 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -21,7 +21,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): def __init__(self, *args: Any, encoding: Optional[str] = None, **kwargs: Any): if "body" not in kwargs and "params" in kwargs: - kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs) + kw = {k: kwargs.pop(k) for k in DUMPS_ARGS if k in kwargs} kwargs["body"] = xmlrpclib.dumps(**kw) # spec defines that requests must use POST method diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index d73dfce4b..92e4852b6 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -7,26 +7,24 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from ipaddress import IPv4Address, IPv6Address from typing import ( TYPE_CHECKING, Any, AnyStr, Callable, Dict, - Generator, Iterable, List, Mapping, Optional, Tuple, + Type, + TypeVar, Union, - cast, + overload, ) from urllib.parse import urljoin -from twisted.internet.ssl import Certificate - from scrapy.exceptions import NotSupported from scrapy.http.headers import Headers from scrapy.http.request import Request @@ -34,9 +32,21 @@ from scrapy.link import Link from scrapy.utils.trackref import object_ref if TYPE_CHECKING: + from ipaddress import IPv4Address, IPv6Address + + from twisted.internet.ssl import Certificate + from twisted.python.failure import Failure + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.http.request import CallbackT, CookiesT from scrapy.selector import SelectorList +ResponseTypeVar = TypeVar("ResponseTypeVar", bound="Response") + + class Response(object_ref): """An object that represents an HTTP response, which is usually downloaded (by the Downloader) and fed to the Spiders for processing. @@ -133,16 +143,27 @@ class Response(object_ref): def __repr__(self) -> str: return f"<{self.status} {self.url}>" - def copy(self) -> Response: + def copy(self) -> Self: """Return a copy of this Response""" return self.replace() - def replace(self, *args: Any, **kwargs: Any) -> Response: + @overload + def replace( + self, *args: Any, cls: Type[ResponseTypeVar], **kwargs: Any + ) -> ResponseTypeVar: ... + + @overload + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... + + def replace( + self, *args: Any, cls: Optional[Type[Response]] = None, **kwargs: Any + ) -> Response: """Create a new Response with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) - cls = kwargs.pop("cls", self.__class__) - return cast(Response, cls(*args, **kwargs)) + if cls is None: + cls = self.__class__ + return cls(*args, **kwargs) def urljoin(self, url: str) -> str: """Join this Response's url with a possible relative url to form an @@ -177,16 +198,16 @@ class Response(object_ref): def follow( self, url: Union[str, Link], - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, ) -> Request: @@ -230,19 +251,19 @@ class Response(object_ref): def follow_all( self, urls: Iterable[Union[str, Link]], - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, - ) -> Generator[Request, None, None]: + ) -> Iterable[Request]: """ .. versionadded:: 2.0 diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 522ffc0d5..588695002 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -15,7 +15,6 @@ from typing import ( AnyStr, Callable, Dict, - Generator, Iterable, List, Mapping, @@ -36,15 +35,18 @@ from w3lib.encoding import ( ) from w3lib.html import strip_html5_whitespace -from scrapy.http import Request from scrapy.http.response import Response from scrapy.link import Link from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url if TYPE_CHECKING: + from twisted.python.failure import Failure + + from scrapy.http.request import CallbackT, CookiesT, Request from scrapy.selector import Selector, SelectorList + _NONE = object() @@ -179,16 +181,16 @@ class TextResponse(Response): def follow( self, url: Union[str, Link, parsel.Selector], - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, ) -> Request: @@ -232,21 +234,21 @@ class TextResponse(Response): def follow_all( self, urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None, - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, css: Optional[str] = None, xpath: Optional[str] = None, - ) -> Generator[Request, None, None]: + ) -> Iterable[Request]: """ A generator that produces :class:`~.Request` instances to follow all links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s diff --git a/scrapy/item.py b/scrapy/item.py index 2daea64cc..3f93809e7 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -27,7 +27,7 @@ if TYPE_CHECKING: from typing_extensions import Self -class Field(dict): +class Field(Dict[str, Any]): """Container of field metadata""" diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 33a10cd6c..d27a132b3 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -2,10 +2,13 @@ Link extractor based on lxml.html """ +from __future__ import annotations + import logging import operator from functools import partial from typing import ( + TYPE_CHECKING, Any, Callable, Iterable, @@ -20,13 +23,10 @@ from typing import ( from urllib.parse import urljoin, urlparse from lxml import etree # nosec -from lxml.html import HtmlElement # nosec from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string -from scrapy import Selector -from scrapy.http import TextResponse from scrapy.link import Link from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches, re from scrapy.utils.misc import arg_to_iter, rel_has_nofollow @@ -34,6 +34,13 @@ from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain +if TYPE_CHECKING: + from lxml.html import HtmlElement # nosec + + from scrapy import Selector + from scrapy.http import TextResponse + + logger = logging.getLogger(__name__) # from lxml/src/lxml/html/__init__.py diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 529fa279e..9644cc093 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -4,11 +4,18 @@ Item Loader See documentation in docs/topics/loaders.rst """ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional + import itemloaders from scrapy.item import Item from scrapy.selector import Selector +if TYPE_CHECKING: + from scrapy.http import TextResponse + class ItemLoader(itemloaders.ItemLoader): """ @@ -82,7 +89,14 @@ class ItemLoader(itemloaders.ItemLoader): default_item_class: type = Item default_selector_class = Selector - def __init__(self, item=None, selector=None, response=None, parent=None, **context): + def __init__( + self, + item: Any = None, + selector: Optional[Selector] = None, + response: Optional[TextResponse] = None, + parent: Optional[itemloaders.ItemLoader] = None, + **context: Any, + ): if selector is None and response is not None: try: selector = self.default_selector_class(response) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index d720b2f38..601209fb0 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -2,12 +2,13 @@ from __future__ import annotations import logging import os -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, TypedDict, Union from twisted.python.failure import Failure -from scrapy import Request, Spider -from scrapy.http import Response +# working around https://github.com/sphinx-doc/sphinx/issues/10400 +from scrapy import Request, Spider # noqa: TC001 +from scrapy.http import Response # noqa: TC001 from scrapy.utils.request import referer_str if TYPE_CHECKING: @@ -26,6 +27,12 @@ DOWNLOADERRORMSG_SHORT = "Error downloading %(request)s" DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s" +class LogFormatterResult(TypedDict): + level: int + msg: str + args: Union[Dict[str, Any], Tuple[Any, ...]] + + class LogFormatter: """Class for generating log messages for different actions. @@ -64,7 +71,9 @@ class LogFormatter: } """ - def crawled(self, request: Request, response: Response, spider: Spider) -> dict: + def crawled( + self, request: Request, response: Response, spider: Spider + ) -> LogFormatterResult: """Logs a message when the crawler finds a webpage.""" request_flags = f" {str(request.flags)}" if request.flags else "" response_flags = f" {str(response.flags)}" if response.flags else "" @@ -84,7 +93,7 @@ class LogFormatter: def scraped( self, item: Any, response: Union[Response, Failure], spider: Spider - ) -> dict: + ) -> LogFormatterResult: """Logs a message when an item is scraped by a spider.""" src: Any if isinstance(response, Failure): @@ -102,7 +111,7 @@ class LogFormatter: def dropped( self, item: Any, exception: BaseException, response: Response, spider: Spider - ) -> dict: + ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" return { "level": logging.WARNING, @@ -115,7 +124,7 @@ class LogFormatter: def item_error( self, item: Any, exception: BaseException, response: Response, spider: Spider - ) -> dict: + ) -> LogFormatterResult: """Logs a message when an item causes an error while it is passing through the item pipeline. @@ -135,7 +144,7 @@ class LogFormatter: request: Request, response: Union[Response, Failure], spider: Spider, - ) -> dict: + ) -> LogFormatterResult: """Logs an error message from a spider. .. versionadded:: 2.0 @@ -155,7 +164,7 @@ class LogFormatter: request: Request, spider: Spider, errmsg: Optional[str] = None, - ) -> dict: + ) -> LogFormatterResult: """Logs a download error message from a spider (typically coming from the engine). diff --git a/scrapy/mail.py b/scrapy/mail.py index fd6302550..c020732f9 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -30,20 +30,22 @@ from typing import ( from twisted import version as twisted_version from twisted.internet import ssl from twisted.internet.defer import Deferred -from twisted.python.failure import Failure from twisted.python.versions import Version -from scrapy.settings import BaseSettings from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes if TYPE_CHECKING: # imports twisted.internet.reactor from twisted.mail.smtp import ESMTPSenderFactory + from twisted.python.failure import Failure # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) @@ -97,11 +99,11 @@ class MailSender: subject: str, body: str, cc: Union[str, List[str], None] = None, - attachs: Sequence[Tuple[str, str, IO]] = (), + attachs: Sequence[Tuple[str, str, IO[Any]]] = (), mimetype: str = "text/plain", charset: Optional[str] = None, _callback: Optional[Callable[..., None]] = None, - ) -> Optional[Deferred]: + ) -> Optional[Deferred[None]]: from twisted.internet import reactor msg: MIMEBase @@ -153,7 +155,9 @@ class MailSender: ) return None - dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8")) + dfd: Deferred[Any] = self._sendmail( + rcpts, msg.as_string().encode(charset or "utf-8") + ) dfd.addCallback(self._sent_ok, to, cc, subject, len(attachs)) dfd.addErrback(self._sent_failed, to, cc, subject, len(attachs)) reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd) @@ -196,11 +200,11 @@ class MailSender: ) return failure - def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred: + def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred[Any]: from twisted.internet import reactor msg_io = BytesIO(msg) - d: Deferred = Deferred() + d: Deferred[Any] = Deferred() factory = self._create_sender_factory(to_addrs, msg_io, d) @@ -214,7 +218,7 @@ class MailSender: return d def _create_sender_factory( - self, to_addrs: List[str], msg: IO, d: Deferred + self, to_addrs: List[str], msg: IO[bytes], d: Deferred[Any] ) -> ESMTPSenderFactory: from twisted.mail.smtp import ESMTPSenderFactory diff --git a/scrapy/middleware.py b/scrapy/middleware.py index f60c726f9..2296db90e 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -13,27 +13,34 @@ from typing import ( List, Optional, Tuple, + TypeVar, Union, cast, ) -from twisted.internet.defer import Deferred - -from scrapy import Spider from scrapy.exceptions import NotConfigured -from scrapy.settings import Settings from scrapy.utils.defer import process_chain, process_parallel from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object if TYPE_CHECKING: - # typing.Self requires Python 3.11 - from typing_extensions import Self + from twisted.internet.defer import Deferred + # typing.Concatenate and typing.ParamSpec require Python 3.10 + # typing.Self requires Python 3.11 + from typing_extensions import Concatenate, ParamSpec, Self + + from scrapy import Spider from scrapy.crawler import Crawler + from scrapy.settings import Settings + + _P = ParamSpec("_P") logger = logging.getLogger(__name__) +_T = TypeVar("_T") +_T2 = TypeVar("_T2") + class MiddlewareManager: """Base class for implementing middleware managers""" @@ -98,16 +105,22 @@ class MiddlewareManager: if hasattr(mw, "close_spider"): self.methods["close_spider"].appendleft(mw.close_spider) - def _process_parallel(self, methodname: str, obj: Any, *args: Any) -> Deferred: - methods = cast(Iterable[Callable], self.methods[methodname]) + def _process_parallel( + self, methodname: str, obj: _T, *args: Any + ) -> Deferred[List[_T2]]: + methods = cast( + "Iterable[Callable[Concatenate[_T, _P], _T2]]", self.methods[methodname] + ) return process_parallel(methods, obj, *args) - def _process_chain(self, methodname: str, obj: Any, *args: Any) -> Deferred: - methods = cast(Iterable[Callable], self.methods[methodname]) + def _process_chain(self, methodname: str, obj: _T, *args: Any) -> Deferred[_T]: + methods = cast( + "Iterable[Callable[Concatenate[_T, _P], _T]]", self.methods[methodname] + ) return process_chain(methods, obj, *args) - def open_spider(self, spider: Spider) -> Deferred: + def open_spider(self, spider: Spider) -> Deferred[List[None]]: return self._process_parallel("open_spider", spider) - def close_spider(self, spider: Spider) -> Deferred: + def close_spider(self, spider: Spider) -> Deferred[List[None]]: return self._process_parallel("close_spider", spider) diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index f9544d329..480a5a58c 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -4,21 +4,26 @@ Item pipeline See documentation in docs/item-pipeline.rst """ -from typing import Any, List +from __future__ import annotations -from twisted.internet.defer import Deferred +from typing import TYPE_CHECKING, Any, List -from scrapy import Spider from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_f_from_coro_f +if TYPE_CHECKING: + from twisted.internet.defer import Deferred + + from scrapy import Spider + from scrapy.settings import Settings + class ItemPipelineManager(MiddlewareManager): component_name = "item pipeline" @classmethod - def _get_mwlist_from_settings(cls, settings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: return build_component_list(settings.getwithbase("ITEM_PIPELINES")) def _add_middleware(self, pipe: Any) -> None: @@ -28,5 +33,5 @@ class ItemPipelineManager(MiddlewareManager): deferred_f_from_coro_f(pipe.process_item) ) - def process_item(self, item: Any, spider: Spider) -> Deferred: + def process_item(self, item: Any, spider: Spider) -> Deferred[Any]: return self._process_chain("process_item", item, spider) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 47457f2a8..1a13aeaf2 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -16,18 +16,34 @@ from collections import defaultdict from contextlib import suppress from ftplib import FTP from io import BytesIO -from os import PathLike from pathlib import Path -from typing import IO, TYPE_CHECKING, DefaultDict, Optional, Set, Type, Union, cast +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + DefaultDict, + Dict, + List, + NoReturn, + Optional, + Protocol, + Set, + Type, + TypedDict, + Union, + cast, +) from urllib.parse import urlparse from itemadapter import ItemAdapter -from twisted.internet import defer, threads +from twisted.internet.defer import Deferred, maybeDeferred +from twisted.internet.threads import deferToThread from scrapy.exceptions import IgnoreRequest, NotConfigured -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.media import MediaPipeline +from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline from scrapy.settings import Settings from scrapy.utils.boto import is_botocore_available from scrapy.utils.datatypes import CaseInsensitiveDict @@ -37,17 +53,24 @@ from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str if TYPE_CHECKING: + from os import PathLike + + from twisted.python.failure import Failure + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + + logger = logging.getLogger(__name__) -def _to_string(path: Union[str, PathLike]) -> str: +def _to_string(path: Union[str, PathLike[str]]) -> str: return str(path) # convert a Path object to string -def _md5sum(file: IO) -> str: +def _md5sum(file: IO[bytes]) -> str: """Calculate the md5 checksum of a file-like object without reading its whole content in memory. @@ -68,23 +91,54 @@ class FileException(Exception): """General media error exception""" +class StatInfo(TypedDict, total=False): + checksum: str + last_modified: float + + +class FilesStoreProtocol(Protocol): + def __init__(self, basedir: str): ... + + def persist_file( + self, + path: str, + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + ) -> Optional[Deferred[Any]]: ... + + def stat_file( + self, path: str, info: MediaPipeline.SpiderInfo + ) -> Union[StatInfo, Deferred[StatInfo]]: ... + + class FSFilesStore: - def __init__(self, basedir: Union[str, PathLike]): + def __init__(self, basedir: Union[str, PathLike[str]]): basedir = _to_string(basedir) if "://" in basedir: basedir = basedir.split("://", 1)[1] - self.basedir = basedir + self.basedir: str = basedir self._mkdir(Path(self.basedir)) - self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set) + self.created_directories: DefaultDict[MediaPipeline.SpiderInfo, Set[str]] = ( + defaultdict(set) + ) def persist_file( - self, path: Union[str, PathLike], buf, info, meta=None, headers=None - ): + self, + path: Union[str, PathLike[str]], + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + ) -> None: absolute_path = self._get_filesystem_path(path) self._mkdir(absolute_path.parent, info) absolute_path.write_bytes(buf.getvalue()) - def stat_file(self, path: Union[str, PathLike], info): + def stat_file( + self, path: Union[str, PathLike[str]], info: MediaPipeline.SpiderInfo + ) -> StatInfo: absolute_path = self._get_filesystem_path(path) try: last_modified = absolute_path.stat().st_mtime @@ -96,12 +150,14 @@ class FSFilesStore: return {"last_modified": last_modified, "checksum": checksum} - def _get_filesystem_path(self, path: Union[str, PathLike]) -> Path: + def _get_filesystem_path(self, path: Union[str, PathLike[str]]) -> Path: path_comps = _to_string(path).split("/") return Path(self.basedir, *path_comps) - def _mkdir(self, dirname: Path, domain: Optional[str] = None): - seen = self.created_directories[domain] if domain else set() + def _mkdir( + self, dirname: Path, domain: Optional[MediaPipeline.SpiderInfo] = None + ) -> None: + seen: Set[str] = self.created_directories[domain] if domain else set() if str(dirname) not in seen: if not dirname.exists(): dirname.mkdir(parents=True) @@ -122,7 +178,7 @@ class S3FilesStore: "Cache-Control": "max-age=172800", } - def __init__(self, uri): + def __init__(self, uri: str): if not is_botocore_available(): raise NotConfigured("missing botocore library") import botocore.session @@ -142,8 +198,10 @@ class S3FilesStore: raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'") self.bucket, self.prefix = uri[5:].split("/", 1) - def stat_file(self, path, info): - def _onsuccess(boto_key): + def stat_file( + self, path: str, info: MediaPipeline.SpiderInfo + ) -> Deferred[StatInfo]: + def _onsuccess(boto_key: Dict[str, Any]) -> StatInfo: checksum = boto_key["ETag"].strip('"') last_modified = boto_key["LastModified"] modified_stamp = time.mktime(last_modified.timetuple()) @@ -151,21 +209,31 @@ class S3FilesStore: return self._get_boto_key(path).addCallback(_onsuccess) - def _get_boto_key(self, path): + def _get_boto_key(self, path: str) -> Deferred[Dict[str, Any]]: key_name = f"{self.prefix}{path}" - return threads.deferToThread( - self.s3_client.head_object, Bucket=self.bucket, Key=key_name + return cast( + "Deferred[Dict[str, Any]]", + deferToThread( + self.s3_client.head_object, Bucket=self.bucket, Key=key_name # type: ignore[attr-defined] + ), ) - def persist_file(self, path, buf, info, meta=None, headers=None): + def persist_file( + self, + path: str, + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + ) -> Deferred[Any]: """Upload file to S3 storage""" key_name = f"{self.prefix}{path}" buf.seek(0) extra = self._headers_to_botocore_kwargs(self.HEADERS) if headers: extra.update(self._headers_to_botocore_kwargs(headers)) - return threads.deferToThread( - self.s3_client.put_object, + return deferToThread( + self.s3_client.put_object, # type: ignore[attr-defined] Bucket=self.bucket, Key=key_name, Body=buf, @@ -174,7 +242,7 @@ class S3FilesStore: **extra, ) - def _headers_to_botocore_kwargs(self, headers): + def _headers_to_botocore_kwargs(self, headers: Dict[str, Any]) -> Dict[str, Any]: """Convert headers to botocore keyword arguments.""" # This is required while we need to support both boto and botocore. mapping = CaseInsensitiveDict( @@ -206,7 +274,7 @@ class S3FilesStore: "X-Amz-Website-Redirect-Location": "WebsiteRedirectLocation", } ) - extra = {} + extra: Dict[str, Any] = {} for key, value in headers.items(): try: kwarg = mapping[key] @@ -226,13 +294,13 @@ class GCSFilesStore: # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. POLICY = None - def __init__(self, uri): + def __init__(self, uri: str): from google.cloud import storage client = storage.Client(project=self.GCS_PROJECT_ID) bucket, prefix = uri[5:].split("/", 1) self.bucket = client.bucket(bucket) - self.prefix = prefix + self.prefix: str = prefix permissions = self.bucket.test_iam_permissions( ["storage.objects.get", "storage.objects.create"] ) @@ -248,8 +316,10 @@ class GCSFilesStore: {"bucket": bucket}, ) - def stat_file(self, path, info): - def _onsuccess(blob): + def stat_file( + self, path: str, info: MediaPipeline.SpiderInfo + ) -> Deferred[StatInfo]: + def _onsuccess(blob) -> StatInfo: if blob: checksum = base64.b64decode(blob.md5_hash).hex() last_modified = time.mktime(blob.updated.timetuple()) @@ -257,24 +327,32 @@ class GCSFilesStore: return {} blob_path = self._get_blob_path(path) - return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback( - _onsuccess + return cast( + Deferred[StatInfo], + deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess), ) - def _get_content_type(self, headers): + def _get_content_type(self, headers: Optional[Dict[str, str]]) -> str: if headers and "Content-Type" in headers: return headers["Content-Type"] return "application/octet-stream" - def _get_blob_path(self, path): + def _get_blob_path(self, path: str) -> str: return self.prefix + path - def persist_file(self, path, buf, info, meta=None, headers=None): + def persist_file( + self, + path: str, + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + ) -> Deferred[Any]: blob_path = self._get_blob_path(path) blob = self.bucket.blob(blob_path) blob.cache_control = self.CACHE_CONTROL blob.metadata = {k: str(v) for k, v in (meta or {}).items()} - return threads.deferToThread( + return deferToThread( blob.upload_from_string, data=buf.getvalue(), content_type=self._get_content_type(headers), @@ -283,24 +361,35 @@ class GCSFilesStore: class FTPFilesStore: - FTP_USERNAME = None - FTP_PASSWORD = None - USE_ACTIVE_MODE = None + FTP_USERNAME: Optional[str] = None + FTP_PASSWORD: Optional[str] = None + USE_ACTIVE_MODE: Optional[bool] = None - def __init__(self, uri): + def __init__(self, uri: str): if not uri.startswith("ftp://"): raise ValueError(f"Incorrect URI scheme in {uri}, expected 'ftp'") u = urlparse(uri) - self.port = u.port - self.host = u.hostname + assert u.port + assert u.hostname + self.port: int = u.port + self.host: str = u.hostname self.port = int(u.port or 21) - self.username = u.username or self.FTP_USERNAME - self.password = u.password or self.FTP_PASSWORD - self.basedir = u.path.rstrip("/") + assert self.FTP_USERNAME + assert self.FTP_PASSWORD + self.username: str = u.username or self.FTP_USERNAME + self.password: str = u.password or self.FTP_PASSWORD + self.basedir: str = u.path.rstrip("/") - def persist_file(self, path, buf, info, meta=None, headers=None): + def persist_file( + self, + path: str, + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + ) -> Deferred[Any]: path = f"{self.basedir}/{path}" - return threads.deferToThread( + return deferToThread( ftp_store_file, path=path, file=buf, @@ -311,8 +400,10 @@ class FTPFilesStore: use_active_mode=self.USE_ACTIVE_MODE, ) - def stat_file(self, path, info): - def _stat_file(path): + def stat_file( + self, path: str, info: MediaPipeline.SpiderInfo + ) -> Deferred[StatInfo]: + def _stat_file(path: str) -> StatInfo: try: ftp = FTP() ftp.connect(self.host, self.port) @@ -328,7 +419,7 @@ class FTPFilesStore: except Exception: return {} - return threads.deferToThread(_stat_file, path) + return cast("Deferred[StatInfo]", deferToThread(_stat_file, path)) class FilesPipeline(MediaPipeline): @@ -350,20 +441,23 @@ class FilesPipeline(MediaPipeline): """ - MEDIA_NAME = "file" - EXPIRES = 90 - STORE_SCHEMES = { + MEDIA_NAME: str = "file" + EXPIRES: int = 90 + STORE_SCHEMES: Dict[str, Type[FilesStoreProtocol]] = { "": FSFilesStore, "file": FSFilesStore, "s3": S3FilesStore, "gs": GCSFilesStore, "ftp": FTPFilesStore, } - DEFAULT_FILES_URLS_FIELD = "file_urls" - DEFAULT_FILES_RESULT_FIELD = "files" + DEFAULT_FILES_URLS_FIELD: str = "file_urls" + DEFAULT_FILES_RESULT_FIELD: str = "files" def __init__( - self, store_uri: Union[str, PathLike], download_func=None, settings=None + self, + store_uri: Union[str, PathLike[str]], + download_func: Optional[Callable[[Request, Spider], Response]] = None, + settings: Union[Settings, Dict[str, Any], None] = None, ): store_uri = _to_string(store_uri) if not store_uri: @@ -372,26 +466,26 @@ class FilesPipeline(MediaPipeline): if isinstance(settings, dict) or settings is None: settings = Settings(settings) cls_name = "FilesPipeline" - self.store = self._get_store(store_uri) + self.store: FilesStoreProtocol = self._get_store(store_uri) resolve = functools.partial( self._key_for_pipe, base_class_name=cls_name, settings=settings ) - self.expires = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES) + self.expires: int = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES) if not hasattr(self, "FILES_URLS_FIELD"): self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELD if not hasattr(self, "FILES_RESULT_FIELD"): self.FILES_RESULT_FIELD = self.DEFAULT_FILES_RESULT_FIELD - self.files_urls_field = settings.get( + self.files_urls_field: str = settings.get( resolve("FILES_URLS_FIELD"), self.FILES_URLS_FIELD ) - self.files_result_field = settings.get( + self.files_result_field: str = settings.get( resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD ) super().__init__(download_func=download_func, settings=settings) @classmethod - def from_settings(cls, settings) -> Self: + def from_settings(cls, settings: Settings) -> Self: s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] @@ -418,7 +512,7 @@ class FilesPipeline(MediaPipeline): store_uri = settings["FILES_STORE"] return cls(store_uri, settings=settings) - def _get_store(self, uri: str): + def _get_store(self, uri: str) -> FilesStoreProtocol: if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir scheme = "file" else: @@ -426,19 +520,21 @@ class FilesPipeline(MediaPipeline): store_cls = self.STORE_SCHEMES[scheme] return store_cls(uri) - def media_to_download(self, request, info, *, item=None): - def _onsuccess(result): + def media_to_download( + self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None + ) -> Deferred[Optional[FileInfo]]: + def _onsuccess(result: StatInfo) -> Optional[FileInfo]: if not result: - return # returning None force download + return None # returning None force download last_modified = result.get("last_modified", None) if not last_modified: - return # returning None force download + return None # returning None force download age_seconds = time.time() - last_modified age_days = age_seconds / 60 / 60 / 24 if age_days > self.expires: - return # returning None force download + return None # returning None force download referer = referer_str(request) logger.debug( @@ -458,19 +554,22 @@ class FilesPipeline(MediaPipeline): } path = self.file_path(request, info=info, item=item) - dfd = defer.maybeDeferred(self.store.stat_file, path, info) - dfd.addCallback(_onsuccess) - dfd.addErrback(lambda _: None) - dfd.addErrback( + # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type + dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type] + dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess) + dfd2.addErrback(lambda _: None) + dfd2.addErrback( lambda f: logger.error( self.__class__.__name__ + ".store.stat_file", exc_info=failure_to_exc_info(f), extra={"spider": info.spider}, ) ) - return dfd + return dfd2 - def media_failed(self, failure, request, info): + def media_failed( + self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo + ) -> NoReturn: if not isinstance(failure.value, IgnoreRequest): referer = referer_str(request) logger.warning( @@ -487,7 +586,14 @@ class FilesPipeline(MediaPipeline): raise FileException - def media_downloaded(self, response, request, info, *, item=None): + def media_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> FileInfo: referer = referer_str(request) if response.status != 200: @@ -546,16 +652,26 @@ class FilesPipeline(MediaPipeline): "status": status, } - def inc_stats(self, spider, status): + def inc_stats(self, spider: Spider, status: str) -> None: + assert spider.crawler.stats spider.crawler.stats.inc_value("file_count", spider=spider) spider.crawler.stats.inc_value(f"file_status_count/{status}", spider=spider) # Overridable Interface - def get_media_requests(self, item, info): + def get_media_requests( + self, item: Any, info: MediaPipeline.SpiderInfo + ) -> List[Request]: urls = ItemAdapter(item).get(self.files_urls_field, []) return [Request(u, callback=NO_CALLBACK) for u in urls] - def file_downloaded(self, response, request, info, *, item=None): + def file_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> str: path = self.file_path(request, response=response, info=info, item=item) buf = BytesIO(response.body) checksum = _md5sum(buf) @@ -563,12 +679,21 @@ class FilesPipeline(MediaPipeline): self.store.persist_file(path, buf, info) return checksum - def item_completed(self, results, item, info): + def item_completed( + self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo + ) -> Any: with suppress(KeyError): ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None, *, item=None): + def file_path( + self, + request: Request, + response: Optional[Response] = None, + info: Optional[MediaPipeline.SpiderInfo] = None, + *, + item: Any = None, + ) -> str: media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec media_ext = Path(request.url).suffix # Handles empty and wild extensions by trying to guess the @@ -577,5 +702,5 @@ class FilesPipeline(MediaPipeline): media_ext = "" media_type = mimetypes.guess_type(request.url)[0] if media_type: - media_ext = mimetypes.guess_extension(media_type) + media_ext = cast(str, mimetypes.guess_extension(media_type)) return f"full/{media_guid}{media_ext}" diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index e7ef06fb3..166f81314 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -11,13 +11,24 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO -from os import PathLike -from typing import TYPE_CHECKING, Dict, Tuple, Type, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Tuple, + Type, + Union, + cast, +) from itemadapter import ItemAdapter from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import ( FileException, @@ -27,20 +38,25 @@ from scrapy.pipelines.files import ( S3FilesStore, _md5sum, ) - -# TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.utils.python import get_func_args, to_bytes if TYPE_CHECKING: + from os import PathLike + + from PIL import Image + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.pipelines.media import FileInfoOrError, MediaPipeline + class NoimagesDrop(DropItem): """Product with no images exception""" - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any): warnings.warn( "The NoimagesDrop class is deprecated", category=ScrapyDeprecationWarning, @@ -56,19 +72,22 @@ class ImageException(FileException): class ImagesPipeline(FilesPipeline): """Abstract pipeline that implement the image thumbnail generation logic""" - MEDIA_NAME = "image" + MEDIA_NAME: str = "image" # Uppercase attributes kept for backward compatibility with code that subclasses # ImagesPipeline. They may be overridden by settings. - MIN_WIDTH = 0 - MIN_HEIGHT = 0 - EXPIRES = 90 + MIN_WIDTH: int = 0 + MIN_HEIGHT: int = 0 + EXPIRES: int = 90 THUMBS: Dict[str, Tuple[int, int]] = {} DEFAULT_IMAGES_URLS_FIELD = "image_urls" DEFAULT_IMAGES_RESULT_FIELD = "images" def __init__( - self, store_uri: Union[str, PathLike], download_func=None, settings=None + self, + store_uri: Union[str, PathLike[str]], + download_func: Optional[Callable[[Request, Spider], Response]] = None, + settings: Union[Settings, Dict[str, Any], None] = None, ): try: from PIL import Image @@ -89,27 +108,33 @@ class ImagesPipeline(FilesPipeline): base_class_name="ImagesPipeline", settings=settings, ) - self.expires = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES) + self.expires: int = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES) if not hasattr(self, "IMAGES_RESULT_FIELD"): - self.IMAGES_RESULT_FIELD = self.DEFAULT_IMAGES_RESULT_FIELD + self.IMAGES_RESULT_FIELD: str = self.DEFAULT_IMAGES_RESULT_FIELD if not hasattr(self, "IMAGES_URLS_FIELD"): - self.IMAGES_URLS_FIELD = self.DEFAULT_IMAGES_URLS_FIELD + self.IMAGES_URLS_FIELD: str = self.DEFAULT_IMAGES_URLS_FIELD - self.images_urls_field = settings.get( + self.images_urls_field: str = settings.get( resolve("IMAGES_URLS_FIELD"), self.IMAGES_URLS_FIELD ) - self.images_result_field = settings.get( + self.images_result_field: str = settings.get( resolve("IMAGES_RESULT_FIELD"), self.IMAGES_RESULT_FIELD ) - self.min_width = settings.getint(resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH) - self.min_height = settings.getint(resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT) - self.thumbs = settings.get(resolve("IMAGES_THUMBS"), self.THUMBS) + self.min_width: int = settings.getint( + resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH + ) + self.min_height: int = settings.getint( + resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT + ) + self.thumbs: Dict[str, Tuple[int, int]] = settings.get( + resolve("IMAGES_THUMBS"), self.THUMBS + ) - self._deprecated_convert_image = None + self._deprecated_convert_image: Optional[bool] = None @classmethod - def from_settings(cls, settings) -> Self: + def from_settings(cls, settings: Settings) -> Self: s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] @@ -136,11 +161,25 @@ class ImagesPipeline(FilesPipeline): store_uri = settings["IMAGES_STORE"] return cls(store_uri, settings=settings) - def file_downloaded(self, response, request, info, *, item=None): + def file_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> str: return self.image_downloaded(response, request, info, item=item) - def image_downloaded(self, response, request, info, *, item=None): - checksum = None + def image_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> str: + checksum: Optional[str] = None for path, image, buf in self.get_images(response, request, info, item=item): if checksum is None: buf.seek(0) @@ -153,9 +192,17 @@ class ImagesPipeline(FilesPipeline): meta={"width": width, "height": height}, headers={"Content-Type": "image/jpeg"}, ) + assert checksum is not None return checksum - def get_images(self, response, request, info, *, item=None): + def get_images( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> Iterable[Tuple[str, Image.Image, BytesIO]]: path = self.file_path(request, response=response, info=info, item=item) orig_image = self._Image.open(BytesIO(response.body)) @@ -196,7 +243,12 @@ class ImagesPipeline(FilesPipeline): thumb_image, thumb_buf = self.convert_image(image, size, buf) yield thumb_path, thumb_image, thumb_buf - def convert_image(self, image, size=None, response_body=None): + def convert_image( + self, + image: Image.Image, + size: Optional[Tuple[int, int]] = None, + response_body: Optional[BytesIO] = None, + ) -> Tuple[Image.Image, BytesIO]: if response_body is None: warnings.warn( f"{self.__class__.__name__}.convert_image() method called in a deprecated way, " @@ -225,7 +277,7 @@ class ImagesPipeline(FilesPipeline): # when updating the minimum requirements for Pillow. resampling_filter = self._Image.Resampling.LANCZOS except AttributeError: - resampling_filter = self._Image.ANTIALIAS + resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined] image.thumbnail(size, resampling_filter) elif response_body is not None and image.format == "JPEG": return image, response_body @@ -234,19 +286,38 @@ class ImagesPipeline(FilesPipeline): image.save(buf, "JPEG") return image, buf - def get_media_requests(self, item, info): + def get_media_requests( + self, item: Any, info: MediaPipeline.SpiderInfo + ) -> List[Request]: urls = ItemAdapter(item).get(self.images_urls_field, []) return [Request(u, callback=NO_CALLBACK) for u in urls] - def item_completed(self, results, item, info): + def item_completed( + self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo + ) -> Any: with suppress(KeyError): ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None, *, item=None): + def file_path( + self, + request: Request, + response: Optional[Response] = None, + info: Optional[MediaPipeline.SpiderInfo] = None, + *, + item: Any = None, + ) -> str: image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec return f"full/{image_guid}.jpg" - def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None): + def thumb_path( + self, + request: Request, + thumb_id: str, + response: Optional[Response] = None, + info: Optional[MediaPipeline.SpiderInfo] = None, + *, + item: Any = None, + ) -> str: thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec return f"thumbs/{thumb_id}/{thumb_guid}.jpg" diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 5f6c5cb07..ea36a9e8a 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -2,13 +2,30 @@ from __future__ import annotations import functools import logging +from abc import ABC, abstractmethod from collections import defaultdict -from typing import TYPE_CHECKING +from typing import ( + TYPE_CHECKING, + Any, + Callable, + DefaultDict, + Dict, + List, + Literal, + NoReturn, + Optional, + Set, + Tuple, + TypedDict, + TypeVar, + Union, + cast, +) from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure -from scrapy.http.request import NO_CALLBACK +from scrapy.http.request import NO_CALLBACK, Request from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import defer_result, mustbe_deferred @@ -19,50 +36,71 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.utils.request import RequestFingerprinter + + +_T = TypeVar("_T") + + +class FileInfo(TypedDict): + url: str + path: str + checksum: Optional[str] + status: str + + +FileInfoOrError = Union[Tuple[Literal[True], FileInfo], Tuple[Literal[False], Failure]] + logger = logging.getLogger(__name__) -def _DUMMY_CALLBACK(response): - return response +class MediaPipeline(ABC): + crawler: Crawler + _fingerprinter: RequestFingerprinter - -class MediaPipeline: - LOG_FAILED_RESULTS = True + LOG_FAILED_RESULTS: bool = True class SpiderInfo: - def __init__(self, spider): - self.spider = spider - self.downloading = set() - self.downloaded = {} - self.waiting = defaultdict(list) + def __init__(self, spider: Spider): + self.spider: Spider = spider + self.downloading: Set[bytes] = set() + self.downloaded: Dict[bytes, Union[FileInfo, Failure]] = {} + self.waiting: DefaultDict[bytes, List[Deferred[FileInfo]]] = defaultdict( + list + ) - def __init__(self, download_func=None, settings=None): + def __init__( + self, + download_func: Optional[Callable[[Request, Spider], Response]] = None, + settings: Union[Settings, Dict[str, Any], None] = None, + ): self.download_func = download_func - self._expects_item = {} if isinstance(settings, dict) or settings is None: settings = Settings(settings) resolve = functools.partial( self._key_for_pipe, base_class_name="MediaPipeline", settings=settings ) - self.allow_redirects = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False) + self.allow_redirects: bool = settings.getbool( + resolve("MEDIA_ALLOW_REDIRECTS"), False + ) self._handle_statuses(self.allow_redirects) - def _handle_statuses(self, allow_redirects): + def _handle_statuses(self, allow_redirects: bool) -> None: self.handle_httpstatus_list = None if allow_redirects: self.handle_httpstatus_list = SequenceExclude(range(300, 400)) - def _key_for_pipe(self, key, base_class_name=None, settings=None): - """ - >>> MediaPipeline()._key_for_pipe("IMAGES") - 'IMAGES' - >>> class MyPipe(MediaPipeline): - ... pass - >>> MyPipe()._key_for_pipe("IMAGES", base_class_name="MediaPipeline") - 'MYPIPE_IMAGES' - """ + def _key_for_pipe( + self, + key: str, + base_class_name: Optional[str] = None, + settings: Optional[Settings] = None, + ) -> str: class_name = self.__class__.__name__ formatted_key = f"{class_name.upper()}_{key}" if ( @@ -75,31 +113,35 @@ class MediaPipeline: return formatted_key @classmethod - def from_crawler(cls, crawler) -> Self: + def from_crawler(cls, crawler: Crawler) -> Self: + pipe: Self try: pipe = cls.from_settings(crawler.settings) # type: ignore[attr-defined] except AttributeError: pipe = cls() pipe.crawler = crawler + assert crawler.request_fingerprinter pipe._fingerprinter = crawler.request_fingerprinter return pipe - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> None: self.spiderinfo = self.SpiderInfo(spider) - def process_item(self, item, spider): + def process_item( + self, item: Any, spider: Spider + ) -> Deferred[List[FileInfoOrError]]: info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) dlist = [self._process_request(r, info, item) for r in requests] - dfd = DeferredList(dlist, consumeErrors=True) + dfd = cast( + "Deferred[List[FileInfoOrError]]", DeferredList(dlist, consumeErrors=True) + ) return dfd.addCallback(self.item_completed, item, info) - def _process_request(self, request, info, item): + def _process_request( + self, request: Request, info: SpiderInfo, item: Any + ) -> Deferred[FileInfo]: fp = self._fingerprinter.fingerprint(request) - if not request.callback or request.callback is NO_CALLBACK: - cb = _DUMMY_CALLBACK - else: - cb = request.callback eb = request.errback request.callback = NO_CALLBACK request.errback = None @@ -107,14 +149,12 @@ class MediaPipeline: # Return cached result if request was already seen if fp in info.downloaded: d = defer_result(info.downloaded[fp]) - d.addCallback(cb) if eb: d.addErrback(eb) return d # Otherwise, wait for result - wad = Deferred() - wad.addCallback(cb) + wad: Deferred[FileInfo] = Deferred() if eb: wad.addErrback(eb) info.waiting[fp].append(wad) @@ -125,36 +165,48 @@ class MediaPipeline: # Download request checking media_to_download hook output first info.downloading.add(fp) - dfd = mustbe_deferred(self.media_to_download, request, info, item=item) - dfd.addCallback(self._check_media_to_download, request, info, item=item) - dfd.addErrback(self._log_exception) - dfd.addBoth(self._cache_result_and_execute_waiters, fp, info) - return dfd.addBoth(lambda _: wad) # it must return wad at last + dfd: Deferred[Optional[FileInfo]] = mustbe_deferred( + self.media_to_download, request, info, item=item + ) + dfd2: Deferred[FileInfo] = dfd.addCallback( + self._check_media_to_download, request, info, item=item + ) + dfd2.addErrback(self._log_exception) + dfd2.addBoth(self._cache_result_and_execute_waiters, fp, info) + return dfd2.addBoth(lambda _: wad) # it must return wad at last - def _log_exception(self, result): + def _log_exception(self, result: Failure) -> Failure: logger.exception(result) return result - def _modify_media_request(self, request): + def _modify_media_request(self, request: Request) -> None: if self.handle_httpstatus_list: request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list else: request.meta["handle_httpstatus_all"] = True - def _check_media_to_download(self, result, request, info, item): + def _check_media_to_download( + self, result: Optional[FileInfo], request: Request, info: SpiderInfo, item: Any + ) -> Union[FileInfo, Deferred[FileInfo]]: if result is not None: return result + dfd: Deferred[Response] if self.download_func: # this ugly code was left only to support tests. TODO: remove dfd = mustbe_deferred(self.download_func, request, info.spider) else: self._modify_media_request(request) + assert self.crawler.engine dfd = self.crawler.engine.download(request) - dfd.addCallback(self.media_downloaded, request, info, item=item) - dfd.addErrback(self.media_failed, request, info) - return dfd + dfd2: Deferred[FileInfo] = dfd.addCallback( + self.media_downloaded, request, info, item=item + ) + dfd2.addErrback(self.media_failed, request, info) + return dfd2 - def _cache_result_and_execute_waiters(self, result, fp, info): + def _cache_result_and_execute_waiters( + self, result: Union[FileInfo, Failure], fp: bytes, info: SpiderInfo + ) -> None: if isinstance(result, Failure): # minimize cached information for failure result.cleanFailure() @@ -184,7 +236,7 @@ class MediaPipeline: # Exception Chaining (https://www.python.org/dev/peps/pep-3134/). context = getattr(result.value, "__context__", None) if isinstance(context, StopIteration): - setattr(result.value, "__context__", None) + result.value.__context__ = None info.downloading.remove(fp) info.downloaded[fp] = result # cache result @@ -192,27 +244,45 @@ class MediaPipeline: defer_result(result).chainDeferred(wad) # Overridable Interface - def media_to_download(self, request, info, *, item=None): + @abstractmethod + def media_to_download( + self, request: Request, info: SpiderInfo, *, item: Any = None + ) -> Deferred[Optional[FileInfo]]: """Check request before starting download""" - pass + raise NotImplementedError() - def get_media_requests(self, item, info): + @abstractmethod + def get_media_requests(self, item: Any, info: SpiderInfo) -> List[Request]: """Returns the media requests to download""" - pass + raise NotImplementedError() - def media_downloaded(self, response, request, info, *, item=None): + @abstractmethod + def media_downloaded( + self, + response: Response, + request: Request, + info: SpiderInfo, + *, + item: Any = None, + ) -> FileInfo: """Handler for success downloads""" - return response + raise NotImplementedError() - def media_failed(self, failure, request, info): + @abstractmethod + def media_failed( + self, failure: Failure, request: Request, info: SpiderInfo + ) -> NoReturn: """Handler for failed downloads""" - return failure + raise NotImplementedError() - def item_completed(self, results, item, info): + def item_completed( + self, results: List[FileInfoOrError], item: Any, info: SpiderInfo + ) -> Any: """Called per item when all media requests has been processed""" if self.LOG_FAILED_RESULTS: for ok, value in results: if not ok: + assert isinstance(value, Failure) logger.error( "%(class)s found errors processing %(item)s", {"class": self.__class__.__name__, "item": item}, @@ -221,6 +291,14 @@ class MediaPipeline: ) return item - def file_path(self, request, response=None, info=None, *, item=None): + @abstractmethod + def file_path( + self, + request: Request, + response: Optional[Response] = None, + info: Optional[SpiderInfo] = None, + *, + item: Any = None, + ) -> str: """Returns the path where downloaded media should be stored""" - pass + raise NotImplementedError() diff --git a/scrapy/resolver.py b/scrapy/resolver.py index ba7cd716b..d5eedf9b1 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Type from twisted.internet import defer from twisted.internet.base import ReactorBase, ThreadedResolver -from twisted.internet.defer import Deferred from twisted.internet.interfaces import ( IAddress, IHostnameResolver, @@ -17,6 +16,8 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index a33f73306..0d282dc37 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -6,7 +6,6 @@ from abc import ABCMeta, abstractmethod from typing import TYPE_CHECKING, Optional, Union from warnings import warn -from scrapy import Spider from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import to_unicode @@ -14,8 +13,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index e852aadc7..bfddb87cb 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -59,6 +59,7 @@ class Selector(_ParselSelector, object_ref): * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type + * ``"json"`` for :class:`~scrapy.http.TextResponse` type * ``"html"`` for anything else Otherwise, if ``type`` is set, the selector type will be forced and no diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index d270a72f4..6703c569f 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -4,7 +4,6 @@ import copy import json from importlib import import_module from pprint import pformat -from types import ModuleType from typing import ( TYPE_CHECKING, Any, @@ -27,6 +26,8 @@ from scrapy.settings import default_settings _SettingsKeyT = Union[bool, float, int, str, None] if TYPE_CHECKING: + from types import ModuleType + # https://github.com/python/typing/issues/445#issuecomment-1131458824 from _typeshed import SupportsItems @@ -275,7 +276,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): assert isinstance(value, (dict, list)) return copy.deepcopy(value) - def getwithbase(self, name: _SettingsKeyT) -> "BaseSettings": + def getwithbase(self, name: _SettingsKeyT) -> BaseSettings: """Get a composition of a dictionary-like setting and its `_BASE` counterpart. @@ -411,7 +412,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ self._assert_mutability() if isinstance(values, str): - values = cast(dict, json.loads(values)) + values = cast(Dict[_SettingsKeyT, Any], json.loads(values)) if values is not None: if isinstance(values, BaseSettings): for name, value in values.items(): @@ -438,7 +439,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): if self.frozen: raise TypeError("Trying to modify an immutable Settings object") - def copy(self) -> "Self": + def copy(self) -> Self: """ Make a deep copy of current settings. @@ -460,7 +461,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ self.frozen = True - def frozencopy(self) -> "Self": + def frozencopy(self) -> Self: """ Return an immutable copy of the current settings. diff --git a/scrapy/shell.py b/scrapy/shell.py index 2c22d3d8f..b7e46274f 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -4,6 +4,8 @@ See documentation in docs/topics/shell.rst """ +from __future__ import annotations + import os import signal from typing import Any, Callable, Dict, Optional, Tuple, Union @@ -92,7 +94,9 @@ class Shell: self.vars, shells=shells, banner=self.vars.pop("banner", "") ) - def _schedule(self, request: Request, spider: Optional[Spider]) -> defer.Deferred: + def _schedule( + self, request: Request, spider: Optional[Spider] + ) -> defer.Deferred[Any]: if is_asyncio_reactor_installed(): # set the asyncio event loop for the current thread event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"] @@ -209,7 +213,7 @@ def inspect_response(response: Response, spider: Spider) -> None: signal.signal(signal.SIGINT, sigint_handler) -def _request_deferred(request: Request) -> defer.Deferred: +def _request_deferred(request: Request) -> defer.Deferred[Any]: """Wrap a request inside a Deferred. This function is harmful, do not use it until you know what you are doing. @@ -228,7 +232,7 @@ def _request_deferred(request: Request) -> defer.Deferred: request.errback = request_errback return result - d: defer.Deferred = defer.Deferred() + d: defer.Deferred[Any] = defer.Deferred() d.addBoth(_restore_callbacks) if request.callback: d.addCallback(request.callback) diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index f6df191d8..b2c6dea5d 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,10 +1,14 @@ -from typing import Any, List, Tuple +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, List, Tuple from pydispatch import dispatcher -from twisted.internet.defer import Deferred from scrapy.utils import signal as _signal +if TYPE_CHECKING: + from twisted.internet.defer import Deferred + class SignalManager: def __init__(self, sender: Any = dispatcher.Anonymous): @@ -46,7 +50,9 @@ class SignalManager: kwargs.setdefault("sender", self.sender) return _signal.send_catch_log(signal, **kwargs) - def send_catch_log_deferred(self, signal: Any, **kwargs: Any) -> Deferred: + def send_catch_log_deferred( + self, signal: Any, **kwargs: Any + ) -> Deferred[List[Tuple[Any, Any]]]: """ Like :meth:`send_catch_log` but supports returning :class:`~twisted.internet.defer.Deferred` objects from signal handlers. diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index d855c962c..b8fe65668 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -3,21 +3,23 @@ from __future__ import annotations import traceback import warnings from collections import defaultdict -from types import ModuleType from typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Type from zope.interface import implementer -from scrapy import Request, Spider from scrapy.interfaces import ISpiderLoader -from scrapy.settings import BaseSettings from scrapy.utils.misc import walk_modules from scrapy.utils.spider import iter_spider_classes if TYPE_CHECKING: + from types import ModuleType + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.settings import BaseSettings + @implementer(ISpiderLoader) class SpiderLoader: diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 1e96654e2..c5b7f0749 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -9,15 +9,17 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable -from scrapy import Spider -from scrapy.crawler import Crawler from scrapy.http import Request, Response -from scrapy.statscollectors import StatsCollector if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 35c869a75..ea1686c25 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -9,16 +9,18 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any, Iterable, List, Optional -from scrapy import Spider -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest -from scrapy.http import Response -from scrapy.settings import BaseSettings if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 50c93ac9f..379c5d0a3 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -12,10 +12,8 @@ import warnings from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable, Set from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response -from scrapy.statscollectors import StatsCollector from scrapy.utils.httpobj import urlparse_cached warnings.warn( @@ -28,6 +26,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a0b6851e5..d35cf8f71 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -23,10 +23,8 @@ from urllib.parse import urlparse from w3lib.url import safe_url_string from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response -from scrapy.settings import BaseSettings from scrapy.utils.misc import load_object from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url @@ -35,6 +33,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + LOCAL_SCHEMES: Tuple[str, ...] = ( "about", "blob", @@ -323,15 +325,18 @@ def _load_policy_class( try: return cast(Type[ReferrerPolicy], load_object(policy)) except ValueError: - try: - return _policy_classes[policy.lower()] - except KeyError: - msg = f"Could not load referrer policy {policy!r}" - if not warning_only: - raise RuntimeError(msg) - else: - warnings.warn(msg, RuntimeWarning) - return None + tokens = [token.strip() for token in policy.lower().split(",")] + # https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header + for token in tokens[::-1]: + if token in _policy_classes: + return _policy_classes[token] + + msg = f"Could not load referrer policy {policy!r}" + if not warning_only: + raise RuntimeError(msg) + else: + warnings.warn(msg, RuntimeWarning) + return None class RefererMiddleware: diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index e2aa554a7..34df54ca7 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -9,15 +9,17 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable -from scrapy import Spider from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response -from scrapy.settings import BaseSettings if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index bef041325..d977acd26 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,9 +7,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union, cast - -from twisted.internet.defer import Deferred +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, cast from scrapy import signals from scrapy.http import Request, Response @@ -17,18 +15,16 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: - from collections.abc import Callable + from twisted.internet.defer import Deferred - # typing.Concatenate requires Python 3.10 # typing.Self requires Python 3.11 - from typing_extensions import Concatenate, Self + from typing_extensions import Self from scrapy.crawler import Crawler - from scrapy.settings import BaseSettings + from scrapy.http.request import CallbackT + from scrapy.settings import BaseSettings, _SettingsKeyT from scrapy.utils.log import SpiderLoggerAdapter - CallbackT = Callable[Concatenate[Response, ...], Any] - class Spider(object_ref): """Base class for scrapy spiders. All spiders must inherit from this @@ -36,7 +32,7 @@ class Spider(object_ref): """ name: str - custom_settings: Optional[dict] = None + custom_settings: Optional[Dict[_SettingsKeyT, Any]] = None def __init__(self, name: Optional[str] = None, **kwargs: Any): if name is not None: @@ -105,10 +101,10 @@ class Spider(object_ref): return url_is_from_spider(request.url, cls) @staticmethod - def close(spider: Spider, reason: str) -> Union[Deferred, None]: + def close(spider: Spider, reason: str) -> Optional[Deferred[None]]: closed = getattr(spider, "closed", None) if callable(closed): - return cast(Union[Deferred, None], closed(reason)) + return cast("Optional[Deferred[None]]", closed(reason)) return None def __repr__(self) -> str: diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 48c830d2a..2639f14b2 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -39,6 +39,7 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import CallbackT _T = TypeVar("_T") @@ -73,7 +74,7 @@ class Rule: def __init__( self, link_extractor: Optional[LinkExtractor] = None, - callback: Union[Callable, str, None] = None, + callback: Union[CallbackT, str, None] = None, cb_kwargs: Optional[Dict[str, Any]] = None, follow: Optional[bool] = None, process_links: Union[ProcessLinksT, str, None] = None, @@ -81,7 +82,7 @@ class Rule: errback: Union[Callable[[Failure], Any], str, None] = None, ): self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor - self.callback: Union[Callable, str, None] = callback + self.callback: Union[CallbackT, str, None] = callback self.errback: Union[Callable[[Failure], Any], str, None] = errback self.cb_kwargs: Dict[str, Any] = cb_kwargs or {} self.process_links: Union[ProcessLinksT, str] = process_links or _identity @@ -92,7 +93,7 @@ class Rule: def _compile(self, spider: Spider) -> None: # this replaces method names with methods and we can't express this in type hints - self.callback = _get_method(self.callback, spider) + self.callback = cast("CallbackT", _get_method(self.callback, spider)) self.errback = cast(Callable[[Failure], Any], _get_method(self.errback, spider)) self.process_links = cast( ProcessLinksT, _get_method(self.process_links, spider) @@ -122,7 +123,9 @@ class CrawlSpider(Spider): def parse_start_url(self, response: Response, **kwargs: Any) -> Any: return [] - def process_results(self, response: Response, results: Any) -> Any: + def process_results( + self, response: Response, results: Iterable[Any] + ) -> Iterable[Any]: return results def _build_request(self, rule_index: int, link: Link) -> Request: @@ -152,7 +155,7 @@ class CrawlSpider(Spider): rule = self._rules[cast(int, response.meta["rule"])] return self._parse_response( response, - cast(Callable, rule.callback), + cast("CallbackT", rule.callback), {**rule.cb_kwargs, **cb_kwargs}, rule.follow, ) @@ -166,7 +169,7 @@ class CrawlSpider(Spider): async def _parse_response( self, response: Response, - callback: Optional[Callable], + callback: Optional[CallbackT], cb_kwargs: Dict[str, Any], follow: bool = True, ) -> AsyncIterable[Any]: diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index a0898a0cf..ce0f1bbaa 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,10 +1,14 @@ -from typing import Any, Iterable, Optional, cast +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Iterable, Optional, cast from scrapy import Request -from scrapy.http import Response from scrapy.spiders import Spider from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + from scrapy.http import Response + class InitSpider(Spider): """Base Spider with initialization facilities""" diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index d082fbfdb..1542ef79c 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -5,7 +5,6 @@ import re from typing import ( TYPE_CHECKING, Any, - Callable, Dict, Iterable, List, @@ -27,6 +26,7 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import CallbackT logger = logging.getLogger(__name__) @@ -34,7 +34,7 @@ logger = logging.getLogger(__name__) class SitemapSpider(Spider): sitemap_urls: Sequence[str] = () sitemap_rules: Sequence[ - Tuple[Union[re.Pattern[str], str], Union[str, Callable]] + Tuple[Union[re.Pattern[str], str], Union[str, CallbackT]] ] = [("", "parse")] sitemap_follow: Sequence[Union[re.Pattern[str], str]] = [""] sitemap_alternate_links: bool = False @@ -54,10 +54,10 @@ class SitemapSpider(Spider): def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) - self._cbs: List[Tuple[re.Pattern[str], Callable]] = [] + self._cbs: List[Tuple[re.Pattern[str], CallbackT]] = [] for r, c in self.sitemap_rules: if isinstance(c, str): - c = cast(Callable, getattr(self, c)) + c = cast("CallbackT", getattr(self, c)) self._cbs.append((regex(r), c)) self._follow: List[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 6f80ee388..d3e7896c5 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -6,20 +6,22 @@ from __future__ import annotations import marshal import pickle # nosec -from os import PathLike from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union from queuelib import queue -from scrapy import Request -from scrapy.crawler import Crawler from scrapy.utils.request import request_from_dict if TYPE_CHECKING: + from os import PathLike + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request + from scrapy.crawler import Crawler + def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]: class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc] diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index ab571a3ab..88e72f366 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -2,15 +2,17 @@ Scrapy extension for collecting scraping stats """ +from __future__ import annotations + import logging import pprint from typing import TYPE_CHECKING, Any, Dict, Optional -from scrapy import Spider - if TYPE_CHECKING: + from scrapy import Spider from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) @@ -18,7 +20,7 @@ StatsT = Dict[str, Any] class StatsCollector: - def __init__(self, crawler: "Crawler"): + def __init__(self, crawler: Crawler): self._dump: bool = crawler.settings.getbool("STATS_DUMP") self._stats: StatsT = {} @@ -67,7 +69,7 @@ class StatsCollector: class MemoryStatsCollector(StatsCollector): - def __init__(self, crawler: "Crawler"): + def __init__(self, crawler: Crawler): super().__init__(crawler) self.spider_stats: Dict[str, StatsT] = {} diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 0505db343..67c8e1a01 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,14 +1,18 @@ -from typing import AsyncGenerator, AsyncIterable, Iterable, Union +from typing import AsyncGenerator, AsyncIterable, Iterable, List, TypeVar, Union + +_T = TypeVar("_T") -async def collect_asyncgen(result: AsyncIterable) -> list: +async def collect_asyncgen(result: AsyncIterable[_T]) -> List[_T]: results = [] async for x in result: results.append(x) return results -async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerator: +async def as_async_generator( + it: Union[Iterable[_T], AsyncIterable[_T]] +) -> AsyncGenerator[_T, None]: """Wraps an iterable (sync or async) into an async generator.""" if isinstance(it, AsyncIterable): async for r in it: diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index f6f704d4b..550516141 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -1,33 +1,33 @@ import random +from typing import Any from urllib.parse import urlencode from twisted.web.resource import Resource -from twisted.web.server import Site +from twisted.web.server import Request, Site class Root(Resource): isLeaf = True - def getChild(self, name, request): + def getChild(self, name: str, request: Request) -> Resource: return self - def render(self, request): + def render(self, request: Request) -> bytes: total = _getarg(request, b"total", 100, int) show = _getarg(request, b"show", 10, int) nlist = [random.randint(1, total) for _ in range(show)] # nosec request.write(b"") + assert request.args is not None args = request.args.copy() for nl in nlist: args["n"] = nl argstr = urlencode(args, doseq=True) - request.write( - f"follow {nl}
".encode("utf8") - ) + request.write(f"follow {nl}
".encode()) request.write(b"") return b"" -def _getarg(request, name, default=None, type=str): +def _getarg(request, name: bytes, default: Any = None, type=str): return type(request.args[name][0]) if name in request.args else default @@ -38,7 +38,7 @@ if __name__ == "__main__": factory = Site(root) httpPort = reactor.listenTCP(8998, Site(root)) - def _print_listening(): + def _print_listening() -> None: httpHost = httpPort.getHost() print(f"Bench server at http://{httpHost.host}:{httpHost.port}") diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 641dfa4a2..c63b69995 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -16,6 +16,7 @@ from typing import ( MutableMapping, Optional, Union, + cast, ) from scrapy.exceptions import ScrapyDeprecationWarning, UsageError @@ -173,7 +174,7 @@ def feed_process_params_from_cli( suitable to be used as the FEEDS setting. """ valid_output_formats: Iterable[str] = without_none_values( - settings.getwithbase("FEED_EXPORTERS") + cast(Dict[str, str], settings.getwithbase("FEED_EXPORTERS")) ).keys() def check_valid_format(output_format: str) -> None: diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index bf1803115..328219831 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -10,10 +10,10 @@ def _embed_ipython_shell( ) -> EmbedFuncT: """Start an IPython Shell""" try: - from IPython.terminal.embed import InteractiveShellEmbed + from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100 from IPython.terminal.ipapp import load_default_config except ImportError: - from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] + from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] # noqa: T100 InteractiveShellEmbed, ) from IPython.frontend.terminal.ipapp import ( # type: ignore[no-redef] diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index f5dbbd64e..c10e48511 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -2,13 +2,20 @@ import argparse import warnings from http.cookies import SimpleCookie from shlex import split +from typing import Any, Dict, List, NoReturn, Optional, Sequence, Tuple, Union from urllib.parse import urlparse from w3lib.http import basic_auth_header class DataAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + values: Union[str, Sequence[Any], None], + option_string: Optional[str] = None, + ) -> None: value = str(values) if value.startswith("$"): value = value[1:] @@ -16,7 +23,7 @@ class DataAction(argparse.Action): class CurlParser(argparse.ArgumentParser): - def error(self, message): + def error(self, message: str) -> NoReturn: error_msg = f"There was an error parsing the curl command: {message}" raise ValueError(error_msg) @@ -42,9 +49,11 @@ for argument in safe_to_ignore_arguments: curl_parser.add_argument(*argument, action="store_true") -def _parse_headers_and_cookies(parsed_args): - headers = [] - cookies = {} +def _parse_headers_and_cookies( + parsed_args: argparse.Namespace, +) -> Tuple[List[Tuple[str, bytes]], Dict[str, str]]: + headers: List[Tuple[str, bytes]] = [] + cookies: Dict[str, str] = {} for header in parsed_args.headers or (): name, val = header.split(":", 1) name = name.strip() @@ -64,7 +73,7 @@ def _parse_headers_and_cookies(parsed_args): def curl_to_request_kwargs( curl_command: str, ignore_unknown_options: bool = True -) -> dict: +) -> Dict[str, Any]: """Convert a cURL command syntax to Request kwargs. :param str curl_command: string containing the curl command @@ -98,7 +107,7 @@ def curl_to_request_kwargs( method = parsed_args.method or "GET" - result = {"method": method.upper(), "url": url} + result: Dict[str, Any] = {"method": method.upper(), "url": url} headers, cookies = _parse_headers_and_cookies(parsed_args) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 0ba2fe4e2..d06887610 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -110,7 +110,7 @@ class CaseInsensitiveDict(collections.UserDict): as keys and allows case-insensitive lookups. """ - def __init__(self, *args, **kwargs) -> None: + def __init__(self, *args: Any, **kwargs: Any) -> None: self._keys: dict = {} super().__init__(*args, **kwargs) @@ -196,8 +196,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): class SequenceExclude: """Object to test if an item is NOT within some sequence.""" - def __init__(self, seq: Sequence): - self.seq: Sequence = seq + def __init__(self, seq: Sequence[Any]): + self.seq: Sequence[Any] = seq def __contains__(self, item: Any) -> bool: return item not in self.seq diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 7e82dd519..2240f0b58 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -4,8 +4,8 @@ import warnings from functools import wraps from typing import TYPE_CHECKING, Any, Callable, TypeVar -from twisted.internet import defer, threads -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, maybeDeferred +from twisted.internet.threads import deferToThread from scrapy.exceptions import ScrapyDeprecationWarning @@ -48,7 +48,7 @@ def defers(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: @wraps(func) def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]: - return defer.maybeDeferred(func, *a, **kw) + return maybeDeferred(func, *a, **kw) return wrapped @@ -60,6 +60,6 @@ def inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: @wraps(func) def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]: - return threads.deferToThread(func, *a, **kw) + return deferToThread(func, *a, **kw) return wrapped diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index abb7e1726..c5763a06c 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio import inspect +import warnings from asyncio import Future from functools import wraps from types import CoroutineType @@ -33,21 +34,23 @@ from twisted.internet import defer from twisted.internet.defer import Deferred, DeferredList, ensureDeferred from twisted.internet.task import Cooperator from twisted.python import failure -from twisted.python.failure import Failure -from scrapy.exceptions import IgnoreRequest +from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed if TYPE_CHECKING: + from twisted.python.failure import Failure + # typing.Concatenate and typing.ParamSpec require Python 3.10 from typing_extensions import Concatenate, ParamSpec _P = ParamSpec("_P") _T = TypeVar("_T") +_T2 = TypeVar("_T2") -def defer_fail(_failure: Failure) -> Deferred: +def defer_fail(_failure: Failure) -> Deferred[Any]: """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -56,12 +59,12 @@ def defer_fail(_failure: Failure) -> Deferred: """ from twisted.internet import reactor - d: Deferred = Deferred() + d: Deferred[Any] = Deferred() reactor.callLater(0.1, d.errback, _failure) return d -def defer_succeed(result: Any) -> Deferred: +def defer_succeed(result: _T) -> Deferred[_T]: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop @@ -70,12 +73,12 @@ def defer_succeed(result: Any) -> Deferred: """ from twisted.internet import reactor - d: Deferred = Deferred() + d: Deferred[_T] = Deferred() reactor.callLater(0.1, d.callback, result) return d -def defer_result(result: Any) -> Deferred: +def defer_result(result: Any) -> Deferred[Any]: if isinstance(result, Deferred): return result if isinstance(result, failure.Failure): @@ -127,17 +130,17 @@ def mustbe_deferred( def parallel( iterable: Iterable[_T], count: int, - callable: Callable[Concatenate[_T, _P], Any], + callable: Callable[Concatenate[_T, _P], _T2], *args: _P.args, **named: _P.kwargs, -) -> Deferred: +) -> Deferred[List[Tuple[bool, Iterator[_T2]]]]: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. Taken from: https://jcalderone.livejournal.com/24285.html """ coop = Cooperator() - work = (callable(elem, *args, **named) for elem in iterable) + work: Iterator[_T2] = (callable(elem, *args, **named) for elem in iterable) return DeferredList([coop.coiterate(work) for _ in range(count)]) @@ -190,16 +193,16 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]): def __init__( self, aiterable: AsyncIterable[_T], - callable: Callable[Concatenate[_T, _P], Any], + callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]], *callable_args: _P.args, **callable_kwargs: _P.kwargs, ): self.aiterator: AsyncIterator[_T] = aiterable.__aiter__() - self.callable: Callable[Concatenate[_T, _P], Any] = callable + self.callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]] = callable self.callable_args: Tuple[Any, ...] = callable_args self.callable_kwargs: Dict[str, Any] = callable_kwargs self.finished: bool = False - self.waiting_deferreds: List[Deferred] = [] + self.waiting_deferreds: List[Deferred[Any]] = [] self.anext_deferred: Optional[Deferred[_T]] = None def _callback(self, result: _T) -> None: @@ -233,12 +236,12 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]): self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) self.anext_deferred.addCallbacks(self._callback, self._errback) - def __next__(self) -> Deferred: + def __next__(self) -> Deferred[Any]: # This puts a new Deferred into self.waiting_deferreds and returns it. # It also calls __anext__() if needed. if self.finished: raise StopIteration - d: Deferred = Deferred() + d: Deferred[Any] = Deferred() self.waiting_deferreds.append(d) if not self.anext_deferred: self._call_anext() @@ -248,25 +251,29 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]): def parallel_async( async_iterable: AsyncIterable[_T], count: int, - callable: Callable[Concatenate[_T, _P], Any], + callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]], *args: _P.args, **named: _P.kwargs, -) -> Deferred: - """Like parallel but for async iterators""" +) -> Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]]: + """Like ``parallel`` but for async iterators""" coop = Cooperator() - work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named) - dl: Deferred = DeferredList([coop.coiterate(work) for _ in range(count)]) + work: Iterator[Deferred[Any]] = _AsyncCooperatorAdapter( + async_iterable, callable, *args, **named + ) + dl: Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]] = DeferredList( + [coop.coiterate(work) for _ in range(count)] + ) return dl def process_chain( - callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], - input: Any, + callbacks: Iterable[Callable[Concatenate[_T, _P], _T]], + input: _T, *a: _P.args, **kw: _P.kwargs, -) -> Deferred: +) -> Deferred[_T]: """Return a Deferred built by chaining the given callbacks""" - d: Deferred = Deferred() + d: Deferred[_T] = Deferred() for x in callbacks: d.addCallback(x, *a, **kw) d.callback(input) @@ -281,6 +288,12 @@ def process_chain_both( **kw: _P.kwargs, ) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" + warnings.warn( + "process_chain_both() is deprecated and will be removed in a future" + " Scrapy version.", + ScrapyDeprecationWarning, + stacklevel=2, + ) d: Deferred = Deferred() for cb, eb in zip(callbacks, errbacks): d.addCallback(cb, *a, **kw) @@ -293,19 +306,21 @@ def process_chain_both( def process_parallel( - callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], - input: Any, + callbacks: Iterable[Callable[Concatenate[_T, _P], _T2]], + input: _T, *a: _P.args, **kw: _P.kwargs, -) -> Deferred: +) -> Deferred[List[_T2]]: """Return a Deferred with the output of all successful calls to the given callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d: Deferred = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) - d.addCallback(lambda r: [x[1] for x in r]) - d.addErrback(lambda f: f.value.subFailure) - return d + d: Deferred[List[Tuple[bool, _T2]]] = DeferredList( + dfds, fireOnOneErrback=True, consumeErrors=True + ) + d2: Deferred[List[_T2]] = d.addCallback(lambda r: [x[1] for x in r]) + d2.addErrback(lambda f: f.value.subFailure) + return d2 def iter_errback( @@ -390,11 +405,11 @@ def deferred_f_from_coro_f( def maybeDeferred_coro( f: Callable[_P, Any], *args: _P.args, **kw: _P.kwargs -) -> Deferred: +) -> Deferred[Any]: """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" try: result = f(*args, **kw) - except: # noqa: E722 + except: # noqa: E722,B001 return defer.fail(failure.Failure(captureVars=Deferred.debug)) if isinstance(result, Deferred): @@ -406,7 +421,7 @@ def maybeDeferred_coro( return defer.succeed(result) -def deferred_to_future(d: Deferred) -> Future: +def deferred_to_future(d: Deferred[_T]) -> Future[_T]: """ .. versionadded:: 2.6.0 @@ -428,7 +443,7 @@ def deferred_to_future(d: Deferred) -> Future: return d.asFuture(_get_asyncio_event_loop()) -def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: +def maybe_deferred_to_future(d: Deferred[_T]) -> Union[Deferred[_T], Future[_T]]: """ .. versionadded:: 2.6.0 diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index fdcf484d4..770ee0b1b 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -4,9 +4,10 @@ from __future__ import annotations # used in global tests code from time import time # noqa: F401 -from typing import Any, List, Tuple +from typing import TYPE_CHECKING, Any, List, Tuple -from scrapy.core.engine import ExecutionEngine +if TYPE_CHECKING: + from scrapy.core.engine import ExecutionEngine def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]: diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index c77681a53..152f3374e 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -21,7 +21,7 @@ def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None: def ftp_store_file( *, path: str, - file: IO, + file: IO[bytes], host: str, port: int, username: str, diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 2e487d88b..85324361c 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,11 +1,15 @@ +from __future__ import annotations + import struct from gzip import GzipFile from io import BytesIO - -from scrapy.http import Response +from typing import TYPE_CHECKING from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded +if TYPE_CHECKING: + from scrapy.http import Response + def gunzip(data: bytes, *, max_size: int = 0) -> bytes: """Gunzip the given data and return as much data as possible. diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index d502e8910..3cf9585ec 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -1,12 +1,16 @@ """Helper functions for scrapy.http objects (Request, Response)""" -from typing import Union +from __future__ import annotations + +from typing import TYPE_CHECKING, Union from urllib.parse import ParseResult, urlparse from weakref import WeakKeyDictionary -from scrapy.http import Request, Response +if TYPE_CHECKING: + from scrapy.http import Request, Response -_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = ( + +_urlparse_cache: WeakKeyDictionary[Union[Request, Response], ParseResult] = ( WeakKeyDictionary() ) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index cd6e9d04e..41a842386 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -6,8 +6,7 @@ from typing import ( Any, Callable, Dict, - Generator, - Iterable, + Iterator, List, Literal, Optional, @@ -22,14 +21,12 @@ from lxml import etree # nosec from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse from scrapy.selector import Selector -from scrapy.utils.python import re_rsearch, to_unicode +from scrapy.utils.python import re_rsearch logger = logging.getLogger(__name__) -def xmliter( - obj: Union[Response, str, bytes], nodename: str -) -> Generator[Selector, Any, None]: +def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Iterator[Selector]: """Return a iterator of Selector's over all nodes of a XML document, given the name of the node to iterate. Useful for parsing XML feeds. @@ -90,7 +87,7 @@ def xmliter_lxml( nodename: str, namespace: Optional[str] = None, prefix: str = "x", -) -> Generator[Selector, Any, None]: +) -> Iterator[Selector]: reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( @@ -168,7 +165,7 @@ def csviter( headers: Optional[List[str]] = None, encoding: Optional[str] = None, quotechar: Optional[str] = None, -) -> Generator[Dict[str, str], Any, None]: +) -> Iterator[Dict[str, str]]: """Returns an iterator of dictionaries from the given csv object obj can be: @@ -184,10 +181,13 @@ def csviter( quotechar is the character used to enclosure fields on the given obj. """ - encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8" - - def row_to_unicode(row_: Iterable) -> List[str]: - return [to_unicode(field, encoding) for field in row_] + if encoding is not None: + warn( + "The encoding argument of csviter() is ignored and will be removed" + " in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) lines = StringIO(_body_or_str(obj, unicode=True)) @@ -200,13 +200,11 @@ def csviter( if not headers: try: - row = next(csv_r) + headers = next(csv_r) except StopIteration: return - headers = row_to_unicode(row) for row in csv_r: - row = row_to_unicode(row) if len(row) != len(headers): logger.warning( "ignoring row %(csvlnum)d (length: %(csvrow)d, " diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index e230e4235..488c7994b 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -1,7 +1,10 @@ -from pathlib import Path -from typing import Optional +from __future__ import annotations -from scrapy.settings import BaseSettings +from pathlib import Path +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from scrapy.settings import BaseSettings def job_dir(settings: BaseSettings) -> Optional[str]: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 430a91e95..439b065a9 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -7,6 +7,7 @@ from types import TracebackType from typing import ( TYPE_CHECKING, Any, + Dict, List, MutableMapping, Optional, @@ -20,11 +21,13 @@ from twisted.python import log as twisted_log from twisted.python.failure import Failure import scrapy -from scrapy.settings import Settings +from scrapy.settings import Settings, _SettingsKeyT from scrapy.utils.versions import scrapy_components_versions if TYPE_CHECKING: from scrapy.crawler import Crawler + from scrapy.logformatter import LogFormatterResult + logger = logging.getLogger(__name__) @@ -86,7 +89,8 @@ DEFAULT_LOGGING = { def configure_logging( - settings: Union[Settings, dict, None] = None, install_root_handler: bool = True + settings: Union[Settings, Dict[_SettingsKeyT, Any], None] = None, + install_root_handler: bool = True, ) -> None: """ Initialize logging defaults for Scrapy. @@ -234,7 +238,9 @@ class LogCounterHandler(logging.Handler): self.crawler.stats.inc_value(sname) -def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]: +def logformatter_adapter( + logkws: LogFormatterResult, +) -> Tuple[int, str, Union[Dict[str, Any], Tuple[Any, ...]]]: """ Helper that takes the dictionary output from the methods in LogFormatter and adapts it into a tuple of positional arguments for logger.log calls, @@ -245,7 +251,7 @@ def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]: message = logkws.get("msg") or "" # NOTE: This also handles 'args' being an empty dict, that case doesn't # play well in logger.log calls - args = logkws if not logkws.get("args") else logkws["args"] + args = cast(Dict[str, Any], logkws) if not logkws.get("args") else logkws["args"] return (level, message, args) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index faf52e44a..3c787e50f 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -13,15 +13,14 @@ from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules -from types import ModuleType from typing import ( IO, TYPE_CHECKING, Any, Callable, Deque, - Generator, Iterable, + Iterator, List, Optional, Type, @@ -35,10 +34,13 @@ from scrapy.item import Item from scrapy.utils.datatypes import LocalWeakReferencedCache if TYPE_CHECKING: + from types import ModuleType + from scrapy import Spider from scrapy.crawler import Crawler from scrapy.settings import BaseSettings + _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes T = TypeVar("T") @@ -56,7 +58,7 @@ def arg_to_iter(arg: Any) -> Iterable[Any]: return [arg] -def load_object(path: Union[str, Callable]) -> Any: +def load_object(path: Union[str, Callable[..., Any]]) -> Any: """Load an object given its absolute object path, and return it. The object can be the import path of a class, function, variable or an @@ -111,7 +113,7 @@ def walk_modules(path: str) -> List[ModuleType]: return mods -def md5sum(file: IO) -> str: +def md5sum(file: IO[bytes]) -> str: """Calculate the md5 checksum of a file-like object without reading its whole content in memory. @@ -227,7 +229,7 @@ def build_from_settings( @contextmanager -def set_environ(**kwargs: str) -> Generator[None, Any, None]: +def set_environ(**kwargs: str) -> Iterator[None]: """Temporarily set environment variables inside the context manager and fully restore previous environment afterwards """ @@ -244,7 +246,7 @@ def set_environ(**kwargs: str) -> Generator[None, Any, None]: os.environ[k] = v -def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]: +def walk_callable(node: ast.AST) -> Iterable[ast.AST]: """Similar to ``ast.walk``, but walks only function body and skips nested functions defined within the node. """ @@ -263,7 +265,7 @@ def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]: _generator_callbacks_cache = LocalWeakReferencedCache(limit=128) -def is_generator_with_return_value(callable: Callable) -> bool: +def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: """ Returns True if a callable is a generator function which includes a 'return' statement with a value different than None, False otherwise @@ -300,7 +302,9 @@ def is_generator_with_return_value(callable: Callable) -> bool: return bool(_generator_callbacks_cache[callable]) -def warn_on_generator_with_return_value(spider: Spider, callable: Callable) -> None: +def warn_on_generator_with_return_value( + spider: Spider, callable: Callable[..., Any] +) -> None: """ Logs a warning if a callable is a generator function and includes a 'return' statement with a value different than None diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index de3c8eaf9..efb6af299 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,7 +1,8 @@ +from __future__ import annotations + import os import warnings from importlib import import_module -from os import PathLike from pathlib import Path from typing import Union @@ -46,7 +47,7 @@ def project_data_dir(project: str = "default") -> str: return str(d) -def data_path(path: Union[str, PathLike], createdir: bool = False) -> str: +def data_path(path: Union[str, os.PathLike[str]], createdir: bool = False) -> str: """ Return the given path joined with the .scrapy data directory. If given an absolute path, return it unmodified. diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 578cde2ac..f56950fdd 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -15,12 +15,10 @@ from itertools import chain from typing import ( TYPE_CHECKING, Any, - AsyncGenerator, AsyncIterable, AsyncIterator, Callable, Dict, - Generator, Iterable, Iterator, List, @@ -42,9 +40,11 @@ if TYPE_CHECKING: _P = ParamSpec("_P") _T = TypeVar("_T") +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") -def flatten(x: Iterable) -> list: +def flatten(x: Iterable[Any]) -> List[Any]: """flatten(sequence) -> list Returns a single, flat list which contains all elements retrieved @@ -64,7 +64,7 @@ def flatten(x: Iterable) -> list: return list(iflatten(x)) -def iflatten(x: Iterable) -> Iterable: +def iflatten(x: Iterable[Any]) -> Iterable[Any]: """iflatten(sequence) -> iterator Similar to ``.flatten()``, but returns iterator instead""" @@ -99,10 +99,10 @@ def is_listlike(x: Any) -> bool: return hasattr(x, "__iter__") and not isinstance(x, (str, bytes)) -def unique(list_: Iterable, key: Callable[[Any], Any] = lambda x: x) -> list: +def unique(list_: Iterable[_T], key: Callable[[_T], Any] = lambda x: x) -> List[_T]: """efficient function to uniquify a list preserving item order""" seen = set() - result = [] + result: List[_T] = [] for item in list_: seenkey = key(item) if seenkey in seen: @@ -146,7 +146,7 @@ def to_bytes( def re_rsearch( - pattern: Union[str, Pattern], text: str, chunk_size: int = 1024 + pattern: Union[str, Pattern[str]], text: str, chunk_size: int = 1024 ) -> Optional[Tuple[int, int]]: """ This function does a reverse search in a text using a regular expression @@ -161,7 +161,7 @@ def re_rsearch( the start position of the match, and the ending (regarding the entire text). """ - def _chunk_iter() -> Generator[Tuple[str, int], Any, None]: + def _chunk_iter() -> Iterable[Tuple[str, int]]: offset = len(text) while True: offset -= chunk_size * 1024 @@ -215,7 +215,7 @@ def binary_is_text(data: bytes) -> bool: return all(c not in _BINARYCHARS for c in data) -def get_func_args(func: Callable, stripself: bool = False) -> List[str]: +def get_func_args(func: Callable[..., Any], stripself: bool = False) -> List[str]: """Return the argument name list of a callable object""" if not callable(func): raise TypeError(f"func must be callable, got '{type(func).__name__}'") @@ -245,7 +245,7 @@ def get_func_args(func: Callable, stripself: bool = False) -> List[str]: return args -def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]: +def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]: """Returns (args, kwargs) tuple for a function >>> import re >>> get_spec(re.match) @@ -269,7 +269,7 @@ def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]: if inspect.isfunction(func) or inspect.ismethod(func): spec = inspect.getfullargspec(func) - elif hasattr(func, "__call__"): + elif hasattr(func, "__call__"): # noqa: B004 spec = inspect.getfullargspec(func.__call__) else: raise TypeError(f"{type(func)} is not callable") @@ -283,7 +283,7 @@ def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]: def equal_attributes( - obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable]]] + obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable[[Any], Any]]]] ) -> bool: """Compare two objects attributes""" # not attributes given return False by default @@ -303,14 +303,16 @@ def equal_attributes( @overload -def without_none_values(iterable: Mapping) -> dict: ... +def without_none_values(iterable: Mapping[_KT, _VT]) -> Dict[_KT, _VT]: ... @overload -def without_none_values(iterable: Iterable) -> Iterable: ... +def without_none_values(iterable: Iterable[_KT]) -> Iterable[_KT]: ... -def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]: +def without_none_values( + iterable: Union[Mapping[_KT, _VT], Iterable[_KT]] +) -> Union[Dict[_KT, _VT], Iterable[_KT]]: """Return a copy of ``iterable`` with all ``None`` entries removed. If ``iterable`` is a mapping, return a dictionary where all pairs that have @@ -347,43 +349,45 @@ else: gc.collect() -class MutableChain(Iterable): +class MutableChain(Iterable[_T]): """ Thin wrapper around itertools.chain, allowing to add iterables "in-place" """ - def __init__(self, *args: Iterable): - self.data = chain.from_iterable(args) + def __init__(self, *args: Iterable[_T]): + self.data: Iterator[_T] = chain.from_iterable(args) - def extend(self, *iterables: Iterable) -> None: + def extend(self, *iterables: Iterable[_T]) -> None: self.data = chain(self.data, chain.from_iterable(iterables)) - def __iter__(self) -> Iterator: + def __iter__(self) -> Iterator[_T]: return self - def __next__(self) -> Any: + def __next__(self) -> _T: return next(self.data) -async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator: +async def _async_chain( + *iterables: Union[Iterable[_T], AsyncIterable[_T]] +) -> AsyncIterator[_T]: for it in iterables: async for o in as_async_generator(it): yield o -class MutableAsyncChain(AsyncIterable): +class MutableAsyncChain(AsyncIterable[_T]): """ Similar to MutableChain but for async iterables """ - def __init__(self, *args: Union[Iterable, AsyncIterable]): - self.data = _async_chain(*args) + def __init__(self, *args: Union[Iterable[_T], AsyncIterable[_T]]): + self.data: AsyncIterator[_T] = _async_chain(*args) - def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None: + def extend(self, *iterables: Union[Iterable[_T], AsyncIterable[_T]]) -> None: self.data = _async_chain(self.data, _async_chain(*iterables)) - def __aiter__(self) -> AsyncIterator: + def __aiter__(self) -> AsyncIterator[_T]: return self - async def __anext__(self) -> Any: + async def __anext__(self) -> _T: return await self.data.__anext__() diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 5af6d22eb..a627db601 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -2,7 +2,6 @@ from __future__ import annotations import asyncio import sys -from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from contextlib import suppress from typing import ( TYPE_CHECKING, @@ -20,13 +19,16 @@ from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error from twisted.internet.base import DelayedCall -from twisted.internet.protocol import ServerFactory -from twisted.internet.tcp import Port from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import load_object if TYPE_CHECKING: + from asyncio import AbstractEventLoop, AbstractEventLoopPolicy + + from twisted.internet.protocol import ServerFactory + from twisted.internet.tcp import Port + # typing.ParamSpec requires Python 3.10 from typing_extensions import ParamSpec diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index c86f9fe39..99ca3b7a0 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -12,7 +12,6 @@ from typing import ( TYPE_CHECKING, Any, Dict, - Generator, Iterable, List, Optional, @@ -40,18 +39,16 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler -def _serialize_headers( - headers: Iterable[bytes], request: Request -) -> Generator[bytes, Any, None]: +def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[bytes]: for header in headers: if header in request.headers: yield header yield from request.headers.getlist(header) -_fingerprint_cache: ( - "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" -) +_fingerprint_cache: WeakKeyDictionary[ + Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes] +] _fingerprint_cache = WeakKeyDictionary() @@ -138,7 +135,7 @@ class RequestFingerprinter: """ @classmethod - def from_crawler(cls, crawler) -> Self: + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) def __init__(self, crawler: Optional[Crawler] = None): @@ -197,7 +194,7 @@ def referer_str(request: Request) -> Optional[str]: return to_unicode(referrer, errors="replace") -def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request: +def request_from_dict(d: Dict[str, Any], *, spider: Optional[Spider] = None) -> Request: """Create a :class:`~scrapy.Request` object from a dict. If a spider is given, it will try to resolve the callbacks looking at the diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index a0b06f75c..320059b3a 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -58,7 +58,7 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str: return f"{status_int} {to_unicode(message)}" -def _remove_html_comments(body): +def _remove_html_comments(body: bytes) -> bytes: start = body.find(b"", start + 1) diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 89cfbd2ec..4310c1d56 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,5 +1,7 @@ """Helper functions for working with signals""" +from __future__ import annotations + import collections.abc import logging from typing import Any as TypingAny @@ -27,7 +29,7 @@ def send_catch_log( signal: TypingAny = Any, sender: TypingAny = Anonymous, *arguments: TypingAny, - **named: TypingAny + **named: TypingAny, ) -> List[Tuple[TypingAny, TypingAny]]: """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. @@ -73,8 +75,8 @@ def send_catch_log_deferred( signal: TypingAny = Any, sender: TypingAny = Anonymous, *arguments: TypingAny, - **named: TypingAny -) -> Deferred: + **named: TypingAny, +) -> Deferred[List[Tuple[TypingAny, TypingAny]]]: """Like send_catch_log but supports returning deferreds on signal handlers. Returns a deferred that gets fired once all signal handlers deferreds were fired. @@ -92,20 +94,25 @@ def send_catch_log_deferred( dont_log = named.pop("dont_log", None) spider = named.get("spider", None) - dfds = [] + dfds: List[Deferred[Tuple[TypingAny, TypingAny]]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): - d = maybeDeferred_coro( + d: Deferred[TypingAny] = maybeDeferred_coro( robustApply, receiver, signal=signal, sender=sender, *arguments, **named ) d.addErrback(logerror, receiver) # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html - d.addBoth( - lambda result: (receiver, result) # pylint: disable=cell-var-from-loop + d2: Deferred[Tuple[TypingAny, TypingAny]] = d.addBoth( + lambda result: ( + receiver, # pylint: disable=cell-var-from-loop # noqa: B023 + result, + ) ) - dfds.append(d) - d = DeferredList(dfds) - d.addCallback(lambda out: [x[1] for x in out]) - return d + dfds.append(d2) + dl = DeferredList(dfds) + d3: Deferred[List[Tuple[TypingAny, TypingAny]]] = dl.addCallback( + lambda out: [x[1] for x in out] + ) + return d3 def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None: diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index cf429043d..7a91afe59 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -5,7 +5,7 @@ Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ -from typing import Any, Dict, Generator, Iterator, Optional, Union +from typing import Any, Dict, Iterable, Iterator, Optional, Union from urllib.parse import urljoin import lxml.etree # nosec @@ -42,7 +42,7 @@ class Sitemap: def sitemap_urls_from_robots( robots_text: str, base_url: Optional[str] = None -) -> Generator[str, Any, None]: +) -> Iterable[str]: """Return an iterator over all sitemap urls contained in the given robots.txt file """ diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index cbbb01d85..ce754fad3 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -2,12 +2,10 @@ from __future__ import annotations import inspect import logging -from types import CoroutineType, ModuleType from typing import ( TYPE_CHECKING, Any, AsyncGenerator, - Generator, Iterable, Literal, Optional, @@ -17,16 +15,19 @@ from typing import ( overload, ) -from twisted.internet.defer import Deferred - -from scrapy import Request from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter if TYPE_CHECKING: + from types import CoroutineType, ModuleType + + from twisted.internet.defer import Deferred + + from scrapy import Request from scrapy.spiderloader import SpiderLoader + logger = logging.getLogger(__name__) _T = TypeVar("_T") @@ -34,18 +35,20 @@ _T = TypeVar("_T") # https://stackoverflow.com/questions/60222982 @overload -def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: ... # type: ignore[overload-overlap] +def iterate_spider_output(result: AsyncGenerator[_T, None]) -> AsyncGenerator[_T, None]: ... # type: ignore[overload-overlap] @overload -def iterate_spider_output(result: CoroutineType) -> Deferred: ... +def iterate_spider_output(result: CoroutineType[Any, Any, _T]) -> Deferred[_T]: ... @overload -def iterate_spider_output(result: _T) -> Iterable: ... +def iterate_spider_output(result: _T) -> Iterable[Any]: ... -def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]: +def iterate_spider_output( + result: Any, +) -> Union[Iterable[Any], AsyncGenerator[_T, None], Deferred[_T]]: if inspect.isasyncgen(result): return result if inspect.iscoroutine(result): @@ -55,7 +58,7 @@ def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferr return arg_to_iter(deferred_from_coro(result)) -def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]: +def iter_spider_classes(module: ModuleType) -> Iterable[Type[Spider]]: """Return an iterator over all spider classes defined in the given module that can be instantiated (i.e. which have name) """ diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index d520ef809..95611ebd9 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,12 +1,16 @@ -from typing import Any, Optional +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL import OpenSSL.version -from OpenSSL.crypto import X509Name from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + from OpenSSL.crypto import X509Name + def ffi_buf_to_string(buf: Any) -> str: return to_unicode(pyOpenSSLutil.ffi.string(buf)) diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 6b22f3bfa..08f3f2dc9 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -1,10 +1,14 @@ """Helper functions for working with templates""" +from __future__ import annotations + import re import string -from os import PathLike from pathlib import Path -from typing import Any, Union +from typing import TYPE_CHECKING, Any, Union + +if TYPE_CHECKING: + from os import PathLike def render_templatefile(path: Union[str, PathLike], **kwargs: Any) -> None: diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 7a8c5c859..30f235592 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -2,21 +2,39 @@ This module contains some assorted functions used in tests """ +from __future__ import annotations + import asyncio import os from importlib import import_module from pathlib import Path from posixpath import split -from typing import Any, Coroutine, Dict, List, Optional, Tuple, Type +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Dict, + List, + Optional, + Tuple, + Type, + TypeVar, +) from unittest import TestCase, mock -from twisted.internet.defer import Deferred from twisted.trial.unittest import SkipTest from scrapy import Spider from scrapy.crawler import Crawler from scrapy.utils.boto import is_botocore_available +if TYPE_CHECKING: + from twisted.internet.defer import Deferred + from twisted.web.client import Response as TxResponse + + +_T = TypeVar("_T") + def assert_gcs_environ() -> None: if "GCS_PROJECT_ID" not in os.environ: @@ -118,8 +136,8 @@ def assert_samelines( testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) -def get_from_asyncio_queue(value: Any) -> Coroutine: - q: asyncio.Queue = asyncio.Queue() +def get_from_asyncio_queue(value: _T) -> Awaitable[_T]: + q: asyncio.Queue[_T] = asyncio.Queue() getter = q.get() q.put_nowait(value) return getter @@ -142,7 +160,7 @@ def mock_google_cloud_storage() -> Tuple[Any, Any, Any]: return (client_mock, bucket_mock, blob_mock) -def get_web_client_agent_req(url: str) -> Deferred: +def get_web_client_agent_req(url: str) -> Deferred[TxResponse]: from twisted.internet import reactor from twisted.web.client import Agent # imports twisted.internet.reactor diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 3bdffcaa7..bb269a9f5 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -2,12 +2,14 @@ from __future__ import annotations import os import sys -from typing import Iterable, List, Optional, Tuple, cast +from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, cast from twisted.internet.defer import Deferred from twisted.internet.error import ProcessTerminated from twisted.internet.protocol import ProcessProtocol -from twisted.python.failure import Failure + +if TYPE_CHECKING: + from twisted.python.failure import Failure class ProcessTest: @@ -20,7 +22,7 @@ class ProcessTest: args: Iterable[str], check_code: bool = True, settings: Optional[str] = None, - ) -> Deferred: + ) -> Deferred[TestProcessProtocol]: from twisted.internet import reactor env = os.environ.copy() @@ -47,7 +49,7 @@ class ProcessTest: class TestProcessProtocol(ProcessProtocol): def __init__(self) -> None: - self.deferred: Deferred = Deferred() + self.deferred: Deferred[TestProcessProtocol] = Deferred() self.out: bytes = b"" self.err: bytes = b"" self.exitcode: Optional[int] = None diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index de9ce992a..ca1f68116 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -15,12 +15,12 @@ class SiteTest: super().tearDown() self.site.stopListening() - def url(self, path): + def url(self, path: str) -> str: return urljoin(self.baseurl, path) class NoMetaRefreshRedirect(util.Redirect): - def render(self, request): + def render(self, request: server.Request) -> bytes: content = util.Redirect.render(self, request) return content.replace( b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"' diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py new file mode 100644 index 000000000..b20aa0c7c --- /dev/null +++ b/tests/CrawlerRunner/change_reactor.py @@ -0,0 +1,31 @@ +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + def start_requests(self): + return [] + + +configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"}) + + +from scrapy.utils.reactor import install_reactor + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + +runner = CrawlerRunner() + +d = runner.crawl(NoRequestsSpider) + +from twisted.internet import reactor + +d.addBoth(callback=lambda _: reactor.stop()) +reactor.run() diff --git a/tests/mockserver.py b/tests/mockserver.py index 647b0682e..6ec46aa3d 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import json import os @@ -7,12 +9,11 @@ from pathlib import Path from shutil import rmtree from subprocess import PIPE, Popen from tempfile import mkdtemp -from typing import Dict +from typing import TYPE_CHECKING, Dict from urllib.parse import urlencode from OpenSSL import SSL from twisted.internet import defer, reactor, ssl -from twisted.internet.protocol import ServerFactory from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory @@ -23,6 +24,9 @@ from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode +if TYPE_CHECKING: + from twisted.internet.protocol import ServerFactory + def getarg(request, name, default=None, type=None): if name in request.args: @@ -189,10 +193,10 @@ class Raw(LeafResource): class Echo(LeafResource): def render_GET(self, request): output = { - "headers": dict( - (to_unicode(k), [to_unicode(v) for v in vs]) + "headers": { + to_unicode(k): [to_unicode(v) for v in vs] for k, vs in request.requestHeaders.getAllRawHeaders() - ), + }, "body": to_unicode(request.content.read()), } return to_bytes(json.dumps(output)) diff --git a/tests/spiders.py b/tests/spiders.py index ea419afbd..743811893 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -283,6 +283,24 @@ class ItemSpider(FollowAllSpider): yield {} +class MaxItemsAndRequestsSpider(FollowAllSpider): + def __init__(self, max_items=10, max_requests=10, *args, **kwargs): + super().__init__(*args, **kwargs) + self.max_items = max_items + self.max_requests = max_requests + + def parse(self, response): + self.items_scraped = 0 + self.pages_crawled = 1 # account for the start url + for request in super().parse(response): + if self.pages_crawled < self.max_requests: + yield request + self.pages_crawled += 1 + if self.items_scraped < self.max_items: + yield Item() + self.items_scraped += 1 + + class DefaultError(Exception): pass diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 38ede70e4..9a837350f 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -3,7 +3,13 @@ from twisted.trial.unittest import TestCase from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider, SlowSpider +from tests.spiders import ( + ErrorSpider, + FollowAllSpider, + ItemSpider, + MaxItemsAndRequestsSpider, + SlowSpider, +) class TestCloseSpider(TestCase): @@ -34,6 +40,43 @@ class TestCloseSpider(TestCase): pagecount = crawler.stats.get_value("response_received_count") self.assertTrue(pagecount >= close_on) + @defer.inlineCallbacks + def test_closespider_pagecount_no_item(self): + close_on = 5 + max_items = 5 + max_requests = close_on + max_items + crawler = get_crawler( + MaxItemsAndRequestsSpider, + { + "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on, + }, + ) + yield crawler.crawl( + max_items=max_items, max_requests=max_requests, mockserver=self.mockserver + ) + reason = crawler.spider.meta["close_reason"] + self.assertEqual(reason, "closespider_pagecount_no_item") + pagecount = crawler.stats.get_value("response_received_count") + itemcount = crawler.stats.get_value("item_scraped_count") + self.assertLessEqual(pagecount, close_on + itemcount) + + @defer.inlineCallbacks + def test_closespider_pagecount_no_item_with_pagecount(self): + close_on_pagecount_no_item = 5 + close_on_pagecount = 20 + crawler = get_crawler( + FollowAllSpider, + { + "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on_pagecount_no_item, + "CLOSESPIDER_PAGECOUNT": close_on_pagecount, + }, + ) + yield crawler.crawl(mockserver=self.mockserver) + reason = crawler.spider.meta["close_reason"] + self.assertEqual(reason, "closespider_pagecount_no_item") + pagecount = crawler.stats.get_value("response_received_count") + self.assertLess(pagecount, close_on_pagecount) + @defer.inlineCallbacks def test_closespider_errorcount(self): close_on = 5 diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 25ded143c..4835e936b 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -20,7 +20,7 @@ class CmdlineTest(unittest.TestCase): self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings" def _execute(self, *new_args, **kwargs): - encoding = getattr(sys.stdout, "encoding") or "utf-8" + encoding = sys.stdout.encoding or "utf-8" args = (sys.executable, "-m", "scrapy.cmdline") + new_args proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs) comm = proc.communicate()[0].strip() diff --git a/tests/test_command_version.py b/tests/test_command_version.py index a52d0d13c..18c1c531c 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -12,7 +12,7 @@ class VersionTest(ProcessTest, unittest.TestCase): @defer.inlineCallbacks def test_output(self): - encoding = getattr(sys.stdout, "encoding") or "utf-8" + encoding = sys.stdout.encoding or "utf-8" _, out, _ = yield self.execute([]) self.assertEqual( out.strip().decode(encoding), @@ -21,7 +21,7 @@ class VersionTest(ProcessTest, unittest.TestCase): @defer.inlineCallbacks def test_verbose_output(self): - encoding = getattr(sys.stdout, "encoding") or "utf-8" + encoding = sys.stdout.encoding or "utf-8" _, out, _ = yield self.execute(["-v"]) headers = [ line.partition(":")[0].strip() diff --git a/tests/test_commands.py b/tests/test_commands.py index b9d468c66..a23b7f4a9 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -13,7 +13,7 @@ from shutil import copytree, rmtree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from tempfile import TemporaryFile, mkdtemp from threading import Timer -from typing import Dict, Generator, Optional, Union +from typing import Dict, Iterator, Optional, Union from unittest import skipIf from pytest import mark @@ -101,7 +101,7 @@ class ProjectTest(unittest.TestCase): def kill_proc(): p.kill() p.communicate() - assert False, "Command took too much time to complete" + raise AssertionError("Command took too much time to complete") timer = Timer(15, kill_proc) try: @@ -200,7 +200,7 @@ def get_permissions_dict( path_obj = Path(path) - renamings = renamings or tuple() + renamings = renamings or () permissions_dict = { ".": get_permissions(path_obj), } @@ -674,7 +674,7 @@ class BadSpider(scrapy.Spider): """ @contextmanager - def _create_file(self, content, name=None) -> Generator[str, None, None]: + def _create_file(self, content, name=None) -> Iterator[str]: tmpdir = Path(self.mktemp()) tmpdir.mkdir() if name: diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 1459e0b5f..c9c12f0d8 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -182,6 +182,19 @@ class TestSpider(Spider): """ pass + def invalid_regex(self, response): + """method with invalid regex + @ Scrapy is awsome + """ + pass + + def invalid_regex_with_valid_contract(self, response): + """method with invalid regex + @ scrapy is awsome + @url http://scrapy.org + """ + pass + class CustomContractSuccessSpider(Spider): name = "custom_contract_success_spider" @@ -385,6 +398,21 @@ class ContractsManagerTest(unittest.TestCase): message = "ContractFail: Missing fields: name, url" assert message in self.results.failures[-1][-1] + def test_regex(self): + spider = TestSpider() + response = ResponseMock() + + # invalid regex + request = self.conman.from_method(spider.invalid_regex, self.results) + self.should_succeed() + + # invalid regex with valid contract + request = self.conman.from_method( + spider.invalid_regex_with_valid_contract, self.results + ) + self.should_succeed() + request.callback(response) + def test_custom_contracts(self): self.conman.from_spider(CustomContractSuccessSpider(), self.results) self.should_succeed() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 989208694..c87e65758 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -21,7 +21,6 @@ import scrapy from scrapy import Spider from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.extensions import telnet from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader @@ -482,7 +481,6 @@ class CrawlerLoggingTestCase(unittest.TestCase): "LOG_FILE": str(log_file), # settings to avoid extra warnings "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", - "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE, } configure_logging() @@ -516,8 +514,6 @@ class CrawlerLoggingTestCase(unittest.TestCase): custom_settings = { "LOG_FILE": str(log_file), "LOG_FILE_APPEND": False, - # disable telnet if not available to avoid an extra warning - "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE, } configure_logging() @@ -926,3 +922,11 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("INFO: Host: not.a.real.domain", log) self.assertIn("INFO: Type: ", log) self.assertIn("INFO: IP address: 127.0.0.1", log) + + def test_change_default_reactor(self): + log = self.run_script("change_reactor.py") + self.assertIn( + "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", + log, + ) + self.assertIn("DEBUG: Using asyncio event loop", log) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index d3fd63847..884491d01 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -892,7 +892,7 @@ class S3TestCase(unittest.TestCase): except Exception as e: self.assertIsInstance(e, (TypeError, NotConfigured)) else: - assert False + raise AssertionError() def test_request_signing1(self): # gets an object from the johnsmith bucket. diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 0155c62eb..dd3f8ceb9 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -36,7 +36,7 @@ class ManagerTestCase(TestCase): if not response: response = Response(request.url) - def download_func(**kwargs): + def download_func(request, spider): return response dfd = self.mwman.download(download_func, request, self.spider) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 5eccd396a..6e343d035 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -362,7 +362,7 @@ class CookiesMiddlewareTest(TestCase): def test_request_cookies_encoding(self): # 1) UTF8-encoded bytes - req1 = Request("http://example.org", cookies={"a": "á".encode("utf8")}) + req1 = Request("http://example.org", cookies={"a": "á".encode()}) assert self.mw.process_request(req1, self.spider) is None self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1") @@ -379,7 +379,7 @@ class CookiesMiddlewareTest(TestCase): @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_request_headers_cookie_encoding(self): # 1) UTF8-encoded bytes - req1 = Request("http://example.org", headers={"Cookie": "a=á".encode("utf8")}) + req1 = Request("http://example.org", headers={"Cookie": "a=á".encode()}) assert self.mw.process_request(req1, self.spider) is None self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1") diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 4bfd34fe2..e37da9715 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1125,7 +1125,7 @@ class RedirectMiddlewareTest(Base.Test): def test_utf8_location(self): req = Request("http://scrapytest.org/first") - utf8_location = "/ação".encode("utf-8") # header using UTF-8 encoding + utf8_location = "/ação".encode() # header using UTF-8 encoding resp = Response( "http://scrapytest.org/first", headers={"Location": utf8_location}, diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 26898a6a1..e166cc000 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -40,9 +40,7 @@ Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: Disallow: /wiki/Käyttäjä: User-Agent: UnicödeBöt Disallow: /some/randome/page.html -""".encode( - "utf-8" - ) +""".encode() response = TextResponse("http://site.local/robots.txt", body=ROBOTS) def return_response(request): diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index aa0975555..f617fc027 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -146,7 +146,7 @@ class RFPDupeFilterTest(unittest.TestCase): case_insensitive_dupefilter.close("finished") def test_seenreq_newlines(self): - """Checks against adding duplicate \r to + r"""Checks against adding duplicate \r to line endings on Windows platforms.""" r1 = Request("http://scrapytest.org/1") diff --git a/tests/test_engine.py b/tests/test_engine.py index 33544e8db..86526420f 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -459,7 +459,7 @@ class EngineTest(unittest.TestCase): def kill_proc(): p.kill() p.communicate() - assert False, "Command took too much time to complete" + raise AssertionError("Command took too much time to complete") timer = Timer(15, kill_proc) try: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d7560b5ff..ea3ed3b05 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import bz2 import csv import gzip @@ -14,10 +16,9 @@ from collections import defaultdict from contextlib import ExitStack from io import BytesIO from logging import getLogger -from os import PathLike from pathlib import Path from string import ascii_letters, digits -from typing import Union +from typing import TYPE_CHECKING, Union from unittest import mock from urllib.parse import quote, urljoin from urllib.request import pathname2url @@ -53,6 +54,9 @@ from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no from tests.mockserver import MockFTPServer, MockServer from tests.spiders import ItemSpider +if TYPE_CHECKING: + from os import PathLike + def path_to_url(path): return urljoin("file:", pathname2url(str(path))) @@ -1356,16 +1360,16 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_encoding(self): - items = [dict({"foo": "Test\xd6"})] + items = [{"foo": "Test\xd6"}] formats = { - "json": '[{"foo": "Test\\u00d6"}]'.encode("utf-8"), - "jsonlines": '{"foo": "Test\\u00d6"}\n'.encode("utf-8"), + "json": b'[{"foo": "Test\\u00d6"}]', + "jsonlines": b'{"foo": "Test\\u00d6"}\n', "xml": ( '\n' "Test\xd6" - ).encode("utf-8"), - "csv": "foo\r\nTest\xd6\r\n".encode("utf-8"), + ).encode(), + "csv": "foo\r\nTest\xd6\r\n".encode(), } for fmt, expected in formats.items(): @@ -1379,13 +1383,13 @@ class FeedExportTest(FeedExportTestBase): self.assertEqual(expected, data[fmt]) formats = { - "json": '[{"foo": "Test\xd6"}]'.encode("latin-1"), - "jsonlines": '{"foo": "Test\xd6"}\n'.encode("latin-1"), + "json": b'[{"foo": "Test\xd6"}]', + "jsonlines": b'{"foo": "Test\xd6"}\n', "xml": ( - '\n' - "Test\xd6" - ).encode("latin-1"), - "csv": "foo\r\nTest\xd6\r\n".encode("latin-1"), + b'\n' + b"Test\xd6" + ), + "csv": b"foo\r\nTest\xd6\r\n", } for fmt, expected in formats.items(): @@ -1401,15 +1405,15 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_configs(self): - items = [dict({"foo": "FOO", "bar": "BAR"})] + items = [{"foo": "FOO", "bar": "BAR"}] formats = { - "json": '[\n{"bar": "BAR"}\n]'.encode("utf-8"), + "json": b'[\n{"bar": "BAR"}\n]', "xml": ( - '\n' - "\n \n FOO\n \n" - ).encode("latin-1"), - "csv": "bar,foo\r\nBAR,FOO\r\n".encode("utf-8"), + b'\n' + b"\n \n FOO\n \n" + ), + "csv": b"bar,foo\r\nBAR,FOO\r\n", } settings = { @@ -1663,8 +1667,8 @@ class FeedExportTest(FeedExportTestBase): def test_extend_kwargs(self): items = [{"foo": "FOO", "bar": "BAR"}] - expected_with_title_csv = "foo,bar\r\nFOO,BAR\r\n".encode("utf-8") - expected_without_title_csv = "FOO,BAR\r\n".encode("utf-8") + expected_with_title_csv = b"foo,bar\r\nFOO,BAR\r\n" + expected_without_title_csv = b"FOO,BAR\r\n" test_cases = [ # with title { @@ -2513,28 +2517,28 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_configs(self): items = [ - dict({"foo": "FOO", "bar": "BAR"}), - dict({"foo": "FOO1", "bar": "BAR1"}), + {"foo": "FOO", "bar": "BAR"}, + {"foo": "FOO1", "bar": "BAR1"}, ] formats = { "json": [ - '[\n{"bar": "BAR"}\n]'.encode("utf-8"), - '[\n{"bar": "BAR1"}\n]'.encode("utf-8"), + b'[\n{"bar": "BAR"}\n]', + b'[\n{"bar": "BAR1"}\n]', ], "xml": [ ( - '\n' - "\n \n FOO\n \n" - ).encode("latin-1"), + b'\n' + b"\n \n FOO\n \n" + ), ( - '\n' - "\n \n FOO1\n \n" - ).encode("latin-1"), + b'\n' + b"\n \n FOO1\n \n" + ), ], "csv": [ - "foo,bar\r\nFOO,BAR\r\n".encode("utf-8"), - "foo,bar\r\nFOO1,BAR1\r\n".encode("utf-8"), + b"foo,bar\r\nFOO,BAR\r\n", + b"foo,bar\r\nFOO1,BAR1\r\n", ], } @@ -2574,11 +2578,11 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_batch_item_count_feeds_setting(self): - items = [dict({"foo": "FOO"}), dict({"foo": "FOO1"})] + items = [{"foo": "FOO"}, {"foo": "FOO1"}] formats = { "json": [ - '[{"foo": "FOO"}]'.encode("utf-8"), - '[{"foo": "FOO1"}]'.encode("utf-8"), + b'[{"foo": "FOO"}]', + b'[{"foo": "FOO1"}]', ], } settings = { diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 995c02a1a..7ea3fe8c9 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import random import re @@ -6,7 +8,7 @@ import string from ipaddress import IPv4Address from pathlib import Path from tempfile import mkdtemp -from typing import Dict +from typing import TYPE_CHECKING, Dict from unittest import mock, skipIf from urllib.parse import urlencode @@ -20,7 +22,6 @@ from twisted.internet.defer import ( from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint from twisted.internet.error import TimeoutError from twisted.internet.ssl import Certificate, PrivateCertificate, optionsForClientTLS -from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from twisted.web.client import URI, ResponseFailed from twisted.web.http import H2_ENABLED @@ -33,6 +34,9 @@ from scrapy.settings import Settings from scrapy.spiders import Spider from tests.mockserver import LeafResource, Status, ssl_context_factory +if TYPE_CHECKING: + from twisted.python.failure import Failure + def generate_random_string(size): return "".join(random.choices(string.ascii_uppercase + string.digits, k=size)) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 80d46274b..b8a277295 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -728,9 +728,7 @@ class TextResponseTest(BaseResponseTest): resp1 = self.response_class( "http://example.com", encoding="utf8", - body='click me'.encode( - "utf8" - ), + body='click me'.encode(), ) req = self._assert_followed_url( resp1.css("a")[0], diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index d9c09a16a..b1043c111 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -186,7 +186,7 @@ class Base: ) def test_nofollow(self): - '''Test the extractor's behaviour for links with rel="nofollow"''' + """Test the extractor's behaviour for links with rel='nofollow'""" html = b"""Page title<title> <body> diff --git a/tests/test_loader.py b/tests/test_loader.py index b0b7f8723..8db929dcf 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -156,7 +156,7 @@ class InitializationTestMixin: self.assertEqual(il.get_output_value("name"), ["foo"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({"name": ["foo"]})) + self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo"]}) def test_get_output_value_list(self): """Getting output value must not remove value from item""" @@ -165,9 +165,7 @@ class InitializationTestMixin: self.assertEqual(il.get_output_value("name"), ["foo", "bar"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual( - ItemAdapter(loaded_item).asdict(), dict({"name": ["foo", "bar"]}) - ) + self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]}) def test_values_single(self): """Values from initial item must be added to loader._values""" diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 528efa142..0d245bec9 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -526,7 +526,7 @@ class InitializationFromDictTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["foo"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, dict({"name": ["foo"]})) + self.assertEqual(loaded_item, {"name": ["foo"]}) def test_get_output_value_list(self): """Getting output value must not remove value from item""" @@ -535,7 +535,7 @@ class InitializationFromDictTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["foo", "bar"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, dict({"name": ["foo", "bar"]})) + self.assertEqual(loaded_item, {"name": ["foo", "bar"]}) def test_values_single(self): """Values from initial item must be added to loader._values""" diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 5a9a217ce..cd3442dd4 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -107,9 +107,7 @@ class FileDownloadCrawlTestCase(TestCase): # check that the images/files checksums are what we know they should be if self.expected_checksums is not None: - checksums = set( - i["checksum"] for item in items for i in item[self.media_key] - ) + checksums = {i["checksum"] for item in items for i in item[self.media_key]} self.assertEqual(checksums, self.expected_checksums) # check that the image files where actually written to the media store diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 18a2454b3..7d7c78920 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -628,7 +628,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): class NoimagesDropTestCase(unittest.TestCase): def test_deprecation_warning(self): - arg = str() + arg = "" with warnings.catch_warnings(record=True) as w: NoimagesDrop(arg) self.assertEqual(len(w), 1) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index d4dde4a40..127775f43 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,4 +1,3 @@ -import io from typing import Optional from testfixtures import LogCapture @@ -11,7 +10,6 @@ from scrapy import signals from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import FileException -from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.spiders import Spider @@ -35,8 +33,26 @@ def _mocked_download_func(request, info): return response() if callable(response) else response +class UserDefinedPipeline(MediaPipeline): + + def media_to_download(self, request, info, *, item=None): + pass + + def get_media_requests(self, item, info): + pass + + def media_downloaded(self, response, request, info, *, item=None): + return {} + + def media_failed(self, failure, request, info): + return failure + + def file_path(self, request, response=None, info=None, *, item=None): + return "" + + class BaseMediaPipelineTestCase(unittest.TestCase): - pipeline_class = MediaPipeline + pipeline_class = UserDefinedPipeline settings = None def setUp(self): @@ -54,54 +70,6 @@ class BaseMediaPipelineTestCase(unittest.TestCase): if not name.startswith("_"): disconnect_all(signal) - def test_default_media_to_download(self): - request = Request("http://url") - assert self.pipe.media_to_download(request, self.info) is None - - def test_default_get_media_requests(self): - item = {"name": "name"} - assert self.pipe.get_media_requests(item, self.info) is None - - def test_default_media_downloaded(self): - request = Request("http://url") - response = Response("http://url", body=b"") - assert self.pipe.media_downloaded(response, request, self.info) is response - - def test_default_media_failed(self): - request = Request("http://url") - fail = Failure(Exception()) - assert self.pipe.media_failed(fail, request, self.info) is fail - - def test_default_item_completed(self): - item = {"name": "name"} - assert self.pipe.item_completed([], item, self.info) is item - - # Check that failures are logged by default - fail = Failure(Exception()) - results = [(True, 1), (False, fail)] - - with LogCapture() as log: - new_item = self.pipe.item_completed(results, item, self.info) - - assert new_item is item - assert len(log.records) == 1 - record = log.records[0] - assert record.levelname == "ERROR" - self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail)) - - # disable failure logging and check again - self.pipe.LOG_FAILED_RESULTS = False - with LogCapture() as log: - new_item = self.pipe.item_completed(results, item, self.info) - assert new_item is item - assert len(log.records) == 0 - - @inlineCallbacks - def test_default_process_item(self): - item = {"name": "name"} - new_item = yield self.pipe.process_item(item, self.spider) - assert new_item is item - def test_modify_media_request(self): request = Request("http://url") self.pipe._modify_media_request(request) @@ -175,8 +143,38 @@ class BaseMediaPipelineTestCase(unittest.TestCase): context = getattr(info.downloaded[fp].value, "__context__", None) self.assertIsNone(context) + def test_default_item_completed(self): + item = {"name": "name"} + assert self.pipe.item_completed([], item, self.info) is item -class MockedMediaPipeline(MediaPipeline): + # Check that failures are logged by default + fail = Failure(Exception()) + results = [(True, 1), (False, fail)] + + with LogCapture() as log: + new_item = self.pipe.item_completed(results, item, self.info) + + assert new_item is item + assert len(log.records) == 1 + record = log.records[0] + assert record.levelname == "ERROR" + self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail)) + + # disable failure logging and check again + self.pipe.LOG_FAILED_RESULTS = False + with LogCapture() as log: + new_item = self.pipe.item_completed(results, item, self.info) + assert new_item is item + assert len(log.records) == 0 + + @inlineCallbacks + def test_default_process_item(self): + item = {"name": "name"} + new_item = yield self.pipe.process_item(item, self.spider) + assert new_item is item + + +class MockedMediaPipeline(UserDefinedPipeline): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._mockcalled = [] @@ -213,10 +211,6 @@ class MockedMediaPipeline(MediaPipeline): class MediaPipelineTestCase(BaseMediaPipelineTestCase): pipeline_class = MockedMediaPipeline - def _callback(self, result): - self.pipe._mockcalled.append("request_callback") - return result - def _errback(self, result): self.pipe._mockcalled.append("request_errback") return result @@ -227,19 +221,17 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): req = Request( "http://url1", meta={"response": rsp}, - callback=self._callback, errback=self._errback, ) item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, rsp)]) + self.assertEqual(new_item["results"], [(True, {})]) self.assertEqual( self.pipe._mockcalled, [ "get_media_requests", "media_to_download", "media_downloaded", - "request_callback", "item_completed", ], ) @@ -251,7 +243,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): req = Request( "http://url1", meta={"response": fail}, - callback=self._callback, errback=self._errback, ) item = {"requests": req} @@ -277,7 +268,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): req2 = Request("http://url2", meta={"response": fail}) item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, rsp1), (False, fail)]) + self.assertEqual(new_item["results"], [(True, {}), (False, fail)]) m = self.pipe._mockcalled # only once self.assertEqual(m[0], "get_media_requests") # first hook called @@ -315,7 +306,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = {"requests": req1} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) - self.assertEqual(new_item["results"], [(True, rsp1)]) + self.assertEqual(new_item["results"], [(True, {})]) # rsp2 is ignored, rsp1 must be in results because request fingerprints are the same req2 = Request( @@ -325,7 +316,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) self.assertEqual(self.fingerprint(req1), self.fingerprint(req2)) - self.assertEqual(new_item["results"], [(True, rsp1)]) + self.assertEqual(new_item["results"], [(True, {})]) @inlineCallbacks def test_results_are_cached_for_requests_of_single_item(self): @@ -337,7 +328,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) - self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) + self.assertEqual(new_item["results"], [(True, {}), (True, {})]) @inlineCallbacks def test_wait_if_request_is_downloading(self): @@ -363,7 +354,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): req2 = Request(req1.url, meta={"response": rsp2_func}) item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) + self.assertEqual(new_item["results"], [(True, {}), (True, {})]) @inlineCallbacks def test_use_media_to_download_result(self): @@ -376,57 +367,15 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ["get_media_requests", "media_to_download", "item_completed"], ) - -class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._mockcalled = [] - - def get_media_requests(self, item, info): - item_url = item["image_urls"][0] - output_img = io.BytesIO() - img = Image.new("RGB", (60, 30), color="red") - img.save(output_img, format="JPEG") - return Request( - item_url, - meta={ - "response": Response(item_url, status=200, body=output_img.getvalue()) - }, + def test_key_for_pipe(self): + self.assertEqual( + self.pipe._key_for_pipe("IMAGES", base_class_name="MediaPipeline"), + "MOCKEDMEDIAPIPELINE_IMAGES", ) - def inc_stats(self, *args, **kwargs): - return True - - def media_to_download(self, request, info): - self._mockcalled.append("media_to_download") - return super().media_to_download(request, info) - - def media_downloaded(self, response, request, info): - self._mockcalled.append("media_downloaded") - return super().media_downloaded(response, request, info) - - def file_downloaded(self, response, request, info): - self._mockcalled.append("file_downloaded") - return super().file_downloaded(response, request, info) - - def file_path(self, request, response=None, info=None): - self._mockcalled.append("file_path") - return super().file_path(request, response, info) - - def thumb_path(self, request, thumb_id, response=None, info=None): - self._mockcalled.append("thumb_path") - return super().thumb_path(request, thumb_id, response, info) - - def get_images(self, response, request, info): - self._mockcalled.append("get_images") - return super().get_images(response, request, info) - - def image_downloaded(self, response, request, info): - self._mockcalled.append("image_downloaded") - return super().image_downloaded(response, request, info) - class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): + def _assert_request_no3xx(self, pipeline_class, settings): pipe = pipeline_class(settings=Settings(settings)) request = Request("http://url") @@ -452,18 +401,11 @@ class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): else: self.assertNotIn(status, request.meta["handle_httpstatus_list"]) - def test_standard_setting(self): - self._assert_request_no3xx(MediaPipeline, {"MEDIA_ALLOW_REDIRECTS": True}) - def test_subclass_standard_setting(self): - class UserDefinedPipeline(MediaPipeline): - pass self._assert_request_no3xx(UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True}) def test_subclass_specific_setting(self): - class UserDefinedPipeline(MediaPipeline): - pass self._assert_request_no3xx( UserDefinedPipeline, {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True} diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 7312eb036..d3f416347 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -147,7 +147,7 @@ class RequestSerializationTest(unittest.TestCase): spider = MySpider() r = Request("http://www.example.com", callback=spider.parse) - setattr(spider, "parse", None) + spider.parse = None self.assertRaises(ValueError, r.to_dict, spider=spider) def test_callback_not_available(self): diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 2633cca5b..7be8150fc 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -29,7 +29,7 @@ class ResponseTypesTest(unittest.TestCase): mappings = [ (b'attachment; filename="data.xml"', XmlResponse), (b"attachment; filename=data.xml", XmlResponse), - ("attachment;filename=data£.tar.gz".encode("utf-8"), Response), + ("attachment;filename=data£.tar.gz".encode(), Response), ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response), ("attachment;filename=data高.doc".encode("gbk"), Response), ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse), diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 6ad30deed..28ad910a8 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -36,10 +36,10 @@ class BaseRobotParserTest: def test_allowed(self): robotstxt_robotstxt_body = ( - "User-agent: * \n" - "Disallow: /disallowed \n" - "Allow: /allowed \n" - "Crawl-delay: 10".encode("utf-8") + b"User-agent: * \n" + b"Disallow: /disallowed \n" + b"Allow: /allowed \n" + b"Crawl-delay: 10" ) rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body @@ -48,15 +48,13 @@ class BaseRobotParserTest: self.assertFalse(rp.allowed("https://www.site.local/disallowed", "*")) def test_allowed_wildcards(self): - robotstxt_robotstxt_body = """User-agent: first + robotstxt_robotstxt_body = b"""User-agent: first Disallow: /disallowed/*/end$ User-agent: second Allow: /*allowed Disallow: / - """.encode( - "utf-8" - ) + """ rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) @@ -77,18 +75,14 @@ class BaseRobotParserTest: self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second")) def test_length_based_precedence(self): - robotstxt_robotstxt_body = ( - "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8") - ) + robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) self.assertTrue(rp.allowed("https://www.site.local/page", "*")) def test_order_based_precedence(self): - robotstxt_robotstxt_body = ( - "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8") - ) + robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) @@ -123,9 +117,7 @@ class BaseRobotParserTest: Disallow: /wiki/Käyttäjä: User-Agent: UnicödeBöt - Disallow: /some/randome/page.html""".encode( - "utf-8" - ) + Disallow: /some/randome/page.html""".encode() rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) @@ -145,14 +137,14 @@ class BaseRobotParserTest: class DecodeRobotsTxtTest(unittest.TestCase): def test_native_string_conversion(self): - robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8") + robotstxt_body = b"User-agent: *\nDisallow: /\n" decoded_content = decode_robotstxt( robotstxt_body, spider=None, to_native_str_type=True ) self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") def test_decode_utf8(self): - robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8") + robotstxt_body = b"User-agent: *\nDisallow: /\n" decoded_content = decode_robotstxt(robotstxt_body, spider=None) self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 02b50baa3..9b7bad4bf 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -284,7 +284,7 @@ class DownloaderAwareSchedulerTestMixin: downloader.decrement(slot) self.assertTrue( - _is_scheduling_fair(list(s for u, s in _URLS_WITH_SLOTS), dequeued_slots) + _is_scheduling_fair([s for u, s in _URLS_WITH_SLOTS], dequeued_slots) ) self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0) diff --git a/tests/test_spider.py b/tests/test_spider.py index d629d33af..18a863350 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -244,7 +244,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -270,7 +270,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -299,7 +299,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 2) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -324,7 +324,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -352,7 +352,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -383,7 +383,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -413,7 +413,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -445,7 +445,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -637,7 +637,7 @@ Sitemap: /sitemap-relative-url.xml class FilteredSitemapSpider(self.spider_class): def sitemap_filter(self, entries): for entry in entries: - alternate_links = entry.get("alternate", tuple()) + alternate_links = entry.get("alternate", ()) for link in alternate_links: if "/deutsch/" in link: entry["loc"] = link diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index afffa87fb..5797edfbd 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -884,6 +884,53 @@ class TestSettingsPolicyByName(TestCase): with self.assertRaises(RuntimeError): RefererMiddleware(settings) + def test_multiple_policy_tokens(self): + # test parsing without space(s) after the comma + settings1 = Settings( + { + "REFERRER_POLICY": ",".join( + [ + "some-custom-unknown-policy", + POLICY_SAME_ORIGIN, + POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, + "another-custom-unknown-policy", + ] + ) + } + ) + mw1 = RefererMiddleware(settings1) + self.assertEqual(mw1.default_policy, StrictOriginWhenCrossOriginPolicy) + + # test parsing with space(s) after the comma + settings2 = Settings( + { + "REFERRER_POLICY": ", ".join( + [ + POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, + "another-custom-unknown-policy", + POLICY_UNSAFE_URL, + ] + ) + } + ) + mw2 = RefererMiddleware(settings2) + self.assertEqual(mw2.default_policy, UnsafeUrlPolicy) + + def test_multiple_policy_tokens_all_invalid(self): + settings = Settings( + { + "REFERRER_POLICY": ",".join( + [ + "some-custom-unknown-policy", + "another-custom-unknown-policy", + "yet-another-custom-unknown-policy", + ] + ) + } + ) + with self.assertRaises(RuntimeError): + RefererMiddleware(settings) + class TestPolicyHeaderPrecedence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"} diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index a7d54b565..ec0399865 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -14,7 +14,6 @@ from scrapy.utils.defer import ( mustbe_deferred, parallel_async, process_chain, - process_chain_both, process_parallel, ) @@ -80,19 +79,6 @@ class DeferUtilsTest(unittest.TestCase): gotexc = True self.assertTrue(gotexc) - @defer.inlineCallbacks - def test_process_chain_both(self): - x = yield process_chain_both( - [cb_fail, cb2, cb3], [None, eb1, None], "res", "v1", "v2" - ) - self.assertEqual(x, "(cb3 (eb1 TypeError v1 v2) v1 v2)") - - fail = Failure(ZeroDivisionError()) - x = yield process_chain_both( - [eb1, cb2, cb3], [eb1, None, None], fail, "v1", "v2" - ) - self.assertEqual(x, "(cb3 (cb2 (eb1 ZeroDivisionError v1 v2) v1 v2) v1 v2)") - @defer.inlineCallbacks def test_process_parallel(self): x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2") diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index a8d080822..0f75bdb5c 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -11,7 +11,6 @@ import pytest from testfixtures import LogCapture from twisted.python.failure import Failure -from scrapy.extensions import telnet from scrapy.utils.log import ( LogCounterHandler, SpiderLoggerAdapter, @@ -70,9 +69,6 @@ class TopLevelFormatterTest(unittest.TestCase): class LogCounterHandlerTest(unittest.TestCase): def setUp(self): settings = {"LOG_LEVEL": "WARNING"} - if not telnet.TWISTED_CONCH_AVAILABLE: - # disable it to avoid the extra warning - settings["TELNETCONSOLE_ENABLED"] = False self.logger = logging.getLogger("test") self.logger.setLevel(logging.NOTSET) self.logger.propagate = False diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing new file mode 100644 index 000000000..665db9088 --- /dev/null +++ b/tests_typing/test_http_request.mypy-testing @@ -0,0 +1,80 @@ +from typing import Any, Dict + +import pytest + +from scrapy import Request +from scrapy.http import JsonRequest + + +class MyRequest(Request): + pass + + +class MyRequest2(Request): + pass + + +@pytest.mark.mypy_testing +def mypy_test_headers(): + Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]" + Request("data:,", headers=None) + Request("data:,", headers={}) + Request("data:,", headers=[]) + Request("data:,", headers={"foo": "bar"}) + Request("data:,", headers={b"foo": "bar"}) + Request("data:,", headers={"foo": b"bar"}) + Request("data:,", headers=[("foo", "bar")]) + Request("data:,", headers=[(b"foo", "bar")]) + Request("data:,", headers=[("foo", b"bar")]) + + +@pytest.mark.mypy_testing +def mypy_test_copy(): + req = Request("data:,") + reveal_type(req) # R: scrapy.http.request.Request + req_copy = req.copy() + reveal_type(req_copy) # R: scrapy.http.request.Request + + +@pytest.mark.mypy_testing +def mypy_test_copy_subclass(): + req = MyRequest("data:,") + reveal_type(req) # R: __main__.MyRequest + req_copy = req.copy() + reveal_type(req_copy) # R: __main__.MyRequest + + +@pytest.mark.mypy_testing +def mypy_test_replace(): + req = Request("data:,") + reveal_type(req) # R: scrapy.http.request.Request + req_copy = req.replace(body=b"a") + reveal_type(req_copy) # R: scrapy.http.request.Request + kwargs: Dict[str, Any] = {} + req_copy2 = req.replace(body=b"a", **kwargs) + reveal_type(req_copy2) # R: Any + + +@pytest.mark.mypy_testing +def mypy_test_replace_subclass(): + req = MyRequest("data:,") + reveal_type(req) # R: __main__.MyRequest + req_copy = req.replace(body=b"a") + reveal_type(req_copy) # R: __main__.MyRequest + req_copy2 = req.replace(body=b"a", cls=MyRequest2) + reveal_type(req_copy2) # R: __main__.MyRequest2 + kwargs: Dict[str, Any] = {} + req_copy3 = req.replace(body=b"a", cls=MyRequest2, **kwargs) + reveal_type(req_copy3) # R: __main__.MyRequest2 + + +@pytest.mark.mypy_testing +def mypy_test_jsonrequest_copy_replace(): + req = JsonRequest("data:,") + reveal_type(req) # R: scrapy.http.request.json_request.JsonRequest + req_copy = req.copy() + reveal_type(req_copy) # R: scrapy.http.request.json_request.JsonRequest + req_copy = req.replace(body=b"a") + reveal_type(req_copy) # R: scrapy.http.request.json_request.JsonRequest + req_copy_my = req.replace(body=b"a", cls=MyRequest) + reveal_type(req_copy_my) # R: __main__.MyRequest diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing new file mode 100644 index 000000000..d58ac1027 --- /dev/null +++ b/tests_typing/test_http_response.mypy-testing @@ -0,0 +1,59 @@ +from typing import Any, Dict + +import pytest + +from scrapy.http import HtmlResponse, Response, TextResponse + + +@pytest.mark.mypy_testing +def mypy_test_headers(): + Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]" + Response("data:,", headers=None) + Response("data:,", headers={}) + Response("data:,", headers=[]) + Response("data:,", headers={"foo": "bar"}) + Response("data:,", headers={b"foo": "bar"}) + Response("data:,", headers={"foo": b"bar"}) + Response("data:,", headers=[("foo", "bar")]) + Response("data:,", headers=[(b"foo", "bar")]) + Response("data:,", headers=[("foo", b"bar")]) + + +@pytest.mark.mypy_testing +def mypy_test_copy(): + resp = Response("data:,") + reveal_type(resp) # R: scrapy.http.response.Response + resp_copy = resp.copy() + reveal_type(resp_copy) # R: scrapy.http.response.Response + + +@pytest.mark.mypy_testing +def mypy_test_copy_subclass(): + resp = HtmlResponse("data:,") + reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse + resp_copy = resp.copy() + reveal_type(resp_copy) # R: scrapy.http.response.html.HtmlResponse + + +@pytest.mark.mypy_testing +def mypy_test_replace(): + resp = Response("data:,") + reveal_type(resp) # R: scrapy.http.response.Response + resp_copy = resp.replace(body=b"a") + reveal_type(resp_copy) # R: scrapy.http.response.Response + kwargs: Dict[str, Any] = {} + resp_copy2 = resp.replace(body=b"a", **kwargs) + reveal_type(resp_copy2) # R: Any + + +@pytest.mark.mypy_testing +def mypy_test_replace_subclass(): + resp = HtmlResponse("data:,") + reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse + resp_copy = resp.replace(body=b"a") + reveal_type(resp_copy) # R: scrapy.http.response.html.HtmlResponse + resp_copy2 = resp.replace(body=b"a", cls=TextResponse) + reveal_type(resp_copy2) # R: scrapy.http.response.text.TextResponse + kwargs: Dict[str, Any] = {} + resp_copy3 = resp.replace(body=b"a", cls=TextResponse, **kwargs) + reveal_type(resp_copy3) # R: scrapy.http.response.text.TextResponse diff --git a/tox.ini b/tox.ini index 5c2f34c6e..ee810eae3 100644 --- a/tox.ini +++ b/tox.ini @@ -48,16 +48,18 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.10.0 - typing-extensions==4.11.0 - types-attrs==19.1.0 + mypy==1.10.1 + typing-extensions==4.12.2 types-lxml==2024.4.14 - types-Pillow==10.2.0.20240423 - types-Pygments==2.17.0.20240310 - types-pyOpenSSL==24.0.0.20240417 - types-setuptools==69.5.0.20240423 - # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 - w3lib >= 2.1.2 + types-Pygments==2.18.0.20240506 + types-pyOpenSSL==24.1.0.20240425 + types-setuptools==70.3.0.20240710 + botocore-stubs==1.34.143 + boto3-stubs[s3]==1.34.143 + attrs >= 18.2.0 + Pillow >= 10.3.0 + pytest >= 8.2.0 + w3lib >= 2.2.0 commands = mypy {posargs: scrapy tests} @@ -81,14 +83,14 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==3.1.0 + pylint==3.2.5 commands = pylint conftest.py docs extras scrapy setup.py tests [testenv:twinecheck] basepython = python3 deps = - twine==5.1.0 + twine==5.1.1 build==1.2.1 commands = python -m build --sdist @@ -147,9 +149,6 @@ deps = {[testenv]deps} boto3 google-cloud-storage - # Twisted[http2] currently forces old mitmproxy because of h2 version - # restrictions in their deps, so we need to pin old markupsafe here too. - markupsafe < 2.1.0 robotexclusionrulesparser Pillow; python_version < '3.13' Twisted[http2]