RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into py313

This commit is contained in:
Andrey Rakhmatullin 2024-08-11 13:46:51 +05:00
parent 1be8aee09c b9ef1326a5
commit 1e68d3c0bf
181 changed files with 2675 additions and 1419 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
.flake8
View File

@ -1,8 +1,67 @@
[flake8]
max-line-length = 119
ignore = E203, E501, E701, E704, W503
extend-select = TC, TC1
ignore =
# black disagrees with flake8 about these
E203, E501, E701, E704, W503
# Assigning to `os.environ` doesn't clear the environment.
B003
# Do not use mutable data structures for argument defaults.
B006
# Loop control variable not used within the loop body.
B007
# Do not perform function calls in argument defaults.
B008
# return/continue/break inside finally blocks cause exceptions to be
# silenced.
B012
# Star-arg unpacking after a keyword argument is strongly discouraged
B026
# No explicit stacklevel argument found.
B028
# docstring does contain unindexed parameters
P102
# other string does contain unindexed parameters
P103
# Missing docstring in public module
D100
# Missing docstring in public class
D101
# Missing docstring in public method
D102
# Missing docstring in public function
D103
# Missing docstring in public package
D104
# Missing docstring in magic method
D105
# Missing docstring in public nested class
D106
# Missing docstring in __init__
D107
# One-line docstring should fit on one line with quotes
D200
# No blank lines allowed after function docstring
D202
# 1 blank line required between summary line and description
D205
# Multi-line docstring closing quotes should be on a separate line
D209
# First line should end with a period
D400
# First line should be in imperative mood; try rephrasing
D401
# First line should not be the function's "signature"
D402
# First word of the first line should be properly capitalized
D403
# Annotation in typing.cast() should be a string literal
TC006
exclude =
docs/conf.py
@ -16,6 +75,7 @@ per-file-ignores =
scrapy/linkextractors/__init__.py:E402,F401
scrapy/selector/__init__.py:F401
scrapy/spiders/__init__.py:E402,F401
tests/CrawlerRunner/change_reactor.py:E402
# Issues pending a review:
scrapy/utils/url.py:F403,F405

6
.github/workflows/checks.yml vendored
View File

@ -21,7 +21,7 @@ jobs:
- python-version: 3.8
env:
TOXENV: typing-tests
- python-version: "3.11" # Keep in sync with .readthedocs.yml
- python-version: "3.12" # Keep in sync with .readthedocs.yml
env:
TOXENV: docs
- python-version: "3.13.0-beta.1"
@ -32,7 +32,7 @@ jobs:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
@ -52,4 +52,4 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: pre-commit/action@v3.0.0
- uses: pre-commit/action@v3.0.1

4
.github/workflows/publish.yml vendored
View File

@ -13,13 +13,13 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
- uses: actions/setup-python@v5
with:
python-version: "3.13.0-beta.1"
- run: |
pip install --upgrade build twine
python -m build
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@v1.6.4
uses: pypa/gh-action-pypi-publish@v1.9.0
with:
password: ${{ secrets.PYPI_TOKEN }}

4
.github/workflows/tests-macos.yml vendored
View File

@ -7,7 +7,7 @@ concurrency:
jobs:
tests:
runs-on: macos-11
runs-on: macos-latest
strategy:
fail-fast: false
matrix:
@ -17,7 +17,7 @@ jobs:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

2
.github/workflows/tests-ubuntu.yml vendored
View File

@ -65,7 +65,7 @@ jobs:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

2
.github/workflows/tests-windows.yml vendored
View File

@ -38,7 +38,7 @@ jobs:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

22
.pre-commit-config.yaml
View File

@ -1,15 +1,22 @@
repos:
- repo: https://github.com/PyCQA/bandit
rev: 1.7.7
rev: 1.7.9
hooks:
- id: bandit
args: [-r, -c, .bandit.yml]
- repo: https://github.com/PyCQA/flake8
rev: 7.0.0
rev: 7.1.0
hooks:
- id: flake8
additional_dependencies:
- flake8-bugbear
- flake8-comprehensions
- flake8-debugger
- flake8-docstrings
- flake8-string-format
- flake8-type-checking
- repo: https://github.com/psf/black.git
rev: 24.2.0
rev: 24.4.2
hooks:
- id: black
- repo: https://github.com/pycqa/isort
@ -17,8 +24,13 @@ repos:
hooks:
- id: isort
- repo: https://github.com/adamchainz/blacken-docs
rev: 1.16.0
rev: 1.18.0
hooks:
- id: blacken-docs
additional_dependencies:
- black==24.2.0
- black==24.4.2
- repo: https://github.com/asottile/pyupgrade
rev: v3.16.0
hooks:
- id: pyupgrade
args: [--py38-plus, --keep-runtime-typing]

2
.readthedocs.yml
View File

@ -9,7 +9,7 @@ build:
tools:
# For available versions, see:
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
python: "3.11" # Keep in sync with .github/workflows/checks.yml
python: "3.12" # Keep in sync with .github/workflows/checks.yml
python:
install:

13
docs/intro/install.rst
View File

@ -37,7 +37,7 @@ Note that sometimes this may require solving compilation issues for some Scrapy
dependencies depending on your operating system, so be sure to check the
:ref:`intro-install-platform-notes`.
For more detailed and platform specifics instructions, as well as
For more detailed and platform-specific instructions, as well as
troubleshooting information, read on.
@ -101,7 +101,7 @@ Windows
-------
Though it's possible to install Scrapy on Windows using pip, we recommend you
to install `Anaconda`_ or `Miniconda`_ and use the package from the
install `Anaconda`_ or `Miniconda`_ and use the package from the
`conda-forge`_ channel, which will avoid most installation issues.
Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with::
@ -141,7 +141,7 @@ But it should support older versions of Ubuntu too, like Ubuntu 14.04,
albeit with potential issues with TLS connections.
**Don't** use the ``python-scrapy`` package provided by Ubuntu, they are
typically too old and slow to catch up with latest Scrapy.
typically too old and slow to catch up with the latest Scrapy release.
To install Scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
@ -170,7 +170,7 @@ macOS
Building Scrapy's dependencies requires the presence of a C compiler and
development headers. On macOS this is typically provided by Apple’s Xcode
development tools. To install the Xcode command line tools open a terminal
development tools. To install the Xcode command-line tools, open a terminal
window and run::
xcode-select --install
@ -200,11 +200,6 @@ solutions:
brew install python
* Latest versions of python have ``pip`` bundled with them so you won't need
to install it separately. If this is not the case, upgrade python::
brew update; brew upgrade python
* *(Optional)* :ref:`Install Scrapy inside a Python virtual environment
<intro-using-virtualenv>`.

20
docs/intro/overview.rst
View File

@ -44,13 +44,13 @@ https://quotes.toscrape.com, following the pagination:
if next_page is not None:
yield response.follow(next_page, self.parse)
Put this in a text file, name it to something like ``quotes_spider.py``
Put this in a text file, name it something like ``quotes_spider.py``
and run the spider using the :command:`runspider` command::
scrapy runspider quotes_spider.py -o quotes.jsonl
When this finishes you will have in the ``quotes.jsonl`` file a list of the
quotes in JSON Lines format, containing text and author, looking like this::
quotes in JSON Lines format, containing the text and author, which will look like this::
{"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}
{"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"}
@ -65,27 +65,27 @@ When you ran the command ``scrapy runspider quotes_spider.py``, Scrapy looked fo
Spider definition inside it and ran it through its crawler engine.
The crawl started by making requests to the URLs defined in the ``start_urls``
attribute (in this case, only the URL for quotes in *humor* category)
attribute (in this case, only the URL for quotes in the *humor* category)
and called the default callback method ``parse``, passing the response object as
an argument. In the ``parse`` callback, we loop through the quote elements
using a CSS Selector, yield a Python dict with the extracted quote text and author,
look for a link to the next page and schedule another request using the same
``parse`` method as callback.
Here you notice one of the main advantages about Scrapy: requests are
Here you will notice one of the main advantages of Scrapy: requests are
:ref:`scheduled and processed asynchronously <topics-architecture>`. This
means that Scrapy doesn't need to wait for a request to be finished and
processed, it can send another request or do other things in the meantime. This
also means that other requests can keep going even if some request fails or an
also means that other requests can keep going even if a request fails or an
error happens while handling it.
While this enables you to do very fast crawls (sending multiple concurrent
requests at the same time, in a fault-tolerant way) Scrapy also gives you
control over the politeness of the crawl through :ref:`a few settings
<topics-settings-ref>`. You can do things like setting a download delay between
each request, limiting amount of concurrent requests per domain or per IP, and
each request, limiting the amount of concurrent requests per domain or per IP, and
even :ref:`using an auto-throttling extension <topics-autothrottle>` that tries
to figure out these automatically.
to figure these settings out automatically.
.. note::
@ -106,10 +106,10 @@ scraping easy and efficient, such as:
* Built-in support for :ref:`selecting and extracting <topics-selectors>` data
from HTML/XML sources using extended CSS selectors and XPath expressions,
with helper methods to extract using regular expressions.
with helper methods for extraction using regular expressions.
* An :ref:`interactive shell console <topics-shell>` (IPython aware) for trying
out the CSS and XPath expressions to scrape data, very useful when writing or
out the CSS and XPath expressions to scrape data, which is very useful when writing or
debugging your spiders.
* Built-in support for :ref:`generating feed exports <topics-feed-exports>` in
@ -124,7 +124,7 @@ scraping easy and efficient, such as:
well-defined API (middlewares, :ref:`extensions <topics-extensions>`, and
:ref:`pipelines <topics-item-pipeline>`).
* Wide range of built-in extensions and middlewares for handling:
* A wide range of built-in extensions and middlewares for handling:
- cookies and session handling
- HTTP features like compression, authentication, caching

55
docs/intro/tutorial.rst
View File

@ -18,11 +18,11 @@ This tutorial will walk you through these tasks:
4. Changing spider to recursively follow links
5. Using spider arguments
Scrapy is written in Python_. If you're new to the language you might want to
start by getting an idea of what the language is like, to get the most out of
Scrapy.
Scrapy is written in Python_. The more you learn about Python, the more you
can get out of Scrapy.
If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource.
If you're already familiar with other languages and want to learn Python quickly, the
`Python Tutorial`_ is a good resource.
If you're new to programming and want to start with Python, the following books
may be useful to you:
@ -76,10 +76,9 @@ This will create a ``tutorial`` directory with the following contents::
Our first Spider
================
Spiders are classes that you define and that Scrapy uses to scrape information
from a website (or a group of websites). They must subclass
:class:`~scrapy.Spider` and define the initial requests to make,
optionally how to follow links in the pages, and how to parse the downloaded
Spiders are classes that you define and that Scrapy uses to scrape information from a website
(or a group of websites). They must subclass :class:`~scrapy.Spider` and define the initial
requests to be made, and optionally, how to follow links in pages and parse the downloaded
page content to extract data.
This is the code for our first Spider. Save it in a file named
@ -138,7 +137,7 @@ To put our spider to work, go to the project's top level directory and run::
scrapy crawl quotes
This command runs the spider with name ``quotes`` that we've just added, that
This command runs the spider named ``quotes`` that we've just added, that
will send some requests for the ``quotes.toscrape.com`` domain. You will get an output
similar to this::
@ -169,7 +168,7 @@ Scrapy schedules the :class:`scrapy.Request <scrapy.Request>` objects
returned by the ``start_requests`` method of the Spider. Upon receiving a
response for each one, it instantiates :class:`~scrapy.http.Response` objects
and calls the callback method associated with the request (in this case, the
``parse`` method) passing the response as argument.
``parse`` method) passing the response as an argument.
A shortcut to the start_requests method
@ -217,8 +216,8 @@ using the :ref:`Scrapy shell <topics-shell>`. Run::
.. note::
Remember to always enclose urls in quotes when running Scrapy shell from
command-line, otherwise urls containing arguments (i.e. ``&`` character)
Remember to always enclose URLs in quotes when running Scrapy shell from the
command line, otherwise URLs containing arguments (i.e. ``&`` character)
will not work.
On Windows, use double quotes instead::
@ -257,7 +256,7 @@ object:
The result of running ``response.css('title')`` is a list-like object called
:class:`~scrapy.selector.SelectorList`, which represents a list of
:class:`~scrapy.Selector` objects that wrap around XML/HTML elements
and allow you to run further queries to fine-grain the selection or extract the
and allow you to run further queries to refine the selection or extract the
data.
To extract the text from the title above, you can do:
@ -354,12 +353,12 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:
XPath expressions are very powerful, and are the foundation of Scrapy
Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You
can see that if you read closely the text representation of the selector
objects in the shell.
can see that if you read the text representation of the selector
objects in the shell closely.
While perhaps not as popular as CSS selectors, XPath expressions offer more
power because besides navigating the structure, it can also look at the
content. Using XPath, you're able to select things like: *select the link
content. Using XPath, you're able to select things like: *the link
that contains the text "Next Page"*. This makes XPath very fitting to the task
of scraping, and we encourage you to learn XPath even if you already know how to
construct CSS selectors, it will make scraping much easier.
@ -422,7 +421,7 @@ variable, so that we can run our CSS selectors directly on a particular quote:
>>> quote = response.css("div.quote")[0]
Now, let's extract ``text``, ``author`` and the ``tags`` from that quote
Now, let's extract the ``text``, ``author`` and ``tags`` from that quote
using the ``quote`` object we just created:
.. code-block:: pycon
@ -448,7 +447,7 @@ to get all of them:
from sys import version_info
Having figured out how to extract each bit, we can now iterate over all the
quotes elements and put them together into a Python dictionary:
quote elements and put them together into a Python dictionary:
.. code-block:: pycon
@ -465,8 +464,8 @@ quotes elements and put them together into a Python dictionary:
Extracting data in our spider
-----------------------------
Let's get back to our spider. Until now, it doesn't extract any data in
particular, just saves the whole HTML page to a local file. Let's integrate the
Let's get back to our spider. Until now, it hasn't extracted any data in
particular, just saving the whole HTML page to a local file. Let's integrate the
extraction logic above into our spider.
A Scrapy spider typically generates many dictionaries containing the data
@ -529,8 +528,8 @@ using a different serialization format, such as `JSON Lines`_::
scrapy crawl quotes -o quotes.jsonl
The `JSON Lines`_ format is useful because it's stream-like, you can easily
append new records to it. It doesn't have the same problem of JSON when you run
The `JSON Lines`_ format is useful because it's stream-like, so you can easily
append new records to it. It doesn't have the same problem as JSON when you run
twice. Also, as each record is a separate line, you can process big files
without having to fit everything in memory, there are tools like `JQ`_ to help
do that at the command-line.
@ -555,7 +554,7 @@ from https://quotes.toscrape.com, you want quotes from all the pages in the webs
Now that you know how to extract data from pages, let's see how to follow links
from them.
First thing is to extract the link to the page we want to follow. Examining
The first thing to do is extract the link to the page we want to follow. Examining
our page, we can see there is a link to the next page with the following
markup:
@ -589,7 +588,7 @@ There is also an ``attrib`` property available
>>> response.css("li.next a").attrib["href"]
'/page/2/'
Let's see now our spider modified to recursively follow the link to the next
Now let's see our spider, modified to recursively follow the link to the next
page, extracting data from it:
.. code-block:: python
@ -756,8 +755,8 @@ Another interesting thing this spider demonstrates is that, even if there are
many quotes from the same author, we don't need to worry about visiting the
same author page multiple times. By default, Scrapy filters out duplicated
requests to URLs already visited, avoiding the problem of hitting servers too
much because of a programming mistake. This can be configured by the setting
:setting:`DUPEFILTER_CLASS`.
much because of a programming mistake. This can be configured in the
:setting:`DUPEFILTER_CLASS` setting.
Hopefully by now you have a good understanding of how to use the mechanism
of following links and callbacks with Scrapy.
@ -824,12 +823,12 @@ Next steps
==========
This tutorial covered only the basics of Scrapy, but there's a lot of other
features not mentioned here. Check the :ref:`topics-whatelse` section in
features not mentioned here. Check the :ref:`topics-whatelse` section in the
:ref:`intro-overview` chapter for a quick overview of the most important ones.
You can continue from the section :ref:`section-basics` to know more about the
command-line tool, spiders, selectors and other things the tutorial hasn't covered like
modeling the scraped data. If you prefer to play with an example project, check
modeling the scraped data. If you'd prefer to play with an example project, check
the :ref:`intro-examples` section.
.. _JSON: https://en.wikipedia.org/wiki/JSON

18
docs/topics/commands.rst
View File

@ -6,7 +6,7 @@
Command line tool
=================
Scrapy is controlled through the ``scrapy`` command-line tool, to be referred
Scrapy is controlled through the ``scrapy`` command-line tool, to be referred to
here as the "Scrapy tool" to differentiate it from the sub-commands, which we
just call "commands" or "Scrapy commands".
@ -185,8 +185,8 @@ And you can see all available commands with::
There are two kinds of commands, those that only work from inside a Scrapy
project (Project-specific commands) and those that also work without an active
Scrapy project (Global commands), though they may behave slightly different
when running from inside a project (as they would use the project overridden
Scrapy project (Global commands), though they may behave slightly differently
when run from inside a project (as they would use the project overridden
settings).
Global commands:
@ -236,7 +236,7 @@ genspider
.. versionadded:: 2.6.0
The ability to pass a URL instead of a domain.
Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
Creates a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
Usage example::
@ -253,7 +253,7 @@ Usage example::
$ scrapy genspider -t crawl scrapyorg scrapy.org
Created spider 'scrapyorg' using template 'crawl'
This is just a convenience shortcut command for creating spiders based on
This is just a convenient shortcut command for creating spiders based on
pre-defined templates, but certainly not the only way to create spiders. You
can just create the spider source code files yourself, instead of using this
command.
@ -274,9 +274,9 @@ Supported options:
* ``-a NAME=VALUE``: set a spider argument (may be repeated)
* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout), to define format set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``)
* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout). To define the output format, set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``)
* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file, to define format set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``)
* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file. To define the output format, set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``)
* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O``
@ -353,7 +353,7 @@ edit
Edit the given spider using the editor defined in the ``EDITOR`` environment
variable or (if unset) the :setting:`EDITOR` setting.
This command is provided only as a convenience shortcut for the most common
This command is provided only as a convenient shortcut for the most common
case, the developer is of course free to choose any tool or IDE to write and
debug spiders.
@ -372,7 +372,7 @@ fetch
Downloads the given URL using the Scrapy downloader and writes the contents to
standard output.
The interesting thing about this command is that it fetches the page how the
The interesting thing about this command is that it fetches the page the way the
spider would download it. For example, if the spider has a ``USER_AGENT``
attribute which overrides the User Agent, it will use that one.

7
docs/topics/dynamic-content.rst
View File

@ -115,15 +115,14 @@ Handling different response formats
Once you have a response with the desired data, how you extract the desired
data from it depends on the type of response:
- If the response is HTML or XML, use :ref:`selectors
- If the response is HTML, XML or JSON, use :ref:`selectors
<topics-selectors>` as usual.
- If the response is JSON, use :func:`json.loads` to load the desired data from
:attr:`response.text <scrapy.http.TextResponse.text>`:
- If the response is JSON, use :func:`response.json()` to load the desired data:
.. code-block:: python
data = json.loads(response.text)
data = response.json()
If the desired data is inside HTML or XML code embedded within JSON data,
you can load that HTML or XML code into a

13
docs/topics/extensions.rst
View File

@ -317,6 +317,19 @@ crawls more than that, the spider will be closed with the reason
``closespider_pagecount``. If zero (or non set), spiders won't be closed by
number of crawled responses.
.. setting:: CLOSESPIDER_PAGECOUNT_NO_ITEM
CLOSESPIDER_PAGECOUNT_NO_ITEM
"""""""""""""""""""""""""""""
Default: ``0``
An integer which specifies the maximum number of consecutive responses to crawl
without items scraped. If the spider crawls more consecutive responses than that
and no items are scraped in the meantime, the spider will be closed with the
reason ``closespider_pagecount_no_item``. If zero (or not set), spiders won't be
closed by number of crawled responses with no items.
.. setting:: CLOSESPIDER_ERRORCOUNT
CLOSESPIDER_ERRORCOUNT

4
docs/topics/item-pipeline.rst
View File

@ -99,7 +99,7 @@ contain a price:
adapter["price"] = adapter["price"] * self.vat_factor
return item
else:
raise DropItem(f"Missing price in {item}")
raise DropItem("Missing price")
Write items to a JSON lines file
@ -254,7 +254,7 @@ returns multiples items with the same id:
def process_item(self, item, spider):
adapter = ItemAdapter(item)
if adapter["id"] in self.ids_seen:
raise DropItem(f"Duplicate item found: {item!r}")
raise DropItem(f"Item ID already seen: {adapter['id']}")
else:
self.ids_seen.add(adapter["id"])
return item

2
docs/topics/link-extractors.rst
View File

@ -85,7 +85,7 @@ LxmlLinkExtractor
:param restrict_xpaths: is an XPath (or list of XPath's) which defines
regions inside the response where links should be extracted from.
If given, only the text selected by those XPath will be scanned for
links. See examples below.
links.
:type restrict_xpaths: str or list
:param restrict_css: a CSS selector (or list of selectors) which defines

40
docs/topics/practices.rst
View File

@ -92,7 +92,6 @@ reactor after ``MySpider`` has finished running.
.. code-block:: python
from twisted.internet import reactor
import scrapy
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
@ -107,6 +106,37 @@ reactor after ``MySpider`` has finished running.
runner = CrawlerRunner()
d = runner.crawl(MySpider)
from twisted.internet import reactor
d.addBoth(lambda _: reactor.stop())
reactor.run() # the script will block here until the crawling is finished
Same example but using a non-default reactor, it's only necessary call
``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically.
.. code-block:: python
import scrapy
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
class MySpider(scrapy.Spider):
# Your spider definition
...
configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"})
from scrapy.utils.reactor import install_reactor
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
runner = CrawlerRunner()
d = runner.crawl(MySpider)
from twisted.internet import reactor
d.addBoth(lambda _: reactor.stop())
reactor.run() # the script will block here until the crawling is finished
@ -151,7 +181,6 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
.. code-block:: python
import scrapy
from twisted.internet import reactor
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
@ -173,6 +202,9 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
runner.crawl(MySpider1)
runner.crawl(MySpider2)
d = runner.join()
from twisted.internet import reactor
d.addBoth(lambda _: reactor.stop())
reactor.run() # the script will block here until all crawling jobs are finished
@ -181,7 +213,7 @@ Same example but running the spiders sequentially by chaining the deferreds:
.. code-block:: python
from twisted.internet import reactor, defer
from twisted.internet import defer
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
@ -209,6 +241,8 @@ Same example but running the spiders sequentially by chaining the deferreds:
reactor.stop()
from twisted.internet import reactor
crawl()
reactor.run() # the script will block here until the last crawl call is finished

8
docs/topics/selectors.rst
View File

@ -1060,6 +1060,12 @@ Selector objects
For convenience, this method can be called as ``response.css()``
.. automethod:: jmespath
.. note::
For convenience, this method can be called as ``response.jmespath()``
.. automethod:: get
See also: :ref:`old-extraction-api`
@ -1092,6 +1098,8 @@ SelectorList objects
.. automethod:: css
.. automethod:: jmespath
.. automethod:: getall
See also: :ref:`old-extraction-api`

6
scrapy/__init__.py
View File

@ -33,12 +33,6 @@ version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."
twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Check minimum required Python version
if sys.version_info < (3, 8):
print(f"Scrapy {__version__} requires Python 3.8+")
sys.exit(1)
# Ignore noisy twisted deprecation warnings
warnings.filterwarnings("ignore", category=DeprecationWarning, module="twisted")

9
scrapy/addons.py
View File

@ -1,13 +1,16 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, List
from scrapy.exceptions import NotConfigured
from scrapy.settings import Settings
from scrapy.utils.conf import build_component_list
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
from scrapy.crawler import Crawler
from scrapy.settings import Settings
logger = logging.getLogger(__name__)
@ -15,8 +18,8 @@ logger = logging.getLogger(__name__)
class AddonManager:
"""This class facilitates loading and storing :ref:`topics-addons`."""
def __init__(self, crawler: "Crawler") -> None:
self.crawler: "Crawler" = crawler
def __init__(self, crawler: Crawler) -> None:
self.crawler: Crawler = crawler
self.addons: List[Any] = []
def load_settings(self, settings: Settings) -> None:

3
scrapy/cmdline.py
View File

@ -12,7 +12,6 @@ import scrapy
from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
from scrapy.crawler import CrawlerProcess
from scrapy.exceptions import UsageError
from scrapy.settings import BaseSettings, Settings
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import get_project_settings, inside_project
from scrapy.utils.python import garbage_collect
@ -21,6 +20,8 @@ if TYPE_CHECKING:
# typing.ParamSpec requires Python 3.10
from typing_extensions import ParamSpec
from scrapy.settings import BaseSettings, Settings
_P = ParamSpec("_P")

8
scrapy/commands/__init__.py
View File

@ -2,18 +2,22 @@
Base class for Scrapy commands
"""
from __future__ import annotations
import argparse
import builtins
import os
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional
from twisted.python import failure
from scrapy.crawler import Crawler, CrawlerProcess
from scrapy.exceptions import UsageError
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
if TYPE_CHECKING:
from scrapy.crawler import Crawler, CrawlerProcess
class ScrapyCommand:
requires_project: bool = False

8
scrapy/commands/bench.py
View File

@ -1,16 +1,20 @@
from __future__ import annotations
import argparse
import subprocess # nosec
import sys
import time
from typing import Any, Iterable, List
from typing import TYPE_CHECKING, Any, Iterable, List
from urllib.parse import urlencode
import scrapy
from scrapy import Request
from scrapy.commands import ScrapyCommand
from scrapy.http import Response, TextResponse
from scrapy.linkextractors import LinkExtractor
if TYPE_CHECKING:
from scrapy import Request
class Command(ScrapyCommand):
default_settings = {

8
scrapy/commands/crawl.py
View File

@ -1,11 +1,15 @@
import argparse
from typing import List, cast
from __future__ import annotations
from typing import TYPE_CHECKING, List, cast
from twisted.python.failure import Failure
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
if TYPE_CHECKING:
import argparse
class Command(BaseRunSpiderCommand):
requires_project = True

8
scrapy/commands/fetch.py
View File

@ -1,6 +1,7 @@
from __future__ import annotations
import sys
from argparse import ArgumentParser, Namespace
from typing import Dict, List, Type
from typing import TYPE_CHECKING, Dict, List, Type
from w3lib.url import is_url
@ -11,6 +12,9 @@ from scrapy.http import Request, Response
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.spider import DefaultSpider, spidercls_for_request
if TYPE_CHECKING:
from argparse import ArgumentParser, Namespace
class Command(ScrapyCommand):
requires_project = False

8
scrapy/commands/list.py
View File

@ -1,8 +1,12 @@
import argparse
from typing import List
from __future__ import annotations
from typing import TYPE_CHECKING, List
from scrapy.commands import ScrapyCommand
if TYPE_CHECKING:
import argparse
class Command(ScrapyCommand):
requires_project = True

41
scrapy/commands/parse.py
View File

@ -1,13 +1,15 @@
from __future__ import annotations
import argparse
import functools
import inspect
import json
import logging
from types import CoroutineType
from typing import (
TYPE_CHECKING,
Any,
AsyncGenerator,
Callable,
Coroutine,
Dict,
Iterable,
List,
@ -20,13 +22,11 @@ from typing import (
from itemadapter import ItemAdapter, is_item
from twisted.internet.defer import Deferred, maybeDeferred
from twisted.python.failure import Failure
from w3lib.url import is_url
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils import display
from scrapy.utils.asyncgen import collect_asyncgen
from scrapy.utils.defer import aiter_errback, deferred_from_coro
@ -34,6 +34,13 @@ from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.spider import spidercls_for_request
if TYPE_CHECKING:
from twisted.python.failure import Failure
from scrapy.http.request import CallbackT
from scrapy.spiders import Spider
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
@ -140,13 +147,13 @@ class Command(BaseRunSpiderCommand):
@overload
def iterate_spider_output(
self, result: Union[AsyncGenerator, CoroutineType]
) -> Deferred: ...
self, result: Union[AsyncGenerator[_T, None], Coroutine[Any, Any, _T]]
) -> Deferred[_T]: ...
@overload
def iterate_spider_output(self, result: _T) -> Iterable: ...
def iterate_spider_output(self, result: _T) -> Iterable[Any]: ...
def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]:
def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred[Any]]:
if inspect.isasyncgen(result):
d = deferred_from_coro(
collect_asyncgen(aiter_errback(result, self.handle_exception))
@ -211,8 +218,8 @@ class Command(BaseRunSpiderCommand):
opts: argparse.Namespace,
depth: int,
spider: Spider,
callback: Callable,
) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, Callable]:
callback: CallbackT,
) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT]:
items, requests = [], []
for x in spider_output:
if is_item(x):
@ -224,16 +231,16 @@ class Command(BaseRunSpiderCommand):
def run_callback(
self,
response: Response,
callback: Callable,
callback: CallbackT,
cb_kwargs: Optional[Dict[str, Any]] = None,
) -> Deferred:
) -> Deferred[Any]:
cb_kwargs = cb_kwargs or {}
d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs))
return d
def get_callback_from_rules(
self, spider: Spider, response: Response
) -> Union[Callable, str, None]:
) -> Union[CallbackT, str, None]:
if getattr(spider, "rules", None):
for rule in spider.rules: # type: ignore[attr-defined]
if rule.link_extractor.matches(response.url):
@ -279,7 +286,7 @@ class Command(BaseRunSpiderCommand):
def scraped_data(
self,
args: Tuple[
List[Any], List[Request], argparse.Namespace, int, Spider, Callable
List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT
],
) -> List[Any]:
items, requests, opts, depth, spider, callback = args
@ -306,8 +313,8 @@ class Command(BaseRunSpiderCommand):
spider: Spider,
opts: argparse.Namespace,
response: Optional[Response] = None,
) -> Callable:
cb: Union[str, Callable, None] = None
) -> CallbackT:
cb: Union[str, CallbackT, None] = None
if response:
cb = response.meta["_callback"]
if not cb:
@ -338,7 +345,7 @@ class Command(BaseRunSpiderCommand):
def prepare_request(
self, spider: Spider, request: Request, opts: argparse.Namespace
) -> Request:
def callback(response: Response, **cb_kwargs: Any) -> Deferred:
def callback(response: Response, **cb_kwargs: Any) -> Deferred[List[Any]]:
# memorize first request
if not self.first_response:
self.first_response = response

12
scrapy/commands/runspider.py
View File

@ -1,17 +1,21 @@
from __future__ import annotations
import argparse
import sys
from importlib import import_module
from os import PathLike
from pathlib import Path
from types import ModuleType
from typing import List, Union
from typing import TYPE_CHECKING, List, Union
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.utils.spider import iter_spider_classes
if TYPE_CHECKING:
from os import PathLike
from types import ModuleType
def _import_file(filepath: Union[str, PathLike]) -> ModuleType:
def _import_file(filepath: Union[str, PathLike[str]]) -> ModuleType:
abspath = Path(filepath).resolve()
if abspath.suffix not in (".py", ".pyw"):
raise ValueError(f"Not a Python source file: {abspath}")

8
scrapy/commands/shell.py
View File

@ -4,9 +4,10 @@ Scrapy Shell
See documentation in docs/topics/shell.rst
"""
from argparse import ArgumentParser, Namespace
from __future__ import annotations
from threading import Thread
from typing import Any, Dict, List, Type
from typing import TYPE_CHECKING, Any, Dict, List, Type
from scrapy import Spider
from scrapy.commands import ScrapyCommand
@ -15,6 +16,9 @@ from scrapy.shell import Shell
from scrapy.utils.spider import DefaultSpider, spidercls_for_request
from scrapy.utils.url import guess_scheme
if TYPE_CHECKING:
from argparse import ArgumentParser, Namespace
class Command(ScrapyCommand):
requires_project = False

88
scrapy/contracts/__init__.py
View File

@ -1,32 +1,53 @@
from __future__ import annotations
import re
import sys
from functools import wraps
from inspect import getmembers
from types import CoroutineType
from typing import AsyncGenerator, Dict, Optional, Type
from unittest import TestCase
from typing import (
TYPE_CHECKING,
Any,
AsyncGenerator,
Callable,
Dict,
Iterable,
List,
Optional,
Tuple,
Type,
cast,
)
from unittest import TestCase, TestResult
from scrapy.http import Request
from scrapy.http import Request, Response
from scrapy.utils.python import get_spec
from scrapy.utils.spider import iterate_spider_output
if TYPE_CHECKING:
from twisted.python.failure import Failure
from scrapy import Spider
class Contract:
"""Abstract class for contracts"""
request_cls: Optional[Type[Request]] = None
name: str
def __init__(self, method, *args):
def __init__(self, method: Callable, *args: Any):
self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook")
self.testcase_post = _create_testcase(method, f"@{self.name} post-hook")
self.args = args
self.args: Tuple[Any, ...] = args
def add_pre_hook(self, request, results):
def add_pre_hook(self, request: Request, results: TestResult) -> Request:
if hasattr(self, "pre_process"):
cb = request.callback
assert cb is not None
@wraps(cb)
def wrapper(response, **cb_kwargs):
def wrapper(response: Response, **cb_kwargs: Any) -> List[Any]:
try:
results.startTest(self.testcase_pre)
self.pre_process(response)
@ -42,23 +63,24 @@ class Contract:
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
raise TypeError("Contracts don't support async callbacks")
return list( # pylint: disable=return-in-finally
iterate_spider_output(cb_result)
cast(Iterable[Any], iterate_spider_output(cb_result))
)
request.callback = wrapper
return request
def add_post_hook(self, request, results):
def add_post_hook(self, request: Request, results: TestResult) -> Request:
if hasattr(self, "post_process"):
cb = request.callback
assert cb is not None
@wraps(cb)
def wrapper(response, **cb_kwargs):
def wrapper(response: Response, **cb_kwargs: Any) -> List[Any]:
cb_result = cb(response, **cb_kwargs)
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
raise TypeError("Contracts don't support async callbacks")
output = list(iterate_spider_output(cb_result))
output = list(cast(Iterable[Any], iterate_spider_output(cb_result)))
try:
results.startTest(self.testcase_post)
self.post_process(output)
@ -76,18 +98,18 @@ class Contract:
return request
def adjust_request_args(self, args):
def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]:
return args
class ContractsManager:
contracts: Dict[str, Contract] = {}
contracts: Dict[str, Type[Contract]] = {}
def __init__(self, contracts):
def __init__(self, contracts: Iterable[Type[Contract]]):
for contract in contracts:
self.contracts[contract.name] = contract
def tested_methods_from_spidercls(self, spidercls):
def tested_methods_from_spidercls(self, spidercls: Type[Spider]) -> List[str]:
is_method = re.compile(r"^\s*@", re.MULTILINE).search
methods = []
for key, value in getmembers(spidercls):
@ -96,21 +118,27 @@ class ContractsManager:
return methods
def extract_contracts(self, method):
contracts = []
def extract_contracts(self, method: Callable) -> List[Contract]:
contracts: List[Contract] = []
assert method.__doc__ is not None
for line in method.__doc__.split("\n"):
line = line.strip()
if line.startswith("@"):
name, args = re.match(r"@(\w+)\s*(.*)", line).groups()
m = re.match(r"@(\w+)\s*(.*)", line)
if m is None:
continue
name, args = m.groups()
args = re.split(r"\s+", args)
contracts.append(self.contracts[name](method, *args))
return contracts
def from_spider(self, spider, results):
requests = []
def from_spider(
self, spider: Spider, results: TestResult
) -> List[Optional[Request]]:
requests: List[Optional[Request]] = []
for method in self.tested_methods_from_spidercls(type(spider)):
bound_method = spider.__getattribute__(method)
try:
@ -121,7 +149,7 @@ class ContractsManager:
return requests
def from_method(self, method, results):
def from_method(self, method: Callable, results: TestResult) -> Optional[Request]:
contracts = self.extract_contracts(method)
if contracts:
request_cls = Request
@ -154,22 +182,26 @@ class ContractsManager:
self._clean_req(request, method, results)
return request
return None
def _clean_req(self, request, method, results):
def _clean_req(
self, request: Request, method: Callable, results: TestResult
) -> None:
"""stop the request from returning objects and records any errors"""
cb = request.callback
assert cb is not None
@wraps(cb)
def cb_wrapper(response, **cb_kwargs):
def cb_wrapper(response: Response, **cb_kwargs: Any) -> None:
try:
output = cb(response, **cb_kwargs)
output = list(iterate_spider_output(output))
output = list(cast(Iterable[Any], iterate_spider_output(output)))
except Exception:
case = _create_testcase(method, "callback")
results.addError(case, sys.exc_info())
def eb_wrapper(failure):
def eb_wrapper(failure: Failure) -> None:
case = _create_testcase(method, "errback")
exc_info = failure.type, failure.value, failure.getTracebackObject()
results.addError(case, exc_info)
@ -178,11 +210,11 @@ class ContractsManager:
request.errback = eb_wrapper
def _create_testcase(method, desc):
spider = method.__self__.name
def _create_testcase(method: Callable, desc: str) -> TestCase:
spider = method.__self__.name # type: ignore[attr-defined]
class ContractTestCase(TestCase):
def __str__(_self):
def __str__(_self) -> str:
return f"[{spider}] {method.__name__} ({desc})"
name = f"{spider}_{method.__name__}"

19
scrapy/contracts/default.py
View File

@ -1,4 +1,5 @@
import json
from typing import Any, Callable, Dict, List, Optional
from itemadapter import ItemAdapter, is_item
@ -15,7 +16,7 @@ class UrlContract(Contract):
name = "url"
def adjust_request_args(self, args):
def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]:
args["url"] = self.args[0]
return args
@ -29,7 +30,7 @@ class CallbackKeywordArgumentsContract(Contract):
name = "cb_kwargs"
def adjust_request_args(self, args):
def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]:
args["cb_kwargs"] = json.loads(" ".join(self.args))
return args
@ -48,14 +49,14 @@ class ReturnsContract(Contract):
"""
name = "returns"
object_type_verifiers = {
object_type_verifiers: Dict[Optional[str], Callable[[Any], bool]] = {
"request": lambda x: isinstance(x, Request),
"requests": lambda x: isinstance(x, Request),
"item": is_item,
"items": is_item,
}
def __init__(self, *args, **kwargs):
def __init__(self, *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
if len(self.args) not in [1, 2, 3]:
@ -66,16 +67,16 @@ class ReturnsContract(Contract):
self.obj_type_verifier = self.object_type_verifiers[self.obj_name]
try:
self.min_bound = int(self.args[1])
self.min_bound: float = int(self.args[1])
except IndexError:
self.min_bound = 1
try:
self.max_bound = int(self.args[2])
self.max_bound: float = int(self.args[2])
except IndexError:
self.max_bound = float("inf")
def post_process(self, output):
def post_process(self, output: List[Any]) -> None:
occurrences = 0
for x in output:
if self.obj_type_verifier(x):
@ -85,7 +86,7 @@ class ReturnsContract(Contract):
if not assertion:
if self.min_bound == self.max_bound:
expected = self.min_bound
expected = str(self.min_bound)
else:
expected = f"{self.min_bound}..{self.max_bound}"
@ -101,7 +102,7 @@ class ScrapesContract(Contract):
name = "scrapes"
def post_process(self, output):
def post_process(self, output: List[Any]) -> None:
for x in output:
if is_item(x):
missing = [arg for arg in self.args if arg not in ItemAdapter(x)]

50
scrapy/core/downloader/__init__.py
View File

@ -1,9 +1,22 @@
from __future__ import annotations
import random
import warnings
from collections import deque
from datetime import datetime
from time import time
from typing import TYPE_CHECKING, Any, Deque, Dict, Optional, Set, Tuple, cast
from typing import (
TYPE_CHECKING,
Any,
Deque,
Dict,
Optional,
Set,
Tuple,
TypeVar,
Union,
cast,
)
from twisted.internet import task
from twisted.internet.defer import Deferred
@ -12,15 +25,18 @@ from scrapy import Request, Spider, signals
from scrapy.core.downloader.handlers import DownloadHandlers
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Response
from scrapy.resolver import dnscache
from scrapy.settings import BaseSettings
from scrapy.signalmanager import SignalManager
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.settings import BaseSettings
_T = TypeVar("_T")
class Slot:
@ -40,7 +56,7 @@ class Slot:
self.throttle = throttle
self.active: Set[Request] = set()
self.queue: Deque[Tuple[Request, Deferred]] = deque()
self.queue: Deque[Tuple[Request, Deferred[Response]]] = deque()
self.transferring: Set[Request] = set()
self.lastseen: float = 0
self.latercall = None
@ -93,7 +109,7 @@ def _get_concurrency_delay(
class Downloader:
DOWNLOAD_SLOT = "download_slot"
def __init__(self, crawler: "Crawler"):
def __init__(self, crawler: Crawler):
self.settings: BaseSettings = crawler.settings
self.signals: SignalManager = crawler.signals
self.slots: Dict[str, Slot] = {}
@ -114,13 +130,17 @@ class Downloader:
"DOWNLOAD_SLOTS", {}
)
def fetch(self, request: Request, spider: Spider) -> Deferred:
def _deactivate(response: Response) -> Response:
def fetch(
self, request: Request, spider: Spider
) -> Deferred[Union[Response, Request]]:
def _deactivate(response: _T) -> _T:
self.active.remove(request)
return response
self.active.add(request)
dfd = self.middleware.download(self._enqueue_request, request, spider)
dfd: Deferred[Union[Response, Request]] = self.middleware.download(
self._enqueue_request, request, spider
)
return dfd.addBoth(_deactivate)
def needs_backout(self) -> bool:
@ -163,7 +183,7 @@ class Downloader:
)
return self.get_slot_key(request)
def _enqueue_request(self, request: Request, spider: Spider) -> Deferred:
def _enqueue_request(self, request: Request, spider: Spider) -> Deferred[Response]:
key, slot = self._get_slot(request, spider)
request.meta[self.DOWNLOAD_SLOT] = key
@ -175,7 +195,7 @@ class Downloader:
self.signals.send_catch_log(
signal=signals.request_reached_downloader, request=request, spider=spider
)
deferred: Deferred = Deferred().addBoth(_deactivate)
deferred: Deferred[Response] = Deferred().addBoth(_deactivate)
slot.queue.append((request, deferred))
self._process_queue(spider, slot)
return deferred
@ -208,11 +228,15 @@ class Downloader:
self._process_queue(spider, slot)
break
def _download(self, slot: Slot, request: Request, spider: Spider) -> Deferred:
def _download(
self, slot: Slot, request: Request, spider: Spider
) -> Deferred[Response]:
# The order is very important for the following deferreds. Do not change!
# 1. Create the download deferred
dfd = mustbe_deferred(self.handlers.download_request, request, spider)
dfd: Deferred[Response] = mustbe_deferred(
self.handlers.download_request, request, spider
)
# 2. Notify response_downloaded listeners about the recent download
# before querying queue for next request
@ -233,7 +257,7 @@ class Downloader:
# middleware itself)
slot.transferring.add(request)
def finish_transferring(_: Any) -> Any:
def finish_transferring(_: _T) -> _T:
slot.transferring.remove(request)
self._process_queue(spider, slot)
self.signals.send_catch_log(

13
scrapy/core/downloader/contextfactory.py
View File

@ -21,8 +21,6 @@ from scrapy.core.downloader.tls import (
ScrapyClientTLSOptions,
openssl_methods,
)
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
@ -31,6 +29,9 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
@implementer(IPolicyForHTTPS)
class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
@ -107,7 +108,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT
return ctx
def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
return ScrapyClientTLSOptions(
hostname.decode("ascii"),
self.getContext(),
@ -134,7 +135,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
``SSLv23_METHOD``) which allows TLS protocol negotiation.
"""
def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
# trustRoot set to platformTrust() will use the platform's root CAs.
#
# This means that a website like https://www.cacert.org will be rejected
@ -158,8 +159,8 @@ class AcceptableProtocolsContextFactory:
self._wrapped_context_factory: Any = context_factory
self._acceptable_protocols: List[bytes] = acceptable_protocols
def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
options: "ClientTLSOptions" = self._wrapped_context_factory.creatorForNetloc(
def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
options: ClientTLSOptions = self._wrapped_context_factory.creatorForNetloc(
hostname, port
)
_setAcceptableProtocols(options._ctx, self._acceptable_protocols)

57
scrapy/core/downloader/handlers/__init__.py
View File

@ -1,10 +1,22 @@
"""Download handlers for different schemes"""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Union, cast
from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
Generator,
Optional,
Protocol,
Type,
Union,
cast,
)
from twisted.internet import defer
from twisted.internet.defer import Deferred
from scrapy import Request, Spider, signals
from scrapy.exceptions import NotConfigured, NotSupported
@ -13,21 +25,36 @@ from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
from scrapy.crawler import Crawler
from scrapy.http import Response
logger = logging.getLogger(__name__)
class DownloadHandlerProtocol(Protocol):
def download_request(
self, request: Request, spider: Spider
) -> Deferred[Response]: ...
class DownloadHandlers:
def __init__(self, crawler: "Crawler"):
self._crawler: "Crawler" = crawler
self._schemes: Dict[str, Union[str, Callable]] = (
def __init__(self, crawler: Crawler):
self._crawler: Crawler = crawler
self._schemes: Dict[str, Union[str, Callable[..., Any]]] = (
{}
) # stores acceptable schemes on instancing
self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes
self._handlers: Dict[str, DownloadHandlerProtocol] = (
{}
) # stores instanced handlers for schemes
self._notconfigured: Dict[str, str] = {} # remembers failed handlers
handlers: Dict[str, Union[str, Callable]] = without_none_values(
crawler.settings.getwithbase("DOWNLOAD_HANDLERS")
handlers: Dict[str, Union[str, Callable[..., Any]]] = without_none_values(
cast(
Dict[str, Union[str, Callable[..., Any]]],
crawler.settings.getwithbase("DOWNLOAD_HANDLERS"),
)
)
for scheme, clspath in handlers.items():
self._schemes[scheme] = clspath
@ -35,7 +62,7 @@ class DownloadHandlers:
crawler.signals.connect(self._close, signals.engine_stopped)
def _get_handler(self, scheme: str) -> Any:
def _get_handler(self, scheme: str) -> Optional[DownloadHandlerProtocol]:
"""Lazy-load the downloadhandler for a scheme
only on the first request for that scheme.
"""
@ -49,10 +76,12 @@ class DownloadHandlers:
return self._load_handler(scheme)
def _load_handler(self, scheme: str, skip_lazy: bool = False) -> Any:
def _load_handler(
self, scheme: str, skip_lazy: bool = False
) -> Optional[DownloadHandlerProtocol]:
path = self._schemes[scheme]
try:
dhcls = load_object(path)
dhcls: Type[DownloadHandlerProtocol] = load_object(path)
if skip_lazy and getattr(dhcls, "lazy", True):
return None
dh = build_from_crawler(
@ -75,17 +104,17 @@ class DownloadHandlers:
self._handlers[scheme] = dh
return dh
def download_request(self, request: Request, spider: Spider) -> Deferred:
def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
scheme = urlparse_cached(request).scheme
handler = self._get_handler(scheme)
if not handler:
raise NotSupported(
f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}"
)
return cast(Deferred, handler.download_request(request, spider))
return handler.download_request(request, spider)
@defer.inlineCallbacks
def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred, Any, None]:
def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred[Any], Any, None]:
for dh in self._handlers.values():
if hasattr(dh, "close"):
yield dh.close()

8
scrapy/core/downloader/handlers/datauri.py
View File

@ -1,12 +1,16 @@
from typing import Any, Dict
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Dict
from w3lib.url import parse_data_uri
from scrapy import Request, Spider
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.utils.decorators import defers
if TYPE_CHECKING:
from scrapy import Request, Spider
class DataURIDownloadHandler:
lazy = False

9
scrapy/core/downloader/handlers/file.py
View File

@ -1,12 +1,17 @@
from __future__ import annotations
from pathlib import Path
from typing import TYPE_CHECKING
from w3lib.url import file_uri_to_path
from scrapy import Request, Spider
from scrapy.http import Response
from scrapy.responsetypes import responsetypes
from scrapy.utils.decorators import defers
if TYPE_CHECKING:
from scrapy import Request, Spider
from scrapy.http import Response
class FileDownloadHandler:
lazy = False

22
scrapy/core/downloader/handlers/ftp.py
View File

@ -35,23 +35,25 @@ from io import BytesIO
from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional
from urllib.parse import unquote
from twisted.internet.defer import Deferred
from twisted.internet.protocol import ClientCreator, Protocol
from twisted.protocols.ftp import CommandFailed, FTPClient
from twisted.python.failure import Failure
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.responsetypes import responsetypes
from scrapy.settings import BaseSettings
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
class ReceivedDataProtocol(Protocol):
def __init__(self, filename: Optional[str] = None):
@ -91,7 +93,7 @@ class FTPDownloadHandler:
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings)
def download_request(self, request: Request, spider: Spider) -> Deferred:
def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
from twisted.internet import reactor
parsed_url = urlparse_cached(request)
@ -103,10 +105,14 @@ class FTPDownloadHandler:
creator = ClientCreator(
reactor, FTPClient, user, password, passive=passive_mode
)
dfd: Deferred = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21)
dfd: Deferred[FTPClient] = creator.connectTCP(
parsed_url.hostname, parsed_url.port or 21
)
return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path))
def gotClient(self, client: FTPClient, request: Request, filepath: str) -> Deferred:
def gotClient(
self, client: FTPClient, request: Request, filepath: str
) -> Deferred[Response]:
self.client = client
protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename"))
d = client.retrieveFile(filepath, protocol)

16
scrapy/core/downloader/handlers/http10.py
View File

@ -5,20 +5,22 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Type
from twisted.internet.defer import Deferred
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
from twisted.internet.interfaces import IConnector
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Request, Spider
from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.settings import BaseSettings
class HTTP10DownloadHandler:
@ -38,13 +40,13 @@ class HTTP10DownloadHandler:
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings, crawler)
def download_request(self, request: Request, spider: Spider) -> Deferred:
def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
"""Return a deferred for the HTTP download"""
factory = self.HTTPClientFactory(request)
self._connect(factory)
return factory.deferred
def _connect(self, factory: ScrapyHTTPClientFactory) -> Deferred:
def _connect(self, factory: ScrapyHTTPClientFactory) -> IConnector:
from twisted.internet import reactor
host, port = to_unicode(factory.host), factory.port

78
scrapy/core/downloader/handlers/http11.py
View File

@ -8,15 +8,13 @@ import re
from contextlib import suppress
from io import BytesIO
from time import time
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, TypeVar, Union
from urllib.parse import urldefrag, urlunparse
from twisted.internet import ssl
from twisted.internet.base import ReactorBase
from twisted.internet.defer import CancelledError, Deferred, succeed
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.internet.error import TimeoutError
from twisted.internet.interfaces import IConsumer
from twisted.internet.protocol import Factory, Protocol, connectionDone
from twisted.python.failure import Failure
from twisted.web.client import URI, Agent, HTTPConnectionPool
@ -30,20 +28,35 @@ from zope.interface import implementer
from scrapy import Request, Spider, signals
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
from scrapy.crawler import Crawler
from scrapy.exceptions import StopDownload
from scrapy.http import Headers, Response
from scrapy.responsetypes import responsetypes
from scrapy.settings import BaseSettings
from scrapy.utils.python import to_bytes, to_unicode
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from twisted.internet.base import ReactorBase
from twisted.internet.interfaces import IConsumer
# typing.NotRequired and typing.Self require Python 3.11
from typing_extensions import NotRequired, Self
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
class _ResultT(TypedDict):
txresponse: TxResponse
body: bytes
flags: Optional[List[str]]
certificate: Optional[ssl.Certificate]
ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None]
failure: NotRequired[Optional[Failure]]
class HTTP11DownloadHandler:
lazy = False
@ -71,7 +84,7 @@ class HTTP11DownloadHandler:
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings, crawler)
def download_request(self, request: Request, spider: Spider) -> Deferred:
def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
"""Return a deferred for the HTTP download"""
agent = ScrapyAgent(
contextFactory=self._contextFactory,
@ -83,10 +96,10 @@ class HTTP11DownloadHandler:
)
return agent.download_request(request)
def close(self) -> Deferred:
def close(self) -> Deferred[None]:
from twisted.internet import reactor
d: Deferred = self._pool.closeCachedConnections()
d: Deferred[None] = self._pool.closeCachedConnections()
# closeCachedConnections will hang on network or server issues, so
# we'll manually timeout the deferred.
#
@ -97,7 +110,7 @@ class HTTP11DownloadHandler:
# issue a callback after `_disconnect_timeout` seconds.
delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, [])
def cancel_delayed_call(result: Any) -> Any:
def cancel_delayed_call(result: _T) -> _T:
if delayed_call.active():
delayed_call.cancel()
return result
@ -137,7 +150,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
):
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
self._tunnelReadyDeferred: Deferred = Deferred()
self._tunnelReadyDeferred: Deferred[Protocol] = Deferred()
self._tunneledHost: str = host
self._tunneledPort: int = port
self._contextFactory: IPolicyForHTTPS = contextFactory
@ -198,7 +211,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
"""Propagates the errback to the appropriate deferred."""
self._tunnelReadyDeferred.errback(reason)
def connect(self, protocolFactory: Factory) -> Deferred:
def connect(self, protocolFactory: Factory) -> Deferred[Protocol]:
self._protocolFactory = protocolFactory
connectDeferred = super().connect(protocolFactory)
connectDeferred.addCallback(self.requestTunnel)
@ -271,7 +284,7 @@ class TunnelingAgent(Agent):
headers: Optional[TxHeaders],
bodyProducer: Optional[IBodyProducer],
requestPath: bytes,
) -> Deferred:
) -> Deferred[TxResponse]:
# proxy host and port are required for HTTP pool `key`
# otherwise, same remote host connection request could reuse
# a cached tunneled connection to a different proxy
@ -310,7 +323,7 @@ class ScrapyProxyAgent(Agent):
uri: bytes,
headers: Optional[TxHeaders] = None,
bodyProducer: Optional[IBodyProducer] = None,
) -> Deferred:
) -> Deferred[TxResponse]:
"""
Issue a new request via the configured proxy.
"""
@ -394,7 +407,7 @@ class ScrapyAgent:
pool=self._pool,
)
def download_request(self, request: Request) -> Deferred:
def download_request(self, request: Request) -> Deferred[Response]:
from twisted.internet import reactor
timeout = request.meta.get("download_timeout") or self._connectTimeout
@ -411,22 +424,20 @@ class ScrapyAgent:
else:
bodyproducer = None
start_time = time()
d: Deferred = agent.request(
d: Deferred[TxResponse] = agent.request(
method, to_bytes(url, encoding="ascii"), headers, bodyproducer
)
# set download latency
d.addCallback(self._cb_latency, request, start_time)
# response body is ready to be consumed
d.addCallback(self._cb_bodyready, request)
d.addCallback(self._cb_bodydone, request, url)
d2: Deferred[_ResultT] = d.addCallback(self._cb_bodyready, request)
d3: Deferred[Response] = d2.addCallback(self._cb_bodydone, request, url)
# check download timeout
self._timeout_cl = reactor.callLater(timeout, d.cancel)
d.addBoth(self._cb_timeout, request, url, timeout)
return d
self._timeout_cl = reactor.callLater(timeout, d3.cancel)
d3.addBoth(self._cb_timeout, request, url, timeout)
return d3
def _cb_timeout(
self, result: Any, request: Request, url: str, timeout: float
) -> Any:
def _cb_timeout(self, result: _T, request: Request, url: str, timeout: float) -> _T:
if self._timeout_cl.active():
self._timeout_cl.cancel()
return result
@ -437,7 +448,7 @@ class ScrapyAgent:
raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
def _cb_latency(self, result: Any, request: Request, start_time: float) -> Any:
def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T:
request.meta["download_latency"] = time() - start_time
return result
@ -451,7 +462,7 @@ class ScrapyAgent:
def _cb_bodyready(
self, txresponse: TxResponse, request: Request
) -> Union[Dict[str, Any], Deferred]:
) -> Union[_ResultT, Deferred[_ResultT]]:
headers_received_result = self._crawler.signals.send_catch_log(
signal=signals.headers_received,
headers=self._headers_from_twisted_response(txresponse),
@ -520,7 +531,7 @@ class ScrapyAgent:
# Abort connection immediately.
txresponse._transport._producer.abortConnection()
d: Deferred = Deferred(_cancel)
d: Deferred[_ResultT] = Deferred(_cancel)
txresponse.deliverBody(
_ResponseReader(
finished=d,
@ -539,7 +550,7 @@ class ScrapyAgent:
return d
def _cb_bodydone(
self, result: Dict[str, Any], request: Request, url: str
self, result: _ResultT, request: Request, url: str
) -> Union[Response, Failure]:
headers = self._headers_from_twisted_response(result["txresponse"])
respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
@ -559,8 +570,9 @@ class ScrapyAgent:
protocol=protocol,
)
if result.get("failure"):
assert result["failure"]
result["failure"].value.response = response
return cast(Failure, result["failure"])
return result["failure"]
return response
@ -570,7 +582,7 @@ class _RequestBodyProducer:
self.body = body
self.length = len(body)
def startProducing(self, consumer: IConsumer) -> Deferred:
def startProducing(self, consumer: IConsumer) -> Deferred[None]:
consumer.write(self.body)
return succeed(None)
@ -584,7 +596,7 @@ class _RequestBodyProducer:
class _ResponseReader(Protocol):
def __init__(
self,
finished: Deferred,
finished: Deferred[_ResultT],
txresponse: TxResponse,
request: Request,
maxsize: int,
@ -592,7 +604,7 @@ class _ResponseReader(Protocol):
fail_on_dataloss: bool,
crawler: Crawler,
):
self._finished: Deferred = finished
self._finished: Deferred[_ResultT] = finished
self._txresponse: TxResponse = txresponse
self._request: Request = request
self._bodybuf: BytesIO = BytesIO()

20
scrapy/core/downloader/handlers/http2.py
View File

@ -4,25 +4,27 @@ from time import time
from typing import TYPE_CHECKING, Optional
from urllib.parse import urldefrag
from twisted.internet.base import DelayedCall
from twisted.internet.defer import Deferred
from twisted.internet.error import TimeoutError
from twisted.web.client import URI
from twisted.web.iweb import IPolicyForHTTPS
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent
from scrapy.crawler import Crawler
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.utils.python import to_bytes
if TYPE_CHECKING:
from twisted.internet.base import DelayedCall
from twisted.internet.defer import Deferred
from twisted.web.iweb import IPolicyForHTTPS
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.spiders import Spider
class H2DownloadHandler:
def __init__(self, settings: Settings, crawler: Crawler):
@ -37,7 +39,7 @@ class H2DownloadHandler:
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings, crawler)
def download_request(self, request: Request, spider: Spider) -> Deferred:
def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
agent = ScrapyH2Agent(
context_factory=self._context_factory,
pool=self._pool,
@ -98,7 +100,7 @@ class ScrapyH2Agent:
pool=self._pool,
)
def download_request(self, request: Request, spider: Spider) -> Deferred:
def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
from twisted.internet import reactor
timeout = request.meta.get("download_timeout") or self._connect_timeout

17
scrapy/core/downloader/handlers/s3.py
View File

@ -2,21 +2,23 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any, Optional, Type
from twisted.internet.defer import Deferred
from scrapy import Request, Spider
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.settings import BaseSettings
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import build_from_crawler
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.settings import BaseSettings
class S3DownloadHandler:
def __init__(
@ -59,7 +61,8 @@ class S3DownloadHandler:
assert aws_access_key_id is not None
assert aws_secret_access_key is not None
SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"]
self._signer = SignerCls(
# botocore.auth.BaseSigner doesn't have an __init__() with args, only subclasses do
self._signer = SignerCls( # type: ignore[call-arg]
botocore.credentials.Credentials(
aws_access_key_id, aws_secret_access_key, aws_session_token
)
@ -75,7 +78,7 @@ class S3DownloadHandler:
def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self:
return cls(crawler.settings, crawler=crawler, **kwargs)
def download_request(self, request: Request, spider: Spider) -> Deferred:
def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
p = urlparse_cached(request)
scheme = "https" if request.meta.get("is_secure") else "http"
bucket = p.hostname

34
scrapy/core/downloader/middleware.py
View File

@ -4,19 +4,24 @@ Downloader Middleware manager
See documentation in docs/topics/downloader-middleware.rst
"""
from typing import Any, Callable, Generator, List, Union, cast
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Callable, Generator, List, Union, cast
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.python.failure import Failure
from scrapy import Spider
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
from scrapy.middleware import MiddlewareManager
from scrapy.settings import BaseSettings
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import deferred_from_coro, mustbe_deferred
if TYPE_CHECKING:
from twisted.python.failure import Failure
from scrapy import Spider
from scrapy.settings import BaseSettings
class DownloaderMiddlewareManager(MiddlewareManager):
component_name = "downloader middleware"
@ -34,10 +39,15 @@ class DownloaderMiddlewareManager(MiddlewareManager):
self.methods["process_exception"].appendleft(mw.process_exception)
def download(
self, download_func: Callable, request: Request, spider: Spider
) -> Deferred:
self,
download_func: Callable[[Request, Spider], Deferred[Response]],
request: Request,
spider: Spider,
) -> Deferred[Union[Response, Request]]:
@inlineCallbacks
def process_request(request: Request) -> Generator[Deferred, Any, Any]:
def process_request(
request: Request,
) -> Generator[Deferred[Any], Any, Union[Response, Request]]:
for method in self.methods["process_request"]:
method = cast(Callable, method)
response = yield deferred_from_coro(
@ -52,12 +62,12 @@ class DownloaderMiddlewareManager(MiddlewareManager):
)
if response:
return response
return (yield download_func(request=request, spider=spider))
return (yield download_func(request, spider))
@inlineCallbacks
def process_response(
response: Union[Response, Request]
) -> Generator[Deferred, Any, Union[Response, Request]]:
) -> Generator[Deferred[Any], Any, Union[Response, Request]]:
if response is None:
raise TypeError("Received None in process_response")
elif isinstance(response, Request):
@ -80,7 +90,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
@inlineCallbacks
def process_exception(
failure: Failure,
) -> Generator[Deferred, Any, Union[Failure, Response, Request]]:
) -> Generator[Deferred[Any], Any, Union[Failure, Response, Request]]:
exception = failure.value
for method in self.methods["process_exception"]:
method = cast(Callable, method)
@ -98,7 +108,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
return response
return failure
deferred = mustbe_deferred(process_request, request)
deferred: Deferred[Union[Response, Request]] = mustbe_deferred(
process_request, request
)
deferred.addErrback(process_exception)
deferred.addCallback(process_response)
return deferred

14
scrapy/core/downloader/webclient.py
View File

@ -1,18 +1,22 @@
from __future__ import annotations
import re
from time import time
from typing import Optional, Tuple
from typing import TYPE_CHECKING, Optional, Tuple
from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
from twisted.internet import defer
from twisted.internet.protocol import ClientFactory
from twisted.web.http import HTTPClient
from scrapy import Request
from scrapy.http import Headers
from scrapy.http import Headers, Response
from scrapy.responsetypes import responsetypes
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes, to_unicode
if TYPE_CHECKING:
from scrapy import Request
def _parsed_url_args(parsed: ParseResult) -> Tuple[bytes, bytes, bytes, int, bytes]:
# Assume parsed is urlparse-d from Request.url,
@ -145,7 +149,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
self.response_headers: Optional[Headers] = None
self.timeout: float = request.meta.get("download_timeout") or timeout
self.start_time: float = time()
self.deferred: defer.Deferred = defer.Deferred().addCallback(
self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback(
self._build_response, request
)
@ -155,7 +159,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
# needed to add the callback _waitForDisconnect.
# Specifically this avoids the AttributeError exception when
# clientConnectionFailed method is called.
self._disconnectedDeferred: defer.Deferred = defer.Deferred()
self._disconnectedDeferred: defer.Deferred[None] = defer.Deferred()
self._set_connection_attributes(request)

96
scrapy/core/engine.py
View File

@ -5,6 +5,8 @@ For more information see docs/topics/architecture.rst
"""
from __future__ import annotations
import logging
from time import time
from typing import (
@ -17,6 +19,7 @@ from typing import (
Optional,
Set,
Type,
TypeVar,
Union,
cast,
)
@ -31,9 +34,8 @@ from scrapy.core.scraper import Scraper
from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest
from scrapy.http import Request, Response
from scrapy.logformatter import LogFormatter
from scrapy.settings import BaseSettings, Settings
from scrapy.settings import Settings
from scrapy.signalmanager import SignalManager
from scrapy.spiders import Spider
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import global_object_name
@ -41,25 +43,31 @@ from scrapy.utils.reactor import CallLaterOnce
if TYPE_CHECKING:
from scrapy.core.scheduler import BaseScheduler
from scrapy.core.scraper import _HandleOutputDeferred
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
class Slot:
def __init__(
self,
start_requests: Iterable[Request],
close_if_idle: bool,
nextcall: CallLaterOnce,
scheduler: "BaseScheduler",
nextcall: CallLaterOnce[None],
scheduler: BaseScheduler,
) -> None:
self.closing: Optional[Deferred] = None
self.closing: Optional[Deferred[None]] = None
self.inprogress: Set[Request] = set()
self.start_requests: Optional[Iterator[Request]] = iter(start_requests)
self.close_if_idle: bool = close_if_idle
self.nextcall: CallLaterOnce = nextcall
self.scheduler: "BaseScheduler" = scheduler
self.nextcall: CallLaterOnce[None] = nextcall
self.scheduler: BaseScheduler = scheduler
self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule)
def add_request(self, request: Request) -> None:
@ -69,7 +77,7 @@ class Slot:
self.inprogress.remove(request)
self._maybe_fire_closing()
def close(self) -> Deferred:
def close(self) -> Deferred[None]:
self.closing = Deferred()
self._maybe_fire_closing()
return self.closing
@ -84,8 +92,12 @@ class Slot:
class ExecutionEngine:
def __init__(self, crawler: "Crawler", spider_closed_callback: Callable) -> None:
self.crawler: "Crawler" = crawler
def __init__(
self,
crawler: Crawler,
spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]],
) -> None:
self.crawler: Crawler = crawler
self.settings: Settings = crawler.settings
self.signals: SignalManager = crawler.signals
assert crawler.logformatter
@ -94,19 +106,21 @@ class ExecutionEngine:
self.spider: Optional[Spider] = None
self.running: bool = False
self.paused: bool = False
self.scheduler_cls: Type["BaseScheduler"] = self._get_scheduler_class(
self.scheduler_cls: Type[BaseScheduler] = self._get_scheduler_class(
crawler.settings
)
downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"])
self.downloader: Downloader = downloader_cls(crawler)
self.scraper = Scraper(crawler)
self._spider_closed_callback: Callable = spider_closed_callback
self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = (
spider_closed_callback
)
self.start_time: Optional[float] = None
def _get_scheduler_class(self, settings: BaseSettings) -> Type["BaseScheduler"]:
def _get_scheduler_class(self, settings: BaseSettings) -> Type[BaseScheduler]:
from scrapy.core.scheduler import BaseScheduler
scheduler_cls: Type = load_object(settings["SCHEDULER"])
scheduler_cls: Type[BaseScheduler] = load_object(settings["SCHEDULER"])
if not issubclass(scheduler_cls, BaseScheduler):
raise TypeError(
f"The provided scheduler class ({settings['SCHEDULER']})"
@ -115,20 +129,20 @@ class ExecutionEngine:
return scheduler_cls
@inlineCallbacks
def start(self) -> Generator[Deferred, Any, None]:
def start(self) -> Generator[Deferred[Any], Any, None]:
if self.running:
raise RuntimeError("Engine already running")
self.start_time = time()
yield self.signals.send_catch_log_deferred(signal=signals.engine_started)
self.running = True
self._closewait: Deferred = Deferred()
self._closewait: Deferred[None] = Deferred()
yield self._closewait
def stop(self) -> Deferred:
def stop(self) -> Deferred[None]:
"""Gracefully stop the execution engine"""
@inlineCallbacks
def _finish_stopping_engine(_: Any) -> Generator[Deferred, Any, None]:
def _finish_stopping_engine(_: Any) -> Generator[Deferred[Any], Any, None]:
yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
self._closewait.callback(None)
@ -143,7 +157,7 @@ class ExecutionEngine:
)
return dfd.addBoth(_finish_stopping_engine)
def close(self) -> Deferred:
def close(self) -> Deferred[None]:
"""
Gracefully close the execution engine.
If it has already been started, stop it. In all cases, close the spider and the downloader.
@ -206,7 +220,7 @@ class ExecutionEngine:
or self.scraper.slot.needs_backout()
)
def _next_request_from_scheduler(self) -> Optional[Deferred]:
def _next_request_from_scheduler(self) -> Optional[Deferred[None]]:
assert self.slot is not None # typing
assert self.spider is not None # typing
@ -214,7 +228,7 @@ class ExecutionEngine:
if request is None:
return None
d = self._download(request)
d: Deferred[Union[Response, Request]] = self._download(request)
d.addBoth(self._handle_downloader_output, request)
d.addErrback(
lambda f: logger.info(
@ -228,8 +242,8 @@ class ExecutionEngine:
assert self.slot
self.slot.remove_request(request)
d.addBoth(_remove_request)
d.addErrback(
d2: Deferred[None] = d.addBoth(_remove_request)
d2.addErrback(
lambda f: logger.info(
"Error while removing request from slot",
exc_info=failure_to_exc_info(f),
@ -237,19 +251,19 @@ class ExecutionEngine:
)
)
slot = self.slot
d.addBoth(lambda _: slot.nextcall.schedule())
d.addErrback(
d2.addBoth(lambda _: slot.nextcall.schedule())
d2.addErrback(
lambda f: logger.info(
"Error while scheduling new request",
exc_info=failure_to_exc_info(f),
extra={"spider": self.spider},
)
)
return d
return d2
def _handle_downloader_output(
self, result: Union[Request, Response, Failure], request: Request
) -> Optional[Deferred]:
) -> Optional[_HandleOutputDeferred]:
assert self.spider is not None # typing
if not isinstance(result, (Request, Response, Failure)):
@ -311,20 +325,23 @@ class ExecutionEngine:
signals.request_dropped, request=request, spider=spider
)
def download(self, request: Request) -> Deferred:
def download(self, request: Request) -> Deferred[Response]:
"""Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
if self.spider is None:
raise RuntimeError(f"No open spider to crawl: {request}")
return self._download(request).addBoth(self._downloaded, request)
d: Deferred[Union[Response, Request]] = self._download(request)
# Deferred.addBoth() overloads don't seem to support a Union[_T, Deferred[_T]] return type
d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[arg-type]
return d2
def _downloaded(
self, result: Union[Response, Request, Failure], request: Request
) -> Union[Deferred, Response, Failure]:
) -> Union[Deferred[Response], Response, Failure]:
assert self.slot is not None # typing
self.slot.remove_request(request)
return self.download(result) if isinstance(result, Request) else result
def _download(self, request: Request) -> Deferred:
def _download(self, request: Request) -> Deferred[Union[Response, Request]]:
assert self.slot is not None # typing
self.slot.add_request(request)
@ -351,21 +368,26 @@ class ExecutionEngine:
)
return result
def _on_complete(_: Any) -> Any:
def _on_complete(_: _T) -> _T:
assert self.slot is not None
self.slot.nextcall.schedule()
return _
assert self.spider is not None
dwld = self.downloader.fetch(request, self.spider)
dwld: Deferred[Union[Response, Request]] = self.downloader.fetch(
request, self.spider
)
dwld.addCallback(_on_success)
dwld.addBoth(_on_complete)
return dwld
@inlineCallbacks
def open_spider(
self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True
) -> Generator[Deferred, Any, None]:
self,
spider: Spider,
start_requests: Iterable[Request] = (),
close_if_idle: bool = True,
) -> Generator[Deferred[Any], Any, None]:
if self.slot is not None:
raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
logger.info("Spider opened", extra={"spider": spider})
@ -411,7 +433,7 @@ class ExecutionEngine:
assert isinstance(ex, CloseSpider) # typing
self.close_spider(self.spider, reason=ex.reason)
def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred:
def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred[None]:
"""Close (cancel) spider and clear all its outstanding requests"""
if self.slot is None:
raise RuntimeError("Engine slot not assigned")
@ -425,7 +447,7 @@ class ExecutionEngine:
dfd = self.slot.close()
def log_failure(msg: str) -> Callable:
def log_failure(msg: str) -> Callable[[Failure], None]:
def errback(failure: Failure) -> None:
logger.error(
msg, exc_info=failure_to_exc_info(failure), extra={"spider": spider}

68
scrapy/core/http2/agent.py
View File

@ -1,10 +1,10 @@
from __future__ import annotations
from collections import deque
from typing import Deque, Dict, List, Optional, Tuple
from typing import TYPE_CHECKING, Deque, Dict, List, Optional, Tuple
from twisted.internet import defer
from twisted.internet.base import ReactorBase
from twisted.internet.defer import Deferred
from twisted.internet.endpoints import HostnameEndpoint
from twisted.python.failure import Failure
from twisted.web.client import (
URI,
@ -16,9 +16,17 @@ from twisted.web.error import SchemeNotSupported
from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory
from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol
from scrapy.http.request import Request
from scrapy.settings import Settings
from scrapy.spiders import Spider
if TYPE_CHECKING:
from twisted.internet.base import ReactorBase
from twisted.internet.endpoints import HostnameEndpoint
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.spiders import Spider
ConnectionKeyT = Tuple[bytes, bytes, int]
class H2ConnectionPool:
@ -28,19 +36,21 @@ class H2ConnectionPool:
# Store a dictionary which is used to get the respective
# H2ClientProtocolInstance using the key as Tuple(scheme, hostname, port)
self._connections: Dict[Tuple, H2ClientProtocol] = {}
self._connections: Dict[ConnectionKeyT, H2ClientProtocol] = {}
# Save all requests that arrive before the connection is established
self._pending_requests: Dict[Tuple, Deque[Deferred]] = {}
self._pending_requests: Dict[
ConnectionKeyT, Deque[Deferred[H2ClientProtocol]]
] = {}
def get_connection(
self, key: Tuple, uri: URI, endpoint: HostnameEndpoint
) -> Deferred:
self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint
) -> Deferred[H2ClientProtocol]:
if key in self._pending_requests:
# Received a request while connecting to remote
# Create a deferred which will fire with the H2ClientProtocol
# instance
d: Deferred = Deferred()
d: Deferred[H2ClientProtocol] = Deferred()
self._pending_requests[key].append(d)
return d
@ -54,22 +64,24 @@ class H2ConnectionPool:
return self._new_connection(key, uri, endpoint)
def _new_connection(
self, key: Tuple, uri: URI, endpoint: HostnameEndpoint
) -> Deferred:
self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint
) -> Deferred[H2ClientProtocol]:
self._pending_requests[key] = deque()
conn_lost_deferred: Deferred = Deferred()
conn_lost_deferred: Deferred[List[BaseException]] = Deferred()
conn_lost_deferred.addCallback(self._remove_connection, key)
factory = H2ClientFactory(uri, self.settings, conn_lost_deferred)
conn_d = endpoint.connect(factory)
conn_d.addCallback(self.put_connection, key)
d: Deferred = Deferred()
d: Deferred[H2ClientProtocol] = Deferred()
self._pending_requests[key].append(d)
return d
def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol:
def put_connection(
self, conn: H2ClientProtocol, key: ConnectionKeyT
) -> H2ClientProtocol:
self._connections[key] = conn
# Now as we have established a proper HTTP/2 connection
@ -81,7 +93,9 @@ class H2ConnectionPool:
return conn
def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None:
def _remove_connection(
self, errors: List[BaseException], key: ConnectionKeyT
) -> None:
self._connections.pop(key)
# Call the errback of all the pending requests for this connection
@ -119,17 +133,17 @@ class H2Agent:
self._reactor, self._context_factory, connect_timeout, bind_address
)
def get_endpoint(self, uri: URI):
def get_endpoint(self, uri: URI) -> HostnameEndpoint:
return self.endpoint_factory.endpointForURI(uri)
def get_key(self, uri: URI) -> Tuple:
def get_key(self, uri: URI) -> ConnectionKeyT:
"""
Arguments:
uri - URI obtained directly from request URL
"""
return uri.scheme, uri.host, uri.port
def request(self, request: Request, spider: Spider) -> Deferred:
def request(self, request: Request, spider: Spider) -> Deferred[Response]:
uri = URI.fromBytes(bytes(request.url, encoding="utf-8"))
try:
endpoint = self.get_endpoint(uri)
@ -137,9 +151,11 @@ class H2Agent:
return defer.fail(Failure())
key = self.get_key(uri)
d = self._pool.get_connection(key, uri, endpoint)
d.addCallback(lambda conn: conn.request(request, spider))
return d
d: Deferred[H2ClientProtocol] = self._pool.get_connection(key, uri, endpoint)
d2: Deferred[Response] = d.addCallback(
lambda conn: conn.request(request, spider)
)
return d2
class ScrapyProxyH2Agent(H2Agent):
@ -161,9 +177,9 @@ class ScrapyProxyH2Agent(H2Agent):
)
self._proxy_uri = proxy_uri
def get_endpoint(self, uri: URI):
def get_endpoint(self, uri: URI) -> HostnameEndpoint:
return self.endpoint_factory.endpointForURI(self._proxy_uri)
def get_key(self, uri: URI) -> Tuple:
def get_key(self, uri: URI) -> ConnectionKeyT:
"""We use the proxy uri instead of uri obtained from request url"""
return "http-proxy", self._proxy_uri.host, self._proxy_uri.port
return b"http-proxy", self._proxy_uri.host, self._proxy_uri.port

51
scrapy/core/http2/protocol.py
View File

@ -1,9 +1,10 @@
from __future__ import annotations
import ipaddress
import itertools
import logging
from collections import deque
from ipaddress import IPv4Address, IPv6Address
from typing import Dict, List, Optional, Union
from typing import TYPE_CHECKING, Any, Deque, Dict, List, Optional, Union
from h2.config import H2Configuration
from h2.connection import H2Connection
@ -20,20 +21,30 @@ from h2.events import (
WindowUpdated,
)
from h2.exceptions import FrameTooLargeError, H2Error
from twisted.internet.defer import Deferred
from twisted.internet.error import TimeoutError
from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory
from twisted.internet.interfaces import (
IAddress,
IHandshakeListener,
IProtocolNegotiationFactory,
)
from twisted.internet.protocol import Factory, Protocol, connectionDone
from twisted.internet.ssl import Certificate
from twisted.protocols.policies import TimeoutMixin
from twisted.python.failure import Failure
from twisted.web.client import URI
from zope.interface import implementer
from scrapy.core.http2.stream import Stream, StreamCloseReason
from scrapy.http import Request
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.http import Request, Response
if TYPE_CHECKING:
from ipaddress import IPv4Address, IPv6Address
from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
from twisted.web.client import URI
from scrapy.settings import Settings
from scrapy.spiders import Spider
logger = logging.getLogger(__name__)
@ -77,7 +88,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
IDLE_TIMEOUT = 240
def __init__(
self, uri: URI, settings: Settings, conn_lost_deferred: Deferred
self,
uri: URI,
settings: Settings,
conn_lost_deferred: Deferred[List[BaseException]],
) -> None:
"""
Arguments:
@ -88,7 +102,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
conn_lost_deferred -- Deferred fires with the reason: Failure to notify
that connection was lost
"""
self._conn_lost_deferred = conn_lost_deferred
self._conn_lost_deferred: Deferred[List[BaseException]] = conn_lost_deferred
config = H2Configuration(client_side=True, header_encoding="utf-8")
self.conn = H2Connection(config=config)
@ -103,7 +117,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
# If requests are received before connection is made we keep
# all requests in a pool and send them as the connection is made
self._pending_request_stream_pool: deque = deque()
self._pending_request_stream_pool: Deque[Stream] = deque()
# Save an instance of errors raised which lead to losing the connection
# We pass these instances to the streams ResponseFailed() failure
@ -111,7 +125,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
# Some meta data of this connection
# initialized when connection is successfully made
self.metadata: Dict = {
self.metadata: Dict[str, Any] = {
# Peer certificate instance
"certificate": None,
# Address of the server we are connected to which
@ -204,14 +218,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
data = self.conn.data_to_send()
self.transport.write(data)
def request(self, request: Request, spider: Spider) -> Deferred:
def request(self, request: Request, spider: Spider) -> Deferred[Response]:
if not isinstance(request, Request):
raise TypeError(
f"Expected scrapy.http.Request, received {request.__class__.__qualname__}"
)
stream = self._new_stream(request, spider)
d = stream.get_response()
d: Deferred[Response] = stream.get_response()
# Add the stream to the request pool
self._pending_request_stream_pool.append(stream)
@ -425,13 +439,16 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
@implementer(IProtocolNegotiationFactory)
class H2ClientFactory(Factory):
def __init__(
self, uri: URI, settings: Settings, conn_lost_deferred: Deferred
self,
uri: URI,
settings: Settings,
conn_lost_deferred: Deferred[List[BaseException]],
) -> None:
self.uri = uri
self.settings = settings
self.conn_lost_deferred = conn_lost_deferred
def buildProtocol(self, addr) -> H2ClientProtocol:
def buildProtocol(self, addr: IAddress) -> H2ClientProtocol:
return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred)
def acceptableProtocols(self) -> List[bytes]:

23
scrapy/core/http2/stream.py
View File

@ -1,23 +1,26 @@
from __future__ import annotations
import logging
from enum import Enum
from io import BytesIO
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
from h2.errors import ErrorCodes
from h2.exceptions import H2Error, ProtocolError, StreamClosedError
from hpack import HeaderTuple
from twisted.internet.defer import CancelledError, Deferred
from twisted.internet.error import ConnectionClosed
from twisted.python.failure import Failure
from twisted.web.client import ResponseFailed
from scrapy.http import Request
from scrapy.http.headers import Headers
from scrapy.responsetypes import responsetypes
from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
from hpack import HeaderTuple
from scrapy.core.http2.protocol import H2ClientProtocol
from scrapy.http import Request, Response
logger = logging.getLogger(__name__)
@ -87,7 +90,7 @@ class Stream:
self,
stream_id: int,
request: Request,
protocol: "H2ClientProtocol",
protocol: H2ClientProtocol,
download_maxsize: int = 0,
download_warnsize: int = 0,
) -> None:
@ -99,7 +102,7 @@ class Stream:
"""
self.stream_id: int = stream_id
self._request: Request = request
self._protocol: "H2ClientProtocol" = protocol
self._protocol: H2ClientProtocol = protocol
self._download_maxsize = self._request.meta.get(
"download_maxsize", download_maxsize
@ -110,7 +113,7 @@ class Stream:
# Metadata of an HTTP/2 connection stream
# initialized when stream is instantiated
self.metadata: Dict = {
self.metadata: Dict[str, Any] = {
"request_content_length": (
0 if self._request.body is None else len(self._request.body)
),
@ -131,7 +134,7 @@ class Stream:
# Private variable used to build the response
# this response is then converted to appropriate Response class
# passed to the response deferred callback
self._response: Dict = {
self._response: Dict[str, Any] = {
# Data received frame by frame from the server is appended
# and passed to the response Deferred when completely received.
"body": BytesIO(),
@ -142,7 +145,7 @@ class Stream:
"headers": Headers({}),
}
def _cancel(_) -> None:
def _cancel(_: Any) -> None:
# Close this stream as gracefully as possible
# If the associated request is initiated we reset this stream
# else we directly call close() method
@ -151,7 +154,7 @@ class Stream:
else:
self.close(StreamCloseReason.CANCELLED)
self._deferred_response: Deferred = Deferred(_cancel)
self._deferred_response: Deferred[Response] = Deferred(_cancel)
def __repr__(self) -> str:
return f"Stream(id={self.stream_id!r})"
@ -177,7 +180,7 @@ class Stream:
and not self.metadata["reached_warnsize"]
)
def get_response(self) -> Deferred:
def get_response(self) -> Deferred[Response]:
"""Simply return a Deferred which fires when response
from the asynchronous request is available
"""

32
scrapy/core/scheduler.py
View File

@ -4,16 +4,12 @@ import json
import logging
from abc import abstractmethod
from pathlib import Path
from typing import TYPE_CHECKING, Any, Optional, Type, cast
from typing import TYPE_CHECKING, Any, List, Optional, Type, cast
from twisted.internet.defer import Deferred
# working around https://github.com/sphinx-doc/sphinx/issues/10400
from twisted.internet.defer import Deferred # noqa: TC002
from scrapy.crawler import Crawler
from scrapy.dupefilters import BaseDupeFilter
from scrapy.http.request import Request
from scrapy.pqueues import ScrapyPriorityQueue
from scrapy.spiders import Spider
from scrapy.statscollectors import StatsCollector
from scrapy.spiders import Spider # noqa: TC001
from scrapy.utils.job import job_dir
from scrapy.utils.misc import build_from_crawler, load_object
@ -24,6 +20,12 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.dupefilters import BaseDupeFilter
from scrapy.http.request import Request
from scrapy.pqueues import ScrapyPriorityQueue
from scrapy.statscollectors import StatsCollector
logger = logging.getLogger(__name__)
@ -71,7 +73,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
"""
return cls()
def open(self, spider: Spider) -> Optional[Deferred]:
def open(self, spider: Spider) -> Optional[Deferred[None]]:
"""
Called when the spider is opened by the engine. It receives the spider
instance as argument and it's useful to execute initialization code.
@ -81,7 +83,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
"""
pass
def close(self, reason: str) -> Optional[Deferred]:
def close(self, reason: str) -> Optional[Deferred[None]]:
"""
Called when the spider is closed by the engine. It receives the reason why the crawl
finished as argument and it's useful to execute cleaning code.
@ -216,7 +218,7 @@ class Scheduler(BaseScheduler):
def has_pending_requests(self) -> bool:
return len(self) > 0
def open(self, spider: Spider) -> Optional[Deferred]:
def open(self, spider: Spider) -> Optional[Deferred[None]]:
"""
(1) initialize the memory queue
(2) initialize the disk queue if the ``jobdir`` attribute is a valid directory
@ -227,7 +229,7 @@ class Scheduler(BaseScheduler):
self.dqs: Optional[ScrapyPriorityQueue] = self._dq() if self.dqdir else None
return self.df.open()
def close(self, reason: str) -> Optional[Deferred]:
def close(self, reason: str) -> Optional[Deferred[None]]:
"""
(1) dump pending requests to disk if there is a disk queue
(2) return the result of the dupefilter's ``close`` method
@ -362,13 +364,13 @@ class Scheduler(BaseScheduler):
return str(dqdir)
return None
def _read_dqs_state(self, dqdir: str) -> list:
def _read_dqs_state(self, dqdir: str) -> List[int]:
path = Path(dqdir, "active.json")
if not path.exists():
return []
with path.open(encoding="utf-8") as f:
return cast(list, json.load(f))
return cast(List[int], json.load(f))
def _write_dqs_state(self, dqdir: str, state: list) -> None:
def _write_dqs_state(self, dqdir: str, state: List[int]) -> None:
with Path(dqdir, "active.json").open("w", encoding="utf-8") as f:
json.dump(state, f)

67
scrapy/core/scraper.py
View File

@ -12,10 +12,13 @@ from typing import (
Deque,
Generator,
Iterable,
Iterator,
List,
Optional,
Set,
Tuple,
Type,
TypeVar,
Union,
cast,
)
@ -47,12 +50,18 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
_ParallelResult = List[Tuple[bool, Iterator[Any]]]
if TYPE_CHECKING:
# parameterized Deferreds require Twisted 21.7.0
_HandleOutputDeferred = Deferred[Union[_ParallelResult, None]]
QueueTuple = Tuple[Union[Response, Failure], Request, _HandleOutputDeferred]
class Slot:
"""Scraper slot (one per running spider)"""
@ -64,12 +73,12 @@ class Slot:
self.active: Set[Request] = set()
self.active_size: int = 0
self.itemproc_size: int = 0
self.closing: Optional[Deferred] = None
self.closing: Optional[Deferred[Spider]] = None
def add_response_request(
self, result: Union[Response, Failure], request: Request
) -> Deferred:
deferred: Deferred = Deferred()
) -> _HandleOutputDeferred:
deferred: _HandleOutputDeferred = Deferred()
self.queue.append((result, request, deferred))
if isinstance(result, Response):
self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE)
@ -115,12 +124,12 @@ class Scraper:
self.logformatter: LogFormatter = crawler.logformatter
@inlineCallbacks
def open_spider(self, spider: Spider) -> Generator[Deferred, Any, None]:
def open_spider(self, spider: Spider) -> Generator[Deferred[Any], Any, None]:
"""Open the given spider for scraping and allocate resources for it"""
self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE"))
yield self.itemproc.open_spider(spider)
def close_spider(self, spider: Spider) -> Deferred:
def close_spider(self, spider: Spider) -> Deferred[Spider]:
"""Close a spider being scraped and release its resources"""
if self.slot is None:
raise RuntimeError("Scraper slot not assigned")
@ -140,12 +149,12 @@ class Scraper:
def enqueue_scrape(
self, result: Union[Response, Failure], request: Request, spider: Spider
) -> Deferred:
) -> _HandleOutputDeferred:
if self.slot is None:
raise RuntimeError("Scraper slot not assigned")
dfd = self.slot.add_response_request(result, request)
def finish_scraping(_: Any) -> Any:
def finish_scraping(_: _T) -> _T:
assert self.slot is not None
self.slot.finish_response(result, request)
self._check_if_closing(spider)
@ -172,7 +181,7 @@ class Scraper:
def _scrape(
self, result: Union[Response, Failure], request: Request, spider: Spider
) -> Deferred:
) -> _HandleOutputDeferred:
"""
Handle the downloaded response or failure through the spider callback/errback
"""
@ -180,32 +189,35 @@ class Scraper:
raise TypeError(
f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}"
)
dfd = self._scrape2(
dfd: Deferred[Union[Iterable[Any], AsyncIterable[Any]]] = self._scrape2(
result, request, spider
) # returns spider's processed output
dfd.addErrback(self.handle_spider_error, request, result, spider)
dfd.addCallback(
dfd2: _HandleOutputDeferred = dfd.addCallback(
self.handle_spider_output, request, cast(Response, result), spider
)
return dfd
return dfd2
def _scrape2(
self, result: Union[Response, Failure], request: Request, spider: Spider
) -> Deferred:
) -> Deferred[Union[Iterable[Any], AsyncIterable[Any]]]:
"""
Handle the different cases of request's result been a Response or a Failure
"""
if isinstance(result, Response):
return self.spidermw.scrape_response(
# Deferreds are invariant so Mutable*Chain isn't matched to *Iterable
return self.spidermw.scrape_response( # type: ignore[return-value]
self.call_spider, result, request, spider
)
# else result is a Failure
dfd = self.call_spider(result, request, spider)
return dfd.addErrback(self._log_download_errors, result, request, spider)
dfd.addErrback(self._log_download_errors, result, request, spider)
return dfd
def call_spider(
self, result: Union[Response, Failure], request: Request, spider: Spider
) -> Deferred:
) -> Deferred[Union[Iterable[Any], AsyncIterable[Any]]]:
dfd: Deferred[Any]
if isinstance(result, Response):
if getattr(result, "request", None) is None:
result.request = request
@ -223,7 +235,10 @@ class Scraper:
if request.errback:
warn_on_generator_with_return_value(spider, request.errback)
dfd.addErrback(request.errback)
return dfd.addCallback(iterate_spider_output)
dfd2: Deferred[Union[Iterable[Any], AsyncIterable[Any]]] = dfd.addCallback(
iterate_spider_output
)
return dfd2
def handle_spider_error(
self,
@ -256,14 +271,15 @@ class Scraper:
def handle_spider_output(
self,
result: Union[Iterable, AsyncIterable],
result: Union[Iterable[_T], AsyncIterable[_T]],
request: Request,
response: Response,
spider: Spider,
) -> Deferred:
) -> _HandleOutputDeferred:
if not result:
return defer_succeed(None)
it: Union[Iterable, AsyncIterable]
it: Union[Iterable[_T], AsyncIterable[_T]]
dfd: Deferred[_ParallelResult]
if isinstance(result, AsyncIterable):
it = aiter_errback(
result, self.handle_spider_error, request, response, spider
@ -288,11 +304,12 @@ class Scraper:
response,
spider,
)
return dfd
# returning Deferred[_ParallelResult] instead of Deferred[Union[_ParallelResult, None]]
return dfd # type: ignore[return-value]
def _process_spidermw_output(
self, output: Any, request: Request, response: Response, spider: Spider
) -> Optional[Deferred]:
) -> Optional[Deferred[Any]]:
"""Process each Request/Item (given in the output parameter) returned
from the given spider
"""
@ -357,7 +374,7 @@ class Scraper:
def _itemproc_finished(
self, output: Any, item: Any, response: Response, spider: Spider
) -> Deferred:
) -> Deferred[Any]:
"""ItemProcessor finished for the given ``item`` and returned ``output``"""
assert self.slot is not None # typing
self.slot.itemproc_size -= 1

101
scrapy/core/spidermw.py
View File

@ -4,12 +4,14 @@ Spider Middleware manager
See documentation in docs/topics/spider-middleware.rst
"""
from __future__ import annotations
import logging
from inspect import isasyncgenfunction, iscoroutine
from itertools import islice
from typing import (
TYPE_CHECKING,
Any,
AsyncGenerator,
AsyncIterable,
Callable,
Generator,
@ -17,6 +19,7 @@ from typing import (
List,
Optional,
Tuple,
TypeVar,
Union,
cast,
)
@ -28,7 +31,6 @@ from scrapy import Request, Spider
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Response
from scrapy.middleware import MiddlewareManager
from scrapy.settings import BaseSettings
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import (
@ -39,10 +41,17 @@ from scrapy.utils.defer import (
)
from scrapy.utils.python import MutableAsyncChain, MutableChain
if TYPE_CHECKING:
from scrapy.settings import BaseSettings
logger = logging.getLogger(__name__)
ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
_T = TypeVar("_T")
ScrapeFunc = Callable[
[Union[Response, Failure], Request, Spider], Union[Iterable[_T], AsyncIterable[_T]]
]
def _isiterable(o: Any) -> bool:
@ -77,7 +86,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
response: Response,
request: Request,
spider: Spider,
) -> Any:
) -> Union[Iterable[_T], AsyncIterable[_T]]:
for method in self.methods["process_spider_input"]:
method = cast(Callable, method)
try:
@ -98,31 +107,39 @@ class SpiderMiddlewareManager(MiddlewareManager):
self,
response: Response,
spider: Spider,
iterable: Union[Iterable, AsyncIterable],
iterable: Union[Iterable[_T], AsyncIterable[_T]],
exception_processor_index: int,
recover_to: Union[MutableChain, MutableAsyncChain],
) -> Union[Generator, AsyncGenerator]:
def process_sync(iterable: Iterable) -> Generator:
recover_to: Union[MutableChain[_T], MutableAsyncChain[_T]],
) -> Union[Iterable[_T], AsyncIterable[_T]]:
def process_sync(iterable: Iterable[_T]) -> Iterable[_T]:
try:
yield from iterable
except Exception as ex:
exception_result = self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index
exception_result = cast(
Union[Failure, MutableChain[_T]],
self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index
),
)
if isinstance(exception_result, Failure):
raise
assert isinstance(recover_to, MutableChain)
recover_to.extend(exception_result)
async def process_async(iterable: AsyncIterable) -> AsyncGenerator:
async def process_async(iterable: AsyncIterable[_T]) -> AsyncIterable[_T]:
try:
async for r in iterable:
yield r
except Exception as ex:
exception_result = self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index
exception_result = cast(
Union[Failure, MutableAsyncChain[_T]],
self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index
),
)
if isinstance(exception_result, Failure):
raise
assert isinstance(recover_to, MutableAsyncChain)
recover_to.extend(exception_result)
if isinstance(iterable, AsyncIterable):
@ -135,7 +152,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
spider: Spider,
_failure: Failure,
start_index: int = 0,
) -> Union[Failure, MutableChain]:
) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:
exception = _failure.value
# don't handle _InvalidOutput exception
if isinstance(exception, _InvalidOutput):
@ -151,14 +168,18 @@ class SpiderMiddlewareManager(MiddlewareManager):
if _isiterable(result):
# stop exception handling by handing control over to the
# process_spider_output chain if an iterable has been returned
dfd: Deferred = self._process_spider_output(
response, spider, result, method_index + 1
dfd: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = (
self._process_spider_output(
response, spider, result, method_index + 1
)
)
# _process_spider_output() returns a Deferred only because of downgrading so this can be
# simplified when downgrading is removed.
if dfd.called:
# the result is available immediately if _process_spider_output didn't do downgrading
return cast(MutableChain, dfd.result)
return cast(
Union[MutableChain[_T], MutableAsyncChain[_T]], dfd.result
)
# we forbid waiting here because otherwise we would need to return a deferred from
# _process_spider_exception too, which complicates the architecture
msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
@ -181,12 +202,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
self,
response: Response,
spider: Spider,
result: Union[Iterable, AsyncIterable],
result: Union[Iterable[_T], AsyncIterable[_T]],
start_index: int = 0,
) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]:
) -> Generator[Deferred[Any], Any, Union[MutableChain[_T], MutableAsyncChain[_T]]]:
# items in this iterable do not need to go through the process_spider_output
# chain, they went through it already from the process_spider_exception method
recovered: Union[MutableChain, MutableAsyncChain]
recovered: Union[MutableChain[_T], MutableAsyncChain[_T]]
last_result_is_async = isinstance(result, AsyncIterable)
if last_result_is_async:
recovered = MutableAsyncChain()
@ -237,7 +258,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
# might fail directly if the output value is not a generator
result = method(response=response, result=result, spider=spider)
except Exception as ex:
exception_result = self._process_spider_exception(
exception_result: Union[
Failure, MutableChain[_T], MutableAsyncChain[_T]
] = self._process_spider_exception(
response, spider, Failure(ex), method_index + 1
)
if isinstance(exception_result, Failure):
@ -267,16 +290,22 @@ class SpiderMiddlewareManager(MiddlewareManager):
return MutableChain(result, recovered) # type: ignore[arg-type]
async def _process_callback_output(
self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
) -> Union[MutableChain, MutableAsyncChain]:
recovered: Union[MutableChain, MutableAsyncChain]
self,
response: Response,
spider: Spider,
result: Union[Iterable[_T], AsyncIterable[_T]],
) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:
recovered: Union[MutableChain[_T], MutableAsyncChain[_T]]
if isinstance(result, AsyncIterable):
recovered = MutableAsyncChain()
else:
recovered = MutableChain()
result = self._evaluate_iterable(response, spider, result, 0, recovered)
result = await maybe_deferred_to_future(
self._process_spider_output(response, spider, result)
cast(
"Deferred[Union[Iterable[_T], AsyncIterable[_T]]]",
self._process_spider_output(response, spider, result),
)
)
if isinstance(result, AsyncIterable):
return MutableAsyncChain(result, recovered)
@ -291,25 +320,29 @@ class SpiderMiddlewareManager(MiddlewareManager):
response: Response,
request: Request,
spider: Spider,
) -> Deferred:
) -> Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]]:
async def process_callback_output(
result: Union[Iterable, AsyncIterable]
) -> Union[MutableChain, MutableAsyncChain]:
result: Union[Iterable[_T], AsyncIterable[_T]]
) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:
return await self._process_callback_output(response, spider, result)
def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
def process_spider_exception(
_failure: Failure,
) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:
return self._process_spider_exception(response, spider, _failure)
dfd = mustbe_deferred(
dfd: Deferred[Union[Iterable[_T], AsyncIterable[_T]]] = mustbe_deferred(
self._process_spider_input, scrape_func, response, request, spider
)
dfd.addCallback(deferred_f_from_coro_f(process_callback_output))
dfd.addErrback(process_spider_exception)
return dfd
dfd2: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = (
dfd.addCallback(deferred_f_from_coro_f(process_callback_output))
)
dfd2.addErrback(process_spider_exception)
return dfd2
def process_start_requests(
self, start_requests: Iterable[Request], spider: Spider
) -> Deferred:
) -> Deferred[Iterable[Request]]:
return self._process_chain("process_start_requests", start_requests, spider)
# This method is only needed until _async compatibility methods are removed.

42
scrapy/crawler.py
View File

@ -4,7 +4,18 @@ import logging
import pprint
import signal
import warnings
from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union, cast
from typing import (
TYPE_CHECKING,
Any,
Dict,
Generator,
Optional,
Set,
Type,
TypeVar,
Union,
cast,
)
from twisted.internet.defer import (
Deferred,
@ -12,13 +23,6 @@ from twisted.internet.defer import (
inlineCallbacks,
maybeDeferred,
)
try:
# zope >= 5.0 only supports MultipleInvalid
from zope.interface.exceptions import MultipleInvalid
except ImportError:
MultipleInvalid = None
from zope.interface.verify import verifyClass
from scrapy import Spider, signals
@ -54,6 +58,8 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
class Crawler:
def __init__(
@ -129,6 +135,8 @@ class Crawler:
if is_asyncio_reactor_installed() and event_loop:
verify_installed_asyncio_event_loop(event_loop)
log_reactor_info()
self.extensions = ExtensionManager.from_crawler(self)
self.settings.freeze()
@ -138,7 +146,7 @@ class Crawler:
)
@inlineCallbacks
def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:
def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred[Any], Any, None]:
if self.crawling:
raise RuntimeError("Crawling already taking place")
if self._started:
@ -170,7 +178,7 @@ class Crawler:
return ExecutionEngine(self, lambda _: self.stop())
@inlineCallbacks
def stop(self) -> Generator[Deferred, Any, None]:
def stop(self) -> Generator[Deferred[Any], Any, None]:
"""Starts a graceful stop of the crawler and returns a deferred that is
fired when the crawler is stopped."""
if self.crawling:
@ -254,7 +262,7 @@ class CrawlerRunner:
self.settings = settings
self.spider_loader = self._get_spider_loader(settings)
self._crawlers: Set[Crawler] = set()
self._active: Set[Deferred] = set()
self._active: Set[Deferred[None]] = set()
self.bootstrap_failed = False
def crawl(
@ -262,7 +270,7 @@ class CrawlerRunner:
crawler_or_spidercls: Union[Type[Spider], str, Crawler],
*args: Any,
**kwargs: Any,
) -> Deferred:
) -> Deferred[None]:
"""
Run a crawler with the provided arguments.
@ -292,12 +300,12 @@ class CrawlerRunner:
crawler = self.create_crawler(crawler_or_spidercls)
return self._crawl(crawler, *args, **kwargs)
def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred:
def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred[None]:
self.crawlers.add(crawler)
d = crawler.crawl(*args, **kwargs)
self._active.add(d)
def _done(result: Any) -> Any:
def _done(result: _T) -> _T:
self.crawlers.discard(crawler)
self._active.discard(d)
self.bootstrap_failed |= not getattr(crawler, "spider", None)
@ -333,7 +341,7 @@ class CrawlerRunner:
# temporary cast until self.spider_loader is typed
return Crawler(cast(Type[Spider], spidercls), self.settings)
def stop(self) -> Deferred:
def stop(self) -> Deferred[Any]:
"""
Stops simultaneously all the crawling jobs taking place.
@ -342,7 +350,7 @@ class CrawlerRunner:
return DeferredList([c.stop() for c in list(self.crawlers)])
@inlineCallbacks
def join(self) -> Generator[Deferred, Any, None]:
def join(self) -> Generator[Deferred[Any], Any, None]:
"""
join()
@ -458,7 +466,7 @@ class CrawlerProcess(CrawlerRunner):
)
reactor.run(installSignalHandlers=install_signal_handlers) # blocking call
def _graceful_stop_reactor(self) -> Deferred:
def _graceful_stop_reactor(self) -> Deferred[Any]:
d = self.stop()
d.addBoth(self._stop_reactor)
return d

8
scrapy/downloadermiddlewares/ajaxcrawl.py
View File

@ -6,16 +6,18 @@ from typing import TYPE_CHECKING, Union
from w3lib import html
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import HtmlResponse, Response
from scrapy.settings import BaseSettings
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
logger = logging.getLogger(__name__)

33
scrapy/downloadermiddlewares/cookies.py
View File

@ -2,22 +2,10 @@ from __future__ import annotations
import logging
from collections import defaultdict
from http.cookiejar import Cookie
from typing import (
TYPE_CHECKING,
Any,
DefaultDict,
Dict,
Iterable,
Optional,
Sequence,
Union,
)
from typing import TYPE_CHECKING, Any, DefaultDict, Iterable, Optional, Sequence, Union
from tldextract import TLDExtract
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.http.cookies import CookieJar
@ -25,9 +13,15 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
from http.cookiejar import Cookie
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.http.request import VerboseCookie
logger = logging.getLogger(__name__)
@ -128,7 +122,7 @@ class CookiesMiddleware:
msg = f"Received cookies from: {response}\n{cookies}"
logger.debug(msg, extra={"spider": spider})
def _format_cookie(self, cookie: Dict[str, Any], request: Request) -> Optional[str]:
def _format_cookie(self, cookie: VerboseCookie, request: Request) -> Optional[str]:
"""
Given a dict consisting of cookie components, return its string representation.
Decode from bytes if necessary.
@ -142,18 +136,19 @@ class CookiesMiddleware:
logger.warning(msg)
return None
continue
if isinstance(cookie[key], (bool, float, int, str)):
decoded[key] = str(cookie[key])
# https://github.com/python/mypy/issues/7178, https://github.com/python/mypy/issues/9168
if isinstance(cookie[key], (bool, float, int, str)): # type: ignore[literal-required]
decoded[key] = str(cookie[key]) # type: ignore[literal-required]
else:
try:
decoded[key] = cookie[key].decode("utf8")
decoded[key] = cookie[key].decode("utf8") # type: ignore[literal-required]
except UnicodeDecodeError:
logger.warning(
"Non UTF-8 encoded cookie found in request %s: %s",
request,
cookie,
)
decoded[key] = cookie[key].decode("latin1", errors="replace")
decoded[key] = cookie[key].decode("latin1", errors="replace") # type: ignore[literal-required]
for flag in ("secure",):
value = cookie.get(flag, _UNSET)
if value is _UNSET or not value:
@ -174,7 +169,7 @@ class CookiesMiddleware:
"""
if not request.cookies:
return []
cookies: Iterable[Dict[str, Any]]
cookies: Iterable[VerboseCookie]
if isinstance(request.cookies, dict):
cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items())
else:

7
scrapy/downloadermiddlewares/defaultheaders.py
View File

@ -8,15 +8,16 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Iterable, Tuple, Union
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.http import Response
class DefaultHeadersMiddleware:
def __init__(self, headers: Iterable[Tuple[str, str]]):

5
scrapy/downloadermiddlewares/downloadtimeout.py
View File

@ -9,13 +9,14 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Union
from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
from scrapy.http import Response
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.http import Response
class DownloadTimeoutMiddleware:
def __init__(self, timeout: float = 180):

5
scrapy/downloadermiddlewares/httpauth.py
View File

@ -11,14 +11,15 @@ from typing import TYPE_CHECKING, Union
from w3lib.http import basic_auth_header
from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.utils.url import url_is_from_any_domain
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.http import Response
class HttpAuthMiddleware:
"""Set Basic HTTP Authorization header

13
scrapy/downloadermiddlewares/httpcache.py
View File

@ -16,19 +16,20 @@ from twisted.internet.error import (
from twisted.web.client import ResponseFailed
from scrapy import signals
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http.request import Request
from scrapy.http.response import Response
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.statscollectors import StatsCollector
from scrapy.utils.misc import load_object
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.http.request import Request
from scrapy.http.response import Response
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.statscollectors import StatsCollector
class HttpCacheMiddleware:
DOWNLOAD_EXCEPTIONS = (

24
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -3,14 +3,12 @@ from __future__ import annotations
import warnings
from itertools import chain
from logging import getLogger
from typing import TYPE_CHECKING, List, Optional, Union
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.statscollectors import StatsCollector
from scrapy.utils._compression import (
_DecompressionMaxSizeExceeded,
_inflate,
@ -24,6 +22,10 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.statscollectors import StatsCollector
logger = getLogger(__name__)
ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]
@ -138,31 +140,35 @@ class HttpCompressionMiddleware:
respcls = responsetypes.from_args(
headers=response.headers, url=response.url, body=decoded_body
)
kwargs = {"cls": respcls, "body": decoded_body}
kwargs: Dict[str, Any] = {"body": decoded_body}
if issubclass(respcls, TextResponse):
# force recalculating the encoding until we make sure the
# responsetypes guessing is reliable
kwargs["encoding"] = None
response = response.replace(**kwargs)
response = response.replace(cls=respcls, **kwargs)
if not content_encoding:
del response.headers["Content-Encoding"]
return response
def _handle_encoding(self, body, content_encoding, max_size):
def _handle_encoding(
self, body: bytes, content_encoding: List[bytes], max_size: int
) -> Tuple[bytes, List[bytes]]:
to_decode, to_keep = self._split_encodings(content_encoding)
for encoding in to_decode:
body = self._decode(body, encoding, max_size)
return body, to_keep
def _split_encodings(self, content_encoding):
to_keep = [
def _split_encodings(
self, content_encoding: List[bytes]
) -> Tuple[List[bytes], List[bytes]]:
to_keep: List[bytes] = [
encoding.strip().lower()
for encoding in chain.from_iterable(
encodings.split(b",") for encodings in content_encoding
)
]
to_decode = []
to_decode: List[bytes] = []
while to_keep:
encoding = to_keep.pop()
if encoding not in ACCEPTED_ENCODINGS:

7
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -9,10 +9,7 @@ from urllib.request import ( # type: ignore[attr-defined]
proxy_bypass,
)
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes
@ -20,6 +17,10 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.http import Response
class HttpProxyMiddleware:
def __init__(self, auth_encoding: Optional[str] = "latin-1"):

32
scrapy/downloadermiddlewares/offsite.py
View File

@ -1,33 +1,45 @@
from __future__ import annotations
import logging
import re
import warnings
from typing import TYPE_CHECKING, Set
from scrapy import signals
from scrapy import Request, Spider, signals
from scrapy.exceptions import IgnoreRequest
from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.statscollectors import StatsCollector
logger = logging.getLogger(__name__)
class OffsiteMiddleware:
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
assert crawler.stats
o = cls(crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled)
return o
def __init__(self, stats):
def __init__(self, stats: StatsCollector):
self.stats = stats
self.domains_seen = set()
self.domains_seen: Set[str] = set()
def spider_opened(self, spider):
self.host_regex = self.get_host_regex(spider)
def spider_opened(self, spider: Spider) -> None:
self.host_regex: re.Pattern[str] = self.get_host_regex(spider)
def request_scheduled(self, request, spider):
def request_scheduled(self, request: Request, spider: Spider) -> None:
self.process_request(request, spider)
def process_request(self, request, spider):
def process_request(self, request: Request, spider: Spider) -> None:
if request.dont_filter or self.should_follow(request, spider):
return None
domain = urlparse_cached(request).hostname
@ -42,13 +54,13 @@ class OffsiteMiddleware:
self.stats.inc_value("offsite/filtered", spider=spider)
raise IgnoreRequest
def should_follow(self, request, spider):
def should_follow(self, request: Request, spider: Spider) -> bool:
regex = self.host_regex
# hostname can be None for wrong urls (like javascript links)
host = urlparse_cached(request).hostname or ""
return bool(regex.search(host))
def get_host_regex(self, spider):
def get_host_regex(self, spider: Spider) -> re.Pattern[str]:
"""Override this method to implement a different offsite policy"""
allowed_domains = getattr(spider, "allowed_domains", None)
if not allowed_domains:

9
scrapy/downloadermiddlewares/redirect.py
View File

@ -6,11 +6,8 @@ from urllib.parse import urljoin
from w3lib.url import safe_url_string
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import HtmlResponse, Response
from scrapy.settings import BaseSettings
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.response import get_meta_refresh
@ -18,6 +15,11 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
logger = logging.getLogger(__name__)
@ -27,6 +29,7 @@ def _build_redirect_request(
redirect_request = source_request.replace(
url=url,
**kwargs,
cls=None,
cookies=None,
)
if "_scheme_proxy" in redirect_request.meta:

14
scrapy/downloadermiddlewares/retry.py
View File

@ -16,12 +16,8 @@ import warnings
from logging import Logger, getLogger
from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Response
from scrapy.http.request import Request
from scrapy.settings import BaseSettings, Settings
from scrapy.spiders import Spider
from scrapy.utils.misc import load_object
from scrapy.utils.python import global_object_name
from scrapy.utils.response import response_status_message
@ -30,6 +26,12 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.http.request import Request
from scrapy.spiders import Spider
retry_logger = getLogger(__name__)
@ -147,9 +149,7 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
if not settings.getbool("RETRY_ENABLED"):
raise NotConfigured
self.max_retry_times = settings.getint("RETRY_TIMES")
self.retry_http_codes = set(
int(x) for x in settings.getlist("RETRY_HTTP_CODES")
)
self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")}
self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
try:

36
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -7,28 +7,32 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
from typing import TYPE_CHECKING, Dict, Optional, TypeVar, Union
from twisted.internet.defer import Deferred, maybeDeferred
from twisted.python.failure import Failure
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.robotstxt import RobotParser
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import load_object
if TYPE_CHECKING:
from twisted.python.failure import Failure
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.robotstxt import RobotParser
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
class RobotsTxtMiddleware:
DOWNLOAD_PRIORITY: int = 1000
@ -41,7 +45,9 @@ class RobotsTxtMiddleware:
"ROBOTSTXT_USER_AGENT", None
)
self.crawler: Crawler = crawler
self._parsers: Dict[str, Union[RobotParser, Deferred, None]] = {}
self._parsers: Dict[
str, Union[RobotParser, Deferred[Optional[RobotParser]], None]
] = {}
self._parserimpl: RobotParser = load_object(
crawler.settings.get("ROBOTSTXT_PARSER")
)
@ -53,14 +59,18 @@ class RobotsTxtMiddleware:
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler)
def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]:
def process_request(
self, request: Request, spider: Spider
) -> Optional[Deferred[None]]:
if request.meta.get("dont_obey_robotstxt"):
return None
if request.url.startswith("data:") or request.url.startswith("file:"):
return None
d: Deferred = maybeDeferred(self.robot_parser, request, spider)
d.addCallback(self.process_request_2, request, spider)
return d
d: Deferred[Optional[RobotParser]] = maybeDeferred(
self.robot_parser, request, spider # type: ignore[arg-type]
)
d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider)
return d2
def process_request_2(
self, rp: Optional[RobotParser], request: Request, spider: Spider
@ -84,7 +94,7 @@ class RobotsTxtMiddleware:
def robot_parser(
self, request: Request, spider: Spider
) -> Union[RobotParser, Deferred, None]:
) -> Union[RobotParser, Deferred[Optional[RobotParser]], None]:
url = urlparse_cached(request)
netloc = url.netloc
@ -107,9 +117,9 @@ class RobotsTxtMiddleware:
parser = self._parsers[netloc]
if isinstance(parser, Deferred):
d: Deferred = Deferred()
d: Deferred[Optional[RobotParser]] = Deferred()
def cb(result: Any) -> Any:
def cb(result: Optional[RobotParser]) -> Optional[RobotParser]:
d.callback(result)
return result

15
scrapy/downloadermiddlewares/stats.py
View File

@ -1,14 +1,10 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Dict, Union
from typing import TYPE_CHECKING, Dict, List, Tuple, Union
from twisted.web import http
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.statscollectors import StatsCollector
from scrapy.utils.python import global_object_name, to_bytes
from scrapy.utils.request import request_httprepr
@ -16,8 +12,15 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.statscollectors import StatsCollector
def get_header_size(headers: Dict[str, Union[list, tuple]]) -> int:
def get_header_size(
headers: Dict[str, Union[List[Union[str, bytes]], Tuple[Union[str, bytes], ...]]]
) -> int:
size = 0
for key, value in headers.items():
if isinstance(value, (list, tuple)):

5
scrapy/downloadermiddlewares/useragent.py
View File

@ -5,13 +5,14 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Union
from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
from scrapy.http import Response
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.http import Response
class UserAgentMiddleware:
"""This middleware allows spiders to override the user_agent"""

14
scrapy/dupefilters.py
View File

@ -4,11 +4,6 @@ import logging
from pathlib import Path
from typing import TYPE_CHECKING, Optional, Set
from twisted.internet.defer import Deferred
from scrapy.http.request import Request
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
from scrapy.utils.job import job_dir
from scrapy.utils.request import (
RequestFingerprinter,
@ -17,10 +12,15 @@ from scrapy.utils.request import (
)
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.http.request import Request
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
class BaseDupeFilter:
@ -31,10 +31,10 @@ class BaseDupeFilter:
def request_seen(self, request: Request) -> bool:
return False
def open(self) -> Optional[Deferred]:
def open(self) -> Optional[Deferred[None]]:
pass
def close(self, reason: str) -> Optional[Deferred]:
def close(self, reason: str) -> Optional[Deferred[None]]:
pass
def log(self, request: Request, spider: Spider) -> None:

8
scrapy/extension.py
View File

@ -4,12 +4,16 @@ The Extension Manager
See documentation in docs/topics/extensions.rst
"""
from typing import Any, List
from __future__ import annotations
from typing import TYPE_CHECKING, Any, List
from scrapy.middleware import MiddlewareManager
from scrapy.settings import Settings
from scrapy.utils.conf import build_component_list
if TYPE_CHECKING:
from scrapy.settings import Settings
class ExtensionManager(MiddlewareManager):
component_name = "extension"

27
scrapy/extensions/closespider.py
View File

@ -10,17 +10,19 @@ import logging
from collections import defaultdict
from typing import TYPE_CHECKING, Any, DefaultDict, Dict
from twisted.python.failure import Failure
from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
if TYPE_CHECKING:
from twisted.python.failure import Failure
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.http import Response
logger = logging.getLogger(__name__)
@ -34,6 +36,9 @@ class CloseSpider:
"pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"),
"errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"),
"timeout_no_item": crawler.settings.getint("CLOSESPIDER_TIMEOUT_NO_ITEM"),
"pagecount_no_item": crawler.settings.getint(
"CLOSESPIDER_PAGECOUNT_NO_ITEM"
),
}
if not any(self.close_on.values()):
@ -43,11 +48,11 @@ class CloseSpider:
if self.close_on.get("errorcount"):
crawler.signals.connect(self.error_count, signal=signals.spider_error)
if self.close_on.get("pagecount"):
if self.close_on.get("pagecount") or self.close_on.get("pagecount_no_item"):
crawler.signals.connect(self.page_count, signal=signals.response_received)
if self.close_on.get("timeout"):
crawler.signals.connect(self.spider_opened, signal=signals.spider_opened)
if self.close_on.get("itemcount"):
if self.close_on.get("itemcount") or self.close_on.get("pagecount_no_item"):
crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
if self.close_on.get("timeout_no_item"):
self.timeout_no_item: int = self.close_on["timeout_no_item"]
@ -58,6 +63,7 @@ class CloseSpider:
crawler.signals.connect(
self.item_scraped_no_item, signal=signals.item_scraped
)
crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)
@classmethod
@ -72,9 +78,17 @@ class CloseSpider:
def page_count(self, response: Response, request: Request, spider: Spider) -> None:
self.counter["pagecount"] += 1
self.counter["pagecount_since_last_item"] += 1
if self.counter["pagecount"] == self.close_on["pagecount"]:
assert self.crawler.engine
self.crawler.engine.close_spider(spider, "closespider_pagecount")
return
if self.close_on["pagecount_no_item"] and (
self.counter["pagecount_since_last_item"]
>= self.close_on["pagecount_no_item"]
):
assert self.crawler.engine
self.crawler.engine.close_spider(spider, "closespider_pagecount_no_item")
def spider_opened(self, spider: Spider) -> None:
from twisted.internet import reactor
@ -89,6 +103,7 @@ class CloseSpider:
def item_scraped(self, item: Any, spider: Spider) -> None:
self.counter["itemcount"] += 1
self.counter["pagecount_since_last_item"] = 0
if self.counter["itemcount"] == self.close_on["itemcount"]:
assert self.crawler.engine
self.crawler.engine.close_spider(spider, "closespider_itemcount")

5
scrapy/extensions/corestats.py
View File

@ -8,13 +8,14 @@ from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any, Optional
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.statscollectors import StatsCollector
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.statscollectors import StatsCollector
class CoreStats:
def __init__(self, stats: StatsCollector):

11
scrapy/extensions/debug.py
View File

@ -12,17 +12,20 @@ import sys
import threading
import traceback
from pdb import Pdb
from types import FrameType
from typing import TYPE_CHECKING, Optional
from scrapy.crawler import Crawler
from scrapy.utils.engine import format_engine_status
from scrapy.utils.trackref import format_live_refs
if TYPE_CHECKING:
from types import FrameType
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
@ -55,7 +58,7 @@ class StackTraceDump:
)
def _thread_stacks(self) -> str:
id2name = dict((th.ident, th.name) for th in threading.enumerate())
id2name = {th.ident: th.name for th in threading.enumerate()}
dumps = ""
for id_, frame in sys._current_frames().items():
name = id2name.get(id_, "")
@ -74,4 +77,4 @@ class Debugger:
def _enter_debugger(self, signum: int, frame: Optional[FrameType]) -> None:
assert frame
Pdb().set_trace(frame.f_back)
Pdb().set_trace(frame.f_back) # noqa: T100

50
scrapy/extensions/feedexport.py
View File

@ -31,18 +31,15 @@ from typing import (
)
from urllib.parse import unquote, urlparse
from twisted.internet import threads
from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
from twisted.python.failure import Failure
from twisted.internet.threads import deferToThread
from w3lib.url import file_uri_to_path
from zope.interface import Interface, implementer
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.exporters import BaseItemExporter
from scrapy.extensions.postprocessing import PostProcessingManager
from scrapy.settings import BaseSettings, Settings
from scrapy.settings import Settings
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.conf import feed_complete_default_values_from_settings
from scrapy.utils.defer import maybe_deferred_to_future
@ -54,11 +51,14 @@ from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
from _typeshed import OpenBinaryMode
from twisted.python.failure import Failure
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
from scrapy.crawler import Crawler
from scrapy.exporters import BaseItemExporter
from scrapy.settings import BaseSettings
try:
import boto3 # noqa: F401
@ -67,6 +67,9 @@ try:
except ImportError:
IS_BOTO3_AVAILABLE = False
logger = logging.getLogger(__name__)
UriParamsCallableT = Callable[[Dict[str, Any], Spider], Optional[Dict[str, Any]]]
_StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol")
@ -104,7 +107,7 @@ class ItemFilter:
for item_class in feed_options.get("item_classes") or ()
)
else:
self.item_classes = tuple()
self.item_classes = ()
def accepts(self, item: Any) -> bool:
"""
@ -146,7 +149,7 @@ class FeedStorageProtocol(Protocol):
"""Open the storage for the given spider. It must return a file-like
object that will be used for the exporters"""
def store(self, file: IO[bytes]) -> Optional[Deferred]:
def store(self, file: IO[bytes]) -> Optional[Deferred[None]]:
"""Store the given file stream"""
@ -159,8 +162,8 @@ class BlockingFeedStorage:
return NamedTemporaryFile(prefix="feed-", dir=path)
def store(self, file: IO[bytes]) -> Optional[Deferred]:
return threads.deferToThread(self._store_in_thread, file)
def store(self, file: IO[bytes]) -> Optional[Deferred[None]]:
return deferToThread(self._store_in_thread, file)
def _store_in_thread(self, file: IO[bytes]) -> None:
raise NotImplementedError
@ -189,7 +192,7 @@ class StdoutFeedStorage:
def open(self, spider: Spider) -> IO[bytes]:
return self._stdout
def store(self, file: IO[bytes]) -> Optional[Deferred]:
def store(self, file: IO[bytes]) -> Optional[Deferred[None]]:
pass
@ -208,7 +211,7 @@ class FileFeedStorage:
dirname.mkdir(parents=True)
return Path(self.path).open(self.write_mode)
def store(self, file: IO[bytes]) -> Optional[Deferred]:
def store(self, file: IO[bytes]) -> Optional[Deferred[None]]:
file.close()
return None
@ -238,13 +241,16 @@ class S3FeedStorage(BlockingFeedStorage):
self.acl: Optional[str] = acl
self.endpoint_url: Optional[str] = endpoint_url
self.region_name: Optional[str] = region_name
# It can be either botocore.client.BaseClient or mypy_boto3_s3.S3Client,
# there seems to be no good way to infer it statically.
self.s3_client: Any
if IS_BOTO3_AVAILABLE:
import boto3.session
session = boto3.session.Session()
boto3_session = boto3.session.Session()
self.s3_client = session.client(
self.s3_client = boto3_session.client(
"s3",
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
@ -261,9 +267,9 @@ class S3FeedStorage(BlockingFeedStorage):
import botocore.session
session = botocore.session.get_session()
botocore_session = botocore.session.get_session()
self.s3_client = session.create_client(
self.s3_client = botocore_session.create_client(
"s3",
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
@ -477,7 +483,7 @@ _FeedSlot = create_deprecated_class(
class FeedExporter:
_pending_deferreds: List[Deferred] = []
_pending_deferreds: List[Deferred[None]] = []
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
@ -564,7 +570,7 @@ class FeedExporter:
self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed)
)
def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred]:
def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred[None]]:
def get_file(slot_: FeedSlot) -> IO[bytes]:
assert slot_.file
if isinstance(slot_.file, PostProcessingManager):
@ -584,7 +590,7 @@ class FeedExporter:
return None
logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
d: Deferred = maybeDeferred(slot.storage.store, get_file(slot))
d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[arg-type]
d.addCallback(
self._handle_store_success, logmsg, spider, type(slot.storage).__name__
@ -615,7 +621,7 @@ class FeedExporter:
self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
def _handle_store_success(
self, f: Failure, logmsg: str, spider: Spider, slot_type: str
self, result: Any, logmsg: str, spider: Spider, slot_type: str
) -> None:
logger.info("Stored %s", logmsg, extra={"spider": spider})
assert self.crawler.stats
@ -691,7 +697,9 @@ class FeedExporter:
self.slots = slots
def _load_components(self, setting_prefix: str) -> Dict[str, Any]:
conf = without_none_values(self.settings.getwithbase(setting_prefix))
conf = without_none_values(
cast(Dict[str, str], self.settings.getwithbase(setting_prefix))
)
d = {}
for k, v in conf.items():
try:

13
scrapy/extensions/httpcache.py
View File

@ -1,3 +1,5 @@
from __future__ import annotations
import gzip
import logging
import os
@ -13,10 +15,7 @@ from weakref import WeakKeyDictionary
from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
from scrapy.http import Headers, Response
from scrapy.http.request import Request
from scrapy.responsetypes import responsetypes
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.project import data_path
from scrapy.utils.python import to_bytes, to_unicode
@ -26,6 +25,10 @@ if TYPE_CHECKING:
# typing.Concatenate requires Python 3.10
from typing_extensions import Concatenate
from scrapy.http.request import Request
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
logger = logging.getLogger(__name__)
@ -315,7 +318,9 @@ class FilesystemCacheStorage:
self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS")
self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP")
# https://github.com/python/mypy/issues/10740
self._open: Callable[Concatenate[Union[str, os.PathLike], str, ...], IO] = (
self._open: Callable[
Concatenate[Union[str, os.PathLike], str, ...], IO[bytes]
] = (
gzip.open if self.use_gzip else open # type: ignore[assignment]
)

6
scrapy/extensions/logstats.py
View File

@ -6,14 +6,16 @@ from typing import TYPE_CHECKING, Optional, Tuple, Union
from twisted.internet import task
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.statscollectors import StatsCollector
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.statscollectors import StatsCollector
logger = logging.getLogger(__name__)

5
scrapy/extensions/memdebug.py
View File

@ -10,15 +10,16 @@ import gc
from typing import TYPE_CHECKING
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.statscollectors import StatsCollector
from scrapy.utils.trackref import live_refs
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.statscollectors import StatsCollector
class MemoryDebugger:
def __init__(self, stats: StatsCollector):

4
scrapy/extensions/memusage.py
View File

@ -16,7 +16,6 @@ from typing import TYPE_CHECKING, List
from twisted.internet import task
from scrapy import signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.mail import MailSender
from scrapy.utils.engine import get_engine_status
@ -25,6 +24,9 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)

6
scrapy/extensions/periodic_log.py
View File

@ -8,15 +8,17 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from twisted.internet import task
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.statscollectors import StatsCollector
from scrapy.utils.serialize import ScrapyJSONEncoder
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.statscollectors import StatsCollector
logger = logging.getLogger(__name__)

3
scrapy/extensions/spiderstate.py
View File

@ -5,7 +5,6 @@ from pathlib import Path
from typing import TYPE_CHECKING, Optional
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.utils.job import job_dir
@ -13,6 +12,8 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
class SpiderState:
"""Store and load spider state during a scraping job"""

11
scrapy/extensions/statsmailer.py
View File

@ -8,18 +8,19 @@ from __future__ import annotations
from typing import TYPE_CHECKING, List, Optional
from twisted.internet.defer import Deferred
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.mail import MailSender
from scrapy.statscollectors import StatsCollector
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.statscollectors import StatsCollector
class StatsMailer:
def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender):
@ -38,7 +39,7 @@ class StatsMailer:
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_closed(self, spider: Spider) -> Optional[Deferred]:
def spider_closed(self, spider: Spider) -> Optional[Deferred[None]]:
spider_stats = self.stats.get_stats(spider)
body = "Global stats\n\n"
body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items())

27
scrapy/extensions/telnet.py
View File

@ -10,23 +10,12 @@ import binascii
import logging
import os
import pprint
import traceback
from typing import TYPE_CHECKING, Any, Dict, List
from twisted.internet import protocol
from twisted.internet.tcp import Port
try:
from twisted.conch import manhole, telnet
from twisted.conch.insults import insults
TWISTED_CONCH_AVAILABLE = True
except (ImportError, SyntaxError):
_TWISTED_CONCH_TRACEBACK = traceback.format_exc()
TWISTED_CONCH_AVAILABLE = False
from scrapy import signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.utils.decorators import defers
from scrapy.utils.engine import print_engine_status
@ -34,8 +23,14 @@ from scrapy.utils.reactor import listen_tcp
from scrapy.utils.trackref import print_live_refs
if TYPE_CHECKING:
from twisted.conch import telnet
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
# signal to update telnet variables
@ -47,11 +42,7 @@ class TelnetConsole(protocol.ServerFactory):
def __init__(self, crawler: Crawler):
if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"):
raise NotConfigured
if not TWISTED_CONCH_AVAILABLE:
raise NotConfigured(
"TELNETCONSOLE_ENABLED setting is True but required twisted "
"modules failed to import:\n" + _TWISTED_CONCH_TRACEBACK
)
self.crawler: Crawler = crawler
self.noisy: bool = False
self.portrange: List[int] = [
@ -85,6 +76,10 @@ class TelnetConsole(protocol.ServerFactory):
self.port.stopListening()
def protocol(self) -> telnet.TelnetTransport: # type: ignore[override]
# these import twisted.internet.reactor
from twisted.conch import manhole, telnet
from twisted.conch.insults import insults
class Portal:
"""An implementation of IPortal"""

8
scrapy/extensions/throttle.py
View File

@ -4,15 +4,17 @@ import logging
from typing import TYPE_CHECKING, Optional, Tuple
from scrapy import Request, Spider, signals
from scrapy.core.downloader import Slot
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.core.downloader import Slot
from scrapy.crawler import Crawler
from scrapy.http import Response
logger = logging.getLogger(__name__)

6
scrapy/http/cookies.py
View File

@ -17,8 +17,6 @@ from typing import (
cast,
)
from scrapy import Request
from scrapy.http import Response
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
@ -26,6 +24,10 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Request
from scrapy.http import Response
# Defined in the http.cookiejar module, but undocumented:
# https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527
IPV4_RE = re.compile(r"\.\d+$", re.ASCII)

3
scrapy/http/headers.py
View File

@ -118,8 +118,7 @@ class Headers(CaselessDict):
]
def to_string(self) -> bytes:
# cast() can be removed if the headers_dict_to_raw() hint is improved
return cast(bytes, headers_dict_to_raw(self))
return headers_dict_to_raw(self)
def to_unicode_dict(self) -> CaseInsensitiveDict:
"""Return headers as a CaseInsensitiveDict with str keys

68
scrapy/http/request/__init__.py
View File

@ -12,7 +12,6 @@ from typing import (
TYPE_CHECKING,
Any,
AnyStr,
Callable,
Dict,
Iterable,
List,
@ -20,8 +19,11 @@ from typing import (
NoReturn,
Optional,
Tuple,
Type,
TypedDict,
TypeVar,
Union,
cast,
overload,
)
from w3lib.url import safe_url_string
@ -34,8 +36,31 @@ from scrapy.utils.trackref import object_ref
from scrapy.utils.url import escape_ajax
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from collections.abc import Callable
from twisted.python.failure import Failure
# typing.Concatenate requires Python 3.10
# typing.NotRequired and typing.Self require Python 3.11
from typing_extensions import Concatenate, NotRequired, Self
from scrapy.http import Response
CallbackT = Callable[Concatenate[Response, ...], Any]
class VerboseCookie(TypedDict):
name: str
value: str
domain: NotRequired[str]
path: NotRequired[str]
secure: NotRequired[bool]
CookiesT = Union[Dict[str, str], List[VerboseCookie]]
RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn:
@ -93,16 +118,16 @@ class Request(object_ref):
def __init__(
self,
url: str,
callback: Optional[Callable] = None,
callback: Optional[CallbackT] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
cookies: Optional[Union[dict, List[dict]]] = None,
cookies: Optional[CookiesT] = None,
meta: Optional[Dict[str, Any]] = None,
encoding: str = "utf-8",
priority: int = 0,
dont_filter: bool = False,
errback: Optional[Callable] = None,
errback: Optional[Callable[[Failure], Any]] = None,
flags: Optional[List[str]] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
) -> None:
@ -120,10 +145,10 @@ class Request(object_ref):
)
if not (callable(errback) or errback is None):
raise TypeError(f"errback must be a callable, got {type(errback).__name__}")
self.callback: Optional[Callable] = callback
self.errback: Optional[Callable] = errback
self.callback: Optional[CallbackT] = callback
self.errback: Optional[Callable[[Failure], Any]] = errback
self.cookies: Union[dict, List[dict]] = cookies or {}
self.cookies: CookiesT = cookies or {}
self.headers: Headers = Headers(headers or {}, encoding=encoding)
self.dont_filter: bool = dont_filter
@ -177,15 +202,26 @@ class Request(object_ref):
def __repr__(self) -> str:
return f"<{self.method} {self.url}>"
def copy(self) -> "Request":
def copy(self) -> Self:
return self.replace()
def replace(self, *args: Any, **kwargs: Any) -> "Request":
@overload
def replace(
self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any
) -> RequestTypeVar: ...
@overload
def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ...
def replace(
self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any
) -> Request:
"""Create a new Request with the same attributes except for those given new values"""
for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))
cls = kwargs.pop("cls", self.__class__)
return cast(Request, cls(*args, **kwargs))
if cls is None:
cls = self.__class__
return cls(*args, **kwargs)
@classmethod
def from_curl(
@ -225,7 +261,7 @@ class Request(object_ref):
request_kwargs.update(kwargs)
return cls(**request_kwargs)
def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]:
def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]:
"""Return a dictionary containing the Request's data.
Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.
@ -254,7 +290,7 @@ class Request(object_ref):
return d
def _find_method(obj: Any, func: Callable) -> str:
def _find_method(obj: Any, func: Callable[..., Any]) -> str:
"""Helper function for Request.to_dict"""
# Only instance methods contain ``__func__``
if obj and hasattr(func, "__func__"):

33
scrapy/http/request/form.py
View File

@ -7,7 +7,17 @@ See documentation in docs/topics/request-response.rst
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast
from typing import (
TYPE_CHECKING,
Any,
Dict,
Iterable,
List,
Optional,
Tuple,
Union,
cast,
)
from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
from lxml.html import FormElement # nosec
@ -18,16 +28,18 @@ from lxml.html import TextareaElement # nosec
from w3lib.html import strip_html5_whitespace
from scrapy.http.request import Request
from scrapy.http.response.text import TextResponse
from scrapy.utils.python import is_listlike, to_bytes
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.http.response.text import TextResponse
FormdataKVType = Tuple[str, Union[str, Iterable[str]]]
FormdataType = Optional[Union[dict, List[FormdataKVType]]]
FormdataVType = Union[str, Iterable[str]]
FormdataKVType = Tuple[str, FormdataVType]
FormdataType = Optional[Union[Dict[str, FormdataVType], List[FormdataKVType]]]
class FormRequest(Request):
@ -62,7 +74,7 @@ class FormRequest(Request):
formid: Optional[str] = None,
formnumber: int = 0,
formdata: FormdataType = None,
clickdata: Optional[dict] = None,
clickdata: Optional[Dict[str, Union[str, int]]] = None,
dont_click: bool = False,
formxpath: Optional[str] = None,
formcss: Optional[str] = None,
@ -156,7 +168,7 @@ def _get_inputs(
form: FormElement,
formdata: FormdataType,
dont_click: bool,
clickdata: Optional[dict],
clickdata: Optional[Dict[str, Union[str, int]]],
) -> List[FormdataKVType]:
"""Return a list of key-value pairs for the inputs found in the given form."""
try:
@ -186,10 +198,8 @@ def _get_inputs(
if clickable and clickable[0] not in formdata and not clickable[0] is None:
values.append(clickable)
if isinstance(formdata, dict):
formdata = formdata.items() # type: ignore[assignment]
values.extend((k, v) for k, v in formdata if v is not None)
formdata_items = formdata.items() if isinstance(formdata, dict) else formdata
values.extend((k, v) for k, v in formdata_items if v is not None)
return values
@ -216,7 +226,7 @@ def _select_value(
def _get_clickable(
clickdata: Optional[dict], form: FormElement
clickdata: Optional[Dict[str, Union[str, int]]], form: FormElement
) -> Optional[Tuple[str, str]]:
"""
Returns the clickable element specified in clickdata,
@ -243,6 +253,7 @@ def _get_clickable(
# because that uniquely identifies the element
nr = clickdata.get("nr", None)
if nr is not None:
assert isinstance(nr, int)
try:
el = list(form.inputs)[nr]
except IndexError:

40
scrapy/http/request/json_request.py
View File

@ -5,27 +5,33 @@ This module implements the JsonRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
from __future__ import annotations
import copy
import json
import warnings
from typing import Any, Optional, Tuple
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type, overload
from scrapy.http.request import Request
from scrapy.http.request import Request, RequestTypeVar
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class JsonRequest(Request):
attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",)
def __init__(
self, *args: Any, dumps_kwargs: Optional[dict] = None, **kwargs: Any
self, *args: Any, dumps_kwargs: Optional[Dict[str, Any]] = None, **kwargs: Any
) -> None:
dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {}
dumps_kwargs.setdefault("sort_keys", True)
self._dumps_kwargs = dumps_kwargs
self._dumps_kwargs: Dict[str, Any] = dumps_kwargs
body_passed = kwargs.get("body", None) is not None
data = kwargs.pop("data", None)
data_passed = data is not None
data: Any = kwargs.pop("data", None)
data_passed: bool = data is not None
if body_passed and data_passed:
warnings.warn("Both body and data passed. data will be ignored")
@ -41,21 +47,31 @@ class JsonRequest(Request):
)
@property
def dumps_kwargs(self) -> dict:
def dumps_kwargs(self) -> Dict[str, Any]:
return self._dumps_kwargs
def replace(self, *args: Any, **kwargs: Any) -> Request:
@overload
def replace(
self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any
) -> RequestTypeVar: ...
@overload
def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ...
def replace(
self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any
) -> Request:
body_passed = kwargs.get("body", None) is not None
data = kwargs.pop("data", None)
data_passed = data is not None
data: Any = kwargs.pop("data", None)
data_passed: bool = data is not None
if body_passed and data_passed:
warnings.warn("Both body and data passed. data will be ignored")
elif not body_passed and data_passed:
kwargs["body"] = self._dumps(data)
return super().replace(*args, **kwargs)
return super().replace(*args, cls=cls, **kwargs)
def _dumps(self, data: dict) -> str:
def _dumps(self, data: Any) -> str:
"""Convert to JSON"""
return json.dumps(data, **self._dumps_kwargs)

2
scrapy/http/request/rpc.py
View File

@ -21,7 +21,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps)
class XmlRpcRequest(Request):
def __init__(self, *args: Any, encoding: Optional[str] = None, **kwargs: Any):
if "body" not in kwargs and "params" in kwargs:
kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs)
kw = {k: kwargs.pop(k) for k in DUMPS_ARGS if k in kwargs}
kwargs["body"] = xmlrpclib.dumps(**kw)
# spec defines that requests must use POST method

53
scrapy/http/response/__init__.py
View File

@ -7,26 +7,24 @@ See documentation in docs/topics/request-response.rst
from __future__ import annotations
from ipaddress import IPv4Address, IPv6Address
from typing import (
TYPE_CHECKING,
Any,
AnyStr,
Callable,
Dict,
Generator,
Iterable,
List,
Mapping,
Optional,
Tuple,
Type,
TypeVar,
Union,
cast,
overload,
)
from urllib.parse import urljoin
from twisted.internet.ssl import Certificate
from scrapy.exceptions import NotSupported
from scrapy.http.headers import Headers
from scrapy.http.request import Request
@ -34,9 +32,21 @@ from scrapy.link import Link
from scrapy.utils.trackref import object_ref
if TYPE_CHECKING:
from ipaddress import IPv4Address, IPv6Address
from twisted.internet.ssl import Certificate
from twisted.python.failure import Failure
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.http.request import CallbackT, CookiesT
from scrapy.selector import SelectorList
ResponseTypeVar = TypeVar("ResponseTypeVar", bound="Response")
class Response(object_ref):
"""An object that represents an HTTP response, which is usually
downloaded (by the Downloader) and fed to the Spiders for processing.
@ -133,16 +143,27 @@ class Response(object_ref):
def __repr__(self) -> str:
return f"<{self.status} {self.url}>"
def copy(self) -> Response:
def copy(self) -> Self:
"""Return a copy of this Response"""
return self.replace()
def replace(self, *args: Any, **kwargs: Any) -> Response:
@overload
def replace(
self, *args: Any, cls: Type[ResponseTypeVar], **kwargs: Any
) -> ResponseTypeVar: ...
@overload
def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ...
def replace(
self, *args: Any, cls: Optional[Type[Response]] = None, **kwargs: Any
) -> Response:
"""Create a new Response with the same attributes except for those given new values"""
for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))
cls = kwargs.pop("cls", self.__class__)
return cast(Response, cls(*args, **kwargs))
if cls is None:
cls = self.__class__
return cls(*args, **kwargs)
def urljoin(self, url: str) -> str:
"""Join this Response's url with a possible relative url to form an
@ -177,16 +198,16 @@ class Response(object_ref):
def follow(
self,
url: Union[str, Link],
callback: Optional[Callable] = None,
callback: Optional[CallbackT] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
cookies: Optional[Union[dict, List[dict]]] = None,
cookies: Optional[CookiesT] = None,
meta: Optional[Dict[str, Any]] = None,
encoding: Optional[str] = "utf-8",
priority: int = 0,
dont_filter: bool = False,
errback: Optional[Callable] = None,
errback: Optional[Callable[[Failure], Any]] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
) -> Request:
@ -230,19 +251,19 @@ class Response(object_ref):
def follow_all(
self,
urls: Iterable[Union[str, Link]],
callback: Optional[Callable] = None,
callback: Optional[CallbackT] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
cookies: Optional[Union[dict, List[dict]]] = None,
cookies: Optional[CookiesT] = None,
meta: Optional[Dict[str, Any]] = None,
encoding: Optional[str] = "utf-8",
priority: int = 0,
dont_filter: bool = False,
errback: Optional[Callable] = None,
errback: Optional[Callable[[Failure], Any]] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
) -> Generator[Request, None, None]:
) -> Iterable[Request]:
"""
.. versionadded:: 2.0

20
scrapy/http/response/text.py
View File

@ -15,7 +15,6 @@ from typing import (
AnyStr,
Callable,
Dict,
Generator,
Iterable,
List,
Mapping,
@ -36,15 +35,18 @@ from w3lib.encoding import (
)
from w3lib.html import strip_html5_whitespace
from scrapy.http import Request
from scrapy.http.response import Response
from scrapy.link import Link
from scrapy.utils.python import memoizemethod_noargs, to_unicode
from scrapy.utils.response import get_base_url
if TYPE_CHECKING:
from twisted.python.failure import Failure
from scrapy.http.request import CallbackT, CookiesT, Request
from scrapy.selector import Selector, SelectorList
_NONE = object()
@ -179,16 +181,16 @@ class TextResponse(Response):
def follow(
self,
url: Union[str, Link, parsel.Selector],
callback: Optional[Callable] = None,
callback: Optional[CallbackT] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
cookies: Optional[Union[dict, List[dict]]] = None,
cookies: Optional[CookiesT] = None,
meta: Optional[Dict[str, Any]] = None,
encoding: Optional[str] = None,
priority: int = 0,
dont_filter: bool = False,
errback: Optional[Callable] = None,
errback: Optional[Callable[[Failure], Any]] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
) -> Request:
@ -232,21 +234,21 @@ class TextResponse(Response):
def follow_all(
self,
urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None,
callback: Optional[Callable] = None,
callback: Optional[CallbackT] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
cookies: Optional[Union[dict, List[dict]]] = None,
cookies: Optional[CookiesT] = None,
meta: Optional[Dict[str, Any]] = None,
encoding: Optional[str] = None,
priority: int = 0,
dont_filter: bool = False,
errback: Optional[Callable] = None,
errback: Optional[Callable[[Failure], Any]] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
css: Optional[str] = None,
xpath: Optional[str] = None,
) -> Generator[Request, None, None]:
) -> Iterable[Request]:
"""
A generator that produces :class:`~.Request` instances to follow all
links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s

2
scrapy/item.py
View File

@ -27,7 +27,7 @@ if TYPE_CHECKING:
from typing_extensions import Self
class Field(dict):
class Field(Dict[str, Any]):
"""Container of field metadata"""

13
scrapy/linkextractors/lxmlhtml.py
View File

@ -2,10 +2,13 @@
Link extractor based on lxml.html
"""
from __future__ import annotations
import logging
import operator
from functools import partial
from typing import (
TYPE_CHECKING,
Any,
Callable,
Iterable,
@ -20,13 +23,10 @@ from typing import (
from urllib.parse import urljoin, urlparse
from lxml import etree # nosec
from lxml.html import HtmlElement # nosec
from parsel.csstranslator import HTMLTranslator
from w3lib.html import strip_html5_whitespace
from w3lib.url import canonicalize_url, safe_url_string
from scrapy import Selector
from scrapy.http import TextResponse
from scrapy.link import Link
from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches, re
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
@ -34,6 +34,13 @@ from scrapy.utils.python import unique as unique_list
from scrapy.utils.response import get_base_url
from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain
if TYPE_CHECKING:
from lxml.html import HtmlElement # nosec
from scrapy import Selector
from scrapy.http import TextResponse
logger = logging.getLogger(__name__)
# from lxml/src/lxml/html/__init__.py

16
scrapy/loader/__init__.py
View File

@ -4,11 +4,18 @@ Item Loader
See documentation in docs/topics/loaders.rst
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Optional
import itemloaders
from scrapy.item import Item
from scrapy.selector import Selector
if TYPE_CHECKING:
from scrapy.http import TextResponse
class ItemLoader(itemloaders.ItemLoader):
"""
@ -82,7 +89,14 @@ class ItemLoader(itemloaders.ItemLoader):
default_item_class: type = Item
default_selector_class = Selector
def __init__(self, item=None, selector=None, response=None, parent=None, **context):
def __init__(
self,
item: Any = None,
selector: Optional[Selector] = None,
response: Optional[TextResponse] = None,
parent: Optional[itemloaders.ItemLoader] = None,
**context: Any,
):
if selector is None and response is not None:
try:
selector = self.default_selector_class(response)

27
scrapy/logformatter.py
View File

@ -2,12 +2,13 @@ from __future__ import annotations
import logging
import os
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, TypedDict, Union
from twisted.python.failure import Failure
from scrapy import Request, Spider
from scrapy.http import Response
# working around https://github.com/sphinx-doc/sphinx/issues/10400
from scrapy import Request, Spider # noqa: TC001
from scrapy.http import Response # noqa: TC001
from scrapy.utils.request import referer_str
if TYPE_CHECKING:
@ -26,6 +27,12 @@ DOWNLOADERRORMSG_SHORT = "Error downloading %(request)s"
DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s"
class LogFormatterResult(TypedDict):
level: int
msg: str
args: Union[Dict[str, Any], Tuple[Any, ...]]
class LogFormatter:
"""Class for generating log messages for different actions.
@ -64,7 +71,9 @@ class LogFormatter:
}
"""
def crawled(self, request: Request, response: Response, spider: Spider) -> dict:
def crawled(
self, request: Request, response: Response, spider: Spider
) -> LogFormatterResult:
"""Logs a message when the crawler finds a webpage."""
request_flags = f" {str(request.flags)}" if request.flags else ""
response_flags = f" {str(response.flags)}" if response.flags else ""
@ -84,7 +93,7 @@ class LogFormatter:
def scraped(
self, item: Any, response: Union[Response, Failure], spider: Spider
) -> dict:
) -> LogFormatterResult:
"""Logs a message when an item is scraped by a spider."""
src: Any
if isinstance(response, Failure):
@ -102,7 +111,7 @@ class LogFormatter:
def dropped(
self, item: Any, exception: BaseException, response: Response, spider: Spider
) -> dict:
) -> LogFormatterResult:
"""Logs a message when an item is dropped while it is passing through the item pipeline."""
return {
"level": logging.WARNING,
@ -115,7 +124,7 @@ class LogFormatter:
def item_error(
self, item: Any, exception: BaseException, response: Response, spider: Spider
) -> dict:
) -> LogFormatterResult:
"""Logs a message when an item causes an error while it is passing
through the item pipeline.
@ -135,7 +144,7 @@ class LogFormatter:
request: Request,
response: Union[Response, Failure],
spider: Spider,
) -> dict:
) -> LogFormatterResult:
"""Logs an error message from a spider.
.. versionadded:: 2.0
@ -155,7 +164,7 @@ class LogFormatter:
request: Request,
spider: Spider,
errmsg: Optional[str] = None,
) -> dict:
) -> LogFormatterResult:
"""Logs a download error message from a spider (typically coming from
the engine).

20
scrapy/mail.py
View File

@ -30,20 +30,22 @@ from typing import (
from twisted import version as twisted_version
from twisted.internet import ssl
from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
from twisted.python.versions import Version
from scrapy.settings import BaseSettings
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import to_bytes
if TYPE_CHECKING:
# imports twisted.internet.reactor
from twisted.mail.smtp import ESMTPSenderFactory
from twisted.python.failure import Failure
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.settings import BaseSettings
logger = logging.getLogger(__name__)
@ -97,11 +99,11 @@ class MailSender:
subject: str,
body: str,
cc: Union[str, List[str], None] = None,
attachs: Sequence[Tuple[str, str, IO]] = (),
attachs: Sequence[Tuple[str, str, IO[Any]]] = (),
mimetype: str = "text/plain",
charset: Optional[str] = None,
_callback: Optional[Callable[..., None]] = None,
) -> Optional[Deferred]:
) -> Optional[Deferred[None]]:
from twisted.internet import reactor
msg: MIMEBase
@ -153,7 +155,9 @@ class MailSender:
)
return None
dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8"))
dfd: Deferred[Any] = self._sendmail(
rcpts, msg.as_string().encode(charset or "utf-8")
)
dfd.addCallback(self._sent_ok, to, cc, subject, len(attachs))
dfd.addErrback(self._sent_failed, to, cc, subject, len(attachs))
reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd)
@ -196,11 +200,11 @@ class MailSender:
)
return failure
def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred:
def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred[Any]:
from twisted.internet import reactor
msg_io = BytesIO(msg)
d: Deferred = Deferred()
d: Deferred[Any] = Deferred()
factory = self._create_sender_factory(to_addrs, msg_io, d)
@ -214,7 +218,7 @@ class MailSender:
return d
def _create_sender_factory(
self, to_addrs: List[str], msg: IO, d: Deferred
self, to_addrs: List[str], msg: IO[bytes], d: Deferred[Any]
) -> ESMTPSenderFactory:
from twisted.mail.smtp import ESMTPSenderFactory

37
scrapy/middleware.py
View File

@ -13,27 +13,34 @@ from typing import (
List,
Optional,
Tuple,
TypeVar,
Union,
cast,
)
from twisted.internet.defer import Deferred
from scrapy import Spider
from scrapy.exceptions import NotConfigured
from scrapy.settings import Settings
from scrapy.utils.defer import process_chain, process_parallel
from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from twisted.internet.defer import Deferred
# typing.Concatenate and typing.ParamSpec require Python 3.10
# typing.Self requires Python 3.11
from typing_extensions import Concatenate, ParamSpec, Self
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.settings import Settings
_P = ParamSpec("_P")
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
_T2 = TypeVar("_T2")
class MiddlewareManager:
"""Base class for implementing middleware managers"""
@ -98,16 +105,22 @@ class MiddlewareManager:
if hasattr(mw, "close_spider"):
self.methods["close_spider"].appendleft(mw.close_spider)
def _process_parallel(self, methodname: str, obj: Any, *args: Any) -> Deferred:
methods = cast(Iterable[Callable], self.methods[methodname])
def _process_parallel(
self, methodname: str, obj: _T, *args: Any
) -> Deferred[List[_T2]]:
methods = cast(
"Iterable[Callable[Concatenate[_T, _P], _T2]]", self.methods[methodname]
)
return process_parallel(methods, obj, *args)
def _process_chain(self, methodname: str, obj: Any, *args: Any) -> Deferred:
methods = cast(Iterable[Callable], self.methods[methodname])
def _process_chain(self, methodname: str, obj: _T, *args: Any) -> Deferred[_T]:
methods = cast(
"Iterable[Callable[Concatenate[_T, _P], _T]]", self.methods[methodname]
)
return process_chain(methods, obj, *args)
def open_spider(self, spider: Spider) -> Deferred:
def open_spider(self, spider: Spider) -> Deferred[List[None]]:
return self._process_parallel("open_spider", spider)
def close_spider(self, spider: Spider) -> Deferred:
def close_spider(self, spider: Spider) -> Deferred[List[None]]:
return self._process_parallel("close_spider", spider)

15
scrapy/pipelines/__init__.py
View File

@ -4,21 +4,26 @@ Item pipeline
See documentation in docs/item-pipeline.rst
"""
from typing import Any, List
from __future__ import annotations
from twisted.internet.defer import Deferred
from typing import TYPE_CHECKING, Any, List
from scrapy import Spider
from scrapy.middleware import MiddlewareManager
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import deferred_f_from_coro_f
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
from scrapy import Spider
from scrapy.settings import Settings
class ItemPipelineManager(MiddlewareManager):
component_name = "item pipeline"
@classmethod
def _get_mwlist_from_settings(cls, settings) -> List[Any]:
def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]:
return build_component_list(settings.getwithbase("ITEM_PIPELINES"))
def _add_middleware(self, pipe: Any) -> None:
@ -28,5 +33,5 @@ class ItemPipelineManager(MiddlewareManager):
deferred_f_from_coro_f(pipe.process_item)
)
def process_item(self, item: Any, spider: Spider) -> Deferred:
def process_item(self, item: Any, spider: Spider) -> Deferred[Any]:
return self._process_chain("process_item", item, spider)

287
scrapy/pipelines/files.py
View File

@ -16,18 +16,34 @@ from collections import defaultdict
from contextlib import suppress
from ftplib import FTP
from io import BytesIO
from os import PathLike
from pathlib import Path
from typing import IO, TYPE_CHECKING, DefaultDict, Optional, Set, Type, Union, cast
from typing import (
IO,
TYPE_CHECKING,
Any,
Callable,
DefaultDict,
Dict,
List,
NoReturn,
Optional,
Protocol,
Set,
Type,
TypedDict,
Union,
cast,
)
from urllib.parse import urlparse
from itemadapter import ItemAdapter
from twisted.internet import defer, threads
from twisted.internet.defer import Deferred, maybeDeferred
from twisted.internet.threads import deferToThread
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.media import MediaPipeline
from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline
from scrapy.settings import Settings
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.datatypes import CaseInsensitiveDict
@ -37,17 +53,24 @@ from scrapy.utils.python import to_bytes
from scrapy.utils.request import referer_str
if TYPE_CHECKING:
from os import PathLike
from twisted.python.failure import Failure
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Spider
logger = logging.getLogger(__name__)
def _to_string(path: Union[str, PathLike]) -> str:
def _to_string(path: Union[str, PathLike[str]]) -> str:
return str(path) # convert a Path object to string
def _md5sum(file: IO) -> str:
def _md5sum(file: IO[bytes]) -> str:
"""Calculate the md5 checksum of a file-like object without reading its
whole content in memory.
@ -68,23 +91,54 @@ class FileException(Exception):
"""General media error exception"""
class StatInfo(TypedDict, total=False):
checksum: str
last_modified: float
class FilesStoreProtocol(Protocol):
def __init__(self, basedir: str): ...
def persist_file(
self,
path: str,
buf: BytesIO,
info: MediaPipeline.SpiderInfo,
meta: Optional[Dict[str, Any]] = None,
headers: Optional[Dict[str, str]] = None,
) -> Optional[Deferred[Any]]: ...
def stat_file(
self, path: str, info: MediaPipeline.SpiderInfo
) -> Union[StatInfo, Deferred[StatInfo]]: ...
class FSFilesStore:
def __init__(self, basedir: Union[str, PathLike]):
def __init__(self, basedir: Union[str, PathLike[str]]):
basedir = _to_string(basedir)
if "://" in basedir:
basedir = basedir.split("://", 1)[1]
self.basedir = basedir
self.basedir: str = basedir
self._mkdir(Path(self.basedir))
self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set)
self.created_directories: DefaultDict[MediaPipeline.SpiderInfo, Set[str]] = (
defaultdict(set)
)
def persist_file(
self, path: Union[str, PathLike], buf, info, meta=None, headers=None
):
self,
path: Union[str, PathLike[str]],
buf: BytesIO,
info: MediaPipeline.SpiderInfo,
meta: Optional[Dict[str, Any]] = None,
headers: Optional[Dict[str, str]] = None,
) -> None:
absolute_path = self._get_filesystem_path(path)
self._mkdir(absolute_path.parent, info)
absolute_path.write_bytes(buf.getvalue())
def stat_file(self, path: Union[str, PathLike], info):
def stat_file(
self, path: Union[str, PathLike[str]], info: MediaPipeline.SpiderInfo
) -> StatInfo:
absolute_path = self._get_filesystem_path(path)
try:
last_modified = absolute_path.stat().st_mtime
@ -96,12 +150,14 @@ class FSFilesStore:
return {"last_modified": last_modified, "checksum": checksum}
def _get_filesystem_path(self, path: Union[str, PathLike]) -> Path:
def _get_filesystem_path(self, path: Union[str, PathLike[str]]) -> Path:
path_comps = _to_string(path).split("/")
return Path(self.basedir, *path_comps)
def _mkdir(self, dirname: Path, domain: Optional[str] = None):
seen = self.created_directories[domain] if domain else set()
def _mkdir(
self, dirname: Path, domain: Optional[MediaPipeline.SpiderInfo] = None
) -> None:
seen: Set[str] = self.created_directories[domain] if domain else set()
if str(dirname) not in seen:
if not dirname.exists():
dirname.mkdir(parents=True)
@ -122,7 +178,7 @@ class S3FilesStore:
"Cache-Control": "max-age=172800",
}
def __init__(self, uri):
def __init__(self, uri: str):
if not is_botocore_available():
raise NotConfigured("missing botocore library")
import botocore.session
@ -142,8 +198,10 @@ class S3FilesStore:
raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
self.bucket, self.prefix = uri[5:].split("/", 1)
def stat_file(self, path, info):
def _onsuccess(boto_key):
def stat_file(
self, path: str, info: MediaPipeline.SpiderInfo
) -> Deferred[StatInfo]:
def _onsuccess(boto_key: Dict[str, Any]) -> StatInfo:
checksum = boto_key["ETag"].strip('"')
last_modified = boto_key["LastModified"]
modified_stamp = time.mktime(last_modified.timetuple())
@ -151,21 +209,31 @@ class S3FilesStore:
return self._get_boto_key(path).addCallback(_onsuccess)
def _get_boto_key(self, path):
def _get_boto_key(self, path: str) -> Deferred[Dict[str, Any]]:
key_name = f"{self.prefix}{path}"
return threads.deferToThread(
self.s3_client.head_object, Bucket=self.bucket, Key=key_name
return cast(
"Deferred[Dict[str, Any]]",
deferToThread(
self.s3_client.head_object, Bucket=self.bucket, Key=key_name # type: ignore[attr-defined]
),
)
def persist_file(self, path, buf, info, meta=None, headers=None):
def persist_file(
self,
path: str,
buf: BytesIO,
info: MediaPipeline.SpiderInfo,
meta: Optional[Dict[str, Any]] = None,
headers: Optional[Dict[str, str]] = None,
) -> Deferred[Any]:
"""Upload file to S3 storage"""
key_name = f"{self.prefix}{path}"
buf.seek(0)
extra = self._headers_to_botocore_kwargs(self.HEADERS)
if headers:
extra.update(self._headers_to_botocore_kwargs(headers))
return threads.deferToThread(
self.s3_client.put_object,
return deferToThread(
self.s3_client.put_object, # type: ignore[attr-defined]
Bucket=self.bucket,
Key=key_name,
Body=buf,
@ -174,7 +242,7 @@ class S3FilesStore:
**extra,
)
def _headers_to_botocore_kwargs(self, headers):
def _headers_to_botocore_kwargs(self, headers: Dict[str, Any]) -> Dict[str, Any]:
"""Convert headers to botocore keyword arguments."""
# This is required while we need to support both boto and botocore.
mapping = CaseInsensitiveDict(
@ -206,7 +274,7 @@ class S3FilesStore:
"X-Amz-Website-Redirect-Location": "WebsiteRedirectLocation",
}
)
extra = {}
extra: Dict[str, Any] = {}
for key, value in headers.items():
try:
kwarg = mapping[key]
@ -226,13 +294,13 @@ class GCSFilesStore:
# Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.
POLICY = None
def __init__(self, uri):
def __init__(self, uri: str):
from google.cloud import storage
client = storage.Client(project=self.GCS_PROJECT_ID)
bucket, prefix = uri[5:].split("/", 1)
self.bucket = client.bucket(bucket)
self.prefix = prefix
self.prefix: str = prefix
permissions = self.bucket.test_iam_permissions(
["storage.objects.get", "storage.objects.create"]
)
@ -248,8 +316,10 @@ class GCSFilesStore:
{"bucket": bucket},
)
def stat_file(self, path, info):
def _onsuccess(blob):
def stat_file(
self, path: str, info: MediaPipeline.SpiderInfo
) -> Deferred[StatInfo]:
def _onsuccess(blob) -> StatInfo:
if blob:
checksum = base64.b64decode(blob.md5_hash).hex()
last_modified = time.mktime(blob.updated.timetuple())
@ -257,24 +327,32 @@ class GCSFilesStore:
return {}
blob_path = self._get_blob_path(path)
return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(
_onsuccess
return cast(
Deferred[StatInfo],
deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess),
)
def _get_content_type(self, headers):
def _get_content_type(self, headers: Optional[Dict[str, str]]) -> str:
if headers and "Content-Type" in headers:
return headers["Content-Type"]
return "application/octet-stream"
def _get_blob_path(self, path):
def _get_blob_path(self, path: str) -> str:
return self.prefix + path
def persist_file(self, path, buf, info, meta=None, headers=None):
def persist_file(
self,
path: str,
buf: BytesIO,
info: MediaPipeline.SpiderInfo,
meta: Optional[Dict[str, Any]] = None,
headers: Optional[Dict[str, str]] = None,
) -> Deferred[Any]:
blob_path = self._get_blob_path(path)
blob = self.bucket.blob(blob_path)
blob.cache_control = self.CACHE_CONTROL
blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
return threads.deferToThread(
return deferToThread(
blob.upload_from_string,
data=buf.getvalue(),
content_type=self._get_content_type(headers),
@ -283,24 +361,35 @@ class GCSFilesStore:
class FTPFilesStore:
FTP_USERNAME = None
FTP_PASSWORD = None
USE_ACTIVE_MODE = None
FTP_USERNAME: Optional[str] = None
FTP_PASSWORD: Optional[str] = None
USE_ACTIVE_MODE: Optional[bool] = None
def __init__(self, uri):
def __init__(self, uri: str):
if not uri.startswith("ftp://"):
raise ValueError(f"Incorrect URI scheme in {uri}, expected 'ftp'")
u = urlparse(uri)
self.port = u.port
self.host = u.hostname
assert u.port
assert u.hostname
self.port: int = u.port
self.host: str = u.hostname
self.port = int(u.port or 21)
self.username = u.username or self.FTP_USERNAME
self.password = u.password or self.FTP_PASSWORD
self.basedir = u.path.rstrip("/")
assert self.FTP_USERNAME
assert self.FTP_PASSWORD
self.username: str = u.username or self.FTP_USERNAME
self.password: str = u.password or self.FTP_PASSWORD
self.basedir: str = u.path.rstrip("/")
def persist_file(self, path, buf, info, meta=None, headers=None):
def persist_file(
self,
path: str,
buf: BytesIO,
info: MediaPipeline.SpiderInfo,
meta: Optional[Dict[str, Any]] = None,
headers: Optional[Dict[str, str]] = None,
) -> Deferred[Any]:
path = f"{self.basedir}/{path}"
return threads.deferToThread(
return deferToThread(
ftp_store_file,
path=path,
file=buf,
@ -311,8 +400,10 @@ class FTPFilesStore:
use_active_mode=self.USE_ACTIVE_MODE,
)
def stat_file(self, path, info):
def _stat_file(path):
def stat_file(
self, path: str, info: MediaPipeline.SpiderInfo
) -> Deferred[StatInfo]:
def _stat_file(path: str) -> StatInfo:
try:
ftp = FTP()
ftp.connect(self.host, self.port)
@ -328,7 +419,7 @@ class FTPFilesStore:
except Exception:
return {}
return threads.deferToThread(_stat_file, path)
return cast("Deferred[StatInfo]", deferToThread(_stat_file, path))
class FilesPipeline(MediaPipeline):
@ -350,20 +441,23 @@ class FilesPipeline(MediaPipeline):
"""
MEDIA_NAME = "file"
EXPIRES = 90
STORE_SCHEMES = {
MEDIA_NAME: str = "file"
EXPIRES: int = 90
STORE_SCHEMES: Dict[str, Type[FilesStoreProtocol]] = {
"": FSFilesStore,
"file": FSFilesStore,
"s3": S3FilesStore,
"gs": GCSFilesStore,
"ftp": FTPFilesStore,
}
DEFAULT_FILES_URLS_FIELD = "file_urls"
DEFAULT_FILES_RESULT_FIELD = "files"
DEFAULT_FILES_URLS_FIELD: str = "file_urls"
DEFAULT_FILES_RESULT_FIELD: str = "files"
def __init__(
self, store_uri: Union[str, PathLike], download_func=None, settings=None
self,
store_uri: Union[str, PathLike[str]],
download_func: Optional[Callable[[Request, Spider], Response]] = None,
settings: Union[Settings, Dict[str, Any], None] = None,
):
store_uri = _to_string(store_uri)
if not store_uri:
@ -372,26 +466,26 @@ class FilesPipeline(MediaPipeline):
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
cls_name = "FilesPipeline"
self.store = self._get_store(store_uri)
self.store: FilesStoreProtocol = self._get_store(store_uri)
resolve = functools.partial(
self._key_for_pipe, base_class_name=cls_name, settings=settings
)
self.expires = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES)
self.expires: int = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES)
if not hasattr(self, "FILES_URLS_FIELD"):
self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELD
if not hasattr(self, "FILES_RESULT_FIELD"):
self.FILES_RESULT_FIELD = self.DEFAULT_FILES_RESULT_FIELD
self.files_urls_field = settings.get(
self.files_urls_field: str = settings.get(
resolve("FILES_URLS_FIELD"), self.FILES_URLS_FIELD
)
self.files_result_field = settings.get(
self.files_result_field: str = settings.get(
resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD
)
super().__init__(download_func=download_func, settings=settings)
@classmethod
def from_settings(cls, settings) -> Self:
def from_settings(cls, settings: Settings) -> Self:
s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"])
s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"]
s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"]
@ -418,7 +512,7 @@ class FilesPipeline(MediaPipeline):
store_uri = settings["FILES_STORE"]
return cls(store_uri, settings=settings)
def _get_store(self, uri: str):
def _get_store(self, uri: str) -> FilesStoreProtocol:
if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir
scheme = "file"
else:
@ -426,19 +520,21 @@ class FilesPipeline(MediaPipeline):
store_cls = self.STORE_SCHEMES[scheme]
return store_cls(uri)
def media_to_download(self, request, info, *, item=None):
def _onsuccess(result):
def media_to_download(
self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None
) -> Deferred[Optional[FileInfo]]:
def _onsuccess(result: StatInfo) -> Optional[FileInfo]:
if not result:
return # returning None force download
return None # returning None force download
last_modified = result.get("last_modified", None)
if not last_modified:
return # returning None force download
return None # returning None force download
age_seconds = time.time() - last_modified
age_days = age_seconds / 60 / 60 / 24
if age_days > self.expires:
return # returning None force download
return None # returning None force download
referer = referer_str(request)
logger.debug(
@ -458,19 +554,22 @@ class FilesPipeline(MediaPipeline):
}
path = self.file_path(request, info=info, item=item)
dfd = defer.maybeDeferred(self.store.stat_file, path, info)
dfd.addCallback(_onsuccess)
dfd.addErrback(lambda _: None)
dfd.addErrback(
# maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type
dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type]
dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess)
dfd2.addErrback(lambda _: None)
dfd2.addErrback(
lambda f: logger.error(
self.__class__.__name__ + ".store.stat_file",
exc_info=failure_to_exc_info(f),
extra={"spider": info.spider},
)
)
return dfd
return dfd2
def media_failed(self, failure, request, info):
def media_failed(
self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo
) -> NoReturn:
if not isinstance(failure.value, IgnoreRequest):
referer = referer_str(request)
logger.warning(
@ -487,7 +586,14 @@ class FilesPipeline(MediaPipeline):
raise FileException
def media_downloaded(self, response, request, info, *, item=None):
def media_downloaded(
self,
response: Response,
request: Request,
info: MediaPipeline.SpiderInfo,
*,
item: Any = None,
) -> FileInfo:
referer = referer_str(request)
if response.status != 200:
@ -546,16 +652,26 @@ class FilesPipeline(MediaPipeline):
"status": status,
}
def inc_stats(self, spider, status):
def inc_stats(self, spider: Spider, status: str) -> None:
assert spider.crawler.stats
spider.crawler.stats.inc_value("file_count", spider=spider)
spider.crawler.stats.inc_value(f"file_status_count/{status}", spider=spider)
# Overridable Interface
def get_media_requests(self, item, info):
def get_media_requests(
self, item: Any, info: MediaPipeline.SpiderInfo
) -> List[Request]:
urls = ItemAdapter(item).get(self.files_urls_field, [])
return [Request(u, callback=NO_CALLBACK) for u in urls]
def file_downloaded(self, response, request, info, *, item=None):
def file_downloaded(
self,
response: Response,
request: Request,
info: MediaPipeline.SpiderInfo,
*,
item: Any = None,
) -> str:
path = self.file_path(request, response=response, info=info, item=item)
buf = BytesIO(response.body)
checksum = _md5sum(buf)
@ -563,12 +679,21 @@ class FilesPipeline(MediaPipeline):
self.store.persist_file(path, buf, info)
return checksum
def item_completed(self, results, item, info):
def item_completed(
self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo
) -> Any:
with suppress(KeyError):
ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok]
return item
def file_path(self, request, response=None, info=None, *, item=None):
def file_path(
self,
request: Request,
response: Optional[Response] = None,
info: Optional[MediaPipeline.SpiderInfo] = None,
*,
item: Any = None,
) -> str:
media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec
media_ext = Path(request.url).suffix
# Handles empty and wild extensions by trying to guess the
@ -577,5 +702,5 @@ class FilesPipeline(MediaPipeline):
media_ext = ""
media_type = mimetypes.guess_type(request.url)[0]
if media_type:
media_ext = mimetypes.guess_extension(media_type)
media_ext = cast(str, mimetypes.guess_extension(media_type))
return f"full/{media_guid}{media_ext}"

133
scrapy/pipelines/images.py
View File

@ -11,13 +11,24 @@ import hashlib
import warnings
from contextlib import suppress
from io import BytesIO
from os import PathLike
from typing import TYPE_CHECKING, Dict, Tuple, Type, Union, cast
from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
Iterable,
List,
Optional,
Tuple,
Type,
Union,
cast,
)
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import (
FileException,
@ -27,20 +38,25 @@ from scrapy.pipelines.files import (
S3FilesStore,
_md5sum,
)
# TODO: from scrapy.pipelines.media import MediaPipeline
from scrapy.settings import Settings
from scrapy.utils.python import get_func_args, to_bytes
if TYPE_CHECKING:
from os import PathLike
from PIL import Image
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Spider
from scrapy.pipelines.media import FileInfoOrError, MediaPipeline
class NoimagesDrop(DropItem):
"""Product with no images exception"""
def __init__(self, *args, **kwargs):
def __init__(self, *args: Any, **kwargs: Any):
warnings.warn(
"The NoimagesDrop class is deprecated",
category=ScrapyDeprecationWarning,
@ -56,19 +72,22 @@ class ImageException(FileException):
class ImagesPipeline(FilesPipeline):
"""Abstract pipeline that implement the image thumbnail generation logic"""
MEDIA_NAME = "image"
MEDIA_NAME: str = "image"
# Uppercase attributes kept for backward compatibility with code that subclasses
# ImagesPipeline. They may be overridden by settings.
MIN_WIDTH = 0
MIN_HEIGHT = 0
EXPIRES = 90
MIN_WIDTH: int = 0
MIN_HEIGHT: int = 0
EXPIRES: int = 90
THUMBS: Dict[str, Tuple[int, int]] = {}
DEFAULT_IMAGES_URLS_FIELD = "image_urls"
DEFAULT_IMAGES_RESULT_FIELD = "images"
def __init__(
self, store_uri: Union[str, PathLike], download_func=None, settings=None
self,
store_uri: Union[str, PathLike[str]],
download_func: Optional[Callable[[Request, Spider], Response]] = None,
settings: Union[Settings, Dict[str, Any], None] = None,
):
try:
from PIL import Image
@ -89,27 +108,33 @@ class ImagesPipeline(FilesPipeline):
base_class_name="ImagesPipeline",
settings=settings,
)
self.expires = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES)
self.expires: int = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES)
if not hasattr(self, "IMAGES_RESULT_FIELD"):
self.IMAGES_RESULT_FIELD = self.DEFAULT_IMAGES_RESULT_FIELD
self.IMAGES_RESULT_FIELD: str = self.DEFAULT_IMAGES_RESULT_FIELD
if not hasattr(self, "IMAGES_URLS_FIELD"):
self.IMAGES_URLS_FIELD = self.DEFAULT_IMAGES_URLS_FIELD
self.IMAGES_URLS_FIELD: str = self.DEFAULT_IMAGES_URLS_FIELD
self.images_urls_field = settings.get(
self.images_urls_field: str = settings.get(
resolve("IMAGES_URLS_FIELD"), self.IMAGES_URLS_FIELD
)
self.images_result_field = settings.get(
self.images_result_field: str = settings.get(
resolve("IMAGES_RESULT_FIELD"), self.IMAGES_RESULT_FIELD
)
self.min_width = settings.getint(resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH)
self.min_height = settings.getint(resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT)
self.thumbs = settings.get(resolve("IMAGES_THUMBS"), self.THUMBS)
self.min_width: int = settings.getint(
resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH
)
self.min_height: int = settings.getint(
resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT
)
self.thumbs: Dict[str, Tuple[int, int]] = settings.get(
resolve("IMAGES_THUMBS"), self.THUMBS
)
self._deprecated_convert_image = None
self._deprecated_convert_image: Optional[bool] = None
@classmethod
def from_settings(cls, settings) -> Self:
def from_settings(cls, settings: Settings) -> Self:
s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"])
s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"]
s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"]
@ -136,11 +161,25 @@ class ImagesPipeline(FilesPipeline):
store_uri = settings["IMAGES_STORE"]
return cls(store_uri, settings=settings)
def file_downloaded(self, response, request, info, *, item=None):
def file_downloaded(
self,
response: Response,
request: Request,
info: MediaPipeline.SpiderInfo,
*,
item: Any = None,
) -> str:
return self.image_downloaded(response, request, info, item=item)
def image_downloaded(self, response, request, info, *, item=None):
checksum = None
def image_downloaded(
self,
response: Response,
request: Request,
info: MediaPipeline.SpiderInfo,
*,
item: Any = None,
) -> str:
checksum: Optional[str] = None
for path, image, buf in self.get_images(response, request, info, item=item):
if checksum is None:
buf.seek(0)
@ -153,9 +192,17 @@ class ImagesPipeline(FilesPipeline):
meta={"width": width, "height": height},
headers={"Content-Type": "image/jpeg"},
)
assert checksum is not None
return checksum
def get_images(self, response, request, info, *, item=None):
def get_images(
self,
response: Response,
request: Request,
info: MediaPipeline.SpiderInfo,
*,
item: Any = None,
) -> Iterable[Tuple[str, Image.Image, BytesIO]]:
path = self.file_path(request, response=response, info=info, item=item)
orig_image = self._Image.open(BytesIO(response.body))
@ -196,7 +243,12 @@ class ImagesPipeline(FilesPipeline):
thumb_image, thumb_buf = self.convert_image(image, size, buf)
yield thumb_path, thumb_image, thumb_buf
def convert_image(self, image, size=None, response_body=None):
def convert_image(
self,
image: Image.Image,
size: Optional[Tuple[int, int]] = None,
response_body: Optional[BytesIO] = None,
) -> Tuple[Image.Image, BytesIO]:
if response_body is None:
warnings.warn(
f"{self.__class__.__name__}.convert_image() method called in a deprecated way, "
@ -225,7 +277,7 @@ class ImagesPipeline(FilesPipeline):
# when updating the minimum requirements for Pillow.
resampling_filter = self._Image.Resampling.LANCZOS
except AttributeError:
resampling_filter = self._Image.ANTIALIAS
resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined]
image.thumbnail(size, resampling_filter)
elif response_body is not None and image.format == "JPEG":
return image, response_body
@ -234,19 +286,38 @@ class ImagesPipeline(FilesPipeline):
image.save(buf, "JPEG")
return image, buf
def get_media_requests(self, item, info):
def get_media_requests(
self, item: Any, info: MediaPipeline.SpiderInfo
) -> List[Request]:
urls = ItemAdapter(item).get(self.images_urls_field, [])
return [Request(u, callback=NO_CALLBACK) for u in urls]
def item_completed(self, results, item, info):
def item_completed(
self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo
) -> Any:
with suppress(KeyError):
ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok]
return item
def file_path(self, request, response=None, info=None, *, item=None):
def file_path(
self,
request: Request,
response: Optional[Response] = None,
info: Optional[MediaPipeline.SpiderInfo] = None,
*,
item: Any = None,
) -> str:
image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec
return f"full/{image_guid}.jpg"
def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
def thumb_path(
self,
request: Request,
thumb_id: str,
response: Optional[Response] = None,
info: Optional[MediaPipeline.SpiderInfo] = None,
*,
item: Any = None,
) -> str:
thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec
return f"thumbs/{thumb_id}/{thumb_guid}.jpg"

200
scrapy/pipelines/media.py
View File

@ -2,13 +2,30 @@ from __future__ import annotations
import functools
import logging
from abc import ABC, abstractmethod
from collections import defaultdict
from typing import TYPE_CHECKING
from typing import (
TYPE_CHECKING,
Any,
Callable,
DefaultDict,
Dict,
List,
Literal,
NoReturn,
Optional,
Set,
Tuple,
TypedDict,
TypeVar,
Union,
cast,
)
from twisted.internet.defer import Deferred, DeferredList
from twisted.python.failure import Failure
from scrapy.http.request import NO_CALLBACK
from scrapy.http.request import NO_CALLBACK, Request
from scrapy.settings import Settings
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.defer import defer_result, mustbe_deferred
@ -19,50 +36,71 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.utils.request import RequestFingerprinter
_T = TypeVar("_T")
class FileInfo(TypedDict):
url: str
path: str
checksum: Optional[str]
status: str
FileInfoOrError = Union[Tuple[Literal[True], FileInfo], Tuple[Literal[False], Failure]]
logger = logging.getLogger(__name__)
def _DUMMY_CALLBACK(response):
return response
class MediaPipeline(ABC):
crawler: Crawler
_fingerprinter: RequestFingerprinter
class MediaPipeline:
LOG_FAILED_RESULTS = True
LOG_FAILED_RESULTS: bool = True
class SpiderInfo:
def __init__(self, spider):
self.spider = spider
self.downloading = set()
self.downloaded = {}
self.waiting = defaultdict(list)
def __init__(self, spider: Spider):
self.spider: Spider = spider
self.downloading: Set[bytes] = set()
self.downloaded: Dict[bytes, Union[FileInfo, Failure]] = {}
self.waiting: DefaultDict[bytes, List[Deferred[FileInfo]]] = defaultdict(
list
)
def __init__(self, download_func=None, settings=None):
def __init__(
self,
download_func: Optional[Callable[[Request, Spider], Response]] = None,
settings: Union[Settings, Dict[str, Any], None] = None,
):
self.download_func = download_func
self._expects_item = {}
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
resolve = functools.partial(
self._key_for_pipe, base_class_name="MediaPipeline", settings=settings
)
self.allow_redirects = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False)
self.allow_redirects: bool = settings.getbool(
resolve("MEDIA_ALLOW_REDIRECTS"), False
)
self._handle_statuses(self.allow_redirects)
def _handle_statuses(self, allow_redirects):
def _handle_statuses(self, allow_redirects: bool) -> None:
self.handle_httpstatus_list = None
if allow_redirects:
self.handle_httpstatus_list = SequenceExclude(range(300, 400))
def _key_for_pipe(self, key, base_class_name=None, settings=None):
"""
>>> MediaPipeline()._key_for_pipe("IMAGES")
'IMAGES'
>>> class MyPipe(MediaPipeline):
... pass
>>> MyPipe()._key_for_pipe("IMAGES", base_class_name="MediaPipeline")
'MYPIPE_IMAGES'
"""
def _key_for_pipe(
self,
key: str,
base_class_name: Optional[str] = None,
settings: Optional[Settings] = None,
) -> str:
class_name = self.__class__.__name__
formatted_key = f"{class_name.upper()}_{key}"
if (
@ -75,31 +113,35 @@ class MediaPipeline:
return formatted_key
@classmethod
def from_crawler(cls, crawler) -> Self:
def from_crawler(cls, crawler: Crawler) -> Self:
pipe: Self
try:
pipe = cls.from_settings(crawler.settings) # type: ignore[attr-defined]
except AttributeError:
pipe = cls()
pipe.crawler = crawler
assert crawler.request_fingerprinter
pipe._fingerprinter = crawler.request_fingerprinter
return pipe
def open_spider(self, spider):
def open_spider(self, spider: Spider) -> None:
self.spiderinfo = self.SpiderInfo(spider)
def process_item(self, item, spider):
def process_item(
self, item: Any, spider: Spider
) -> Deferred[List[FileInfoOrError]]:
info = self.spiderinfo
requests = arg_to_iter(self.get_media_requests(item, info))
dlist = [self._process_request(r, info, item) for r in requests]
dfd = DeferredList(dlist, consumeErrors=True)
dfd = cast(
"Deferred[List[FileInfoOrError]]", DeferredList(dlist, consumeErrors=True)
)
return dfd.addCallback(self.item_completed, item, info)
def _process_request(self, request, info, item):
def _process_request(
self, request: Request, info: SpiderInfo, item: Any
) -> Deferred[FileInfo]:
fp = self._fingerprinter.fingerprint(request)
if not request.callback or request.callback is NO_CALLBACK:
cb = _DUMMY_CALLBACK
else:
cb = request.callback
eb = request.errback
request.callback = NO_CALLBACK
request.errback = None
@ -107,14 +149,12 @@ class MediaPipeline:
# Return cached result if request was already seen
if fp in info.downloaded:
d = defer_result(info.downloaded[fp])
d.addCallback(cb)
if eb:
d.addErrback(eb)
return d
# Otherwise, wait for result
wad = Deferred()
wad.addCallback(cb)
wad: Deferred[FileInfo] = Deferred()
if eb:
wad.addErrback(eb)
info.waiting[fp].append(wad)
@ -125,36 +165,48 @@ class MediaPipeline:
# Download request checking media_to_download hook output first
info.downloading.add(fp)
dfd = mustbe_deferred(self.media_to_download, request, info, item=item)
dfd.addCallback(self._check_media_to_download, request, info, item=item)
dfd.addErrback(self._log_exception)
dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
return dfd.addBoth(lambda _: wad) # it must return wad at last
dfd: Deferred[Optional[FileInfo]] = mustbe_deferred(
self.media_to_download, request, info, item=item
)
dfd2: Deferred[FileInfo] = dfd.addCallback(
self._check_media_to_download, request, info, item=item
)
dfd2.addErrback(self._log_exception)
dfd2.addBoth(self._cache_result_and_execute_waiters, fp, info)
return dfd2.addBoth(lambda _: wad) # it must return wad at last
def _log_exception(self, result):
def _log_exception(self, result: Failure) -> Failure:
logger.exception(result)
return result
def _modify_media_request(self, request):
def _modify_media_request(self, request: Request) -> None:
if self.handle_httpstatus_list:
request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list
else:
request.meta["handle_httpstatus_all"] = True
def _check_media_to_download(self, result, request, info, item):
def _check_media_to_download(
self, result: Optional[FileInfo], request: Request, info: SpiderInfo, item: Any
) -> Union[FileInfo, Deferred[FileInfo]]:
if result is not None:
return result
dfd: Deferred[Response]
if self.download_func:
# this ugly code was left only to support tests. TODO: remove
dfd = mustbe_deferred(self.download_func, request, info.spider)
else:
self._modify_media_request(request)
assert self.crawler.engine
dfd = self.crawler.engine.download(request)
dfd.addCallback(self.media_downloaded, request, info, item=item)
dfd.addErrback(self.media_failed, request, info)
return dfd
dfd2: Deferred[FileInfo] = dfd.addCallback(
self.media_downloaded, request, info, item=item
)
dfd2.addErrback(self.media_failed, request, info)
return dfd2
def _cache_result_and_execute_waiters(self, result, fp, info):
def _cache_result_and_execute_waiters(
self, result: Union[FileInfo, Failure], fp: bytes, info: SpiderInfo
) -> None:
if isinstance(result, Failure):
# minimize cached information for failure
result.cleanFailure()
@ -184,7 +236,7 @@ class MediaPipeline:
# Exception Chaining (https://www.python.org/dev/peps/pep-3134/).
context = getattr(result.value, "__context__", None)
if isinstance(context, StopIteration):
setattr(result.value, "__context__", None)
result.value.__context__ = None
info.downloading.remove(fp)
info.downloaded[fp] = result # cache result
@ -192,27 +244,45 @@ class MediaPipeline:
defer_result(result).chainDeferred(wad)
# Overridable Interface
def media_to_download(self, request, info, *, item=None):
@abstractmethod
def media_to_download(
self, request: Request, info: SpiderInfo, *, item: Any = None
) -> Deferred[Optional[FileInfo]]:
"""Check request before starting download"""
pass
raise NotImplementedError()
def get_media_requests(self, item, info):
@abstractmethod
def get_media_requests(self, item: Any, info: SpiderInfo) -> List[Request]:
"""Returns the media requests to download"""
pass
raise NotImplementedError()
def media_downloaded(self, response, request, info, *, item=None):
@abstractmethod
def media_downloaded(
self,
response: Response,
request: Request,
info: SpiderInfo,
*,
item: Any = None,
) -> FileInfo:
"""Handler for success downloads"""
return response
raise NotImplementedError()
def media_failed(self, failure, request, info):
@abstractmethod
def media_failed(
self, failure: Failure, request: Request, info: SpiderInfo
) -> NoReturn:
"""Handler for failed downloads"""
return failure
raise NotImplementedError()
def item_completed(self, results, item, info):
def item_completed(
self, results: List[FileInfoOrError], item: Any, info: SpiderInfo
) -> Any:
"""Called per item when all media requests has been processed"""
if self.LOG_FAILED_RESULTS:
for ok, value in results:
if not ok:
assert isinstance(value, Failure)
logger.error(
"%(class)s found errors processing %(item)s",
{"class": self.__class__.__name__, "item": item},
@ -221,6 +291,14 @@ class MediaPipeline:
)
return item
def file_path(self, request, response=None, info=None, *, item=None):
@abstractmethod
def file_path(
self,
request: Request,
response: Optional[Response] = None,
info: Optional[SpiderInfo] = None,
*,
item: Any = None,
) -> str:
"""Returns the path where downloaded media should be stored"""
pass
raise NotImplementedError()

3
scrapy/resolver.py
View File

@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Type
from twisted.internet import defer
from twisted.internet.base import ReactorBase, ThreadedResolver
from twisted.internet.defer import Deferred
from twisted.internet.interfaces import (
IAddress,
IHostnameResolver,
@ -17,6 +16,8 @@ from zope.interface.declarations import implementer, provider
from scrapy.utils.datatypes import LocalCache
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
# typing.Self requires Python 3.11
from typing_extensions import Self

3
scrapy/robotstxt.py
View File

@ -6,7 +6,6 @@ from abc import ABCMeta, abstractmethod
from typing import TYPE_CHECKING, Optional, Union
from warnings import warn
from scrapy import Spider
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.python import to_unicode
@ -14,8 +13,10 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Spider
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)

Some files were not shown because too many files have changed in this diff Show More