RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into py313

This commit is contained in:
Andrey Rakhmatullin 2024-05-22 13:14:59 +05:00
parent c66b517706 631fc65fad
commit 85d7458651
230 changed files with 7881 additions and 2925 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
.bandit.yml
View File

@ -1,21 +1,7 @@
skips:
- B101
- B113 # https://github.com/PyCQA/bandit/issues/1010
- B105
- B301
- B303
- B306
- B307
- B311
- B320
- B321
- B324
- B402 # https://github.com/scrapy/scrapy/issues/4180
- B403
- B404
- B406
- B410
- B503
- B603
- B605
- B101 # assert_used, needed for mypy
- B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180
- B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180
- B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082
- B503 # ssl_with_bad_defaults
exclude_dirs: ['tests']

6
.bumpversion.cfg
View File

@ -1,7 +1,11 @@
[bumpversion]
current_version = 2.11.0
current_version = 2.11.2
commit = True
tag = True
tag_name = {new_version}
[bumpversion:file:scrapy/VERSION]
[bumpversion:file:SECURITY.md]
parse = (?P<major>\d+)\.(?P<minor>\d+)\.x
serialize = {major}.{minor}.x

6
.coveragerc
View File

@ -4,3 +4,9 @@ include = scrapy/*
omit =
tests/*
disable_warnings = include-ignored
[report]
# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185
exclude_lines =
pragma: no cover
if TYPE_CHECKING:

2
.flake8
View File

@ -1,7 +1,7 @@
[flake8]
max-line-length = 119
ignore = W503, E203
ignore = E203, E501, E701, E704, W503
exclude =
docs/conf.py

4
.git-blame-ignore-revs
View File

@ -1,7 +1,7 @@
# .git-blame-ignore-revs
# adding black formatter to all the code
e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d
# re applying black to the code with default line length
# reapplying black to the code with default line length
303f0a70fcf8067adf0a909c2096a5009162383a
# reaplying black again and removing line length on pre-commit black config
# reapplying black again and removing line length on pre-commit black config
c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962

3
.github/workflows/checks.yml vendored
View File

@ -18,6 +18,9 @@ jobs:
- python-version: 3.8
env:
TOXENV: typing
- python-version: 3.8
env:
TOXENV: typing-tests
- python-version: "3.11" # Keep in sync with .readthedocs.yml
env:
TOXENV: docs

10
.pre-commit-config.yaml
View File

@ -1,19 +1,19 @@
repos:
- repo: https://github.com/PyCQA/bandit
rev: 1.7.5
rev: 1.7.7
hooks:
- id: bandit
args: [-r, -c, .bandit.yml]
- repo: https://github.com/PyCQA/flake8
rev: 6.1.0
rev: 7.0.0
hooks:
- id: flake8
- repo: https://github.com/psf/black.git
rev: 23.9.1
rev: 24.2.0
hooks:
- id: black
- repo: https://github.com/pycqa/isort
rev: 5.12.0
rev: 5.13.2
hooks:
- id: isort
- repo: https://github.com/adamchainz/blacken-docs
@ -21,4 +21,4 @@ repos:
hooks:
- id: blacken-docs
additional_dependencies:
- black==23.9.1
- black==24.2.0

12
MANIFEST.in
View File

@ -1,9 +1,8 @@
include README.rst
include AUTHORS
include INSTALL
include LICENSE
include MANIFEST.in
include CODE_OF_CONDUCT.md
include CONTRIBUTING.md
include INSTALL.md
include NEWS
include SECURITY.md
include scrapy/VERSION
include scrapy/mime.types
@ -12,16 +11,13 @@ include scrapy/py.typed
include codecov.yml
include conftest.py
include pytest.ini
include requirements-*.txt
include tox.ini
recursive-include scrapy/templates *
recursive-include scrapy license.txt
recursive-include docs *
prune docs/build
recursive-include extras *
recursive-include bin *
recursive-include tests *
global-exclude __pycache__ *.py[cod]

12
SECURITY.md Normal file
View File

@ -0,0 +1,12 @@
# Security Policy
## Supported Versions
| Version | Supported |
| ------- | ------------------ |
| 2.11.x | :white_check_mark: |
| < 2.11.x | :x: |
## Reporting a Vulnerability
Please report the vulnerability using https://github.com/scrapy/scrapy/security/advisories/new.

16
conftest.py
View File

@ -1,10 +1,6 @@
import platform
import sys
from pathlib import Path
import pytest
from twisted import version as twisted_version
from twisted.python.versions import Version
from twisted.web.http import H2_ENABLED
from scrapy.utils.reactor import install_reactor
@ -85,12 +81,12 @@ def only_not_asyncio(request, reactor_pytest):
def requires_uvloop(request):
if not request.node.get_closest_marker("requires_uvloop"):
return
if sys.implementation.name == "pypy":
pytest.skip("uvloop does not support pypy properly")
if platform.system() == "Windows":
pytest.skip("uvloop does not support Windows")
if twisted_version == Version("twisted", 21, 2, 0):
pytest.skip("https://twistedmatrix.com/trac/ticket/10106")
try:
import uvloop
del uvloop
except ImportError:
pytest.skip("uvloop is not installed")
def pytest_configure(config):

2
docs/_tests/quotes.html
View File

@ -273,7 +273,7 @@
Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
</p>
<p class="copyright">
Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
Made with <span class='sh-red'>❤</span> by <a href="https://www.zyte.com">Zyte</a>
</p>
</div>
</footer>

2
docs/_tests/quotes1.html
View File

@ -273,7 +273,7 @@
Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
</p>
<p class="copyright">
Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
Made with <span class='sh-red'>❤</span> by <a href="https://www.zyte.com">Zyte</a>
</p>
</div>
</footer>

5
docs/conf.py
View File

@ -10,7 +10,6 @@
# serve to show the default.
import sys
from datetime import datetime
from pathlib import Path
# If your extensions are in another directory, add it here. If the directory
@ -48,7 +47,7 @@ master_doc = "index"
# General information about the project.
project = "Scrapy"
copyright = f"2008–{datetime.now().year}, Scrapy developers"
copyright = "Scrapy developers"
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@ -227,7 +226,7 @@ latex_documents = [
# A list of regular expressions that match URIs that should not be checked when
# doing a linkcheck build.
linkcheck_ignore = [
"http://localhost:\d+",
r"http://localhost:\d+",
"http://hg.scrapy.org",
"http://directory.google.com/",
]

2
docs/contributing.rst
View File

@ -178,7 +178,7 @@ Scrapy:
* We use `black <https://black.readthedocs.io/en/stable/>`_ for code formatting.
There is a hook in the pre-commit config
that will automatically format your code before every commit. You can also
run black manually with ``tox -e black``.
run black manually with ``tox -e pre-commit``.
* Don't put your name in the code you contribute; git provides enough
metadata to identify author of the code.

67
docs/faq.rst
View File

@ -138,39 +138,37 @@ See previous question.
How can I prevent memory errors due to many allowed domains?
------------------------------------------------------------
If you have a spider with a long list of
:attr:`~scrapy.Spider.allowed_domains` (e.g. 50,000+), consider
replacing the default
:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware
with a :ref:`custom spider middleware <custom-spider-middleware>` that requires
less memory. For example:
If you have a spider with a long list of :attr:`~scrapy.Spider.allowed_domains`
(e.g. 50,000+), consider replacing the default
:class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` downloader
middleware with a :ref:`custom downloader middleware
<topics-downloader-middleware-custom>` that requires less memory. For example:
- If your domain names are similar enough, use your own regular expression
instead joining the strings in
:attr:`~scrapy.Spider.allowed_domains` into a complex regular
expression.
instead joining the strings in :attr:`~scrapy.Spider.allowed_domains` into
a complex regular expression.
- If you can `meet the installation requirements`_, use pyre2_ instead of
Python’s re_ to compile your URL-filtering regular expression. See
:issue:`1908`.
See also other suggestions at `StackOverflow`_.
See also `other suggestions at StackOverflow
<https://stackoverflow.com/q/36440681>`__.
.. note:: Remember to disable
:class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable
your custom implementation:
:class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` when you
enable your custom implementation:
.. code-block:: python
SPIDER_MIDDLEWARES = {
"scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None,
"myproject.middlewares.CustomOffsiteMiddleware": 500,
DOWNLOADER_MIDDLEWARES = {
"scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": None,
"myproject.middlewares.CustomOffsiteMiddleware": 50,
}
.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation
.. _pyre2: https://github.com/andreasvc/pyre2
.. _re: https://docs.python.org/library/re.html
.. _StackOverflow: https://stackoverflow.com/q/36440681/939364
Can I use Basic HTTP Authentication in my spiders?
--------------------------------------------------
@ -206,12 +204,10 @@ I get "Filtered offsite request" messages. How can I fix them?
Those messages (logged with ``DEBUG`` level) don't necessarily mean there is a
problem, so you may not need to fix them.
Those messages are thrown by the Offsite Spider Middleware, which is a spider
middleware (enabled by default) whose purpose is to filter out requests to
domains outside the ones covered by the spider.
For more info see:
:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware`.
Those messages are thrown by
:class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`, which is a
downloader middleware (enabled by default) whose purpose is to filter out
requests to domains outside the ones covered by the spider.
What is the recommended way to deploy a Scrapy crawler in production?
---------------------------------------------------------------------
@ -297,9 +293,13 @@ build the DOM of the entire feed in memory, and this can be quite slow and
consume a lot of memory.
In order to avoid parsing all the entire feed at once in memory, you can use
the functions ``xmliter`` and ``csviter`` from ``scrapy.utils.iterators``
module. In fact, this is what the feed spiders (see :ref:`topics-spiders`) use
under the cover.
the :func:`~scrapy.utils.iterators.xmliter_lxml` and
:func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what
:class:`~scrapy.spiders.XMLFeedSpider` uses.
.. autofunction:: scrapy.utils.iterators.xmliter_lxml
.. autofunction:: scrapy.utils.iterators.csviter
Does Scrapy manage cookies automatically?
-----------------------------------------
@ -405,6 +405,23 @@ or :class:`~scrapy.signals.headers_received` signals and raising a
:ref:`topics-stop-response-download` topic for additional information and examples.
.. _faq-blank-request:
How can I make a blank request?
-------------------------------
.. code-block:: python
from scrapy import Request
blank_request = Request("data:,")
In this case, the URL is set to a data URI scheme. Data URLs allow you to include data
in-line in web pages as if they were external resources. The "data:" scheme with an empty
content (",") essentially creates a request to a data URL without any specific content.
Running ``runspider`` I get ``error: No spider found in file: <filename>``
--------------------------------------------------------------------------

261
docs/news.rst
View File

@ -3,6 +3,229 @@
Release notes
=============
.. _release-VERSION:
Scrapy VERSION (YYYY-MM-DD)
---------------------------
Deprecations
~~~~~~~~~~~~
- :meth:`scrapy.core.downloader.Downloader._get_slot_key` is deprecated, use
:meth:`scrapy.core.downloader.Downloader.get_slot_key` instead.
(:issue:`6340`)
.. _release-2.11.2:
Scrapy 2.11.2 (2024-05-14)
--------------------------
Security bug fixes
~~~~~~~~~~~~~~~~~~
- Redirects to non-HTTP protocols are no longer followed. Please, see the
`23j4-mw76-5v7h security advisory`_ for more information. (:issue:`457`)
.. _23j4-mw76-5v7h security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-23j4-mw76-5v7h
- The ``Authorization`` header is now dropped on redirects to a different
scheme (``http://`` or ``https://``) or port, even if the domain is the
same. Please, see the `4qqq-9vqf-3h3f security advisory`_ for more
information.
.. _4qqq-9vqf-3h3f security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-4qqq-9vqf-3h3f
- When using system proxy settings that are different for ``http://`` and
``https://``, redirects to a different URL scheme will now also trigger the
corresponding change in proxy settings for the redirected request. Please,
see the `jm3v-qxmh-hxwv security advisory`_ for more information.
(:issue:`767`)
.. _jm3v-qxmh-hxwv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-jm3v-qxmh-hxwv
- :attr:`Spider.allowed_domains <scrapy.Spider.allowed_domains>` is now
enforced for all requests, and not only requests from spider callbacks.
(:issue:`1042`, :issue:`2241`, :issue:`6358`)
- :func:`~scrapy.utils.iterators.xmliter_lxml` no longer resolves XML
entities. (:issue:`6265`)
- defusedxml_ is now used to make
:class:`scrapy.http.request.rpc.XmlRpcRequest` more secure.
(:issue:`6250`, :issue:`6251`)
.. _defusedxml: https://github.com/tiran/defusedxml
Bug fixes
~~~~~~~~~
- Restored support for brotlipy_, which had been dropped in Scrapy 2.11.1 in
favor of brotli_. (:issue:`6261`)
.. _brotli: https://github.com/google/brotli
.. note:: brotlipy is deprecated, both in Scrapy and upstream. Use brotli
instead if you can.
- Make :setting:`METAREFRESH_IGNORE_TAGS` ``["noscript"]`` by default. This
prevents
:class:`~scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware` from
following redirects that would not be followed by web browsers with
JavaScript enabled. (:issue:`6342`, :issue:`6347`)
- During :ref:`feed export <topics-feed-exports>`, do not close the
underlying file from :ref:`built-in post-processing plugins
<builtin-plugins>`.
(:issue:`5932`, :issue:`6178`, :issue:`6239`)
- :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
now properly applies the ``unique`` and ``canonicalize`` parameters.
(:issue:`3273`, :issue:`6221`)
- Do not initialize the scheduler disk queue if :setting:`JOBDIR` is an empty
string. (:issue:`6121`, :issue:`6124`)
- Fix :attr:`Spider.logger <scrapy.Spider.logger>` not logging custom extra
information. (:issue:`6323`, :issue:`6324`)
- ``robots.txt`` files with a non-UTF-8 encoding no longer prevent parsing
the UTF-8-compatible (e.g. ASCII) parts of the document.
(:issue:`6292`, :issue:`6298`)
- :meth:`scrapy.http.cookies.WrappedRequest.get_header` no longer raises an
exception if ``default`` is ``None``.
(:issue:`6308`, :issue:`6310`)
- :class:`~scrapy.selector.Selector` now uses
:func:`scrapy.utils.response.get_base_url` to determine the base URL of a
given :class:`~scrapy.http.Response`. (:issue:`6265`)
- The :meth:`media_to_download` method of :ref:`media pipelines
<topics-media-pipeline>` now logs exceptions before stripping them.
(:issue:`5067`, :issue:`5068`)
- When passing a callback to the :command:`parse` command, build the callback
callable with the right signature.
(:issue:`6182`)
Documentation
~~~~~~~~~~~~~
- Add a FAQ entry about :ref:`creating blank requests <faq-blank-request>`.
(:issue:`6203`, :issue:`6208`)
- Document that :attr:`scrapy.selector.Selector.type` can be ``"json"``.
(:issue:`6328`, :issue:`6334`)
Quality assurance
~~~~~~~~~~~~~~~~~
- Make builds reproducible. (:issue:`5019`, :issue:`6322`)
- Packaging and test fixes.
(:issue:`6286`, :issue:`6290`, :issue:`6312`, :issue:`6316`, :issue:`6344`)
.. _release-2.11.1:
Scrapy 2.11.1 (2024-02-14)
--------------------------
Highlights:
- Security bug fixes.
- Support for Twisted >= 23.8.0.
- Documentation improvements.
Security bug fixes
~~~~~~~~~~~~~~~~~~
- Addressed `ReDoS vulnerabilities`_:
- ``scrapy.utils.iterators.xmliter`` is now deprecated in favor of
:func:`~scrapy.utils.iterators.xmliter_lxml`, which
:class:`~scrapy.spiders.XMLFeedSpider` now uses.
To minimize the impact of this change on existing code,
:func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating
the node namespace with a prefix in the node name, and big files with
highly nested trees when using libxml2 2.7+.
- Fixed regular expressions in the implementation of the
:func:`~scrapy.utils.response.open_in_browser` function.
Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information.
.. _ReDoS vulnerabilities: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS
.. _cc65-xxvf-f7r9 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9
- :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` now also apply
to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security
advisory`_ for more information.
.. _7j7m-v7m3-jqm7 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-7j7m-v7m3-jqm7
- Also in relation with the `7j7m-v7m3-jqm7 security advisory`_, the
deprecated ``scrapy.downloadermiddlewares.decompression`` module has been
removed.
- The ``Authorization`` header is now dropped on redirects to a different
domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more
information.
.. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv
Modified requirements
~~~~~~~~~~~~~~~~~~~~~
- The Twisted dependency is no longer restricted to < 23.8.0. (:issue:`6024`,
:issue:`6064`, :issue:`6142`)
Bug fixes
~~~~~~~~~
- The OS signal handling code was refactored to no longer use private Twisted
functions. (:issue:`6024`, :issue:`6064`, :issue:`6112`)
Documentation
~~~~~~~~~~~~~
- Improved documentation for :class:`~scrapy.crawler.Crawler` initialization
changes made in the 2.11.0 release. (:issue:`6057`, :issue:`6147`)
- Extended documentation for :attr:`Request.meta <scrapy.http.Request.meta>`.
(:issue:`5565`)
- Fixed the :reqmeta:`dont_merge_cookies` documentation. (:issue:`5936`,
:issue:`6077`)
- Added a link to Zyte's export guides to the :ref:`feed exports
<topics-feed-exports>` documentation. (:issue:`6183`)
- Added a missing note about backward-incompatible changes in
:class:`~scrapy.exporters.PythonItemExporter` to the 2.11.0 release notes.
(:issue:`6060`, :issue:`6081`)
- Added a missing note about removing the deprecated
``scrapy.utils.boto.is_botocore()`` function to the 2.8.0 release notes.
(:issue:`6056`, :issue:`6061`)
- Other documentation improvements. (:issue:`6128`, :issue:`6144`,
:issue:`6163`, :issue:`6190`, :issue:`6192`)
Quality assurance
~~~~~~~~~~~~~~~~~
- Added Python 3.12 to the CI configuration, re-enabled tests that were
disabled when the pre-release support was added. (:issue:`5985`,
:issue:`6083`, :issue:`6098`)
- Fixed a test issue on PyPy 7.3.14. (:issue:`6204`, :issue:`6205`)
.. _release-2.11.0:
Scrapy 2.11.0 (2023-09-18)
@ -62,6 +285,9 @@ Deprecation removals
1.0.0, use :attr:`CrawlerRunner.spider_loader
<scrapy.crawler.CrawlerRunner.spider_loader>` instead. (:issue:`6010`)
- The :func:`scrapy.utils.response.response_httprepr` function, deprecated in
Scrapy 2.6.0, has now been removed. (:issue:`6111`)
Deprecations
~~~~~~~~~~~~
@ -1157,6 +1383,9 @@ Deprecations
Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`
first to set the :class:`~scrapy.Spider` object.
- :func:`scrapy.utils.response.response_httprepr` is now deprecated.
(:issue:`4972`)
New features
~~~~~~~~~~~~
@ -2871,6 +3100,38 @@ affect subclasses:
(:issue:`3884`)
.. _release-1.8.4:
Scrapy 1.8.4 (2024-02-14)
-------------------------
**Security bug fixes:**
- Due to its `ReDoS vulnerabilities`_, ``scrapy.utils.iterators.xmliter`` is
now deprecated in favor of :func:`~scrapy.utils.iterators.xmliter_lxml`,
which :class:`~scrapy.spiders.XMLFeedSpider` now uses.
To minimize the impact of this change on existing code,
:func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating
the node namespace as a prefix in the node name, and big files with highly
nested trees when using libxml2 2.7+.
Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information.
- :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` now also apply
to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security
advisory`_ for more information.
- Also in relation with the `7j7m-v7m3-jqm7 security advisory`_, use of the
``scrapy.downloadermiddlewares.decompression`` module is discouraged and
will trigger a warning.
- The ``Authorization`` header is now dropped on redirects to a different
domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more
information.
.. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv
.. _release-1.8.3:

8
docs/requirements.txt
View File

@ -1,4 +1,4 @@
sphinx==5.0.2
sphinx-hoverxref==1.1.1
sphinx-notfound-page==0.8
sphinx-rtd-theme==1.0.0
sphinx==6.2.1
sphinx-hoverxref==1.3.0
sphinx-notfound-page==1.0.0
sphinx-rtd-theme==2.0.0

3
docs/topics/addons.rst
View File

@ -150,8 +150,7 @@ Access the crawler instance:
def from_crawler(cls, crawler):
return cls(crawler)
def update_settings(self, settings):
...
def update_settings(self, settings): ...
Use a fallback component:

14
docs/topics/autothrottle.rst
View File

@ -47,6 +47,18 @@ effect, but there are some important differences:
AutoThrottle doesn't have these issues.
Disabling throttling on a downloader slot
=========================================
It is possible to disable AutoThrottle for a specific download slot at run time
by setting its ``throttle`` attribute to ``False``, e.g. using
:setting:`DOWNLOAD_SLOTS`.
Note, however, that AutoThrottle still determines the starting delay of every
slot by setting the ``download_delay`` attribute on the running spider. You
might want to set a custom value for the ``delay`` attribute of the slot, e.g.
using :setting:`DOWNLOAD_SLOTS`.
Throttling algorithm
====================
@ -131,7 +143,7 @@ AUTOTHROTTLE_TARGET_CONCURRENCY
Default: ``1.0``
Average number of requests Scrapy should be sending in parallel to remote
websites.
websites. It must be higher than ``0.0``.
By default, AutoThrottle adjusts the delay to send a single
concurrent request to each of the remote websites. Set this option to

4
docs/topics/benchmarking.rst
View File

@ -24,7 +24,8 @@ You should see an output like this::
'scrapy.extensions.telnet.TelnetConsole',
'scrapy.extensions.corestats.CoreStats']
2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled downloader middlewares:
['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware',
['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',
'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware',
'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',
'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',
'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',
@ -37,7 +38,6 @@ You should see an output like this::
'scrapy.downloadermiddlewares.stats.DownloaderStats']
2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled spider middlewares:
['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',
'scrapy.spidermiddlewares.referer.RefererMiddleware',
'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',
'scrapy.spidermiddlewares.depth.DepthMiddleware']

2
docs/topics/broad-crawls.rst
View File

@ -116,7 +116,7 @@ Reduce log level
When doing broad crawls you are often only interested in the crawl rates you
get and any errors found. These stats are reported by Scrapy when using the
``INFO`` log level. In order to save CPU (and log storage requirements) you
should not use ``DEBUG`` log level when preforming large broad crawls in
should not use ``DEBUG`` log level when performing large broad crawls in
production. Using ``DEBUG`` level when developing your (broad) crawler may be
fine though.

18
docs/topics/debug.rst
View File

@ -125,25 +125,15 @@ Fortunately, the :command:`shell` is your bread and butter in this case (see
See also: :ref:`topics-shell-inspect-response`.
Open in browser
===============
Sometimes you just want to see how a certain response looks in a browser, you
can use the ``open_in_browser`` function for that. Here is an example of how
you would use it:
can use the :func:`~scrapy.utils.response.open_in_browser` function for that:
.. code-block:: python
.. autofunction:: scrapy.utils.response.open_in_browser
from scrapy.utils.response import open_in_browser
def parse_details(self, response):
if "item name" not in response.body:
open_in_browser(response)
``open_in_browser`` will open a browser with the response received by Scrapy at
that point, adjusting the `base tag`_ so that images and styles are displayed
properly.
Logging
=======
@ -163,8 +153,6 @@ available in all future runs should they be necessary again:
For more information, check the :ref:`topics-logging` section.
.. _base tag: https://www.w3schools.com/tags/tag_base.asp
.. _debug-vscode:
Visual Studio Code

48
docs/topics/downloader-middleware.rst
View File

@ -763,6 +763,44 @@ HttpProxyMiddleware
Keep in mind this value will take precedence over ``http_proxy``/``https_proxy``
environment variables, and it will also ignore ``no_proxy`` environment variable.
OffsiteMiddleware
-----------------
.. module:: scrapy.downloadermiddlewares.offsite
:synopsis: Offsite Middleware
.. class:: OffsiteMiddleware
.. versionadded:: 2.11.2
Filters out Requests for URLs outside the domains covered by the spider.
This middleware filters out every request whose host names aren't in the
spider's :attr:`~scrapy.Spider.allowed_domains` attribute.
All subdomains of any domain in the list are also allowed.
E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org``
but not ``www2.example.com`` nor ``example.com``.
When your spider returns a request for a domain not belonging to those
covered by the spider, this middleware will log a debug message similar to
this one::
DEBUG: Filtered offsite request to 'offsite.example': <GET http://offsite.example/some/page.html>
To avoid filling the log with too much noise, it will only print one of
these messages for each new domain filtered. So, for example, if another
request for ``offsite.example`` is filtered, no log message will be
printed. But if a request for ``other.example`` is filtered, a message
will be printed (but only for the first request filtered).
If the spider doesn't define an
:attr:`~scrapy.Spider.allowed_domains` attribute, or the
attribute is empty, the offsite middleware will allow all requests.
If the request has the :attr:`~scrapy.Request.dont_filter` attribute
set, the offsite middleware will allow the request even if its domain is not
listed in allowed domains.
RedirectMiddleware
------------------
@ -882,7 +920,15 @@ Meta tags within these tags are ignored.
.. versionchanged:: 2.0
The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
``['script', 'noscript']`` to ``[]``.
``["script", "noscript"]`` to ``[]``.
.. versionchanged:: 2.11.2
The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
``[]`` to ``["noscript"]``.
.. versionchanged:: VERSION
The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
``[]`` to ``['noscript']``.
.. setting:: METAREFRESH_MAXDELAY

13
docs/topics/feed-exports.rst
View File

@ -13,6 +13,11 @@ Scrapy provides this functionality out of the box with the Feed Exports, which
allows you to generate feeds with the scraped items, using multiple
serialization formats and storage backends.
This page provides detailed documentation for all feed export features. If you
are looking for a step-by-step guide, check out `Zyte’s export guides`_.
.. _Zyte’s export guides: https://docs.zyte.com/web-scraping/guides/export/index.html#exporting-scraped-data
.. _topics-feed-format:
Serialization formats
@ -385,7 +390,13 @@ Each plugin is a class that must implement the following methods:
.. method:: close(self)
Close the target file object.
Clean up the plugin.
For example, you might want to close a file wrapper that you might have
used to compress data written into the file received in the ``__init__``
method.
.. warning:: Do not close the file from the ``__init__`` method.
To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You
can then access those parameters from the ``__init__`` method of your plugin.

7
docs/topics/items.rst
View File

@ -399,12 +399,7 @@ In code that receives an item, such as methods of :ref:`item pipelines
<topics-spider-middleware>`, it is a good practice to use the
:class:`~itemadapter.ItemAdapter` class and the
:func:`~itemadapter.is_item` function to write code that works for
any :ref:`supported item type <item-types>`:
.. autoclass:: itemadapter.ItemAdapter
.. autofunction:: itemadapter.is_item
any supported item type.
Other classes related to items
==============================

8
docs/topics/media-pipeline.rst
View File

@ -532,14 +532,14 @@ See here the methods that you can override in your custom Files Pipeline:
.. code-block:: python
from pathlib import PurePosixPath
from urllib.parse import urlparse
from scrapy.utils.httpobj import urlparse_cached
from scrapy.pipelines.files import FilesPipeline
class MyFilesPipeline(FilesPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
return "files/" + PurePosixPath(urlparse(request.url).path).name
return "files/" + PurePosixPath(urlparse_cached(request).path).name
Similarly, you can use the ``item`` to determine the file path based on some item
property.
@ -690,14 +690,14 @@ See here the methods that you can override in your custom Images Pipeline:
.. code-block:: python
from pathlib import PurePosixPath
from urllib.parse import urlparse
from scrapy.utils.httpobj import urlparse_cached
from scrapy.pipelines.images import ImagesPipeline
class MyImagesPipeline(ImagesPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
return "files/" + PurePosixPath(urlparse(request.url).path).name
return "files/" + PurePosixPath(urlparse_cached(request).path).name
Similarly, you can use the ``item`` to determine the file path based on some item
property.

3
docs/topics/practices.rst
View File

@ -289,7 +289,8 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
super proxy that you can attach your own proxies to.
* use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy
plugin <https://github.com/scrapy-plugins/scrapy-zyte-api>`__
plugin <https://github.com/scrapy-plugins/scrapy-zyte-api>`__ and additional
features, like `AI web scraping <https://www.zyte.com/ai-web-scraping/>`__
If you are still unable to prevent your bot getting banned, consider contacting
`commercial support`_.

68
docs/topics/request-response.rst
View File

@ -94,13 +94,14 @@ Request objects
.. code-block:: python
request_with_cookies = Request(
url="http://www.example.com",
url="https://www.example.com",
cookies=[
{
"name": "currency",
"value": "USD",
"domain": "example.com",
"path": "/currency",
"secure": True,
},
],
)
@ -469,60 +470,6 @@ import path.
.. autoclass:: scrapy.utils.request.RequestFingerprinter
.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
REQUEST_FINGERPRINTER_IMPLEMENTATION
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: 2.7
Default: ``'2.6'``
Determines which request fingerprinting algorithm is used by the default
request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
Possible values are:
- ``'2.6'`` (default)
This implementation uses the same request fingerprinting algorithm as
Scrapy 2.6 and earlier versions.
Even though this is the default value for backward compatibility reasons,
it is a deprecated value.
- ``'2.7'``
This implementation was introduced in Scrapy 2.7 to fix an issue of the
previous implementation.
New projects should use this value. The :command:`startproject` command
sets this value in the generated ``settings.py`` file.
If you are using the default value (``'2.6'``) for this setting, and you are
using Scrapy components where changing the request fingerprinting algorithm
would cause undesired results, you need to carefully decide when to change the
value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
setting to a custom request fingerprinter class that implements the 2.6 request
fingerprinting algorithm and does not log this warning (
:ref:`2.6-request-fingerprinter` includes an example implementation of such a
class).
Scenarios where changing the request fingerprinting algorithm may cause
undesired results include, for example, using the HTTP cache middleware (see
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
Changing the request fingerprinting algorithm would invalidate the current
cache, requiring you to redownload all requests again.
Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in
your settings to switch already to the request fingerprinting implementation
that will be the only request fingerprinting implementation available in a
future version of Scrapy, and remove the deprecation warning triggered by using
the default value (``'2.6'``).
.. _2.6-request-fingerprinter:
.. _custom-request-fingerprinter:
Writing your own request fingerprinter
@ -731,6 +678,7 @@ Those are:
* :reqmeta:`download_fail_on_dataloss`
* :reqmeta:`download_latency`
* :reqmeta:`download_maxsize`
* :reqmeta:`download_warnsize`
* :reqmeta:`download_timeout`
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
* ``ftp_user`` (See :setting:`FTP_USER` for more info)
@ -1357,3 +1305,13 @@ XmlResponse objects
line. See :attr:`TextResponse.encoding`.
.. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241
JsonResponse objects
--------------------
.. class:: JsonResponse(url[, ...])
The :class:`JsonResponse` class is a subclass of :class:`TextResponse`
that is used when the response has a `JSON MIME type
<https://mimesniff.spec.whatwg.org/#json-mime-type>`_ in its `Content-Type`
header.

6
docs/topics/selectors.rst
View File

@ -1032,10 +1032,8 @@ whereas the CSS lookup is translated into XPath and thus runs more efficiently,
so performance-wise its uses are limited to situations that are not easily
described with CSS selectors.
Parsel also simplifies adding your own XPath extensions.
.. autofunction:: parsel.xpathfuncs.set_xpathfunc
Parsel also simplifies adding your own XPath extensions with
:func:`~parsel.xpathfuncs.set_xpathfunc`.
.. _topics-selectors-ref:

50
docs/topics/settings.rst
View File

@ -674,6 +674,7 @@ Default:
.. code-block:: python
{
"scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50,
"scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware": 100,
"scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware": 300,
"scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware": 350,
@ -835,7 +836,7 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
.. setting:: DOWNLOAD_SLOTS
DOWNLOAD_SLOTS
----------------
--------------
Default: ``{}``
@ -844,7 +845,12 @@ Allows to define concurrency/delay parameters on per slot (domain) basis:
.. code-block:: python
DOWNLOAD_SLOTS = {
"quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False},
"quotes.toscrape.com": {
"concurrency": 1,
"delay": 2,
"randomize_delay": False,
"throttle": False,
},
"books.toscrape.com": {"delay": 3, "randomize_delay": False},
}
@ -856,6 +862,9 @@ Allows to define concurrency/delay parameters on per slot (domain) basis:
- :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency``
- :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay``
There is no global setting for ``throttle``, whose default value is
``None``.
.. setting:: DOWNLOAD_TIMEOUT
@ -873,40 +882,42 @@ The amount of time (in secs) that the downloader will wait before timing out.
Request.meta key.
.. setting:: DOWNLOAD_MAXSIZE
.. reqmeta:: download_maxsize
DOWNLOAD_MAXSIZE
----------------
Default: ``1073741824`` (1024MB)
Default: ``1073741824`` (1 GiB)
The maximum response size (in bytes) that downloader will download.
The maximum response body size (in bytes) allowed. Bigger responses are
aborted and ignored.
If you want to disable it set to 0.
This applies both before and after compression. If decompressing a response
body would exceed this limit, decompression is aborted and the response is
ignored.
.. reqmeta:: download_maxsize
Use ``0`` to disable this limit.
.. note::
This size can be set per spider using :attr:`download_maxsize`
spider attribute and per-request using :reqmeta:`download_maxsize`
Request.meta key.
This limit can be set per spider using the :attr:`download_maxsize` spider
attribute and per request using the :reqmeta:`download_maxsize` Request.meta
key.
.. setting:: DOWNLOAD_WARNSIZE
.. reqmeta:: download_warnsize
DOWNLOAD_WARNSIZE
-----------------
Default: ``33554432`` (32MB)
Default: ``33554432`` (32 MiB)
The response size (in bytes) that downloader will start to warn.
If the size of a response exceeds this value, before or after compression, a
warning will be logged about it.
If you want to disable it set to 0.
Use ``0`` to disable this limit.
.. note::
This size can be set per spider using :attr:`download_warnsize`
spider attribute and per-request using :reqmeta:`download_warnsize`
Request.meta key.
This limit can be set per spider using the :attr:`download_warnsize` spider
attribute and per request using the :reqmeta:`download_warnsize` Request.meta
key.
.. setting:: DOWNLOAD_FAIL_ON_DATALOSS
@ -1603,7 +1614,6 @@ Default:
{
"scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50,
"scrapy.spidermiddlewares.offsite.OffsiteMiddleware": 500,
"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
"scrapy.spidermiddlewares.depth.DepthMiddleware": 900,

11
docs/topics/signals.rst
View File

@ -343,11 +343,18 @@ request_scheduled
.. signal:: request_scheduled
.. function:: request_scheduled(request, spider)
Sent when the engine schedules a :class:`~scrapy.Request`, to be
downloaded later.
Sent when the engine is asked to schedule a :class:`~scrapy.Request`, to be
downloaded later, before the request reaches the :ref:`scheduler
<topics-scheduler>`.
Raise :exc:`~scrapy.exceptions.IgnoreRequest` to drop a request before it
reaches the scheduler.
This signal does not support returning deferreds from its handlers.
.. versionadded:: 2.11.2
Allow dropping requests with :exc:`~scrapy.exceptions.IgnoreRequest`.
:param request: the request that reached the scheduler
:type request: :class:`~scrapy.Request` object

40
docs/topics/spider-middleware.rst
View File

@ -51,8 +51,8 @@ value. For example, if you want to disable the off-site middleware:
.. code-block:: python
SPIDER_MIDDLEWARES = {
"myproject.middlewares.CustomSpiderMiddleware": 543,
"scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None,
"scrapy.spidermiddlewares.referer.RefererMiddleware": None,
"myproject.middlewares.CustomRefererSpiderMiddleware": 700,
}
Finally, keep in mind that some middlewares may need to be enabled through a
@ -313,42 +313,6 @@ Default: ``False``
Pass all responses, regardless of its status code.
OffsiteMiddleware
-----------------
.. module:: scrapy.spidermiddlewares.offsite
:synopsis: Offsite Spider Middleware
.. class:: OffsiteMiddleware
Filters out Requests for URLs outside the domains covered by the spider.
This middleware filters out every request whose host names aren't in the
spider's :attr:`~scrapy.Spider.allowed_domains` attribute.
All subdomains of any domain in the list are also allowed.
E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org``
but not ``www2.example.com`` nor ``example.com``.
When your spider returns a request for a domain not belonging to those
covered by the spider, this middleware will log a debug message similar to
this one::
DEBUG: Filtered offsite request to 'www.othersite.com': <GET http://www.othersite.com/some/page.html>
To avoid filling the log with too much noise, it will only print one of
these messages for each new domain filtered. So, for example, if another
request for ``www.othersite.com`` is filtered, no log message will be
printed. But if a request for ``someothersite.com`` is filtered, a message
will be printed (but only for the first request filtered).
If the spider doesn't define an
:attr:`~scrapy.Spider.allowed_domains` attribute, or the
attribute is empty, the offsite middleware will allow all requests.
If the request has the :attr:`~scrapy.Request.dont_filter` attribute
set, the offsite middleware will allow the request even if its domain is not
listed in allowed domains.
RefererMiddleware
-----------------

3
docs/topics/spiders.rst
View File

@ -75,7 +75,8 @@ scrapy.Spider
An optional list of strings containing domains that this spider is
allowed to crawl. Requests for URLs not belonging to the domain names
specified in this list (or their subdomains) won't be followed if
:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled.
:class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` is
enabled.
Let's say your target url is ``https://www.example.com/1.html``,
then add ``'example.com'`` to the list.

4
docs/topics/telnetconsole.rst
View File

@ -172,8 +172,8 @@ TELNETCONSOLE_PORT
Default: ``[6023, 6073]``
The port range to use for the telnet console. If set to ``None`` or ``0``, a
dynamically assigned port is used.
The port range to use for the telnet console. If set to ``None``, a dynamically
assigned port is used.
.. setting:: TELNETCONSOLE_HOST

19
pylintrc
View File

@ -4,21 +4,14 @@ jobs=1 # >1 hides results
[MESSAGES CONTROL]
disable=abstract-method,
anomalous-backslash-in-string,
arguments-differ,
arguments-renamed,
attribute-defined-outside-init,
bad-classmethod-argument,
bad-mcs-classmethod-argument,
bare-except,
broad-except,
broad-exception-raised,
c-extension-no-member,
catching-non-exception,
cell-var-from-loop,
comparison-with-callable,
consider-using-dict-items,
consider-using-in,
consider-using-with,
cyclic-import,
dangerous-default-value,
@ -32,7 +25,6 @@ disable=abstract-method,
implicit-str-concat,
import-error,
import-outside-toplevel,
import-self,
inconsistent-return-statements,
inherit-non-class,
invalid-name,
@ -44,7 +36,6 @@ disable=abstract-method,
logging-fstring-interpolation,
logging-not-lazy,
lost-exception,
method-hidden,
missing-docstring,
no-else-raise,
no-else-return,
@ -52,7 +43,7 @@ disable=abstract-method,
no-method-argument,
no-name-in-module,
no-self-argument,
no-value-for-parameter,
no-value-for-parameter, # https://github.com/pylint-dev/pylint/issues/3268
not-callable,
pointless-exception-statement,
pointless-statement,
@ -77,23 +68,15 @@ disable=abstract-method,
too-many-public-methods,
too-many-return-statements,
unbalanced-tuple-unpacking,
undefined-variable,
undefined-loop-variable,
unexpected-special-method-signature,
unnecessary-comprehension,
unnecessary-dunder-call,
unnecessary-pass,
unreachable,
unsubscriptable-object,
unused-argument,
unused-import,
unused-private-member,
unused-variable,
unused-wildcard-import,
use-dict-literal,
used-before-assignment,
useless-object-inheritance, # Required for Python 2 support
useless-return,
useless-super-delegation,
wildcard-import,
wrong-import-position

2
scrapy/VERSION
View File

@ -1 +1 @@
2.11.0
2.11.2

6
scrapy/addons.py
View File

@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, List
from scrapy.exceptions import NotConfigured
from scrapy.settings import Settings
from scrapy.utils.conf import build_component_list
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
from scrapy.crawler import Crawler
@ -32,9 +32,7 @@ class AddonManager:
for clspath in build_component_list(settings["ADDONS"]):
try:
addoncls = load_object(clspath)
addon = create_instance(
addoncls, settings=settings, crawler=self.crawler
)
addon = build_from_crawler(addoncls, self.crawler)
addon.update_settings(settings)
self.addons.append(addon)
except NotConfigured as e:

58
scrapy/cmdline.py
View File

@ -1,21 +1,33 @@
from __future__ import annotations
import argparse
import cProfile
import inspect
import os
import sys
from importlib.metadata import entry_points
from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple, Type
import scrapy
from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
from scrapy.crawler import CrawlerProcess
from scrapy.exceptions import UsageError
from scrapy.settings import BaseSettings, Settings
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import get_project_settings, inside_project
from scrapy.utils.python import garbage_collect
if TYPE_CHECKING:
# typing.ParamSpec requires Python 3.10
from typing_extensions import ParamSpec
_P = ParamSpec("_P")
class ScrapyArgumentParser(argparse.ArgumentParser):
def _parse_optional(self, arg_string):
def _parse_optional(
self, arg_string: str
) -> Optional[Tuple[Optional[argparse.Action], str, Optional[str]]]:
# if starts with -: it means that is a parameter not a argument
if arg_string[:2] == "-:":
return None
@ -23,7 +35,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser):
return super()._parse_optional(arg_string)
def _iter_command_classes(module_name):
def _iter_command_classes(module_name: str) -> Iterable[Type[ScrapyCommand]]:
# TODO: add `name` attribute to commands and merge this function with
# scrapy.utils.spider.iter_spider_classes
for module in walk_modules(module_name):
@ -37,8 +49,8 @@ def _iter_command_classes(module_name):
yield obj
def _get_commands_from_module(module, inproject):
d = {}
def _get_commands_from_module(module: str, inproject: bool) -> Dict[str, ScrapyCommand]:
d: Dict[str, ScrapyCommand] = {}
for cmd in _iter_command_classes(module):
if inproject or not cmd.requires_project:
cmdname = cmd.__module__.split(".")[-1]
@ -46,8 +58,10 @@ def _get_commands_from_module(module, inproject):
return d
def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
cmds = {}
def _get_commands_from_entry_points(
inproject: bool, group: str = "scrapy.commands"
) -> Dict[str, ScrapyCommand]:
cmds: Dict[str, ScrapyCommand] = {}
if sys.version_info >= (3, 10):
eps = entry_points(group=group)
else:
@ -61,7 +75,9 @@ def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
return cmds
def _get_commands_dict(settings, inproject):
def _get_commands_dict(
settings: BaseSettings, inproject: bool
) -> Dict[str, ScrapyCommand]:
cmds = _get_commands_from_module("scrapy.commands", inproject)
cmds.update(_get_commands_from_entry_points(inproject))
cmds_module = settings["COMMANDS_MODULE"]
@ -70,16 +86,17 @@ def _get_commands_dict(settings, inproject):
return cmds
def _pop_command_name(argv):
def _pop_command_name(argv: List[str]) -> Optional[str]:
i = 0
for arg in argv[1:]:
if not arg.startswith("-"):
del argv[i]
return arg
i += 1
return None
def _print_header(settings, inproject):
def _print_header(settings: BaseSettings, inproject: bool) -> None:
version = scrapy.__version__
if inproject:
print(f"Scrapy {version} - active project: {settings['BOT_NAME']}\n")
@ -88,7 +105,7 @@ def _print_header(settings, inproject):
print(f"Scrapy {version} - no active project\n")
def _print_commands(settings, inproject):
def _print_commands(settings: BaseSettings, inproject: bool) -> None:
_print_header(settings, inproject)
print("Usage:")
print(" scrapy <command> [options] [args]\n")
@ -103,13 +120,20 @@ def _print_commands(settings, inproject):
print('Use "scrapy <command> -h" to see more info about a command')
def _print_unknown_command(settings, cmdname, inproject):
def _print_unknown_command(
settings: BaseSettings, cmdname: str, inproject: bool
) -> None:
_print_header(settings, inproject)
print(f"Unknown command: {cmdname}\n")
print('Use "scrapy" to see available commands')
def _run_print_help(parser, func, *a, **kw):
def _run_print_help(
parser: argparse.ArgumentParser,
func: Callable[_P, None],
*a: _P.args,
**kw: _P.kwargs,
) -> None:
try:
func(*a, **kw)
except UsageError as e:
@ -120,7 +144,9 @@ def _run_print_help(parser, func, *a, **kw):
sys.exit(2)
def execute(argv=None, settings=None):
def execute(
argv: Optional[List[str]] = None, settings: Optional[Settings] = None
) -> None:
if argv is None:
argv = sys.argv
@ -162,14 +188,16 @@ def execute(argv=None, settings=None):
sys.exit(cmd.exitcode)
def _run_command(cmd, args, opts):
def _run_command(cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace) -> None:
if opts.profile:
_run_command_profiled(cmd, args, opts)
else:
cmd.run(args, opts)
def _run_command_profiled(cmd, args, opts):
def _run_command_profiled(
cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace
) -> None:
if opts.profile:
sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
loc = locals()

49
scrapy/commands/__init__.py
View File

@ -1,62 +1,64 @@
"""
Base class for Scrapy commands
"""
import argparse
import builtins
import os
from pathlib import Path
from typing import Any, Dict, List, Optional
from typing import Any, Dict, Iterable, List, Optional
from twisted.python import failure
from scrapy.crawler import CrawlerProcess
from scrapy.crawler import Crawler, CrawlerProcess
from scrapy.exceptions import UsageError
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
class ScrapyCommand:
requires_project = False
requires_project: bool = False
crawler_process: Optional[CrawlerProcess] = None
# default settings to be used for this command instead of global defaults
default_settings: Dict[str, Any] = {}
exitcode = 0
exitcode: int = 0
def __init__(self) -> None:
self.settings: Any = None # set in scrapy.cmdline
def set_crawler(self, crawler):
def set_crawler(self, crawler: Crawler) -> None:
if hasattr(self, "_crawler"):
raise RuntimeError("crawler already set")
self._crawler = crawler
self._crawler: Crawler = crawler
def syntax(self):
def syntax(self) -> str:
"""
Command syntax (preferably one-line). Do not include command name.
"""
return ""
def short_desc(self):
def short_desc(self) -> str:
"""
A short description of the command
"""
return ""
def long_desc(self):
def long_desc(self) -> str:
"""A long description of the command. Return short description when not
available. It cannot contain newlines since contents will be formatted
by optparser which removes newlines and wraps text.
"""
return self.short_desc()
def help(self):
def help(self) -> str:
"""An extensive help for the command. It will be shown when using the
"help" command. It can contain newlines since no post-formatting will
be applied to its contents.
"""
return self.long_desc()
def add_options(self, parser):
def add_options(self, parser: argparse.ArgumentParser) -> None:
"""
Populate option parse with options available for this command
"""
@ -91,7 +93,7 @@ class ScrapyCommand:
)
group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
def process_options(self, args, opts):
def process_options(self, args: List[str], opts: argparse.Namespace) -> None:
try:
self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline")
except ValueError:
@ -128,8 +130,8 @@ class BaseRunSpiderCommand(ScrapyCommand):
Common class used to share functionality between the crawl, parse and runspider commands
"""
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
def add_options(self, parser: argparse.ArgumentParser) -> None:
super().add_options(parser)
parser.add_argument(
"-a",
dest="spargs",
@ -161,8 +163,8 @@ class BaseRunSpiderCommand(ScrapyCommand):
help="format to use for dumping items",
)
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
def process_options(self, args: List[str], opts: argparse.Namespace) -> None:
super().process_options(args, opts)
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
@ -182,7 +184,13 @@ class ScrapyHelpFormatter(argparse.HelpFormatter):
Help Formatter for scrapy command line help messages.
"""
def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
def __init__(
self,
prog: str,
indent_increment: int = 2,
max_help_position: int = 24,
width: Optional[int] = None,
):
super().__init__(
prog,
indent_increment=indent_increment,
@ -190,11 +198,12 @@ class ScrapyHelpFormatter(argparse.HelpFormatter):
width=width,
)
def _join_parts(self, part_strings):
parts = self.format_part_strings(part_strings)
def _join_parts(self, part_strings: Iterable[str]) -> str:
# scrapy.commands.list shadows builtins.list
parts = self.format_part_strings(builtins.list(part_strings))
return super()._join_parts(parts)
def format_part_strings(self, part_strings):
def format_part_strings(self, part_strings: List[str]) -> List[str]:
"""
Underline and title case command line help message headers.
"""

25
scrapy/commands/bench.py
View File

@ -1,10 +1,14 @@
import subprocess
import argparse
import subprocess # nosec
import sys
import time
from typing import Any, Iterable, List
from urllib.parse import urlencode
import scrapy
from scrapy import Request
from scrapy.commands import ScrapyCommand
from scrapy.http import Response, TextResponse
from scrapy.linkextractors import LinkExtractor
@ -15,24 +19,28 @@ class Command(ScrapyCommand):
"CLOSESPIDER_TIMEOUT": 10,
}
def short_desc(self):
def short_desc(self) -> str:
return "Run quick benchmark test"
def run(self, args, opts):
def run(self, args: List[str], opts: argparse.Namespace) -> None:
with _BenchServer():
assert self.crawler_process
self.crawler_process.crawl(_BenchSpider, total=100000)
self.crawler_process.start()
class _BenchServer:
def __enter__(self):
def __enter__(self) -> None:
from scrapy.utils.test import get_testenv
pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]
self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE, env=get_testenv())
self.proc = subprocess.Popen(
pargs, stdout=subprocess.PIPE, env=get_testenv()
) # nosec
assert self.proc.stdout
self.proc.stdout.readline()
def __exit__(self, exc_type, exc_value, traceback):
def __exit__(self, exc_type, exc_value, traceback) -> None:
self.proc.kill()
self.proc.wait()
time.sleep(0.2)
@ -47,11 +55,12 @@ class _BenchSpider(scrapy.Spider):
baseurl = "http://localhost:8998"
link_extractor = LinkExtractor()
def start_requests(self):
def start_requests(self) -> Iterable[Request]:
qargs = {"total": self.total, "show": self.show}
url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}"
return [scrapy.Request(url, dont_filter=True)]
def parse(self, response):
def parse(self, response: Response) -> Any:
assert isinstance(Response, TextResponse)
for link in self.link_extractor.extract_links(response):
yield scrapy.Request(link.url, callback=self.parse)

18
scrapy/commands/check.py
View File

@ -1,5 +1,7 @@
import argparse
import time
from collections import defaultdict
from typing import List
from unittest import TextTestResult as _TextTestResult
from unittest import TextTestRunner
@ -10,9 +12,10 @@ from scrapy.utils.misc import load_object, set_environ
class TextTestResult(_TextTestResult):
def printSummary(self, start, stop):
def printSummary(self, start: float, stop: float) -> None:
write = self.stream.write
writeln = self.stream.writeln
# _WritelnDecorator isn't implemented in typeshed yet
writeln = self.stream.writeln # type: ignore[attr-defined]
run = self.testsRun
plural = "s" if run != 1 else ""
@ -42,14 +45,14 @@ class Command(ScrapyCommand):
requires_project = True
default_settings = {"LOG_ENABLED": False}
def syntax(self):
def syntax(self) -> str:
return "[options] <spider>"
def short_desc(self):
def short_desc(self) -> str:
return "Check spider contracts"
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
def add_options(self, parser: argparse.ArgumentParser) -> None:
super().add_options(parser)
parser.add_argument(
"-l",
"--list",
@ -66,7 +69,7 @@ class Command(ScrapyCommand):
help="print contract tests for all spiders",
)
def run(self, args, opts):
def run(self, args: List[str], opts: argparse.Namespace) -> None:
# load contracts
contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS"))
conman = ContractsManager(load_object(c) for c in contracts)
@ -76,6 +79,7 @@ class Command(ScrapyCommand):
# contract requests
contract_reqs = defaultdict(list)
assert self.crawler_process
spider_loader = self.crawler_process.spider_loader
with set_environ(SCRAPY_CHECK="true"):

14
scrapy/commands/crawl.py
View File

@ -1,3 +1,8 @@
import argparse
from typing import List, cast
from twisted.python.failure import Failure
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
@ -5,13 +10,13 @@ from scrapy.exceptions import UsageError
class Command(BaseRunSpiderCommand):
requires_project = True
def syntax(self):
def syntax(self) -> str:
return "[options] <spider>"
def short_desc(self):
def short_desc(self) -> str:
return "Run a spider"
def run(self, args, opts):
def run(self, args: List[str], opts: argparse.Namespace) -> None:
if len(args) < 1:
raise UsageError()
elif len(args) > 1:
@ -20,10 +25,11 @@ class Command(BaseRunSpiderCommand):
)
spname = args[0]
assert self.crawler_process
crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)
if getattr(crawl_defer, "result", None) is not None and issubclass(
crawl_defer.result.type, Exception
cast(Failure, crawl_defer.result).type, Exception
):
self.exitcode = 1
else:

16
scrapy/commands/edit.py
View File

@ -1,5 +1,7 @@
import argparse
import os
import sys
from typing import List
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
@ -9,32 +11,34 @@ class Command(ScrapyCommand):
requires_project = True
default_settings = {"LOG_ENABLED": False}
def syntax(self):
def syntax(self) -> str:
return "<spider>"
def short_desc(self):
def short_desc(self) -> str:
return "Edit spider"
def long_desc(self):
def long_desc(self) -> str:
return (
"Edit a spider using the editor defined in the EDITOR environment"
" variable or else the EDITOR setting"
)
def _err(self, msg):
def _err(self, msg: str) -> None:
sys.stderr.write(msg + os.linesep)
self.exitcode = 1
def run(self, args, opts):
def run(self, args: List[str], opts: argparse.Namespace) -> None:
if len(args) != 1:
raise UsageError()
editor = self.settings["EDITOR"]
assert self.crawler_process
try:
spidercls = self.crawler_process.spider_loader.load(args[0])
except KeyError:
return self._err(f"Spider not found: {args[0]}")
sfile = sys.modules[spidercls.__module__].__file__
assert sfile
sfile = sfile.replace(".pyc", ".py")
self.exitcode = os.system(f'{editor} "{sfile}"')
self.exitcode = os.system(f'{editor} "{sfile}"') # nosec

23
scrapy/commands/fetch.py
View File

@ -1,13 +1,13 @@
import sys
from argparse import Namespace
from typing import List, Type
from argparse import ArgumentParser, Namespace
from typing import Dict, List, Type
from w3lib.url import is_url
from scrapy import Spider
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.http import Request
from scrapy.http import Request, Response
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.spider import DefaultSpider, spidercls_for_request
@ -15,20 +15,20 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request
class Command(ScrapyCommand):
requires_project = False
def syntax(self):
def syntax(self) -> str:
return "[options] <url>"
def short_desc(self):
def short_desc(self) -> str:
return "Fetch a URL using the Scrapy downloader"
def long_desc(self):
def long_desc(self) -> str:
return (
"Fetch a URL using the Scrapy downloader and print its content"
" to stdout. You may want to use --nolog to disable logging"
)
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
def add_options(self, parser: ArgumentParser) -> None:
super().add_options(parser)
parser.add_argument("--spider", dest="spider", help="use this spider")
parser.add_argument(
"--headers",
@ -44,20 +44,21 @@ class Command(ScrapyCommand):
help="do not handle HTTP 3xx status codes and print response as-is",
)
def _print_headers(self, headers, prefix):
def _print_headers(self, headers: Dict[bytes, List[bytes]], prefix: bytes) -> None:
for key, values in headers.items():
for value in values:
self._print_bytes(prefix + b" " + key + b": " + value)
def _print_response(self, response, opts):
def _print_response(self, response: Response, opts: Namespace) -> None:
if opts.headers:
assert response.request
self._print_headers(response.request.headers, b">")
print(">")
self._print_headers(response.headers, b"<")
else:
self._print_bytes(response.body)
def _print_bytes(self, bytes_):
def _print_bytes(self, bytes_: bytes) -> None:
sys.stdout.buffer.write(bytes_ + b"\n")
def run(self, args: List[str], opts: Namespace) -> None:

33
scrapy/commands/genspider.py
View File

@ -1,9 +1,10 @@
import argparse
import os
import shutil
import string
from importlib import import_module
from pathlib import Path
from typing import Optional, cast
from typing import List, Optional, Union, cast
from urllib.parse import urlparse
import scrapy
@ -12,7 +13,7 @@ from scrapy.exceptions import UsageError
from scrapy.utils.template import render_templatefile, string_camelcase
def sanitize_module_name(module_name):
def sanitize_module_name(module_name: str) -> str:
"""Sanitize the given module name, by replacing dashes and points
with underscores and prefixing it with a letter if it doesn't start
with one
@ -23,7 +24,7 @@ def sanitize_module_name(module_name):
return module_name
def extract_domain(url):
def extract_domain(url: str) -> str:
"""Extract domain name from URL string"""
o = urlparse(url)
if o.scheme == "" and o.netloc == "":
@ -31,7 +32,7 @@ def extract_domain(url):
return o.netloc
def verify_url_scheme(url):
def verify_url_scheme(url: str) -> str:
"""Check url for scheme and insert https if none found."""
parsed = urlparse(url)
if parsed.scheme == "" and parsed.netloc == "":
@ -43,14 +44,14 @@ class Command(ScrapyCommand):
requires_project = False
default_settings = {"LOG_ENABLED": False}
def syntax(self):
def syntax(self) -> str:
return "[options] <name> <domain>"
def short_desc(self):
def short_desc(self) -> str:
return "Generate new spider using pre-defined templates"
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
def add_options(self, parser: argparse.ArgumentParser) -> None:
super().add_options(parser)
parser.add_argument(
"-l",
"--list",
@ -86,7 +87,7 @@ class Command(ScrapyCommand):
help="If the spider already exists, overwrite it with the template",
)
def run(self, args, opts):
def run(self, args: List[str], opts: argparse.Namespace) -> None:
if opts.list:
self._list_templates()
return
@ -113,9 +114,16 @@ class Command(ScrapyCommand):
if template_file:
self._genspider(module, name, url, opts.template, template_file)
if opts.edit:
self.exitcode = os.system(f'scrapy edit "{name}"')
self.exitcode = os.system(f'scrapy edit "{name}"') # nosec
def _genspider(self, module, name, url, template_name, template_file):
def _genspider(
self,
module: str,
name: str,
url: str,
template_name: str,
template_file: Union[str, os.PathLike],
) -> None:
"""Generate the spider module, based on the given template"""
capitalized_module = "".join(s.capitalize() for s in module.split("_"))
domain = extract_domain(url)
@ -130,6 +138,7 @@ class Command(ScrapyCommand):
}
if self.settings.get("NEWSPIDER_MODULE"):
spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
assert spiders_module.__file__
spiders_dir = Path(spiders_module.__file__).parent.resolve()
else:
spiders_module = None
@ -152,7 +161,7 @@ class Command(ScrapyCommand):
print('Use "scrapy genspider --list" to see all available templates.')
return None
def _list_templates(self):
def _list_templates(self) -> None:
print("Available templates:")
for file in sorted(Path(self.templates_dir).iterdir()):
if file.suffix == ".tmpl":

8
scrapy/commands/list.py
View File

@ -1,3 +1,6 @@
import argparse
from typing import List
from scrapy.commands import ScrapyCommand
@ -5,9 +8,10 @@ class Command(ScrapyCommand):
requires_project = True
default_settings = {"LOG_ENABLED": False}
def short_desc(self):
def short_desc(self) -> str:
return "List available spiders"
def run(self, args, opts):
def run(self, args: List[str], opts: argparse.Namespace) -> None:
assert self.crawler_process
for s in sorted(self.crawler_process.spider_loader.list()):
print(s)

183
scrapy/commands/parse.py
View File

@ -1,15 +1,32 @@
import argparse
import functools
import inspect
import json
import logging
from typing import Dict
from types import CoroutineType
from typing import (
Any,
AsyncGenerator,
Callable,
Dict,
Iterable,
List,
Optional,
Tuple,
TypeVar,
Union,
overload,
)
from itemadapter import ItemAdapter, is_item
from twisted.internet.defer import maybeDeferred
from twisted.internet.defer import Deferred, maybeDeferred
from twisted.python.failure import Failure
from w3lib.url import is_url
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.http import Request
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils import display
from scrapy.utils.asyncgen import collect_asyncgen
from scrapy.utils.defer import aiter_errback, deferred_from_coro
@ -19,24 +36,26 @@ from scrapy.utils.spider import spidercls_for_request
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
class Command(BaseRunSpiderCommand):
requires_project = True
spider = None
items: Dict[int, list] = {}
requests: Dict[int, list] = {}
items: Dict[int, List[Any]] = {}
requests: Dict[int, List[Request]] = {}
first_response = None
def syntax(self):
def syntax(self) -> str:
return "[options] <url>"
def short_desc(self):
def short_desc(self) -> str:
return "Parse URL (using its spider) and print the results"
def add_options(self, parser):
BaseRunSpiderCommand.add_options(self, parser)
def add_options(self, parser: argparse.ArgumentParser) -> None:
super().add_options(parser)
parser.add_argument(
"--spider",
dest="spider",
@ -105,7 +124,7 @@ class Command(BaseRunSpiderCommand):
)
@property
def max_level(self):
def max_level(self) -> int:
max_items, max_requests = 0, 0
if self.items:
max_items = max(self.items)
@ -113,13 +132,21 @@ class Command(BaseRunSpiderCommand):
max_requests = max(self.requests)
return max(max_items, max_requests)
def handle_exception(self, _failure):
def handle_exception(self, _failure: Failure) -> None:
logger.error(
"An error is caught while iterating the async iterable",
exc_info=failure_to_exc_info(_failure),
)
def iterate_spider_output(self, result):
@overload
def iterate_spider_output(
self, result: Union[AsyncGenerator, CoroutineType]
) -> Deferred: ...
@overload
def iterate_spider_output(self, result: _T) -> Iterable: ...
def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]:
if inspect.isasyncgen(result):
d = deferred_from_coro(
collect_asyncgen(aiter_errback(result, self.handle_exception))
@ -132,15 +159,15 @@ class Command(BaseRunSpiderCommand):
return d
return arg_to_iter(deferred_from_coro(result))
def add_items(self, lvl, new_items):
def add_items(self, lvl: int, new_items: List[Any]) -> None:
old_items = self.items.get(lvl, [])
self.items[lvl] = old_items + new_items
def add_requests(self, lvl, new_reqs):
def add_requests(self, lvl: int, new_reqs: List[Request]) -> None:
old_reqs = self.requests.get(lvl, [])
self.requests[lvl] = old_reqs + new_reqs
def print_items(self, lvl=None, colour=True):
def print_items(self, lvl: Optional[int] = None, colour: bool = True) -> None:
if lvl is None:
items = [item for lst in self.items.values() for item in lst]
else:
@ -149,7 +176,7 @@ class Command(BaseRunSpiderCommand):
print("# Scraped Items ", "-" * 60)
display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)
def print_requests(self, lvl=None, colour=True):
def print_requests(self, lvl: Optional[int] = None, colour: bool = True) -> None:
if lvl is None:
if self.requests:
requests = self.requests[max(self.requests)]
@ -161,7 +188,7 @@ class Command(BaseRunSpiderCommand):
print("# Requests ", "-" * 65)
display.pprint(requests, colorize=colour)
def print_results(self, opts):
def print_results(self, opts: argparse.Namespace) -> None:
colour = not opts.nocolour
if opts.verbose:
@ -178,7 +205,14 @@ class Command(BaseRunSpiderCommand):
if not opts.nolinks:
self.print_requests(colour=colour)
def _get_items_and_requests(self, spider_output, opts, depth, spider, callback):
def _get_items_and_requests(
self,
spider_output: Iterable[Any],
opts: argparse.Namespace,
depth: int,
spider: Spider,
callback: Callable,
) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, Callable]:
items, requests = [], []
for x in spider_output:
if is_item(x):
@ -187,14 +221,21 @@ class Command(BaseRunSpiderCommand):
requests.append(x)
return items, requests, opts, depth, spider, callback
def run_callback(self, response, callback, cb_kwargs=None):
def run_callback(
self,
response: Response,
callback: Callable,
cb_kwargs: Optional[Dict[str, Any]] = None,
) -> Deferred:
cb_kwargs = cb_kwargs or {}
d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs))
return d
def get_callback_from_rules(self, spider, response):
def get_callback_from_rules(
self, spider: Spider, response: Response
) -> Union[Callable, str, None]:
if getattr(spider, "rules", None):
for rule in spider.rules:
for rule in spider.rules: # type: ignore[attr-defined]
if rule.link_extractor.matches(response.url):
return rule.callback or "parse"
else:
@ -203,8 +244,10 @@ class Command(BaseRunSpiderCommand):
"please specify a callback to use for parsing",
{"spider": spider.name},
)
return None
def set_spidercls(self, url, opts):
def set_spidercls(self, url: str, opts: argparse.Namespace) -> None:
assert self.crawler_process
spider_loader = self.crawler_process.spider_loader
if opts.spider:
try:
@ -218,13 +261,14 @@ class Command(BaseRunSpiderCommand):
if not self.spidercls:
logger.error("Unable to find spider for: %(url)s", {"url": url})
def _start_requests(spider):
def _start_requests(spider: Spider) -> Iterable[Request]:
yield self.prepare_request(spider, Request(url), opts)
if self.spidercls:
self.spidercls.start_requests = _start_requests
def start_parsing(self, url, opts):
def start_parsing(self, url: str, opts: argparse.Namespace) -> None:
assert self.crawler_process
self.crawler_process.crawl(self.spidercls, **opts.spargs)
self.pcrawler = list(self.crawler_process.crawlers)[0]
self.crawler_process.start()
@ -232,7 +276,12 @@ class Command(BaseRunSpiderCommand):
if not self.first_response:
logger.error("No response downloaded for: %(url)s", {"url": url})
def scraped_data(self, args):
def scraped_data(
self,
args: Tuple[
List[Any], List[Request], argparse.Namespace, int, Spider, Callable
],
) -> List[Any]:
items, requests, opts, depth, spider, callback = args
if opts.pipelines:
itemproc = self.pcrawler.engine.scraper.itemproc
@ -251,42 +300,53 @@ class Command(BaseRunSpiderCommand):
return scraped_data
def prepare_request(self, spider, request, opts):
def callback(response, **cb_kwargs):
def _get_callback(
self,
*,
spider: Spider,
opts: argparse.Namespace,
response: Optional[Response] = None,
) -> Callable:
cb: Union[str, Callable, None] = None
if response:
cb = response.meta["_callback"]
if not cb:
if opts.callback:
cb = opts.callback
elif response and opts.rules and self.first_response == response:
cb = self.get_callback_from_rules(spider, response)
if not cb:
raise ValueError(
f"Cannot find a rule that matches {response.url!r} in spider: "
f"{spider.name}"
)
else:
cb = "parse"
if not callable(cb):
assert cb is not None
cb_method = getattr(spider, cb, None)
if callable(cb_method):
cb = cb_method
else:
raise ValueError(
f"Cannot find callback {cb!r} in spider: {spider.name}"
)
assert callable(cb)
return cb
def prepare_request(
self, spider: Spider, request: Request, opts: argparse.Namespace
) -> Request:
def callback(response: Response, **cb_kwargs: Any) -> Deferred:
# memorize first request
if not self.first_response:
self.first_response = response
# determine real callback
cb = response.meta["_callback"]
if not cb:
if opts.callback:
cb = opts.callback
elif opts.rules and self.first_response == response:
cb = self.get_callback_from_rules(spider, response)
if not cb:
logger.error(
"Cannot find a rule that matches %(url)r in spider: %(spider)s",
{"url": response.url, "spider": spider.name},
)
return
else:
cb = "parse"
if not callable(cb):
cb_method = getattr(spider, cb, None)
if callable(cb_method):
cb = cb_method
else:
logger.error(
"Cannot find callback %(callback)r in spider: %(spider)s",
{"callback": cb, "spider": spider.name},
)
return
cb = self._get_callback(spider=spider, opts=opts, response=response)
# parse items and requests
depth = response.meta["_depth"]
depth: int = response.meta["_depth"]
d = self.run_callback(response, cb, cb_kwargs)
d.addCallback(self._get_items_and_requests, opts, depth, spider, callback)
@ -303,16 +363,19 @@ class Command(BaseRunSpiderCommand):
request.meta["_depth"] = 1
request.meta["_callback"] = request.callback
if not request.callback and not opts.rules:
cb = self._get_callback(spider=spider, opts=opts)
functools.update_wrapper(callback, cb)
request.callback = callback
return request
def process_options(self, args, opts):
BaseRunSpiderCommand.process_options(self, args, opts)
def process_options(self, args: List[str], opts: argparse.Namespace) -> None:
super().process_options(args, opts)
self.process_request_meta(opts)
self.process_request_cb_kwargs(opts)
def process_request_meta(self, opts):
def process_request_meta(self, opts: argparse.Namespace) -> None:
if opts.meta:
try:
opts.meta = json.loads(opts.meta)
@ -323,7 +386,7 @@ class Command(BaseRunSpiderCommand):
print_help=False,
)
def process_request_cb_kwargs(self, opts):
def process_request_cb_kwargs(self, opts: argparse.Namespace) -> None:
if opts.cbkwargs:
try:
opts.cbkwargs = json.loads(opts.cbkwargs)
@ -334,7 +397,7 @@ class Command(BaseRunSpiderCommand):
print_help=False,
)
def run(self, args, opts):
def run(self, args: List[str], opts: argparse.Namespace) -> None:
# parse arguments
if not len(args) == 1 or not is_url(args[0]):
raise UsageError()

12
scrapy/commands/runspider.py
View File

@ -1,9 +1,10 @@
import argparse
import sys
from importlib import import_module
from os import PathLike
from pathlib import Path
from types import ModuleType
from typing import Union
from typing import List, Union
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
@ -27,16 +28,16 @@ class Command(BaseRunSpiderCommand):
requires_project = False
default_settings = {"SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):
def syntax(self) -> str:
return "[options] <spider_file>"
def short_desc(self):
def short_desc(self) -> str:
return "Run a self-contained spider (without creating a project)"
def long_desc(self):
def long_desc(self) -> str:
return "Run the spider defined in the given file"
def run(self, args, opts):
def run(self, args: List[str], opts: argparse.Namespace) -> None:
if len(args) != 1:
raise UsageError()
filename = Path(args[0])
@ -51,6 +52,7 @@ class Command(BaseRunSpiderCommand):
raise UsageError(f"No spider found in file: {filename}\n")
spidercls = spclasses.pop()
assert self.crawler_process
self.crawler_process.crawl(spidercls, **opts.spargs)
self.crawler_process.start()

13
scrapy/commands/settings.py
View File

@ -1,4 +1,6 @@
import argparse
import json
from typing import List
from scrapy.commands import ScrapyCommand
from scrapy.settings import BaseSettings
@ -8,14 +10,14 @@ class Command(ScrapyCommand):
requires_project = False
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):
def syntax(self) -> str:
return "[options]"
def short_desc(self):
def short_desc(self) -> str:
return "Get settings values"
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
def add_options(self, parser: argparse.ArgumentParser) -> None:
super().add_options(parser)
parser.add_argument(
"--get", dest="get", metavar="SETTING", help="print raw setting value"
)
@ -44,7 +46,8 @@ class Command(ScrapyCommand):
help="print setting value, interpreted as a list",
)
def run(self, args, opts):
def run(self, args: List[str], opts: argparse.Namespace) -> None:
assert self.crawler_process
settings = self.crawler_process.settings
if opts.get:
s = settings.get(opts.get)

20
scrapy/commands/shell.py
View File

@ -3,9 +3,10 @@ Scrapy Shell
See documentation in docs/topics/shell.rst
"""
from argparse import Namespace
from argparse import ArgumentParser, Namespace
from threading import Thread
from typing import List, Type
from typing import Any, Dict, List, Type
from scrapy import Spider
from scrapy.commands import ScrapyCommand
@ -23,20 +24,20 @@ class Command(ScrapyCommand):
"DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
}
def syntax(self):
def syntax(self) -> str:
return "[url|file]"
def short_desc(self):
def short_desc(self) -> str:
return "Interactive scraping console"
def long_desc(self):
def long_desc(self) -> str:
return (
"Interactive console for scraping the given url or file. "
"Use ./file.html syntax or full path for local file."
)
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
def add_options(self, parser: ArgumentParser) -> None:
super().add_options(parser)
parser.add_argument(
"-c",
dest="code",
@ -51,7 +52,7 @@ class Command(ScrapyCommand):
help="do not handle HTTP 3xx status codes and print response as-is",
)
def update_vars(self, vars):
def update_vars(self, vars: Dict[str, Any]) -> None:
"""You can use this function to update the Scrapy objects that will be
available in the shell
"""
@ -87,7 +88,8 @@ class Command(ScrapyCommand):
shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
shell.start(url=url, redirect=not opts.no_redirect)
def _start_crawler_thread(self):
def _start_crawler_thread(self) -> None:
assert self.crawler_process
t = Thread(
target=self.crawler_process.start,
kwargs={"stop_after_crawl": False, "install_signal_handlers": False},

22
scrapy/commands/startproject.py
View File

@ -1,3 +1,4 @@
import argparse
import os
import re
import string
@ -5,13 +6,14 @@ from importlib.util import find_spec
from pathlib import Path
from shutil import copy2, copystat, ignore_patterns, move
from stat import S_IWUSR as OWNER_WRITE_PERMISSION
from typing import List, Tuple, Union
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.utils.template import render_templatefile, string_camelcase
TEMPLATES_TO_RENDER = (
TEMPLATES_TO_RENDER: Tuple[Tuple[str, ...], ...] = (
("scrapy.cfg",),
("${project_name}", "settings.py.tmpl"),
("${project_name}", "items.py.tmpl"),
@ -22,7 +24,7 @@ TEMPLATES_TO_RENDER = (
IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn")
def _make_writable(path):
def _make_writable(path: Union[str, os.PathLike]) -> None:
current_permissions = os.stat(path).st_mode
os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION)
@ -31,14 +33,14 @@ class Command(ScrapyCommand):
requires_project = False
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):
def syntax(self) -> str:
return "<project_name> [project_dir]"
def short_desc(self):
def short_desc(self) -> str:
return "Create new project"
def _is_valid_name(self, project_name):
def _module_exists(module_name):
def _is_valid_name(self, project_name: str) -> bool:
def _module_exists(module_name: str) -> bool:
spec = find_spec(module_name)
return spec is not None and spec.loader is not None
@ -53,7 +55,7 @@ class Command(ScrapyCommand):
return True
return False
def _copytree(self, src: Path, dst: Path):
def _copytree(self, src: Path, dst: Path) -> None:
"""
Since the original function always creates the directory, to resolve
the issue a new function had to be created. It's a simple copy and
@ -84,7 +86,7 @@ class Command(ScrapyCommand):
copystat(src, dst)
_make_writable(dst)
def run(self, args, opts):
def run(self, args: List[str], opts: argparse.Namespace) -> None:
if len(args) not in (1, 2):
raise UsageError()
@ -105,7 +107,9 @@ class Command(ScrapyCommand):
return
self._copytree(Path(self.templates_dir), project_dir.resolve())
move(project_dir / "module", project_dir / project_name)
# On 3.8 shutil.move doesn't fully support Path args, but it supports our use case
# See https://bugs.python.org/issue32689
move(project_dir / "module", project_dir / project_name) # type: ignore[arg-type]
for paths in TEMPLATES_TO_RENDER:
tplfile = Path(
project_dir,

13
scrapy/commands/version.py
View File

@ -1,3 +1,6 @@
import argparse
from typing import List
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.utils.versions import scrapy_components_versions
@ -6,14 +9,14 @@ from scrapy.utils.versions import scrapy_components_versions
class Command(ScrapyCommand):
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):
def syntax(self) -> str:
return "[-v]"
def short_desc(self):
def short_desc(self) -> str:
return "Print Scrapy version"
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
def add_options(self, parser: argparse.ArgumentParser) -> None:
super().add_options(parser)
parser.add_argument(
"--verbose",
"-v",
@ -22,7 +25,7 @@ class Command(ScrapyCommand):
help="also display twisted/python/platform info (useful for bug reports)",
)
def run(self, args, opts):
def run(self, args: List[str], opts: argparse.Namespace) -> None:
if opts.verbose:
versions = scrapy_components_versions()
width = max(len(n) for (n, _) in versions)

15
scrapy/commands/view.py
View File

@ -1,21 +1,28 @@
import argparse
import logging
from scrapy.commands import fetch
from scrapy.http import Response, TextResponse
from scrapy.utils.response import open_in_browser
logger = logging.getLogger(__name__)
class Command(fetch.Command):
def short_desc(self):
def short_desc(self) -> str:
return "Open URL in browser, as seen by Scrapy"
def long_desc(self):
def long_desc(self) -> str:
return (
"Fetch a URL using the Scrapy downloader and show its contents in a browser"
)
def add_options(self, parser):
def add_options(self, parser: argparse.ArgumentParser) -> None:
super().add_options(parser)
parser.add_argument("--headers", help=argparse.SUPPRESS)
def _print_response(self, response, opts):
def _print_response(self, response: Response, opts: argparse.Namespace) -> None:
if not isinstance(response, TextResponse):
logger.error("Cannot view a non-text response.")
return
open_in_browser(response)

35
scrapy/core/downloader/__init__.py
View File

@ -1,8 +1,9 @@
import random
import warnings
from collections import deque
from datetime import datetime
from time import time
from typing import TYPE_CHECKING, Any, Deque, Dict, Set, Tuple, cast
from typing import TYPE_CHECKING, Any, Deque, Dict, Optional, Set, Tuple, cast
from twisted.internet import task
from twisted.internet.defer import Deferred
@ -10,6 +11,7 @@ from twisted.internet.defer import Deferred
from scrapy import Request, Spider, signals
from scrapy.core.downloader.handlers import DownloadHandlers
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Response
from scrapy.resolver import dnscache
from scrapy.settings import BaseSettings
@ -24,10 +26,18 @@ if TYPE_CHECKING:
class Slot:
"""Downloader slot"""
def __init__(self, concurrency: int, delay: float, randomize_delay: bool):
def __init__(
self,
concurrency: int,
delay: float,
randomize_delay: bool,
*,
throttle: Optional[bool] = None,
):
self.concurrency: int = concurrency
self.delay: float = delay
self.randomize_delay: bool = randomize_delay
self.throttle = throttle
self.active: Set[Request] = set()
self.queue: Deque[Tuple[Request, Deferred]] = deque()
@ -40,7 +50,7 @@ class Slot:
def download_delay(self) -> float:
if self.randomize_delay:
return random.uniform(0.5 * self.delay, 1.5 * self.delay)
return random.uniform(0.5 * self.delay, 1.5 * self.delay) # nosec
return self.delay
def close(self) -> None:
@ -52,13 +62,15 @@ class Slot:
return (
f"{cls_name}(concurrency={self.concurrency!r}, "
f"delay={self.delay:.2f}, "
f"randomize_delay={self.randomize_delay!r})"
f"randomize_delay={self.randomize_delay!r}, "
f"throttle={self.throttle!r})"
)
def __str__(self) -> str:
return (
f"<downloader.Slot concurrency={self.concurrency!r} "
f"delay={self.delay:.2f} randomize_delay={self.randomize_delay!r} "
f"throttle={self.throttle!r} "
f"len(active)={len(self.active)} len(queue)={len(self.queue)} "
f"len(transferring)={len(self.transferring)} "
f"lastseen={datetime.fromtimestamp(self.lastseen).isoformat()}>"
@ -115,7 +127,7 @@ class Downloader:
return len(self.active) >= self.total_concurrency
def _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]:
key = self._get_slot_key(request, spider)
key = self.get_slot_key(request)
if key not in self.slots:
slot_settings = self.per_slot_settings.get(key, {})
conc = (
@ -127,12 +139,13 @@ class Downloader:
slot_settings.get("delay", delay),
)
randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
new_slot = Slot(conc, delay, randomize_delay)
throttle = slot_settings.get("throttle", None)
new_slot = Slot(conc, delay, randomize_delay, throttle=throttle)
self.slots[key] = new_slot
return key, self.slots[key]
def _get_slot_key(self, request: Request, spider: Spider) -> str:
def get_slot_key(self, request: Request) -> str:
if self.DOWNLOAD_SLOT in request.meta:
return cast(str, request.meta[self.DOWNLOAD_SLOT])
@ -142,6 +155,14 @@ class Downloader:
return key
def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str:
warnings.warn(
"Use of this protected method is deprecated. Consider using its corresponding public method get_slot_key() instead.",
ScrapyDeprecationWarning,
stacklevel=2,
)
return self.get_slot_key(request)
def _enqueue_request(self, request: Request, spider: Spider) -> Deferred:
key, slot = self._get_slot(request, spider)
request.meta[self.DOWNLOAD_SLOT] = key

30
scrapy/core/downloader/contextfactory.py
View File

@ -1,3 +1,5 @@
from __future__ import annotations
import warnings
from typing import TYPE_CHECKING, Any, List, Optional
@ -19,12 +21,16 @@ from scrapy.core.downloader.tls import (
ScrapyClientTLSOptions,
openssl_methods,
)
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
from twisted.internet._sslverify import ClientTLSOptions
# typing.Self requires Python 3.11
from typing_extensions import Self
@implementer(IPolicyForHTTPS)
class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
@ -62,7 +68,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
method: int = SSL.SSLv23_METHOD,
*args: Any,
**kwargs: Any,
):
) -> Self:
tls_verbose_logging: bool = settings.getbool(
"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
)
@ -97,7 +103,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
# kept for old-style HTTP/1.0 downloader context twisted calls,
# e.g. connectSSL()
def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context:
ctx = self.getCertificateOptions().getContext()
ctx: SSL.Context = self.getCertificateOptions().getContext()
ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT
return ctx
@ -160,23 +166,23 @@ class AcceptableProtocolsContextFactory:
return options
def load_context_factory_from_settings(settings, crawler):
def load_context_factory_from_settings(
settings: BaseSettings, crawler: Crawler
) -> IPolicyForHTTPS:
ssl_method = openssl_methods[settings.get("DOWNLOADER_CLIENT_TLS_METHOD")]
context_factory_cls = load_object(settings["DOWNLOADER_CLIENTCONTEXTFACTORY"])
# try method-aware context factory
try:
context_factory = create_instance(
objcls=context_factory_cls,
settings=settings,
crawler=crawler,
context_factory = build_from_crawler(
context_factory_cls,
crawler,
method=ssl_method,
)
except TypeError:
# use context factory defaults
context_factory = create_instance(
objcls=context_factory_cls,
settings=settings,
crawler=crawler,
context_factory = build_from_crawler(
context_factory_cls,
crawler,
)
msg = (
f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept "

15
scrapy/core/downloader/handlers/__init__.py
View File

@ -9,7 +9,7 @@ from twisted.internet.defer import Deferred
from scrapy import Request, Spider, signals
from scrapy.exceptions import NotConfigured, NotSupported
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
@ -21,9 +21,9 @@ logger = logging.getLogger(__name__)
class DownloadHandlers:
def __init__(self, crawler: "Crawler"):
self._crawler: "Crawler" = crawler
self._schemes: Dict[
str, Union[str, Callable]
] = {} # stores acceptable schemes on instancing
self._schemes: Dict[str, Union[str, Callable]] = (
{}
) # stores acceptable schemes on instancing
self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes
self._notconfigured: Dict[str, str] = {} # remembers failed handlers
handlers: Dict[str, Union[str, Callable]] = without_none_values(
@ -55,10 +55,9 @@ class DownloadHandlers:
dhcls = load_object(path)
if skip_lazy and getattr(dhcls, "lazy", True):
return None
dh = create_instance(
objcls=dhcls,
settings=self._crawler.settings,
crawler=self._crawler,
dh = build_from_crawler(
dhcls,
self._crawler,
)
except NotConfigured as ex:
self._notconfigured[scheme] = str(ex)

4
scrapy/core/downloader/handlers/file.py
View File

@ -2,6 +2,8 @@ from pathlib import Path
from w3lib.url import file_uri_to_path
from scrapy import Request, Spider
from scrapy.http import Response
from scrapy.responsetypes import responsetypes
from scrapy.utils.decorators import defers
@ -10,7 +12,7 @@ class FileDownloadHandler:
lazy = False
@defers
def download_request(self, request, spider):
def download_request(self, request: Request, spider: Spider) -> Response:
filepath = file_uri_to_path(request.url)
body = Path(filepath).read_bytes()
respcls = responsetypes.from_args(filename=filepath, body=body)

58
scrapy/core/downloader/handlers/ftp.py
View File

@ -28,34 +28,46 @@ In case of status 200 request, response.headers will come with two keys:
'Size' - with size of the downloaded data
"""
from __future__ import annotations
import re
from io import BytesIO
from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional
from urllib.parse import unquote
from twisted.internet.defer import Deferred
from twisted.internet.protocol import ClientCreator, Protocol
from twisted.protocols.ftp import CommandFailed, FTPClient
from twisted.python.failure import Failure
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.responsetypes import responsetypes
from scrapy.settings import BaseSettings
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class ReceivedDataProtocol(Protocol):
def __init__(self, filename=None):
self.__filename = filename
self.body = open(filename, "wb") if filename else BytesIO()
self.size = 0
def __init__(self, filename: Optional[str] = None):
self.__filename: Optional[str] = filename
self.body: BinaryIO = open(filename, "wb") if filename else BytesIO()
self.size: int = 0
def dataReceived(self, data):
def dataReceived(self, data: bytes) -> None:
self.body.write(data)
self.size += len(data)
@property
def filename(self):
def filename(self) -> Optional[str]:
return self.__filename
def close(self):
def close(self) -> None:
self.body.close() if self.filename else self.body.seek(0)
@ -65,21 +77,21 @@ _CODE_RE = re.compile(r"\d+")
class FTPDownloadHandler:
lazy = False
CODE_MAPPING = {
CODE_MAPPING: Dict[str, int] = {
"550": 404,
"default": 503,
}
def __init__(self, settings):
def __init__(self, settings: BaseSettings):
self.default_user = settings["FTP_USER"]
self.default_password = settings["FTP_PASSWORD"]
self.passive_mode = settings["FTP_PASSIVE_MODE"]
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings)
def download_request(self, request, spider):
def download_request(self, request: Request, spider: Spider) -> Deferred:
from twisted.internet import reactor
parsed_url = urlparse_cached(request)
@ -91,28 +103,29 @@ class FTPDownloadHandler:
creator = ClientCreator(
reactor, FTPClient, user, password, passive=passive_mode
)
dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21)
dfd: Deferred = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21)
return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path))
def gotClient(self, client, request, filepath):
def gotClient(self, client: FTPClient, request: Request, filepath: str) -> Deferred:
self.client = client
protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename"))
return client.retrieveFile(filepath, protocol).addCallbacks(
callback=self._build_response,
callbackArgs=(request, protocol),
errback=self._failed,
errbackArgs=(request,),
)
d = client.retrieveFile(filepath, protocol)
d.addCallback(self._build_response, request, protocol)
d.addErrback(self._failed, request)
return d
def _build_response(self, result, request, protocol):
def _build_response(
self, result: Any, request: Request, protocol: ReceivedDataProtocol
) -> Response:
self.result = result
protocol.close()
headers = {"local filename": protocol.filename or "", "size": protocol.size}
body = to_bytes(protocol.filename or protocol.body.read())
respcls = responsetypes.from_args(url=request.url, body=body)
return respcls(url=request.url, status=200, body=body, headers=headers)
# hints for Headers-related types may need to be fixed to not use AnyStr
return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type]
def _failed(self, result, request):
def _failed(self, result: Failure, request: Request) -> Response:
message = result.getErrorMessage()
if result.type == CommandFailed:
m = _CODE_RE.search(message)
@ -122,4 +135,5 @@ class FTPDownloadHandler:
return Response(
url=request.url, status=httpcode, body=to_bytes(message)
)
assert result.type
raise result.type(result.value)

44
scrapy/core/downloader/handlers/http10.py
View File

@ -1,39 +1,57 @@
"""Download handlers for http and https schemes
"""
from scrapy.utils.misc import create_instance, load_object
from __future__ import annotations
from typing import TYPE_CHECKING, Type
from twisted.internet.defer import Deferred
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory
class HTTP10DownloadHandler:
lazy = False
def __init__(self, settings, crawler=None):
self.HTTPClientFactory = load_object(settings["DOWNLOADER_HTTPCLIENTFACTORY"])
self.ClientContextFactory = load_object(
def __init__(self, settings: BaseSettings, crawler: Crawler):
self.HTTPClientFactory: Type[ScrapyHTTPClientFactory] = load_object(
settings["DOWNLOADER_HTTPCLIENTFACTORY"]
)
self.ClientContextFactory: Type[ScrapyClientContextFactory] = load_object(
settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]
)
self._settings = settings
self._crawler = crawler
self._settings: BaseSettings = settings
self._crawler: Crawler = crawler
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings, crawler)
def download_request(self, request, spider):
def download_request(self, request: Request, spider: Spider) -> Deferred:
"""Return a deferred for the HTTP download"""
factory = self.HTTPClientFactory(request)
self._connect(factory)
return factory.deferred
def _connect(self, factory):
def _connect(self, factory: ScrapyHTTPClientFactory) -> Deferred:
from twisted.internet import reactor
host, port = to_unicode(factory.host), factory.port
if factory.scheme == b"https":
client_context_factory = create_instance(
objcls=self.ClientContextFactory,
settings=self._settings,
crawler=self._crawler,
client_context_factory = build_from_crawler(
self.ClientContextFactory,
self._crawler,
)
return reactor.connectSSL(host, port, factory, client_context_factory)
return reactor.connectTCP(host, port, factory)

293
scrapy/core/downloader/handlers/http11.py
View File

@ -1,65 +1,77 @@
"""Download handlers for http and https schemes"""
from __future__ import annotations
import ipaddress
import logging
import re
from contextlib import suppress
from io import BytesIO
from time import time
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
from urllib.parse import urldefrag, urlunparse
from twisted.internet import defer, protocol, ssl
from twisted.internet import ssl
from twisted.internet.base import ReactorBase
from twisted.internet.defer import CancelledError, Deferred, succeed
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.internet.error import TimeoutError
from twisted.internet.interfaces import IConsumer
from twisted.internet.protocol import Factory, Protocol, connectionDone
from twisted.python.failure import Failure
from twisted.web.client import (
URI,
Agent,
HTTPConnectionPool,
ResponseDone,
ResponseFailed,
)
from twisted.web.client import URI, Agent, HTTPConnectionPool
from twisted.web.client import Response as TxResponse
from twisted.web.client import ResponseDone, ResponseFailed
from twisted.web.http import PotentialDataLoss, _DataLoss
from twisted.web.http_headers import Headers as TxHeaders
from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer
from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS
from zope.interface import implementer
from scrapy import signals
from scrapy import Request, Spider, signals
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
from scrapy.crawler import Crawler
from scrapy.exceptions import StopDownload
from scrapy.http import Headers
from scrapy.http import Headers, Response
from scrapy.responsetypes import responsetypes
from scrapy.settings import BaseSettings
from scrapy.utils.python import to_bytes, to_unicode
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
class HTTP11DownloadHandler:
lazy = False
def __init__(self, settings, crawler=None):
def __init__(self, settings: BaseSettings, crawler: Crawler):
self._crawler = crawler
from twisted.internet import reactor
self._pool = HTTPConnectionPool(reactor, persistent=True)
self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True)
self._pool.maxPersistentPerHost = settings.getint(
"CONCURRENT_REQUESTS_PER_DOMAIN"
)
self._pool._factory.noisy = False
self._contextFactory = load_context_factory_from_settings(settings, crawler)
self._default_maxsize = settings.getint("DOWNLOAD_MAXSIZE")
self._default_warnsize = settings.getint("DOWNLOAD_WARNSIZE")
self._fail_on_dataloss = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS")
self._disconnect_timeout = 1
self._contextFactory: IPolicyForHTTPS = load_context_factory_from_settings(
settings, crawler
)
self._default_maxsize: int = settings.getint("DOWNLOAD_MAXSIZE")
self._default_warnsize: int = settings.getint("DOWNLOAD_WARNSIZE")
self._fail_on_dataloss: bool = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS")
self._disconnect_timeout: int = 1
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings, crawler)
def download_request(self, request, spider):
def download_request(self, request: Request, spider: Spider) -> Deferred:
"""Return a deferred for the HTTP download"""
agent = ScrapyAgent(
contextFactory=self._contextFactory,
@ -71,10 +83,10 @@ class HTTP11DownloadHandler:
)
return agent.download_request(request)
def close(self):
def close(self) -> Deferred:
from twisted.internet import reactor
d = self._pool.closeCachedConnections()
d: Deferred = self._pool.closeCachedConnections()
# closeCachedConnections will hang on network or server issues, so
# we'll manually timeout the deferred.
#
@ -85,7 +97,7 @@ class HTTP11DownloadHandler:
# issue a callback after `_disconnect_timeout` seconds.
delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, [])
def cancel_delayed_call(result):
def cancel_delayed_call(result: Any) -> Any:
if delayed_call.active():
delayed_call.cancel()
return result
@ -115,39 +127,41 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def __init__(
self,
reactor,
host,
port,
proxyConf,
contextFactory,
timeout=30,
bindAddress=None,
reactor: ReactorBase,
host: str,
port: int,
proxyConf: Tuple[str, int, Optional[bytes]],
contextFactory: IPolicyForHTTPS,
timeout: float = 30,
bindAddress: Optional[Tuple[str, int]] = None,
):
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
self._tunnelReadyDeferred = defer.Deferred()
self._tunneledHost = host
self._tunneledPort = port
self._contextFactory = contextFactory
self._connectBuffer = bytearray()
self._tunnelReadyDeferred: Deferred = Deferred()
self._tunneledHost: str = host
self._tunneledPort: int = port
self._contextFactory: IPolicyForHTTPS = contextFactory
self._connectBuffer: bytearray = bytearray()
def requestTunnel(self, protocol):
def requestTunnel(self, protocol: Protocol) -> Protocol:
"""Asks the proxy to open a tunnel."""
assert protocol.transport
tunnelReq = tunnel_request_data(
self._tunneledHost, self._tunneledPort, self._proxyAuthHeader
)
protocol.transport.write(tunnelReq)
self._protocolDataReceived = protocol.dataReceived
protocol.dataReceived = self.processProxyResponse
protocol.dataReceived = self.processProxyResponse # type: ignore[method-assign]
self._protocol = protocol
return protocol
def processProxyResponse(self, rcvd_bytes):
def processProxyResponse(self, data: bytes) -> None:
"""Processes the response from the proxy. If the tunnel is successfully
created, notifies the client that we are ready to send requests. If not
raises a TunnelError.
"""
self._connectBuffer += rcvd_bytes
assert self._protocol.transport
self._connectBuffer += data
# make sure that enough (all) bytes are consumed
# and that we've got all HTTP headers (ending with a blank line)
# from the proxy so that we don't send those bytes to the TLS layer
@ -155,23 +169,24 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
# see https://github.com/scrapy/scrapy/issues/2491
if b"\r\n\r\n" not in self._connectBuffer:
return
self._protocol.dataReceived = self._protocolDataReceived
self._protocol.dataReceived = self._protocolDataReceived # type: ignore[method-assign]
respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)
if respm and int(respm.group("status")) == 200:
# set proper Server Name Indication extension
sslOptions = self._contextFactory.creatorForNetloc(
sslOptions = self._contextFactory.creatorForNetloc( # type: ignore[call-arg,misc]
self._tunneledHost, self._tunneledPort
)
self._protocol.transport.startTLS(sslOptions, self._protocolFactory)
self._tunnelReadyDeferred.callback(self._protocol)
else:
extra: Any
if respm:
extra = {
"status": int(respm.group("status")),
"reason": respm.group("reason").strip(),
}
else:
extra = rcvd_bytes[: self._truncatedLength]
extra = data[: self._truncatedLength]
self._tunnelReadyDeferred.errback(
TunnelError(
"Could not open CONNECT tunnel with proxy "
@ -179,11 +194,11 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
)
)
def connectFailed(self, reason):
def connectFailed(self, reason: Failure) -> None:
"""Propagates the errback to the appropriate deferred."""
self._tunnelReadyDeferred.errback(reason)
def connect(self, protocolFactory):
def connect(self, protocolFactory: Factory) -> Deferred:
self._protocolFactory = protocolFactory
connectDeferred = super().connect(protocolFactory)
connectDeferred.addCallback(self.requestTunnel)
@ -191,7 +206,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
return self._tunnelReadyDeferred
def tunnel_request_data(host, port, proxy_auth_header=None):
def tunnel_request_data(
host: str, port: int, proxy_auth_header: Optional[bytes] = None
) -> bytes:
r"""
Return binary content of a CONNECT request.
@ -222,18 +239,19 @@ class TunnelingAgent(Agent):
def __init__(
self,
reactor,
proxyConf,
contextFactory=None,
connectTimeout=None,
bindAddress=None,
pool=None,
*,
reactor: ReactorBase,
proxyConf: Tuple[str, int, Optional[bytes]],
contextFactory: IPolicyForHTTPS,
connectTimeout: Optional[float] = None,
bindAddress: Optional[bytes] = None,
pool: Optional[HTTPConnectionPool] = None,
):
super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
self._proxyConf = proxyConf
self._contextFactory = contextFactory
self._proxyConf: Tuple[str, int, Optional[bytes]] = proxyConf
self._contextFactory: IPolicyForHTTPS = contextFactory
def _getEndpoint(self, uri):
def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint:
return TunnelingTCP4ClientEndpoint(
reactor=self._reactor,
host=uri.host,
@ -245,8 +263,15 @@ class TunnelingAgent(Agent):
)
def _requestWithEndpoint(
self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath
):
self,
key: Any,
endpoint: TCP4ClientEndpoint,
method: bytes,
parsedURI: bytes,
headers: Optional[TxHeaders],
bodyProducer: Optional[IBodyProducer],
requestPath: bytes,
) -> Deferred:
# proxy host and port are required for HTTP pool `key`
# otherwise, same remote host connection request could reuse
# a cached tunneled connection to a different proxy
@ -264,7 +289,12 @@ class TunnelingAgent(Agent):
class ScrapyProxyAgent(Agent):
def __init__(
self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None
self,
reactor: ReactorBase,
proxyURI: bytes,
connectTimeout: Optional[float] = None,
bindAddress: Optional[bytes] = None,
pool: Optional[HTTPConnectionPool] = None,
):
super().__init__(
reactor=reactor,
@ -272,9 +302,15 @@ class ScrapyProxyAgent(Agent):
bindAddress=bindAddress,
pool=pool,
)
self._proxyURI = URI.fromBytes(proxyURI)
self._proxyURI: URI = URI.fromBytes(proxyURI)
def request(self, method, uri, headers=None, bodyProducer=None):
def request(
self,
method: bytes,
uri: bytes,
headers: Optional[TxHeaders] = None,
bodyProducer: Optional[IBodyProducer] = None,
) -> Deferred:
"""
Issue a new request via the configured proxy.
"""
@ -298,26 +334,27 @@ class ScrapyAgent:
def __init__(
self,
contextFactory=None,
connectTimeout=10,
bindAddress=None,
pool=None,
maxsize=0,
warnsize=0,
fail_on_dataloss=True,
crawler=None,
*,
contextFactory: IPolicyForHTTPS,
connectTimeout: float = 10,
bindAddress: Optional[bytes] = None,
pool: Optional[HTTPConnectionPool] = None,
maxsize: int = 0,
warnsize: int = 0,
fail_on_dataloss: bool = True,
crawler: Crawler,
):
self._contextFactory = contextFactory
self._connectTimeout = connectTimeout
self._bindAddress = bindAddress
self._pool = pool
self._maxsize = maxsize
self._warnsize = warnsize
self._fail_on_dataloss = fail_on_dataloss
self._txresponse = None
self._crawler = crawler
self._contextFactory: IPolicyForHTTPS = contextFactory
self._connectTimeout: float = connectTimeout
self._bindAddress: Optional[bytes] = bindAddress
self._pool: Optional[HTTPConnectionPool] = pool
self._maxsize: int = maxsize
self._warnsize: int = warnsize
self._fail_on_dataloss: bool = fail_on_dataloss
self._txresponse: Optional[TxResponse] = None
self._crawler: Crawler = crawler
def _get_agent(self, request, timeout):
def _get_agent(self, request: Request, timeout: float) -> Agent:
from twisted.internet import reactor
bindaddress = request.meta.get("bindaddress") or self._bindAddress
@ -325,10 +362,10 @@ class ScrapyAgent:
if proxy:
proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy)
scheme = _parse(request.url)[0]
proxyHost = to_unicode(proxyHost)
proxyHost_str = to_unicode(proxyHost)
if scheme == b"https":
proxyAuth = request.headers.get(b"Proxy-Authorization", None)
proxyConf = (proxyHost, proxyPort, proxyAuth)
proxyConf = (proxyHost_str, proxyPort, proxyAuth)
return self._TunnelingAgent(
reactor=reactor,
proxyConf=proxyConf,
@ -338,7 +375,9 @@ class ScrapyAgent:
pool=self._pool,
)
proxyScheme = proxyScheme or b"http"
proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, "", "", ""))
proxyURI = urlunparse(
(proxyScheme, proxyNetloc, proxyParams, b"", b"", b"")
)
return self._ProxyAgent(
reactor=reactor,
proxyURI=to_bytes(proxyURI, encoding="ascii"),
@ -355,7 +394,7 @@ class ScrapyAgent:
pool=self._pool,
)
def download_request(self, request):
def download_request(self, request: Request) -> Deferred:
from twisted.internet import reactor
timeout = request.meta.get("download_timeout") or self._connectTimeout
@ -372,7 +411,7 @@ class ScrapyAgent:
else:
bodyproducer = None
start_time = time()
d = agent.request(
d: Deferred = agent.request(
method, to_bytes(url, encoding="ascii"), headers, bodyproducer
)
# set download latency
@ -385,7 +424,9 @@ class ScrapyAgent:
d.addBoth(self._cb_timeout, request, url, timeout)
return d
def _cb_timeout(self, result, request, url, timeout):
def _cb_timeout(
self, result: Any, request: Request, url: str, timeout: float
) -> Any:
if self._timeout_cl.active():
self._timeout_cl.cancel()
return result
@ -396,19 +437,21 @@ class ScrapyAgent:
raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
def _cb_latency(self, result, request, start_time):
def _cb_latency(self, result: Any, request: Request, start_time: float) -> Any:
request.meta["download_latency"] = time() - start_time
return result
@staticmethod
def _headers_from_twisted_response(response):
def _headers_from_twisted_response(response: TxResponse) -> Headers:
headers = Headers()
if response.length != UNKNOWN_LENGTH:
headers[b"Content-Length"] = str(response.length).encode()
headers.update(response.headers.getAllRawHeaders())
return headers
def _cb_bodyready(self, txresponse, request):
def _cb_bodyready(
self, txresponse: TxResponse, request: Request
) -> Union[Dict[str, Any], Deferred]:
headers_received_result = self._crawler.signals.send_catch_log(
signal=signals.headers_received,
headers=self._headers_from_twisted_response(txresponse),
@ -464,7 +507,7 @@ class ScrapyAgent:
logger.warning(warning_msg, warning_args)
txresponse._transport.loseConnection()
raise defer.CancelledError(warning_msg % warning_args)
raise CancelledError(warning_msg % warning_args)
if warnsize and expected_size > warnsize:
logger.warning(
@ -473,11 +516,11 @@ class ScrapyAgent:
{"size": expected_size, "warnsize": warnsize, "request": request},
)
def _cancel(_):
def _cancel(_: Any) -> None:
# Abort connection immediately.
txresponse._transport._producer.abortConnection()
d = defer.Deferred(_cancel)
d: Deferred = Deferred(_cancel)
txresponse.deliverBody(
_ResponseReader(
finished=d,
@ -495,7 +538,9 @@ class ScrapyAgent:
return d
def _cb_bodydone(self, result, request, url):
def _cb_bodydone(
self, result: Dict[str, Any], request: Request, url: str
) -> Union[Response, Failure]:
headers = self._headers_from_twisted_response(result["txresponse"])
respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
try:
@ -515,53 +560,57 @@ class ScrapyAgent:
)
if result.get("failure"):
result["failure"].value.response = response
return result["failure"]
return cast(Failure, result["failure"])
return response
@implementer(IBodyProducer)
class _RequestBodyProducer:
def __init__(self, body):
def __init__(self, body: bytes):
self.body = body
self.length = len(body)
def startProducing(self, consumer):
def startProducing(self, consumer: IConsumer) -> Deferred:
consumer.write(self.body)
return defer.succeed(None)
return succeed(None)
def pauseProducing(self):
def pauseProducing(self) -> None:
pass
def stopProducing(self):
def stopProducing(self) -> None:
pass
class _ResponseReader(protocol.Protocol):
class _ResponseReader(Protocol):
def __init__(
self,
finished,
txresponse,
request,
maxsize,
warnsize,
fail_on_dataloss,
crawler,
finished: Deferred,
txresponse: TxResponse,
request: Request,
maxsize: int,
warnsize: int,
fail_on_dataloss: bool,
crawler: Crawler,
):
self._finished = finished
self._txresponse = txresponse
self._request = request
self._bodybuf = BytesIO()
self._maxsize = maxsize
self._warnsize = warnsize
self._fail_on_dataloss = fail_on_dataloss
self._fail_on_dataloss_warned = False
self._reached_warnsize = False
self._bytes_received = 0
self._certificate = None
self._ip_address = None
self._crawler = crawler
self._finished: Deferred = finished
self._txresponse: TxResponse = txresponse
self._request: Request = request
self._bodybuf: BytesIO = BytesIO()
self._maxsize: int = maxsize
self._warnsize: int = warnsize
self._fail_on_dataloss: bool = fail_on_dataloss
self._fail_on_dataloss_warned: bool = False
self._reached_warnsize: bool = False
self._bytes_received: int = 0
self._certificate: Optional[ssl.Certificate] = None
self._ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None] = (
None
)
self._crawler: Crawler = crawler
def _finish_response(self, flags=None, failure=None):
def _finish_response(
self, flags: Optional[List[str]] = None, failure: Optional[Failure] = None
) -> None:
self._finished.callback(
{
"txresponse": self._txresponse,
@ -573,7 +622,8 @@ class _ResponseReader(protocol.Protocol):
}
)
def connectionMade(self):
def connectionMade(self) -> None:
assert self.transport
if self._certificate is None:
with suppress(AttributeError):
self._certificate = ssl.Certificate(
@ -585,11 +635,12 @@ class _ResponseReader(protocol.Protocol):
self.transport._producer.getPeer().host
)
def dataReceived(self, bodyBytes):
def dataReceived(self, bodyBytes: bytes) -> None:
# This maybe called several times after cancel was called with buffered data.
if self._finished.called:
return
assert self.transport
self._bodybuf.write(bodyBytes)
self._bytes_received += len(bodyBytes)
@ -636,7 +687,7 @@ class _ResponseReader(protocol.Protocol):
{"warnsize": self._warnsize, "request": self._request},
)
def connectionLost(self, reason):
def connectionLost(self, reason: Failure = connectionDone) -> None:
if self._finished.called:
return

19
scrapy/core/downloader/handlers/http2.py
View File

@ -1,11 +1,14 @@
from __future__ import annotations
from time import time
from typing import Optional, Type, TypeVar
from typing import TYPE_CHECKING, Optional
from urllib.parse import urldefrag
from twisted.internet.base import DelayedCall
from twisted.internet.defer import Deferred
from twisted.internet.error import TimeoutError
from twisted.web.client import URI
from twisted.web.iweb import IPolicyForHTTPS
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
@ -16,13 +19,13 @@ from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.utils.python import to_bytes
H2DownloadHandlerOrSubclass = TypeVar(
"H2DownloadHandlerOrSubclass", bound="H2DownloadHandler"
)
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class H2DownloadHandler:
def __init__(self, settings: Settings, crawler: Optional[Crawler] = None):
def __init__(self, settings: Settings, crawler: Crawler):
self._crawler = crawler
from twisted.internet import reactor
@ -31,9 +34,7 @@ class H2DownloadHandler:
self._context_factory = load_context_factory_from_settings(settings, crawler)
@classmethod
def from_crawler(
cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler
) -> H2DownloadHandlerOrSubclass:
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings, crawler)
def download_request(self, request: Request, spider: Spider) -> Deferred:
@ -54,7 +55,7 @@ class ScrapyH2Agent:
def __init__(
self,
context_factory,
context_factory: IPolicyForHTTPS,
pool: H2ConnectionPool,
connect_timeout: int = 10,
bind_address: Optional[bytes] = None,

43
scrapy/core/downloader/handlers/s3.py
View File

@ -1,21 +1,34 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Optional, Type
from twisted.internet.defer import Deferred
from scrapy import Request, Spider
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.settings import BaseSettings
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import create_instance
from scrapy.utils.misc import build_from_crawler
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class S3DownloadHandler:
def __init__(
self,
settings,
settings: BaseSettings,
*,
crawler=None,
aws_access_key_id=None,
aws_secret_access_key=None,
aws_session_token=None,
httpdownloadhandler=HTTPDownloadHandler,
**kw,
crawler: Crawler,
aws_access_key_id: Optional[str] = None,
aws_secret_access_key: Optional[str] = None,
aws_session_token: Optional[str] = None,
httpdownloadhandler: Type[HTTPDownloadHandler] = HTTPDownloadHandler,
**kw: Any,
):
if not is_botocore_available():
raise NotConfigured("missing botocore library")
@ -43,6 +56,8 @@ class S3DownloadHandler:
if kw:
raise TypeError(f"Unexpected keyword arguments: {kw}")
if not self.anon:
assert aws_access_key_id is not None
assert aws_secret_access_key is not None
SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"]
self._signer = SignerCls(
botocore.credentials.Credentials(
@ -50,18 +65,17 @@ class S3DownloadHandler:
)
)
_http_handler = create_instance(
objcls=httpdownloadhandler,
settings=settings,
crawler=crawler,
_http_handler = build_from_crawler(
httpdownloadhandler,
crawler,
)
self._download_http = _http_handler.download_request
@classmethod
def from_crawler(cls, crawler, **kwargs):
def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self:
return cls(crawler.settings, crawler=crawler, **kwargs)
def download_request(self, request, spider):
def download_request(self, request: Request, spider: Spider) -> Deferred:
p = urlparse_cached(request)
scheme = "https" if request.meta.get("is_secure") else "http"
bucket = p.hostname
@ -78,6 +92,7 @@ class S3DownloadHandler:
headers=request.headers.to_unicode_dict(),
data=request.body,
)
assert self._signer
self._signer.add_auth(awsrequest)
request = request.replace(url=url, headers=awsrequest.headers.items())
return self._download_http(request, spider)

1
scrapy/core/downloader/middleware.py
View File

@ -3,6 +3,7 @@ Downloader Middleware manager
See documentation in docs/topics/downloader-middleware.rst
"""
from typing import Any, Callable, Generator, List, Union, cast
from twisted.internet.defer import Deferred, inlineCallbacks

29
scrapy/core/engine.py
View File

@ -4,6 +4,7 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spider.
For more information see docs/topics/architecture.rst
"""
import logging
from time import time
from typing import (
@ -27,14 +28,15 @@ from twisted.python.failure import Failure
from scrapy import signals
from scrapy.core.downloader import Downloader
from scrapy.core.scraper import Scraper
from scrapy.exceptions import CloseSpider, DontCloseSpider
from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest
from scrapy.http import Request, Response
from scrapy.logformatter import LogFormatter
from scrapy.settings import BaseSettings, Settings
from scrapy.signalmanager import SignalManager
from scrapy.spiders import Spider
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import global_object_name
from scrapy.utils.reactor import CallLaterOnce
if TYPE_CHECKING:
@ -291,9 +293,19 @@ class ExecutionEngine:
self.slot.nextcall.schedule() # type: ignore[union-attr]
def _schedule_request(self, request: Request, spider: Spider) -> None:
self.signals.send_catch_log(
signals.request_scheduled, request=request, spider=spider
request_scheduled_result = self.signals.send_catch_log(
signals.request_scheduled,
request=request,
spider=spider,
dont_log=IgnoreRequest,
)
for handler, result in request_scheduled_result:
if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest):
logger.debug(
f"Signal handler {global_object_name(handler)} dropped "
f"request {request} before it reached the scheduler."
)
return
if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr]
self.signals.send_catch_log(
signals.request_dropped, request=request, spider=spider
@ -346,7 +358,7 @@ class ExecutionEngine:
assert self.spider is not None
dwld = self.downloader.fetch(request, self.spider)
dwld.addCallbacks(_on_success)
dwld.addCallback(_on_success)
dwld.addBoth(_on_complete)
return dwld
@ -358,16 +370,15 @@ class ExecutionEngine:
raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
logger.info("Spider opened", extra={"spider": spider})
nextcall = CallLaterOnce(self._next_request)
scheduler = create_instance(
self.scheduler_cls, settings=None, crawler=self.crawler
)
scheduler = build_from_crawler(self.scheduler_cls, self.crawler)
start_requests = yield self.scraper.spidermw.process_start_requests(
start_requests, spider
)
self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
self.spider = spider
if hasattr(scheduler, "open"):
yield scheduler.open(spider)
if d := scheduler.open(spider):
yield d
yield self.scraper.open_spider(spider)
assert self.crawler.stats
self.crawler.stats.open_spider(spider)

18
scrapy/core/http2/stream.py
View File

@ -2,7 +2,6 @@ import logging
from enum import Enum
from io import BytesIO
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
from urllib.parse import urlparse
from h2.errors import ErrorCodes
from h2.exceptions import H2Error, ProtocolError, StreamClosedError
@ -15,6 +14,7 @@ from twisted.web.client import ResponseFailed
from scrapy.http import Request
from scrapy.http.headers import Headers
from scrapy.responsetypes import responsetypes
from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
from scrapy.core.http2.protocol import H2ClientProtocol
@ -111,17 +111,17 @@ class Stream:
# Metadata of an HTTP/2 connection stream
# initialized when stream is instantiated
self.metadata: Dict = {
"request_content_length": 0
if self._request.body is None
else len(self._request.body),
"request_content_length": (
0 if self._request.body is None else len(self._request.body)
),
# Flag to keep track whether the stream has initiated the request
"request_sent": False,
# Flag to track whether we have logged about exceeding download warnsize
"reached_warnsize": False,
# Each time we send a data frame, we will decrease value by the amount send.
"remaining_content_length": 0
if self._request.body is None
else len(self._request.body),
"remaining_content_length": (
0 if self._request.body is None else len(self._request.body)
),
# Flag to keep track whether client (self) have closed this stream
"stream_closed_local": False,
# Flag to keep track whether the server has closed the stream
@ -185,7 +185,7 @@ class Stream:
def check_request_url(self) -> bool:
# Make sure that we are sending the request to the correct URL
url = urlparse(self._request.url)
url = urlparse_cached(self._request)
return (
url.netloc == str(self._protocol.metadata["uri"].host, "utf-8")
or url.netloc == str(self._protocol.metadata["uri"].netloc, "utf-8")
@ -194,7 +194,7 @@ class Stream:
)
def _get_request_headers(self) -> List[Tuple[str, str]]:
url = urlparse(self._request.url)
url = urlparse_cached(self._request)
path = url.path
if url.query:

51
scrapy/core/scheduler.py
View File

@ -4,19 +4,23 @@ import json
import logging
from abc import abstractmethod
from pathlib import Path
from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, cast
from typing import TYPE_CHECKING, Any, Optional, Type, cast
from twisted.internet.defer import Deferred
from scrapy.crawler import Crawler
from scrapy.dupefilters import BaseDupeFilter
from scrapy.http.request import Request
from scrapy.pqueues import ScrapyPriorityQueue
from scrapy.spiders import Spider
from scrapy.statscollectors import StatsCollector
from scrapy.utils.job import job_dir
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
# requires queuelib >= 1.6.2
from queuelib.queue import BaseQueue
# typing.Self requires Python 3.11
from typing_extensions import Self
@ -121,9 +125,6 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
raise NotImplementedError()
SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler")
class Scheduler(BaseScheduler):
"""
Default Scrapy scheduler. This implementation also handles duplication
@ -179,30 +180,30 @@ class Scheduler(BaseScheduler):
self,
dupefilter: BaseDupeFilter,
jobdir: Optional[str] = None,
dqclass=None,
mqclass=None,
dqclass: Optional[Type[BaseQueue]] = None,
mqclass: Optional[Type[BaseQueue]] = None,
logunser: bool = False,
stats: Optional[StatsCollector] = None,
pqclass=None,
pqclass: Optional[Type[ScrapyPriorityQueue]] = None,
crawler: Optional[Crawler] = None,
):
self.df: BaseDupeFilter = dupefilter
self.dqdir: Optional[str] = self._dqdir(jobdir)
self.pqclass = pqclass
self.dqclass = dqclass
self.mqclass = mqclass
self.pqclass: Optional[Type[ScrapyPriorityQueue]] = pqclass
self.dqclass: Optional[Type[BaseQueue]] = dqclass
self.mqclass: Optional[Type[BaseQueue]] = mqclass
self.logunser: bool = logunser
self.stats: Optional[StatsCollector] = stats
self.crawler: Optional[Crawler] = crawler
@classmethod
def from_crawler(cls: Type[SchedulerTV], crawler: Crawler) -> SchedulerTV:
def from_crawler(cls, crawler: Crawler) -> Self:
"""
Factory method, initializes the scheduler with arguments taken from the crawl settings
"""
dupefilter_cls = load_object(crawler.settings["DUPEFILTER_CLASS"])
return cls(
dupefilter=create_instance(dupefilter_cls, crawler.settings, crawler),
dupefilter=build_from_crawler(dupefilter_cls, crawler),
jobdir=job_dir(crawler.settings),
dqclass=load_object(crawler.settings["SCHEDULER_DISK_QUEUE"]),
mqclass=load_object(crawler.settings["SCHEDULER_MEMORY_QUEUE"]),
@ -221,9 +222,9 @@ class Scheduler(BaseScheduler):
(2) initialize the disk queue if the ``jobdir`` attribute is a valid directory
(3) return the result of the dupefilter's ``open`` method
"""
self.spider = spider
self.mqs = self._mq()
self.dqs = self._dq() if self.dqdir else None
self.spider: Spider = spider
self.mqs: ScrapyPriorityQueue = self._mq()
self.dqs: Optional[ScrapyPriorityQueue] = self._dq() if self.dqdir else None
return self.df.open()
def close(self, reason: str) -> Optional[Deferred]:
@ -320,24 +321,26 @@ class Scheduler(BaseScheduler):
return self.dqs.pop()
return None
def _mq(self):
def _mq(self) -> ScrapyPriorityQueue:
"""Create a new priority queue instance, with in-memory storage"""
return create_instance(
assert self.crawler
assert self.pqclass
return build_from_crawler(
self.pqclass,
settings=None,
crawler=self.crawler,
self.crawler,
downstream_queue_cls=self.mqclass,
key="",
)
def _dq(self):
def _dq(self) -> ScrapyPriorityQueue:
"""Create a new priority queue instance, with disk storage"""
assert self.crawler
assert self.dqdir
assert self.pqclass
state = self._read_dqs_state(self.dqdir)
q = create_instance(
q = build_from_crawler(
self.pqclass,
settings=None,
crawler=self.crawler,
self.crawler,
downstream_queue_cls=self.dqclass,
key=self.dqdir,
startprios=state,

11
scrapy/core/scraper.py
View File

@ -1,5 +1,6 @@
"""This module implements the Scraper component which parses responses and
extracts information from them"""
from __future__ import annotations
import logging
@ -7,7 +8,6 @@ from collections import deque
from typing import (
TYPE_CHECKING,
Any,
AsyncGenerator,
AsyncIterable,
Deque,
Generator,
@ -17,6 +17,7 @@ from typing import (
Tuple,
Type,
Union,
cast,
)
from itemadapter import is_item
@ -183,7 +184,9 @@ class Scraper:
result, request, spider
) # returns spider's processed output
dfd.addErrback(self.handle_spider_error, request, result, spider)
dfd.addCallback(self.handle_spider_output, request, result, spider)
dfd.addCallback(
self.handle_spider_output, request, cast(Response, result), spider
)
return dfd
def _scrape2(
@ -255,12 +258,12 @@ class Scraper:
self,
result: Union[Iterable, AsyncIterable],
request: Request,
response: Union[Response, Failure],
response: Response,
spider: Spider,
) -> Deferred:
if not result:
return defer_succeed(None)
it: Union[Generator, AsyncGenerator]
it: Union[Iterable, AsyncIterable]
if isinstance(result, AsyncIterable):
it = aiter_errback(
result, self.handle_spider_error, request, response, spider

10
scrapy/core/spidermw.py
View File

@ -3,6 +3,7 @@ Spider Middleware manager
See documentation in docs/topics/spider-middleware.rst
"""
import logging
from inspect import isasyncgenfunction, iscoroutine
from itertools import islice
@ -103,8 +104,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
) -> Union[Generator, AsyncGenerator]:
def process_sync(iterable: Iterable) -> Generator:
try:
for r in iterable:
yield r
yield from iterable
except Exception as ex:
exception_result = self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index
@ -303,10 +303,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
dfd = mustbe_deferred(
self._process_spider_input, scrape_func, response, request, spider
)
dfd.addCallbacks(
callback=deferred_f_from_coro_f(process_callback_output),
errback=process_spider_exception,
)
dfd.addCallback(deferred_f_from_coro_f(process_callback_output))
dfd.addErrback(process_spider_exception)
return dfd
def process_start_requests(

53
scrapy/crawler.py
View File

@ -39,7 +39,7 @@ from scrapy.utils.log import (
log_reactor_info,
log_scrapy_info,
)
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
from scrapy.utils.reactor import (
install_reactor,
@ -109,10 +109,9 @@ class Crawler:
lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
self.logformatter = lf_cls.from_crawler(self)
self.request_fingerprinter = create_instance(
self.request_fingerprinter = build_from_crawler(
load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),
settings=self.settings,
crawler=self,
self,
)
reactor_class: str = self.settings["TWISTED_REACTOR"]
@ -179,6 +178,48 @@ class Crawler:
assert self.engine
yield maybeDeferred(self.engine.stop)
@staticmethod
def _get_component(component_class, components):
for component in components:
if isinstance(component, component_class):
return component
return None
def get_addon(self, cls):
return self._get_component(cls, self.addons.addons)
def get_downloader_middleware(self, cls):
if not self.engine:
raise RuntimeError(
"Crawler.get_downloader_middleware() can only be called after "
"the crawl engine has been created."
)
return self._get_component(cls, self.engine.downloader.middleware.middlewares)
def get_extension(self, cls):
if not self.extensions:
raise RuntimeError(
"Crawler.get_extension() can only be called after the "
"extension manager has been created."
)
return self._get_component(cls, self.extensions.middlewares)
def get_item_pipeline(self, cls):
if not self.engine:
raise RuntimeError(
"Crawler.get_item_pipeline() can only be called after the "
"crawl engine has been created."
)
return self._get_component(cls, self.engine.scraper.itemproc.middlewares)
def get_spider_middleware(self, cls):
if not self.engine:
raise RuntimeError(
"Crawler.get_spider_middleware() can only be called after the "
"crawl engine has been created."
)
return self._get_component(cls, self.engine.scraper.spidermw.middlewares)
class CrawlerRunner:
"""
@ -404,7 +445,9 @@ class CrawlerProcess(CrawlerRunner):
d.addBoth(self._stop_reactor)
resolver_class = load_object(self.settings["DNS_RESOLVER"])
resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
# We pass self, which is CrawlerProcess, instead of Crawler here,
# which works because the default resolvers only use crawler.settings.
resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[arg-type]
resolver.install_on_reactor()
tp = reactor.getThreadPool()
tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))

14
scrapy/downloadermiddlewares/cookies.py
View File

@ -33,6 +33,7 @@ logger = logging.getLogger(__name__)
_split_domain = TLDExtract(include_psl_private_domains=True)
_UNSET = object()
def _is_public_domain(domain: str) -> bool:
@ -133,6 +134,7 @@ class CookiesMiddleware:
Decode from bytes if necessary.
"""
decoded = {}
flags = set()
for key in ("name", "value", "path", "domain"):
if cookie.get(key) is None:
if key in ("name", "value"):
@ -152,10 +154,16 @@ class CookiesMiddleware:
cookie,
)
decoded[key] = cookie[key].decode("latin1", errors="replace")
for flag in ("secure",):
value = cookie.get(flag, _UNSET)
if value is _UNSET or not value:
continue
flags.add(flag)
cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}"
for key, value in decoded.items(): # path, domain
cookie_str += f"; {key.capitalize()}={value}"
for flag in flags: # secure
cookie_str += f"; {flag.capitalize()}"
return cookie_str
def _get_request_cookies(
@ -168,9 +176,11 @@ class CookiesMiddleware:
return []
cookies: Iterable[Dict[str, Any]]
if isinstance(request.cookies, dict):
cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items())
else:
cookies = request.cookies
for cookie in cookies:
cookie.setdefault("secure", urlparse_cached(request).scheme == "https")
formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
response = Response(request.url, headers={"Set-Cookie": formatted})
return jar.make_cookies(response, request)

1
scrapy/downloadermiddlewares/defaultheaders.py
View File

@ -3,6 +3,7 @@ DefaultHeaders downloader middleware
See documentation in docs/topics/downloader-middleware.rst
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Iterable, Tuple, Union

1
scrapy/downloadermiddlewares/downloadtimeout.py
View File

@ -3,6 +3,7 @@ Download timeout middleware
See documentation in docs/topics/downloader-middleware.rst
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Union

140
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -1,50 +1,96 @@
from __future__ import annotations
import io
import zlib
import warnings
from itertools import chain
from logging import getLogger
from typing import TYPE_CHECKING, List, Optional, Union
from scrapy import Request, Spider
from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.statscollectors import StatsCollector
from scrapy.utils._compression import (
_DecompressionMaxSizeExceeded,
_inflate,
_unbrotli,
_unzstd,
)
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.gz import gunzip
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = getLogger(__name__)
ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]
try:
import brotli
ACCEPTED_ENCODINGS.append(b"br")
try:
import brotli # noqa: F401
except ImportError:
import brotlicffi # noqa: F401
except ImportError:
pass
else:
ACCEPTED_ENCODINGS.append(b"br")
try:
import zstandard
ACCEPTED_ENCODINGS.append(b"zstd")
import zstandard # noqa: F401
except ImportError:
pass
else:
ACCEPTED_ENCODINGS.append(b"zstd")
class HttpCompressionMiddleware:
"""This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites"""
def __init__(self, stats: Optional[StatsCollector] = None):
self.stats = stats
def __init__(
self,
stats: Optional[StatsCollector] = None,
*,
crawler: Optional[Crawler] = None,
):
if not crawler:
self.stats = stats
self._max_size = 1073741824
self._warn_size = 33554432
return
self.stats = crawler.stats
self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
self._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")
crawler.signals.connect(self.open_spider, signals.spider_opened)
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("COMPRESSION_ENABLED"):
raise NotConfigured
return cls(stats=crawler.stats)
try:
return cls(crawler=crawler)
except TypeError:
warnings.warn(
"HttpCompressionMiddleware subclasses must either modify "
"their '__init__' method to support a 'crawler' parameter or "
"reimplement their 'from_crawler' method.",
ScrapyDeprecationWarning,
)
mw = cls()
mw.stats = crawler.stats
mw._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
mw._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")
crawler.signals.connect(mw.open_spider, signals.spider_opened)
return mw
def open_spider(self, spider):
if hasattr(spider, "download_maxsize"):
self._max_size = spider.download_maxsize
if hasattr(spider, "download_warnsize"):
self._warn_size = spider.download_warnsize
def process_request(
self, request: Request, spider: Spider
@ -60,8 +106,26 @@ class HttpCompressionMiddleware:
if isinstance(response, Response):
content_encoding = response.headers.getlist("Content-Encoding")
if content_encoding:
encoding = content_encoding.pop()
decoded_body = self._decode(response.body, encoding.lower())
max_size = request.meta.get("download_maxsize", self._max_size)
warn_size = request.meta.get("download_warnsize", self._warn_size)
try:
decoded_body, content_encoding = self._handle_encoding(
response.body, content_encoding, max_size
)
except _DecompressionMaxSizeExceeded:
raise IgnoreRequest(
f"Ignored response {response} because its body "
f"({len(response.body)} B compressed) exceeded "
f"DOWNLOAD_MAXSIZE ({max_size} B) during "
f"decompression."
)
if len(response.body) < warn_size <= len(decoded_body):
logger.warning(
f"{response} body size after decompression "
f"({len(decoded_body)} B) is larger than the "
f"download warning size ({warn_size} B)."
)
response.headers["Content-Encoding"] = content_encoding
if self.stats:
self.stats.inc_value(
"httpcompression/response_bytes",
@ -74,7 +138,7 @@ class HttpCompressionMiddleware:
respcls = responsetypes.from_args(
headers=response.headers, url=response.url, body=decoded_body
)
kwargs = dict(cls=respcls, body=decoded_body)
kwargs = {"cls": respcls, "body": decoded_body}
if issubclass(respcls, TextResponse):
# force recalculating the encoding until we make sure the
# responsetypes guessing is reliable
@ -85,25 +149,35 @@ class HttpCompressionMiddleware:
return response
def _decode(self, body: bytes, encoding: bytes) -> bytes:
if encoding == b"gzip" or encoding == b"x-gzip":
body = gunzip(body)
def _handle_encoding(self, body, content_encoding, max_size):
to_decode, to_keep = self._split_encodings(content_encoding)
for encoding in to_decode:
body = self._decode(body, encoding, max_size)
return body, to_keep
def _split_encodings(self, content_encoding):
to_keep = [
encoding.strip().lower()
for encoding in chain.from_iterable(
encodings.split(b",") for encodings in content_encoding
)
]
to_decode = []
while to_keep:
encoding = to_keep.pop()
if encoding not in ACCEPTED_ENCODINGS:
to_keep.append(encoding)
return to_decode, to_keep
to_decode.append(encoding)
return to_decode, to_keep
def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes:
if encoding in {b"gzip", b"x-gzip"}:
return gunzip(body, max_size=max_size)
if encoding == b"deflate":
try:
body = zlib.decompress(body)
except zlib.error:
# ugly hack to work with raw deflate content that may
# be sent by microsoft servers. For more information, see:
# http://carsten.codimi.de/gzip.yaws/
# http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
# http://www.gzip.org/zlib/zlib_faq.html#faq38
body = zlib.decompress(body, -15)
return _inflate(body, max_size=max_size)
if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:
body = brotli.decompress(body)
return _unbrotli(body, max_size=max_size)
if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:
# Using its streaming API since its simple API could handle only cases
# where there is content size data embedded in the frame
reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body))
body = reader.read()
return _unzstd(body, max_size=max_size)
return body

19
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -60,26 +60,33 @@ class HttpProxyMiddleware:
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
creds, proxy_url = None, None
creds, proxy_url, scheme = None, None, None
if "proxy" in request.meta:
if request.meta["proxy"] is not None:
creds, proxy_url = self._get_proxy(request.meta["proxy"], "")
elif self.proxies:
parsed = urlparse_cached(request)
scheme = parsed.scheme
_scheme = parsed.scheme
if (
# 'no_proxy' is only supported by http schemes
scheme not in ("http", "https")
_scheme not in ("http", "https")
or (parsed.hostname and not proxy_bypass(parsed.hostname))
) and scheme in self.proxies:
) and _scheme in self.proxies:
scheme = _scheme
creds, proxy_url = self.proxies[scheme]
self._set_proxy_and_creds(request, proxy_url, creds)
self._set_proxy_and_creds(request, proxy_url, creds, scheme)
return None
def _set_proxy_and_creds(
self, request: Request, proxy_url: Optional[str], creds: Optional[bytes]
self,
request: Request,
proxy_url: Optional[str],
creds: Optional[bytes],
scheme: Optional[str],
) -> None:
if scheme:
request.meta["_scheme_proxy"] = True
if proxy_url:
request.meta["proxy"] = proxy_url
elif request.meta.get("proxy") is not None:

77
scrapy/downloadermiddlewares/offsite.py Normal file
View File

@ -0,0 +1,77 @@
import logging
import re
import warnings
from scrapy import signals
from scrapy.exceptions import IgnoreRequest
from scrapy.utils.httpobj import urlparse_cached
logger = logging.getLogger(__name__)
class OffsiteMiddleware:
@classmethod
def from_crawler(cls, crawler):
o = cls(crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled)
return o
def __init__(self, stats):
self.stats = stats
self.domains_seen = set()
def spider_opened(self, spider):
self.host_regex = self.get_host_regex(spider)
def request_scheduled(self, request, spider):
self.process_request(request, spider)
def process_request(self, request, spider):
if request.dont_filter or self.should_follow(request, spider):
return None
domain = urlparse_cached(request).hostname
if domain and domain not in self.domains_seen:
self.domains_seen.add(domain)
logger.debug(
"Filtered offsite request to %(domain)r: %(request)s",
{"domain": domain, "request": request},
extra={"spider": spider},
)
self.stats.inc_value("offsite/domains", spider=spider)
self.stats.inc_value("offsite/filtered", spider=spider)
raise IgnoreRequest
def should_follow(self, request, spider):
regex = self.host_regex
# hostname can be None for wrong urls (like javascript links)
host = urlparse_cached(request).hostname or ""
return bool(regex.search(host))
def get_host_regex(self, spider):
"""Override this method to implement a different offsite policy"""
allowed_domains = getattr(spider, "allowed_domains", None)
if not allowed_domains:
return re.compile("") # allow all by default
url_pattern = re.compile(r"^https?://.*$")
port_pattern = re.compile(r":\d+$")
domains = []
for domain in allowed_domains:
if domain is None:
continue
if url_pattern.match(domain):
message = (
"allowed_domains accepts only domains, not URLs. "
f"Ignoring URL entry {domain} in allowed_domains."
)
warnings.warn(message)
elif port_pattern.search(domain):
message = (
"allowed_domains accepts only domains without ports. "
f"Ignoring entry {domain} in allowed_domains."
)
warnings.warn(message)
else:
domains.append(re.escape(domain))
regex = rf'^(.*\.)?({"|".join(domains)})$'
return re.compile(regex)

64
scrapy/downloadermiddlewares/redirect.py
View File

@ -2,7 +2,7 @@ from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, List, Union, cast
from urllib.parse import urljoin, urlparse
from urllib.parse import urljoin
from w3lib.url import safe_url_string
@ -29,11 +29,49 @@ def _build_redirect_request(
**kwargs,
cookies=None,
)
if "Cookie" in redirect_request.headers:
source_request_netloc = urlparse_cached(source_request).netloc
redirect_request_netloc = urlparse_cached(redirect_request).netloc
if source_request_netloc != redirect_request_netloc:
if "_scheme_proxy" in redirect_request.meta:
source_request_scheme = urlparse_cached(source_request).scheme
redirect_request_scheme = urlparse_cached(redirect_request).scheme
if source_request_scheme != redirect_request_scheme:
redirect_request.meta.pop("_scheme_proxy")
redirect_request.meta.pop("proxy", None)
redirect_request.meta.pop("_auth_proxy", None)
redirect_request.headers.pop(b"Proxy-Authorization", None)
has_cookie_header = "Cookie" in redirect_request.headers
has_authorization_header = "Authorization" in redirect_request.headers
if has_cookie_header or has_authorization_header:
default_ports = {"http": 80, "https": 443}
parsed_source_request = urlparse_cached(source_request)
source_scheme, source_host, source_port = (
parsed_source_request.scheme,
parsed_source_request.hostname,
parsed_source_request.port
or default_ports.get(parsed_source_request.scheme),
)
parsed_redirect_request = urlparse_cached(redirect_request)
redirect_scheme, redirect_host, redirect_port = (
parsed_redirect_request.scheme,
parsed_redirect_request.hostname,
parsed_redirect_request.port
or default_ports.get(parsed_redirect_request.scheme),
)
if has_cookie_header and (
redirect_scheme not in {source_scheme, "https"}
or source_host != redirect_host
):
del redirect_request.headers["Cookie"]
# https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name
if has_authorization_header and (
source_scheme != redirect_scheme
or source_host != redirect_host
or source_port != redirect_port
):
del redirect_request.headers["Authorization"]
return redirect_request
@ -119,13 +157,15 @@ class RedirectMiddleware(BaseRedirectMiddleware):
assert response.headers["Location"] is not None
location = safe_url_string(response.headers["Location"])
if response.headers["Location"].startswith(b"//"):
request_scheme = urlparse(request.url).scheme
request_scheme = urlparse_cached(request).scheme
location = request_scheme + "://" + location.lstrip("/")
redirected_url = urljoin(request.url, location)
redirected = _build_redirect_request(request, url=redirected_url)
if urlparse_cached(redirected).scheme not in {"http", "https"}:
return response
if response.status in (301, 307, 308) or request.method == "HEAD":
redirected = _build_redirect_request(request, url=redirected_url)
return self._redirect(redirected, request, spider, response.status)
redirected = self._redirect_request_using_get(request, redirected_url)
@ -147,12 +187,16 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
request.meta.get("dont_redirect", False)
or request.method == "HEAD"
or not isinstance(response, HtmlResponse)
or urlparse_cached(request).scheme not in {"http", "https"}
):
return response
interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags)
if url and cast(float, interval) < self._maxdelay:
redirected = self._redirect_request_using_get(request, url)
if not url:
return response
redirected = self._redirect_request_using_get(request, url)
if urlparse_cached(redirected).scheme not in {"http", "https"}:
return response
if cast(float, interval) < self._maxdelay:
return self._redirect(redirected, request, spider, "meta refresh")
return response

1
scrapy/downloadermiddlewares/retry.py
View File

@ -9,6 +9,7 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
Failed pages are collected on the scraping process and rescheduled at the end,
once the spider has finished crawling all regular (non failed) pages.
"""
from __future__ import annotations
import warnings

1
scrapy/exceptions.py
View File

@ -4,6 +4,7 @@ Scrapy core exceptions
These exceptions are documented in docs/topics/exceptions.rst. Please don't add
new exceptions here without documenting them there.
"""
from typing import Any
# Internal

154
scrapy/exporters.py
View File

@ -3,16 +3,18 @@ Item Exporters are used to export/serialize items into different formats.
"""
import csv
import io
import marshal
import pickle
import pickle # nosec
import pprint
from collections.abc import Mapping
from xml.sax.saxutils import XMLGenerator
from io import BytesIO, TextIOWrapper
from json import JSONEncoder
from typing import Any, Callable, Dict, Iterable, Mapping, Optional, Tuple, Union
from xml.sax.saxutils import XMLGenerator # nosec
from xml.sax.xmlreader import AttributesImpl # nosec
from itemadapter import ItemAdapter, is_item
from scrapy.item import Item
from scrapy.item import Field, Item
from scrapy.utils.python import is_listlike, to_bytes, to_unicode
from scrapy.utils.serialize import ScrapyJSONEncoder
@ -29,36 +31,42 @@ __all__ = [
class BaseItemExporter:
def __init__(self, *, dont_fail=False, **kwargs):
self._kwargs = kwargs
def __init__(self, *, dont_fail: bool = False, **kwargs: Any):
self._kwargs: Dict[str, Any] = kwargs
self._configure(kwargs, dont_fail=dont_fail)
def _configure(self, options, dont_fail=False):
def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None:
"""Configure the exporter by popping options from the ``options`` dict.
If dont_fail is set, it won't raise an exception on unexpected options
(useful for using with keyword arguments in subclasses ``__init__`` methods)
"""
self.encoding = options.pop("encoding", None)
self.fields_to_export = options.pop("fields_to_export", None)
self.export_empty_fields = options.pop("export_empty_fields", False)
self.indent = options.pop("indent", None)
self.encoding: Optional[str] = options.pop("encoding", None)
self.fields_to_export: Union[Mapping[str, str], Iterable[str], None] = (
options.pop("fields_to_export", None)
)
self.export_empty_fields: bool = options.pop("export_empty_fields", False)
self.indent: Optional[int] = options.pop("indent", None)
if not dont_fail and options:
raise TypeError(f"Unexpected options: {', '.join(options.keys())}")
def export_item(self, item):
def export_item(self, item: Any) -> None:
raise NotImplementedError
def serialize_field(self, field, name, value):
serializer = field.get("serializer", lambda x: x)
def serialize_field(
self, field: Union[Mapping[str, Any], Field], name: str, value: Any
) -> Any:
serializer: Callable[[Any], Any] = field.get("serializer", lambda x: x)
return serializer(value)
def start_exporting(self):
def start_exporting(self) -> None:
pass
def finish_exporting(self):
def finish_exporting(self) -> None:
pass
def _get_serialized_fields(self, item, default_value=None, include_empty=None):
def _get_serialized_fields(
self, item: Any, default_value: Any = None, include_empty: Optional[bool] = None
) -> Iterable[Tuple[str, Any]]:
"""Return the fields to export as an iterable of tuples
(name, serialized_value)
"""
@ -100,22 +108,22 @@ class BaseItemExporter:
class JsonLinesItemExporter(BaseItemExporter):
def __init__(self, file, **kwargs):
def __init__(self, file: BytesIO, **kwargs: Any):
super().__init__(dont_fail=True, **kwargs)
self.file = file
self.file: BytesIO = file
self._kwargs.setdefault("ensure_ascii", not self.encoding)
self.encoder = ScrapyJSONEncoder(**self._kwargs)
self.encoder: JSONEncoder = ScrapyJSONEncoder(**self._kwargs)
def export_item(self, item):
def export_item(self, item: Any) -> None:
itemdict = dict(self._get_serialized_fields(item))
data = self.encoder.encode(itemdict) + "\n"
self.file.write(to_bytes(data, self.encoding))
class JsonItemExporter(BaseItemExporter):
def __init__(self, file, **kwargs):
def __init__(self, file: BytesIO, **kwargs: Any):
super().__init__(dont_fail=True, **kwargs)
self.file = file
self.file: BytesIO = file
# there is a small difference between the behaviour or JsonItemExporter.indent
# and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent
# the addition of newlines everywhere
@ -127,26 +135,26 @@ class JsonItemExporter(BaseItemExporter):
self.encoder = ScrapyJSONEncoder(**self._kwargs)
self.first_item = True
def _beautify_newline(self):
def _beautify_newline(self) -> None:
if self.indent is not None:
self.file.write(b"\n")
def _add_comma_after_first(self):
def _add_comma_after_first(self) -> None:
if self.first_item:
self.first_item = False
else:
self.file.write(b",")
self._beautify_newline()
def start_exporting(self):
def start_exporting(self) -> None:
self.file.write(b"[")
self._beautify_newline()
def finish_exporting(self):
def finish_exporting(self) -> None:
self._beautify_newline()
self.file.write(b"]")
def export_item(self, item):
def export_item(self, item: Any) -> None:
itemdict = dict(self._get_serialized_fields(item))
data = to_bytes(self.encoder.encode(itemdict), self.encoding)
self._add_comma_after_first()
@ -154,7 +162,7 @@ class JsonItemExporter(BaseItemExporter):
class XmlItemExporter(BaseItemExporter):
def __init__(self, file, **kwargs):
def __init__(self, file: BytesIO, **kwargs: Any):
self.item_element = kwargs.pop("item_element", "item")
self.root_element = kwargs.pop("root_element", "items")
super().__init__(**kwargs)
@ -162,22 +170,22 @@ class XmlItemExporter(BaseItemExporter):
self.encoding = "utf-8"
self.xg = XMLGenerator(file, encoding=self.encoding)
def _beautify_newline(self, new_item=False):
def _beautify_newline(self, new_item: bool = False) -> None:
if self.indent is not None and (self.indent > 0 or new_item):
self.xg.characters("\n")
def _beautify_indent(self, depth=1):
def _beautify_indent(self, depth: int = 1) -> None:
if self.indent:
self.xg.characters(" " * self.indent * depth)
def start_exporting(self):
def start_exporting(self) -> None:
self.xg.startDocument()
self.xg.startElement(self.root_element, {})
self.xg.startElement(self.root_element, AttributesImpl({}))
self._beautify_newline(new_item=True)
def export_item(self, item):
def export_item(self, item: Any) -> None:
self._beautify_indent(depth=1)
self.xg.startElement(self.item_element, {})
self.xg.startElement(self.item_element, AttributesImpl({}))
self._beautify_newline()
for name, value in self._get_serialized_fields(item, default_value=""):
self._export_xml_field(name, value, depth=2)
@ -185,13 +193,13 @@ class XmlItemExporter(BaseItemExporter):
self.xg.endElement(self.item_element)
self._beautify_newline(new_item=True)
def finish_exporting(self):
def finish_exporting(self) -> None:
self.xg.endElement(self.root_element)
self.xg.endDocument()
def _export_xml_field(self, name, serialized_value, depth):
def _export_xml_field(self, name: str, serialized_value: Any, depth: int) -> None:
self._beautify_indent(depth=depth)
self.xg.startElement(name, {})
self.xg.startElement(name, AttributesImpl({}))
if hasattr(serialized_value, "items"):
self._beautify_newline()
for subname, value in serialized_value.items():
@ -213,17 +221,17 @@ class XmlItemExporter(BaseItemExporter):
class CsvItemExporter(BaseItemExporter):
def __init__(
self,
file,
include_headers_line=True,
join_multivalued=",",
errors=None,
**kwargs,
file: BytesIO,
include_headers_line: bool = True,
join_multivalued: str = ",",
errors: Optional[str] = None,
**kwargs: Any,
):
super().__init__(dont_fail=True, **kwargs)
if not self.encoding:
self.encoding = "utf-8"
self.include_headers_line = include_headers_line
self.stream = io.TextIOWrapper(
self.stream = TextIOWrapper(
file,
line_buffering=False,
write_through=True,
@ -235,11 +243,13 @@ class CsvItemExporter(BaseItemExporter):
self._headers_not_written = True
self._join_multivalued = join_multivalued
def serialize_field(self, field, name, value):
serializer = field.get("serializer", self._join_if_needed)
def serialize_field(
self, field: Union[Mapping[str, Any], Field], name: str, value: Any
) -> Any:
serializer: Callable[[Any], Any] = field.get("serializer", self._join_if_needed)
return serializer(value)
def _join_if_needed(self, value):
def _join_if_needed(self, value: Any) -> Any:
if isinstance(value, (list, tuple)):
try:
return self._join_multivalued.join(value)
@ -247,7 +257,7 @@ class CsvItemExporter(BaseItemExporter):
pass
return value
def export_item(self, item):
def export_item(self, item: Any) -> None:
if self._headers_not_written:
self._headers_not_written = False
self._write_headers_and_set_fields_to_export(item)
@ -256,36 +266,38 @@ class CsvItemExporter(BaseItemExporter):
values = list(self._build_row(x for _, x in fields))
self.csv_writer.writerow(values)
def finish_exporting(self):
def finish_exporting(self) -> None:
self.stream.detach() # Avoid closing the wrapped file.
def _build_row(self, values):
def _build_row(self, values: Iterable[Any]) -> Iterable[Any]:
for s in values:
try:
yield to_unicode(s, self.encoding)
except TypeError:
yield s
def _write_headers_and_set_fields_to_export(self, item):
def _write_headers_and_set_fields_to_export(self, item: Any) -> None:
if self.include_headers_line:
if not self.fields_to_export:
# use declared field names, or keys if the item is a dict
self.fields_to_export = ItemAdapter(item).field_names()
fields: Iterable[str]
if isinstance(self.fields_to_export, Mapping):
fields = self.fields_to_export.values()
else:
assert self.fields_to_export
fields = self.fields_to_export
row = list(self._build_row(fields))
self.csv_writer.writerow(row)
class PickleItemExporter(BaseItemExporter):
def __init__(self, file, protocol=4, **kwargs):
def __init__(self, file: BytesIO, protocol: int = 4, **kwargs: Any):
super().__init__(**kwargs)
self.file = file
self.protocol = protocol
self.file: BytesIO = file
self.protocol: int = protocol
def export_item(self, item):
def export_item(self, item: Any) -> None:
d = dict(self._get_serialized_fields(item))
pickle.dump(d, self.file, self.protocol)
@ -299,20 +311,20 @@ class MarshalItemExporter(BaseItemExporter):
opened in binary mode, a :class:`~io.BytesIO` object, etc)
"""
def __init__(self, file, **kwargs):
def __init__(self, file: BytesIO, **kwargs: Any):
super().__init__(**kwargs)
self.file = file
self.file: BytesIO = file
def export_item(self, item):
def export_item(self, item: Any) -> None:
marshal.dump(dict(self._get_serialized_fields(item)), self.file)
class PprintItemExporter(BaseItemExporter):
def __init__(self, file, **kwargs):
def __init__(self, file: BytesIO, **kwargs: Any):
super().__init__(**kwargs)
self.file = file
self.file: BytesIO = file
def export_item(self, item):
def export_item(self, item: Any) -> None:
itemdict = dict(self._get_serialized_fields(item))
self.file.write(to_bytes(pprint.pformat(itemdict) + "\n"))
@ -327,16 +339,20 @@ class PythonItemExporter(BaseItemExporter):
.. _msgpack: https://pypi.org/project/msgpack/
"""
def _configure(self, options, dont_fail=False):
def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None:
super()._configure(options, dont_fail)
if not self.encoding:
self.encoding = "utf-8"
def serialize_field(self, field, name, value):
serializer = field.get("serializer", self._serialize_value)
def serialize_field(
self, field: Union[Mapping[str, Any], Field], name: str, value: Any
) -> Any:
serializer: Callable[[Any], Any] = field.get(
"serializer", self._serialize_value
)
return serializer(value)
def _serialize_value(self, value):
def _serialize_value(self, value: Any) -> Any:
if isinstance(value, Item):
return self.export_item(value)
if is_item(value):
@ -347,10 +363,10 @@ class PythonItemExporter(BaseItemExporter):
return to_unicode(value, encoding=self.encoding)
return value
def _serialize_item(self, item):
def _serialize_item(self, item: Any) -> Iterable[Tuple[Union[str, bytes], Any]]:
for key, value in ItemAdapter(item).items():
yield key, self._serialize_value(value)
def export_item(self, item):
result = dict(self._get_serialized_fields(item))
def export_item(self, item: Any) -> Dict[Union[str, bytes], Any]: # type: ignore[override]
result: Dict[Union[str, bytes], Any] = dict(self._get_serialized_fields(item))
return result

6
scrapy/extension.py
View File

@ -3,7 +3,11 @@ The Extension Manager
See documentation in docs/topics/extensions.rst
"""
from typing import Any, List
from scrapy.middleware import MiddlewareManager
from scrapy.settings import Settings
from scrapy.utils.conf import build_component_list
@ -11,5 +15,5 @@ class ExtensionManager(MiddlewareManager):
component_name = "extension"
@classmethod
def _get_mwlist_from_settings(cls, settings):
def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]:
return build_component_list(settings.getwithbase("EXTENSIONS"))

52
scrapy/extensions/closespider.py
View File

@ -4,20 +4,31 @@ conditions are met.
See documentation in docs/topics/extensions.rst
"""
from __future__ import annotations
import logging
from collections import defaultdict
from typing import TYPE_CHECKING, Any, DefaultDict, Dict
from scrapy import signals
from twisted.python.failure import Failure
from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
class CloseSpider:
def __init__(self, crawler):
self.crawler = crawler
def __init__(self, crawler: Crawler):
self.crawler: Crawler = crawler
self.close_on = {
self.close_on: Dict[str, Any] = {
"timeout": crawler.settings.getfloat("CLOSESPIDER_TIMEOUT"),
"itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"),
"pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"),
@ -28,7 +39,7 @@ class CloseSpider:
if not any(self.close_on.values()):
raise NotConfigured
self.counter = defaultdict(int)
self.counter: DefaultDict[str, int] = defaultdict(int)
if self.close_on.get("errorcount"):
crawler.signals.connect(self.error_count, signal=signals.spider_error)
@ -39,8 +50,8 @@ class CloseSpider:
if self.close_on.get("itemcount"):
crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
if self.close_on.get("timeout_no_item"):
self.timeout_no_item = self.close_on["timeout_no_item"]
self.items_in_period = 0
self.timeout_no_item: int = self.close_on["timeout_no_item"]
self.items_in_period: int = 0
crawler.signals.connect(
self.spider_opened_no_item, signal=signals.spider_opened
)
@ -50,22 +61,25 @@ class CloseSpider:
crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler)
def error_count(self, failure, response, spider):
def error_count(self, failure: Failure, response: Response, spider: Spider) -> None:
self.counter["errorcount"] += 1
if self.counter["errorcount"] == self.close_on["errorcount"]:
assert self.crawler.engine
self.crawler.engine.close_spider(spider, "closespider_errorcount")
def page_count(self, response, request, spider):
def page_count(self, response: Response, request: Request, spider: Spider) -> None:
self.counter["pagecount"] += 1
if self.counter["pagecount"] == self.close_on["pagecount"]:
assert self.crawler.engine
self.crawler.engine.close_spider(spider, "closespider_pagecount")
def spider_opened(self, spider):
def spider_opened(self, spider: Spider) -> None:
from twisted.internet import reactor
assert self.crawler.engine
self.task = reactor.callLater(
self.close_on["timeout"],
self.crawler.engine.close_spider,
@ -73,21 +87,22 @@ class CloseSpider:
reason="closespider_timeout",
)
def item_scraped(self, item, spider):
def item_scraped(self, item: Any, spider: Spider) -> None:
self.counter["itemcount"] += 1
if self.counter["itemcount"] == self.close_on["itemcount"]:
assert self.crawler.engine
self.crawler.engine.close_spider(spider, "closespider_itemcount")
def spider_closed(self, spider):
task = getattr(self, "task", False)
def spider_closed(self, spider: Spider) -> None:
task = getattr(self, "task", None)
if task and task.active():
task.cancel()
task_no_item = getattr(self, "task_no_item", False)
task_no_item = getattr(self, "task_no_item", None)
if task_no_item and task_no_item.running:
task_no_item.stop()
def spider_opened_no_item(self, spider):
def spider_opened_no_item(self, spider: Spider) -> None:
from twisted.internet import task
self.task_no_item = task.LoopingCall(self._count_items_produced, spider)
@ -98,10 +113,10 @@ class CloseSpider:
f"{self.timeout_no_item} seconds."
)
def item_scraped_no_item(self, item, spider):
def item_scraped_no_item(self, item: Any, spider: Spider) -> None:
self.items_in_period += 1
def _count_items_produced(self, spider):
def _count_items_produced(self, spider: Spider) -> None:
if self.items_in_period >= 1:
self.items_in_period = 0
else:
@ -109,4 +124,5 @@ class CloseSpider:
f"Closing spider since no items were produced in the last "
f"{self.timeout_no_item} seconds."
)
assert self.crawler.engine
self.crawler.engine.close_spider(spider, "closespider_timeout_no_item")

34
scrapy/extensions/corestats.py
View File

@ -1,18 +1,29 @@
"""
Extension for collecting core stats like items scraped and start/finish times
"""
from datetime import datetime, timezone
from scrapy import signals
from __future__ import annotations
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any, Optional
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.statscollectors import StatsCollector
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class CoreStats:
def __init__(self, stats):
self.stats = stats
self.start_time = None
def __init__(self, stats: StatsCollector):
self.stats: StatsCollector = stats
self.start_time: Optional[datetime] = None
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
assert crawler.stats
o = cls(crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
@ -21,11 +32,12 @@ class CoreStats:
crawler.signals.connect(o.response_received, signal=signals.response_received)
return o
def spider_opened(self, spider):
def spider_opened(self, spider: Spider) -> None:
self.start_time = datetime.now(tz=timezone.utc)
self.stats.set_value("start_time", self.start_time, spider=spider)
def spider_closed(self, spider, reason):
def spider_closed(self, spider: Spider, reason: str) -> None:
assert self.start_time is not None
finish_time = datetime.now(tz=timezone.utc)
elapsed_time = finish_time - self.start_time
elapsed_time_seconds = elapsed_time.total_seconds()
@ -35,13 +47,13 @@ class CoreStats:
self.stats.set_value("finish_time", finish_time, spider=spider)
self.stats.set_value("finish_reason", reason, spider=spider)
def item_scraped(self, item, spider):
def item_scraped(self, item: Any, spider: Spider) -> None:
self.stats.inc_value("item_scraped_count", spider=spider)
def response_received(self, spider):
def response_received(self, spider: Spider) -> None:
self.stats.inc_value("response_received_count", spider=spider)
def item_dropped(self, item, spider, exception):
def item_dropped(self, item: Any, spider: Spider, exception: BaseException) -> None:
reason = exception.__class__.__name__
self.stats.inc_value("item_dropped_count", spider=spider)
self.stats.inc_value(f"item_dropped_reasons_count/{reason}", spider=spider)

25
scrapy/extensions/debug.py
View File

@ -4,22 +4,31 @@ Extensions for debugging Scrapy
See documentation in docs/topics/extensions.rst
"""
from __future__ import annotations
import logging
import signal
import sys
import threading
import traceback
from pdb import Pdb
from types import FrameType
from typing import TYPE_CHECKING, Optional
from scrapy.crawler import Crawler
from scrapy.utils.engine import format_engine_status
from scrapy.utils.trackref import format_live_refs
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
class StackTraceDump:
def __init__(self, crawler=None):
self.crawler = crawler
def __init__(self, crawler: Crawler):
self.crawler: Crawler = crawler
try:
signal.signal(signal.SIGUSR2, self.dump_stacktrace)
signal.signal(signal.SIGQUIT, self.dump_stacktrace)
@ -28,10 +37,11 @@ class StackTraceDump:
pass
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler)
def dump_stacktrace(self, signum, frame):
def dump_stacktrace(self, signum: int, frame: Optional[FrameType]) -> None:
assert self.crawler.engine
log_args = {
"stackdumps": self._thread_stacks(),
"enginestatus": format_engine_status(self.crawler.engine),
@ -44,7 +54,7 @@ class StackTraceDump:
extra={"crawler": self.crawler},
)
def _thread_stacks(self):
def _thread_stacks(self) -> str:
id2name = dict((th.ident, th.name) for th in threading.enumerate())
dumps = ""
for id_, frame in sys._current_frames().items():
@ -55,12 +65,13 @@ class StackTraceDump:
class Debugger:
def __init__(self):
def __init__(self) -> None:
try:
signal.signal(signal.SIGUSR2, self._enter_debugger)
except AttributeError:
# win32 platforms don't support SIGUSR signals
pass
def _enter_debugger(self, signum, frame):
def _enter_debugger(self, signum: int, frame: Optional[FrameType]) -> None:
assert frame
Pdb().set_trace(frame.f_back)

342
scrapy/extensions/feedexport.py
View File

@ -4,6 +4,8 @@ Feed Exports extension
See documentation in docs/topics/feed-exports.rst
"""
from __future__ import annotations
import logging
import re
import sys
@ -11,26 +13,51 @@ import warnings
from datetime import datetime, timezone
from pathlib import Path, PureWindowsPath
from tempfile import NamedTemporaryFile
from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union
from typing import (
IO,
TYPE_CHECKING,
Any,
Callable,
Dict,
Iterable,
List,
Optional,
Protocol,
Tuple,
Type,
TypeVar,
Union,
cast,
)
from urllib.parse import unquote, urlparse
from twisted.internet import defer, threads
from twisted.internet.defer import DeferredList
from twisted.internet import threads
from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
from twisted.python.failure import Failure
from w3lib.url import file_uri_to_path
from zope.interface import Interface, implementer
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.exporters import BaseItemExporter
from scrapy.extensions.postprocessing import PostProcessingManager
from scrapy.settings import BaseSettings, Settings
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.conf import feed_complete_default_values_from_settings
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.deprecate import create_deprecated_class
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
from _typeshed import OpenBinaryMode
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
try:
@ -40,8 +67,19 @@ try:
except ImportError:
IS_BOTO3_AVAILABLE = False
UriParamsCallableT = Callable[[Dict[str, Any], Spider], Optional[Dict[str, Any]]]
def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
_StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol")
def build_storage(
builder: Callable[..., _StorageT],
uri: str,
*args: Any,
feed_options: Optional[Dict[str, Any]] = None,
preargs: Iterable[Any] = (),
**kwargs: Any,
) -> _StorageT:
kwargs["feed_options"] = feed_options
return builder(*preargs, uri, *args, **kwargs)
@ -55,10 +93,10 @@ class ItemFilter:
:type feed_options: dict
"""
feed_options: Optional[dict]
item_classes: Tuple
feed_options: Optional[Dict[str, Any]]
item_classes: Tuple[type, ...]
def __init__(self, feed_options: Optional[dict]) -> None:
def __init__(self, feed_options: Optional[Dict[str, Any]]) -> None:
self.feed_options = feed_options
if feed_options is not None:
self.item_classes = tuple(
@ -97,28 +135,49 @@ class IFeedStorage(Interface):
"""Store the given file stream"""
class FeedStorageProtocol(Protocol):
"""Reimplementation of ``IFeedStorage`` that can be used in type hints."""
def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None):
"""Initialize the storage with the parameters given in the URI and the
feed-specific options (see :setting:`FEEDS`)"""
def open(self, spider: Spider) -> IO[bytes]:
"""Open the storage for the given spider. It must return a file-like
object that will be used for the exporters"""
def store(self, file: IO[bytes]) -> Optional[Deferred]:
"""Store the given file stream"""
@implementer(IFeedStorage)
class BlockingFeedStorage:
def open(self, spider):
def open(self, spider: Spider) -> IO[bytes]:
path = spider.crawler.settings["FEED_TEMPDIR"]
if path and not Path(path).is_dir():
raise OSError("Not a Directory: " + str(path))
return NamedTemporaryFile(prefix="feed-", dir=path)
def store(self, file):
def store(self, file: IO[bytes]) -> Optional[Deferred]:
return threads.deferToThread(self._store_in_thread, file)
def _store_in_thread(self, file):
def _store_in_thread(self, file: IO[bytes]) -> None:
raise NotImplementedError
@implementer(IFeedStorage)
class StdoutFeedStorage:
def __init__(self, uri, _stdout=None, *, feed_options=None):
def __init__(
self,
uri: str,
_stdout: Optional[IO[bytes]] = None,
*,
feed_options: Optional[Dict[str, Any]] = None,
):
if not _stdout:
_stdout = sys.stdout.buffer
self._stdout = _stdout
self._stdout: IO[bytes] = _stdout
if feed_options and feed_options.get("overwrite", False) is True:
logger.warning(
"Standard output (stdout) storage does not support "
@ -127,54 +186,58 @@ class StdoutFeedStorage:
"it to False."
)
def open(self, spider):
def open(self, spider: Spider) -> IO[bytes]:
return self._stdout
def store(self, file):
def store(self, file: IO[bytes]) -> Optional[Deferred]:
pass
@implementer(IFeedStorage)
class FileFeedStorage:
def __init__(self, uri, *, feed_options=None):
self.path = file_uri_to_path(uri)
def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None):
self.path: str = file_uri_to_path(uri)
feed_options = feed_options or {}
self.write_mode = "wb" if feed_options.get("overwrite", False) else "ab"
self.write_mode: OpenBinaryMode = (
"wb" if feed_options.get("overwrite", False) else "ab"
)
def open(self, spider) -> IO[Any]:
def open(self, spider: Spider) -> IO[bytes]:
dirname = Path(self.path).parent
if dirname and not dirname.exists():
dirname.mkdir(parents=True)
return Path(self.path).open(self.write_mode)
def store(self, file):
def store(self, file: IO[bytes]) -> Optional[Deferred]:
file.close()
return None
class S3FeedStorage(BlockingFeedStorage):
def __init__(
self,
uri,
access_key=None,
secret_key=None,
acl=None,
endpoint_url=None,
uri: str,
access_key: Optional[str] = None,
secret_key: Optional[str] = None,
acl: Optional[str] = None,
endpoint_url: Optional[str] = None,
*,
feed_options=None,
session_token=None,
region_name=None,
feed_options: Optional[Dict[str, Any]] = None,
session_token: Optional[str] = None,
region_name: Optional[str] = None,
):
if not is_botocore_available():
raise NotConfigured("missing botocore library")
u = urlparse(uri)
self.bucketname = u.hostname
self.access_key = u.username or access_key
self.secret_key = u.password or secret_key
self.session_token = session_token
self.keyname = u.path[1:] # remove first "/"
self.acl = acl
self.endpoint_url = endpoint_url
self.region_name = region_name
assert u.hostname
self.bucketname: str = u.hostname
self.access_key: Optional[str] = u.username or access_key
self.secret_key: Optional[str] = u.password or secret_key
self.session_token: Optional[str] = session_token
self.keyname: str = u.path[1:] # remove first "/"
self.acl: Optional[str] = acl
self.endpoint_url: Optional[str] = endpoint_url
self.region_name: Optional[str] = region_name
if IS_BOTO3_AVAILABLE:
import boto3.session
@ -217,7 +280,13 @@ class S3FeedStorage(BlockingFeedStorage):
)
@classmethod
def from_crawler(cls, crawler, uri, *, feed_options=None):
def from_crawler(
cls,
crawler: Crawler,
uri: str,
*,
feed_options: Optional[Dict[str, Any]] = None,
) -> Self:
return build_storage(
cls,
uri,
@ -230,8 +299,9 @@ class S3FeedStorage(BlockingFeedStorage):
feed_options=feed_options,
)
def _store_in_thread(self, file):
def _store_in_thread(self, file: IO[bytes]) -> None:
file.seek(0)
kwargs: Dict[str, Any]
if IS_BOTO3_AVAILABLE:
kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}
self.s3_client.upload_fileobj(
@ -246,22 +316,23 @@ class S3FeedStorage(BlockingFeedStorage):
class GCSFeedStorage(BlockingFeedStorage):
def __init__(self, uri, project_id, acl):
self.project_id = project_id
self.acl = acl
def __init__(self, uri: str, project_id: Optional[str], acl: Optional[str]):
self.project_id: Optional[str] = project_id
self.acl: Optional[str] = acl
u = urlparse(uri)
self.bucket_name = u.hostname
self.blob_name = u.path[1:] # remove first "/"
assert u.hostname
self.bucket_name: str = u.hostname
self.blob_name: str = u.path[1:] # remove first "/"
@classmethod
def from_crawler(cls, crawler, uri):
def from_crawler(cls, crawler: Crawler, uri: str) -> Self:
return cls(
uri,
crawler.settings["GCS_PROJECT_ID"],
crawler.settings["FEED_STORAGE_GCS_ACL"] or None,
)
def _store_in_thread(self, file):
def _store_in_thread(self, file: IO[bytes]) -> None:
file.seek(0)
from google.cloud.storage import Client
@ -291,7 +362,13 @@ class FTPFeedStorage(BlockingFeedStorage):
self.overwrite: bool = not feed_options or feed_options.get("overwrite", True)
@classmethod
def from_crawler(cls, crawler, uri, *, feed_options=None):
def from_crawler(
cls,
crawler: Crawler,
uri: str,
*,
feed_options: Optional[Dict[str, Any]] = None,
) -> Self:
return build_storage(
cls,
uri,
@ -299,7 +376,7 @@ class FTPFeedStorage(BlockingFeedStorage):
feed_options=feed_options,
)
def _store_in_thread(self, file):
def _store_in_thread(self, file: IO[bytes]) -> None:
ftp_store_file(
path=self.path,
file=file,
@ -315,46 +392,51 @@ class FTPFeedStorage(BlockingFeedStorage):
class FeedSlot:
def __init__(
self,
storage,
uri,
format,
store_empty,
batch_id,
uri_template,
filter,
feed_options,
spider,
exporters,
settings,
crawler,
storage: FeedStorageProtocol,
uri: str,
format: str,
store_empty: bool,
batch_id: int,
uri_template: str,
filter: ItemFilter,
feed_options: Dict[str, Any],
spider: Spider,
exporters: Dict[str, Type[BaseItemExporter]],
settings: BaseSettings,
crawler: Crawler,
):
self.file = None
self.exporter = None
self.storage = storage
self.file: Optional[IO[bytes]] = None
self.exporter: Optional[BaseItemExporter] = None
self.storage: FeedStorageProtocol = storage
# feed params
self.batch_id = batch_id
self.format = format
self.store_empty = store_empty
self.uri_template = uri_template
self.uri = uri
self.filter = filter
self.batch_id: int = batch_id
self.format: str = format
self.store_empty: bool = store_empty
self.uri_template: str = uri_template
self.uri: str = uri
self.filter: ItemFilter = filter
# exporter params
self.feed_options = feed_options
self.spider = spider
self.exporters = exporters
self.settings = settings
self.crawler = crawler
self.feed_options: Dict[str, Any] = feed_options
self.spider: Spider = spider
self.exporters: Dict[str, Type[BaseItemExporter]] = exporters
self.settings: BaseSettings = settings
self.crawler: Crawler = crawler
# flags
self.itemcount = 0
self._exporting = False
self._fileloaded = False
self.itemcount: int = 0
self._exporting: bool = False
self._fileloaded: bool = False
def start_exporting(self):
def start_exporting(self) -> None:
if not self._fileloaded:
self.file = self.storage.open(self.spider)
if "postprocessing" in self.feed_options:
self.file = PostProcessingManager(
self.feed_options["postprocessing"], self.file, self.feed_options
self.file = cast(
IO[bytes],
PostProcessingManager(
self.feed_options["postprocessing"],
self.file,
self.feed_options,
),
)
self.exporter = self._get_exporter(
file=self.file,
@ -367,17 +449,23 @@ class FeedSlot:
self._fileloaded = True
if not self._exporting:
assert self.exporter
self.exporter.start_exporting()
self._exporting = True
def _get_instance(self, objcls, *args, **kwargs):
return create_instance(objcls, self.settings, self.crawler, *args, **kwargs)
def _get_instance(
self, objcls: Type[BaseItemExporter], *args: Any, **kwargs: Any
) -> BaseItemExporter:
return build_from_crawler(objcls, self.crawler, *args, **kwargs)
def _get_exporter(self, file, format, *args, **kwargs):
def _get_exporter(
self, file: IO[bytes], format: str, *args: Any, **kwargs: Any
) -> BaseItemExporter:
return self._get_instance(self.exporters[format], file, *args, **kwargs)
def finish_exporting(self):
def finish_exporting(self) -> None:
if self._exporting:
assert self.exporter
self.exporter.finish_exporting()
self._exporting = False
@ -389,22 +477,22 @@ _FeedSlot = create_deprecated_class(
class FeedExporter:
_pending_deferreds: List[defer.Deferred] = []
_pending_deferreds: List[Deferred] = []
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
exporter = cls(crawler)
crawler.signals.connect(exporter.open_spider, signals.spider_opened)
crawler.signals.connect(exporter.close_spider, signals.spider_closed)
crawler.signals.connect(exporter.item_scraped, signals.item_scraped)
return exporter
def __init__(self, crawler):
self.crawler = crawler
self.settings = crawler.settings
def __init__(self, crawler: Crawler):
self.crawler: Crawler = crawler
self.settings: Settings = crawler.settings
self.feeds = {}
self.slots = []
self.filters = {}
self.slots: List[FeedSlot] = []
self.filters: Dict[str, ItemFilter] = {}
if not self.settings["FEEDS"] and not self.settings["FEED_URI"]:
raise NotConfigured
@ -436,8 +524,12 @@ class FeedExporter:
)
self.filters[uri] = self._load_filter(feed_options)
self.storages = self._load_components("FEED_STORAGES")
self.exporters = self._load_components("FEED_EXPORTERS")
self.storages: Dict[str, Type[FeedStorageProtocol]] = self._load_components(
"FEED_STORAGES"
)
self.exporters: Dict[str, Type[BaseItemExporter]] = self._load_components(
"FEED_EXPORTERS"
)
for uri, feed_options in self.feeds.items():
if not self._storage_supported(uri, feed_options):
raise NotConfigured
@ -446,7 +538,7 @@ class FeedExporter:
if not self._exporter_supported(feed_options["format"]):
raise NotConfigured
def open_spider(self, spider):
def open_spider(self, spider: Spider) -> None:
for uri, feed_options in self.feeds.items():
uri_params = self._get_uri_params(spider, feed_options["uri_params"])
self.slots.append(
@ -459,7 +551,7 @@ class FeedExporter:
)
)
async def close_spider(self, spider):
async def close_spider(self, spider: Spider) -> None:
for slot in self.slots:
self._close_slot(slot, spider)
@ -472,8 +564,9 @@ class FeedExporter:
self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed)
)
def _close_slot(self, slot, spider):
def get_file(slot_):
def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred]:
def get_file(slot_: FeedSlot) -> IO[bytes]:
assert slot_.file
if isinstance(slot_.file, PostProcessingManager):
slot_.file.close()
return slot_.file.file
@ -491,7 +584,7 @@ class FeedExporter:
return None
logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
d = defer.maybeDeferred(slot.storage.store, get_file(slot))
d: Deferred = maybeDeferred(slot.storage.store, get_file(slot))
d.addCallback(
self._handle_store_success, logmsg, spider, type(slot.storage).__name__
@ -509,20 +602,33 @@ class FeedExporter:
return d
def _handle_store_error(self, f, logmsg, spider, slot_type):
def _handle_store_error(
self, f: Failure, logmsg: str, spider: Spider, slot_type: str
) -> None:
logger.error(
"Error storing %s",
logmsg,
exc_info=failure_to_exc_info(f),
extra={"spider": spider},
)
assert self.crawler.stats
self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
def _handle_store_success(self, f, logmsg, spider, slot_type):
def _handle_store_success(
self, f: Failure, logmsg: str, spider: Spider, slot_type: str
) -> None:
logger.info("Stored %s", logmsg, extra={"spider": spider})
assert self.crawler.stats
self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}")
def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template):
def _start_new_batch(
self,
batch_id: int,
uri: str,
feed_options: Dict[str, Any],
spider: Spider,
uri_template: str,
) -> FeedSlot:
"""
Redirect the output data stream to a new file.
Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
@ -545,11 +651,11 @@ class FeedExporter:
spider=spider,
exporters=self.exporters,
settings=self.settings,
crawler=getattr(self, "crawler", None),
crawler=self.crawler,
)
return slot
def item_scraped(self, item, spider):
def item_scraped(self, item: Any, spider: Spider) -> None:
slots = []
for slot in self.slots:
if not slot.filter.accepts(item):
@ -559,6 +665,7 @@ class FeedExporter:
continue
slot.start_exporting()
assert slot.exporter
slot.exporter.export_item(item)
slot.itemcount += 1
# create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one
@ -583,7 +690,7 @@ class FeedExporter:
slots.append(slot)
self.slots = slots
def _load_components(self, setting_prefix):
def _load_components(self, setting_prefix: str) -> Dict[str, Any]:
conf = without_none_values(self.settings.getwithbase(setting_prefix))
d = {}
for k, v in conf.items():
@ -593,12 +700,13 @@ class FeedExporter:
pass
return d
def _exporter_supported(self, format):
def _exporter_supported(self, format: str) -> bool:
if format in self.exporters:
return True
logger.error("Unknown feed format: %(format)s", {"format": format})
return False
def _settings_are_valid(self):
def _settings_are_valid(self) -> bool:
"""
If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
%(batch_time)s or %(batch_id)d to distinguish different files of partial output
@ -616,7 +724,7 @@ class FeedExporter:
return False
return True
def _storage_supported(self, uri, feed_options):
def _storage_supported(self, uri: str, feed_options: Dict[str, Any]) -> bool:
scheme = urlparse(uri).scheme
if scheme in self.storages or PureWindowsPath(uri).drive:
try:
@ -629,8 +737,11 @@ class FeedExporter:
)
else:
logger.error("Unknown feed storage scheme: %(scheme)s", {"scheme": scheme})
return False
def _get_storage(self, uri, feed_options):
def _get_storage(
self, uri: str, feed_options: Dict[str, Any]
) -> FeedStorageProtocol:
"""Fork of create_instance specific to feed storage classes
It supports not passing the *feed_options* parameters to classes that
@ -639,11 +750,14 @@ class FeedExporter:
feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"])
crawler = getattr(self, "crawler", None)
def build_instance(builder, *preargs):
def build_instance(
builder: Type[FeedStorageProtocol], *preargs: Any
) -> FeedStorageProtocol:
return build_storage(
builder, uri, feed_options=feed_options, preargs=preargs
)
instance: FeedStorageProtocol
if crawler and hasattr(feedcls, "from_crawler"):
instance = build_instance(feedcls.from_crawler, crawler)
method_name = "from_crawler"
@ -660,9 +774,9 @@ class FeedExporter:
def _get_uri_params(
self,
spider: Spider,
uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]],
uri_params_function: Union[str, UriParamsCallableT, None],
slot: Optional[FeedSlot] = None,
) -> dict:
) -> Dict[str, Any]:
params = {}
for k in dir(spider):
params[k] = getattr(spider, k)
@ -670,7 +784,7 @@ class FeedExporter:
params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-")
params["batch_time"] = utc_now.isoformat().replace(":", "-")
params["batch_id"] = slot.batch_id + 1 if slot is not None else 1
uripar_function = (
uripar_function: UriParamsCallableT = (
load_object(uri_params_function)
if uri_params_function
else lambda params, _: params
@ -678,7 +792,9 @@ class FeedExporter:
new_params = uripar_function(params, spider)
return new_params if new_params is not None else params
def _load_filter(self, feed_options):
def _load_filter(self, feed_options: Dict[str, Any]) -> ItemFilter:
# load the item filter if declared else load the default filter class
item_filter_class = load_object(feed_options.get("item_filter", ItemFilter))
item_filter_class: Type[ItemFilter] = load_object(
feed_options.get("item_filter", ItemFilter)
)
return item_filter_class(feed_options)

155
scrapy/extensions/httpcache.py
View File

@ -1,10 +1,13 @@
import gzip
import logging
import pickle
import os
import pickle # nosec
from email.utils import mktime_tz, parsedate_tz
from importlib import import_module
from pathlib import Path
from time import time
from types import ModuleType
from typing import IO, TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union, cast
from weakref import WeakKeyDictionary
from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
@ -12,49 +15,65 @@ from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
from scrapy.http import Headers, Response
from scrapy.http.request import Request
from scrapy.responsetypes import responsetypes
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.project import data_path
from scrapy.utils.python import to_bytes, to_unicode
from scrapy.utils.request import RequestFingerprinter
if TYPE_CHECKING:
# typing.Concatenate requires Python 3.10
from typing_extensions import Concatenate
logger = logging.getLogger(__name__)
class DummyPolicy:
def __init__(self, settings):
self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES")
self.ignore_http_codes = [
def __init__(self, settings: BaseSettings):
self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES")
self.ignore_http_codes: List[int] = [
int(x) for x in settings.getlist("HTTPCACHE_IGNORE_HTTP_CODES")
]
def should_cache_request(self, request):
def should_cache_request(self, request: Request) -> bool:
return urlparse_cached(request).scheme not in self.ignore_schemes
def should_cache_response(self, response, request):
def should_cache_response(self, response: Response, request: Request) -> bool:
return response.status not in self.ignore_http_codes
def is_cached_response_fresh(self, cachedresponse, request):
def is_cached_response_fresh(
self, cachedresponse: Response, request: Request
) -> bool:
return True
def is_cached_response_valid(self, cachedresponse, response, request):
def is_cached_response_valid(
self, cachedresponse: Response, response: Response, request: Request
) -> bool:
return True
class RFC2616Policy:
MAXAGE = 3600 * 24 * 365 # one year
def __init__(self, settings):
self.always_store = settings.getbool("HTTPCACHE_ALWAYS_STORE")
self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES")
self._cc_parsed = WeakKeyDictionary()
self.ignore_response_cache_controls = [
def __init__(self, settings: BaseSettings):
self.always_store: bool = settings.getbool("HTTPCACHE_ALWAYS_STORE")
self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES")
self._cc_parsed: WeakKeyDictionary[
Union[Request, Response], Dict[bytes, Optional[bytes]]
] = WeakKeyDictionary()
self.ignore_response_cache_controls: List[bytes] = [
to_bytes(cc)
for cc in settings.getlist("HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS")
]
def _parse_cachecontrol(self, r):
def _parse_cachecontrol(
self, r: Union[Request, Response]
) -> Dict[bytes, Optional[bytes]]:
if r not in self._cc_parsed:
cch = r.headers.get(b"Cache-Control", b"")
assert cch is not None
parsed = parse_cachecontrol(cch)
if isinstance(r, Response):
for key in self.ignore_response_cache_controls:
@ -62,7 +81,7 @@ class RFC2616Policy:
self._cc_parsed[r] = parsed
return self._cc_parsed[r]
def should_cache_request(self, request):
def should_cache_request(self, request: Request) -> bool:
if urlparse_cached(request).scheme in self.ignore_schemes:
return False
cc = self._parse_cachecontrol(request)
@ -72,7 +91,7 @@ class RFC2616Policy:
# Any other is eligible for caching
return True
def should_cache_response(self, response, request):
def should_cache_response(self, response: Response, request: Request) -> bool:
# What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
# Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4
# Status code 206 is not included because cache can not deal with partial contents
@ -100,7 +119,9 @@ class RFC2616Policy:
# info and can not be revalidated
return False
def is_cached_response_fresh(self, cachedresponse, request):
def is_cached_response_fresh(
self, cachedresponse: Response, request: Request
) -> bool:
cc = self._parse_cachecontrol(cachedresponse)
ccreq = self._parse_cachecontrol(request)
if b"no-cache" in cc or b"no-cache" in ccreq:
@ -141,7 +162,9 @@ class RFC2616Policy:
self._set_conditional_validators(request, cachedresponse)
return False
def is_cached_response_valid(self, cachedresponse, response, request):
def is_cached_response_valid(
self, cachedresponse: Response, response: Response, request: Request
) -> bool:
# Use the cached response if the new response is a server error,
# as long as the old response didn't specify must-revalidate.
if response.status >= 500:
@ -152,7 +175,9 @@ class RFC2616Policy:
# Use the cached response if the server says it hasn't changed.
return response.status == 304
def _set_conditional_validators(self, request, cachedresponse):
def _set_conditional_validators(
self, request: Request, cachedresponse: Response
) -> None:
if b"Last-Modified" in cachedresponse.headers:
request.headers[b"If-Modified-Since"] = cachedresponse.headers[
b"Last-Modified"
@ -161,13 +186,15 @@ class RFC2616Policy:
if b"ETag" in cachedresponse.headers:
request.headers[b"If-None-Match"] = cachedresponse.headers[b"ETag"]
def _get_max_age(self, cc):
def _get_max_age(self, cc: Dict[bytes, Optional[bytes]]) -> Optional[int]:
try:
return max(0, int(cc[b"max-age"]))
return max(0, int(cc[b"max-age"])) # type: ignore[arg-type]
except (KeyError, ValueError):
return None
def _compute_freshness_lifetime(self, response, request, now):
def _compute_freshness_lifetime(
self, response: Response, request: Request, now: float
) -> float:
# Reference nsHttpResponseHead::ComputeFreshnessLifetime
# https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#706
cc = self._parse_cachecontrol(response)
@ -198,10 +225,12 @@ class RFC2616Policy:
# Insufficient information to compute freshness lifetime
return 0
def _compute_current_age(self, response, request, now):
def _compute_current_age(
self, response: Response, request: Request, now: float
) -> float:
# Reference nsHttpResponseHead::ComputeCurrentAge
# https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#658
currentage = 0
currentage: float = 0
# If Date header is not set we assume it is a fast connection, and
# clock is in sync with the server
date = rfc1123_to_epoch(response.headers.get(b"Date")) or now
@ -210,7 +239,7 @@ class RFC2616Policy:
if b"Age" in response.headers:
try:
age = int(response.headers[b"Age"])
age = int(response.headers[b"Age"]) # type: ignore[arg-type]
currentage = max(currentage, age)
except ValueError:
pass
@ -219,13 +248,13 @@ class RFC2616Policy:
class DbmCacheStorage:
def __init__(self, settings):
self.cachedir = data_path(settings["HTTPCACHE_DIR"], createdir=True)
self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS")
self.dbmodule = import_module(settings["HTTPCACHE_DBM_MODULE"])
self.db = None
def __init__(self, settings: BaseSettings):
self.cachedir: str = data_path(settings["HTTPCACHE_DIR"], createdir=True)
self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS")
self.dbmodule: ModuleType = import_module(settings["HTTPCACHE_DBM_MODULE"])
self.db: Any = None # the real type is private
def open_spider(self, spider: Spider):
def open_spider(self, spider: Spider) -> None:
dbpath = Path(self.cachedir, f"{spider.name}.db")
self.db = self.dbmodule.open(str(dbpath), "c")
@ -235,15 +264,16 @@ class DbmCacheStorage:
extra={"spider": spider},
)
self._fingerprinter = spider.crawler.request_fingerprinter
assert spider.crawler.request_fingerprinter
self._fingerprinter: RequestFingerprinter = spider.crawler.request_fingerprinter
def close_spider(self, spider):
def close_spider(self, spider: Spider) -> None:
self.db.close()
def retrieve_response(self, spider, request):
def retrieve_response(self, spider: Spider, request: Request) -> Optional[Response]:
data = self._read_data(spider, request)
if data is None:
return # not cached
return None # not cached
url = data["url"]
status = data["status"]
headers = Headers(data["headers"])
@ -252,7 +282,9 @@ class DbmCacheStorage:
response = respcls(url=url, headers=headers, status=status, body=body)
return response
def store_response(self, spider, request, response):
def store_response(
self, spider: Spider, request: Request, response: Response
) -> None:
key = self._fingerprinter.fingerprint(request).hex()
data = {
"status": response.status,
@ -263,28 +295,31 @@ class DbmCacheStorage:
self.db[f"{key}_data"] = pickle.dumps(data, protocol=4)
self.db[f"{key}_time"] = str(time())
def _read_data(self, spider, request):
def _read_data(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]:
key = self._fingerprinter.fingerprint(request).hex()
db = self.db
tkey = f"{key}_time"
if tkey not in db:
return # not found
return None # not found
ts = db[tkey]
if 0 < self.expiration_secs < time() - float(ts):
return # expired
return None # expired
return pickle.loads(db[f"{key}_data"])
return cast(Dict[str, Any], pickle.loads(db[f"{key}_data"])) # nosec
class FilesystemCacheStorage:
def __init__(self, settings):
self.cachedir = data_path(settings["HTTPCACHE_DIR"])
self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS")
self.use_gzip = settings.getbool("HTTPCACHE_GZIP")
self._open = gzip.open if self.use_gzip else open
def __init__(self, settings: BaseSettings):
self.cachedir: str = data_path(settings["HTTPCACHE_DIR"])
self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS")
self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP")
# https://github.com/python/mypy/issues/10740
self._open: Callable[Concatenate[Union[str, os.PathLike], str, ...], IO] = (
gzip.open if self.use_gzip else open # type: ignore[assignment]
)
def open_spider(self, spider: Spider):
def open_spider(self, spider: Spider) -> None:
logger.debug(
"Using filesystem cache storage in %(cachedir)s",
{"cachedir": self.cachedir},
@ -294,27 +329,29 @@ class FilesystemCacheStorage:
assert spider.crawler.request_fingerprinter
self._fingerprinter = spider.crawler.request_fingerprinter
def close_spider(self, spider):
def close_spider(self, spider: Spider) -> None:
pass
def retrieve_response(self, spider: Spider, request: Request):
def retrieve_response(self, spider: Spider, request: Request) -> Optional[Response]:
"""Return response if present in cache, or None otherwise."""
metadata = self._read_meta(spider, request)
if metadata is None:
return # not cached
return None # not cached
rpath = Path(self._get_request_path(spider, request))
with self._open(rpath / "response_body", "rb") as f:
body = f.read()
with self._open(rpath / "response_headers", "rb") as f:
rawheaders = f.read()
url = metadata.get("response_url")
url = metadata["response_url"]
status = metadata["status"]
headers = Headers(headers_raw_to_dict(rawheaders))
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
response = respcls(url=url, headers=headers, status=status, body=body)
return response
def store_response(self, spider: Spider, request: Request, response):
def store_response(
self, spider: Spider, request: Request, response: Response
) -> None:
"""Store the given response in the cache."""
rpath = Path(self._get_request_path(spider, request))
if not rpath.exists():
@ -343,19 +380,19 @@ class FilesystemCacheStorage:
key = self._fingerprinter.fingerprint(request).hex()
return str(Path(self.cachedir, spider.name, key[0:2], key))
def _read_meta(self, spider: Spider, request: Request):
def _read_meta(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]:
rpath = Path(self._get_request_path(spider, request))
metapath = rpath / "pickled_meta"
if not metapath.exists():
return # not found
return None # not found
mtime = metapath.stat().st_mtime
if 0 < self.expiration_secs < time() - mtime:
return # expired
return None # expired
with self._open(metapath, "rb") as f:
return pickle.load(f)
return cast(Dict[str, Any], pickle.load(f)) # nosec
def parse_cachecontrol(header):
def parse_cachecontrol(header: bytes) -> Dict[bytes, Optional[bytes]]:
"""Parse Cache-Control header
https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9
@ -375,9 +412,9 @@ def parse_cachecontrol(header):
return directives
def rfc1123_to_epoch(date_str):
def rfc1123_to_epoch(date_str: Union[str, bytes, None]) -> Optional[int]:
try:
date_str = to_unicode(date_str, encoding="ascii")
return mktime_tz(parsedate_tz(date_str))
date_str = to_unicode(date_str, encoding="ascii") # type: ignore[arg-type]
return mktime_tz(parsedate_tz(date_str)) # type: ignore[arg-type]
except Exception:
return None

82
scrapy/extensions/logstats.py
View File

@ -1,58 +1,94 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Optional, Tuple, Union
from twisted.internet import task
from scrapy import signals
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.statscollectors import StatsCollector
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
class LogStats:
"""Log basic scraping stats periodically"""
"""Log basic scraping stats periodically like:
* RPM - Requests per Minute
* IPM - Items per Minute
"""
def __init__(self, stats, interval=60.0):
self.stats = stats
self.interval = interval
self.multiplier = 60.0 / self.interval
self.task = None
def __init__(self, stats: StatsCollector, interval: float = 60.0):
self.stats: StatsCollector = stats
self.interval: float = interval
self.multiplier: float = 60.0 / self.interval
self.task: Optional[task.LoopingCall] = None
@classmethod
def from_crawler(cls, crawler):
interval = crawler.settings.getfloat("LOGSTATS_INTERVAL")
def from_crawler(cls, crawler: Crawler) -> Self:
interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL")
if not interval:
raise NotConfigured
assert crawler.stats
o = cls(crawler.stats, interval)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_opened(self, spider):
self.pagesprev = 0
self.itemsprev = 0
def spider_opened(self, spider: Spider) -> None:
self.pagesprev: int = 0
self.itemsprev: int = 0
self.task = task.LoopingCall(self.log, spider)
self.task.start(self.interval)
def log(self, spider):
items = self.stats.get_value("item_scraped_count", 0)
pages = self.stats.get_value("response_received_count", 0)
irate = (items - self.itemsprev) * self.multiplier
prate = (pages - self.pagesprev) * self.multiplier
self.pagesprev, self.itemsprev = pages, items
def log(self, spider: Spider) -> None:
self.calculate_stats()
msg = (
"Crawled %(pages)d pages (at %(pagerate)d pages/min), "
"scraped %(items)d items (at %(itemrate)d items/min)"
)
log_args = {
"pages": pages,
"pagerate": prate,
"items": items,
"itemrate": irate,
"pages": self.pages,
"pagerate": self.prate,
"items": self.items,
"itemrate": self.irate,
}
logger.info(msg, log_args, extra={"spider": spider})
def spider_closed(self, spider, reason):
def calculate_stats(self) -> None:
self.items: int = self.stats.get_value("item_scraped_count", 0)
self.pages: int = self.stats.get_value("response_received_count", 0)
self.irate: float = (self.items - self.itemsprev) * self.multiplier
self.prate: float = (self.pages - self.pagesprev) * self.multiplier
self.pagesprev, self.itemsprev = self.pages, self.items
def spider_closed(self, spider: Spider, reason: str) -> None:
if self.task and self.task.running:
self.task.stop()
rpm_final, ipm_final = self.calculate_final_stats(spider)
self.stats.set_value("responses_per_minute", rpm_final)
self.stats.set_value("items_per_minute", ipm_final)
def calculate_final_stats(
self, spider: Spider
) -> Union[Tuple[None, None], Tuple[float, float]]:
start_time = self.stats.get_value("start_time")
finished_time = self.stats.get_value("finished_time")
if not start_time or not finished_time:
return None, None
mins_elapsed = (finished_time - start_time).seconds / 60
items = self.stats.get_value("item_scraped_count", 0)
pages = self.stats.get_value("response_received_count", 0)
return (pages / mins_elapsed), (items / mins_elapsed)

22
scrapy/extensions/memdebug.py
View File

@ -4,26 +4,36 @@ MemoryDebugger extension
See documentation in docs/topics/extensions.rst
"""
import gc
from __future__ import annotations
from scrapy import signals
import gc
from typing import TYPE_CHECKING
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.statscollectors import StatsCollector
from scrapy.utils.trackref import live_refs
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class MemoryDebugger:
def __init__(self, stats):
self.stats = stats
def __init__(self, stats: StatsCollector):
self.stats: StatsCollector = stats
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("MEMDEBUG_ENABLED"):
raise NotConfigured
assert crawler.stats
o = cls(crawler.stats)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_closed(self, spider, reason):
def spider_closed(self, spider: Spider, reason: str) -> None:
gc.collect()
self.stats.set_value(
"memdebug/gc_garbage_count", len(gc.garbage), spider=spider

58
scrapy/extensions/memusage.py
View File

@ -3,24 +3,33 @@ MemoryUsage extension
See documentation in docs/topics/extensions.rst
"""
from __future__ import annotations
import logging
import socket
import sys
from importlib import import_module
from pprint import pformat
from typing import TYPE_CHECKING, List
from twisted.internet import task
from scrapy import signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.mail import MailSender
from scrapy.utils.engine import get_engine_status
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
class MemoryUsage:
def __init__(self, crawler):
def __init__(self, crawler: Crawler):
if not crawler.settings.getbool("MEMUSAGE_ENABLED"):
raise NotConfigured
try:
@ -29,32 +38,33 @@ class MemoryUsage:
except ImportError:
raise NotConfigured
self.crawler = crawler
self.warned = False
self.notify_mails = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL")
self.limit = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024
self.warning = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024
self.check_interval = crawler.settings.getfloat(
self.crawler: Crawler = crawler
self.warned: bool = False
self.notify_mails: List[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL")
self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024
self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024
self.check_interval: float = crawler.settings.getfloat(
"MEMUSAGE_CHECK_INTERVAL_SECONDS"
)
self.mail = MailSender.from_settings(crawler.settings)
self.mail: MailSender = MailSender.from_settings(crawler.settings)
crawler.signals.connect(self.engine_started, signal=signals.engine_started)
crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped)
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler)
def get_virtual_size(self):
size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss
def get_virtual_size(self) -> int:
size: int = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss
if sys.platform != "darwin":
# on macOS ru_maxrss is in bytes, on Linux it is in KB
size *= 1024
return size
def engine_started(self):
def engine_started(self) -> None:
assert self.crawler.stats
self.crawler.stats.set_value("memusage/startup", self.get_virtual_size())
self.tasks = []
self.tasks: List[task.LoopingCall] = []
tsk = task.LoopingCall(self.update)
self.tasks.append(tsk)
tsk.start(self.check_interval, now=True)
@ -67,15 +77,18 @@ class MemoryUsage:
self.tasks.append(tsk)
tsk.start(self.check_interval, now=True)
def engine_stopped(self):
def engine_stopped(self) -> None:
for tsk in self.tasks:
if tsk.running:
tsk.stop()
def update(self):
def update(self) -> None:
assert self.crawler.stats
self.crawler.stats.max_value("memusage/max", self.get_virtual_size())
def _check_limit(self):
def _check_limit(self) -> None:
assert self.crawler.engine
assert self.crawler.stats
peak_mem_usage = self.get_virtual_size()
if peak_mem_usage > self.limit:
self.crawler.stats.set_value("memusage/limit_reached", 1)
@ -105,9 +118,10 @@ class MemoryUsage:
{"virtualsize": peak_mem_usage / 1024 / 1024},
)
def _check_warning(self):
def _check_warning(self) -> None:
if self.warned: # warn only once
return
assert self.crawler.stats
if self.get_virtual_size() > self.warning:
self.crawler.stats.set_value("memusage/warning_reached", 1)
mem = self.warning / 1024 / 1024
@ -125,12 +139,14 @@ class MemoryUsage:
self.crawler.stats.set_value("memusage/warning_notified", 1)
self.warned = True
def _send_report(self, rcpts, subject):
def _send_report(self, rcpts: List[str], subject: str) -> None:
"""send notification mail with some additional useful info"""
assert self.crawler.engine
assert self.crawler.stats
stats = self.crawler.stats
s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n"
s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n"
s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n"
s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"
s += f"Maximum memory usage : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n"
s += f"Current memory usage : {self.get_virtual_size() / 1024 / 1024}M\r\n"
s += (
"ENGINE STATUS ------------------------------------------------------- \r\n"

89
scrapy/extensions/periodic_log.py
View File

@ -1,12 +1,22 @@
from __future__ import annotations
import logging
from datetime import datetime, timezone
from json import JSONEncoder
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from twisted.internet import task
from scrapy import signals
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.statscollectors import StatsCollector
from scrapy.utils.serialize import ScrapyJSONEncoder
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
@ -15,32 +25,34 @@ class PeriodicLog:
def __init__(
self,
stats,
interval=60.0,
ext_stats={},
ext_delta={},
ext_timing_enabled=False,
stats: StatsCollector,
interval: float = 60.0,
ext_stats: Dict[str, Any] = {},
ext_delta: Dict[str, Any] = {},
ext_timing_enabled: bool = False,
):
self.stats = stats
self.interval = interval
self.multiplier = 60.0 / self.interval
self.task = None
self.encoder = ScrapyJSONEncoder(sort_keys=True, indent=4)
self.ext_stats_enabled = bool(ext_stats)
self.ext_stats_include = ext_stats.get("include", [])
self.ext_stats_exclude = ext_stats.get("exclude", [])
self.ext_delta_enabled = bool(ext_delta)
self.ext_delta_include = ext_delta.get("include", [])
self.ext_delta_exclude = ext_delta.get("exclude", [])
self.ext_timing_enabled = ext_timing_enabled
self.stats: StatsCollector = stats
self.interval: float = interval
self.multiplier: float = 60.0 / self.interval
self.task: Optional[task.LoopingCall] = None
self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4)
self.ext_stats_enabled: bool = bool(ext_stats)
self.ext_stats_include: List[str] = ext_stats.get("include", [])
self.ext_stats_exclude: List[str] = ext_stats.get("exclude", [])
self.ext_delta_enabled: bool = bool(ext_delta)
self.ext_delta_include: List[str] = ext_delta.get("include", [])
self.ext_delta_exclude: List[str] = ext_delta.get("exclude", [])
self.ext_timing_enabled: bool = ext_timing_enabled
@classmethod
def from_crawler(cls, crawler):
interval = crawler.settings.getfloat("LOGSTATS_INTERVAL")
def from_crawler(cls, crawler: Crawler) -> Self:
interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL")
if not interval:
raise NotConfigured
try:
ext_stats = crawler.settings.getdict("PERIODIC_LOG_STATS")
ext_stats: Optional[Dict[str, Any]] = crawler.settings.getdict(
"PERIODIC_LOG_STATS"
)
except (TypeError, ValueError):
ext_stats = (
{"enabled": True}
@ -48,7 +60,9 @@ class PeriodicLog:
else None
)
try:
ext_delta = crawler.settings.getdict("PERIODIC_LOG_DELTA")
ext_delta: Optional[Dict[str, Any]] = crawler.settings.getdict(
"PERIODIC_LOG_DELTA"
)
except (TypeError, ValueError):
ext_delta = (
{"enabled": True}
@ -56,11 +70,14 @@ class PeriodicLog:
else None
)
ext_timing_enabled = crawler.settings.getbool(
ext_timing_enabled: bool = crawler.settings.getbool(
"PERIODIC_LOG_TIMING_ENABLED", False
)
if not (ext_stats or ext_delta or ext_timing_enabled):
raise NotConfigured
assert crawler.stats
assert ext_stats is not None
assert ext_delta is not None
o = cls(
crawler.stats,
interval,
@ -72,16 +89,16 @@ class PeriodicLog:
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_opened(self, spider):
self.time_prev = datetime.now(tz=timezone.utc)
self.delta_prev = {}
self.stats_prev = {}
def spider_opened(self, spider: Spider) -> None:
self.time_prev: datetime = datetime.now(tz=timezone.utc)
self.delta_prev: Dict[str, Union[int, float]] = {}
self.stats_prev: Dict[str, Union[int, float]] = {}
self.task = task.LoopingCall(self.log)
self.task.start(self.interval)
def log(self):
data = {}
def log(self) -> None:
data: Dict[str, Any] = {}
if self.ext_timing_enabled:
data.update(self.log_timing())
if self.ext_delta_enabled:
@ -90,8 +107,8 @@ class PeriodicLog:
data.update(self.log_crawler_stats())
logger.info(self.encoder.encode(data))
def log_delta(self):
num_stats = {
def log_delta(self) -> Dict[str, Any]:
num_stats: Dict[str, Union[int, float]] = {
k: v
for k, v in self.stats._stats.items()
if isinstance(v, (int, float))
@ -101,7 +118,7 @@ class PeriodicLog:
self.delta_prev = num_stats
return {"delta": delta}
def log_timing(self):
def log_timing(self) -> Dict[str, Any]:
now = datetime.now(tz=timezone.utc)
time = {
"log_interval": self.interval,
@ -113,7 +130,7 @@ class PeriodicLog:
self.time_prev = now
return {"time": time}
def log_crawler_stats(self):
def log_crawler_stats(self) -> Dict[str, Any]:
stats = {
k: v
for k, v in self.stats._stats.items()
@ -121,7 +138,9 @@ class PeriodicLog:
}
return {"stats": stats}
def param_allowed(self, stat_name, include, exclude):
def param_allowed(
self, stat_name: str, include: List[str], exclude: List[str]
) -> bool:
if not include and not exclude:
return True
for p in exclude:
@ -134,7 +153,7 @@ class PeriodicLog:
return True
return False
def spider_closed(self, spider, reason):
def spider_closed(self, spider: Spider, reason: str) -> None:
self.log()
if self.task and self.task.running:
self.task.stop()

10
scrapy/extensions/postprocessing.py
View File

@ -1,11 +1,12 @@
"""
Extension for processing data before they are exported to feeds.
"""
from bz2 import BZ2File
from gzip import GzipFile
from io import IOBase
from lzma import LZMAFile
from typing import Any, BinaryIO, Dict, List
from typing import IO, Any, BinaryIO, Dict, List, cast
from scrapy.utils.misc import load_object
@ -42,7 +43,6 @@ class GzipPlugin:
def close(self) -> None:
self.gzipfile.close()
self.file.close()
class Bz2Plugin:
@ -69,7 +69,6 @@ class Bz2Plugin:
def close(self) -> None:
self.bz2file.close()
self.file.close()
class LZMAPlugin:
@ -111,7 +110,6 @@ class LZMAPlugin:
def close(self) -> None:
self.lzmafile.close()
self.file.close()
# io.IOBase is subclassed here, so that exporters can use the PostProcessingManager
@ -128,7 +126,7 @@ class PostProcessingManager(IOBase):
"""
def __init__(
self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any]
self, plugins: List[Any], file: IO[bytes], feed_options: Dict[str, Any]
) -> None:
self.plugins = self._load_plugins(plugins)
self.file = file
@ -144,7 +142,7 @@ class PostProcessingManager(IOBase):
:return: returns number of bytes written
:rtype: int
"""
return self.head_plugin.write(data)
return cast(int, self.head_plugin.write(data))
def tell(self) -> int:
return self.file.tell()

30
scrapy/extensions/spiderstate.py
View File

@ -1,19 +1,27 @@
import pickle
from pathlib import Path
from __future__ import annotations
from scrapy import signals
import pickle # nosec
from pathlib import Path
from typing import TYPE_CHECKING, Optional
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.utils.job import job_dir
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class SpiderState:
"""Store and load spider state during a scraping job"""
def __init__(self, jobdir=None):
self.jobdir = jobdir
def __init__(self, jobdir: Optional[str] = None):
self.jobdir: Optional[str] = jobdir
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
jobdir = job_dir(crawler.settings)
if not jobdir:
raise NotConfigured
@ -23,18 +31,20 @@ class SpiderState:
crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened)
return obj
def spider_closed(self, spider):
def spider_closed(self, spider: Spider) -> None:
if self.jobdir:
with Path(self.statefn).open("wb") as f:
assert hasattr(spider, "state") # set in spider_opened
pickle.dump(spider.state, f, protocol=4)
def spider_opened(self, spider):
def spider_opened(self, spider: Spider) -> None:
if self.jobdir and Path(self.statefn).exists():
with Path(self.statefn).open("rb") as f:
spider.state = pickle.load(f)
spider.state = pickle.load(f) # type: ignore[attr-defined] # nosec
else:
spider.state = {}
spider.state = {} # type: ignore[attr-defined]
@property
def statefn(self) -> str:
assert self.jobdir
return str(Path(self.jobdir, "spider.state"))

31
scrapy/extensions/statsmailer.py
View File

@ -4,28 +4,41 @@ StatsMailer extension sends an email when a spider finishes scraping.
Use STATSMAILER_RCPTS setting to enable and give the recipient mail address
"""
from scrapy import signals
from __future__ import annotations
from typing import TYPE_CHECKING, List, Optional
from twisted.internet.defer import Deferred
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.mail import MailSender
from scrapy.statscollectors import StatsCollector
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class StatsMailer:
def __init__(self, stats, recipients, mail):
self.stats = stats
self.recipients = recipients
self.mail = mail
def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender):
self.stats: StatsCollector = stats
self.recipients: List[str] = recipients
self.mail: MailSender = mail
@classmethod
def from_crawler(cls, crawler):
recipients = crawler.settings.getlist("STATSMAILER_RCPTS")
def from_crawler(cls, crawler: Crawler) -> Self:
recipients: List[str] = crawler.settings.getlist("STATSMAILER_RCPTS")
if not recipients:
raise NotConfigured
mail = MailSender.from_settings(crawler.settings)
mail: MailSender = MailSender.from_settings(crawler.settings)
assert crawler.stats
o = cls(crawler.stats, recipients, mail)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_closed(self, spider):
def spider_closed(self, spider: Spider) -> Optional[Deferred]:
spider_stats = self.stats.get_stats(spider)
body = "Global stats\n\n"
body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items())

37
scrapy/extensions/telnet.py
View File

@ -4,13 +4,17 @@ Scrapy Telnet Console extension
See documentation in docs/topics/telnetconsole.rst
"""
from __future__ import annotations
import binascii
import logging
import os
import pprint
import traceback
from typing import TYPE_CHECKING, Any, Dict, List
from twisted.internet import protocol
from twisted.internet.tcp import Port
try:
from twisted.conch import manhole, telnet
@ -22,12 +26,16 @@ except (ImportError, SyntaxError):
TWISTED_CONCH_AVAILABLE = False
from scrapy import signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.utils.decorators import defers
from scrapy.utils.engine import print_engine_status
from scrapy.utils.reactor import listen_tcp
from scrapy.utils.trackref import print_live_refs
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
# signal to update telnet variables
@ -36,7 +44,7 @@ update_telnet_vars = object()
class TelnetConsole(protocol.ServerFactory):
def __init__(self, crawler):
def __init__(self, crawler: Crawler):
if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"):
raise NotConfigured
if not TWISTED_CONCH_AVAILABLE:
@ -44,14 +52,14 @@ class TelnetConsole(protocol.ServerFactory):
"TELNETCONSOLE_ENABLED setting is True but required twisted "
"modules failed to import:\n" + _TWISTED_CONCH_TRACEBACK
)
self.crawler = crawler
self.noisy = False
self.portrange = [
self.crawler: Crawler = crawler
self.noisy: bool = False
self.portrange: List[int] = [
int(x) for x in crawler.settings.getlist("TELNETCONSOLE_PORT")
]
self.host = crawler.settings["TELNETCONSOLE_HOST"]
self.username = crawler.settings["TELNETCONSOLE_USERNAME"]
self.password = crawler.settings["TELNETCONSOLE_PASSWORD"]
self.host: str = crawler.settings["TELNETCONSOLE_HOST"]
self.username: str = crawler.settings["TELNETCONSOLE_USERNAME"]
self.password: str = crawler.settings["TELNETCONSOLE_PASSWORD"]
if not self.password:
self.password = binascii.hexlify(os.urandom(8)).decode("utf8")
@ -61,11 +69,11 @@ class TelnetConsole(protocol.ServerFactory):
self.crawler.signals.connect(self.stop_listening, signals.engine_stopped)
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler)
def start_listening(self):
self.port = listen_tcp(self.portrange, self.host, self)
def start_listening(self) -> None:
self.port: Port = listen_tcp(self.portrange, self.host, self)
h = self.port.getHost()
logger.info(
"Telnet console listening on %(host)s:%(port)d",
@ -73,10 +81,10 @@ class TelnetConsole(protocol.ServerFactory):
extra={"crawler": self.crawler},
)
def stop_listening(self):
def stop_listening(self) -> None:
self.port.stopListening()
def protocol(self):
def protocol(self) -> telnet.TelnetTransport: # type: ignore[override]
class Portal:
"""An implementation of IPortal"""
@ -95,9 +103,10 @@ class TelnetConsole(protocol.ServerFactory):
return telnet.TelnetTransport(telnet.AuthenticatingTelnetProtocol, Portal())
def _get_telnet_vars(self):
def _get_telnet_vars(self) -> Dict[str, Any]:
# Note: if you add entries here also update topics/telnetconsole.rst
telnet_vars = {
assert self.crawler.engine
telnet_vars: Dict[str, Any] = {
"engine": self.crawler.engine,
"spider": self.crawler.engine.spider,
"slot": self.crawler.engine.slot,

56
scrapy/extensions/throttle.py
View File

@ -1,51 +1,68 @@
import logging
from __future__ import annotations
from scrapy import signals
import logging
from typing import TYPE_CHECKING, Optional, Tuple
from scrapy import Request, Spider, signals
from scrapy.core.downloader import Slot
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
class AutoThrottle:
def __init__(self, crawler):
self.crawler = crawler
def __init__(self, crawler: Crawler):
self.crawler: Crawler = crawler
if not crawler.settings.getbool("AUTOTHROTTLE_ENABLED"):
raise NotConfigured
self.debug = crawler.settings.getbool("AUTOTHROTTLE_DEBUG")
self.target_concurrency = crawler.settings.getfloat(
self.debug: bool = crawler.settings.getbool("AUTOTHROTTLE_DEBUG")
self.target_concurrency: float = crawler.settings.getfloat(
"AUTOTHROTTLE_TARGET_CONCURRENCY"
)
if self.target_concurrency <= 0.0:
raise NotConfigured(
f"AUTOTHROTTLE_TARGET_CONCURRENCY "
f"({self.target_concurrency!r}) must be higher than 0."
)
crawler.signals.connect(self._spider_opened, signal=signals.spider_opened)
crawler.signals.connect(
self._response_downloaded, signal=signals.response_downloaded
)
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler)
def _spider_opened(self, spider):
def _spider_opened(self, spider: Spider) -> None:
self.mindelay = self._min_delay(spider)
self.maxdelay = self._max_delay(spider)
spider.download_delay = self._start_delay(spider)
spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined]
def _min_delay(self, spider):
def _min_delay(self, spider: Spider) -> float:
s = self.crawler.settings
return getattr(spider, "download_delay", s.getfloat("DOWNLOAD_DELAY"))
def _max_delay(self, spider):
def _max_delay(self, spider: Spider) -> float:
return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY")
def _start_delay(self, spider):
def _start_delay(self, spider: Spider) -> float:
return max(
self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY")
)
def _response_downloaded(self, response, request, spider):
def _response_downloaded(
self, response: Response, request: Request, spider: Spider
) -> None:
key, slot = self._get_slot(request, spider)
latency = request.meta.get("download_latency")
if latency is None or slot is None:
if latency is None or slot is None or slot.throttle is False:
return
olddelay = slot.delay
@ -69,11 +86,16 @@ class AutoThrottle:
extra={"spider": spider},
)
def _get_slot(self, request, spider):
key = request.meta.get("download_slot")
def _get_slot(
self, request: Request, spider: Spider
) -> Tuple[Optional[str], Optional[Slot]]:
key: Optional[str] = request.meta.get("download_slot")
if key is None:
return None, None
assert self.crawler.engine
return key, self.crawler.engine.downloader.slots.get(key)
def _adjust_delay(self, slot, latency, response):
def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None:
"""Define delay adjustment policy"""
# If a server needs `latency` seconds to respond then

1
scrapy/http/__init__.py
View File

@ -12,5 +12,6 @@ from scrapy.http.request.json_request import JsonRequest
from scrapy.http.request.rpc import XmlRpcRequest
from scrapy.http.response import Response
from scrapy.http.response.html import HtmlResponse
from scrapy.http.response.json import JsonResponse
from scrapy.http.response.text import TextResponse
from scrapy.http.response.xml import XmlResponse

126
scrapy/http/cookies.py
View File

@ -1,36 +1,56 @@
from __future__ import annotations
import re
import time
from http.cookiejar import Cookie
from http.cookiejar import CookieJar as _CookieJar
from http.cookiejar import DefaultCookiePolicy
from typing import Sequence
from http.cookiejar import CookiePolicy, DefaultCookiePolicy
from typing import (
TYPE_CHECKING,
Any,
Dict,
Iterator,
List,
Optional,
Sequence,
Tuple,
cast,
)
from scrapy import Request
from scrapy.http import Response
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
# Defined in the http.cookiejar module, but undocumented:
# https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527
IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
class CookieJar:
def __init__(self, policy=None, check_expired_frequency=10000):
self.policy = policy or DefaultCookiePolicy()
self.jar = _CookieJar(self.policy)
self.jar._cookies_lock = _DummyLock()
self.check_expired_frequency = check_expired_frequency
self.processed = 0
def __init__(
self,
policy: Optional[CookiePolicy] = None,
check_expired_frequency: int = 10000,
):
self.policy: CookiePolicy = policy or DefaultCookiePolicy()
self.jar: _CookieJar = _CookieJar(self.policy)
self.jar._cookies_lock = _DummyLock() # type: ignore[attr-defined]
self.check_expired_frequency: int = check_expired_frequency
self.processed: int = 0
def extract_cookies(self, response, request):
def extract_cookies(self, response: Response, request: Request) -> None:
wreq = WrappedRequest(request)
wrsp = WrappedResponse(response)
return self.jar.extract_cookies(wrsp, wreq)
self.jar.extract_cookies(wrsp, wreq) # type: ignore[arg-type]
def add_cookie_header(self, request: Request) -> None:
wreq = WrappedRequest(request)
self.policy._now = self.jar._now = int(time.time())
self.policy._now = self.jar._now = int(time.time()) # type: ignore[attr-defined]
# the cookiejar implementation iterates through all domains
# instead we restrict to potential matches on the domain
@ -47,10 +67,10 @@ class CookieJar:
cookies = []
for host in hosts:
if host in self.jar._cookies:
cookies += self.jar._cookies_for_domain(host, wreq)
if host in self.jar._cookies: # type: ignore[attr-defined]
cookies += self.jar._cookies_for_domain(host, wreq) # type: ignore[attr-defined]
attrs = self.jar._cookie_attrs(cookies)
attrs = self.jar._cookie_attrs(cookies) # type: ignore[attr-defined]
if attrs:
if not wreq.has_header("Cookie"):
wreq.add_unredirected_header("Cookie", "; ".join(attrs))
@ -61,37 +81,42 @@ class CookieJar:
self.jar.clear_expired_cookies()
@property
def _cookies(self):
return self.jar._cookies
def _cookies(self) -> Dict[str, Dict[str, Dict[str, Cookie]]]:
return self.jar._cookies # type: ignore[attr-defined,no-any-return]
def clear_session_cookies(self, *args, **kwargs):
return self.jar.clear_session_cookies(*args, **kwargs)
def clear_session_cookies(self) -> None:
return self.jar.clear_session_cookies()
def clear(self, domain=None, path=None, name=None):
return self.jar.clear(domain, path, name)
def clear(
self,
domain: Optional[str] = None,
path: Optional[str] = None,
name: Optional[str] = None,
) -> None:
self.jar.clear(domain, path, name)
def __iter__(self):
def __iter__(self) -> Iterator[Cookie]:
return iter(self.jar)
def __len__(self):
def __len__(self) -> int:
return len(self.jar)
def set_policy(self, pol):
return self.jar.set_policy(pol)
def set_policy(self, pol: CookiePolicy) -> None:
self.jar.set_policy(pol)
def make_cookies(self, response: Response, request: Request) -> Sequence[Cookie]:
wreq = WrappedRequest(request)
wrsp = WrappedResponse(response)
return self.jar.make_cookies(wrsp, wreq)
return self.jar.make_cookies(wrsp, wreq) # type: ignore[arg-type]
def set_cookie(self, cookie):
def set_cookie(self, cookie: Cookie) -> None:
self.jar.set_cookie(cookie)
def set_cookie_if_ok(self, cookie: Cookie, request: Request) -> None:
self.jar.set_cookie_if_ok(cookie, WrappedRequest(request))
self.jar.set_cookie_if_ok(cookie, WrappedRequest(request)) # type: ignore[arg-type]
def potential_domain_matches(domain):
def potential_domain_matches(domain: str) -> List[str]:
"""Potential domain matches for a cookie
>>> potential_domain_matches('www.example.com')
@ -111,10 +136,10 @@ def potential_domain_matches(domain):
class _DummyLock:
def acquire(self):
def acquire(self) -> None:
pass
def release(self):
def release(self) -> None:
pass
@ -124,19 +149,19 @@ class WrappedRequest:
see http://docs.python.org/library/urllib2.html#urllib2.Request
"""
def __init__(self, request):
def __init__(self, request: Request):
self.request = request
def get_full_url(self):
def get_full_url(self) -> str:
return self.request.url
def get_host(self):
def get_host(self) -> str:
return urlparse_cached(self.request).netloc
def get_type(self):
def get_type(self) -> str:
return urlparse_cached(self.request).scheme
def is_unverifiable(self):
def is_unverifiable(self) -> bool:
"""Unverifiable should indicate whether the request is unverifiable, as defined by RFC 2965.
It defaults to False. An unverifiable request is one whose URL the user did not have the
@ -144,35 +169,36 @@ class WrappedRequest:
HTML document, and the user had no option to approve the automatic
fetching of the image, this should be true.
"""
return self.request.meta.get("is_unverifiable", False)
return cast(bool, self.request.meta.get("is_unverifiable", False))
@property
def full_url(self):
def full_url(self) -> str:
return self.get_full_url()
@property
def host(self):
def host(self) -> str:
return self.get_host()
@property
def type(self):
def type(self) -> str:
return self.get_type()
@property
def unverifiable(self):
def unverifiable(self) -> bool:
return self.is_unverifiable()
@property
def origin_req_host(self):
return urlparse_cached(self.request).hostname
def origin_req_host(self) -> str:
return cast(str, urlparse_cached(self.request).hostname)
def has_header(self, name):
def has_header(self, name: str) -> bool:
return name in self.request.headers
def get_header(self, name, default=None):
return to_unicode(self.request.headers.get(name, default), errors="replace")
def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]:
value = self.request.headers.get(name, default)
return to_unicode(value, errors="replace") if value is not None else None
def header_items(self):
def header_items(self) -> List[Tuple[str, List[str]]]:
return [
(
to_unicode(k, errors="replace"),
@ -181,18 +207,18 @@ class WrappedRequest:
for k, v in self.request.headers.items()
]
def add_unredirected_header(self, name, value):
def add_unredirected_header(self, name: str, value: str) -> None:
self.request.headers.appendlist(name, value)
class WrappedResponse:
def __init__(self, response):
def __init__(self, response: Response):
self.response = response
def info(self):
def info(self) -> Self:
return self
def get_all(self, name, default=None):
def get_all(self, name: str, default: Any = None) -> List[str]:
return [
to_unicode(v, errors="replace") for v in self.response.headers.getlist(name)
]

4
scrapy/http/headers.py
View File

@ -113,7 +113,9 @@ class Headers(CaselessDict):
return ((k, self.getlist(k)) for k in self.keys())
def values(self) -> List[Optional[bytes]]: # type: ignore[override]
return [self[k] for k in self.keys()]
return [
self[k] for k in self.keys() # pylint: disable=consider-using-dict-items
]
def to_string(self) -> bytes:
# cast() can be removed if the headers_dict_to_raw() hint is improved

30
scrapy/http/request/__init__.py
View File

@ -4,8 +4,12 @@ requests in Scrapy.
See documentation in docs/topics/request-response.rst
"""
from __future__ import annotations
import inspect
from typing import (
TYPE_CHECKING,
Any,
AnyStr,
Callable,
@ -16,8 +20,6 @@ from typing import (
NoReturn,
Optional,
Tuple,
Type,
TypeVar,
Union,
cast,
)
@ -31,7 +33,9 @@ from scrapy.utils.python import to_bytes
from scrapy.utils.trackref import object_ref
from scrapy.utils.url import escape_ajax
RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn:
@ -185,11 +189,11 @@ class Request(object_ref):
@classmethod
def from_curl(
cls: Type[RequestTypeVar],
cls,
curl_command: str,
ignore_unknown_options: bool = True,
**kwargs: Any,
) -> RequestTypeVar:
) -> Self:
"""Create a Request object from a string containing a `cURL
<https://curl.haxx.se/>`_ command. It populates the HTTP method, the
URL, the headers, the cookies and the body. It accepts the same
@ -231,12 +235,16 @@ class Request(object_ref):
"""
d = {
"url": self.url, # urls are safe (safe_string_url)
"callback": _find_method(spider, self.callback)
if callable(self.callback)
else self.callback,
"errback": _find_method(spider, self.errback)
if callable(self.errback)
else self.errback,
"callback": (
_find_method(spider, self.callback)
if callable(self.callback)
else self.callback
),
"errback": (
_find_method(spider, self.errback)
if callable(self.errback)
else self.errback
),
"headers": dict(self.headers),
}
for attr in self.attributes:

17
scrapy/http/request/form.py
View File

@ -10,21 +10,16 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast
from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
from lxml.html import (
FormElement,
HTMLParser,
InputElement,
MultipleSelectOptions,
SelectElement,
TextareaElement,
)
from parsel.selector import create_root_node
from lxml.html import FormElement # nosec
from lxml.html import InputElement # nosec
from lxml.html import MultipleSelectOptions # nosec
from lxml.html import SelectElement # nosec
from lxml.html import TextareaElement # nosec
from w3lib.html import strip_html5_whitespace
from scrapy.http.request import Request
from scrapy.http.response.text import TextResponse
from scrapy.utils.python import is_listlike, to_bytes
from scrapy.utils.response import get_base_url
if TYPE_CHECKING:
# typing.Self requires Python 3.11
@ -120,7 +115,7 @@ def _get_form(
formxpath: Optional[str],
) -> FormElement:
"""Find the wanted form element within the given response."""
root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response))
root = response.selector.root
forms = root.xpath("//form")
if not forms:
raise ValueError(f"No <form> element found in {response}")

5
scrapy/http/request/rpc.py
View File

@ -4,12 +4,17 @@ This module implements the XmlRpcRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
import xmlrpc.client as xmlrpclib
from typing import Any, Optional
import defusedxml.xmlrpc
from scrapy.http.request import Request
from scrapy.utils.python import get_func_args
defusedxml.xmlrpc.monkey_patch()
DUMPS_ARGS = get_func_args(xmlrpclib.dumps)

Some files were not shown because too many files have changed in this diff Show More