RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into HassaanNaushahi-BR_fix_issue-1998

This commit is contained in:
Hassaan Naushahi 2023-12-15 18:15:24 +05:00 committed by GitHub
parent 72962d6f5b 70ba3a0868
commit 32c7da5c5b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
201 changed files with 5019 additions and 2919 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.bumpversion.cfg
View File

@ -1,5 +1,5 @@
[bumpversion]
current_version = 2.9.0
current_version = 2.11.0
commit = True
tag = True
tag_name = {new_version}

13
.github/ISSUE_TEMPLATE/question.md vendored Normal file
View File

@ -0,0 +1,13 @@
---
name: Question / Help
about: Ask a question about Scrapy or ask for help with your Scrapy code.
---
Thanks for taking an interest in Scrapy!
The Scrapy GitHub issue tracker is not meant for questions or help. Please ask
for help in the [Scrapy community resources](https://scrapy.org/community/)
instead.
The GitHub issue tracker's purpose is to deal with bug reports and feature
requests for the project itself.

12
.github/workflows/checks.yml vendored
View File

@ -1,6 +1,10 @@
name: Checks
on: [push, pull_request]
concurrency:
group: ${{github.workflow}}-${{ github.ref }}
cancel-in-progress: true
jobs:
checks:
runs-on: ubuntu-latest
@ -8,7 +12,7 @@ jobs:
fail-fast: false
matrix:
include:
- python-version: "3.11"
- python-version: "3.12"
env:
TOXENV: pylint
- python-version: 3.8
@ -17,12 +21,12 @@ jobs:
- python-version: "3.11" # Keep in sync with .readthedocs.yml
env:
TOXENV: docs
- python-version: "3.11"
- python-version: "3.12"
env:
TOXENV: twinecheck
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
@ -38,5 +42,5 @@ jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: pre-commit/action@v3.0.0

8
.github/workflows/publish.yml vendored
View File

@ -4,14 +4,18 @@ on:
tags:
- '[0-9]+.[0-9]+.[0-9]+'
concurrency:
group: ${{github.workflow}}-${{ github.ref }}
cancel-in-progress: true
jobs:
publish:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: 3.11
python-version: 3.12
- run: |
pip install --upgrade build twine
python -m build

8
.github/workflows/tests-macos.yml vendored
View File

@ -1,16 +1,20 @@
name: macOS
on: [push, pull_request]
concurrency:
group: ${{github.workflow}}-${{ github.ref }}
cancel-in-progress: true
jobs:
tests:
runs-on: macos-11
strategy:
fail-fast: false
matrix:
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4

36
.github/workflows/tests-ubuntu.yml vendored
View File

@ -1,6 +1,10 @@
name: Ubuntu
on: [push, pull_request]
concurrency:
group: ${{github.workflow}}-${{ github.ref }}
cancel-in-progress: true
jobs:
tests:
runs-on: ubuntu-latest
@ -8,9 +12,6 @@ jobs:
fail-fast: false
matrix:
include:
- python-version: 3.8
env:
TOXENV: py
- python-version: 3.9
env:
TOXENV: py
@ -20,30 +21,45 @@ jobs:
- python-version: "3.11"
env:
TOXENV: py
- python-version: "3.11"
- python-version: "3.12"
env:
TOXENV: py
- python-version: "3.12"
env:
TOXENV: asyncio
- python-version: pypy3.9
env:
TOXENV: pypy3
- python-version: pypy3.10
env:
TOXENV: pypy3
# pinned deps
- python-version: 3.7.13
- python-version: 3.8.17
env:
TOXENV: pinned
- python-version: 3.7.13
- python-version: 3.8.17
env:
TOXENV: asyncio-pinned
- python-version: pypy3.7
- python-version: pypy3.8
env:
TOXENV: pypy3-pinned
- python-version: 3.8.17
env:
TOXENV: extra-deps-pinned
- python-version: 3.8.17
env:
TOXENV: botocore-pinned
- python-version: "3.11"
- python-version: "3.12"
env:
TOXENV: extra-deps
- python-version: "3.12"
env:
TOXENV: botocore
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
@ -51,7 +67,7 @@ jobs:
python-version: ${{ matrix.python-version }}
- name: Install system libraries
if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned')
if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned')
run: |
sudo apt-get update
sudo apt-get install libxml2-dev libxslt-dev

26
.github/workflows/tests-windows.yml vendored
View File

@ -1,6 +1,10 @@
name: Windows
on: [push, pull_request]
concurrency:
group: ${{github.workflow}}-${{ github.ref }}
cancel-in-progress: true
jobs:
tests:
runs-on: windows-latest
@ -8,31 +12,27 @@ jobs:
fail-fast: false
matrix:
include:
- python-version: 3.7
env:
TOXENV: windows-pinned
- python-version: 3.8
env:
TOXENV: py
TOXENV: windows-pinned
- python-version: 3.9
env:
TOXENV: py
- python-version: "3.10"
env:
TOXENV: py
- python-version: "3.10"
- python-version: "3.11"
env:
TOXENV: py
- python-version: "3.12"
env:
TOXENV: py
- python-version: "3.12"
env:
TOXENV: asyncio
# no binary package for lxml for 3.11 yet
# - python-version: "3.11"
# env:
# TOXENV: py
# - python-version: "3.11"
# env:
# TOXENV: asyncio
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4

10
.pre-commit-config.yaml
View File

@ -5,20 +5,20 @@ repos:
- id: bandit
args: [-r, -c, .bandit.yml]
- repo: https://github.com/PyCQA/flake8
rev: 5.0.4 # 6.0.0 drops Python 3.7 support
rev: 6.1.0
hooks:
- id: flake8
- repo: https://github.com/psf/black.git
rev: 23.3.0
rev: 23.9.1
hooks:
- id: black
- repo: https://github.com/pycqa/isort
rev: 5.11.5 # 5.12 drops Python 3.7 support
rev: 5.12.0
hooks:
- id: isort
- repo: https://github.com/adamchainz/blacken-docs
rev: 1.13.0
rev: 1.16.0
hooks:
- id: blacken-docs
additional_dependencies:
- black==23.3.0
- black==23.9.1

1
MANIFEST.in
View File

@ -7,6 +7,7 @@ include NEWS
include scrapy/VERSION
include scrapy/mime.types
include scrapy/py.typed
include codecov.yml
include conftest.py

13
README.rst
View File

@ -17,9 +17,10 @@ Scrapy
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu
:alt: Ubuntu
.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
:alt: macOS
.. .. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
.. :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
.. :alt: macOS
.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows
@ -41,7 +42,7 @@ Scrapy
Overview
========
Scrapy is a fast high-level web crawling and web scraping framework, used to
Scrapy is a BSD-licensed fast high-level web crawling and web scraping framework, used to
crawl websites and extract structured data from their pages. It can be used for
a wide range of purposes, from data mining to monitoring and automated testing.
@ -58,7 +59,7 @@ including a list of features.
Requirements
============
* Python 3.7+
* Python 3.8+
* Works on Linux, Windows, macOS, BSD
Install
@ -110,4 +111,4 @@ See https://scrapy.org/companies/ for a list.
Commercial Support
==================
See https://scrapy.org/support/ for details.
See https://scrapy.org/support/ for details.

22
conftest.py
View File

@ -1,6 +1,10 @@
import platform
import sys
from pathlib import Path
import pytest
from twisted import version as twisted_version
from twisted.python.versions import Version
from twisted.web.http import H2_ENABLED
from scrapy.utils.reactor import install_reactor
@ -14,6 +18,10 @@ def _py_files(folder):
collect_ignore = [
# not a test, but looks like a test
"scrapy/utils/testsite.py",
"tests/ftpserver.py",
"tests/mockserver.py",
"tests/pipelines.py",
"tests/spiders.py",
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
*_py_files("tests/CrawlerProcess"),
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
@ -73,6 +81,20 @@ def only_not_asyncio(request, reactor_pytest):
pytest.skip("This test is only run without --reactor=asyncio")
@pytest.fixture(autouse=True)
def requires_uvloop(request):
if not request.node.get_closest_marker("requires_uvloop"):
return
if sys.implementation.name == "pypy":
pytest.skip("uvloop does not support pypy properly")
if platform.system() == "Windows":
pytest.skip("uvloop does not support Windows")
if twisted_version == Version("twisted", 21, 2, 0):
pytest.skip("https://twistedmatrix.com/trac/ticket/10106")
if sys.version_info >= (3, 12):
pytest.skip("uvloop doesn't support Python 3.12 yet")
def pytest_configure(config):
if config.getoption("--reactor") == "asyncio":
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")

2
docs/conf.py
View File

@ -276,7 +276,7 @@ coverage_ignore_pyobjects = [
intersphinx_mapping = {
"attrs": ("https://www.attrs.org/en/stable/", None),
"coverage": ("https://coverage.readthedocs.io/en/stable", None),
"coverage": ("https://coverage.readthedocs.io/en/latest", None),
"cryptography": ("https://cryptography.io/en/latest/", None),
"cssselect": ("https://cssselect.readthedocs.io/en/latest", None),
"itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None),

12
docs/contributing.rst
View File

@ -265,15 +265,15 @@ To run a specific test (say ``tests/test_loader.py``) use:
To run the tests on a specific :doc:`tox <tox:index>` environment, use
``-e <name>`` with an environment name from ``tox.ini``. For example, to run
the tests with Python 3.7 use::
the tests with Python 3.10 use::
tox -e py37
tox -e py310
You can also specify a comma-separated list of environments, and use :ref:`tox’s
parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
parallel::
tox -e py37,py38 -p auto
tox -e py39,py310 -p auto
To pass command-line options to :doc:`pytest <pytest:index>`, add them after
``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
@ -283,9 +283,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
tox -- scrapy tests -x # stop after first failure
You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
the Python 3.7 :doc:`tox <tox:index>` environment using all your CPU cores::
the Python 3.10 :doc:`tox <tox:index>` environment using all your CPU cores::
tox -e py37 -- scrapy tests -n auto
tox -e py310 -- scrapy tests -n auto
To see coverage report install :doc:`coverage <coverage:index>`
(``pip install coverage``) and run:
@ -322,4 +322,4 @@ And their unit-tests are in::
.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
.. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22
.. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22
.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy
.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy

4
docs/index.rst
View File

@ -222,6 +222,7 @@ Extending Scrapy
:hidden:
topics/architecture
topics/addons
topics/downloader-middleware
topics/spider-middleware
topics/extensions
@ -235,6 +236,9 @@ Extending Scrapy
:doc:`topics/architecture`
Understand the Scrapy architecture.
:doc:`topics/addons`
Enable and configure third-party extensions.
:doc:`topics/downloader-middleware`
Customize how pages get requested and downloaded.

2
docs/intro/install.rst
View File

@ -9,7 +9,7 @@ Installation guide
Supported Python versions
=========================
Scrapy requires Python 3.7+, either the CPython implementation (default) or
Scrapy requires Python 3.8+, either the CPython implementation (default) or
the PyPy implementation (see :ref:`python:implementations`).
.. _intro-install-scrapy:

10
docs/intro/tutorial.rst
View File

@ -493,7 +493,15 @@ in the callback, as you can see below:
"tags": quote.css("div.tags a.tag::text").getall(),
}
If you run this spider, it will output the extracted data with the log::
To run this spider, exit the scrapy shell by entering::
quit()
Then, run::
scrapy crawl quotes
Now, it should output the extracted data with the log::
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
{'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'}

308
docs/news.rst
View File

@ -3,6 +3,307 @@
Release notes
=============
.. _release-2.11.0:
Scrapy 2.11.0 (2023-09-18)
--------------------------
Highlights:
- Spiders can now modify :ref:`settings <topics-settings>` in their
:meth:`~scrapy.Spider.from_crawler` methods, e.g. based on :ref:`spider
arguments <spiderargs>`.
- Periodic logging of stats.
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Most of the initialization of :class:`scrapy.crawler.Crawler` instances is
now done in :meth:`~scrapy.crawler.Crawler.crawl`, so the state of
instances before that method is called is now different compared to older
Scrapy versions. We do not recommend using the
:class:`~scrapy.crawler.Crawler` instances before
:meth:`~scrapy.crawler.Crawler.crawl` is called. (:issue:`6038`)
- :meth:`scrapy.Spider.from_crawler` is now called before the initialization
of various components previously initialized in
:meth:`scrapy.crawler.Crawler.__init__` and before the settings are
finalized and frozen. This change was needed to allow changing the settings
in :meth:`scrapy.Spider.from_crawler`. If you want to access the final
setting values and the initialized :class:`~scrapy.crawler.Crawler`
attributes in the spider code as early as possible you can do this in
:meth:`~scrapy.Spider.start_requests` or in a handler of the
:signal:`engine_started` signal. (:issue:`6038`)
- The :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method now
requires the response to be in a valid JSON encoding (UTF-8, UTF-16, or
UTF-32). If you need to deal with JSON documents in an invalid encoding,
use ``json.loads(response.text)`` instead. (:issue:`6016`)
- :class:`~scrapy.exporters.PythonItemExporter` used the binary output by
default but it no longer does. (:issue:`6006`, :issue:`6007`)
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
- Removed the binary export mode of
:class:`~scrapy.exporters.PythonItemExporter`, deprecated in Scrapy 1.1.0.
(:issue:`6006`, :issue:`6007`)
.. note:: If you are using this Scrapy version on Scrapy Cloud with a stack
that includes an older Scrapy version and get a "TypeError:
Unexpected options: binary" error, you may need to add
``scrapinghub-entrypoint-scrapy >= 0.14.1`` to your project
requirements or switch to a stack that includes Scrapy 2.11.
- Removed the ``CrawlerRunner.spiders`` attribute, deprecated in Scrapy
1.0.0, use :attr:`CrawlerRunner.spider_loader
<scrapy.crawler.CrawlerRunner.spider_loader>` instead. (:issue:`6010`)
Deprecations
~~~~~~~~~~~~
- Running :meth:`~scrapy.crawler.Crawler.crawl` more than once on the same
:class:`scrapy.crawler.Crawler` instance is now deprecated. (:issue:`1587`,
:issue:`6040`)
New features
~~~~~~~~~~~~
- Spiders can now modify settings in their
:meth:`~scrapy.Spider.from_crawler` method, e.g. based on :ref:`spider
arguments <spiderargs>`. (:issue:`1305`, :issue:`1580`, :issue:`2392`,
:issue:`3663`, :issue:`6038`)
- Added the :class:`~scrapy.extensions.periodic_log.PeriodicLog` extension
which can be enabled to log stats and/or their differences periodically.
(:issue:`5926`)
- Optimized the memory usage in :meth:`TextResponse.json
<scrapy.http.TextResponse.json>` by removing unnecessary body decoding.
(:issue:`5968`, :issue:`6016`)
- Links to ``.webp`` files are now ignored by :ref:`link extractors
<topics-link-extractors>`. (:issue:`6021`)
Bug fixes
~~~~~~~~~
- Fixed logging enabled add-ons. (:issue:`6036`)
- Fixed :class:`~scrapy.mail.MailSender` producing invalid message bodies
when the ``charset`` argument is passed to
:meth:`~scrapy.mail.MailSender.send`. (:issue:`5096`, :issue:`5118`)
- Fixed an exception when accessing ``self.EXCEPTIONS_TO_RETRY`` from a
subclass of :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware`.
(:issue:`6049`, :issue:`6050`)
- :meth:`scrapy.settings.BaseSettings.getdictorlist`, used to parse
:setting:`FEED_EXPORT_FIELDS`, now handles tuple values. (:issue:`6011`,
:issue:`6013`)
- Calls to ``datetime.utcnow()``, no longer recommended to be used, have been
replaced with calls to ``datetime.now()`` with a timezone. (:issue:`6014`)
Documentation
~~~~~~~~~~~~~
- Updated a deprecated function call in a pipeline example. (:issue:`6008`,
:issue:`6009`)
Quality assurance
~~~~~~~~~~~~~~~~~
- Extended typing hints. (:issue:`6003`, :issue:`6005`, :issue:`6031`,
:issue:`6034`)
- Pinned brotli_ to 1.0.9 for the PyPy tests as 1.1.0 breaks them.
(:issue:`6044`, :issue:`6045`)
- Other CI and pre-commit improvements. (:issue:`6002`, :issue:`6013`,
:issue:`6046`)
.. _release-2.10.1:
Scrapy 2.10.1 (2023-08-30)
--------------------------
Marked ``Twisted >= 23.8.0`` as unsupported. (:issue:`6024`, :issue:`6026`)
.. _release-2.10.0:
Scrapy 2.10.0 (2023-08-04)
--------------------------
Highlights:
- Added Python 3.12 support, dropped Python 3.7 support.
- The new add-ons framework simplifies configuring 3rd-party components that
support it.
- Exceptions to retry can now be configured.
- Many fixes and improvements for feed exports.
Modified requirements
~~~~~~~~~~~~~~~~~~~~~
- Dropped support for Python 3.7. (:issue:`5953`)
- Added support for the upcoming Python 3.12. (:issue:`5984`)
- Minimum versions increased for these dependencies:
- lxml_: 4.3.0 → 4.4.1
- cryptography_: 3.4.6 → 36.0.0
- ``pkg_resources`` is no longer used. (:issue:`5956`, :issue:`5958`)
- boto3_ is now recommended instead of botocore_ for exporting to S3.
(:issue:`5833`).
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True``
instead of ``False``. In earlier Scrapy versions empty files were created
even when this setting was ``False`` (which was a bug that is now fixed),
so the new default should keep the old behavior. (:issue:`872`,
:issue:`5847`)
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting,
returning ``None`` or modifying the ``params`` input parameter, deprecated
in Scrapy 2.6, is no longer supported. (:issue:`5994`, :issue:`5996`)
- The ``scrapy.utils.reqser`` module, deprecated in Scrapy 2.6, is removed.
(:issue:`5994`, :issue:`5996`)
- The ``scrapy.squeues`` classes ``PickleFifoDiskQueueNonRequest``,
``PickleLifoDiskQueueNonRequest``, ``MarshalFifoDiskQueueNonRequest``,
and ``MarshalLifoDiskQueueNonRequest``, deprecated in
Scrapy 2.6, are removed. (:issue:`5994`, :issue:`5996`)
- The property ``open_spiders`` and the methods ``has_capacity`` and
``schedule`` of :class:`scrapy.core.engine.ExecutionEngine`,
deprecated in Scrapy 2.6, are removed. (:issue:`5994`, :issue:`5998`)
- Passing a ``spider`` argument to the
:meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`,
:meth:`~scrapy.core.engine.ExecutionEngine.crawl` and
:meth:`~scrapy.core.engine.ExecutionEngine.download` methods of
:class:`scrapy.core.engine.ExecutionEngine`, deprecated in Scrapy 2.6, is
no longer supported. (:issue:`5994`, :issue:`5998`)
Deprecations
~~~~~~~~~~~~
- :class:`scrapy.utils.datatypes.CaselessDict` is deprecated, use
:class:`scrapy.utils.datatypes.CaseInsensitiveDict` instead.
(:issue:`5146`)
- Passing the ``custom`` argument to
:func:`scrapy.utils.conf.build_component_list` is deprecated, it was used
in the past to merge ``FOO`` and ``FOO_BASE`` setting values but now Scrapy
uses :func:`scrapy.settings.BaseSettings.getwithbase` to do the same.
Code that uses this argument and cannot be switched to ``getwithbase()``
can be switched to merging the values explicitly. (:issue:`5726`,
:issue:`5923`)
New features
~~~~~~~~~~~~
- Added support for :ref:`Scrapy add-ons <topics-addons>`. (:issue:`5950`)
- Added the :setting:`RETRY_EXCEPTIONS` setting that configures which
exceptions will be retried by
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware`.
(:issue:`2701`, :issue:`5929`)
- Added the possiiblity to close the spider if no items were produced in the
specified time, configured by :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`.
(:issue:`5979`)
- Added support for the :setting:`AWS_REGION_NAME` setting to feed exports.
(:issue:`5980`)
- Added support for using :class:`pathlib.Path` objects that refer to
absolute Windows paths in the :setting:`FEEDS` setting. (:issue:`5939`)
Bug fixes
~~~~~~~~~
- Fixed creating empty feeds even with ``FEED_STORE_EMPTY=False``.
(:issue:`872`, :issue:`5847`)
- Fixed using absolute Windows paths when specifying output files.
(:issue:`5969`, :issue:`5971`)
- Fixed problems with uploading large files to S3 by switching to multipart
uploads (requires boto3_). (:issue:`960`, :issue:`5735`, :issue:`5833`)
- Fixed the JSON exporter writing extra commas when some exceptions occur.
(:issue:`3090`, :issue:`5952`)
- Fixed the "read of closed file" error in the CSV exporter. (:issue:`5043`,
:issue:`5705`)
- Fixed an error when a component added by the class object throws
:exc:`~scrapy.exceptions.NotConfigured` with a message. (:issue:`5950`,
:issue:`5992`)
- Added the missing :meth:`scrapy.settings.BaseSettings.pop` method.
(:issue:`5959`, :issue:`5960`, :issue:`5963`)
- Added :class:`~scrapy.utils.datatypes.CaseInsensitiveDict` as a replacement
for :class:`~scrapy.utils.datatypes.CaselessDict` that fixes some API
inconsistencies. (:issue:`5146`)
Documentation
~~~~~~~~~~~~~
- Documented :meth:`scrapy.Spider.update_settings`. (:issue:`5745`,
:issue:`5846`)
- Documented possible problems with early Twisted reactor installation and
their solutions. (:issue:`5981`, :issue:`6000`)
- Added examples of making additional requests in callbacks. (:issue:`5927`)
- Improved the feed export docs. (:issue:`5579`, :issue:`5931`)
- Clarified the docs about request objects on redirection. (:issue:`5707`,
:issue:`5937`)
Quality assurance
~~~~~~~~~~~~~~~~~
- Added support for running tests against the installed Scrapy version.
(:issue:`4914`, :issue:`5949`)
- Extended typing hints. (:issue:`5925`, :issue:`5977`)
- Fixed the ``test_utils_asyncio.AsyncioTest.test_set_asyncio_event_loop``
test. (:issue:`5951`)
- Fixed the ``test_feedexport.BatchDeliveriesTest.test_batch_path_differ``
test on Windows. (:issue:`5847`)
- Enabled CI runs for Python 3.11 on Windows. (:issue:`5999`)
- Simplified skipping tests that depend on ``uvloop``. (:issue:`5984`)
- Fixed the ``extra-deps-pinned`` tox env. (:issue:`5948`)
- Implemented cleanups. (:issue:`5965`, :issue:`5986`)
.. _release-2.9.0:
Scrapy 2.9.0 (2023-05-08)
@ -25,7 +326,7 @@ Deprecations
New features
~~~~~~~~~~~~
- Settings correponding to :setting:`DOWNLOAD_DELAY`,
- Settings corresponding to :setting:`DOWNLOAD_DELAY`,
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
:setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis
via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`)
@ -173,6 +474,10 @@ Deprecation removals
has now been removed.
(:issue:`5719`)
- The ``scrapy.utils.boto.is_botocore()`` function, deprecated in Scrapy 2.4,
has now been removed.
(:issue:`5719`)
Deprecations
~~~~~~~~~~~~
@ -5748,6 +6053,7 @@ First release of Scrapy.
.. _AJAX crawlable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1
.. _boto3: https://github.com/boto/boto3
.. _botocore: https://github.com/boto/botocore
.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
.. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/

193
docs/topics/addons.rst Normal file
View File

@ -0,0 +1,193 @@
.. _topics-addons:
=======
Add-ons
=======
Scrapy's add-on system is a framework which unifies managing and configuring
components that extend Scrapy's core functionality, such as middlewares,
extensions, or pipelines. It provides users with a plug-and-play experience in
Scrapy extension management, and grants extensive configuration control to
developers.
Activating and configuring add-ons
==================================
During :class:`~scrapy.crawler.Crawler` initialization, the list of enabled
add-ons is read from your ``ADDONS`` setting.
The ``ADDONS`` setting is a dict in which every key is an add-on class or its
import path and the value is its priority.
This is an example where two add-ons are enabled in a project's
``settings.py``::
ADDONS = {
'path.to.someaddon': 0,
SomeAddonClass: 1,
}
Writing your own add-ons
========================
Add-ons are Python classes that include the following method:
.. method:: update_settings(settings)
This method is called during the initialization of the
:class:`~scrapy.crawler.Crawler`. Here, you should perform dependency checks
(e.g. for external Python libraries) and update the
:class:`~scrapy.settings.Settings` object as wished, e.g. enable components
for this add-on or set required configuration of other extensions.
:param settings: The settings object storing Scrapy/component configuration
:type settings: :class:`~scrapy.settings.Settings`
They can also have the following method:
.. classmethod:: from_crawler(cls, crawler)
:noindex:
If present, this class method is called to create an add-on instance
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
of the add-on. The crawler object provides access to all Scrapy core
components like settings and signals; it is a way for the add-on to access
them and hook its functionality into Scrapy.
:param crawler: The crawler that uses this add-on
:type crawler: :class:`~scrapy.crawler.Crawler`
The settings set by the add-on should use the ``addon`` priority (see
:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`)::
class MyAddon:
def update_settings(self, settings):
settings.set("DNSCACHE_ENABLED", True, "addon")
This allows users to override these settings in the project or spider
configuration. This is not possible with settings that are mutable objects,
such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases
you can provide an add-on-specific setting that governs whether the add-on will
modify :setting:`ITEM_PIPELINES`::
class MyAddon:
def update_settings(self, settings):
if settings.getbool("MYADDON_ENABLE_PIPELINE"):
settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200
If the ``update_settings`` method raises
:exc:`scrapy.exceptions.NotConfigured`, the add-on will be skipped. This makes
it easy to enable an add-on only when some conditions are met.
Fallbacks
---------
Some components provided by add-ons need to fall back to "default"
implementations, e.g. a custom download handler needs to send the request that
it doesn't handle via the default download handler, or a stats collector that
includes some additional processing but otherwise uses the default stats
collector. And it's possible that a project needs to use several custom
components of the same type, e.g. two custom download handlers that support
different kinds of custom requests and still need to use the default download
handler for other requests. To make such use cases easier to configure, we
recommend that such custom components should be written in the following way:
1. The custom component (e.g. ``MyDownloadHandler``) shouldn't inherit from the
default Scrapy one (e.g.
``scrapy.core.downloader.handlers.http.HTTPDownloadHandler``), but instead
be able to load the class of the fallback component from a special setting
(e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use
it.
2. The add-ons that include these components should read the current value of
the default setting (e.g. ``DOWNLOAD_HANDLERS``) in their
``update_settings()`` methods, save that value into the fallback setting
(``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default
setting to the component provided by the add-on (e.g.
``MyDownloadHandler``). If the fallback setting is already set by the user,
they shouldn't change it.
3. This way, if there are several add-ons that want to modify the same setting,
all of them will fallback to the component from the previous one and then to
the Scrapy default. The order of that depends on the priority order in the
``ADDONS`` setting.
Add-on examples
===============
Set some basic configuration:
.. code-block:: python
class MyAddon:
def update_settings(self, settings):
settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200
settings.set("DNSCACHE_ENABLED", True, "addon")
Check dependencies:
.. code-block:: python
class MyAddon:
def update_settings(self, settings):
try:
import boto
except ImportError:
raise NotConfigured("MyAddon requires the boto library")
...
Access the crawler instance:
.. code-block:: python
class MyAddon:
def __init__(self, crawler) -> None:
super().__init__()
self.crawler = crawler
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def update_settings(self, settings):
...
Use a fallback component:
.. code-block:: python
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER"
class MyHandler:
lazy = False
def __init__(self, settings, crawler):
dhcls = load_object(settings.get(FALLBACK_SETTING))
self._fallback_handler = create_instance(
dhcls,
settings=None,
crawler=crawler,
)
def download_request(self, request, spider):
if request.meta.get("my_params"):
# handle the request
...
else:
return self._fallback_handler.download_request(request, spider)
class MyAddon:
def update_settings(self, settings):
if not settings.get(FALLBACK_SETTING):
settings.set(
FALLBACK_SETTING,
settings.getwithbase("DOWNLOAD_HANDLERS")["https"],
"addon",
)
settings["DOWNLOAD_HANDLERS"]["https"] = MyHandler

3
docs/topics/api.rst
View File

@ -100,7 +100,7 @@ how you :ref:`configure the downloader middlewares
Starts the crawler by instantiating its spider class with the given
``args`` and ``kwargs`` arguments, while setting the execution engine in
motion.
motion. Should be called only once.
Returns a deferred that is fired when the crawl is finished.
@ -137,6 +137,7 @@ Settings API
SETTINGS_PRIORITIES = {
"default": 0,
"command": 10,
"addon": 15,
"project": 20,
"spider": 30,
"cmdline": 40,

100
docs/topics/asyncio.rst
View File

@ -27,54 +27,43 @@ reactor manually. You can do that using
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
.. _using-custom-loops:
.. _asyncio-preinstalled-reactor:
Using custom asyncio loops
==========================
Handling a pre-installed reactor
================================
You can also use custom asyncio event loops with the asyncio reactor. Set the
:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to
use it instead of the default asyncio event loop.
``twisted.internet.reactor`` and some other Twisted imports install the default
Twisted reactor as a side effect. Once a Twisted reactor is installed, it is
not possible to switch to a different reactor at run time.
If you :ref:`configure the asyncio Twisted reactor <install-asyncio>` and, at
run time, Scrapy complains that a different reactor is already installed,
chances are you have some such imports in your code.
You can usually fix the issue by moving those offending module-level Twisted
imports to the method or function definitions where they are used. For example,
if you have something like:
.. code-block:: python
from twisted.internet import reactor
.. _asyncio-windows:
def my_function():
reactor.callLater(...)
Windows-specific notes
======================
Switch to something like:
The Windows implementation of :mod:`asyncio` can use two event loop
implementations:
.. code-block:: python
- :class:`~asyncio.SelectorEventLoop`, default before Python 3.8, required
when using Twisted.
def my_function():
from twisted.internet import reactor
- :class:`~asyncio.ProactorEventLoop`, default since Python 3.8, cannot work
with Twisted.
reactor.callLater(...)
So on Python 3.8+ the event loop class needs to be changed.
.. versionchanged:: 2.6.0
The event loop class is changed automatically when you change the
:setting:`TWISTED_REACTOR` setting or call
:func:`~scrapy.utils.reactor.install_reactor`.
To change the event loop class manually, call the following code before
installing the reactor::
import asyncio
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
You can put this in the same function that installs the reactor, if you do that
yourself, or in some code that runs before the reactor is installed, e.g.
``settings.py``.
.. note:: Other libraries you use may require
:class:`~asyncio.ProactorEventLoop`, e.g. because it supports
subprocesses (this is the case with `playwright`_), so you cannot use
them together with Scrapy on Windows (but you should be able to use
them on WSL or native Linux).
.. _playwright: https://github.com/microsoft/playwright-python
Alternatively, you can try to :ref:`manually install the asyncio reactor
<install-asyncio>`, with :func:`~scrapy.utils.reactor.install_reactor`, before
those imports happen.
.. _asyncio-await-dfd:
@ -122,3 +111,36 @@ example:
f"TWISTED_REACTOR setting. See the asyncio documentation "
f"of Scrapy for more information."
)
.. _asyncio-windows:
Windows-specific notes
======================
The Windows implementation of :mod:`asyncio` can use two event loop
implementations, :class:`~asyncio.ProactorEventLoop` (default) and
:class:`~asyncio.SelectorEventLoop`. However, only
:class:`~asyncio.SelectorEventLoop` works with Twisted.
Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop`
automatically when you change the :setting:`TWISTED_REACTOR` setting or call
:func:`~scrapy.utils.reactor.install_reactor`.
.. note:: Other libraries you use may require
:class:`~asyncio.ProactorEventLoop`, e.g. because it supports
subprocesses (this is the case with `playwright`_), so you cannot use
them together with Scrapy on Windows (but you should be able to use
them on WSL or native Linux).
.. _playwright: https://github.com/microsoft/playwright-python
.. _using-custom-loops:
Using custom asyncio loops
==========================
You can also use custom asyncio event loops with the asyncio reactor. Set the
:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event
loop class to use it instead of the default asyncio event loop.

2
docs/topics/components.rst
View File

@ -70,7 +70,7 @@ If your requirement is a minimum Scrapy version, you may use
.. code-block:: python
from pkg_resources import parse_version
from packaging.version import parse as parse_version
import scrapy

35
docs/topics/downloader-middleware.rst
View File

@ -915,6 +915,7 @@ settings (see the settings documentation for more info):
* :setting:`RETRY_ENABLED`
* :setting:`RETRY_TIMES`
* :setting:`RETRY_HTTP_CODES`
* :setting:`RETRY_EXCEPTIONS`
.. reqmeta:: dont_retry
@ -966,6 +967,37 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because
it is a common code used to indicate server overload. It is not included by
default because HTTP specs say so.
.. setting:: RETRY_EXCEPTIONS
RETRY_EXCEPTIONS
^^^^^^^^^^^^^^^^
Default::
[
'twisted.internet.defer.TimeoutError',
'twisted.internet.error.TimeoutError',
'twisted.internet.error.DNSLookupError',
'twisted.internet.error.ConnectionRefusedError',
'twisted.internet.error.ConnectionDone',
'twisted.internet.error.ConnectError',
'twisted.internet.error.ConnectionLost',
'twisted.internet.error.TCPTimedOutError',
'twisted.web.client.ResponseFailed',
IOError,
'scrapy.core.downloader.handlers.http11.TunnelError',
]
List of exceptions to retry.
Each list entry may be an exception type or its import path as a string.
An exception will not be caught when the exception type is not in
:setting:`RETRY_EXCEPTIONS` or when the maximum number of retries for a request
has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught
exception propagation, see
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`.
.. setting:: RETRY_PRIORITY_ADJUST
RETRY_PRIORITY_ADJUST
@ -1007,8 +1039,8 @@ RobotsTxtMiddleware
* :ref:`Protego <protego-parser>` (default)
* :ref:`RobotFileParser <python-robotfileparser>`
* :ref:`Reppy <reppy-parser>`
* :ref:`Robotexclusionrulesparser <rerp-parser>`
* :ref:`Reppy <reppy-parser>` (deprecated)
You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
@ -1101,6 +1133,7 @@ In order to use this parser:
.. warning:: `Upstream issue #122
<https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in Python 3.9+.
Because of this the Reppy parser is deprecated.
* Set :setting:`ROBOTSTXT_PARSER` setting to
``scrapy.robotstxt.ReppyRobotParser``

129
docs/topics/extensions.rst
View File

@ -258,6 +258,7 @@ The conditions for closing a spider can be configured through the following
settings:
* :setting:`CLOSESPIDER_TIMEOUT`
* :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`
* :setting:`CLOSESPIDER_ITEMCOUNT`
* :setting:`CLOSESPIDER_PAGECOUNT`
* :setting:`CLOSESPIDER_ERRORCOUNT`
@ -280,6 +281,18 @@ more than that number of second, it will be automatically closed with the
reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by
timeout.
.. setting:: CLOSESPIDER_TIMEOUT_NO_ITEM
CLOSESPIDER_TIMEOUT_NO_ITEM
"""""""""""""""""""""""""""
Default: ``0``
An integer which specifies a number of seconds. If the spider has not produced
any items in the last number of seconds, it will be closed with the reason
``closespider_timeout_no_item``. If zero (or non set), spiders won't be closed
regardless if it hasn't produced any items.
.. setting:: CLOSESPIDER_ITEMCOUNT
CLOSESPIDER_ITEMCOUNT
@ -337,6 +350,122 @@ full list of parameters, including examples on how to instantiate
.. module:: scrapy.extensions.debug
:synopsis: Extensions for debugging Scrapy
.. module:: scrapy.extensions.periodic_log
:synopsis: Periodic stats logging
Periodic log extension
~~~~~~~~~~~~~~~~~~~~~~
.. class:: PeriodicLog
This extension periodically logs rich stat data as a JSON object::
2023-08-04 02:30:57 [scrapy.extensions.logstats] INFO: Crawled 976 pages (at 162 pages/min), scraped 925 items (at 161 items/min)
2023-08-04 02:30:57 [scrapy.extensions.periodic_log] INFO: {
"delta": {
"downloader/request_bytes": 55582,
"downloader/request_count": 162,
"downloader/request_method_count/GET": 162,
"downloader/response_bytes": 618133,
"downloader/response_count": 162,
"downloader/response_status_count/200": 162,
"item_scraped_count": 161
},
"stats": {
"downloader/request_bytes": 338243,
"downloader/request_count": 992,
"downloader/request_method_count/GET": 992,
"downloader/response_bytes": 3836736,
"downloader/response_count": 976,
"downloader/response_status_count/200": 976,
"item_scraped_count": 925,
"log_count/INFO": 21,
"log_count/WARNING": 1,
"scheduler/dequeued": 992,
"scheduler/dequeued/memory": 992,
"scheduler/enqueued": 1050,
"scheduler/enqueued/memory": 1050
},
"time": {
"elapsed": 360.008903,
"log_interval": 60.0,
"log_interval_real": 60.006694,
"start_time": "2023-08-03 23:24:57",
"utcnow": "2023-08-03 23:30:57"
}
}
This extension logs the following configurable sections:
- ``"delta"`` shows how some numeric stats have changed since the last stats
log message.
The :setting:`PERIODIC_LOG_DELTA` setting determines the target stats. They
must have ``int`` or ``float`` values.
- ``"stats"`` shows the current value of some stats.
The :setting:`PERIODIC_LOG_STATS` setting determines the target stats.
- ``"time"`` shows detailed timing data.
The :setting:`PERIODIC_LOG_TIMING_ENABLED` setting determines whether or
not to show this section.
This extension logs data at the start, then on a fixed time interval
configurable through the :setting:`LOGSTATS_INTERVAL` setting, and finally
right before the crawl ends.
Example extension configuration:
.. code-block:: python
custom_settings = {
"LOG_LEVEL": "INFO",
"PERIODIC_LOG_STATS": {
"include": ["downloader/", "scheduler/", "log_count/", "item_scraped_count/"],
},
"PERIODIC_LOG_DELTA": {"include": ["downloader/"]},
"PERIODIC_LOG_TIMING_ENABLED": True,
"EXTENSIONS": {
"scrapy.extensions.periodic_log.PeriodicLog": 0,
},
}
.. setting:: PERIODIC_LOG_DELTA
PERIODIC_LOG_DELTA
""""""""""""""""""
Default: ``None``
* ``"PERIODIC_LOG_DELTA": True`` - show deltas for all ``int`` and ``float`` stat values.
* ``"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}`` - show deltas for stats with names containing any configured substring.
* ``"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}`` - show deltas for all stats with names not containing any configured substring.
.. setting:: PERIODIC_LOG_STATS
PERIODIC_LOG_STATS
""""""""""""""""""
Default: ``None``
* ``"PERIODIC_LOG_STATS": True`` - show the current value of all stats.
* ``"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}`` - show current values for stats with names containing any configured substring.
* ``"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}`` - show current values for all stats with names not containing any configured substring.
.. setting:: PERIODIC_LOG_TIMING_ENABLED
PERIODIC_LOG_TIMING_ENABLED
"""""""""""""""""""""""""""
Default: ``False``
``True`` enables logging of timing data (i.e. the ``"time"`` section).
Debugging extensions
--------------------

21
docs/topics/feed-exports.rst
View File

@ -101,12 +101,12 @@ The storages backends supported out of the box are:
- :ref:`topics-feed-storage-fs`
- :ref:`topics-feed-storage-ftp`
- :ref:`topics-feed-storage-s3` (requires botocore_)
- :ref:`topics-feed-storage-s3` (requires boto3_)
- :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
- :ref:`topics-feed-storage-stdout`
Some storage backends may be unavailable if the required external libraries are
not available. For example, the S3 backend is only available if the botocore_
not available. For example, the S3 backend is only available if the boto3_
library is installed.
@ -156,8 +156,8 @@ The feeds are stored in the local filesystem.
- Required external libraries: none
Note that for the local filesystem storage (only) you can omit the scheme if
you specify an absolute path like ``/tmp/export.csv``. This only works on Unix
systems though.
you specify an absolute path like ``/tmp/export.csv`` (Unix systems only).
Alternatively you can also use a :class:`pathlib.Path` object.
.. _topics-feed-storage-ftp:
@ -199,7 +199,7 @@ The feeds are stored on `Amazon S3`_.
- ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
- Required external libraries: `botocore`_ >= 1.4.87
- Required external libraries: `boto3`_ >= 1.20.0
The AWS credentials can be passed as user/password in the URI, or they can be
passed through the following settings:
@ -210,10 +210,12 @@ passed through the following settings:
.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
You can also define a custom ACL and custom endpoint for exported feeds using this setting:
You can also define a custom ACL, custom endpoint, and region name for exported
feeds using these settings:
- :setting:`FEED_STORAGE_S3_ACL`
- :setting:`AWS_ENDPOINT_URL`
- :setting:`AWS_REGION_NAME`
The default value for the ``overwrite`` key in the :setting:`FEEDS` for this
storage backend is: ``True``.
@ -572,9 +574,12 @@ to ``.json`` or ``.xml``.
FEED_STORE_EMPTY
----------------
Default: ``False``
Default: ``True``
Whether to export empty feeds (i.e. feeds with no items).
If ``False``, and there are no items to export, no new files are created and
existing files are not modified, even if the :ref:`overwrite feed option
<feed-options>` is enabled.
.. setting:: FEED_STORAGES
@ -799,6 +804,6 @@ source spider in the feed URI:
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _Amazon S3: https://aws.amazon.com/s3/
.. _botocore: https://github.com/boto/botocore
.. _boto3: https://github.com/boto/boto3
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. _Google Cloud Storage: https://cloud.google.com/storage/

2
docs/topics/item-pipeline.rst
View File

@ -215,7 +215,7 @@ item.
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
request = scrapy.Request(screenshot_url, callback=NO_CALLBACK)
response = await maybe_deferred_to_future(
spider.crawler.engine.download(request, spider)
spider.crawler.engine.download(request)
)
if response.status != 200:

7
docs/topics/practices.rst
View File

@ -288,9 +288,8 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
super proxy that you can attach your own proxies to.
* use a highly distributed downloader that circumvents bans internally, so you
can just focus on parsing clean pages. One example of such downloaders is
`Zyte Smart Proxy Manager`_
* use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy
plugin <https://github.com/scrapy-plugins/scrapy-zyte-api>`__
If you are still unable to prevent your bot getting banned, consider contacting
`commercial support`_.
@ -301,4 +300,4 @@ If you are still unable to prevent your bot getting banned, consider contacting
.. _Common Crawl: https://commoncrawl.org/
.. _testspiders: https://github.com/scrapinghub/testspiders
.. _scrapoxy: https://scrapoxy.io/
.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/
.. _Zyte API: https://docs.zyte.com/zyte-api/get-started.html

71
docs/topics/request-response.rst
View File

@ -115,20 +115,9 @@ Request objects
cookies for that domain and will be sent again in future requests.
That's the typical behaviour of any regular web browser.
To create a request that does not send stored cookies and does not
store received cookies, set the ``dont_merge_cookies`` key to ``True``
in :attr:`request.meta <scrapy.Request.meta>`.
Example of a request that sends manually-defined cookies and ignores
cookie storage:
.. code-block:: python
Request(
url="http://www.example.com",
cookies={"currency": "USD", "country": "UY"},
meta={"dont_merge_cookies": True},
)
Note that setting the :reqmeta:`dont_merge_cookies` key to ``True`` in
:attr:`request.meta <scrapy.Request.meta>` causes custom cookies to be
ignored.
For more info see :ref:`cookies-mw`.
@ -204,18 +193,47 @@ Request objects
:meth:`replace`.
.. attribute:: Request.meta
:value: {}
A dict that contains arbitrary metadata for this request. This dict is
empty for new Requests, and is usually populated by different Scrapy
components (extensions, middlewares, etc). So the data contained in this
dict depends on the extensions you have enabled.
A dictionary of arbitrary metadata for the request.
See :ref:`topics-request-meta` for a list of special meta keys
recognized by Scrapy.
You may extend request metadata as you see fit.
This dict is :doc:`shallow copied <library/copy>` when the request is
cloned using the ``copy()`` or ``replace()`` methods, and can also be
accessed, in your spider, from the ``response.meta`` attribute.
Request metadata can also be accessed through the
:attr:`~scrapy.http.Response.meta` attribute of a response.
To pass data from one spider callback to another, consider using
:attr:`cb_kwargs` instead. However, request metadata may be the right
choice in certain scenarios, such as to maintain some debugging data
across all follow-up requests (e.g. the source URL).
A common use of request metadata is to define request-specific
parameters for Scrapy components (extensions, middlewares, etc.). For
example, if you set ``dont_retry`` to ``True``,
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` will never
retry that request, even if it fails. See :ref:`topics-request-meta`.
You may also use request metadata in your custom Scrapy components, for
example, to keep request state information relevant to your component.
For example,
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses the
``retry_times`` metadata key to keep track of how many times a request
has been retried so far.
Copying all the metadata of a previous request into a new, follow-up
request in a spider callback is a bad practice, because request
metadata may include metadata set by Scrapy components that is not
meant to be copied into other requests. For example, copying the
``retry_times`` metadata key into follow-up requests can lower the
amount of retries allowed for those follow-up requests.
You should only copy all request metadata from one request to another
if the new request is meant to replace the old request, as is often the
case when returning a request from a :ref:`downloader middleware
<topics-downloader-middleware>` method.
Also mind that the :meth:`copy` and :meth:`replace` request methods
:doc:`shallow-copy <library/copy>` request metadata.
.. attribute:: Request.cb_kwargs
@ -1103,9 +1121,10 @@ Response objects
through all :ref:`Downloader Middlewares <topics-downloader-middleware>`.
In particular, this means that:
- HTTP redirections will cause the original request (to the URL before
redirection) to be assigned to the redirected response (with the final
URL after redirection).
- HTTP redirections will create a new request from the request before
redirection. It has the majority of the same metadata and original
request attributes and gets assigned to the redirected response
instead of the propagation of the original request.
- Response.request.url doesn't always equal Response.url

71
docs/topics/settings.rst
View File

@ -40,8 +40,9 @@ precedence:
1. Command line options (most precedence)
2. Settings per-spider
3. Project settings module
4. Default settings per-command
5. Default global settings (less precedence)
4. Settings set by add-ons
5. Default settings per-command
6. Default global settings (less precedence)
The population of these settings sources is taken care of internally, but a
manual handling is possible using API calls. See the
@ -66,8 +67,8 @@ Example::
----------------------
Spiders (See the :ref:`topics-spiders` chapter for reference) can define their
own settings that will take precedence and override the project ones. They can
do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute:
own settings that will take precedence and override the project ones. One way
to do so is by setting their :attr:`~scrapy.Spider.custom_settings` attribute:
.. code-block:: python
@ -81,6 +82,45 @@ do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute:
"SOME_SETTING": "some value",
}
It's often better to implement :meth:`~scrapy.Spider.update_settings` instead,
and settings set there should use the "spider" priority explicitly:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = "myspider"
@classmethod
def update_settings(cls, settings):
super().update_settings(settings)
settings.set("SOME_SETTING", "some value", priority="spider")
.. versionadded:: 2.11
It's also possible to modify the settings in the
:meth:`~scrapy.Spider.from_crawler` method, e.g. based on :ref:`spider
arguments <spiderargs>` or other logic:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = "myspider"
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super().from_crawler(crawler, *args, **kwargs)
if "some_argument" in kwargs:
spider.settings.set(
"SOME_SETTING", kwargs["some_argument"], priority="spider"
)
return spider
3. Project settings module
--------------------------
@ -89,7 +129,13 @@ project, it's where most of your custom settings will be populated. For a
standard Scrapy project, this means you'll be adding or changing the settings
in the ``settings.py`` file created for your project.
4. Default settings per-command
4. Settings set by add-ons
--------------------------
:ref:`Add-ons <topics-addons>` can modify settings. They should do this with
this priority, though this is not enforced.
5. Default settings per-command
-------------------------------
Each :doc:`Scrapy tool </topics/commands>` command can have its own default
@ -97,7 +143,7 @@ settings, which override the global default settings. Those custom command
settings are specified in the ``default_settings`` attribute of the command
class.
5. Default global settings
6. Default global settings
--------------------------
The global defaults are located in the ``scrapy.settings.default_settings``
@ -201,6 +247,16 @@ to any particular component. In that case the module of that component will be
shown, typically an extension, middleware or pipeline. It also means that the
component must be enabled in order for the setting to have any effect.
.. setting:: ADDONS
ADDONS
------
Default: ``{}``
A dict containing paths to the add-ons enabled in your project and their
priorities. For more information, see :ref:`topics-addons`.
.. setting:: AWS_ACCESS_KEY_ID
AWS_ACCESS_KEY_ID
@ -964,7 +1020,6 @@ some of them need to be enabled through a setting.
For more information See the :ref:`extensions user guide <topics-extensions>`
and the :ref:`list of available extensions <topics-extensions-ref>`.
.. setting:: FEED_TEMPDIR
FEED_TEMPDIR
@ -1065,7 +1120,7 @@ modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead.
JOBDIR
------
Default: ``''``
Default: ``None``
A string indicating the directory for storing the state of a crawl when
:ref:`pausing and resuming crawls <topics-jobs>`.

55
docs/topics/spiders.rst
View File

@ -136,6 +136,21 @@ scrapy.Spider
attributes in the new instance so they can be accessed later inside the
spider's code.
.. versionchanged:: 2.11
The settings in ``crawler.settings`` can now be modified in this
method, which is handy if you want to modify them based on
arguments. As a consequence, these settings aren't the final values
as they can be modified later by e.g. :ref:`add-ons
<topics-addons>`. For the same reason, most of the
:class:`~scrapy.crawler.Crawler` attributes aren't initialized at
this point.
The final settings and the initialized
:class:`~scrapy.crawler.Crawler` attributes are available in the
:meth:`start_requests` method, handlers of the
:signal:`engine_started` signal and later.
:param crawler: crawler to which the spider will be bound
:type crawler: :class:`~scrapy.crawler.Crawler` instance
@ -145,6 +160,46 @@ scrapy.Spider
:param kwargs: keyword arguments passed to the :meth:`__init__` method
:type kwargs: dict
.. classmethod:: update_settings(settings)
The ``update_settings()`` method is used to modify the spider's settings
and is called during initialization of a spider instance.
It takes a :class:`~scrapy.settings.Settings` object as a parameter and
can add or update the spider's configuration values. This method is a
class method, meaning that it is called on the :class:`~scrapy.Spider`
class and allows all instances of the spider to share the same
configuration.
While per-spider settings can be set in
:attr:`~scrapy.Spider.custom_settings`, using ``update_settings()``
allows you to dynamically add, remove or change settings based on other
settings, spider attributes or other factors and use setting priorities
other than ``'spider'``. Also, it's easy to extend ``update_settings()``
in a subclass by overriding it, while doing the same with
:attr:`~scrapy.Spider.custom_settings` can be hard.
For example, suppose a spider needs to modify :setting:`FEEDS`:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = "myspider"
custom_feed = {
"/home/user/documents/items.json": {
"format": "json",
"indent": 4,
}
}
@classmethod
def update_settings(cls, settings):
super().update_settings(settings)
settings.setdefault("FEEDS", {}).update(cls.custom_feed)
.. method:: start_requests()
This method must return an iterable with the first Requests to crawl for

2
docs/utils/linkfix.py
View File

@ -30,7 +30,7 @@ def main():
try:
with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out:
output_lines = out.readlines()
except IOError:
except OSError:
print("linkcheck output not found; please run linkcheck first.")
sys.exit(1)

1
pytest.ini
View File

@ -20,6 +20,7 @@ addopts =
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
requires_uvloop: marks tests as only enabled when uvloop is known to be working
filterwarnings =
ignore:scrapy.downloadermiddlewares.decompression is deprecated
ignore:Module scrapy.utils.reqser is deprecated

2
scrapy/VERSION
View File

@ -1 +1 @@
2.9.0
2.11.0

4
scrapy/__init__.py
View File

@ -34,8 +34,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Check minimum required Python version
if sys.version_info < (3, 7):
print(f"Scrapy {__version__} requires Python 3.7+")
if sys.version_info < (3, 8):
print(f"Scrapy {__version__} requires Python 3.8+")
sys.exit(1)

53
scrapy/addons.py Normal file
View File

@ -0,0 +1,53 @@
import logging
from typing import TYPE_CHECKING, Any, List
from scrapy.exceptions import NotConfigured
from scrapy.settings import Settings
from scrapy.utils.conf import build_component_list
from scrapy.utils.misc import create_instance, load_object
if TYPE_CHECKING:
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
class AddonManager:
"""This class facilitates loading and storing :ref:`topics-addons`."""
def __init__(self, crawler: "Crawler") -> None:
self.crawler: "Crawler" = crawler
self.addons: List[Any] = []
def load_settings(self, settings: Settings) -> None:
"""Load add-ons and configurations from a settings object and apply them.
This will load the add-on for every add-on path in the
``ADDONS`` setting and execute their ``update_settings`` methods.
:param settings: The :class:`~scrapy.settings.Settings` object from \
which to read the add-on configuration
:type settings: :class:`~scrapy.settings.Settings`
"""
for clspath in build_component_list(settings["ADDONS"]):
try:
addoncls = load_object(clspath)
addon = create_instance(
addoncls, settings=settings, crawler=self.crawler
)
addon.update_settings(settings)
self.addons.append(addon)
except NotConfigured as e:
if e.args:
logger.warning(
"Disabled %(clspath)s: %(eargs)s",
{"clspath": clspath, "eargs": e.args[0]},
extra={"crawler": self.crawler},
)
logger.info(
"Enabled addons:\n%(addons)s",
{
"addons": self.addons,
},
extra={"crawler": self.crawler},
)

9
scrapy/cmdline.py
View File

@ -3,8 +3,7 @@ import cProfile
import inspect
import os
import sys
import pkg_resources
from importlib.metadata import entry_points
import scrapy
from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
@ -49,7 +48,11 @@ def _get_commands_from_module(module, inproject):
def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
cmds = {}
for entry_point in pkg_resources.iter_entry_points(group):
if sys.version_info >= (3, 10):
eps = entry_points(group=group)
else:
eps = entry_points().get(group, ())
for entry_point in eps:
obj = entry_point.load()
if inspect.isclass(obj):
cmds[entry_point.name] = obj()

4
scrapy/commands/__init__.py
View File

@ -4,7 +4,7 @@ Base class for Scrapy commands
import argparse
import os
from pathlib import Path
from typing import Any, Dict, Optional
from typing import Any, Dict, List, Optional
from twisted.python import failure
@ -116,7 +116,7 @@ class ScrapyCommand:
if opts.pdb:
failure.startDebugMode()
def run(self, args, opts):
def run(self, args: List[str], opts: argparse.Namespace) -> None:
"""
Entry point for running commands
"""

8
scrapy/commands/fetch.py
View File

@ -1,7 +1,10 @@
import sys
from argparse import Namespace
from typing import List, Type
from w3lib.url import is_url
from scrapy import Spider
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.http import Request
@ -57,7 +60,7 @@ class Command(ScrapyCommand):
def _print_bytes(self, bytes_):
sys.stdout.buffer.write(bytes_ + b"\n")
def run(self, args, opts):
def run(self, args: List[str], opts: Namespace) -> None:
if len(args) != 1 or not is_url(args[0]):
raise UsageError()
request = Request(
@ -73,7 +76,8 @@ class Command(ScrapyCommand):
else:
request.meta["handle_httpstatus_all"] = True
spidercls = DefaultSpider
spidercls: Type[Spider] = DefaultSpider
assert self.crawler_process
spider_loader = self.crawler_process.spider_loader
if opts.spider:
spidercls = spider_loader.load(opts.spider)

9
scrapy/commands/shell.py
View File

@ -3,8 +3,11 @@ Scrapy Shell
See documentation in docs/topics/shell.rst
"""
from argparse import Namespace
from threading import Thread
from typing import List, Type
from scrapy import Spider
from scrapy.commands import ScrapyCommand
from scrapy.http import Request
from scrapy.shell import Shell
@ -54,15 +57,16 @@ class Command(ScrapyCommand):
"""
pass
def run(self, args, opts):
def run(self, args: List[str], opts: Namespace) -> None:
url = args[0] if args else None
if url:
# first argument may be a local file
url = guess_scheme(url)
assert self.crawler_process
spider_loader = self.crawler_process.spider_loader
spidercls = DefaultSpider
spidercls: Type[Spider] = DefaultSpider
if opts.spider:
spidercls = spider_loader.load(opts.spider)
elif url:
@ -73,6 +77,7 @@ class Command(ScrapyCommand):
# The crawler is created this way since the Shell manually handles the
# crawling engine, so the set up in the crawl method won't work
crawler = self.crawler_process._create_crawler(spidercls)
crawler._apply_settings()
# The Shell class needs a persistent engine in the crawler
crawler.engine = crawler._create_engine()
crawler.engine.start()

19
scrapy/contracts/__init__.py
View File

@ -2,7 +2,8 @@ import re
import sys
from functools import wraps
from inspect import getmembers
from typing import Dict
from types import CoroutineType
from typing import AsyncGenerator, Dict, Optional, Type
from unittest import TestCase
from scrapy.http import Request
@ -13,7 +14,7 @@ from scrapy.utils.spider import iterate_spider_output
class Contract:
"""Abstract class for contracts"""
request_cls = None
request_cls: Optional[Type[Request]] = None
def __init__(self, method, *args):
self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook")
@ -37,7 +38,12 @@ class Contract:
else:
results.addSuccess(self.testcase_pre)
finally:
return list(iterate_spider_output(cb(response, **cb_kwargs)))
cb_result = cb(response, **cb_kwargs)
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
raise TypeError("Contracts don't support async callbacks")
return list( # pylint: disable=return-in-finally
iterate_spider_output(cb_result)
)
request.callback = wrapper
@ -49,7 +55,10 @@ class Contract:
@wraps(cb)
def wrapper(response, **cb_kwargs):
output = list(iterate_spider_output(cb(response, **cb_kwargs)))
cb_result = cb(response, **cb_kwargs)
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
raise TypeError("Contracts don't support async callbacks")
output = list(iterate_spider_output(cb_result))
try:
results.startTest(self.testcase_post)
self.post_process(output)
@ -61,7 +70,7 @@ class Contract:
else:
results.addSuccess(self.testcase_post)
finally:
return output
return output # pylint: disable=return-in-finally
request.callback = wrapper

2
scrapy/core/downloader/__init__.py
View File

@ -154,7 +154,7 @@ class Downloader:
self.signals.send_catch_log(
signal=signals.request_reached_downloader, request=request, spider=spider
)
deferred = Deferred().addBoth(_deactivate)
deferred: Deferred = Deferred().addBoth(_deactivate)
slot.queue.append((request, deferred))
self._process_queue(spider, slot)
return deferred

9
scrapy/core/downloader/handlers/datauri.py
View File

@ -1,6 +1,9 @@
from typing import Any, Dict
from w3lib.url import parse_data_uri
from scrapy.http import TextResponse
from scrapy import Request, Spider
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.utils.decorators import defers
@ -9,11 +12,11 @@ class DataURIDownloadHandler:
lazy = False
@defers
def download_request(self, request, spider):
def download_request(self, request: Request, spider: Spider) -> Response:
uri = parse_data_uri(request.url)
respcls = responsetypes.from_mimetype(uri.media_type)
resp_kwargs = {}
resp_kwargs: Dict[str, Any] = {}
if issubclass(respcls, TextResponse) and uri.media_type.split("/")[0] == "text":
charset = uri.media_type_parameters.get("charset")
resp_kwargs["encoding"] = charset

110
scrapy/core/engine.py
View File

@ -5,7 +5,6 @@ For more information see docs/topics/architecture.rst
"""
import logging
import warnings
from time import time
from typing import (
TYPE_CHECKING,
@ -14,7 +13,6 @@ from typing import (
Generator,
Iterable,
Iterator,
List,
Optional,
Set,
Type,
@ -29,7 +27,7 @@ from twisted.python.failure import Failure
from scrapy import signals
from scrapy.core.downloader import Downloader
from scrapy.core.scraper import Scraper
from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning
from scrapy.exceptions import CloseSpider, DontCloseSpider
from scrapy.http import Request, Response
from scrapy.logformatter import LogFormatter
from scrapy.settings import BaseSettings, Settings
@ -88,6 +86,7 @@ class ExecutionEngine:
self.crawler: "Crawler" = crawler
self.settings: Settings = crawler.settings
self.signals: SignalManager = crawler.signals
assert crawler.logformatter
self.logformatter: LogFormatter = crawler.logformatter
self.slot: Optional[Slot] = None
self.spider: Optional[Spider] = None
@ -213,7 +212,7 @@ class ExecutionEngine:
if request is None:
return None
d = self._download(request, self.spider)
d = self._download(request)
d.addBoth(self._handle_downloader_output, request)
d.addErrback(
lambda f: logger.info(
@ -222,7 +221,12 @@ class ExecutionEngine:
extra={"spider": self.spider},
)
)
d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request)) # type: ignore[arg-type]
def _remove_request(_: Any) -> None:
assert self.slot
self.slot.remove_request(request)
d.addBoth(_remove_request)
d.addErrback(
lambda f: logger.info(
"Error while removing request from slot",
@ -266,13 +270,7 @@ class ExecutionEngine:
)
return d
def spider_is_idle(self, spider: Optional[Spider] = None) -> bool:
if spider is not None:
warnings.warn(
"Passing a 'spider' argument to ExecutionEngine.spider_is_idle is deprecated",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
def spider_is_idle(self) -> bool:
if self.slot is None:
raise RuntimeError("Engine slot not assigned")
if not self.scraper.slot.is_idle(): # type: ignore[union-attr]
@ -285,18 +283,8 @@ class ExecutionEngine:
return False
return True
def crawl(self, request: Request, spider: Optional[Spider] = None) -> None:
def crawl(self, request: Request) -> None:
"""Inject the request into the spider <-> downloader pipeline"""
if spider is not None:
warnings.warn(
"Passing a 'spider' argument to ExecutionEngine.crawl is deprecated",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
if spider is not self.spider:
raise RuntimeError(
f"The spider {spider.name!r} does not match the open spider"
)
if self.spider is None:
raise RuntimeError(f"No open spider to crawl: {request}")
self._schedule_request(request, self.spider)
@ -311,39 +299,24 @@ class ExecutionEngine:
signals.request_dropped, request=request, spider=spider
)
def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred:
def download(self, request: Request) -> Deferred:
"""Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
if spider is not None:
warnings.warn(
"Passing a 'spider' argument to ExecutionEngine.download is deprecated",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
if spider is not self.spider:
logger.warning(
"The spider '%s' does not match the open spider", spider.name
)
if self.spider is None:
raise RuntimeError(f"No open spider to crawl: {request}")
return self._download(request, spider).addBoth(
self._downloaded, request, spider
)
return self._download(request).addBoth(self._downloaded, request)
def _downloaded(
self, result: Union[Response, Request], request: Request, spider: Spider
) -> Union[Deferred, Response]:
self, result: Union[Response, Request, Failure], request: Request
) -> Union[Deferred, Response, Failure]:
assert self.slot is not None # typing
self.slot.remove_request(request)
return self.download(result, spider) if isinstance(result, Request) else result
return self.download(result) if isinstance(result, Request) else result
def _download(self, request: Request, spider: Optional[Spider]) -> Deferred:
def _download(self, request: Request) -> Deferred:
assert self.slot is not None # typing
self.slot.add_request(request)
if spider is None:
spider = self.spider
def _on_success(result: Union[Response, Request]) -> Union[Response, Request]:
if not isinstance(result, (Response, Request)):
raise TypeError(
@ -352,15 +325,17 @@ class ExecutionEngine:
if isinstance(result, Response):
if result.request is None:
result.request = request
assert spider is not None
logkws = self.logformatter.crawled(result.request, result, spider)
assert self.spider is not None
logkws = self.logformatter.crawled(result.request, result, self.spider)
if logkws is not None:
logger.log(*logformatter_adapter(logkws), extra={"spider": spider})
logger.log(
*logformatter_adapter(logkws), extra={"spider": self.spider}
)
self.signals.send_catch_log(
signal=signals.response_received,
response=result,
request=result.request,
spider=spider,
spider=self.spider,
)
return result
@ -369,8 +344,8 @@ class ExecutionEngine:
self.slot.nextcall.schedule()
return _
assert spider is not None
dwld = self.downloader.fetch(request, spider)
assert self.spider is not None
dwld = self.downloader.fetch(request, self.spider)
dwld.addCallbacks(_on_success)
dwld.addBoth(_on_complete)
return dwld
@ -394,6 +369,7 @@ class ExecutionEngine:
if hasattr(scheduler, "open"):
yield scheduler.open(spider)
yield self.scraper.open_spider(spider)
assert self.crawler.stats
self.crawler.stats.open_spider(spider)
yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider)
self.slot.nextcall.schedule()
@ -465,7 +441,11 @@ class ExecutionEngine:
)
dfd.addErrback(log_failure("Error while sending spider_close signal"))
dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason))
def close_stats(_: Any) -> None:
assert self.crawler.stats
self.crawler.stats.close_spider(spider, reason=reason)
dfd.addBoth(close_stats)
dfd.addErrback(log_failure("Stats close failure"))
dfd.addBoth(
@ -485,31 +465,3 @@ class ExecutionEngine:
dfd.addBoth(lambda _: self._spider_closed_callback(spider))
return dfd
@property
def open_spiders(self) -> List[Spider]:
warnings.warn(
"ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return [self.spider] if self.spider is not None else []
def has_capacity(self) -> bool:
warnings.warn(
"ExecutionEngine.has_capacity is deprecated",
ScrapyDeprecationWarning,
stacklevel=2,
)
return not bool(self.slot)
def schedule(self, request: Request, spider: Spider) -> None:
warnings.warn(
"ExecutionEngine.schedule is deprecated, please use "
"ExecutionEngine.crawl or ExecutionEngine.download instead",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
if self.slot is None:
raise RuntimeError("Engine slot not assigned")
self._schedule_request(request, spider)

4
scrapy/core/http2/stream.py
View File

@ -228,8 +228,8 @@ class Stream:
content_length_name = self._request.headers.normkey(b"Content-Length")
for name, values in self._request.headers.items():
for value in values:
value = str(value, "utf-8")
for value_bytes in values:
value = str(value_bytes, "utf-8")
if name == content_length_name:
if value != content_length:
logger.warning(

14
scrapy/core/scheduler.py
View File

@ -1,8 +1,10 @@
from __future__ import annotations
import json
import logging
from abc import abstractmethod
from pathlib import Path
from typing import Any, Optional, Type, TypeVar, cast
from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, cast
from twisted.internet.defer import Deferred
@ -14,6 +16,11 @@ from scrapy.statscollectors import StatsCollector
from scrapy.utils.job import job_dir
from scrapy.utils.misc import create_instance, load_object
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
@ -54,7 +61,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
"""
@classmethod
def from_crawler(cls, crawler: Crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
"""
Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument.
"""
@ -325,6 +332,7 @@ class Scheduler(BaseScheduler):
def _dq(self):
"""Create a new priority queue instance, with disk storage"""
assert self.dqdir
state = self._read_dqs_state(self.dqdir)
q = create_instance(
self.pqclass,
@ -344,7 +352,7 @@ class Scheduler(BaseScheduler):
def _dqdir(self, jobdir: Optional[str]) -> Optional[str]:
"""Return a folder name to keep disk queue state at"""
if jobdir is not None:
if jobdir:
dqdir = Path(jobdir, "requests.queue")
if not dqdir.exists():
dqdir.mkdir(parents=True)

12
scrapy/core/scraper.py
View File

@ -110,6 +110,7 @@ class Scraper:
self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS")
self.crawler: Crawler = crawler
self.signals: SignalManager = crawler.signals
assert crawler.logformatter
self.logformatter: LogFormatter = crawler.logformatter
@inlineCallbacks
@ -205,6 +206,7 @@ class Scraper:
if isinstance(result, Response):
if getattr(result, "request", None) is None:
result.request = request
assert result.request
callback = result.request.callback or spider._parse
warn_on_generator_with_return_value(spider, callback)
dfd = defer_succeed(result)
@ -221,7 +223,11 @@ class Scraper:
return dfd.addCallback(iterate_spider_output)
def handle_spider_error(
self, _failure: Failure, request: Request, response: Response, spider: Spider
self,
_failure: Failure,
request: Request,
response: Union[Response, Failure],
spider: Spider,
) -> None:
exc = _failure.value
if isinstance(exc, CloseSpider):
@ -240,6 +246,7 @@ class Scraper:
response=response,
spider=spider,
)
assert self.crawler.stats
self.crawler.stats.inc_value(
f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider
)
@ -248,7 +255,7 @@ class Scraper:
self,
result: Union[Iterable, AsyncIterable],
request: Request,
response: Response,
response: Union[Response, Failure],
spider: Spider,
) -> Deferred:
if not result:
@ -364,6 +371,7 @@ class Scraper:
spider=spider,
exception=output.value,
)
assert ex
logkws = self.logformatter.item_error(item, ex, response, spider)
logger.log(
*logformatter_adapter(logkws),

190
scrapy/crawler.py
View File

@ -4,10 +4,14 @@ import logging
import pprint
import signal
import warnings
from typing import TYPE_CHECKING, Optional, Type, Union
from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union, cast
from twisted.internet import defer
from zope.interface.exceptions import DoesNotImplement
from twisted.internet.defer import (
Deferred,
DeferredList,
inlineCallbacks,
maybeDeferred,
)
try:
# zope >= 5.0 only supports MultipleInvalid
@ -18,12 +22,13 @@ except ImportError:
from zope.interface.verify import verifyClass
from scrapy import Spider, signals
from scrapy.addons import AddonManager
from scrapy.core.engine import ExecutionEngine
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extension import ExtensionManager
from scrapy.interfaces import ISpiderLoader
from scrapy.logformatter import LogFormatter
from scrapy.settings import Settings, overridden_settings
from scrapy.settings import BaseSettings, Settings, overridden_settings
from scrapy.signalmanager import SignalManager
from scrapy.statscollectors import StatsCollector
from scrapy.utils.log import (
@ -54,7 +59,7 @@ class Crawler:
def __init__(
self,
spidercls: Type[Spider],
settings: Union[None, dict, Settings] = None,
settings: Union[None, Dict[str, Any], Settings] = None,
init_reactor: bool = False,
):
if isinstance(spidercls, Spider):
@ -66,31 +71,45 @@ class Crawler:
self.spidercls: Type[Spider] = spidercls
self.settings: Settings = settings.copy()
self.spidercls.update_settings(self.settings)
self._update_root_log_handler()
self.addons: AddonManager = AddonManager(self)
self.signals: SignalManager = SignalManager(self)
self.stats: StatsCollector = load_object(self.settings["STATS_CLASS"])(self)
self._init_reactor: bool = init_reactor
self.crawling: bool = False
self._started: bool = False
handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL"))
logging.root.addHandler(handler)
d = dict(overridden_settings(self.settings))
logger.info(
"Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}
)
self.extensions: Optional[ExtensionManager] = None
self.stats: Optional[StatsCollector] = None
self.logformatter: Optional[LogFormatter] = None
self.request_fingerprinter: Optional[RequestFingerprinter] = None
self.spider: Optional[Spider] = None
self.engine: Optional[ExecutionEngine] = None
def _update_root_log_handler(self) -> None:
if get_scrapy_root_handler() is not None:
# scrapy root handler already installed: update it with new settings
install_scrapy_root_handler(self.settings)
def _apply_settings(self) -> None:
if self.settings.frozen:
return
self.addons.load_settings(self.settings)
self.stats = load_object(self.settings["STATS_CLASS"])(self)
handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL"))
logging.root.addHandler(handler)
# lambda is assigned to Crawler attribute because this way it is not
# garbage collected after leaving __init__ scope
# garbage collected after leaving the scope
self.__remove_handler = lambda: logging.root.removeHandler(handler)
self.signals.connect(self.__remove_handler, signals.engine_stopped)
lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
self.logformatter: LogFormatter = lf_cls.from_crawler(self)
self.logformatter = lf_cls.from_crawler(self)
self.request_fingerprinter: RequestFingerprinter = create_instance(
self.request_fingerprinter = create_instance(
load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),
settings=self.settings,
crawler=self,
@ -98,7 +117,7 @@ class Crawler:
reactor_class: str = self.settings["TWISTED_REACTOR"]
event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"]
if init_reactor:
if self._init_reactor:
# this needs to be done after the spider settings are merged,
# but before something imports twisted.internet.reactor
if reactor_class:
@ -111,44 +130,54 @@ class Crawler:
if is_asyncio_reactor_installed() and event_loop:
verify_installed_asyncio_event_loop(event_loop)
self.extensions: ExtensionManager = ExtensionManager.from_crawler(self)
self.extensions = ExtensionManager.from_crawler(self)
self.settings.freeze()
self.crawling: bool = False
self.spider: Optional[Spider] = None
self.engine: Optional[ExecutionEngine] = None
@defer.inlineCallbacks
def crawl(self, *args, **kwargs):
d = dict(overridden_settings(self.settings))
logger.info(
"Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}
)
@inlineCallbacks
def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:
if self.crawling:
raise RuntimeError("Crawling already taking place")
self.crawling = True
if self._started:
warnings.warn(
"Running Crawler.crawl() more than once is deprecated.",
ScrapyDeprecationWarning,
stacklevel=2,
)
self.crawling = self._started = True
try:
self.spider = self._create_spider(*args, **kwargs)
self._apply_settings()
self._update_root_log_handler()
self.engine = self._create_engine()
start_requests = iter(self.spider.start_requests())
yield self.engine.open_spider(self.spider, start_requests)
yield defer.maybeDeferred(self.engine.start)
yield maybeDeferred(self.engine.start)
except Exception:
self.crawling = False
if self.engine is not None:
yield self.engine.close()
raise
def _create_spider(self, *args, **kwargs):
def _create_spider(self, *args: Any, **kwargs: Any) -> Spider:
return self.spidercls.from_crawler(self, *args, **kwargs)
def _create_engine(self):
def _create_engine(self) -> ExecutionEngine:
return ExecutionEngine(self, lambda _: self.stop())
@defer.inlineCallbacks
def stop(self):
@inlineCallbacks
def stop(self) -> Generator[Deferred, Any, None]:
"""Starts a graceful stop of the crawler and returns a deferred that is
fired when the crawler is stopped."""
if self.crawling:
self.crawling = False
yield defer.maybeDeferred(self.engine.stop)
assert self.engine
yield maybeDeferred(self.engine.stop)
class CrawlerRunner:
@ -171,45 +200,28 @@ class CrawlerRunner:
)
@staticmethod
def _get_spider_loader(settings):
def _get_spider_loader(settings: BaseSettings):
"""Get SpiderLoader instance from settings"""
cls_path = settings.get("SPIDER_LOADER_CLASS")
loader_cls = load_object(cls_path)
excs = (
(DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement
)
try:
verifyClass(ISpiderLoader, loader_cls)
except excs:
warnings.warn(
"SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does "
"not fully implement scrapy.interfaces.ISpiderLoader interface. "
"Please add all missing methods to avoid unexpected runtime errors.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
verifyClass(ISpiderLoader, loader_cls)
return loader_cls.from_settings(settings.frozencopy())
def __init__(self, settings=None):
def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None):
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
self.settings = settings
self.spider_loader = self._get_spider_loader(settings)
self._crawlers = set()
self._active = set()
self._crawlers: Set[Crawler] = set()
self._active: Set[Deferred] = set()
self.bootstrap_failed = False
@property
def spiders(self):
warnings.warn(
"CrawlerRunner.spiders attribute is renamed to "
"CrawlerRunner.spider_loader.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return self.spider_loader
def crawl(self, crawler_or_spidercls, *args, **kwargs):
def crawl(
self,
crawler_or_spidercls: Union[Type[Spider], str, Crawler],
*args: Any,
**kwargs: Any,
) -> Deferred:
"""
Run a crawler with the provided arguments.
@ -239,12 +251,12 @@ class CrawlerRunner:
crawler = self.create_crawler(crawler_or_spidercls)
return self._crawl(crawler, *args, **kwargs)
def _crawl(self, crawler, *args, **kwargs):
def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred:
self.crawlers.add(crawler)
d = crawler.crawl(*args, **kwargs)
self._active.add(d)
def _done(result):
def _done(result: Any) -> Any:
self.crawlers.discard(crawler)
self._active.discard(d)
self.bootstrap_failed |= not getattr(crawler, "spider", None)
@ -252,7 +264,9 @@ class CrawlerRunner:
return d.addBoth(_done)
def create_crawler(self, crawler_or_spidercls):
def create_crawler(
self, crawler_or_spidercls: Union[Type[Spider], str, Crawler]
) -> Crawler:
"""
Return a :class:`~scrapy.crawler.Crawler` object.
@ -272,21 +286,22 @@ class CrawlerRunner:
return crawler_or_spidercls
return self._create_crawler(crawler_or_spidercls)
def _create_crawler(self, spidercls):
def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler:
if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
return Crawler(spidercls, self.settings)
# temporary cast until self.spider_loader is typed
return Crawler(cast(Type[Spider], spidercls), self.settings)
def stop(self):
def stop(self) -> Deferred:
"""
Stops simultaneously all the crawling jobs taking place.
Returns a deferred that is fired when they all have ended.
"""
return defer.DeferredList([c.stop() for c in list(self.crawlers)])
return DeferredList([c.stop() for c in list(self.crawlers)])
@defer.inlineCallbacks
def join(self):
@inlineCallbacks
def join(self) -> Generator[Deferred, Any, None]:
"""
join()
@ -294,7 +309,7 @@ class CrawlerRunner:
completed their executions.
"""
while self._active:
yield defer.DeferredList(self._active)
yield DeferredList(self._active)
class CrawlerProcess(CrawlerRunner):
@ -321,13 +336,17 @@ class CrawlerProcess(CrawlerRunner):
process. See :ref:`run-from-script` for an example.
"""
def __init__(self, settings=None, install_root_handler=True):
def __init__(
self,
settings: Union[Dict[str, Any], Settings, None] = None,
install_root_handler: bool = True,
):
super().__init__(settings)
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)
self._initialized_reactor = False
def _signal_shutdown(self, signum, _):
def _signal_shutdown(self, signum: int, _: Any) -> None:
from twisted.internet import reactor
install_shutdown_handlers(self._signal_kill)
@ -338,7 +357,7 @@ class CrawlerProcess(CrawlerRunner):
)
reactor.callFromThread(self._graceful_stop_reactor)
def _signal_kill(self, signum, _):
def _signal_kill(self, signum: int, _: Any) -> None:
from twisted.internet import reactor
install_shutdown_handlers(signal.SIG_IGN)
@ -348,14 +367,19 @@ class CrawlerProcess(CrawlerRunner):
)
reactor.callFromThread(self._stop_reactor)
def _create_crawler(self, spidercls):
def _create_crawler(self, spidercls: Union[Type[Spider], str]) -> Crawler:
if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
init_reactor = not self._initialized_reactor
self._initialized_reactor = True
return Crawler(spidercls, self.settings, init_reactor=init_reactor)
# temporary cast until self.spider_loader is typed
return Crawler(
cast(Type[Spider], spidercls), self.settings, init_reactor=init_reactor
)
def start(self, stop_after_crawl=True, install_signal_handlers=True):
def start(
self, stop_after_crawl: bool = True, install_signal_handlers: bool = True
) -> None:
"""
This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
@ -367,8 +391,8 @@ class CrawlerProcess(CrawlerRunner):
:param bool stop_after_crawl: stop or not the reactor when all
crawlers have finished
:param bool install_signal_handlers: whether to install the shutdown
handlers (default: True)
:param bool install_signal_handlers: whether to install the OS signal
handlers from Twisted and Scrapy (default: True)
"""
from twisted.internet import reactor
@ -379,22 +403,24 @@ class CrawlerProcess(CrawlerRunner):
return
d.addBoth(self._stop_reactor)
if install_signal_handlers:
install_shutdown_handlers(self._signal_shutdown)
resolver_class = load_object(self.settings["DNS_RESOLVER"])
resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
resolver.install_on_reactor()
tp = reactor.getThreadPool()
tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))
reactor.addSystemEventTrigger("before", "shutdown", self.stop)
reactor.run(installSignalHandlers=False) # blocking call
if install_signal_handlers:
reactor.addSystemEventTrigger(
"after", "startup", install_shutdown_handlers, self._signal_shutdown
)
reactor.run(installSignalHandlers=install_signal_handlers) # blocking call
def _graceful_stop_reactor(self):
def _graceful_stop_reactor(self) -> Deferred:
d = self.stop()
d.addBoth(self._stop_reactor)
return d
def _stop_reactor(self, _=None):
def _stop_reactor(self, _: Any = None) -> None:
from twisted.internet import reactor
try:

28
scrapy/downloadermiddlewares/ajaxcrawl.py
View File

@ -1,10 +1,20 @@
from __future__ import annotations
import logging
import re
from typing import TYPE_CHECKING, Union
from w3lib import html
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import HtmlResponse
from scrapy.http import HtmlResponse, Response
from scrapy.settings import BaseSettings
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
@ -15,7 +25,7 @@ class AjaxCrawlMiddleware:
For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started.
"""
def __init__(self, settings):
def __init__(self, settings: BaseSettings):
if not settings.getbool("AJAXCRAWL_ENABLED"):
raise NotConfigured
@ -23,13 +33,15 @@ class AjaxCrawlMiddleware:
# middleware parses first 4k. 4k turns out to be insufficient
# for this middleware, and parsing 100k could be slow.
# We use something in between (32K) by default.
self.lookup_bytes = settings.getint("AJAXCRAWL_MAXSIZE", 32768)
self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE", 32768)
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings)
def process_response(self, request, response, spider):
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Union[Request, Response]:
if not isinstance(response, HtmlResponse) or response.status != 200:
return response
@ -54,7 +66,7 @@ class AjaxCrawlMiddleware:
ajax_crawl_request.meta["ajax_crawlable"] = True
return ajax_crawl_request
def _has_ajax_crawlable_variant(self, response):
def _has_ajax_crawlable_variant(self, response: Response) -> bool:
"""
Return True if a page without hash fragment could be "AJAX crawlable"
according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started.
@ -64,12 +76,12 @@ class AjaxCrawlMiddleware:
# XXX: move it to w3lib?
_ajax_crawlable_re = re.compile(
_ajax_crawlable_re: re.Pattern[str] = re.compile(
r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'
)
def _has_ajaxcrawlable_meta(text):
def _has_ajaxcrawlable_meta(text: str) -> bool:
"""
>>> _has_ajaxcrawlable_meta('<html><head><meta name="fragment" content="!"/></head><body></body></html>')
True

62
scrapy/downloadermiddlewares/cookies.py
View File

@ -1,21 +1,41 @@
from __future__ import annotations
import logging
from collections import defaultdict
from http.cookiejar import Cookie
from typing import (
TYPE_CHECKING,
Any,
DefaultDict,
Dict,
Iterable,
Optional,
Sequence,
Union,
)
from tldextract import TLDExtract
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.http.cookies import CookieJar
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
_split_domain = TLDExtract(include_psl_private_domains=True)
def _is_public_domain(domain):
def _is_public_domain(domain: str) -> bool:
parts = _split_domain(domain)
return not parts.domain
@ -23,23 +43,27 @@ def _is_public_domain(domain):
class CookiesMiddleware:
"""This middleware enables working with sites that need cookies"""
def __init__(self, debug=False):
self.jars = defaultdict(CookieJar)
self.debug = debug
def __init__(self, debug: bool = False):
self.jars: DefaultDict[Any, CookieJar] = defaultdict(CookieJar)
self.debug: bool = debug
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("COOKIES_ENABLED"):
raise NotConfigured
return cls(crawler.settings.getbool("COOKIES_DEBUG"))
def _process_cookies(self, cookies, *, jar, request):
def _process_cookies(
self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request
) -> None:
for cookie in cookies:
cookie_domain = cookie.domain
if cookie_domain.startswith("."):
cookie_domain = cookie_domain[1:]
request_domain = urlparse_cached(request).hostname.lower()
hostname = urlparse_cached(request).hostname
assert hostname is not None
request_domain = hostname.lower()
if cookie_domain and _is_public_domain(cookie_domain):
if cookie_domain != request_domain:
@ -48,9 +72,11 @@ class CookiesMiddleware:
jar.set_cookie_if_ok(cookie, request)
def process_request(self, request, spider):
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
if request.meta.get("dont_merge_cookies", False):
return
return None
cookiejarkey = request.meta.get("cookiejar")
jar = self.jars[cookiejarkey]
@ -61,8 +87,11 @@ class CookiesMiddleware:
request.headers.pop("Cookie", None)
jar.add_cookie_header(request)
self._debug_cookie(request, spider)
return None
def process_response(self, request, response, spider):
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Union[Request, Response]:
if request.meta.get("dont_merge_cookies", False):
return response
@ -76,7 +105,7 @@ class CookiesMiddleware:
return response
def _debug_cookie(self, request, spider):
def _debug_cookie(self, request: Request, spider: Spider) -> None:
if self.debug:
cl = [
to_unicode(c, errors="replace")
@ -87,7 +116,7 @@ class CookiesMiddleware:
msg = f"Sending cookies to: {request}\n{cookies}"
logger.debug(msg, extra={"spider": spider})
def _debug_set_cookie(self, response, spider):
def _debug_set_cookie(self, response: Response, spider: Spider) -> None:
if self.debug:
cl = [
to_unicode(c, errors="replace")
@ -98,7 +127,7 @@ class CookiesMiddleware:
msg = f"Received cookies from: {response}\n{cookies}"
logger.debug(msg, extra={"spider": spider})
def _format_cookie(self, cookie, request):
def _format_cookie(self, cookie: Dict[str, Any], request: Request) -> Optional[str]:
"""
Given a dict consisting of cookie components, return its string representation.
Decode from bytes if necessary.
@ -109,7 +138,7 @@ class CookiesMiddleware:
if key in ("name", "value"):
msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
logger.warning(msg)
return
return None
continue
if isinstance(cookie[key], (bool, float, int, str)):
decoded[key] = str(cookie[key])
@ -129,12 +158,15 @@ class CookiesMiddleware:
cookie_str += f"; {key.capitalize()}={value}"
return cookie_str
def _get_request_cookies(self, jar, request):
def _get_request_cookies(
self, jar: CookieJar, request: Request
) -> Sequence[Cookie]:
"""
Extract cookies from the Request.cookies attribute
"""
if not request.cookies:
return []
cookies: Iterable[Dict[str, Any]]
if isinstance(request.cookies, dict):
cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
else:

94
scrapy/downloadermiddlewares/decompression.py
View File

@ -1,94 +0,0 @@
""" This module implements the DecompressionMiddleware which tries to recognise
and extract the potentially compressed responses that may arrive.
"""
import bz2
import gzip
import logging
import tarfile
import zipfile
from io import BytesIO
from tempfile import mktemp
from warnings import warn
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.responsetypes import responsetypes
warn(
"scrapy.downloadermiddlewares.decompression is deprecated",
ScrapyDeprecationWarning,
stacklevel=2,
)
logger = logging.getLogger(__name__)
class DecompressionMiddleware:
"""This middleware tries to recognise and extract the possibly compressed
responses that may arrive."""
def __init__(self):
self._formats = {
"tar": self._is_tar,
"zip": self._is_zip,
"gz": self._is_gzip,
"bz2": self._is_bzip2,
}
def _is_tar(self, response):
archive = BytesIO(response.body)
try:
tar_file = tarfile.open(name=mktemp(), fileobj=archive)
except tarfile.ReadError:
return
body = tar_file.extractfile(tar_file.members[0]).read()
respcls = responsetypes.from_args(filename=tar_file.members[0].name, body=body)
return response.replace(body=body, cls=respcls)
def _is_zip(self, response):
archive = BytesIO(response.body)
try:
zip_file = zipfile.ZipFile(archive)
except zipfile.BadZipFile:
return
namelist = zip_file.namelist()
body = zip_file.read(namelist[0])
respcls = responsetypes.from_args(filename=namelist[0], body=body)
return response.replace(body=body, cls=respcls)
def _is_gzip(self, response):
archive = BytesIO(response.body)
try:
body = gzip.GzipFile(fileobj=archive).read()
except IOError:
return
respcls = responsetypes.from_args(body=body)
return response.replace(body=body, cls=respcls)
def _is_bzip2(self, response):
try:
body = bz2.decompress(response.body)
except IOError:
return
respcls = responsetypes.from_args(body=body)
return response.replace(body=body, cls=respcls)
def process_response(self, request, response, spider):
if not response.body:
return response
for fmt, func in self._formats.items():
new_response = func(response)
if new_response:
logger.debug(
"Decompressed response with format: %(responsefmt)s",
{"responsefmt": fmt},
extra={"spider": spider},
)
return new_response
return response

21
scrapy/downloadermiddlewares/defaultheaders.py
View File

@ -3,19 +3,32 @@ DefaultHeaders downloader middleware
See documentation in docs/topics/downloader-middleware.rst
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Iterable, Tuple, Union
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class DefaultHeadersMiddleware:
def __init__(self, headers):
self._headers = headers
def __init__(self, headers: Iterable[Tuple[str, str]]):
self._headers: Iterable[Tuple[str, str]] = headers
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
headers = without_none_values(crawler.settings["DEFAULT_REQUEST_HEADERS"])
return cls(headers.items())
def process_request(self, request, spider):
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
for k, v in self._headers:
request.headers.setdefault(k, v)
return None

24
scrapy/downloadermiddlewares/downloadtimeout.py
View File

@ -3,23 +3,35 @@ Download timeout middleware
See documentation in docs/topics/downloader-middleware.rst
"""
from __future__ import annotations
from scrapy import signals
from typing import TYPE_CHECKING, Union
from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
from scrapy.http import Response
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class DownloadTimeoutMiddleware:
def __init__(self, timeout=180):
self._timeout = timeout
def __init__(self, timeout: float = 180):
self._timeout: float = timeout
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
o = cls(crawler.settings.getfloat("DOWNLOAD_TIMEOUT"))
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o
def spider_opened(self, spider):
def spider_opened(self, spider: Spider) -> None:
self._timeout = getattr(spider, "download_timeout", self._timeout)
def process_request(self, request, spider):
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
if self._timeout:
request.meta.setdefault("download_timeout", self._timeout)
return None

41
scrapy/downloadermiddlewares/httpauth.py
View File

@ -3,50 +3,45 @@ HTTP basic auth downloader middleware
See documentation in docs/topics/downloader-middleware.rst
"""
import warnings
from __future__ import annotations
from typing import TYPE_CHECKING, Union
from w3lib.http import basic_auth_header
from scrapy import signals
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.httpobj import urlparse_cached
from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.utils.url import url_is_from_any_domain
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class HttpAuthMiddleware:
"""Set Basic HTTP Authorization header
(http_user and http_pass spider class attributes)"""
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
o = cls()
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o
def spider_opened(self, spider):
def spider_opened(self, spider: Spider) -> None:
usr = getattr(spider, "http_user", "")
pwd = getattr(spider, "http_pass", "")
if usr or pwd:
self.auth = basic_auth_header(usr, pwd)
if not hasattr(spider, "http_auth_domain"):
warnings.warn(
"Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security "
"problems if the spider makes requests to several different domains. http_auth_domain "
"will be set to the domain of the first request, please set it to the correct value "
"explicitly.",
category=ScrapyDeprecationWarning,
)
self.domain_unset = True
else:
self.domain = spider.http_auth_domain
self.domain_unset = False
self.domain = spider.http_auth_domain # type: ignore[attr-defined]
def process_request(self, request, spider):
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
auth = getattr(self, "auth", None)
if auth and b"Authorization" not in request.headers:
domain = urlparse_cached(request).hostname
if self.domain_unset:
self.domain = domain
self.domain_unset = False
if not self.domain or url_is_from_any_domain(request.url, [self.domain]):
request.headers[b"Authorization"] = auth
return None

31
scrapy/downloadermiddlewares/httpcache.py
View File

@ -1,5 +1,7 @@
from __future__ import annotations
from email.utils import formatdate
from typing import Optional, Type, TypeVar
from typing import TYPE_CHECKING, Optional, Union
from twisted.internet import defer
from twisted.internet.error import (
@ -23,7 +25,9 @@ from scrapy.spiders import Spider
from scrapy.statscollectors import StatsCollector
from scrapy.utils.misc import load_object
HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware")
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class HttpCacheMiddleware:
@ -37,7 +41,7 @@ class HttpCacheMiddleware:
ConnectionLost,
TCPTimedOutError,
ResponseFailed,
IOError,
OSError,
)
def __init__(self, settings: Settings, stats: StatsCollector) -> None:
@ -49,9 +53,8 @@ class HttpCacheMiddleware:
self.stats = stats
@classmethod
def from_crawler(
cls: Type[HttpCacheMiddlewareTV], crawler: Crawler
) -> HttpCacheMiddlewareTV:
def from_crawler(cls, crawler: Crawler) -> Self:
assert crawler.stats
o = cls(crawler.settings, crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
@ -63,7 +66,9 @@ class HttpCacheMiddleware:
def spider_closed(self, spider: Spider) -> None:
self.storage.close_spider(spider)
def process_request(self, request: Request, spider: Spider) -> Optional[Response]:
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
if request.meta.get("dont_cache", False):
return None
@ -73,7 +78,9 @@ class HttpCacheMiddleware:
return None
# Look for cached response and check if expired
cachedresponse = self.storage.retrieve_response(spider, request)
cachedresponse: Optional[Response] = self.storage.retrieve_response(
spider, request
)
if cachedresponse is None:
self.stats.inc_value("httpcache/miss", spider=spider)
if self.ignore_missing:
@ -95,7 +102,7 @@ class HttpCacheMiddleware:
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Response:
) -> Union[Request, Response]:
if request.meta.get("dont_cache", False):
return response
@ -110,7 +117,7 @@ class HttpCacheMiddleware:
response.headers["Date"] = formatdate(usegmt=True)
# Do not validate first-hand responses
cachedresponse = request.meta.pop("cached_response", None)
cachedresponse: Optional[Response] = request.meta.pop("cached_response", None)
if cachedresponse is None:
self.stats.inc_value("httpcache/firsthand", spider=spider)
self._cache_response(spider, response, request, cachedresponse)
@ -126,8 +133,8 @@ class HttpCacheMiddleware:
def process_exception(
self, request: Request, exception: Exception, spider: Spider
) -> Optional[Response]:
cachedresponse = request.meta.pop("cached_response", None)
) -> Union[Request, Response, None]:
cachedresponse: Optional[Response] = request.meta.pop("cached_response", None)
if cachedresponse is not None and isinstance(
exception, self.DOWNLOAD_EXCEPTIONS
):

33
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -1,14 +1,22 @@
import io
import warnings
import zlib
from __future__ import annotations
import io
import zlib
from typing import TYPE_CHECKING, List, Optional, Union
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.statscollectors import StatsCollector
from scrapy.utils.gz import gunzip
ACCEPTED_ENCODINGS = [b"gzip", b"deflate"]
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]
try:
import brotli
@ -29,7 +37,7 @@ class HttpCompressionMiddleware:
"""This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites"""
def __init__(self, stats=None, settings=None):
def __init__(self, stats: Optional[StatsCollector] = None):
self.stats = stats
if not stats:
warnings.warn(
@ -61,7 +69,7 @@ class HttpCompressionMiddleware:
)
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("COMPRESSION_ENABLED"):
raise NotConfigured
try:
@ -78,10 +86,15 @@ class HttpCompressionMiddleware:
result.keep_encoding_header = False
return result
def process_request(self, request, spider):
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
return None
def process_response(self, request, response, spider):
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Union[Request, Response]:
if request.method == "HEAD":
return response
if b'decoded' in response.flags:
@ -110,7 +123,7 @@ class HttpCompressionMiddleware:
del response.headers['Content-Encoding']
return response
def _decode(self, body, encoding):
def _decode(self, body: bytes, encoding: bytes) -> bytes:
if encoding == b"gzip" or encoding == b"x-gzip":
body = gunzip(body)

45
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -1,16 +1,30 @@
import base64
from urllib.parse import unquote, urlunparse
from urllib.request import _parse_proxy, getproxies, proxy_bypass
from __future__ import annotations
import base64
from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union
from urllib.parse import unquote, urlunparse
from urllib.request import ( # type: ignore[attr-defined]
_parse_proxy,
getproxies,
proxy_bypass,
)
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class HttpProxyMiddleware:
def __init__(self, auth_encoding="latin-1"):
self.auth_encoding = auth_encoding
self.proxies = {}
def __init__(self, auth_encoding: Optional[str] = "latin-1"):
self.auth_encoding: Optional[str] = auth_encoding
self.proxies: Dict[str, Tuple[Optional[bytes], str]] = {}
for type_, url in getproxies().items():
try:
self.proxies[type_] = self._get_proxy(url, type_)
@ -20,19 +34,19 @@ class HttpProxyMiddleware:
continue
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("HTTPPROXY_ENABLED"):
raise NotConfigured
auth_encoding = crawler.settings.get("HTTPPROXY_AUTH_ENCODING")
auth_encoding: Optional[str] = crawler.settings.get("HTTPPROXY_AUTH_ENCODING")
return cls(auth_encoding)
def _basic_auth_header(self, username, password):
def _basic_auth_header(self, username: str, password: str) -> bytes:
user_pass = to_bytes(
f"{unquote(username)}:{unquote(password)}", encoding=self.auth_encoding
)
return base64.b64encode(user_pass)
def _get_proxy(self, url, orig_type):
def _get_proxy(self, url: str, orig_type: str) -> Tuple[Optional[bytes], str]:
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", ""))
@ -43,7 +57,9 @@ class HttpProxyMiddleware:
return creds, proxy_url
def process_request(self, request, spider):
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
creds, proxy_url = None, None
if "proxy" in request.meta:
if request.meta["proxy"] is not None:
@ -54,13 +70,16 @@ class HttpProxyMiddleware:
if (
# 'no_proxy' is only supported by http schemes
scheme not in ("http", "https")
or not proxy_bypass(parsed.hostname)
or (parsed.hostname and not proxy_bypass(parsed.hostname))
) and scheme in self.proxies:
creds, proxy_url = self.proxies[scheme]
self._set_proxy_and_creds(request, proxy_url, creds)
return None
def _set_proxy_and_creds(self, request, proxy_url, creds):
def _set_proxy_and_creds(
self, request: Request, proxy_url: Optional[str], creds: Optional[bytes]
) -> None:
if proxy_url:
request.meta["proxy"] = proxy_url
elif request.meta.get("proxy") is not None:

51
scrapy/downloadermiddlewares/redirect.py
View File

@ -1,17 +1,29 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, List, Union, cast
from urllib.parse import urljoin, urlparse
from w3lib.url import safe_url_string
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import HtmlResponse
from scrapy.http import HtmlResponse, Response
from scrapy.settings import BaseSettings
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.response import get_meta_refresh
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
def _build_redirect_request(source_request, *, url, **kwargs):
def _build_redirect_request(
source_request: Request, *, url: str, **kwargs: Any
) -> Request:
redirect_request = source_request.replace(
url=url,
**kwargs,
@ -26,20 +38,22 @@ def _build_redirect_request(source_request, *, url, **kwargs):
class BaseRedirectMiddleware:
enabled_setting = "REDIRECT_ENABLED"
enabled_setting: str = "REDIRECT_ENABLED"
def __init__(self, settings):
def __init__(self, settings: BaseSettings):
if not settings.getbool(self.enabled_setting):
raise NotConfigured
self.max_redirect_times = settings.getint("REDIRECT_MAX_TIMES")
self.priority_adjust = settings.getint("REDIRECT_PRIORITY_ADJUST")
self.max_redirect_times: int = settings.getint("REDIRECT_MAX_TIMES")
self.priority_adjust: int = settings.getint("REDIRECT_PRIORITY_ADJUST")
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings)
def _redirect(self, redirected, request, spider, reason):
def _redirect(
self, redirected: Request, request: Request, spider: Spider, reason: Any
) -> Request:
ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times)
redirects = request.meta.get("redirect_times", 0) + 1
@ -67,7 +81,9 @@ class BaseRedirectMiddleware:
)
raise IgnoreRequest("max redirections reached")
def _redirect_request_using_get(self, request, redirect_url):
def _redirect_request_using_get(
self, request: Request, redirect_url: str
) -> Request:
redirect_request = _build_redirect_request(
request,
url=redirect_url,
@ -85,7 +101,9 @@ class RedirectMiddleware(BaseRedirectMiddleware):
and meta-refresh html tag.
"""
def process_response(self, request, response, spider):
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Union[Request, Response]:
if (
request.meta.get("dont_redirect", False)
or response.status in getattr(spider, "handle_httpstatus_list", [])
@ -98,6 +116,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
if "Location" not in response.headers or response.status not in allowed_status:
return response
assert response.headers["Location"] is not None
location = safe_url_string(response.headers["Location"])
if response.headers["Location"].startswith(b"//"):
request_scheme = urlparse(request.url).scheme
@ -116,12 +135,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):
class MetaRefreshMiddleware(BaseRedirectMiddleware):
enabled_setting = "METAREFRESH_ENABLED"
def __init__(self, settings):
def __init__(self, settings: BaseSettings):
super().__init__(settings)
self._ignore_tags = settings.getlist("METAREFRESH_IGNORE_TAGS")
self._maxdelay = settings.getint("METAREFRESH_MAXDELAY")
self._ignore_tags: List[str] = settings.getlist("METAREFRESH_IGNORE_TAGS")
self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY")
def process_response(self, request, response, spider):
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Union[Request, Response]:
if (
request.meta.get("dont_redirect", False)
or request.method == "HEAD"
@ -130,7 +151,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
return response
interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags)
if url and interval < self._maxdelay:
if url and cast(float, interval) < self._maxdelay:
redirected = self._redirect_request_using_get(request, url)
return self._redirect(redirected, request, spider, "meta refresh")

105
scrapy/downloadermiddlewares/retry.py
View File

@ -9,41 +9,60 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
Failed pages are collected on the scraping process and rescheduled at the end,
once the spider has finished crawling all regular (non failed) pages.
"""
from __future__ import annotations
import warnings
from logging import Logger, getLogger
from typing import Optional, Union
from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union
from twisted.internet import defer
from twisted.internet.error import (
ConnectError,
ConnectionDone,
ConnectionLost,
ConnectionRefusedError,
DNSLookupError,
TCPTimedOutError,
TimeoutError,
)
from twisted.web.client import ResponseFailed
from scrapy.core.downloader.handlers.http11 import TunnelError
from scrapy.exceptions import NotConfigured
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Response
from scrapy.http.request import Request
from scrapy.settings import BaseSettings, Settings
from scrapy.spiders import Spider
from scrapy.utils.misc import load_object
from scrapy.utils.python import global_object_name
from scrapy.utils.response import response_status_message
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
retry_logger = getLogger(__name__)
def backwards_compatibility_getattr(self: Any, name: str) -> Tuple[Any, ...]:
if name == "EXCEPTIONS_TO_RETRY":
warnings.warn(
"Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. "
"Use the RETRY_EXCEPTIONS setting instead.",
ScrapyDeprecationWarning,
stacklevel=2,
)
return tuple(
load_object(x) if isinstance(x, str) else x
for x in Settings().getlist("RETRY_EXCEPTIONS")
)
raise AttributeError(
f"{self.__class__.__name__!r} object has no attribute {name!r}"
)
class BackwardsCompatibilityMetaclass(type):
__getattr__ = backwards_compatibility_getattr
def get_retry_request(
request: Request,
*,
spider: Spider,
reason: Union[str, Exception] = "unspecified",
reason: Union[str, Exception, Type[Exception]] = "unspecified",
max_retry_times: Optional[int] = None,
priority_adjust: Optional[int] = None,
logger: Logger = retry_logger,
stats_base_key: str = "retry",
):
) -> Optional[Request]:
"""
Returns a new :class:`~scrapy.Request` object to retry the specified
request, or ``None`` if retries of the specified request have been
@ -85,6 +104,7 @@ def get_retry_request(
retry-related job stats
"""
settings = spider.crawler.settings
assert spider.crawler.stats
stats = spider.crawler.stats
retry_times = request.meta.get("retry_times", 0) + 1
if max_retry_times is None:
@ -121,24 +141,8 @@ def get_retry_request(
return None
class RetryMiddleware:
# IOError is raised by the HttpCompression middleware when trying to
# decompress an empty response
EXCEPTIONS_TO_RETRY = (
defer.TimeoutError,
TimeoutError,
DNSLookupError,
ConnectionRefusedError,
ConnectionDone,
ConnectError,
ConnectionLost,
TCPTimedOutError,
ResponseFailed,
IOError,
TunnelError,
)
def __init__(self, settings):
class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
def __init__(self, settings: BaseSettings):
if not settings.getbool("RETRY_ENABLED"):
raise NotConfigured
self.max_retry_times = settings.getint("RETRY_TIMES")
@ -147,11 +151,22 @@ class RetryMiddleware:
)
self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
try:
self.exceptions_to_retry = self.__getattribute__("EXCEPTIONS_TO_RETRY")
except AttributeError:
# If EXCEPTIONS_TO_RETRY is not "overridden"
self.exceptions_to_retry = tuple(
load_object(x) if isinstance(x, str) else x
for x in settings.getlist("RETRY_EXCEPTIONS")
)
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings)
def process_response(self, request, response, spider):
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Union[Request, Response]:
if request.meta.get("dont_retry", False):
return response
if response.status in self.retry_http_codes:
@ -159,13 +174,21 @@ class RetryMiddleware:
return self._retry(request, reason, spider) or response
return response
def process_exception(self, request, exception, spider):
if isinstance(exception, self.EXCEPTIONS_TO_RETRY) and not request.meta.get(
def process_exception(
self, request: Request, exception: Exception, spider: Spider
) -> Union[Request, Response, None]:
if isinstance(exception, self.exceptions_to_retry) and not request.meta.get(
"dont_retry", False
):
return self._retry(request, exception, spider)
return None
def _retry(self, request, reason, spider):
def _retry(
self,
request: Request,
reason: Union[str, Exception, Type[Exception]],
spider: Spider,
) -> Optional[Request]:
max_retry_times = request.meta.get("max_retry_times", self.max_retry_times)
priority_adjust = request.meta.get("priority_adjust", self.priority_adjust)
return get_retry_request(
@ -175,3 +198,5 @@ class RetryMiddleware:
max_retry_times=max_retry_times,
priority_adjust=priority_adjust,
)
__getattr__ = backwards_compatibility_getattr

77
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -4,65 +4,87 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
"""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
from twisted.internet.defer import Deferred, maybeDeferred
from twisted.python.failure import Failure
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.robotstxt import RobotParser
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import load_object
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
class RobotsTxtMiddleware:
DOWNLOAD_PRIORITY = 1000
DOWNLOAD_PRIORITY: int = 1000
def __init__(self, crawler):
def __init__(self, crawler: Crawler):
if not crawler.settings.getbool("ROBOTSTXT_OBEY"):
raise NotConfigured
self._default_useragent = crawler.settings.get("USER_AGENT", "Scrapy")
self._robotstxt_useragent = crawler.settings.get("ROBOTSTXT_USER_AGENT", None)
self.crawler = crawler
self._parsers = {}
self._parserimpl = load_object(crawler.settings.get("ROBOTSTXT_PARSER"))
self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy")
self._robotstxt_useragent: Optional[str] = crawler.settings.get(
"ROBOTSTXT_USER_AGENT", None
)
self.crawler: Crawler = crawler
self._parsers: Dict[str, Union[RobotParser, Deferred, None]] = {}
self._parserimpl: RobotParser = load_object(
crawler.settings.get("ROBOTSTXT_PARSER")
)
# check if parser dependencies are met, this should throw an error otherwise.
self._parserimpl.from_crawler(self.crawler, b"")
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler)
def process_request(self, request, spider):
def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]:
if request.meta.get("dont_obey_robotstxt"):
return
return None
if request.url.startswith("data:") or request.url.startswith("file:"):
return
d = maybeDeferred(self.robot_parser, request, spider)
return None
d: Deferred = maybeDeferred(self.robot_parser, request, spider)
d.addCallback(self.process_request_2, request, spider)
return d
def process_request_2(self, rp, request, spider):
def process_request_2(
self, rp: Optional[RobotParser], request: Request, spider: Spider
) -> None:
if rp is None:
return
useragent = self._robotstxt_useragent
useragent: Union[str, bytes, None] = self._robotstxt_useragent
if not useragent:
useragent = request.headers.get(b"User-Agent", self._default_useragent)
assert useragent is not None
if not rp.allowed(request.url, useragent):
logger.debug(
"Forbidden by robots.txt: %(request)s",
{"request": request},
extra={"spider": spider},
)
assert self.crawler.stats
self.crawler.stats.inc_value("robotstxt/forbidden")
raise IgnoreRequest("Forbidden by robots.txt")
def robot_parser(self, request, spider):
def robot_parser(
self, request: Request, spider: Spider
) -> Union[RobotParser, Deferred, None]:
url = urlparse_cached(request)
netloc = url.netloc
@ -75,24 +97,27 @@ class RobotsTxtMiddleware:
meta={"dont_obey_robotstxt": True},
callback=NO_CALLBACK,
)
assert self.crawler.engine
assert self.crawler.stats
dfd = self.crawler.engine.download(robotsreq)
dfd.addCallback(self._parse_robots, netloc, spider)
dfd.addErrback(self._logerror, robotsreq, spider)
dfd.addErrback(self._robots_error, netloc)
self.crawler.stats.inc_value("robotstxt/request_count")
if isinstance(self._parsers[netloc], Deferred):
d = Deferred()
parser = self._parsers[netloc]
if isinstance(parser, Deferred):
d: Deferred = Deferred()
def cb(result):
def cb(result: Any) -> Any:
d.callback(result)
return result
self._parsers[netloc].addCallback(cb)
parser.addCallback(cb)
return d
return self._parsers[netloc]
return parser
def _logerror(self, failure, request, spider):
def _logerror(self, failure: Failure, request: Request, spider: Spider) -> Failure:
if failure.type is not IgnoreRequest:
logger.error(
"Error downloading %(request)s: %(f_exception)s",
@ -102,20 +127,24 @@ class RobotsTxtMiddleware:
)
return failure
def _parse_robots(self, response, netloc, spider):
def _parse_robots(self, response: Response, netloc: str, spider: Spider) -> None:
assert self.crawler.stats
self.crawler.stats.inc_value("robotstxt/response_count")
self.crawler.stats.inc_value(
f"robotstxt/response_status_count/{response.status}"
)
rp = self._parserimpl.from_crawler(self.crawler, response.body)
rp_dfd = self._parsers[netloc]
assert isinstance(rp_dfd, Deferred)
self._parsers[netloc] = rp
rp_dfd.callback(rp)
def _robots_error(self, failure, netloc):
def _robots_error(self, failure: Failure, netloc: str) -> None:
if failure.type is not IgnoreRequest:
key = f"robotstxt/exception_count/{failure.type}"
assert self.crawler.stats
self.crawler.stats.inc_value(key)
rp_dfd = self._parsers[netloc]
assert isinstance(rp_dfd, Deferred)
self._parsers[netloc] = None
rp_dfd.callback(None)

37
scrapy/downloadermiddlewares/stats.py
View File

@ -1,11 +1,23 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Dict, Union
from twisted.web import http
from scrapy import Request, Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.statscollectors import StatsCollector
from scrapy.utils.python import global_object_name, to_bytes
from scrapy.utils.request import request_httprepr
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
def get_header_size(headers):
def get_header_size(headers: Dict[str, Union[list, tuple]]) -> int:
size = 0
for key, value in headers.items():
if isinstance(value, (list, tuple)):
@ -14,30 +26,36 @@ def get_header_size(headers):
return size + len(b"\r\n") * (len(headers.keys()) - 1)
def get_status_size(response_status):
def get_status_size(response_status: int) -> int:
return len(to_bytes(http.RESPONSES.get(response_status, b""))) + 15
# resp.status + b"\r\n" + b"HTTP/1.1 <100-599> "
class DownloaderStats:
def __init__(self, stats):
self.stats = stats
def __init__(self, stats: StatsCollector):
self.stats: StatsCollector = stats
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("DOWNLOADER_STATS"):
raise NotConfigured
assert crawler.stats
return cls(crawler.stats)
def process_request(self, request, spider):
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
self.stats.inc_value("downloader/request_count", spider=spider)
self.stats.inc_value(
f"downloader/request_method_count/{request.method}", spider=spider
)
reqlen = len(request_httprepr(request))
self.stats.inc_value("downloader/request_bytes", reqlen, spider=spider)
return None
def process_response(self, request, response, spider):
def process_response(
self, request: Request, response: Response, spider: Spider
) -> Union[Request, Response]:
self.stats.inc_value("downloader/response_count", spider=spider)
self.stats.inc_value(
f"downloader/response_status_count/{response.status}", spider=spider
@ -52,9 +70,12 @@ class DownloaderStats:
self.stats.inc_value("downloader/response_bytes", reslen, spider=spider)
return response
def process_exception(self, request, exception, spider):
def process_exception(
self, request: Request, exception: Exception, spider: Spider
) -> Union[Request, Response, None]:
ex_class = global_object_name(exception.__class__)
self.stats.inc_value("downloader/exception_count", spider=spider)
self.stats.inc_value(
f"downloader/exception_type_count/{ex_class}", spider=spider
)
return None

23
scrapy/downloadermiddlewares/useragent.py
View File

@ -1,23 +1,36 @@
"""Set User-Agent header per spider or use a default value from settings"""
from scrapy import signals
from __future__ import annotations
from typing import TYPE_CHECKING, Union
from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
from scrapy.http import Response
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
class UserAgentMiddleware:
"""This middleware allows spiders to override the user_agent"""
def __init__(self, user_agent="Scrapy"):
def __init__(self, user_agent: str = "Scrapy"):
self.user_agent = user_agent
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
o = cls(crawler.settings["USER_AGENT"])
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o
def spider_opened(self, spider):
def spider_opened(self, spider: Spider) -> None:
self.user_agent = getattr(spider, "user_agent", self.user_agent)
def process_request(self, request, spider):
def process_request(
self, request: Request, spider: Spider
) -> Union[Request, Response, None]:
if self.user_agent:
request.headers.setdefault(b"User-Agent", self.user_agent)
return None

75
scrapy/dupefilters.py
View File

@ -1,25 +1,31 @@
from __future__ import annotations
import logging
from pathlib import Path
from typing import Optional, Set, Type, TypeVar
from warnings import warn
from typing import TYPE_CHECKING, Optional, Set
from twisted.internet.defer import Deferred
from scrapy.http.request import Request
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.job import job_dir
from scrapy.utils.request import RequestFingerprinter, referer_str
from scrapy.utils.request import (
RequestFingerprinter,
RequestFingerprinterProtocol,
referer_str,
)
BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
class BaseDupeFilter:
@classmethod
def from_settings(
cls: Type[BaseDupeFilterTV], settings: BaseSettings
) -> BaseDupeFilterTV:
def from_settings(cls, settings: BaseSettings) -> Self:
return cls()
def request_seen(self, request: Request) -> bool:
@ -36,9 +42,6 @@ class BaseDupeFilter:
pass
RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter")
class RFPDupeFilter(BaseDupeFilter):
"""Request Fingerprint duplicates filter"""
@ -47,10 +50,12 @@ class RFPDupeFilter(BaseDupeFilter):
path: Optional[str] = None,
debug: bool = False,
*,
fingerprinter=None,
fingerprinter: Optional[RequestFingerprinterProtocol] = None,
) -> None:
self.file = None
self.fingerprinter = fingerprinter or RequestFingerprinter()
self.fingerprinter: RequestFingerprinterProtocol = (
fingerprinter or RequestFingerprinter()
)
self.fingerprints: Set[str] = set()
self.logdupes = True
self.debug = debug
@ -62,40 +67,21 @@ class RFPDupeFilter(BaseDupeFilter):
@classmethod
def from_settings(
cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None
) -> RFPDupeFilterTV:
cls,
settings: BaseSettings,
*,
fingerprinter: Optional[RequestFingerprinterProtocol] = None,
) -> Self:
debug = settings.getbool("DUPEFILTER_DEBUG")
try:
return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
except TypeError:
warn(
"RFPDupeFilter subclasses must either modify their '__init__' "
"method to support a 'fingerprinter' parameter or reimplement "
"the 'from_settings' class method.",
ScrapyDeprecationWarning,
)
result = cls(job_dir(settings), debug)
result.fingerprinter = fingerprinter
return result
return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
@classmethod
def from_crawler(cls, crawler):
try:
return cls.from_settings(
crawler.settings,
fingerprinter=crawler.request_fingerprinter,
)
except TypeError:
warn(
"RFPDupeFilter subclasses must either modify their overridden "
"'__init__' method and 'from_settings' class method to "
"support a 'fingerprinter' parameter, or reimplement the "
"'from_crawler' class method.",
ScrapyDeprecationWarning,
)
result = cls.from_settings(crawler.settings)
result.fingerprinter = crawler.request_fingerprinter
return result
def from_crawler(cls, crawler: Crawler) -> Self:
assert crawler.request_fingerprinter
return cls.from_settings(
crawler.settings,
fingerprinter=crawler.request_fingerprinter,
)
def request_seen(self, request: Request) -> bool:
fp = self.request_fingerprint(request)
@ -127,4 +113,5 @@ class RFPDupeFilter(BaseDupeFilter):
self.logger.debug(msg, {"request": request}, extra={"spider": spider})
self.logdupes = False
assert spider.crawler.stats
spider.crawler.stats.inc_value("dupefilter/filtered", spider=spider)

5
scrapy/exceptions.py
View File

@ -4,6 +4,7 @@ Scrapy core exceptions
These exceptions are documented in docs/topics/exceptions.rst. Please don't add
new exceptions here without documenting them there.
"""
from typing import Any
# Internal
@ -51,7 +52,7 @@ class StopDownload(Exception):
should be handled by the request errback. Note that 'fail' is a keyword-only argument.
"""
def __init__(self, *, fail=True):
def __init__(self, *, fail: bool = True):
super().__init__()
self.fail = fail
@ -77,7 +78,7 @@ class NotSupported(Exception):
class UsageError(Exception):
"""To indicate a command-line usage error"""
def __init__(self, *a, **kw):
def __init__(self, *a: Any, **kw: Any):
self.print_help = kw.pop("print_help", True)
super().__init__(*a, **kw)

34
scrapy/exporters.py
View File

@ -7,13 +7,11 @@ import io
import marshal
import pickle
import pprint
import warnings
from collections.abc import Mapping
from xml.sax.saxutils import XMLGenerator
from itemadapter import ItemAdapter, is_item
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import Item
from scrapy.utils.python import is_listlike, to_bytes, to_unicode
from scrapy.utils.serialize import ScrapyJSONEncoder
@ -133,6 +131,13 @@ class JsonItemExporter(BaseItemExporter):
if self.indent is not None:
self.file.write(b"\n")
def _add_comma_after_first(self):
if self.first_item:
self.first_item = False
else:
self.file.write(b",")
self._beautify_newline()
def start_exporting(self):
self.file.write(b"[")
self._beautify_newline()
@ -142,14 +147,10 @@ class JsonItemExporter(BaseItemExporter):
self.file.write(b"]")
def export_item(self, item):
if self.first_item:
self.first_item = False
else:
self.file.write(b",")
self._beautify_newline()
itemdict = dict(self._get_serialized_fields(item))
data = self.encoder.encode(itemdict)
self.file.write(to_bytes(data, self.encoding))
data = to_bytes(self.encoder.encode(itemdict), self.encoding)
self._add_comma_after_first()
self.file.write(data)
class XmlItemExporter(BaseItemExporter):
@ -255,6 +256,9 @@ class CsvItemExporter(BaseItemExporter):
values = list(self._build_row(x for _, x in fields))
self.csv_writer.writerow(values)
def finish_exporting(self):
self.stream.detach() # Avoid closing the wrapped file.
def _build_row(self, values):
for s in values:
try:
@ -324,13 +328,7 @@ class PythonItemExporter(BaseItemExporter):
"""
def _configure(self, options, dont_fail=False):
self.binary = options.pop("binary", True)
super()._configure(options, dont_fail)
if self.binary:
warnings.warn(
"PythonItemExporter will drop support for binary export in the future",
ScrapyDeprecationWarning,
)
if not self.encoding:
self.encoding = "utf-8"
@ -345,18 +343,14 @@ class PythonItemExporter(BaseItemExporter):
return dict(self._serialize_item(value))
if is_listlike(value):
return [self._serialize_value(v) for v in value]
encode_func = to_bytes if self.binary else to_unicode
if isinstance(value, (str, bytes)):
return encode_func(value, encoding=self.encoding)
return to_unicode(value, encoding=self.encoding)
return value
def _serialize_item(self, item):
for key, value in ItemAdapter(item).items():
key = to_bytes(key) if self.binary else key
yield key, self._serialize_value(value)
def export_item(self, item):
result = dict(self._get_serialized_fields(item))
if self.binary:
result = dict(self._serialize_item(result))
return result

41
scrapy/extensions/closespider.py
View File

@ -4,11 +4,14 @@ conditions are met.
See documentation in docs/topics/extensions.rst
"""
import logging
from collections import defaultdict
from scrapy import signals
from scrapy.exceptions import NotConfigured
logger = logging.getLogger(__name__)
class CloseSpider:
def __init__(self, crawler):
@ -19,6 +22,7 @@ class CloseSpider:
"itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"),
"pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"),
"errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"),
"timeout_no_item": crawler.settings.getint("CLOSESPIDER_TIMEOUT_NO_ITEM"),
}
if not any(self.close_on.values()):
@ -34,6 +38,15 @@ class CloseSpider:
crawler.signals.connect(self.spider_opened, signal=signals.spider_opened)
if self.close_on.get("itemcount"):
crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
if self.close_on.get("timeout_no_item"):
self.timeout_no_item = self.close_on["timeout_no_item"]
self.items_in_period = 0
crawler.signals.connect(
self.spider_opened_no_item, signal=signals.spider_opened
)
crawler.signals.connect(
self.item_scraped_no_item, signal=signals.item_scraped
)
crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)
@classmethod
@ -69,3 +82,31 @@ class CloseSpider:
task = getattr(self, "task", False)
if task and task.active():
task.cancel()
task_no_item = getattr(self, "task_no_item", False)
if task_no_item and task_no_item.running:
task_no_item.stop()
def spider_opened_no_item(self, spider):
from twisted.internet import task
self.task_no_item = task.LoopingCall(self._count_items_produced, spider)
self.task_no_item.start(self.timeout_no_item, now=False)
logger.info(
f"Spider will stop when no items are produced after "
f"{self.timeout_no_item} seconds."
)
def item_scraped_no_item(self, item, spider):
self.items_in_period += 1
def _count_items_produced(self, spider):
if self.items_in_period >= 1:
self.items_in_period = 0
else:
logger.info(
f"Closing spider since no items were produced in the last "
f"{self.timeout_no_item} seconds."
)
self.crawler.engine.close_spider(spider, "closespider_timeout_no_item")

6
scrapy/extensions/corestats.py
View File

@ -1,7 +1,7 @@
"""
Extension for collecting core stats like items scraped and start/finish times
"""
from datetime import datetime
from datetime import datetime, timezone
from scrapy import signals
@ -22,11 +22,11 @@ class CoreStats:
return o
def spider_opened(self, spider):
self.start_time = datetime.utcnow()
self.start_time = datetime.now(tz=timezone.utc)
self.stats.set_value("start_time", self.start_time, spider=spider)
def spider_closed(self, spider, reason):
finish_time = datetime.utcnow()
finish_time = datetime.now(tz=timezone.utc)
elapsed_time = finish_time - self.start_time
elapsed_time_seconds = elapsed_time.total_seconds()
self.stats.set_value(

218
scrapy/extensions/feedexport.py
View File

@ -8,10 +8,10 @@ import logging
import re
import sys
import warnings
from datetime import datetime
from pathlib import Path
from datetime import datetime, timezone
from pathlib import Path, PureWindowsPath
from tempfile import NamedTemporaryFile
from typing import IO, Any, Callable, List, Optional, Tuple, Union
from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union
from urllib.parse import unquote, urlparse
from twisted.internet import defer, threads
@ -29,23 +29,20 @@ from scrapy.utils.deprecate import create_deprecated_class
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import get_func_args, without_none_values
from scrapy.utils.python import without_none_values
logger = logging.getLogger(__name__)
try:
import boto3 # noqa: F401
IS_BOTO3_AVAILABLE = True
except ImportError:
IS_BOTO3_AVAILABLE = False
def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
argument_names = get_func_args(builder)
if "feed_options" in argument_names:
kwargs["feed_options"] = feed_options
else:
warnings.warn(
f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a "
"'feed_options' parameter to its signature to remove this "
"warning. This parameter will become mandatory in a future "
"version of Scrapy.",
category=ScrapyDeprecationWarning,
)
kwargs["feed_options"] = feed_options
return builder(*preargs, uri, *args, **kwargs)
@ -165,6 +162,7 @@ class S3FeedStorage(BlockingFeedStorage):
*,
feed_options=None,
session_token=None,
region_name=None,
):
if not is_botocore_available():
raise NotConfigured("missing botocore library")
@ -176,16 +174,41 @@ class S3FeedStorage(BlockingFeedStorage):
self.keyname = u.path[1:] # remove first "/"
self.acl = acl
self.endpoint_url = endpoint_url
import botocore.session
self.region_name = region_name
if IS_BOTO3_AVAILABLE:
import boto3.session
session = boto3.session.Session()
self.s3_client = session.client(
"s3",
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
region_name=self.region_name,
)
else:
warnings.warn(
"`botocore` usage has been deprecated for S3 feed "
"export, please use `boto3` to avoid problems",
category=ScrapyDeprecationWarning,
)
import botocore.session
session = botocore.session.get_session()
self.s3_client = session.create_client(
"s3",
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
region_name=self.region_name,
)
session = botocore.session.get_session()
self.s3_client = session.create_client(
"s3",
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
)
if feed_options and feed_options.get("overwrite", True) is False:
logger.warning(
"S3 does not support appending to files. To "
@ -203,15 +226,22 @@ class S3FeedStorage(BlockingFeedStorage):
session_token=crawler.settings["AWS_SESSION_TOKEN"],
acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None,
endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None,
region_name=crawler.settings["AWS_REGION_NAME"] or None,
feed_options=feed_options,
)
def _store_in_thread(self, file):
file.seek(0)
kwargs = {"ACL": self.acl} if self.acl else {}
self.s3_client.put_object(
Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs
)
if IS_BOTO3_AVAILABLE:
kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}
self.s3_client.upload_fileobj(
Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs
)
else:
kwargs = {"ACL": self.acl} if self.acl else {}
self.s3_client.put_object(
Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs
)
file.close()
@ -242,15 +272,23 @@ class GCSFeedStorage(BlockingFeedStorage):
class FTPFeedStorage(BlockingFeedStorage):
def __init__(self, uri, use_active_mode=False, *, feed_options=None):
def __init__(
self,
uri: str,
use_active_mode: bool = False,
*,
feed_options: Optional[Dict[str, Any]] = None,
):
u = urlparse(uri)
self.host = u.hostname
self.port = int(u.port or "21")
self.username = u.username
self.password = unquote(u.password or "")
self.path = u.path
self.use_active_mode = use_active_mode
self.overwrite = not feed_options or feed_options.get("overwrite", True)
if not u.hostname:
raise ValueError(f"Got a storage URI without a hostname: {uri}")
self.host: str = u.hostname
self.port: int = int(u.port or "21")
self.username: str = u.username or ""
self.password: str = unquote(u.password or "")
self.path: str = u.path
self.use_active_mode: bool = use_active_mode
self.overwrite: bool = not feed_options or feed_options.get("overwrite", True)
@classmethod
def from_crawler(cls, crawler, uri, *, feed_options=None):
@ -277,8 +315,6 @@ class FTPFeedStorage(BlockingFeedStorage):
class FeedSlot:
def __init__(
self,
file,
exporter,
storage,
uri,
format,
@ -286,9 +322,14 @@ class FeedSlot:
batch_id,
uri_template,
filter,
feed_options,
spider,
exporters,
settings,
crawler,
):
self.file = file
self.exporter = exporter
self.file = None
self.exporter = None
self.storage = storage
# feed params
self.batch_id = batch_id
@ -297,15 +338,44 @@ class FeedSlot:
self.uri_template = uri_template
self.uri = uri
self.filter = filter
# exporter params
self.feed_options = feed_options
self.spider = spider
self.exporters = exporters
self.settings = settings
self.crawler = crawler
# flags
self.itemcount = 0
self._exporting = False
self._fileloaded = False
def start_exporting(self):
if not self._fileloaded:
self.file = self.storage.open(self.spider)
if "postprocessing" in self.feed_options:
self.file = PostProcessingManager(
self.feed_options["postprocessing"], self.file, self.feed_options
)
self.exporter = self._get_exporter(
file=self.file,
format=self.feed_options["format"],
fields_to_export=self.feed_options["fields"],
encoding=self.feed_options["encoding"],
indent=self.feed_options["indent"],
**self.feed_options["item_export_kwargs"],
)
self._fileloaded = True
if not self._exporting:
self.exporter.start_exporting()
self._exporting = True
def _get_instance(self, objcls, *args, **kwargs):
return create_instance(objcls, self.settings, self.crawler, *args, **kwargs)
def _get_exporter(self, file, format, *args, **kwargs):
return self._get_instance(self.exporters[format], file, *args, **kwargs)
def finish_exporting(self):
if self._exporting:
self.exporter.finish_exporting()
@ -347,7 +417,9 @@ class FeedExporter:
category=ScrapyDeprecationWarning,
stacklevel=2,
)
uri = str(self.settings["FEED_URI"]) # handle pathlib.Path objects
uri = self.settings["FEED_URI"]
# handle pathlib.Path objects
uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")}
self.feeds[uri] = feed_complete_default_values_from_settings(
feed_options, self.settings
@ -357,7 +429,8 @@ class FeedExporter:
# 'FEEDS' setting takes precedence over 'FEED_URI'
for uri, feed_options in self.settings.getdict("FEEDS").items():
uri = str(uri) # handle pathlib.Path objects
# handle pathlib.Path objects
uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
self.feeds[uri] = feed_complete_default_values_from_settings(
feed_options, self.settings
)
@ -406,11 +479,16 @@ class FeedExporter:
return slot_.file.file
return slot_.file
slot.finish_exporting()
if not slot.itemcount and not slot.store_empty:
# We need to call slot.storage.store nonetheless to get the file
# properly closed.
return defer.maybeDeferred(slot.storage.store, get_file(slot))
if slot.itemcount:
# Normal case
slot.finish_exporting()
elif slot.store_empty and slot.batch_id == 1:
# Need to store the empty file
slot.start_exporting()
slot.finish_exporting()
else:
# In this case, the file is not stored, so no processing is required.
return None
logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
d = defer.maybeDeferred(slot.storage.store, get_file(slot))
@ -455,23 +533,7 @@ class FeedExporter:
:param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri
"""
storage = self._get_storage(uri, feed_options)
file = storage.open(spider)
if "postprocessing" in feed_options:
file = PostProcessingManager(
feed_options["postprocessing"], file, feed_options
)
exporter = self._get_exporter(
file=file,
format=feed_options["format"],
fields_to_export=feed_options["fields"],
encoding=feed_options["encoding"],
indent=feed_options["indent"],
**feed_options["item_export_kwargs"],
)
slot = FeedSlot(
file=file,
exporter=exporter,
storage=storage,
uri=uri,
format=feed_options["format"],
@ -479,9 +541,12 @@ class FeedExporter:
batch_id=batch_id,
uri_template=uri_template,
filter=self.filters[uri_template],
feed_options=feed_options,
spider=spider,
exporters=self.exporters,
settings=self.settings,
crawler=getattr(self, "crawler", None),
)
if slot.store_empty:
slot.start_exporting()
return slot
def item_scraped(self, item, spider):
@ -553,7 +618,7 @@ class FeedExporter:
def _storage_supported(self, uri, feed_options):
scheme = urlparse(uri).scheme
if scheme in self.storages:
if scheme in self.storages or PureWindowsPath(uri).drive:
try:
self._get_storage(uri, feed_options)
return True
@ -565,21 +630,13 @@ class FeedExporter:
else:
logger.error("Unknown feed storage scheme: %(scheme)s", {"scheme": scheme})
def _get_instance(self, objcls, *args, **kwargs):
return create_instance(
objcls, self.settings, getattr(self, "crawler", None), *args, **kwargs
)
def _get_exporter(self, file, format, *args, **kwargs):
return self._get_instance(self.exporters[format], file, *args, **kwargs)
def _get_storage(self, uri, feed_options):
"""Fork of create_instance specific to feed storage classes
It supports not passing the *feed_options* parameters to classes that
do not support it, and issuing a deprecation warning instead.
"""
feedcls = self.storages[urlparse(uri).scheme]
feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"])
crawler = getattr(self, "crawler", None)
def build_instance(builder, *preargs):
@ -609,25 +666,16 @@ class FeedExporter:
params = {}
for k in dir(spider):
params[k] = getattr(spider, k)
utc_now = datetime.utcnow()
utc_now = datetime.now(tz=timezone.utc)
params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-")
params["batch_time"] = utc_now.isoformat().replace(":", "-")
params["batch_id"] = slot.batch_id + 1 if slot is not None else 1
original_params = params.copy()
uripar_function = (
load_object(uri_params_function)
if uri_params_function
else lambda params, _: params
)
new_params = uripar_function(params, spider)
if new_params is None or original_params != params:
warnings.warn(
"Modifying the params dictionary in-place in the function defined in "
"the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS "
"setting is deprecated. The function must return a new dictionary "
"instead.",
category=ScrapyDeprecationWarning,
)
return new_params if new_params is not None else params
def _load_filter(self, feed_options):

1
scrapy/extensions/httpcache.py
View File

@ -291,6 +291,7 @@ class FilesystemCacheStorage:
extra={"spider": spider},
)
assert spider.crawler.request_fingerprinter
self._fingerprinter = spider.crawler.request_fingerprinter
def close_spider(self, spider):

140
scrapy/extensions/periodic_log.py Normal file
View File

@ -0,0 +1,140 @@
import logging
from datetime import datetime, timezone
from twisted.internet import task
from scrapy import signals
from scrapy.exceptions import NotConfigured
from scrapy.utils.serialize import ScrapyJSONEncoder
logger = logging.getLogger(__name__)
class PeriodicLog:
"""Log basic scraping stats periodically"""
def __init__(
self,
stats,
interval=60.0,
ext_stats={},
ext_delta={},
ext_timing_enabled=False,
):
self.stats = stats
self.interval = interval
self.multiplier = 60.0 / self.interval
self.task = None
self.encoder = ScrapyJSONEncoder(sort_keys=True, indent=4)
self.ext_stats_enabled = bool(ext_stats)
self.ext_stats_include = ext_stats.get("include", [])
self.ext_stats_exclude = ext_stats.get("exclude", [])
self.ext_delta_enabled = bool(ext_delta)
self.ext_delta_include = ext_delta.get("include", [])
self.ext_delta_exclude = ext_delta.get("exclude", [])
self.ext_timing_enabled = ext_timing_enabled
@classmethod
def from_crawler(cls, crawler):
interval = crawler.settings.getfloat("LOGSTATS_INTERVAL")
if not interval:
raise NotConfigured
try:
ext_stats = crawler.settings.getdict("PERIODIC_LOG_STATS")
except (TypeError, ValueError):
ext_stats = (
{"enabled": True}
if crawler.settings.getbool("PERIODIC_LOG_STATS")
else None
)
try:
ext_delta = crawler.settings.getdict("PERIODIC_LOG_DELTA")
except (TypeError, ValueError):
ext_delta = (
{"enabled": True}
if crawler.settings.getbool("PERIODIC_LOG_DELTA")
else None
)
ext_timing_enabled = crawler.settings.getbool(
"PERIODIC_LOG_TIMING_ENABLED", False
)
if not (ext_stats or ext_delta or ext_timing_enabled):
raise NotConfigured
o = cls(
crawler.stats,
interval,
ext_stats,
ext_delta,
ext_timing_enabled,
)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_opened(self, spider):
self.time_prev = datetime.now(tz=timezone.utc)
self.delta_prev = {}
self.stats_prev = {}
self.task = task.LoopingCall(self.log)
self.task.start(self.interval)
def log(self):
data = {}
if self.ext_timing_enabled:
data.update(self.log_timing())
if self.ext_delta_enabled:
data.update(self.log_delta())
if self.ext_stats_enabled:
data.update(self.log_crawler_stats())
logger.info(self.encoder.encode(data))
def log_delta(self):
num_stats = {
k: v
for k, v in self.stats._stats.items()
if isinstance(v, (int, float))
and self.param_allowed(k, self.ext_delta_include, self.ext_delta_exclude)
}
delta = {k: v - self.delta_prev.get(k, 0) for k, v in num_stats.items()}
self.delta_prev = num_stats
return {"delta": delta}
def log_timing(self):
now = datetime.now(tz=timezone.utc)
time = {
"log_interval": self.interval,
"start_time": self.stats._stats["start_time"],
"utcnow": now,
"log_interval_real": (now - self.time_prev).total_seconds(),
"elapsed": (now - self.stats._stats["start_time"]).total_seconds(),
}
self.time_prev = now
return {"time": time}
def log_crawler_stats(self):
stats = {
k: v
for k, v in self.stats._stats.items()
if self.param_allowed(k, self.ext_stats_include, self.ext_stats_exclude)
}
return {"stats": stats}
def param_allowed(self, stat_name, include, exclude):
if not include and not exclude:
return True
for p in exclude:
if p in stat_name:
return False
if exclude and not include:
return True
for p in include:
if p in stat_name:
return True
return False
def spider_closed(self, spider, reason):
self.log()
if self.task and self.task.running:
self.task.stop()

7
scrapy/http/common.py
View File

@ -1,7 +0,0 @@
def obsolete_setter(setter, attrname):
def newsetter(self, value):
c = self.__class__.__name__
msg = f"{c}.{attrname} is not modifiable, use {c}.replace() instead"
raise AttributeError(msg)
return newsetter

10
scrapy/http/cookies.py
View File

@ -1,8 +1,12 @@
import re
import time
from http.cookiejar import Cookie
from http.cookiejar import CookieJar as _CookieJar
from http.cookiejar import DefaultCookiePolicy
from typing import Sequence
from scrapy import Request
from scrapy.http import Response
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
@ -24,7 +28,7 @@ class CookieJar:
wrsp = WrappedResponse(response)
return self.jar.extract_cookies(wrsp, wreq)
def add_cookie_header(self, request):
def add_cookie_header(self, request: Request) -> None:
wreq = WrappedRequest(request)
self.policy._now = self.jar._now = int(time.time())
@ -75,7 +79,7 @@ class CookieJar:
def set_policy(self, pol):
return self.jar.set_policy(pol)
def make_cookies(self, response, request):
def make_cookies(self, response: Response, request: Request) -> Sequence[Cookie]:
wreq = WrappedRequest(request)
wrsp = WrappedResponse(response)
return self.jar.make_cookies(wrsp, wreq)
@ -83,7 +87,7 @@ class CookieJar:
def set_cookie(self, cookie):
self.jar.set_cookie(cookie)
def set_cookie_if_ok(self, cookie, request):
def set_cookie_if_ok(self, cookie: Cookie, request: Request) -> None:
self.jar.set_cookie_if_ok(cookie, WrappedRequest(request))

97
scrapy/http/headers.py
View File

@ -1,41 +1,73 @@
from __future__ import annotations
from collections.abc import Mapping
from typing import (
TYPE_CHECKING,
Any,
AnyStr,
Dict,
Iterable,
List,
Optional,
Tuple,
Union,
cast,
)
from w3lib.http import headers_dict_to_raw
from scrapy.utils.datatypes import CaselessDict
from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
_RawValueT = Union[bytes, str, int]
# isn't fully compatible typing-wise with either dict or CaselessDict,
# but it needs refactoring anyway, see also https://github.com/scrapy/scrapy/pull/5146
class Headers(CaselessDict):
"""Case insensitive http headers dictionary"""
def __init__(self, seq=None, encoding="utf-8"):
self.encoding = encoding
def __init__(
self,
seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
encoding: str = "utf-8",
):
self.encoding: str = encoding
super().__init__(seq)
def update(self, seq):
def update( # type: ignore[override]
self, seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]]]
) -> None:
seq = seq.items() if isinstance(seq, Mapping) else seq
iseq = {}
iseq: Dict[bytes, List[bytes]] = {}
for k, v in seq:
iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
super().update(iseq)
def normkey(self, key):
def normkey(self, key: AnyStr) -> bytes: # type: ignore[override]
"""Normalize key to bytes"""
return self._tobytes(key.title())
def normvalue(self, value):
def normvalue(self, value: Union[_RawValueT, Iterable[_RawValueT]]) -> List[bytes]:
"""Normalize values to bytes"""
_value: Iterable[_RawValueT]
if value is None:
value = []
_value = []
elif isinstance(value, (str, bytes)):
value = [value]
elif not hasattr(value, "__iter__"):
value = [value]
_value = [value]
elif hasattr(value, "__iter__"):
_value = value
else:
_value = [value]
return [self._tobytes(x) for x in value]
return [self._tobytes(x) for x in _value]
def _tobytes(self, x):
def _tobytes(self, x: _RawValueT) -> bytes:
if isinstance(x, bytes):
return x
if isinstance(x, str):
@ -44,51 +76,54 @@ class Headers(CaselessDict):
return str(x).encode(self.encoding)
raise TypeError(f"Unsupported value type: {type(x)}")
def __getitem__(self, key):
def __getitem__(self, key: AnyStr) -> Optional[bytes]:
try:
return super().__getitem__(key)[-1]
return cast(List[bytes], super().__getitem__(key))[-1]
except IndexError:
return None
def get(self, key, def_val=None):
def get(self, key: AnyStr, def_val: Any = None) -> Optional[bytes]:
try:
return super().get(key, def_val)[-1]
return cast(List[bytes], super().get(key, def_val))[-1]
except IndexError:
return None
def getlist(self, key, def_val=None):
def getlist(self, key: AnyStr, def_val: Any = None) -> List[bytes]:
try:
return super().__getitem__(key)
return cast(List[bytes], super().__getitem__(key))
except KeyError:
if def_val is not None:
return self.normvalue(def_val)
return []
def setlist(self, key, list_):
def setlist(self, key: AnyStr, list_: Iterable[_RawValueT]) -> None:
self[key] = list_
def setlistdefault(self, key, default_list=()):
def setlistdefault(
self, key: AnyStr, default_list: Iterable[_RawValueT] = ()
) -> Any:
return self.setdefault(key, default_list)
def appendlist(self, key, value):
def appendlist(self, key: AnyStr, value: Iterable[_RawValueT]) -> None:
lst = self.getlist(key)
lst.extend(self.normvalue(value))
self[key] = lst
def items(self):
def items(self) -> Iterable[Tuple[bytes, List[bytes]]]: # type: ignore[override]
return ((k, self.getlist(k)) for k in self.keys())
def values(self):
def values(self) -> List[Optional[bytes]]: # type: ignore[override]
return [self[k] for k in self.keys()]
def to_string(self):
return headers_dict_to_raw(self)
def to_string(self) -> bytes:
# cast() can be removed if the headers_dict_to_raw() hint is improved
return cast(bytes, headers_dict_to_raw(self))
def to_unicode_dict(self):
"""Return headers as a CaselessDict with unicode keys
and unicode values. Multiple values are joined with ','.
def to_unicode_dict(self) -> CaseInsensitiveDict:
"""Return headers as a CaseInsensitiveDict with str keys
and str values. Multiple values are joined with ','.
"""
return CaselessDict(
return CaseInsensitiveDict(
(
to_unicode(key, encoding=self.encoding),
to_unicode(b",".join(value), encoding=self.encoding),
@ -96,7 +131,7 @@ class Headers(CaselessDict):
for key, value in self.items()
)
def __copy__(self):
def __copy__(self) -> Self:
return self.__class__(self)
copy = __copy__

76
scrapy/http/request/__init__.py
View File

@ -5,12 +5,26 @@ requests in Scrapy.
See documentation in docs/topics/request-response.rst
"""
import inspect
from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union
from typing import (
Any,
AnyStr,
Callable,
Dict,
Iterable,
List,
Mapping,
NoReturn,
Optional,
Tuple,
Type,
TypeVar,
Union,
cast,
)
from w3lib.url import safe_url_string
import scrapy
from scrapy.http.common import obsolete_setter
from scrapy.http.headers import Headers
from scrapy.utils.curl import curl_to_request_kwargs
from scrapy.utils.python import to_bytes
@ -20,7 +34,7 @@ from scrapy.utils.url import escape_ajax
RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
def NO_CALLBACK(*args, **kwargs):
def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn:
"""When assigned to the ``callback`` parameter of
:class:`~scrapy.http.Request`, it indicates that the request is not meant
to have a spider callback at all.
@ -77,24 +91,24 @@ class Request(object_ref):
url: str,
callback: Optional[Callable] = None,
method: str = "GET",
headers: Optional[dict] = None,
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
cookies: Optional[Union[dict, List[dict]]] = None,
meta: Optional[dict] = None,
meta: Optional[Dict[str, Any]] = None,
encoding: str = "utf-8",
priority: int = 0,
dont_filter: bool = False,
errback: Optional[Callable] = None,
flags: Optional[List[str]] = None,
cb_kwargs: Optional[dict] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
) -> None:
self._encoding = encoding # this one has to be set first
self.method = str(method).upper()
self._encoding: str = encoding # this one has to be set first
self.method: str = str(method).upper()
self._set_url(url)
self._set_body(body)
if not isinstance(priority, int):
raise TypeError(f"Request priority not an integer: {priority!r}")
self.priority = priority
self.priority: int = priority
if not (callable(callback) or callback is None):
raise TypeError(
@ -102,30 +116,33 @@ class Request(object_ref):
)
if not (callable(errback) or errback is None):
raise TypeError(f"errback must be a callable, got {type(errback).__name__}")
self.callback = callback
self.errback = errback
self.callback: Optional[Callable] = callback
self.errback: Optional[Callable] = errback
self.cookies = cookies or {}
self.headers = Headers(headers or {}, encoding=encoding)
self.dont_filter = dont_filter
self.cookies: Union[dict, List[dict]] = cookies or {}
self.headers: Headers = Headers(headers or {}, encoding=encoding)
self.dont_filter: bool = dont_filter
self._meta = dict(meta) if meta else None
self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None
self.flags = [] if flags is None else list(flags)
self._meta: Optional[Dict[str, Any]] = dict(meta) if meta else None
self._cb_kwargs: Optional[Dict[str, Any]] = (
dict(cb_kwargs) if cb_kwargs else None
)
self.flags: List[str] = [] if flags is None else list(flags)
@property
def cb_kwargs(self) -> dict:
def cb_kwargs(self) -> Dict[str, Any]:
if self._cb_kwargs is None:
self._cb_kwargs = {}
return self._cb_kwargs
@property
def meta(self) -> dict:
def meta(self) -> Dict[str, Any]:
if self._meta is None:
self._meta = {}
return self._meta
def _get_url(self) -> str:
@property
def url(self) -> str:
return self._url
def _set_url(self, url: str) -> None:
@ -142,16 +159,13 @@ class Request(object_ref):
):
raise ValueError(f"Missing scheme in request url: {self._url}")
url = property(_get_url, obsolete_setter(_set_url, "url"))
def _get_body(self) -> bytes:
@property
def body(self) -> bytes:
return self._body
def _set_body(self, body: Optional[Union[str, bytes]]) -> None:
self._body = b"" if body is None else to_bytes(body, self.encoding)
body = property(_get_body, obsolete_setter(_set_body, "body"))
@property
def encoding(self) -> str:
return self._encoding
@ -162,19 +176,19 @@ class Request(object_ref):
def copy(self) -> "Request":
return self.replace()
def replace(self, *args, **kwargs) -> "Request":
def replace(self, *args: Any, **kwargs: Any) -> "Request":
"""Create a new Request with the same attributes except for those given new values"""
for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))
cls = kwargs.pop("cls", self.__class__)
return cls(*args, **kwargs)
return cast(Request, cls(*args, **kwargs))
@classmethod
def from_curl(
cls: Type[RequestTypeVar],
curl_command: str,
ignore_unknown_options: bool = True,
**kwargs,
**kwargs: Any,
) -> RequestTypeVar:
"""Create a Request object from a string containing a `cURL
<https://curl.haxx.se/>`_ command. It populates the HTTP method, the
@ -188,7 +202,7 @@ class Request(object_ref):
``ignore_unknown_options=False``.
.. caution:: Using :meth:`from_curl` from :class:`~scrapy.http.Request`
subclasses, such as :class:`~scrapy.http.JSONRequest`, or
subclasses, such as :class:`~scrapy.http.JsonRequest`, or
:class:`~scrapy.http.XmlRpcRequest`, as well as having
:ref:`downloader middlewares <topics-downloader-middleware>`
and
@ -207,7 +221,7 @@ class Request(object_ref):
request_kwargs.update(kwargs)
return cls(**request_kwargs)
def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> dict:
def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]:
"""Return a dictionary containing the Request's data.
Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.
@ -232,7 +246,7 @@ class Request(object_ref):
return d
def _find_method(obj, func):
def _find_method(obj: Any, func: Callable) -> str:
"""Helper function for Request.to_dict"""
# Only instance methods contain ``__func__``
if obj and hasattr(func, "__func__"):

27
scrapy/http/request/form.py
View File

@ -5,7 +5,9 @@ This module implements the FormRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union, cast
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast
from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
from lxml.html import (
@ -24,7 +26,10 @@ from scrapy.http.response.text import TextResponse
from scrapy.utils.python import is_listlike, to_bytes
from scrapy.utils.response import get_base_url
FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest")
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
FormdataKVType = Tuple[str, Union[str, Iterable[str]]]
FormdataType = Optional[Union[dict, List[FormdataKVType]]]
@ -33,7 +38,9 @@ FormdataType = Optional[Union[dict, List[FormdataKVType]]]
class FormRequest(Request):
valid_form_methods = ["GET", "POST"]
def __init__(self, *args, formdata: FormdataType = None, **kwargs) -> None:
def __init__(
self, *args: Any, formdata: FormdataType = None, **kwargs: Any
) -> None:
if formdata and kwargs.get("method") is None:
kwargs["method"] = "POST"
@ -54,7 +61,7 @@ class FormRequest(Request):
@classmethod
def from_response(
cls: Type[FormRequestTypeVar],
cls,
response: TextResponse,
formname: Optional[str] = None,
formid: Optional[str] = None,
@ -64,8 +71,8 @@ class FormRequest(Request):
dont_click: bool = False,
formxpath: Optional[str] = None,
formcss: Optional[str] = None,
**kwargs,
) -> FormRequestTypeVar:
**kwargs: Any,
) -> Self:
kwargs.setdefault("encoding", response.encoding)
if formcss is not None:
@ -121,12 +128,12 @@ def _get_form(
if formname is not None:
f = root.xpath(f'//form[@name="{formname}"]')
if f:
return f[0]
return cast(FormElement, f[0])
if formid is not None:
f = root.xpath(f'//form[@id="{formid}"]')
if f:
return f[0]
return cast(FormElement, f[0])
# Get form element from xpath, if not found, go up
if formxpath is not None:
@ -135,7 +142,7 @@ def _get_form(
el = nodes[0]
while True:
if el.tag == "form":
return el
return cast(FormElement, el)
el = el.getparent()
if el is None:
break
@ -147,7 +154,7 @@ def _get_form(
except IndexError:
raise IndexError(f"Form number {formnumber} not found in {response}")
else:
return form
return cast(FormElement, form)
def _get_inputs(

12
scrapy/http/request/json_request.py
View File

@ -8,16 +8,17 @@ See documentation in docs/topics/request-response.rst
import copy
import json
import warnings
from typing import Optional, Tuple
from typing import Any, Optional, Tuple
from scrapy.http.request import Request
from scrapy.utils.deprecate import create_deprecated_class
class JsonRequest(Request):
attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",)
def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None:
def __init__(
self, *args: Any, dumps_kwargs: Optional[dict] = None, **kwargs: Any
) -> None:
dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {}
dumps_kwargs.setdefault("sort_keys", True)
self._dumps_kwargs = dumps_kwargs
@ -43,7 +44,7 @@ class JsonRequest(Request):
def dumps_kwargs(self) -> dict:
return self._dumps_kwargs
def replace(self, *args, **kwargs) -> Request:
def replace(self, *args: Any, **kwargs: Any) -> Request:
body_passed = kwargs.get("body", None) is not None
data = kwargs.pop("data", None)
data_passed = data is not None
@ -58,6 +59,3 @@ class JsonRequest(Request):
def _dumps(self, data: dict) -> str:
"""Convert to JSON"""
return json.dumps(data, **self._dumps_kwargs)
JSONRequest = create_deprecated_class("JSONRequest", JsonRequest)

4
scrapy/http/request/rpc.py
View File

@ -5,7 +5,7 @@ This module implements the XmlRpcRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
import xmlrpc.client as xmlrpclib
from typing import Optional
from typing import Any, Optional
from scrapy.http.request import Request
from scrapy.utils.python import get_func_args
@ -14,7 +14,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps)
class XmlRpcRequest(Request):
def __init__(self, *args, encoding: Optional[str] = None, **kwargs):
def __init__(self, *args: Any, encoding: Optional[str] = None, **kwargs: Any):
if "body" not in kwargs and "params" in kwargs:
kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs)
kwargs["body"] = xmlrpclib.dumps(**kw)

151
scrapy/http/response/__init__.py
View File

@ -4,16 +4,37 @@ responses in Scrapy.
See documentation in docs/topics/request-response.rst
"""
from typing import Generator, Tuple
from __future__ import annotations
from ipaddress import IPv4Address, IPv6Address
from typing import (
TYPE_CHECKING,
Any,
AnyStr,
Callable,
Dict,
Generator,
Iterable,
List,
Mapping,
Optional,
Tuple,
Union,
cast,
)
from urllib.parse import urljoin
from twisted.internet.ssl import Certificate
from scrapy.exceptions import NotSupported
from scrapy.http.common import obsolete_setter
from scrapy.http.headers import Headers
from scrapy.http.request import Request
from scrapy.link import Link
from scrapy.utils.trackref import object_ref
if TYPE_CHECKING:
from scrapy.selector import SelectorList
class Response(object_ref):
"""An object that represents an HTTP response, which is usually
@ -41,29 +62,29 @@ class Response(object_ref):
def __init__(
self,
url: str,
status=200,
headers=None,
body=b"",
flags=None,
request=None,
certificate=None,
ip_address=None,
protocol=None,
status: int = 200,
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: bytes = b"",
flags: Optional[List[str]] = None,
request: Optional[Request] = None,
certificate: Optional[Certificate] = None,
ip_address: Union[IPv4Address, IPv6Address, None] = None,
protocol: Optional[str] = None,
):
self.headers = Headers(headers or {})
self.status = int(status)
self.headers: Headers = Headers(headers or {})
self.status: int = int(status)
self._set_body(body)
self._set_url(url)
self.request = request
self.flags = [] if flags is None else list(flags)
self.certificate = certificate
self.ip_address = ip_address
self.protocol = protocol
self.request: Optional[Request] = request
self.flags: List[str] = [] if flags is None else list(flags)
self.certificate: Optional[Certificate] = certificate
self.ip_address: Union[IPv4Address, IPv6Address, None] = ip_address
self.protocol: Optional[str] = protocol
@property
def cb_kwargs(self):
def cb_kwargs(self) -> Dict[str, Any]:
try:
return self.request.cb_kwargs
return self.request.cb_kwargs # type: ignore[union-attr]
except AttributeError:
raise AttributeError(
"Response.cb_kwargs not available, this response "
@ -71,32 +92,32 @@ class Response(object_ref):
)
@property
def meta(self):
def meta(self) -> Dict[str, Any]:
try:
return self.request.meta
return self.request.meta # type: ignore[union-attr]
except AttributeError:
raise AttributeError(
"Response.meta not available, this response "
"is not tied to any request"
)
def _get_url(self):
@property
def url(self) -> str:
return self._url
def _set_url(self, url: str):
def _set_url(self, url: str) -> None:
if isinstance(url, str):
self._url = url
self._url: str = url
else:
raise TypeError(
f"{type(self).__name__} url must be str, " f"got {type(url).__name__}"
)
url = property(_get_url, obsolete_setter(_set_url, "url"))
def _get_body(self):
@property
def body(self) -> bytes:
return self._body
def _set_body(self, body):
def _set_body(self, body: Optional[bytes]) -> None:
if body is None:
self._body = b""
elif not isinstance(body, bytes):
@ -108,47 +129,45 @@ class Response(object_ref):
else:
self._body = body
body = property(_get_body, obsolete_setter(_set_body, "body"))
def __repr__(self):
def __repr__(self) -> str:
return f"<{self.status} {self.url}>"
def copy(self):
def copy(self) -> Response:
"""Return a copy of this Response"""
return self.replace()
def replace(self, *args, **kwargs):
def replace(self, *args: Any, **kwargs: Any) -> Response:
"""Create a new Response with the same attributes except for those given new values"""
for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))
cls = kwargs.pop("cls", self.__class__)
return cls(*args, **kwargs)
return cast(Response, cls(*args, **kwargs))
def urljoin(self, url):
def urljoin(self, url: str) -> str:
"""Join this Response's url with a possible relative url to form an
absolute interpretation of the latter."""
return urljoin(self.url, url)
@property
def text(self):
def text(self) -> str:
"""For subclasses of TextResponse, this will return the body
as str
"""
raise AttributeError("Response content isn't text")
def css(self, *a, **kw):
def css(self, *a: Any, **kw: Any) -> SelectorList:
"""Shortcut method implemented only by responses whose content
is text (subclasses of TextResponse).
"""
raise NotSupported("Response content isn't text")
def jmespath(self, *a, **kw):
def jmespath(self, *a: Any, **kw: Any) -> SelectorList:
"""Shortcut method implemented only by responses whose content
is text (subclasses of TextResponse).
"""
raise NotSupported("Response content isn't text")
def xpath(self, *a, **kw):
def xpath(self, *a: Any, **kw: Any) -> SelectorList:
"""Shortcut method implemented only by responses whose content
is text (subclasses of TextResponse).
"""
@ -156,19 +175,19 @@ class Response(object_ref):
def follow(
self,
url,
callback=None,
method="GET",
headers=None,
body=None,
cookies=None,
meta=None,
encoding="utf-8",
priority=0,
dont_filter=False,
errback=None,
cb_kwargs=None,
flags=None,
url: Union[str, Link],
callback: Optional[Callable] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
cookies: Optional[Union[dict, List[dict]]] = None,
meta: Optional[Dict[str, Any]] = None,
encoding: Optional[str] = "utf-8",
priority: int = 0,
dont_filter: bool = False,
errback: Optional[Callable] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
) -> Request:
"""
Return a :class:`~.Request` instance to follow a link ``url``.
@ -183,6 +202,8 @@ class Response(object_ref):
.. versionadded:: 2.0
The *flags* parameter.
"""
if encoding is None:
raise ValueError("encoding can't be None")
if isinstance(url, Link):
url = url.url
elif url is None:
@ -207,19 +228,19 @@ class Response(object_ref):
def follow_all(
self,
urls,
callback=None,
method="GET",
headers=None,
body=None,
cookies=None,
meta=None,
encoding="utf-8",
priority=0,
dont_filter=False,
errback=None,
cb_kwargs=None,
flags=None,
urls: Iterable[Union[str, Link]],
callback: Optional[Callable] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
cookies: Optional[Union[dict, List[dict]]] = None,
meta: Optional[Dict[str, Any]] = None,
encoding: Optional[str] = "utf-8",
priority: int = 0,
dont_filter: bool = False,
errback: Optional[Callable] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
) -> Generator[Request, None, None]:
"""
.. versionadded:: 2.0

149
scrapy/http/response/text.py
View File

@ -4,10 +4,25 @@ discovering (through HTTP headers) to base Response class.
See documentation in docs/topics/request-response.rst
"""
from __future__ import annotations
import json
from contextlib import suppress
from typing import Generator, Tuple
from typing import (
TYPE_CHECKING,
Any,
AnyStr,
Callable,
Dict,
Generator,
Iterable,
List,
Mapping,
Optional,
Tuple,
Union,
cast,
)
from urllib.parse import urljoin
import parsel
@ -22,9 +37,13 @@ from w3lib.html import strip_html5_whitespace
from scrapy.http import Request
from scrapy.http.response import Response
from scrapy.link import Link
from scrapy.utils.python import memoizemethod_noargs, to_unicode
from scrapy.utils.response import get_base_url
if TYPE_CHECKING:
from scrapy.selector import Selector, SelectorList
_NONE = object()
@ -34,21 +53,15 @@ class TextResponse(Response):
attributes: Tuple[str, ...] = Response.attributes + ("encoding",)
def __init__(self, *args, **kwargs):
self._encoding = kwargs.pop("encoding", None)
self._cached_benc = None
self._cached_ubody = None
self._cached_selector = None
def __init__(self, *args: Any, **kwargs: Any):
self._encoding: Optional[str] = kwargs.pop("encoding", None)
self._cached_benc: Optional[str] = None
self._cached_ubody: Optional[str] = None
self._cached_selector: Optional[Selector] = None
super().__init__(*args, **kwargs)
def _set_url(self, url):
if isinstance(url, str):
self._url = to_unicode(url, self.encoding)
else:
super()._set_url(url)
def _set_body(self, body):
self._body = b"" # used by encoding detection
def _set_body(self, body: Union[str, bytes, None]) -> None:
self._body: bytes = b"" # used by encoding detection
if isinstance(body, str):
if self._encoding is None:
raise TypeError(
@ -60,10 +73,10 @@ class TextResponse(Response):
super()._set_body(body)
@property
def encoding(self):
def encoding(self) -> str:
return self._declared_encoding() or self._body_inferred_encoding()
def _declared_encoding(self):
def _declared_encoding(self) -> Optional[str]:
return (
self._encoding
or self._bom_encoding()
@ -71,18 +84,18 @@ class TextResponse(Response):
or self._body_declared_encoding()
)
def json(self):
def json(self) -> Any:
"""
.. versionadded:: 2.2
Deserialize a JSON document to a Python object.
"""
if self._cached_decoded_json is _NONE:
self._cached_decoded_json = json.loads(self.text)
self._cached_decoded_json = json.loads(self.body)
return self._cached_decoded_json
@property
def text(self):
def text(self) -> str:
"""Body as unicode"""
# access self.encoding before _cached_ubody to make sure
# _body_inferred_encoding is called
@ -92,20 +105,20 @@ class TextResponse(Response):
self._cached_ubody = html_to_unicode(charset, self.body)[1]
return self._cached_ubody
def urljoin(self, url):
def urljoin(self, url: str) -> str:
"""Join this Response's url with a possible relative url to form an
absolute interpretation of the latter."""
return urljoin(get_base_url(self), url)
@memoizemethod_noargs
def _headers_encoding(self):
content_type = self.headers.get(b"Content-Type", b"")
def _headers_encoding(self) -> Optional[str]:
content_type = cast(bytes, self.headers.get(b"Content-Type", b""))
return http_content_type_encoding(to_unicode(content_type, encoding="latin-1"))
def _body_inferred_encoding(self):
def _body_inferred_encoding(self) -> str:
if self._cached_benc is None:
content_type = to_unicode(
self.headers.get(b"Content-Type", b""), encoding="latin-1"
cast(bytes, self.headers.get(b"Content-Type", b"")), encoding="latin-1"
)
benc, ubody = html_to_unicode(
content_type,
@ -117,59 +130,66 @@ class TextResponse(Response):
self._cached_ubody = ubody
return self._cached_benc
def _auto_detect_fun(self, text):
def _auto_detect_fun(self, text: bytes) -> Optional[str]:
for enc in (self._DEFAULT_ENCODING, "utf-8", "cp1252"):
try:
text.decode(enc)
except UnicodeError:
continue
return resolve_encoding(enc)
return None
@memoizemethod_noargs
def _body_declared_encoding(self):
def _body_declared_encoding(self) -> Optional[str]:
return html_body_declared_encoding(self.body)
@memoizemethod_noargs
def _bom_encoding(self):
def _bom_encoding(self) -> Optional[str]:
return read_bom(self.body)[0]
@property
def selector(self):
def selector(self) -> Selector:
from scrapy.selector import Selector
if self._cached_selector is None:
self._cached_selector = Selector(self)
return self._cached_selector
def jmespath(self, query, **kwargs):
def jmespath(self, query: str, **kwargs: Any) -> SelectorList:
from scrapy.selector import SelectorList
if not hasattr(self.selector, "jmespath"): # type: ignore[attr-defined]
raise AttributeError(
"Please install parsel >= 1.8.1 to get jmespath support"
)
return self.selector.jmespath(query, **kwargs) # type: ignore[attr-defined]
return cast(SelectorList, self.selector.jmespath(query, **kwargs)) # type: ignore[attr-defined]
def xpath(self, query, **kwargs):
return self.selector.xpath(query, **kwargs)
def xpath(self, query: str, **kwargs: Any) -> SelectorList:
from scrapy.selector import SelectorList
def css(self, query):
return self.selector.css(query)
return cast(SelectorList, self.selector.xpath(query, **kwargs))
def css(self, query: str) -> SelectorList:
from scrapy.selector import SelectorList
return cast(SelectorList, self.selector.css(query))
def follow(
self,
url,
callback=None,
method="GET",
headers=None,
body=None,
cookies=None,
meta=None,
encoding=None,
priority=0,
dont_filter=False,
errback=None,
cb_kwargs=None,
flags=None,
url: Union[str, Link, parsel.Selector],
callback: Optional[Callable] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
cookies: Optional[Union[dict, List[dict]]] = None,
meta: Optional[Dict[str, Any]] = None,
encoding: Optional[str] = None,
priority: int = 0,
dont_filter: bool = False,
errback: Optional[Callable] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
) -> Request:
"""
Return a :class:`~.Request` instance to follow a link ``url``.
@ -210,21 +230,21 @@ class TextResponse(Response):
def follow_all(
self,
urls=None,
callback=None,
method="GET",
headers=None,
body=None,
cookies=None,
meta=None,
encoding=None,
priority=0,
dont_filter=False,
errback=None,
cb_kwargs=None,
flags=None,
css=None,
xpath=None,
urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None,
callback: Optional[Callable] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
cookies: Optional[Union[dict, List[dict]]] = None,
meta: Optional[Dict[str, Any]] = None,
encoding: Optional[str] = None,
priority: int = 0,
dont_filter: bool = False,
errback: Optional[Callable] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
css: Optional[str] = None,
xpath: Optional[str] = None,
) -> Generator[Request, None, None]:
"""
A generator that produces :class:`~.Request` instances to follow all
@ -266,7 +286,7 @@ class TextResponse(Response):
with suppress(_InvalidSelector):
urls.append(_url_from_selector(sel))
return super().follow_all(
urls=urls,
urls=cast(Iterable[Union[str, Link]], urls),
callback=callback,
method=method,
headers=headers,
@ -288,8 +308,7 @@ class _InvalidSelector(ValueError):
"""
def _url_from_selector(sel):
# type: (parsel.Selector) -> str
def _url_from_selector(sel: parsel.Selector) -> str:
if isinstance(sel.root, str):
# e.g. ::attr(href) result
return strip_html5_whitespace(sel.root)

21
scrapy/link.py
View File

@ -4,6 +4,7 @@ This module defines the Link object used in Link extractors.
For actual link extractors implementation see scrapy.linkextractors, or
its documentation in: docs/topics/link-extractors.rst
"""
from typing import Any
class Link:
@ -26,16 +27,20 @@ class Link:
__slots__ = ["url", "text", "fragment", "nofollow"]
def __init__(self, url, text="", fragment="", nofollow=False):
def __init__(
self, url: str, text: str = "", fragment: str = "", nofollow: bool = False
):
if not isinstance(url, str):
got = url.__class__.__name__
raise TypeError(f"Link urls must be str objects, got {got}")
self.url = url
self.text = text
self.fragment = fragment
self.nofollow = nofollow
self.url: str = url
self.text: str = text
self.fragment: str = fragment
self.nofollow: bool = nofollow
def __eq__(self, other):
def __eq__(self, other: Any) -> bool:
if not isinstance(other, Link):
raise NotImplementedError
return (
self.url == other.url
and self.text == other.text
@ -43,12 +48,12 @@ class Link:
and self.nofollow == other.nofollow
)
def __hash__(self):
def __hash__(self) -> int:
return (
hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow)
)
def __repr__(self):
def __repr__(self) -> str:
return (
f"Link(url={self.url!r}, text={self.text!r}, "
f"fragment={self.fragment!r}, nofollow={self.nofollow!r})"

21
scrapy/linkextractors/__init__.py
View File

@ -38,6 +38,7 @@ IGNORED_EXTENSIONS = [
"svg",
"cdr",
"ico",
"webp",
# audio
"mp3",
"wma",
@ -66,12 +67,22 @@ IGNORED_EXTENSIONS = [
"webm",
# office suites
"xls",
"xlsm",
"xlsx",
"xltm",
"xltx",
"potm",
"potx",
"ppt",
"pptm",
"pptx",
"pps",
"doc",
"docb",
"docm",
"docx",
"dotm",
"dotx",
"odt",
"ods",
"odg",
@ -85,6 +96,16 @@ IGNORED_EXTENSIONS = [
"dmg",
"iso",
"apk",
"jar",
"sh",
"rb",
"js",
"hta",
"bat",
"cpl",
"msi",
"msp",
"py",
]

2
scrapy/loader/__init__.py
View File

@ -78,7 +78,7 @@ class ItemLoader(itemloaders.ItemLoader):
read-only.
"""
default_item_class = Item
default_item_class: type = Item
default_selector_class = Selector
def __init__(self, item=None, selector=None, response=None, parent=None, **context):

21
scrapy/loader/common.py
View File

@ -1,21 +0,0 @@
"""Common functions used in Item Loaders code"""
import warnings
from itemloaders import common
from scrapy.utils.deprecate import ScrapyDeprecationWarning
def wrap_loader_context(function, context):
"""Wrap functions that receive loader_context to contain the context
"pre-loaded" and expose a interface that receives only one argument
"""
warnings.warn(
"scrapy.loader.common.wrap_loader_context has moved to a new library."
"Please update your reference to itemloaders.common.wrap_loader_context",
ScrapyDeprecationWarning,
stacklevel=2,
)
return common.wrap_loader_context(function, context)

20
scrapy/loader/processors.py
View File

@ -1,20 +0,0 @@
"""
This module provides some commonly used processors for Item Loaders.
See documentation in docs/topics/loaders.rst
"""
from itemloaders import processors
from scrapy.utils.deprecate import create_deprecated_class
MapCompose = create_deprecated_class("MapCompose", processors.MapCompose)
Compose = create_deprecated_class("Compose", processors.Compose)
TakeFirst = create_deprecated_class("TakeFirst", processors.TakeFirst)
Identity = create_deprecated_class("Identity", processors.Identity)
SelectJmes = create_deprecated_class("SelectJmes", processors.SelectJmes)
Join = create_deprecated_class("Join", processors.Join)

21
scrapy/logformatter.py
View File

@ -1,6 +1,8 @@
from __future__ import annotations
import logging
import os
from typing import Any, Dict, Optional, Union
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
from twisted.python.failure import Failure
@ -8,6 +10,13 @@ from scrapy import Request, Spider
from scrapy.http import Response
from scrapy.utils.request import referer_str
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s"
DROPPEDMSG = "Dropped: %(exception)s" + os.linesep + "%(item)s"
CRAWLEDMSG = "Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
@ -105,7 +114,7 @@ class LogFormatter:
}
def item_error(
self, item: Any, exception, response: Response, spider: Spider
self, item: Any, exception: BaseException, response: Response, spider: Spider
) -> dict:
"""Logs a message when an item causes an error while it is passing
through the item pipeline.
@ -121,7 +130,11 @@ class LogFormatter:
}
def spider_error(
self, failure: Failure, request: Request, response: Response, spider: Spider
self,
failure: Failure,
request: Request,
response: Union[Response, Failure],
spider: Spider,
) -> dict:
"""Logs an error message from a spider.
@ -161,5 +174,5 @@ class LogFormatter:
}
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls()

7
scrapy/mail.py
View File

@ -96,10 +96,9 @@ class MailSender:
rcpts.extend(cc)
msg["Cc"] = COMMASPACE.join(cc)
if charset:
msg.set_charset(charset)
if attachs:
if charset:
msg.set_charset(charset)
msg.attach(MIMEText(body, "plain", charset or "us-ascii"))
for attach_name, mimetype, f in attachs:
part = MIMEBase(*mimetype.split("/"))
@ -110,7 +109,7 @@ class MailSender:
)
msg.attach(part)
else:
msg.set_payload(body)
msg.set_payload(body, charset)
if _callback:
_callback(to=to, subject=subject, body=body, cc=cc, attach=attachs, msg=msg)

34
scrapy/middleware.py
View File

@ -1,7 +1,21 @@
from __future__ import annotations
import logging
import pprint
from collections import defaultdict, deque
from typing import Any, Callable, Deque, Dict, Iterable, List, Tuple, Union, cast
from typing import (
TYPE_CHECKING,
Any,
Callable,
Deque,
Dict,
Iterable,
List,
Optional,
Tuple,
Union,
cast,
)
from twisted.internet.defer import Deferred
@ -11,6 +25,13 @@ from scrapy.settings import Settings
from scrapy.utils.defer import process_chain, process_parallel
from scrapy.utils.misc import create_instance, load_object
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
@ -34,7 +55,9 @@ class MiddlewareManager:
raise NotImplementedError
@classmethod
def from_settings(cls, settings: Settings, crawler=None):
def from_settings(
cls, settings: Settings, crawler: Optional[Crawler] = None
) -> Self:
mwlist = cls._get_mwlist_from_settings(settings)
middlewares = []
enabled = []
@ -46,10 +69,9 @@ class MiddlewareManager:
enabled.append(clspath)
except NotConfigured as e:
if e.args:
clsname = clspath.split(".")[-1]
logger.warning(
"Disabled %(clsname)s: %(eargs)s",
{"clsname": clsname, "eargs": e.args[0]},
"Disabled %(clspath)s: %(eargs)s",
{"clspath": clspath, "eargs": e.args[0]},
extra={"crawler": crawler},
)
@ -64,7 +86,7 @@ class MiddlewareManager:
return cls(*middlewares)
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls.from_settings(crawler.settings, crawler)
def _add_middleware(self, mw: Any) -> None:

4
scrapy/pipelines/files.py
View File

@ -28,7 +28,7 @@ from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.media import MediaPipeline
from scrapy.settings import Settings
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.datatypes import CaselessDict
from scrapy.utils.datatypes import CaseInsensitiveDict
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import md5sum
@ -155,7 +155,7 @@ class S3FilesStore:
def _headers_to_botocore_kwargs(self, headers):
"""Convert headers to botocore keyword arguments."""
# This is required while we need to support both boto and botocore.
mapping = CaselessDict(
mapping = CaseInsensitiveDict(
{
"Content-Type": "ContentType",
"Cache-Control": "CacheControl",

3
scrapy/pipelines/images.py
View File

@ -8,6 +8,7 @@ import hashlib
import warnings
from contextlib import suppress
from io import BytesIO
from typing import Dict, Tuple
from itemadapter import ItemAdapter
@ -48,7 +49,7 @@ class ImagesPipeline(FilesPipeline):
MIN_WIDTH = 0
MIN_HEIGHT = 0
EXPIRES = 90
THUMBS = {}
THUMBS: Dict[str, Tuple[int, int]] = {}
DEFAULT_IMAGES_URLS_FIELD = "image_urls"
DEFAULT_IMAGES_RESULT_FIELD = "images"

52
scrapy/pipelines/media.py
View File

@ -1,8 +1,6 @@
import functools
import logging
from collections import defaultdict
from inspect import signature
from warnings import warn
from twisted.internet.defer import Deferred, DeferredList
from twisted.python.failure import Failure
@ -11,7 +9,6 @@ from scrapy.http.request import NO_CALLBACK
from scrapy.settings import Settings
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.defer import defer_result, mustbe_deferred
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import arg_to_iter
@ -44,9 +41,6 @@ class MediaPipeline:
self.allow_redirects = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False)
self._handle_statuses(self.allow_redirects)
# Check if deprecated methods are being used and make them compatible
self._make_compatible()
def _handle_statuses(self, allow_redirects):
self.handle_httpstatus_list = None
if allow_redirects:
@ -126,52 +120,6 @@ class MediaPipeline:
)
return dfd.addBoth(lambda _: wad) # it must return wad at last
def _make_compatible(self):
"""Make overridable methods of MediaPipeline and subclasses backwards compatible"""
methods = [
"file_path",
"thumb_path",
"media_to_download",
"media_downloaded",
"file_downloaded",
"image_downloaded",
"get_images",
]
for method_name in methods:
method = getattr(self, method_name, None)
if callable(method):
setattr(self, method_name, self._compatible(method))
def _compatible(self, func):
"""Wrapper for overridable methods to allow backwards compatibility"""
self._check_signature(func)
@functools.wraps(func)
def wrapper(*args, **kwargs):
if self._expects_item[func.__name__]:
return func(*args, **kwargs)
kwargs.pop("item", None)
return func(*args, **kwargs)
return wrapper
def _check_signature(self, func):
sig = signature(func)
self._expects_item[func.__name__] = True
if "item" not in sig.parameters:
old_params = str(sig)[1:-1]
new_params = old_params + ", *, item=None"
warn(
f"{func.__name__}(self, {old_params}) is deprecated, "
f"please use {func.__name__}(self, {new_params})",
ScrapyDeprecationWarning,
stacklevel=2,
)
self._expects_item[func.__name__] = False
def _modify_media_request(self, request):
if self.handle_httpstatus_list:
request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list

0
scrapy/py.typed Normal file
View File

8
scrapy/resolver.py
View File

@ -1,3 +1,5 @@
from typing import Any
from twisted.internet import defer
from twisted.internet.base import ThreadedResolver
from twisted.internet.interfaces import (
@ -11,7 +13,7 @@ from zope.interface.declarations import implementer, provider
from scrapy.utils.datatypes import LocalCache
# TODO: cache misses
dnscache = LocalCache(10000)
dnscache: LocalCache[str, Any] = LocalCache(10000)
@implementer(IResolverSimple)
@ -36,7 +38,7 @@ class CachingThreadedResolver(ThreadedResolver):
def install_on_reactor(self):
self.reactor.installResolver(self)
def getHostByName(self, name, timeout=None):
def getHostByName(self, name: str, timeout=None):
if name in dnscache:
return defer.succeed(dnscache[name])
# in Twisted<=16.6, getHostByName() is always called with
@ -110,7 +112,7 @@ class CachingHostnameResolver:
def resolveHostName(
self,
resolutionReceiver,
hostName,
hostName: str,
portNumber=0,
addressTypes=None,
transportSemantics="TCP",

39
scrapy/responsetypes.py
View File

@ -5,6 +5,7 @@ based on different criteria.
from io import StringIO
from mimetypes import MimeTypes
from pkgutil import get_data
from typing import Dict, Mapping, Optional, Type, Union
from scrapy.http import Response
from scrapy.utils.misc import load_object
@ -29,15 +30,19 @@ class ResponseTypes:
"text/*": "scrapy.http.TextResponse",
}
def __init__(self):
self.classes = {}
self.mimetypes = MimeTypes()
mimedata = get_data("scrapy", "mime.types").decode("utf8")
self.mimetypes.readfp(StringIO(mimedata))
def __init__(self) -> None:
self.classes: Dict[str, Type[Response]] = {}
self.mimetypes: MimeTypes = MimeTypes()
mimedata = get_data("scrapy", "mime.types")
if not mimedata:
raise ValueError(
"The mime.types file is not found in the Scrapy installation"
)
self.mimetypes.readfp(StringIO(mimedata.decode("utf8")))
for mimetype, cls in self.CLASSES.items():
self.classes[mimetype] = load_object(cls)
def from_mimetype(self, mimetype):
def from_mimetype(self, mimetype: str) -> Type[Response]:
"""Return the most appropriate Response class for the given mimetype"""
if mimetype is None:
return Response
@ -46,7 +51,9 @@ class ResponseTypes:
basetype = f"{mimetype.split('/')[0]}/*"
return self.classes.get(basetype, Response)
def from_content_type(self, content_type, content_encoding=None):
def from_content_type(
self, content_type: Union[str, bytes], content_encoding: Optional[bytes] = None
) -> Type[Response]:
"""Return the most appropriate Response class from an HTTP Content-Type
header"""
if content_encoding:
@ -56,7 +63,9 @@ class ResponseTypes:
)
return self.from_mimetype(mimetype)
def from_content_disposition(self, content_disposition):
def from_content_disposition(
self, content_disposition: Union[str, bytes]
) -> Type[Response]:
try:
filename = (
to_unicode(content_disposition, encoding="latin-1", errors="replace")
@ -68,7 +77,7 @@ class ResponseTypes:
except IndexError:
return Response
def from_headers(self, headers):
def from_headers(self, headers: Mapping[bytes, bytes]) -> Type[Response]:
"""Return the most appropriate Response class by looking at the HTTP
headers"""
cls = Response
@ -81,14 +90,14 @@ class ResponseTypes:
cls = self.from_content_disposition(headers[b"Content-Disposition"])
return cls
def from_filename(self, filename):
def from_filename(self, filename: str) -> Type[Response]:
"""Return the most appropriate Response class from a file name"""
mimetype, encoding = self.mimetypes.guess_type(filename)
if mimetype and not encoding:
return self.from_mimetype(mimetype)
return Response
def from_body(self, body):
def from_body(self, body: bytes) -> Type[Response]:
"""Try to guess the appropriate response based on the body content.
This method is a bit magic and could be improved in the future, but
it's not meant to be used except for special cases where response types
@ -106,7 +115,13 @@ class ResponseTypes:
return self.from_mimetype("text/html")
return self.from_mimetype("text")
def from_args(self, headers=None, url=None, filename=None, body=None):
def from_args(
self,
headers: Optional[Mapping[bytes, bytes]] = None,
url: Optional[str] = None,
filename: Optional[str] = None,
body: Optional[bytes] = None,
) -> Type[Response]:
"""Guess the most appropriate Response class based on
the given arguments."""
cls = Response

20
scrapy/robotstxt.py
View File

@ -1,9 +1,20 @@
from __future__ import annotations
import logging
import sys
from abc import ABCMeta, abstractmethod
from typing import TYPE_CHECKING, Union
from warnings import warn
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
@ -29,7 +40,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
class RobotParser(metaclass=ABCMeta):
@classmethod
@abstractmethod
def from_crawler(cls, crawler, robotstxt_body):
def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
"""Parse the content of a robots.txt_ file as bytes. This must be a class method.
It must return a new instance of the parser backend.
@ -42,14 +53,14 @@ class RobotParser(metaclass=ABCMeta):
pass
@abstractmethod
def allowed(self, url, user_agent):
def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool:
"""Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``.
:param url: Absolute URL
:type url: str
:type url: str or bytes
:param user_agent: User agent
:type user_agent: str
:type user_agent: str or bytes
"""
pass
@ -79,6 +90,7 @@ class PythonRobotParser(RobotParser):
class ReppyRobotParser(RobotParser):
def __init__(self, robotstxt_body, spider):
warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2)
from reppy.robots import Robots
self.spider = spider

18
scrapy/selector/unified.py
View File

@ -1,10 +1,11 @@
"""
XPath selectors based on lxml
"""
from typing import Any, Optional, Type, Union
from parsel import Selector as _ParselSelector
from scrapy.http import HtmlResponse, XmlResponse
from scrapy.http import HtmlResponse, TextResponse, XmlResponse
from scrapy.utils.python import to_bytes
from scrapy.utils.trackref import object_ref
@ -13,14 +14,14 @@ __all__ = ["Selector", "SelectorList"]
_NOT_SET = object()
def _st(response, st):
def _st(response: Optional[TextResponse], st: Optional[str]) -> str:
if st is None:
return "xml" if isinstance(response, XmlResponse) else "html"
return st
def _response_from_text(text, st):
rt = XmlResponse if st == "xml" else HtmlResponse
def _response_from_text(text: Union[str, bytes], st: Optional[str]) -> TextResponse:
rt: Type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse
return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8"))
@ -65,7 +66,14 @@ class Selector(_ParselSelector, object_ref):
__slots__ = ["response"]
selectorlist_cls = SelectorList
def __init__(self, response=None, text=None, type=None, root=_NOT_SET, **kwargs):
def __init__(
self,
response: Optional[TextResponse] = None,
text: Optional[str] = None,
type: Optional[str] = None,
root: Optional[Any] = _NOT_SET,
**kwargs: Any,
):
if response is not None and text is not None:
raise ValueError(
f"{self.__class__.__name__}.__init__() received "

184
scrapy/settings/__init__.py
View File

@ -1,21 +1,52 @@
from __future__ import annotations
import copy
import json
from collections.abc import MutableMapping
from importlib import import_module
from pprint import pformat
from types import ModuleType
from typing import (
TYPE_CHECKING,
Any,
Dict,
Iterable,
Iterator,
List,
Mapping,
MutableMapping,
Optional,
Tuple,
Union,
cast,
)
from scrapy.settings import default_settings
SETTINGS_PRIORITIES = {
# The key types are restricted in BaseSettings._get_key() to ones supported by JSON,
# see https://github.com/scrapy/scrapy/issues/5383.
_SettingsKeyT = Union[bool, float, int, str, None]
if TYPE_CHECKING:
# https://github.com/python/typing/issues/445#issuecomment-1131458824
from _typeshed import SupportsItems
# typing.Self requires Python 3.11
from typing_extensions import Self
_SettingsInputT = Union[SupportsItems[_SettingsKeyT, Any], str, None]
SETTINGS_PRIORITIES: Dict[str, int] = {
"default": 0,
"command": 10,
"addon": 15,
"project": 20,
"spider": 30,
"cmdline": 40,
}
def get_settings_priority(priority):
def get_settings_priority(priority: Union[int, str]) -> int:
"""
Small helper function that looks up a given string priority in the
:attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its
@ -34,14 +65,15 @@ class SettingsAttribute:
for settings configuration, not this one.
"""
def __init__(self, value, priority):
self.value = value
def __init__(self, value: Any, priority: int):
self.value: Any = value
self.priority: int
if isinstance(self.value, BaseSettings):
self.priority = max(self.value.maxpriority(), priority)
else:
self.priority = priority
def set(self, value, priority):
def set(self, value: Any, priority: int) -> None:
"""Sets value if priority is higher or equal than current priority."""
if priority >= self.priority:
if isinstance(self.value, BaseSettings):
@ -49,11 +81,11 @@ class SettingsAttribute:
self.value = value
self.priority = priority
def __repr__(self):
def __repr__(self) -> str:
return f"<SettingsAttribute value={self.value!r} priority={self.priority}>"
class BaseSettings(MutableMapping):
class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
"""
Instances of this class behave like dictionaries, but store priorities
along with their ``(key, value)`` pairs, and can be frozen (i.e. marked
@ -75,21 +107,25 @@ class BaseSettings(MutableMapping):
highest priority will be retrieved.
"""
def __init__(self, values=None, priority="project"):
self.frozen = False
self.attributes = {}
__default = object()
def __init__(
self, values: _SettingsInputT = None, priority: Union[int, str] = "project"
):
self.frozen: bool = False
self.attributes: dict[_SettingsKeyT, SettingsAttribute] = {}
if values:
self.update(values, priority)
def __getitem__(self, opt_name):
def __getitem__(self, opt_name: _SettingsKeyT) -> Any:
if opt_name not in self:
return None
return self.attributes[opt_name].value
def __contains__(self, name):
def __contains__(self, name: Any) -> bool:
return name in self.attributes
def get(self, name, default=None):
def get(self, name: _SettingsKeyT, default: Any = None) -> Any:
"""
Get a setting value without affecting its original type.
@ -101,7 +137,7 @@ class BaseSettings(MutableMapping):
"""
return self[name] if self[name] is not None else default
def getbool(self, name, default=False):
def getbool(self, name: _SettingsKeyT, default: bool = False) -> bool:
"""
Get a setting value as a boolean.
@ -131,7 +167,7 @@ class BaseSettings(MutableMapping):
"'True'/'False' and 'true'/'false'"
)
def getint(self, name, default=0):
def getint(self, name: _SettingsKeyT, default: int = 0) -> int:
"""
Get a setting value as an int.
@ -143,7 +179,7 @@ class BaseSettings(MutableMapping):
"""
return int(self.get(name, default))
def getfloat(self, name, default=0.0):
def getfloat(self, name: _SettingsKeyT, default: float = 0.0) -> float:
"""
Get a setting value as a float.
@ -155,7 +191,9 @@ class BaseSettings(MutableMapping):
"""
return float(self.get(name, default))
def getlist(self, name, default=None):
def getlist(
self, name: _SettingsKeyT, default: Optional[List[Any]] = None
) -> List[Any]:
"""
Get a setting value as a list. If the setting original type is a list, a
copy of it will be returned. If it's a string it will be split by ",".
@ -174,7 +212,9 @@ class BaseSettings(MutableMapping):
value = value.split(",")
return list(value)
def getdict(self, name, default=None):
def getdict(
self, name: _SettingsKeyT, default: Optional[Dict[Any, Any]] = None
) -> Dict[Any, Any]:
"""
Get a setting value as a dictionary. If the setting original type is a
dictionary, a copy of it will be returned. If it is a string it will be
@ -195,7 +235,11 @@ class BaseSettings(MutableMapping):
value = json.loads(value)
return dict(value)
def getdictorlist(self, name, default=None):
def getdictorlist(
self,
name: _SettingsKeyT,
default: Union[Dict[Any, Any], List[Any], Tuple[Any], None] = None,
) -> Union[Dict[Any, Any], List[Any]]:
"""Get a setting value as either a :class:`dict` or a :class:`list`.
If the setting is already a dict or a list, a copy of it will be
@ -222,24 +266,31 @@ class BaseSettings(MutableMapping):
return {}
if isinstance(value, str):
try:
return json.loads(value)
value_loaded = json.loads(value)
assert isinstance(value_loaded, (dict, list))
return value_loaded
except ValueError:
return value.split(",")
if isinstance(value, tuple):
return list(value)
assert isinstance(value, (dict, list))
return copy.deepcopy(value)
def getwithbase(self, name):
def getwithbase(self, name: _SettingsKeyT) -> "BaseSettings":
"""Get a composition of a dictionary-like setting and its `_BASE`
counterpart.
:param name: name of the dictionary-like setting
:type name: str
"""
if not isinstance(name, str):
raise ValueError(f"Base setting key must be a string, got {name}")
compbs = BaseSettings()
compbs.update(self[name + "_BASE"])
compbs.update(self[name])
return compbs
def getpriority(self, name):
def getpriority(self, name: _SettingsKeyT) -> Optional[int]:
"""
Return the current numerical priority value of a setting, or ``None`` if
the given ``name`` does not exist.
@ -251,7 +302,7 @@ class BaseSettings(MutableMapping):
return None
return self.attributes[name].priority
def maxpriority(self):
def maxpriority(self) -> int:
"""
Return the numerical value of the highest priority present throughout
all settings, or the numerical value for ``default`` from
@ -259,13 +310,15 @@ class BaseSettings(MutableMapping):
stored.
"""
if len(self) > 0:
return max(self.getpriority(name) for name in self)
return max(cast(int, self.getpriority(name)) for name in self)
return get_settings_priority("default")
def __setitem__(self, name, value):
def __setitem__(self, name: _SettingsKeyT, value: Any) -> None:
self.set(name, value)
def set(self, name, value, priority="project"):
def set(
self, name: _SettingsKeyT, value: Any, priority: Union[int, str] = "project"
) -> None:
"""
Store a key/value attribute with a given priority.
@ -293,17 +346,26 @@ class BaseSettings(MutableMapping):
else:
self.attributes[name].set(value, priority)
def setdefault(self, name, default=None, priority="project"):
def setdefault(
self,
name: _SettingsKeyT,
default: Any = None,
priority: Union[int, str] = "project",
) -> Any:
if name not in self:
self.set(name, default, priority)
return default
return self.attributes[name].value
def setdict(self, values, priority="project"):
def setdict(
self, values: _SettingsInputT, priority: Union[int, str] = "project"
) -> None:
self.update(values, priority)
def setmodule(self, module, priority="project"):
def setmodule(
self, module: Union[ModuleType, str], priority: Union[int, str] = "project"
) -> None:
"""
Store settings from a module with a given priority.
@ -325,7 +387,8 @@ class BaseSettings(MutableMapping):
if key.isupper():
self.set(key, getattr(module, key), priority)
def update(self, values, priority="project"):
# BaseSettings.update() doesn't support all inputs that MutableMapping.update() supports
def update(self, values: _SettingsInputT, priority: Union[int, str] = "project") -> None: # type: ignore[override]
"""
Store key/value pairs with a given priority.
@ -349,30 +412,34 @@ class BaseSettings(MutableMapping):
"""
self._assert_mutability()
if isinstance(values, str):
values = json.loads(values)
values = cast(dict, json.loads(values))
if values is not None:
if isinstance(values, BaseSettings):
for name, value in values.items():
self.set(name, value, values.getpriority(name))
self.set(name, value, cast(int, values.getpriority(name)))
else:
for name, value in values.items():
self.set(name, value, priority)
def delete(self, name, priority="project"):
def delete(
self, name: _SettingsKeyT, priority: Union[int, str] = "project"
) -> None:
if name not in self:
raise KeyError(name)
self._assert_mutability()
priority = get_settings_priority(priority)
if priority >= self.getpriority(name):
if priority >= cast(int, self.getpriority(name)):
del self.attributes[name]
def __delitem__(self, name):
def __delitem__(self, name: _SettingsKeyT) -> None:
self._assert_mutability()
del self.attributes[name]
def _assert_mutability(self):
def _assert_mutability(self) -> None:
if self.frozen:
raise TypeError("Trying to modify an immutable Settings object")
def copy(self):
def copy(self) -> "Self":
"""
Make a deep copy of current settings.
@ -384,7 +451,7 @@ class BaseSettings(MutableMapping):
"""
return copy.deepcopy(self)
def freeze(self):
def freeze(self) -> None:
"""
Disable further changes to the current settings.
@ -394,7 +461,7 @@ class BaseSettings(MutableMapping):
"""
self.frozen = True
def frozencopy(self):
def frozencopy(self) -> "Self":
"""
Return an immutable copy of the current settings.
@ -404,26 +471,26 @@ class BaseSettings(MutableMapping):
copy.freeze()
return copy
def __iter__(self):
def __iter__(self) -> Iterator[_SettingsKeyT]:
return iter(self.attributes)
def __len__(self):
def __len__(self) -> int:
return len(self.attributes)
def _to_dict(self):
def _to_dict(self) -> Dict[_SettingsKeyT, Any]:
return {
self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v)
for k, v in self.items()
}
def _get_key(self, key_value):
def _get_key(self, key_value: Any) -> _SettingsKeyT:
return (
key_value
if isinstance(key_value, (bool, float, int, str, type(None)))
else str(key_value)
)
def copy_to_dict(self):
def copy_to_dict(self) -> Dict[_SettingsKeyT, Any]:
"""
Make a copy of current settings and convert to a dict.
@ -439,12 +506,25 @@ class BaseSettings(MutableMapping):
settings = self.copy()
return settings._to_dict()
def _repr_pretty_(self, p, cycle):
# https://ipython.readthedocs.io/en/stable/config/integrating.html#pretty-printing
def _repr_pretty_(self, p: Any, cycle: bool) -> None:
if cycle:
p.text(repr(self))
else:
p.text(pformat(self.copy_to_dict()))
def pop(self, name: _SettingsKeyT, default: Any = __default) -> Any:
try:
value = self.attributes[name].value
except KeyError:
if default is self.__default:
raise
return default
else:
self.__delitem__(name)
return value
class Settings(BaseSettings):
"""
@ -457,7 +537,9 @@ class Settings(BaseSettings):
described on :ref:`topics-settings-ref` already populated.
"""
def __init__(self, values=None, priority="project"):
def __init__(
self, values: _SettingsInputT = None, priority: Union[int, str] = "project"
):
# Do not pass kwarg values here. We don't want to promote user-defined
# dicts, and we want to update, not replace, default dicts with the
# values given by the user
@ -471,15 +553,17 @@ class Settings(BaseSettings):
self.update(values, priority)
def iter_default_settings():
def iter_default_settings() -> Iterable[Tuple[str, Any]]:
"""Return the default settings as an iterator of (name, value) tuples"""
for name in dir(default_settings):
if name.isupper():
yield name, getattr(default_settings, name)
def overridden_settings(settings):
"""Return a dict of the settings that have been overridden"""
def overridden_settings(
settings: Mapping[_SettingsKeyT, Any]
) -> Iterable[Tuple[str, Any]]:
"""Return an iterable of the settings that have been overridden"""
for name, defvalue in iter_default_settings():
value = settings[name]
if not isinstance(defvalue, dict) and value != defvalue:

25
scrapy/settings/default_settings.py
View File

@ -17,6 +17,8 @@ import sys
from importlib import import_module
from pathlib import Path
ADDONS = {}
AJAXCRAWL_ENABLED = False
ASYNCIO_EVENT_LOOP = None
@ -142,7 +144,7 @@ EXTENSIONS_BASE = {
FEED_TEMPDIR = None
FEEDS = {}
FEED_URI_PARAMS = None # a function to extend uri arguments
FEED_STORE_EMPTY = False
FEED_STORE_EMPTY = True
FEED_EXPORT_ENCODING = None
FEED_EXPORT_FIELDS = None
FEED_STORAGES = {}
@ -205,6 +207,8 @@ ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager"
ITEM_PIPELINES = {}
ITEM_PIPELINES_BASE = {}
JOBDIR = None
LOG_ENABLED = True
LOG_ENCODING = "utf-8"
LOG_FORMATTER = "scrapy.logformatter.LogFormatter"
@ -241,6 +245,10 @@ METAREFRESH_MAXDELAY = 100
NEWSPIDER_MODULE = ""
PERIODIC_LOG_DELTA = None
PERIODIC_LOG_STATS = None
PERIODIC_LOG_TIMING_ENABLED = False
RANDOMIZE_DOWNLOAD_DELAY = True
REACTOR_THREADPOOL_MAXSIZE = 10
@ -259,6 +267,21 @@ RETRY_ENABLED = True
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
RETRY_PRIORITY_ADJUST = -1
RETRY_EXCEPTIONS = [
"twisted.internet.defer.TimeoutError",
"twisted.internet.error.TimeoutError",
"twisted.internet.error.DNSLookupError",
"twisted.internet.error.ConnectionRefusedError",
"twisted.internet.error.ConnectionDone",
"twisted.internet.error.ConnectError",
"twisted.internet.error.ConnectionLost",
"twisted.internet.error.TCPTimedOutError",
"twisted.web.client.ResponseFailed",
# OSError is raised by the HttpCompression middleware when trying to
# decompress an empty response
OSError,
"scrapy.core.downloader.handlers.http11.TunnelError",
]
ROBOTSTXT_OBEY = False
ROBOTSTXT_PARSER = "scrapy.robotstxt.ProtegoRobotParser"

34
scrapy/spiderloader.py
View File

@ -1,13 +1,23 @@
from __future__ import annotations
import traceback
import warnings
from collections import defaultdict
from types import ModuleType
from typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Type
from zope.interface import implementer
from scrapy import Request, Spider
from scrapy.interfaces import ISpiderLoader
from scrapy.settings import BaseSettings
from scrapy.utils.misc import walk_modules
from scrapy.utils.spider import iter_spider_classes
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
@implementer(ISpiderLoader)
class SpiderLoader:
@ -16,14 +26,14 @@ class SpiderLoader:
in a Scrapy project.
"""
def __init__(self, settings):
self.spider_modules = settings.getlist("SPIDER_MODULES")
self.warn_only = settings.getbool("SPIDER_LOADER_WARN_ONLY")
self._spiders = {}
self._found = defaultdict(list)
def __init__(self, settings: BaseSettings):
self.spider_modules: List[str] = settings.getlist("SPIDER_MODULES")
self.warn_only: bool = settings.getbool("SPIDER_LOADER_WARN_ONLY")
self._spiders: Dict[str, Type[Spider]] = {}
self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list)
self._load_all_spiders()
def _check_name_duplicates(self):
def _check_name_duplicates(self) -> None:
dupes = []
for name, locations in self._found.items():
dupes.extend(
@ -42,12 +52,12 @@ class SpiderLoader:
category=UserWarning,
)
def _load_spiders(self, module):
def _load_spiders(self, module: ModuleType) -> None:
for spcls in iter_spider_classes(module):
self._found[spcls.name].append((module.__name__, spcls.__name__))
self._spiders[spcls.name] = spcls
def _load_all_spiders(self):
def _load_all_spiders(self) -> None:
for name in self.spider_modules:
try:
for module in walk_modules(name):
@ -65,10 +75,10 @@ class SpiderLoader:
self._check_name_duplicates()
@classmethod
def from_settings(cls, settings):
def from_settings(cls, settings: BaseSettings) -> Self:
return cls(settings)
def load(self, spider_name):
def load(self, spider_name: str) -> Type[Spider]:
"""
Return the Spider class for the given spider name. If the spider
name is not found, raise a KeyError.
@ -78,7 +88,7 @@ class SpiderLoader:
except KeyError:
raise KeyError(f"Spider not found: {spider_name}")
def find_by_request(self, request):
def find_by_request(self, request: Request) -> List[str]:
"""
Return the list of spider names that can handle the given request.
"""
@ -86,7 +96,7 @@ class SpiderLoader:
name for name, cls in self._spiders.items() if cls.handles_request(request)
]
def list(self):
def list(self) -> List[str]:
"""
Return a list with the names of all spiders available in the project.
"""

41
scrapy/spidermiddlewares/depth.py
View File

@ -4,46 +4,67 @@ Depth Spider Middleware
See documentation in docs/topics/spider-middleware.rst
"""
import logging
from __future__ import annotations
from scrapy.http import Request
import logging
from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.http import Request, Response
from scrapy.statscollectors import StatsCollector
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
class DepthMiddleware:
def __init__(self, maxdepth, stats, verbose_stats=False, prio=1):
def __init__(
self,
maxdepth: int,
stats: StatsCollector,
verbose_stats: bool = False,
prio: int = 1,
):
self.maxdepth = maxdepth
self.stats = stats
self.verbose_stats = verbose_stats
self.prio = prio
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
settings = crawler.settings
maxdepth = settings.getint("DEPTH_LIMIT")
verbose = settings.getbool("DEPTH_STATS_VERBOSE")
prio = settings.getint("DEPTH_PRIORITY")
assert crawler.stats
return cls(maxdepth, crawler.stats, verbose, prio)
def process_spider_output(self, response, result, spider):
def process_spider_output(
self, response: Response, result: Iterable[Any], spider: Spider
) -> Iterable[Any]:
self._init_depth(response, spider)
return (r for r in result or () if self._filter(r, response, spider))
return (r for r in result if self._filter(r, response, spider))
async def process_spider_output_async(self, response, result, spider):
async def process_spider_output_async(
self, response: Response, result: AsyncIterable[Any], spider: Spider
) -> AsyncIterable[Any]:
self._init_depth(response, spider)
async for r in result or ():
async for r in result:
if self._filter(r, response, spider):
yield r
def _init_depth(self, response, spider):
def _init_depth(self, response: Response, spider: Spider) -> None:
# base case (depth=0)
if "depth" not in response.meta:
response.meta["depth"] = 0
if self.verbose_stats:
self.stats.inc_value("request_depth_count/0", spider=spider)
def _filter(self, request, response, spider):
def _filter(self, request: Any, response: Response, spider: Spider) -> bool:
if not isinstance(request, Request):
return True
depth = response.meta["depth"] + 1

33
scrapy/spidermiddlewares/httperror.py
View File

@ -3,9 +3,20 @@ HttpError Spider Middleware
See documentation in docs/topics/spider-middleware.rst
"""
import logging
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, Iterable, List, Optional
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest
from scrapy.http import Response
from scrapy.settings import BaseSettings
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
@ -13,21 +24,23 @@ logger = logging.getLogger(__name__)
class HttpError(IgnoreRequest):
"""A non-200 response was filtered"""
def __init__(self, response, *args, **kwargs):
def __init__(self, response: Response, *args: Any, **kwargs: Any):
self.response = response
super().__init__(*args, **kwargs)
class HttpErrorMiddleware:
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings)
def __init__(self, settings):
self.handle_httpstatus_all = settings.getbool("HTTPERROR_ALLOW_ALL")
self.handle_httpstatus_list = settings.getlist("HTTPERROR_ALLOWED_CODES")
def __init__(self, settings: BaseSettings):
self.handle_httpstatus_all: bool = settings.getbool("HTTPERROR_ALLOW_ALL")
self.handle_httpstatus_list: List[int] = settings.getlist(
"HTTPERROR_ALLOWED_CODES"
)
def process_spider_input(self, response, spider):
def process_spider_input(self, response: Response, spider: Spider) -> None:
if 200 <= response.status < 300: # common case
return
meta = response.meta
@ -45,8 +58,11 @@ class HttpErrorMiddleware:
return
raise HttpError(response, "Ignoring non-200 response")
def process_spider_exception(self, response, exception, spider):
def process_spider_exception(
self, response: Response, exception: Exception, spider: Spider
) -> Optional[Iterable[Any]]:
if isinstance(exception, HttpError):
assert spider.crawler.stats
spider.crawler.stats.inc_value("httperror/response_ignored_count")
spider.crawler.stats.inc_value(
f"httperror/response_ignored_status_count/{response.status}"
@ -57,3 +73,4 @@ class HttpErrorMiddleware:
extra={"spider": spider},
)
return []
return None

45
scrapy/spidermiddlewares/offsite.py
View File

@ -3,36 +3,51 @@ Offsite Spider Middleware
See documentation in docs/topics/spider-middleware.rst
"""
from __future__ import annotations
import logging
import re
import warnings
from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable, Set
from scrapy import signals
from scrapy.http import Request
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.http import Request, Response
from scrapy.statscollectors import StatsCollector
from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
logger = logging.getLogger(__name__)
class OffsiteMiddleware:
def __init__(self, stats):
self.stats = stats
def __init__(self, stats: StatsCollector):
self.stats: StatsCollector = stats
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
assert crawler.stats
o = cls(crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o
def process_spider_output(self, response, result, spider):
return (r for r in result or () if self._filter(r, spider))
def process_spider_output(
self, response: Response, result: Iterable[Any], spider: Spider
) -> Iterable[Any]:
return (r for r in result if self._filter(r, spider))
async def process_spider_output_async(self, response, result, spider):
async for r in result or ():
async def process_spider_output_async(
self, response: Response, result: AsyncIterable[Any], spider: Spider
) -> AsyncIterable[Any]:
async for r in result:
if self._filter(r, spider):
yield r
def _filter(self, request, spider) -> bool:
def _filter(self, request: Any, spider: Spider) -> bool:
if not isinstance(request, Request):
return True
if request.dont_filter or self.should_follow(request, spider):
@ -49,13 +64,13 @@ class OffsiteMiddleware:
self.stats.inc_value("offsite/filtered", spider=spider)
return False
def should_follow(self, request, spider):
def should_follow(self, request: Request, spider: Spider) -> bool:
regex = self.host_regex
# hostname can be None for wrong urls (like javascript links)
host = urlparse_cached(request).hostname or ""
return bool(regex.search(host))
def get_host_regex(self, spider):
def get_host_regex(self, spider: Spider) -> re.Pattern[str]:
"""Override this method to implement a different offsite policy"""
allowed_domains = getattr(spider, "allowed_domains", None)
if not allowed_domains:
@ -83,9 +98,9 @@ class OffsiteMiddleware:
regex = rf'^(.*\.)?({"|".join(domains)})$'
return re.compile(regex)
def spider_opened(self, spider):
self.host_regex = self.get_host_regex(spider)
self.domains_seen = set()
def spider_opened(self, spider: Spider) -> None:
self.host_regex: re.Pattern[str] = self.get_host_regex(spider)
self.domains_seen: Set[str] = set()
class URLWarning(Warning):

101
scrapy/spidermiddlewares/referer.py
View File

@ -2,20 +2,39 @@
RefererMiddleware: populates Request referer field, based on the Response which
originated it.
"""
from __future__ import annotations
import warnings
from typing import Tuple
from typing import (
TYPE_CHECKING,
Any,
AsyncIterable,
Dict,
Iterable,
Optional,
Tuple,
Type,
Union,
cast,
)
from urllib.parse import urlparse
from w3lib.url import safe_url_string
from scrapy import signals
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.settings import BaseSettings
from scrapy.utils.misc import load_object
from scrapy.utils.python import to_unicode
from scrapy.utils.url import strip_url
LOCAL_SCHEMES = (
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
LOCAL_SCHEMES: Tuple[str, ...] = (
"about",
"blob",
"data",
@ -37,18 +56,20 @@ class ReferrerPolicy:
NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES
name: str
def referrer(self, response_url, request_url):
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
raise NotImplementedError()
def stripped_referrer(self, url):
def stripped_referrer(self, url: str) -> Optional[str]:
if urlparse(url).scheme not in self.NOREFERRER_SCHEMES:
return self.strip_url(url)
return None
def origin_referrer(self, url):
def origin_referrer(self, url: str) -> Optional[str]:
if urlparse(url).scheme not in self.NOREFERRER_SCHEMES:
return self.origin(url)
return None
def strip_url(self, url, origin_only=False):
def strip_url(self, url: str, origin_only: bool = False) -> Optional[str]:
"""
https://www.w3.org/TR/referrer-policy/#strip-url
@ -72,18 +93,18 @@ class ReferrerPolicy:
origin_only=origin_only,
)
def origin(self, url):
def origin(self, url: str) -> Optional[str]:
"""Return serialized origin (scheme, host, path) for a request or response URL."""
return self.strip_url(url, origin_only=True)
def potentially_trustworthy(self, url):
def potentially_trustworthy(self, url: str) -> bool:
# Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy
parsed_url = urlparse(url)
if parsed_url.scheme in ("data",):
return False
return self.tls_protected(url)
def tls_protected(self, url):
def tls_protected(self, url: str) -> bool:
return urlparse(url).scheme in ("https", "ftps")
@ -98,7 +119,7 @@ class NoReferrerPolicy(ReferrerPolicy):
name: str = POLICY_NO_REFERRER
def referrer(self, response_url, request_url):
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
return None
@ -119,9 +140,10 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE
def referrer(self, response_url, request_url):
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
if not self.tls_protected(response_url) or self.tls_protected(request_url):
return self.stripped_referrer(response_url)
return None
class SameOriginPolicy(ReferrerPolicy):
@ -137,9 +159,10 @@ class SameOriginPolicy(ReferrerPolicy):
name: str = POLICY_SAME_ORIGIN
def referrer(self, response_url, request_url):
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
if self.origin(response_url) == self.origin(request_url):
return self.stripped_referrer(response_url)
return None
class OriginPolicy(ReferrerPolicy):
@ -154,7 +177,7 @@ class OriginPolicy(ReferrerPolicy):
name: str = POLICY_ORIGIN
def referrer(self, response_url, request_url):
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
return self.origin_referrer(response_url)
@ -174,13 +197,14 @@ class StrictOriginPolicy(ReferrerPolicy):
name: str = POLICY_STRICT_ORIGIN
def referrer(self, response_url, request_url):
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
if (
self.tls_protected(response_url)
and self.potentially_trustworthy(request_url)
or not self.tls_protected(response_url)
):
return self.origin_referrer(response_url)
return None
class OriginWhenCrossOriginPolicy(ReferrerPolicy):
@ -197,7 +221,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):
name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
def referrer(self, response_url, request_url):
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
origin = self.origin(response_url)
if origin == self.origin(request_url):
return self.stripped_referrer(response_url)
@ -224,7 +248,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN
def referrer(self, response_url, request_url):
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
origin = self.origin(response_url)
if origin == self.origin(request_url):
return self.stripped_referrer(response_url)
@ -234,6 +258,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
or not self.tls_protected(response_url)
):
return self.origin_referrer(response_url)
return None
class UnsafeUrlPolicy(ReferrerPolicy):
@ -252,7 +277,7 @@ class UnsafeUrlPolicy(ReferrerPolicy):
name: str = POLICY_UNSAFE_URL
def referrer(self, response_url, request_url):
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
return self.stripped_referrer(response_url)
@ -267,7 +292,7 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
name: str = POLICY_SCRAPY_DEFAULT
_policy_classes = {
_policy_classes: Dict[str, Type[ReferrerPolicy]] = {
p.name: p
for p in (
NoReferrerPolicy,
@ -286,14 +311,16 @@ _policy_classes = {
_policy_classes[""] = NoReferrerWhenDowngradePolicy
def _load_policy_class(policy, warning_only=False):
def _load_policy_class(
policy: str, warning_only: bool = False
) -> Optional[Type[ReferrerPolicy]]:
"""
Expect a string for the path to the policy class,
otherwise try to interpret the string as a standard value
from https://www.w3.org/TR/referrer-policy/#referrer-policies
"""
try:
return load_object(policy)
return cast(Type[ReferrerPolicy], load_object(policy))
except ValueError:
try:
return _policy_classes[policy.lower()]
@ -307,13 +334,15 @@ def _load_policy_class(policy, warning_only=False):
class RefererMiddleware:
def __init__(self, settings=None):
self.default_policy = DefaultReferrerPolicy
def __init__(self, settings: Optional[BaseSettings] = None):
self.default_policy: Type[ReferrerPolicy] = DefaultReferrerPolicy
if settings is not None:
self.default_policy = _load_policy_class(settings.get("REFERRER_POLICY"))
settings_policy = _load_policy_class(settings.get("REFERRER_POLICY"))
assert settings_policy
self.default_policy = settings_policy
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("REFERER_ENABLED"):
raise NotConfigured
mw = cls(crawler.settings)
@ -323,7 +352,9 @@ class RefererMiddleware:
return mw
def policy(self, resp_or_url, request):
def policy(
self, resp_or_url: Union[Response, str], request: Request
) -> ReferrerPolicy:
"""
Determine Referrer-Policy to use from a parent Response (or URL),
and a Request to be sent.
@ -348,21 +379,25 @@ class RefererMiddleware:
cls = _load_policy_class(policy_name, warning_only=True)
return cls() if cls else self.default_policy()
def process_spider_output(self, response, result, spider):
return (self._set_referer(r, response) for r in result or ())
def process_spider_output(
self, response: Response, result: Iterable[Any], spider: Spider
) -> Iterable[Any]:
return (self._set_referer(r, response) for r in result)
async def process_spider_output_async(self, response, result, spider):
async for r in result or ():
async def process_spider_output_async(
self, response: Response, result: AsyncIterable[Any], spider: Spider
) -> AsyncIterable[Any]:
async for r in result:
yield self._set_referer(r, response)
def _set_referer(self, r, response):
def _set_referer(self, r: Any, response: Response) -> Any:
if isinstance(r, Request):
referrer = self.policy(response, r).referrer(response.url, r.url)
if referrer is not None:
r.headers.setdefault("Referer", referrer)
return r
def request_scheduled(self, request, spider):
def request_scheduled(self, request: Request, spider: Spider) -> None:
# check redirected request to patch "Referer" header if necessary
redirected_urls = request.meta.get("redirect_urls", [])
if redirected_urls:
@ -378,7 +413,7 @@ class RefererMiddleware:
policy_referrer = self.policy(parent_url, request).referrer(
parent_url, request.url
)
if policy_referrer != request_referrer:
if policy_referrer != request_referrer.decode("latin1"):
if policy_referrer is None:
request.headers.pop("Referer")
else:

Some files were not shown because too many files have changed in this diff Show More