mirror of https://github.com/scrapy/scrapy.git
Compare commits
No commits in common. "master" and "2.16.0" have entirely different histories.
|
|
@ -22,10 +22,10 @@ jobs:
|
|||
TOXENV: pylint
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: mypy
|
||||
TOXENV: typing
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: mypy-tests
|
||||
TOXENV: typing-tests
|
||||
# Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version.
|
||||
- python-version: "3.14"
|
||||
env:
|
||||
|
|
|
|||
|
|
@ -45,26 +45,26 @@ jobs:
|
|||
env:
|
||||
TOXENV: pypy3
|
||||
|
||||
# min deps
|
||||
# pinned deps
|
||||
- python-version: "3.10.19"
|
||||
env:
|
||||
TOXENV: min
|
||||
TOXENV: pinned
|
||||
- python-version: "3.10.19"
|
||||
env:
|
||||
TOXENV: min-default-reactor
|
||||
TOXENV: default-reactor-pinned
|
||||
- python-version: "3.10.19"
|
||||
env:
|
||||
TOXENV: min-no-reactor
|
||||
TOXENV: no-reactor-pinned
|
||||
# pinned due to https://github.com/pypy/pypy/issues/5388
|
||||
- python-version: pypy3.11-7.3.20
|
||||
env:
|
||||
TOXENV: min-pypy3
|
||||
TOXENV: pypy3-pinned
|
||||
- python-version: "3.10.19"
|
||||
env:
|
||||
TOXENV: min-extra-deps
|
||||
TOXENV: extra-deps-pinned
|
||||
- python-version: "3.10.19"
|
||||
env:
|
||||
TOXENV: min-botocore
|
||||
TOXENV: botocore-pinned
|
||||
|
||||
- python-version: "3.14"
|
||||
env:
|
||||
|
|
@ -79,6 +79,9 @@ jobs:
|
|||
- python-version: "3.14"
|
||||
env:
|
||||
TOXENV: botocore
|
||||
- python-version: "3.14"
|
||||
env:
|
||||
TOXENV: mitmproxy
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
|
@ -89,14 +92,11 @@ jobs:
|
|||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install system libraries
|
||||
if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'min')
|
||||
if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned')
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install libxml2-dev libxslt-dev
|
||||
|
||||
- name: Install mitmproxy
|
||||
run: pipx install mitmproxy
|
||||
|
||||
- name: Run tests
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
|
|
|
|||
|
|
@ -41,13 +41,13 @@ jobs:
|
|||
env:
|
||||
TOXENV: no-reactor
|
||||
|
||||
# min deps
|
||||
# pinned deps
|
||||
- python-version: "3.10.11"
|
||||
env:
|
||||
TOXENV: min
|
||||
TOXENV: pinned
|
||||
- python-version: "3.10.11"
|
||||
env:
|
||||
TOXENV: min-extra-deps
|
||||
TOXENV: extra-deps-pinned
|
||||
|
||||
- python-version: "3.14"
|
||||
env:
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ exclude: |
|
|||
)
|
||||
repos:
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.15.20
|
||||
rev: v0.15.2
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
args: [ --fix ]
|
||||
|
|
@ -16,7 +16,7 @@ repos:
|
|||
hooks:
|
||||
- id: blacken-docs
|
||||
additional_dependencies:
|
||||
- black==26.5.1
|
||||
- black==25.9.0
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v6.0.0
|
||||
hooks:
|
||||
|
|
@ -27,6 +27,6 @@ repos:
|
|||
hooks:
|
||||
- id: sphinx-lint
|
||||
- repo: https://github.com/scrapy/sphinx-scrapy
|
||||
rev: 0.8.8
|
||||
rev: 0.8.6
|
||||
hooks:
|
||||
- id: sphinx-scrapy
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@
|
|||
|
||||
| Version | Supported |
|
||||
| ------- | ------------------ |
|
||||
| 2.17.x | :white_check_mark: |
|
||||
| < 2.17.x | :x: |
|
||||
| 2.16.x | :white_check_mark: |
|
||||
| < 2.16.x | :x: |
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
|
|
|
|||
38
conftest.py
38
conftest.py
|
|
@ -11,7 +11,7 @@ from scrapy.utils.reactor import set_asyncio_event_loop_policy
|
|||
from scrapy.utils.reactorless import install_reactor_import_hook
|
||||
from tests.keys import generate_keys
|
||||
from tests.mockserver.http import MockServer
|
||||
from tests.mockserver.mitm_proxy import MitmProxy, mitmdump_cmd
|
||||
from tests.mockserver.mitm_proxy import MitmProxy
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
|
|
@ -24,13 +24,13 @@ def _py_files(folder):
|
|||
collect_ignore = [
|
||||
# may need extra deps
|
||||
"docs/_ext",
|
||||
# contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerProcessSubprocess
|
||||
# contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess
|
||||
*_py_files("tests/AsyncCrawlerProcess"),
|
||||
# contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerRunnerSubprocess
|
||||
# contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess
|
||||
*_py_files("tests/AsyncCrawlerRunner"),
|
||||
# contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerProcessSubprocess
|
||||
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
|
||||
*_py_files("tests/CrawlerProcess"),
|
||||
# contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerRunnerSubprocess
|
||||
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
|
||||
*_py_files("tests/CrawlerRunner"),
|
||||
]
|
||||
|
||||
|
|
@ -74,19 +74,27 @@ def mockserver() -> Generator[MockServer]:
|
|||
|
||||
|
||||
@pytest.fixture # function scope because it modifies os.environ
|
||||
def proxy_server(
|
||||
request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch
|
||||
) -> Generator[str]:
|
||||
kind = request.param
|
||||
proxy = MitmProxy(mode="socks5" if kind == "socks5" else None)
|
||||
def mitm_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
|
||||
proxy = MitmProxy()
|
||||
url = proxy.start()
|
||||
if kind == "https":
|
||||
url = url.replace("http://", "https://")
|
||||
monkeypatch.setenv("http_proxy", url)
|
||||
monkeypatch.setenv("https_proxy", url)
|
||||
|
||||
try:
|
||||
yield kind
|
||||
yield proxy
|
||||
finally:
|
||||
proxy.stop()
|
||||
|
||||
|
||||
@pytest.fixture # function scope because it modifies os.environ
|
||||
def mitm_proxy_server_https(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
|
||||
proxy = MitmProxy()
|
||||
url = proxy.start().replace("http://", "https://")
|
||||
monkeypatch.setenv("http_proxy", url)
|
||||
monkeypatch.setenv("https_proxy", url)
|
||||
|
||||
try:
|
||||
yield proxy
|
||||
finally:
|
||||
proxy.stop()
|
||||
|
||||
|
|
@ -127,6 +135,7 @@ def pytest_runtest_setup(item):
|
|||
"uvloop",
|
||||
"botocore",
|
||||
"boto3",
|
||||
"mitmproxy",
|
||||
]
|
||||
|
||||
for module in optional_deps:
|
||||
|
|
@ -136,9 +145,6 @@ def pytest_runtest_setup(item):
|
|||
except ImportError:
|
||||
pytest.skip(f"{module} is not installed")
|
||||
|
||||
if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None:
|
||||
pytest.skip("mitmdump is not available")
|
||||
|
||||
|
||||
# Generate localhost certificate files, needed by some tests
|
||||
generate_keys()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
:orphan:
|
||||
|
||||
======================================
|
||||
Scrapy documentation quick start guide
|
||||
======================================
|
||||
|
||||
This file provides a quick guide on how to compile the Scrapy documentation.
|
||||
|
||||
|
||||
Setup the environment
|
||||
---------------------
|
||||
|
||||
To compile the documentation you need Sphinx Python library. To install it
|
||||
and all its dependencies run the following command from this dir
|
||||
|
||||
::
|
||||
|
||||
pip install -r requirements.txt
|
||||
|
||||
|
||||
Compile the documentation
|
||||
-------------------------
|
||||
|
||||
To compile the documentation (to classic HTML output) run the following command
|
||||
from this dir::
|
||||
|
||||
make html
|
||||
|
||||
Documentation will be generated (in HTML format) inside the ``build/html`` dir.
|
||||
|
||||
|
||||
View the documentation
|
||||
----------------------
|
||||
|
||||
To view the documentation run the following command::
|
||||
|
||||
make htmlview
|
||||
|
||||
This command will fire up your default browser and open the main page of your
|
||||
(previously generated) HTML documentation.
|
||||
|
||||
|
||||
Start over
|
||||
----------
|
||||
|
||||
To clean up all generated documentation files and start from scratch run::
|
||||
|
||||
make clean
|
||||
|
||||
Keep in mind that this command won't touch any documentation source files.
|
||||
|
||||
|
||||
Recreating documentation on the fly
|
||||
-----------------------------------
|
||||
|
||||
There is a way to recreate the doc automatically when you make changes, you
|
||||
need to install watchdog (``pip install watchdog``) and then use::
|
||||
|
||||
make watch
|
||||
|
||||
Alternative method using tox
|
||||
----------------------------
|
||||
|
||||
To compile the documentation to HTML run the following command::
|
||||
|
||||
tox -e docs
|
||||
|
||||
Documentation will be generated inside the ``docs/_build/all`` dir.
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
{% extends "!layout.html" %}
|
||||
|
||||
{# Overridden to include a link to scrapy.org, not just to the docs root #}
|
||||
{# Overriden to include a link to scrapy.org, not just to the docs root #}
|
||||
{%- block sidebartitle %}
|
||||
|
||||
{# the logo helper function was removed in Sphinx 6 and deprecated since Sphinx 4 #}
|
||||
|
|
|
|||
|
|
@ -158,7 +158,6 @@ scrapy_intersphinx_enable = [
|
|||
"itemloaders",
|
||||
"parsel",
|
||||
"pytest",
|
||||
"scrapy-lint",
|
||||
"sphinx",
|
||||
"tox",
|
||||
"twisted",
|
||||
|
|
|
|||
|
|
@ -323,10 +323,9 @@ deprecation removals are documented in the :ref:`release notes <news>`.
|
|||
Tests
|
||||
=====
|
||||
|
||||
Tests are implemented using pytest_. Running tests requires :doc:`tox
|
||||
<tox:index>`.
|
||||
|
||||
.. _pytest: https://pytest.org
|
||||
Tests are implemented using the :doc:`Twisted unit-testing framework
|
||||
<twisted:development/test-standard>`. Running tests requires
|
||||
:doc:`tox <tox:index>`.
|
||||
|
||||
.. _running-tests:
|
||||
|
||||
|
|
@ -372,21 +371,6 @@ To see coverage report install :doc:`coverage <coverage:index>`
|
|||
|
||||
see output of ``coverage --help`` for more options like html or xml report.
|
||||
|
||||
Some tests need a ``mitmdump`` executable (from mitmproxy_) to test against a
|
||||
fully featured proxy server; they are skipped when one cannot be found
|
||||
(``mitmproxy`` is intentionally not a test dependency that would be installed
|
||||
into test venvs, as that sometimes leads to various dependency conflicts).
|
||||
To run these tests, make ``mitmdump`` available in one of these ways:
|
||||
|
||||
* install ``mitmproxy`` so that ``mitmdump`` is on your ``PATH``, e.g. with
|
||||
pipx_ (``pipx install mitmproxy``) or uv_ (``uv tool install mitmproxy``);
|
||||
|
||||
* have uv_ installed, in which case the tests will run
|
||||
``uvx --from mitmproxy mitmdump``;
|
||||
|
||||
* set the ``MITMDUMP`` environment variable to the path of a ``mitmdump``
|
||||
executable.
|
||||
|
||||
Writing tests
|
||||
-------------
|
||||
|
||||
|
|
@ -414,6 +398,3 @@ And their unit-tests are in::
|
|||
.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
|
||||
.. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22
|
||||
.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy
|
||||
.. _mitmproxy: https://mitmproxy.org/
|
||||
.. _pipx: https://pipx.pypa.io/
|
||||
.. _uv: https://docs.astral.sh/uv/
|
||||
|
|
|
|||
31
docs/faq.rst
31
docs/faq.rst
|
|
@ -82,18 +82,10 @@ to steal from us!
|
|||
Does Scrapy work with HTTP proxies?
|
||||
-----------------------------------
|
||||
|
||||
Yes. Support for HTTP proxies is provided through the HTTP Proxy downloader
|
||||
middleware. See
|
||||
Yes. Support for HTTP proxies is provided (since Scrapy 0.8) through the HTTP
|
||||
Proxy downloader middleware. See
|
||||
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`.
|
||||
|
||||
Does Scrapy work with SOCKS proxies?
|
||||
------------------------------------
|
||||
|
||||
Yes, when using
|
||||
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. See
|
||||
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and the
|
||||
handler documentation.
|
||||
|
||||
How can I scrape an item with attributes in different pages?
|
||||
------------------------------------------------------------
|
||||
|
||||
|
|
@ -285,8 +277,7 @@ consume a lot of memory.
|
|||
In order to avoid parsing all the entire feed at once in memory, you can use
|
||||
the :func:`~scrapy.utils.iterators.xmliter_lxml` and
|
||||
:func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what
|
||||
:class:`~scrapy.spiders.XMLFeedSpider` and
|
||||
:class:`~scrapy.spiders.CSVFeedSpider` use.
|
||||
:class:`~scrapy.spiders.XMLFeedSpider` uses.
|
||||
|
||||
.. autofunction:: scrapy.utils.iterators.xmliter_lxml
|
||||
|
||||
|
|
@ -361,19 +352,15 @@ method for this purpose. For example:
|
|||
def process_spider_output(self, response, result):
|
||||
for item_or_request in result:
|
||||
if isinstance(item_or_request, Request):
|
||||
yield item_or_request
|
||||
continue
|
||||
adapter = ItemAdapter(item_or_request)
|
||||
adapter = ItemAdapter(item)
|
||||
for _ in range(adapter["multiply_by"]):
|
||||
yield deepcopy(item_or_request)
|
||||
yield deepcopy(item)
|
||||
|
||||
Does Scrapy support IPv6 addresses?
|
||||
-----------------------------------
|
||||
|
||||
Yes, but when using
|
||||
:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` or
|
||||
:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` you need to
|
||||
set :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
|
||||
Yes, by setting :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
|
||||
Note that by doing so, you lose the ability to set a specific timeout for DNS requests
|
||||
(the value of the :setting:`DNS_TIMEOUT` setting is ignored).
|
||||
|
||||
|
|
@ -384,9 +371,8 @@ How to deal with ``<class 'ValueError'>: filedescriptor out of range in select()
|
|||
----------------------------------------------------------------------------------------------
|
||||
|
||||
This issue `has been reported`_ to appear when running broad crawls in macOS, where the default
|
||||
Twisted reactor was :class:`twisted.internet.selectreactor.SelectReactor` at that time.
|
||||
If you have switched to this reactor using the :setting:`TWISTED_REACTOR` setting you can switch
|
||||
to a different one in the same way.
|
||||
Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a
|
||||
different reactor is possible by using the :setting:`TWISTED_REACTOR` setting.
|
||||
|
||||
|
||||
.. _faq-stop-response-download:
|
||||
|
|
@ -412,6 +398,7 @@ How can I make a blank request?
|
|||
|
||||
from scrapy import Request
|
||||
|
||||
|
||||
blank_request = Request("data:,")
|
||||
|
||||
In this case, the URL is set to a data URI scheme. Data URLs allow you to include data
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ Having trouble? We'd like to help!
|
|||
* Ask or search questions in `StackOverflow using the scrapy tag`_.
|
||||
* Ask or search questions in the `Scrapy subreddit`_.
|
||||
* Search for questions on the archives of the `scrapy-users mailing list`_.
|
||||
* Ask a question in the `#scrapy IRC channel`_.
|
||||
* Ask a question in the `#scrapy IRC channel`_,
|
||||
* Report bugs with Scrapy in our `issue tracker`_.
|
||||
* Join the Discord community `Scrapy Discord`_.
|
||||
|
||||
|
|
@ -91,15 +91,15 @@ Basic concepts
|
|||
:doc:`topics/selectors`
|
||||
Extract the data from web pages using XPath.
|
||||
|
||||
:doc:`topics/shell`
|
||||
Test your extraction code in an interactive environment.
|
||||
|
||||
:doc:`topics/items`
|
||||
Define the data you want to scrape.
|
||||
|
||||
:doc:`topics/loaders`
|
||||
Populate your items with the extracted data.
|
||||
|
||||
:doc:`topics/shell`
|
||||
Test your extraction code in an interactive environment.
|
||||
|
||||
:doc:`topics/item-pipeline`
|
||||
Post-process and store your scraped data.
|
||||
|
||||
|
|
@ -151,7 +151,6 @@ Solving specific problems
|
|||
topics/debug
|
||||
topics/contracts
|
||||
topics/practices
|
||||
topics/security
|
||||
topics/broad-crawls
|
||||
topics/developer-tools
|
||||
topics/dynamic-content
|
||||
|
|
@ -176,10 +175,6 @@ Solving specific problems
|
|||
:doc:`topics/practices`
|
||||
Get familiar with some Scrapy common practices.
|
||||
|
||||
:doc:`topics/security`
|
||||
Understand the security implications of Scrapy defaults and how to harden
|
||||
them.
|
||||
|
||||
:doc:`topics/broad-crawls`
|
||||
Tune Scrapy for crawling a lot domains in parallel.
|
||||
|
||||
|
|
|
|||
|
|
@ -230,8 +230,8 @@ Installing Scrapy with PyPy on Windows is not tested.
|
|||
You can check that Scrapy is installed correctly by running ``scrapy bench``.
|
||||
If this command gives errors such as
|
||||
``TypeError: ... got 2 unexpected keyword arguments``, this means
|
||||
that the ``PyPyDispatcher`` dependency wasn't installed. To fix this issue, run
|
||||
``pip install 'PyPyDispatcher>=2.1.0'``.
|
||||
that setuptools was unable to pick up one PyPy-specific dependency.
|
||||
To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``.
|
||||
|
||||
|
||||
.. _intro-install-troubleshooting:
|
||||
|
|
|
|||
|
|
@ -83,17 +83,16 @@ While this enables you to do very fast crawls (sending multiple concurrent
|
|||
requests at the same time, in a fault-tolerant way) Scrapy also gives you
|
||||
control over the politeness of the crawl through :ref:`a few settings
|
||||
<topics-settings-ref>`. You can do things like setting a download delay between
|
||||
each request, limiting the amount of concurrent requests per domain, and
|
||||
each request, limiting the amount of concurrent requests per domain or per IP, and
|
||||
even :ref:`using an auto-throttling extension <topics-autothrottle>` that tries
|
||||
to figure these settings out automatically.
|
||||
|
||||
.. note::
|
||||
|
||||
This is using :ref:`feed exports <topics-feed-exports>` to generate the
|
||||
JSON Lines file, you can easily change the export format (XML or CSV, for
|
||||
example) or the storage backend (FTP or `Amazon S3`_, for example). You can
|
||||
also write an :ref:`item pipeline <topics-item-pipeline>` to store the
|
||||
items in a database.
|
||||
JSON file, you can easily change the export format (XML or CSV, for example) or the
|
||||
storage backend (FTP or `Amazon S3`_, for example). You can also write an
|
||||
:ref:`item pipeline <topics-item-pipeline>` to store the items in a database.
|
||||
|
||||
|
||||
.. _topics-whatelse:
|
||||
|
|
|
|||
368
docs/news.rst
368
docs/news.rst
|
|
@ -3,352 +3,6 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
Scrapy VERSION (unreleased)
|
||||
---------------------------
|
||||
|
||||
Backward-incompatible changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- The following runtime usage of zope.interface_ interfaces is removed:
|
||||
|
||||
- :class:`~scrapy.spiderloader.SpiderLoader` and
|
||||
:class:`~scrapy.spiderloader.DummySpiderLoader` are no longer marked
|
||||
as implementing the ``ISpiderLoader`` interface.
|
||||
|
||||
- :func:`~scrapy.spiderloader.get_spider_loader` no longer checks that the
|
||||
configured spider loader implements the ``ISpiderLoader`` interface.
|
||||
|
||||
- :class:`~scrapy.extensions.feedexport.BlockingFeedStorage`,
|
||||
:class:`~scrapy.extensions.feedexport.FileFeedStorage` and
|
||||
:class:`~scrapy.extensions.feedexport.StdoutFeedStorage` are no longer
|
||||
marked as implementing the ``IFeedStorage`` interface.
|
||||
|
||||
(:issue:`6585`, :issue:`7731`)
|
||||
|
||||
.. _release-2.17.0:
|
||||
|
||||
Scrapy 2.17.0 (2026-07-07)
|
||||
--------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
- Security bug fixes
|
||||
|
||||
- HTTP/2 and SOCKS proxy support for ``HttpxDownloadHandler``
|
||||
|
||||
- Improved settings for changing allowed TLS versions
|
||||
|
||||
Security bug fixes
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- ``s3://`` requests now use HTTPS by default, instead of plaintext HTTP.
|
||||
|
||||
Previously, :class:`~scrapy.core.downloader.handlers.s3.S3DownloadHandler`
|
||||
sent signed S3 requests over plaintext HTTP unless
|
||||
``request.meta["is_secure"]`` was set to a true value, exposing the request
|
||||
path, the AWS ``Authorization`` header, the ``X-Amz-Security-Token`` header
|
||||
(when using temporary credentials), and the response contents to network
|
||||
attackers, who could also tamper with responses. See the `76g3-c3x4-crvx`_
|
||||
security advisory for details.
|
||||
|
||||
To restore the previous behavior for a given request, set
|
||||
``request.meta["is_secure"]`` to ``False``.
|
||||
|
||||
.. _76g3-c3x4-crvx: https://github.com/scrapy/scrapy/security/advisories/GHSA-76g3-c3x4-crvx
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- The ``DOWNLOADER_CLIENT_TLS_METHOD`` setting is deprecated. You should use
|
||||
the :setting:`DOWNLOAD_TLS_MIN_VERSION` and/or
|
||||
:setting:`DOWNLOAD_TLS_MAX_VERSION` settings instead if you want to change
|
||||
the TLS method selection.
|
||||
(:issue:`3288`, :issue:`6546`)
|
||||
|
||||
- The following spider attributes are deprecated in favor of settings:
|
||||
|
||||
- ``http_user`` (use :setting:`HTTPAUTH_USER`)
|
||||
|
||||
- ``http_pass`` (use :setting:`HTTPAUTH_PASS`)
|
||||
|
||||
- ``http_auth_domain`` (use :setting:`HTTPAUTH_DOMAIN`)
|
||||
|
||||
(:issue:`7590`)
|
||||
|
||||
- The ``scrapy.commands.ScrapyCommand.help()`` method is deprecated. It was
|
||||
never called by Scrapy.
|
||||
(:issue:`7626`, :issue:`7633`)
|
||||
|
||||
- The following TLS-related functions and constants, intended for internal
|
||||
use, are deprecated:
|
||||
|
||||
- ``scrapy.core.downloader.tls.METHOD_TLS``
|
||||
|
||||
- ``scrapy.core.downloader.tls.METHOD_TLSv10``
|
||||
|
||||
- ``scrapy.core.downloader.tls.METHOD_TLSv11``
|
||||
|
||||
- ``scrapy.core.downloader.tls.METHOD_TLSv12``
|
||||
|
||||
- ``scrapy.core.downloader.tls.openssl_methods``
|
||||
|
||||
- ``scrapy.core.downloader.tls.DEFAULT_CIPHERS``
|
||||
|
||||
- ``scrapy.utils.ssl.ffi_buf_to_string()``
|
||||
|
||||
- ``scrapy.utils.ssl.get_temp_key_info()``
|
||||
|
||||
- ``scrapy.utils.ssl.x509name_to_string()``
|
||||
|
||||
(:issue:`6546`, :issue:`7619`, :issue:`7665`)
|
||||
|
||||
- The ``CRAWLSPIDER_FOLLOW_LINKS`` setting is deprecated. You can set
|
||||
``follow=False`` in your rules to achieve the same effect.
|
||||
(:issue:`7592`)
|
||||
|
||||
- Instantiating
|
||||
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
|
||||
without a ``crawler`` argument is deprecated.
|
||||
(:issue:`7655`)
|
||||
|
||||
- Instantiating
|
||||
:class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` without a
|
||||
``settings`` argument is deprecated.
|
||||
(:issue:`7664`)
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- Added support for HTTP/2 requests to
|
||||
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. It
|
||||
requires setting the new :setting:`HTTPX_HTTP2_ENABLED` setting to
|
||||
``True``.
|
||||
(:issue:`7575`)
|
||||
|
||||
- Added support for SOCKS proxies to
|
||||
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.
|
||||
(:issue:`747`, :issue:`7575`)
|
||||
|
||||
- Added :setting:`DOWNLOAD_TLS_MIN_VERSION` and
|
||||
:setting:`DOWNLOAD_TLS_MAX_VERSION` settings as replacements for the
|
||||
``DOWNLOADER_CLIENT_TLS_METHOD`` setting (which is now deprecated).
|
||||
Compared to the old setting, they support specifying a range of allowed
|
||||
versions and support newer TLS versions.
|
||||
(:issue:`4821`, :issue:`6546`)
|
||||
|
||||
- Added :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS` and
|
||||
:setting:`HTTPAUTH_DOMAIN` settings and :reqmeta:`http_user`,
|
||||
:reqmeta:`http_pass` and :reqmeta:`http_auth_domain` meta keys as more
|
||||
flexible ways to set HTTP authentication data.
|
||||
(:issue:`7590`)
|
||||
|
||||
- Added a :reqmeta:`verbatim_url` meta key that can be set to ``True`` to
|
||||
skip request URL canonicalization.
|
||||
(:issue:`7473`)
|
||||
|
||||
- Added ``deny_tags`` and ``deny_attrs`` arguments to :class:`LinkExtractor
|
||||
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`.
|
||||
(:issue:`6321`, :issue:`7679`)
|
||||
|
||||
- :attr:`scrapy.Item.fields` now returns the fields in the definition order
|
||||
instead of the alphabetical one.
|
||||
(:issue:`7015`, :issue:`7694`)
|
||||
|
||||
- Added a :setting:`RETRY_GIVE_UP_LOG_LEVEL` setting, a
|
||||
:reqmeta:`give_up_log_level` meta key and a ``give_up_log_level`` argument
|
||||
of the
|
||||
:func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function that
|
||||
allow changing the log level of the message logged when the retry limit has
|
||||
been reached.
|
||||
(:issue:`4622`, :issue:`5297`, :issue:`7567`)
|
||||
|
||||
- It's now possible to set :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` to
|
||||
``None`` to use the default ciphers of the underlying TLS implementation.
|
||||
(:issue:`7499`, :issue:`7665`)
|
||||
|
||||
Improvements
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- :class:`~scrapy.FormRequest` is no longer deprecated, only its
|
||||
``from_response()`` method is still deprecated.
|
||||
(:issue:`7561`, :issue:`7671`)
|
||||
|
||||
- Switched the item definition in the default project template from a
|
||||
:class:`scrapy.item.Item` to a dataclass.
|
||||
(:issue:`7493`, :issue:`7513`)
|
||||
|
||||
- Fixed deprecation warnings with pyOpenSSL 26.3.0.
|
||||
(:issue:`7619`)
|
||||
|
||||
- Removed the runtime warnings for :attr:`Spider.allowed_domains
|
||||
<scrapy.Spider.allowed_domains>` containing URLs or domains with ports
|
||||
instead of just domains and for spider classes having a ``start_url``
|
||||
attribute instead of :class:`~scrapy.spiders.Spider.start_urls`. Please use
|
||||
:doc:`scrapy-lint <scrapy-lint:index>` to find mistakes in your spider code
|
||||
instead.
|
||||
(:issue:`4421`, :issue:`7627`)
|
||||
|
||||
- :func:`scrapy.utils.test.get_crawler` now disables
|
||||
:setting:`TELNETCONSOLE_ENABLED` by default.
|
||||
(:issue:`7644`)
|
||||
|
||||
- Other code refactoring and improvements.
|
||||
(:issue:`7409`, :issue:`7593`, :issue:`7594`, :issue:`7611`, :issue:`7649`)
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
- :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` no
|
||||
longer ignores proxy credentials for redirected or retried requests.
|
||||
(:issue:`7601`, :issue:`7630`)
|
||||
|
||||
- :class:`~scrapy.extensions.feedexport.GCSFeedStorage` now closes the
|
||||
temporary file after the upload.
|
||||
(:issue:`7546`)
|
||||
|
||||
- Fixed ``scrapy shell <URL>`` running a full spider crawl when there is a
|
||||
spider for the requested URL. This bug was introduced in Scrapy 2.13.0.
|
||||
(:issue:`7552`, :issue:`7557`)
|
||||
|
||||
- The :setting:`IMAGES_STORE_S3_ACL` and :setting:`IMAGES_STORE_GCS_ACL`
|
||||
settings are no longer ignored. This bug was introduced in Scrapy 2.12.0.
|
||||
(:issue:`7597`, :issue:`7614`)
|
||||
|
||||
- :class:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler` now closes
|
||||
the connection after making the request.
|
||||
(:issue:`7602`, :issue:`7667`)
|
||||
|
||||
- Removed the deprecated ``spider`` argument from the pipeline defined in the
|
||||
default project template.
|
||||
(:issue:`7676`)
|
||||
|
||||
- Fixed ``scrapy genspider --edit`` not working.
|
||||
(:issue:`7260`, :issue:`7683`)
|
||||
|
||||
- When a :class:`~scrapy.crawler.Crawler` instance is passed to
|
||||
:meth:`AsyncCrawlerRunner.create_crawler()
|
||||
<scrapy.crawler.AsyncCrawlerRunner.create_crawler>` or
|
||||
:meth:`CrawlerRunner.create_crawler()
|
||||
<scrapy.crawler.CrawlerRunner.create_crawler>`, settings from both classes
|
||||
are now merged, previously only the settings from the
|
||||
:class:`~scrapy.crawler.Crawler` instance were used.
|
||||
(:issue:`1280`, :issue:`7647`)
|
||||
|
||||
- Fixed several issues with cookie handling in
|
||||
:func:`scrapy.utils.request.request_to_curl`.
|
||||
(:issue:`7603`, :issue:`7675`, :issue:`7684`)
|
||||
|
||||
- Fixed :class:`scrapy.resolver.CachingThreadedResolver` not disabling the
|
||||
cache when :setting:`DNSCACHE_ENABLED` is set to ``False``.
|
||||
(:issue:`7663`)
|
||||
|
||||
- Fixed :func:`scrapy.utils.response.open_in_browser` not removing comments
|
||||
when looking for the ``<base>`` tag.
|
||||
(:issue:`7506`)
|
||||
|
||||
- Fixed checking for deprecated methods in custom :setting:`ITEM_PROCESSOR`
|
||||
implementations.
|
||||
(:issue:`7589`)
|
||||
|
||||
- Fixed :func:`scrapy.utils.url.strip_url` corrupting some URLs with
|
||||
credentials.
|
||||
(:issue:`7604`, :issue:`7605`)
|
||||
|
||||
- :func:`scrapy.utils.misc.rel_has_nofollow` now ignores the case when
|
||||
looking for "nofollow" strings.
|
||||
(:issue:`7632`)
|
||||
|
||||
- Fixed an exception in :class:`scrapy.utils.sitemap.Sitemap` when parsing
|
||||
some malformed sitemaps.
|
||||
(:issue:`7686`, :issue:`7687`)
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
- Mentioned :doc:`scrapy-lint <scrapy-lint:index>` in the docs.
|
||||
(:issue:`4421`, :issue:`7627`)
|
||||
|
||||
- Added the docs about :ref:`security considerations <security>`.
|
||||
(:issue:`7389`, :issue:`7678`)
|
||||
|
||||
- Improved the :ref:`item pipeline docs <topics-item-pipeline>`.
|
||||
(:issue:`2350`, :issue:`7676`)
|
||||
|
||||
- Documented which stats are collected by
|
||||
:class:`~scrapy.extensions.corestats.CoreStats`.
|
||||
(:issue:`7421`)
|
||||
|
||||
- Switched documentation examples from using :class:`scrapy.item.Item` to
|
||||
using dataclasses.
|
||||
(:issue:`7493`, :issue:`7513`)
|
||||
|
||||
- Added feature comparison tables to the :ref:`download handler
|
||||
<download-handlers-ref>` docs.
|
||||
(:issue:`7575`)
|
||||
|
||||
- Improved the docs for :ref:`logging settings <logging-settings>`.
|
||||
(:issue:`6909`, :issue:`7668`)
|
||||
|
||||
- Documented a way to :ref:`improve startup time and memory usage
|
||||
<large-project-startup>` by using :setting:`SPIDER_MODULES`.
|
||||
(:issue:`7576`, :issue:`7600`)
|
||||
|
||||
- Clarified handling of the ``type`` argument of :class:`~scrapy.Selector`.
|
||||
(:issue:`7704`)
|
||||
|
||||
- Other documentation improvements and fixes.
|
||||
(:issue:`4954`,
|
||||
:issue:`6120`,
|
||||
:issue:`7286`,
|
||||
:issue:`7564`,
|
||||
:issue:`7573`,
|
||||
:issue:`7598`,
|
||||
:issue:`7599`,
|
||||
:issue:`7698`)
|
||||
|
||||
Quality assurance
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Fixed deprecation warnings with pytest 9.1.0.
|
||||
(:issue:`7621`)
|
||||
|
||||
- Type hints improvements and fixes.
|
||||
(:issue:`6958`, :issue:`7586`)
|
||||
|
||||
- CI and test improvements and fixes.
|
||||
(:issue:`5954`,
|
||||
:issue:`7002`,
|
||||
:issue:`7017`,
|
||||
:issue:`7247`,
|
||||
:issue:`7508`,
|
||||
:issue:`7545`,
|
||||
:issue:`7566`,
|
||||
:issue:`7574`,
|
||||
:issue:`7585`,
|
||||
:issue:`7595`,
|
||||
:issue:`7608`,
|
||||
:issue:`7610`,
|
||||
:issue:`7612`,
|
||||
:issue:`7616`,
|
||||
:issue:`7625`,
|
||||
:issue:`7637`,
|
||||
:issue:`7639`,
|
||||
:issue:`7640`,
|
||||
:issue:`7641`,
|
||||
:issue:`7642`,
|
||||
:issue:`7643`,
|
||||
:issue:`7644`,
|
||||
:issue:`7645`,
|
||||
:issue:`7646`,
|
||||
:issue:`7654`,
|
||||
:issue:`7655`,
|
||||
:issue:`7664`,
|
||||
:issue:`7672`,
|
||||
:issue:`7677`,
|
||||
:issue:`7680`,
|
||||
:issue:`7682`,
|
||||
:issue:`7692`)
|
||||
|
||||
.. _release-2.16.0:
|
||||
|
||||
Scrapy 2.16.0 (2026-05-19)
|
||||
|
|
@ -600,7 +254,7 @@ Documentation
|
|||
- Added a ``CITATION.cff`` file.
|
||||
(:issue:`7502`, :issue:`7519`)
|
||||
|
||||
- Mentioned ``DOWNLOADER_CLIENT_TLS_METHOD`` in :ref:`bans`.
|
||||
- Mentioned :setting:`DOWNLOADER_CLIENT_TLS_METHOD` in :ref:`bans`.
|
||||
(:issue:`5232`, :issue:`7518`)
|
||||
|
||||
- Other documentation improvements and fixes.
|
||||
|
|
@ -1057,7 +711,7 @@ Deprecations
|
|||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- Added a new setting, :setting:`REFERRER_POLICIES`, to allow customizing
|
||||
- Added a new setting, :setting:`REFERER_POLICIES`, to allow customizing
|
||||
supported referrer policies.
|
||||
|
||||
Bug fixes
|
||||
|
|
@ -2546,7 +2200,7 @@ Backward-incompatible changes
|
|||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- User-defined cookies for HTTPS requests will have the ``secure`` flag set
|
||||
to ``True`` unless it's set to ``False`` explicitly. This is important when
|
||||
to ``True`` unless it's set to ``False`` explictly. This is important when
|
||||
these cookies are reused in HTTP requests, e.g. after a redirect to an HTTP
|
||||
URL.
|
||||
(:issue:`6357`)
|
||||
|
|
@ -2581,7 +2235,7 @@ Backward-incompatible changes
|
|||
``crawler.settings`` instead. When they call ``__init__()`` of the base
|
||||
class they should pass the ``crawler`` argument to it too.
|
||||
- A ``from_settings()`` method shouldn't be defined. Class-specific
|
||||
initialization code should go into either an overridden ``from_crawler()``
|
||||
initialization code should go into either an overriden ``from_crawler()``
|
||||
method or into ``__init__()``.
|
||||
- It's now possible to override ``from_crawler()`` and it's not necessary
|
||||
to call ``MediaPipeline.from_crawler()`` in it if other recommendations
|
||||
|
|
@ -2665,7 +2319,7 @@ Deprecation removals
|
|||
(:issue:`6109`, :issue:`6116`)
|
||||
|
||||
- A custom class assigned to the :setting:`SPIDER_LOADER_CLASS` setting that
|
||||
does not implement the ``ISpiderLoader`` interface
|
||||
does not implement the :class:`~scrapy.interfaces.ISpiderLoader` interface
|
||||
will now raise a :exc:`zope.interface.verify.DoesNotImplement` exception at
|
||||
run time. Non-compliant classes have been triggering a deprecation warning
|
||||
since Scrapy 1.0.0.
|
||||
|
|
@ -6068,7 +5722,7 @@ The following changes may impact custom priority queue classes:
|
|||
|
||||
* A new keyword parameter has been added: ``key``. It is a string
|
||||
that is always an empty string for memory queues and indicates the
|
||||
:setting:`JOBDIR` value for disk queues.
|
||||
:setting:`JOB_DIR` value for disk queues.
|
||||
|
||||
* The parameter for disk queues that contains data from the previous
|
||||
crawl, ``startprios`` or ``slot_startprios``, is now passed as a
|
||||
|
|
@ -6589,8 +6243,8 @@ Backward-incompatible changes
|
|||
``429``, you must override :setting:`RETRY_HTTP_CODES` accordingly.
|
||||
|
||||
* :class:`~scrapy.crawler.Crawler`,
|
||||
:meth:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>` and
|
||||
:meth:`CrawlerRunner.create_crawler <scrapy.crawler.CrawlerRunner.create_crawler>`
|
||||
:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>` and
|
||||
:class:`CrawlerRunner.create_crawler <scrapy.crawler.CrawlerRunner.create_crawler>`
|
||||
no longer accept a :class:`~scrapy.spiders.Spider` subclass instance, they
|
||||
only accept a :class:`~scrapy.spiders.Spider` subclass now.
|
||||
|
||||
|
|
@ -6622,7 +6276,7 @@ New features
|
|||
``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
|
||||
:ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
|
||||
scheduling improvement on crawls targeting multiple web domains, at the
|
||||
cost of no ``CONCURRENT_REQUESTS_PER_IP`` support (:issue:`3520`)
|
||||
cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
|
||||
|
||||
* A new :attr:`.Request.cb_kwargs` attribute
|
||||
provides a cleaner way to pass keyword arguments to callback methods
|
||||
|
|
@ -7810,7 +7464,7 @@ This 1.1 release brings a lot of interesting features and bug fixes:
|
|||
selectors engine without needing to upgrade Scrapy.
|
||||
- HTTPS downloader now does TLS protocol negotiation by default,
|
||||
instead of forcing TLS 1.0. You can also set the SSL/TLS method
|
||||
using the new ``DOWNLOADER_CLIENT_TLS_METHOD`` setting.
|
||||
using the new :setting:`DOWNLOADER_CLIENT_TLS_METHOD`.
|
||||
|
||||
- These bug fixes may require your attention:
|
||||
|
||||
|
|
@ -9101,7 +8755,7 @@ New features and settings
|
|||
- In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`)
|
||||
- Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`)
|
||||
- ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by:
|
||||
- :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, ``CONCURRENT_REQUESTS_PER_IP``
|
||||
- :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP`
|
||||
- check the documentation for more details
|
||||
- Added builtin caching DNS resolver (:rev:`2728`)
|
||||
- Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`)
|
||||
|
|
|
|||
|
|
@ -5,4 +5,4 @@ sphinx
|
|||
sphinx-notfound-page
|
||||
sphinx-rtd-theme
|
||||
sphinx-rtd-dark-mode
|
||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8
|
||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.6
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0
|
|||
# via
|
||||
# -r docs/requirements.in
|
||||
# sphinx-rtd-dark-mode
|
||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737
|
||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@b1d55db4d16a5425fc68576d63519bbfe26dd9c0
|
||||
# via -r docs/requirements.in
|
||||
sphinx-sitemap==2.9.0
|
||||
# via sphinx-scrapy
|
||||
|
|
|
|||
|
|
@ -21,14 +21,10 @@ The ``ADDONS`` setting is a dict in which every key is an add-on class or its
|
|||
import path and the value is its priority.
|
||||
|
||||
This is an example where two add-ons are enabled in a project's
|
||||
``settings.py``:
|
||||
|
||||
.. skip: next
|
||||
|
||||
.. code-block:: python
|
||||
``settings.py``::
|
||||
|
||||
ADDONS = {
|
||||
"path.to.someaddon": 0,
|
||||
'path.to.someaddon': 0,
|
||||
SomeAddonClass: 1,
|
||||
}
|
||||
|
||||
|
|
@ -60,9 +56,7 @@ the following methods:
|
|||
:type settings: :class:`~scrapy.settings.BaseSettings`
|
||||
|
||||
The settings set by the add-on should use the ``addon`` priority (see
|
||||
:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):
|
||||
|
||||
.. code-block:: python
|
||||
:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`)::
|
||||
|
||||
class MyAddon:
|
||||
def update_settings(self, settings):
|
||||
|
|
@ -174,6 +168,7 @@ Use a fallback component:
|
|||
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
|
||||
|
||||
FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -172,15 +172,46 @@ SpiderLoader API
|
|||
.. module:: scrapy.spiderloader
|
||||
:synopsis: The spider loader
|
||||
|
||||
Custom spider loaders can be employed by specifying their path in the
|
||||
:setting:`SPIDER_LOADER_CLASS` project setting. They must implement
|
||||
:class:`SpiderLoaderProtocol`.
|
||||
.. class:: SpiderLoader
|
||||
|
||||
.. autoclass:: SpiderLoaderProtocol
|
||||
:members:
|
||||
This class is in charge of retrieving and handling the spider classes
|
||||
defined across the project.
|
||||
|
||||
.. autoclass:: SpiderLoader
|
||||
:members:
|
||||
Custom spider loaders can be employed by specifying their path in the
|
||||
:setting:`SPIDER_LOADER_CLASS` project setting. They must fully implement
|
||||
the :class:`scrapy.interfaces.ISpiderLoader` interface to guarantee an
|
||||
errorless execution.
|
||||
|
||||
.. method:: from_settings(settings)
|
||||
|
||||
This class method is used by Scrapy to create an instance of the class.
|
||||
It's called with the current project settings, and it loads the spiders
|
||||
found recursively in the modules of the :setting:`SPIDER_MODULES`
|
||||
setting.
|
||||
|
||||
:param settings: project settings
|
||||
:type settings: :class:`~scrapy.settings.Settings` instance
|
||||
|
||||
.. method:: load(spider_name)
|
||||
|
||||
Get the Spider class with the given name. It'll look into the previously
|
||||
loaded spiders for a spider class with name ``spider_name`` and will raise
|
||||
a KeyError if not found.
|
||||
|
||||
:param spider_name: spider class name
|
||||
:type spider_name: str
|
||||
|
||||
.. method:: list()
|
||||
|
||||
Get the names of the available spiders in the project.
|
||||
|
||||
.. method:: find_by_request(request)
|
||||
|
||||
List the spiders' names that can handle the given request. Will try to
|
||||
match the request's url against the domains of the spiders.
|
||||
|
||||
:param request: queried request
|
||||
:type request: :class:`~scrapy.Request` instance
|
||||
|
||||
.. autoclass:: DummySpiderLoader
|
||||
|
||||
|
|
|
|||
|
|
@ -4,27 +4,19 @@
|
|||
asyncio
|
||||
=======
|
||||
|
||||
Scrapy supports :mod:`asyncio` natively. New projects created with
|
||||
:command:`startproject` have asyncio enabled by default, and you can use
|
||||
:mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine
|
||||
<coroutines>`.
|
||||
|
||||
The rest of this page covers advanced topics. If you are starting a new project,
|
||||
no additional setup is needed.
|
||||
Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
|
||||
asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
|
||||
:mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
|
||||
|
||||
|
||||
.. _install-asyncio:
|
||||
|
||||
Configuring the asyncio reactor
|
||||
===============================
|
||||
Installing the asyncio reactor
|
||||
==============================
|
||||
|
||||
New projects generated with :command:`startproject` have the asyncio
|
||||
reactor configured by default. No manual setup is needed.
|
||||
|
||||
The :setting:`TWISTED_REACTOR` setting controls which Twisted reactor Scrapy
|
||||
uses. Its default value is
|
||||
``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, which enables
|
||||
:mod:`asyncio` support.
|
||||
To enable :mod:`asyncio` support, your :setting:`TWISTED_REACTOR` setting needs
|
||||
to be set to ``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``,
|
||||
which is the default value.
|
||||
|
||||
If you are using :class:`~scrapy.crawler.AsyncCrawlerRunner` or
|
||||
:class:`~scrapy.crawler.CrawlerRunner`, you also need to
|
||||
|
|
@ -105,9 +97,6 @@ Scrapy API requires passing a Deferred to it) using the following helpers:
|
|||
|
||||
.. autofunction:: scrapy.utils.defer.deferred_from_coro
|
||||
.. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f
|
||||
|
||||
The following function helps with a reverse wrapping:
|
||||
|
||||
.. autofunction:: scrapy.utils.defer.ensure_awaitable
|
||||
|
||||
|
||||
|
|
@ -150,12 +139,6 @@ Using Scrapy without a Twisted reactor
|
|||
.. warning::
|
||||
This is currently experimental and may not be suitable for production use.
|
||||
|
||||
.. note:: As the Twisted download handlers cannot be used without a reactor,
|
||||
the default download handler in this mode is
|
||||
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. You
|
||||
will need to additionally install the ``httpx`` library to use it, unless
|
||||
you switch to some different handler.
|
||||
|
||||
It's possible to use Scrapy without installing a Twisted reactor at all, by
|
||||
setting the :setting:`TWISTED_REACTOR_ENABLED` setting to ``False``. In this
|
||||
mode Scrapy will use the asyncio event loop directly, and most of the Scrapy
|
||||
|
|
@ -199,7 +182,7 @@ in future Scrapy versions. The following features are not available:
|
|||
:class:`~scrapy.crawler.CrawlerProcess`
|
||||
(:class:`~scrapy.crawler.AsyncCrawlerProcess` and
|
||||
:class:`~scrapy.crawler.AsyncCrawlerRunner` are available)
|
||||
* Twisted-specific DNS resolvers (the :setting:`TWISTED_DNS_RESOLVER` setting)
|
||||
* Twisted-specific DNS resolvers (the :setting:`DNS_RESOLVER` setting)
|
||||
* User and 3rd-party code that requires a reactor (see :ref:`below
|
||||
<asyncio-without-reactor-migrate>` for examples)
|
||||
|
||||
|
|
@ -227,8 +210,7 @@ for its differences and limitations compared to
|
|||
|
||||
Additionally, :class:`~scrapy.crawler.AsyncCrawlerProcess` will install a
|
||||
:term:`meta path finder` that prevents :mod:`twisted.internet.reactor` from
|
||||
being imported. It will be uninstalled when :meth:`AsyncCrawlerProcess.start()
|
||||
<scrapy.crawler.AsyncCrawlerProcess.start>` exits.
|
||||
being imported.
|
||||
|
||||
.. _asyncio-without-reactor-migrate:
|
||||
|
||||
|
|
@ -325,7 +307,8 @@ implementations, :class:`~asyncio.ProactorEventLoop` (default) and
|
|||
:class:`~asyncio.SelectorEventLoop` works with Twisted.
|
||||
|
||||
Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop`
|
||||
automatically when installing the asyncio reactor.
|
||||
automatically when you change the :setting:`TWISTED_REACTOR` setting or call
|
||||
:func:`~scrapy.utils.reactor.install_reactor`.
|
||||
|
||||
.. note:: Other libraries you use may require
|
||||
:class:`~asyncio.ProactorEventLoop`, e.g. because it supports
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ AutoThrottle algorithm adjusts download delays based on the following rules:
|
|||
.. _download-latency:
|
||||
|
||||
In Scrapy, the download latency is measured as the time elapsed between
|
||||
sending the request and receiving the HTTP headers.
|
||||
establishing the TCP connection and receiving the HTTP headers.
|
||||
|
||||
Note that these latencies are very hard to measure accurately in a cooperative
|
||||
multitasking environment because Scrapy may be busy processing a spider
|
||||
|
|
@ -88,8 +88,6 @@ server) is, and this extension builds on that premise.
|
|||
Prevent specific requests from triggering slot delay adjustments
|
||||
================================================================
|
||||
|
||||
.. versionadded:: 2.12.0
|
||||
|
||||
AutoThrottle adjusts the delay of download slots based on the latencies of
|
||||
responses that belong to that download slot. The only exceptions are non-200
|
||||
responses, which are only taken into account to increase that delay, but
|
||||
|
|
|
|||
|
|
@ -199,7 +199,6 @@ Global commands:
|
|||
* :command:`fetch`
|
||||
* :command:`view`
|
||||
* :command:`version`
|
||||
* :command:`bench`
|
||||
|
||||
Project-only commands:
|
||||
|
||||
|
|
@ -208,6 +207,7 @@ Project-only commands:
|
|||
* :command:`list`
|
||||
* :command:`edit`
|
||||
* :command:`parse`
|
||||
* :command:`bench`
|
||||
|
||||
.. command:: startproject
|
||||
|
||||
|
|
@ -309,25 +309,11 @@ Usage examples::
|
|||
* parse_item
|
||||
|
||||
$ scrapy check
|
||||
F.F.
|
||||
======================================================================
|
||||
FAIL: [first_spider] parse (@returns post-hook)
|
||||
----------------------------------------------------------------------
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
scrapy.exceptions.ContractFail: Returned 92 requests, expected 0..4
|
||||
[FAILED] first_spider:parse_item
|
||||
>>> 'RetailPricex' field is missing
|
||||
|
||||
======================================================================
|
||||
FAIL: [first_spider] parse_item (@scrapes post-hook)
|
||||
----------------------------------------------------------------------
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
scrapy.exceptions.ContractFail: Missing fields: RetailPricex
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 4 contracts in 0.174s
|
||||
|
||||
FAILED (failures=2)
|
||||
[FAILED] first_spider:parse
|
||||
>>> Returned 92 requests, expected 0..4
|
||||
|
||||
.. skip: end
|
||||
|
||||
|
|
@ -391,7 +377,7 @@ Supported options:
|
|||
|
||||
* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
|
||||
|
||||
* ``--headers``: print the request's and response's HTTP headers instead of the response's body
|
||||
* ``--headers``: print the response's HTTP headers instead of the response's body
|
||||
|
||||
* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them)
|
||||
|
||||
|
|
@ -401,19 +387,15 @@ Usage examples::
|
|||
[ ... html content here ... ]
|
||||
|
||||
$ scrapy fetch --nolog --headers http://www.example.com/
|
||||
> Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
|
||||
> Accept-Language: en
|
||||
> User-Agent: Scrapy/2.16.0 (+https://scrapy.org)
|
||||
> Accept-Encoding: gzip, deflate, br
|
||||
>
|
||||
< Date: Wed, 08 Jul 2026 06:15:01 GMT
|
||||
< Content-Type: text/html
|
||||
< Server: cloudflare
|
||||
< Last-Modified: Wed, 01 Jul 2026 17:50:18 GMT
|
||||
< Allow: GET, HEAD
|
||||
< Cf-Cache-Status: HIT
|
||||
< Age: 8184
|
||||
< Cf-Ray: a17cf3b80eddf141-DME
|
||||
{'Accept-Ranges': ['bytes'],
|
||||
'Age': ['1263 '],
|
||||
'Connection': ['close '],
|
||||
'Content-Length': ['596'],
|
||||
'Content-Type': ['text/html; charset=UTF-8'],
|
||||
'Date': ['Wed, 18 Aug 2010 23:59:46 GMT'],
|
||||
'Etag': ['"573c1-254-48c9c87349680"'],
|
||||
'Last-Modified': ['Fri, 30 Jul 2010 15:30:18 GMT'],
|
||||
'Server': ['Apache/2.2.3 (CentOS)']}
|
||||
|
||||
.. command:: view
|
||||
|
||||
|
|
@ -494,7 +476,7 @@ Supported options:
|
|||
|
||||
* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
|
||||
|
||||
* ``-a NAME=VALUE``: set spider argument (may be repeated)
|
||||
* ``--a NAME=VALUE``: set spider argument (may be repeated)
|
||||
|
||||
* ``--callback`` or ``-c``: spider method to use as callback for parsing the
|
||||
response
|
||||
|
|
@ -623,10 +605,7 @@ shouldn't matter to the user running the command, but when the user :ref:`needs
|
|||
a non-default Twisted reactor <disable-asyncio>`, it may be important.
|
||||
|
||||
Scrapy decides which of these two classes to use based on the value of the
|
||||
:setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED` settings.
|
||||
With :setting:`TWISTED_REACTOR_ENABLED` set to ``False`` it will use
|
||||
:class:`~scrapy.crawler.AsyncCrawlerProcess`. Otherwise, if the
|
||||
:setting:`TWISTED_REACTOR` value is the default one
|
||||
:setting:`TWISTED_REACTOR` setting. If the setting value is the default one
|
||||
(``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``),
|
||||
:class:`~scrapy.crawler.AsyncCrawlerProcess` will be used, otherwise
|
||||
:class:`~scrapy.crawler.CrawlerProcess` will be used. The :ref:`spider settings
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ Supported callables
|
|||
The following callables may be defined as coroutines using ``async def``, and
|
||||
hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
|
||||
|
||||
- The :meth:`~scrapy.Spider.start` spider method, which *must* be
|
||||
- The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be
|
||||
defined as an :term:`asynchronous generator`.
|
||||
|
||||
.. versionadded:: 2.13
|
||||
|
|
@ -204,15 +204,13 @@ This means you can use many useful Python libraries providing such code:
|
|||
Common use cases for asynchronous code include:
|
||||
|
||||
* requesting data from websites, databases and other services (in
|
||||
:meth:`~scrapy.Spider.start`, callbacks, pipelines and
|
||||
:meth:`~scrapy.spiders.Spider.start`, callbacks, pipelines and
|
||||
middlewares);
|
||||
* storing data in databases (in pipelines and middlewares);
|
||||
* delaying the spider initialization until some external event (in the
|
||||
:signal:`spider_opened` handler);
|
||||
* calling asynchronous Scrapy methods like
|
||||
:meth:`ExecutionEngine.download_async()
|
||||
<scrapy.core.engine.ExecutionEngine.download_async>` (see :ref:`the
|
||||
screenshot pipeline example <ScreenshotPipeline>`).
|
||||
* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download`
|
||||
(see :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
|
||||
|
||||
.. _aio-libs: https://github.com/aio-libs
|
||||
|
||||
|
|
@ -269,6 +267,6 @@ You can also send multiple requests in parallel:
|
|||
responses = await asyncio.gather(*tasks)
|
||||
yield {
|
||||
"h1": response.css("h1::text").get(),
|
||||
"price": responses[0].css(".price::text").get(),
|
||||
"color": responses[1].css(".color::text").get(),
|
||||
"price": responses[0][1].css(".price::text").get(),
|
||||
"price2": responses[1][1].css(".color::text").get(),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -246,6 +246,7 @@ also request each page to get every quote on the site:
|
|||
.. code-block:: python
|
||||
|
||||
import scrapy
|
||||
import json
|
||||
|
||||
|
||||
class QuoteSpider(scrapy.Spider):
|
||||
|
|
@ -255,7 +256,7 @@ also request each page to get every quote on the site:
|
|||
start_urls = ["https://quotes.toscrape.com/api/quotes?page=1"]
|
||||
|
||||
def parse(self, response):
|
||||
data = response.json()
|
||||
data = json.loads(response.text)
|
||||
for quote in data["quotes"]:
|
||||
yield {"quote": quote["text"]}
|
||||
if data["has_next"]:
|
||||
|
|
|
|||
|
|
@ -39,10 +39,6 @@ for additional schemes and to replace or disable default ones:
|
|||
"sftp": "my.download_handlers.SftpHandler",
|
||||
}
|
||||
|
||||
.. seealso:: :ref:`security-unencrypted-protocols` and
|
||||
:ref:`security-local-resources`, for the security implications of the
|
||||
default ``http``, ``ftp``, ``file`` and ``data`` handlers.
|
||||
|
||||
Replacing HTTP(S) download handlers
|
||||
-----------------------------------
|
||||
|
||||
|
|
@ -89,7 +85,7 @@ the following API:
|
|||
If ``True``, the handler will only be instantiated when the first
|
||||
request handled by it needs to be downloaded.
|
||||
|
||||
.. method:: download_request(request: Request) -> Response
|
||||
.. method:: download_request(request: Request) -> Response:
|
||||
:async:
|
||||
|
||||
Download the given request and return a response.
|
||||
|
|
@ -134,39 +130,44 @@ these exceptions.
|
|||
|
||||
.. _download-handlers-ref:
|
||||
|
||||
Built-in HTTP download handlers reference
|
||||
=========================================
|
||||
Built-in download handlers reference
|
||||
====================================
|
||||
|
||||
Scrapy ships several handlers for HTTP and HTTPS requests. While all of them
|
||||
support basic features, they may differ in support of specific Scrapy features
|
||||
and settings and HTTP protocol features. See the documentation of specific
|
||||
handlers and specific settings for more information. Additionally, as the
|
||||
underlying HTTP client implementations differ between handlers, the behavior of
|
||||
specific websites may be different when doing the same Scrapy requests but
|
||||
using different handlers.
|
||||
DataURIDownloadHandler
|
||||
----------------------
|
||||
|
||||
Here is a comparison of some features of the built-in HTTP handlers, see the
|
||||
individual handler docs for more differences:
|
||||
.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler
|
||||
|
||||
================== ================= ===================== ====================
|
||||
Feature H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler
|
||||
================== ================= ===================== ====================
|
||||
Requires asyncio No No Yes
|
||||
Requires a reactor Yes Yes No
|
||||
HTTP/1.1 No Yes Yes
|
||||
HTTP/2 Yes No Yes
|
||||
TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl``
|
||||
HTTP proxies No Yes Yes
|
||||
SOCKS proxies No No Yes
|
||||
================== ================= ===================== ====================
|
||||
| Supported scheme: ``data``.
|
||||
| Lazy: no.
|
||||
|
||||
You can find additional HTTP download handlers in the
|
||||
scrapy-download-handlers-incubator_ package. This package is made by the Scrapy
|
||||
developers and contains experimental handlers that may be included in some
|
||||
later Scrapy version but can already be used. Please refer to the documentation
|
||||
of this package for more information.
|
||||
This handler supports RFC 2397 ``data:content/type;base64,`` data URIs.
|
||||
|
||||
.. _scrapy-download-handlers-incubator: https://github.com/scrapy-plugins/scrapy-download-handlers-incubator
|
||||
FileDownloadHandler
|
||||
-------------------
|
||||
|
||||
.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler
|
||||
|
||||
| Supported scheme: ``file``.
|
||||
| Lazy: no.
|
||||
|
||||
This handler supports ``file:///path`` local file URIs. It doesn't
|
||||
support remote files.
|
||||
|
||||
FTPDownloadHandler
|
||||
------------------
|
||||
|
||||
.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler
|
||||
|
||||
| Supported scheme: ``ftp``.
|
||||
| Lazy: no.
|
||||
|
||||
This handler supports ``ftp://host/path`` FTP URIs.
|
||||
|
||||
It's implemented using :mod:`twisted.protocols.ftp`.
|
||||
|
||||
.. note::
|
||||
This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
||||
|
||||
.. _twisted-http2-handler:
|
||||
|
||||
|
|
@ -176,9 +177,7 @@ H2DownloadHandler
|
|||
.. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler
|
||||
|
||||
| Supported scheme: ``https``.
|
||||
| :ref:`Lazy <lazy-download-handlers>`: yes.
|
||||
| :ref:`Requires asyncio support <using-asyncio>`: no.
|
||||
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: yes.
|
||||
| Lazy: yes.
|
||||
|
||||
This handler supports ``https://host/path`` URLs and uses the HTTP/2 protocol
|
||||
for them.
|
||||
|
|
@ -197,96 +196,63 @@ If you want to use this handler you need to replace the default one for the
|
|||
"https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",
|
||||
}
|
||||
|
||||
Features and limitations
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. warning::
|
||||
|
||||
This handler is experimental, and not yet recommended for production
|
||||
environments. Future Scrapy versions may introduce related changes without
|
||||
a deprecation period or warning.
|
||||
|
||||
=========================== ================================================
|
||||
HTTP proxies No (not implemented)
|
||||
SOCKS proxies No (not supported by the library)
|
||||
HTTP/2 Yes
|
||||
``response.certificate`` :class:`twisted.internet.ssl.Certificate` object
|
||||
Per-request ``bindaddress`` Yes
|
||||
TLS implementation ``pyOpenSSL``/``cryptography``
|
||||
=========================== ================================================
|
||||
.. note::
|
||||
|
||||
Other limitations:
|
||||
Known limitations of the HTTP/2 implementation in this handler include:
|
||||
|
||||
- No support for HTTP/1.1.
|
||||
- No support for proxies.
|
||||
|
||||
- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER`
|
||||
to ``scrapy.resolver.CachingHostnameResolver``.
|
||||
- No support for HTTP/2 Cleartext (h2c), since no major browser supports
|
||||
HTTP/2 unencrypted (refer `http2 faq`_).
|
||||
|
||||
- No support for the :signal:`bytes_received` and :signal:`headers_received`
|
||||
signals.
|
||||
- No setting to specify a maximum `frame size`_ larger than the default
|
||||
value, 16384. Connections to servers that send a larger frame will
|
||||
fail.
|
||||
|
||||
Known limitations of the HTTP/2 support:
|
||||
- No support for `server pushes`_, which are ignored.
|
||||
|
||||
- No support for HTTP/2 Cleartext (h2c), since no major browser supports
|
||||
HTTP/2 unencrypted (refer `http2 faq`_).
|
||||
|
||||
- No setting to specify a maximum `frame size`_ larger than the default
|
||||
value, 16384. Connections to servers that send a larger frame will fail.
|
||||
|
||||
- No support for `server pushes`_, which are ignored.
|
||||
- No support for the :signal:`bytes_received` and
|
||||
:signal:`headers_received` signals.
|
||||
|
||||
.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2
|
||||
.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
|
||||
.. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2
|
||||
|
||||
.. note::
|
||||
This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
||||
|
||||
HTTP11DownloadHandler
|
||||
---------------------
|
||||
|
||||
.. autoclass:: scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler
|
||||
|
||||
| Supported schemes: ``http``, ``https``.
|
||||
| :ref:`Lazy <lazy-download-handlers>`: no.
|
||||
| :ref:`Requires asyncio support <using-asyncio>`: no.
|
||||
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: yes.
|
||||
| Lazy: no.
|
||||
|
||||
This handler supports ``http://host/path`` and ``https://host/path`` URLs and
|
||||
uses the HTTP/1.1 protocol for them.
|
||||
|
||||
It's implemented using :mod:`twisted.web.client`.
|
||||
|
||||
Features and limitations
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
=========================== ================================================
|
||||
HTTP proxies Yes
|
||||
SOCKS proxies No (not supported by the library)
|
||||
HTTP/2 No (implemented as a separate handler)
|
||||
``response.certificate`` :class:`twisted.internet.ssl.Certificate` object
|
||||
Per-request ``bindaddress`` Yes
|
||||
TLS implementation ``pyOpenSSL``/``cryptography``
|
||||
=========================== ================================================
|
||||
|
||||
Other limitations:
|
||||
|
||||
- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER`
|
||||
to ``scrapy.resolver.CachingHostnameResolver``.
|
||||
|
||||
- HTTPS proxies to HTTPS destinations are not supported.
|
||||
.. note::
|
||||
This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
||||
|
||||
HttpxDownloadHandler
|
||||
--------------------
|
||||
|
||||
.. versionadded:: 2.15.0
|
||||
|
||||
.. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler
|
||||
|
||||
| Supported schemes: ``http``, ``https``.
|
||||
| :ref:`Lazy <lazy-download-handlers>`: no.
|
||||
| :ref:`Requires asyncio support <using-asyncio>`: yes.
|
||||
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.
|
||||
| Lazy: no.
|
||||
|
||||
This handler supports ``http://host/path`` and ``https://host/path`` URLs and
|
||||
uses the HTTP/1.1 or HTTP/2 protocol for them.
|
||||
uses the HTTP/1.1 protocol for them.
|
||||
|
||||
It's implemented using the ``httpx`` library and needs it to be installed.
|
||||
|
||||
|
|
@ -300,83 +266,30 @@ If you want to use this handler you need to replace the default ones for the
|
|||
"https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
|
||||
}
|
||||
|
||||
Features and limitations
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. warning::
|
||||
|
||||
This handler is experimental, and not yet recommended for production
|
||||
environments. Future Scrapy versions may introduce related changes without
|
||||
a deprecation period or warning or even remove it altogether.
|
||||
|
||||
=========================== =======================================
|
||||
HTTP proxies Yes
|
||||
SOCKS proxies Yes (SOCKS5; requires ``httpx[socks]``)
|
||||
HTTP/2 Yes (requires ``httpx[http2]``)
|
||||
``response.certificate`` DER bytes
|
||||
Per-request ``bindaddress`` No (not supported by the library)
|
||||
TLS implementation Standard library ``ssl``
|
||||
=========================== =======================================
|
||||
.. note::
|
||||
|
||||
Other limitations:
|
||||
As this handler is based on a different HTTP client implementation compared
|
||||
to :class:`~.HTTP11DownloadHandler`, it's expected that its behavior on
|
||||
some websites may be different. Additionally, these are the Scrapy features
|
||||
that are explicitly not supported when using it:
|
||||
|
||||
- The handler creates a separate connection pool for each proxy URL (due to
|
||||
limitations of ``httpx``) which may lead to higher resource usage when
|
||||
using proxy rotation.
|
||||
- Per-request bind address support (the :reqmeta:`bindaddress` meta key).
|
||||
The global :setting:`DOWNLOAD_BIND_ADDRESS` setting is supported but the
|
||||
port number, if specified, will be ignored.
|
||||
|
||||
.. setting:: HTTPX_HTTP2_ENABLED
|
||||
- The :setting:`DOWNLOADER_CLIENT_TLS_METHOD` setting.
|
||||
|
||||
HTTPX_HTTP2_ENABLED
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
- Settings specific to the Twisted networking or HTTP implementation, like
|
||||
:setting:`DNS_RESOLVER`.
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Default: ``False``
|
||||
|
||||
Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra
|
||||
needs to be installed if you want to enable this setting.
|
||||
|
||||
Built-in non-HTTP download handlers reference
|
||||
=============================================
|
||||
|
||||
DataURIDownloadHandler
|
||||
----------------------
|
||||
|
||||
.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler
|
||||
|
||||
| Supported scheme: ``data``.
|
||||
| :ref:`Lazy <lazy-download-handlers>`: no.
|
||||
| :ref:`Requires asyncio support <using-asyncio>`: no.
|
||||
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.
|
||||
|
||||
This handler supports RFC 2397 ``data:content/type;base64,`` data URIs.
|
||||
|
||||
FileDownloadHandler
|
||||
-------------------
|
||||
|
||||
.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler
|
||||
|
||||
| Supported scheme: ``file``.
|
||||
| :ref:`Lazy <lazy-download-handlers>`: no.
|
||||
| :ref:`Requires asyncio support <using-asyncio>`: no.
|
||||
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.
|
||||
|
||||
This handler supports ``file:///path`` local file URIs. It doesn't
|
||||
support remote files.
|
||||
|
||||
FTPDownloadHandler
|
||||
------------------
|
||||
|
||||
.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler
|
||||
|
||||
| Supported scheme: ``ftp``.
|
||||
| :ref:`Lazy <lazy-download-handlers>`: no.
|
||||
| :ref:`Requires asyncio support <using-asyncio>`: no.
|
||||
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: yes.
|
||||
|
||||
This handler supports ``ftp://host/path`` FTP URIs.
|
||||
|
||||
It's implemented using :mod:`twisted.protocols.ftp`.
|
||||
- Using :ref:`non-asyncio reactors <disable-asyncio>` (``httpx`` requires
|
||||
``asyncio``).
|
||||
|
||||
S3DownloadHandler
|
||||
-----------------
|
||||
|
|
@ -384,9 +297,7 @@ S3DownloadHandler
|
|||
.. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler
|
||||
|
||||
| Supported scheme: ``s3``.
|
||||
| :ref:`Lazy <lazy-download-handlers>`: yes.
|
||||
| :ref:`Requires asyncio support <using-asyncio>`: no.
|
||||
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.
|
||||
| Lazy: yes.
|
||||
|
||||
This handler supports ``s3://bucket/path`` S3 URIs.
|
||||
|
||||
|
|
|
|||
|
|
@ -307,15 +307,26 @@ HttpAuthMiddleware
|
|||
|
||||
.. class:: HttpAuthMiddleware
|
||||
|
||||
This middleware authenticates requests using `Basic access authentication`_
|
||||
(aka. HTTP auth).
|
||||
This middleware authenticates all requests generated from certain spiders
|
||||
using `Basic access authentication`_ (aka. HTTP auth).
|
||||
|
||||
Use the :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS`, and
|
||||
:setting:`HTTPAUTH_DOMAIN` settings to configure it. You can also override
|
||||
the credentials per request via :attr:`~scrapy.Request.meta` keys
|
||||
:reqmeta:`http_user`, :reqmeta:`http_pass`, and :reqmeta:`http_auth_domain`.
|
||||
To enable HTTP authentication for a spider, set the ``http_user`` and
|
||||
``http_pass`` spider attributes to the authentication data and the
|
||||
``http_auth_domain`` spider attribute to the domain which requires this
|
||||
authentication (its subdomains will be also handled in the same way).
|
||||
You can set ``http_auth_domain`` to ``None`` to enable the
|
||||
authentication for all requests but you risk leaking your authentication
|
||||
credentials to unrelated domains.
|
||||
|
||||
Example using settings (e.g. in :attr:`~scrapy.Spider.custom_settings`):
|
||||
.. warning::
|
||||
In previous Scrapy versions HttpAuthMiddleware sent the authentication
|
||||
data with all requests, which is a security problem if the spider
|
||||
makes requests to several different domains. Currently if the
|
||||
``http_auth_domain`` attribute is not set, the middleware will use the
|
||||
domain of the first request, which will work for some spiders but not
|
||||
for others. In the future the middleware will produce an error instead.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
|
|
@ -323,70 +334,13 @@ HttpAuthMiddleware
|
|||
|
||||
|
||||
class SomeIntranetSiteSpider(CrawlSpider):
|
||||
http_user = "someuser"
|
||||
http_pass = "somepass"
|
||||
http_auth_domain = "intranet.example.com"
|
||||
name = "intranet.example.com"
|
||||
custom_settings = {
|
||||
"HTTPAUTH_USER": "someuser",
|
||||
"HTTPAUTH_PASS": "somepass",
|
||||
"HTTPAUTH_DOMAIN": "intranet.example.com",
|
||||
}
|
||||
|
||||
# .. rest of the spider code omitted ...
|
||||
|
||||
Example using per-request meta:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
async def start(self):
|
||||
yield Request(
|
||||
"https://intranet.example.com/protected/",
|
||||
meta={
|
||||
"http_user": "someuser",
|
||||
"http_pass": "somepass",
|
||||
"http_auth_domain": "intranet.example.com",
|
||||
},
|
||||
)
|
||||
|
||||
.. setting:: HTTPAUTH_USER
|
||||
|
||||
HTTPAUTH_USER
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Default: ``""``
|
||||
|
||||
The username to use for HTTP basic authentication, applied to all requests
|
||||
whose URL matches :setting:`HTTPAUTH_DOMAIN`.
|
||||
|
||||
.. setting:: HTTPAUTH_PASS
|
||||
|
||||
HTTPAUTH_PASS
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Default: ``""``
|
||||
|
||||
The password to use for HTTP basic authentication.
|
||||
|
||||
.. setting:: HTTPAUTH_DOMAIN
|
||||
|
||||
HTTPAUTH_DOMAIN
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The domain (and its subdomains) to which HTTP basic authentication credentials
|
||||
are sent. Set to ``None`` to send credentials with all requests, but be aware
|
||||
that this risks leaking credentials to unrelated domains.
|
||||
|
||||
This setting must be explicitly configured whenever :setting:`HTTPAUTH_USER`
|
||||
or :setting:`HTTPAUTH_PASS` is set.
|
||||
|
||||
.. seealso:: :ref:`security-credential-leakage`
|
||||
|
||||
.. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication
|
||||
|
||||
|
||||
|
|
@ -505,7 +459,7 @@ Filesystem storage backend (default)
|
|||
|
||||
* ``response_body`` - the plain response body
|
||||
|
||||
* ``response_headers`` - the response headers (in raw HTTP format)
|
||||
* ``response_headers`` - the request headers (in raw HTTP format)
|
||||
|
||||
* ``meta`` - some metadata of this cache resource in Python ``repr()``
|
||||
format (grep-friendly format)
|
||||
|
|
@ -547,7 +501,7 @@ defines the methods described below.
|
|||
.. method:: open_spider(spider)
|
||||
|
||||
This method gets called after a spider has been opened for crawling. It handles
|
||||
the :signal:`spider_opened` signal.
|
||||
the :signal:`open_spider <spider_opened>` signal.
|
||||
|
||||
:param spider: the spider which has been opened
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
|
@ -555,7 +509,7 @@ defines the methods described below.
|
|||
.. method:: close_spider(spider)
|
||||
|
||||
This method gets called after a spider has been closed. It handles
|
||||
the :signal:`spider_closed` signal.
|
||||
the :signal:`close_spider <spider_closed>` signal.
|
||||
|
||||
:param spider: the spider which has been closed
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
|
@ -591,8 +545,8 @@ In order to use your storage backend, set:
|
|||
HTTPCache middleware settings
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` can be
|
||||
configured through the following settings:
|
||||
The :class:`HttpCacheMiddleware` can be configured through the following
|
||||
settings:
|
||||
|
||||
.. setting:: HTTPCACHE_ENABLED
|
||||
|
||||
|
|
@ -791,7 +745,8 @@ HttpProxyMiddleware
|
|||
Handling of this meta key needs to be implemented inside the :ref:`download
|
||||
handler <topics-download-handlers>`, so it's not guaranteed to be supported
|
||||
by all 3rd-party handlers. It's currently unsupported by
|
||||
:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`.
|
||||
:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` and
|
||||
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.
|
||||
|
||||
.. note::
|
||||
|
||||
|
|
@ -803,17 +758,11 @@ HttpProxyMiddleware
|
|||
:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`
|
||||
supports HTTPS proxies only for HTTP destinations.
|
||||
|
||||
.. note::
|
||||
|
||||
If the download handler supports it, you can use a SOCKS proxy URL (e.g.
|
||||
``socks5://username:password@some_proxy_server:port``).
|
||||
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`
|
||||
supports SOCKS proxies while other built-in handlers don't.
|
||||
|
||||
HttpProxyMiddleware settings
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. setting:: HTTPPROXY_ENABLED
|
||||
.. setting:: HTTPPROXY_AUTH_ENCODING
|
||||
|
||||
HTTPPROXY_ENABLED
|
||||
^^^^^^^^^^^^^^^^^
|
||||
|
|
@ -822,8 +771,6 @@ Default: ``True``
|
|||
|
||||
Whether or not to enable the :class:`HttpProxyMiddleware`.
|
||||
|
||||
.. setting:: HTTPPROXY_AUTH_ENCODING
|
||||
|
||||
HTTPPROXY_AUTH_ENCODING
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
|
|
@ -868,9 +815,9 @@ OffsiteMiddleware
|
|||
.. reqmeta:: allow_offsite
|
||||
|
||||
If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to
|
||||
``True`` or :attr:`Request.meta <scrapy.Request.meta>` has ``allow_offsite``
|
||||
set to ``True``, then the OffsiteMiddleware will allow the request even if
|
||||
its domain is not listed in allowed domains.
|
||||
``True`` or :attr:`Request.meta` has ``allow_offsite`` set to ``True``, then
|
||||
the OffsiteMiddleware will allow the request even if its domain is not listed
|
||||
in allowed domains.
|
||||
|
||||
RedirectMiddleware
|
||||
------------------
|
||||
|
|
@ -985,7 +932,7 @@ Whether the Meta Refresh middleware will be enabled.
|
|||
METAREFRESH_IGNORE_TAGS
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Default: ``["noscript"]``
|
||||
Default: ``[]``
|
||||
|
||||
Meta tags within these tags are ignored.
|
||||
|
||||
|
|
@ -1015,6 +962,17 @@ RetryMiddleware
|
|||
A middleware to retry failed requests that are potentially caused by
|
||||
temporary problems such as a connection timeout or HTTP 500 error.
|
||||
|
||||
Failed pages are collected on the scraping process and rescheduled at the
|
||||
end, once the spider has finished crawling all regular (non failed) pages.
|
||||
|
||||
The :class:`RetryMiddleware` can be configured through the following
|
||||
settings (see the settings documentation for more info):
|
||||
|
||||
* :setting:`RETRY_ENABLED`
|
||||
* :setting:`RETRY_TIMES`
|
||||
* :setting:`RETRY_HTTP_CODES`
|
||||
* :setting:`RETRY_EXCEPTIONS`
|
||||
|
||||
.. reqmeta:: dont_retry
|
||||
|
||||
If :attr:`Request.meta <scrapy.Request.meta>` has ``dont_retry`` key
|
||||
|
|
@ -1081,7 +1039,7 @@ Default::
|
|||
'twisted.internet.error.ConnectionDone',
|
||||
'twisted.internet.error.ConnectError',
|
||||
'twisted.internet.error.ConnectionLost',
|
||||
OSError,
|
||||
IOError,
|
||||
'scrapy.core.downloader.handlers.http11.TunnelError',
|
||||
]
|
||||
|
||||
|
|
@ -1095,23 +1053,6 @@ has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught
|
|||
exception propagation, see
|
||||
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`.
|
||||
|
||||
.. setting:: RETRY_GIVE_UP_LOG_LEVEL
|
||||
|
||||
RETRY_GIVE_UP_LOG_LEVEL
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Default: ``"ERROR"``
|
||||
|
||||
:ref:`Logging level <levels>` used for the message logged when a request
|
||||
exceeds its retries.
|
||||
|
||||
Can be a level name (e.g. ``"WARNING"``) or a number (e.g. ``logging.WARNING``
|
||||
or ``30``).
|
||||
|
||||
See also: :reqmeta:`give_up_log_level`, :func:`get_retry_request`.
|
||||
|
||||
.. setting:: RETRY_PRIORITY_ADJUST
|
||||
|
||||
RETRY_PRIORITY_ADJUST
|
||||
|
|
|
|||
|
|
@ -133,7 +133,7 @@ data from it depends on the type of response:
|
|||
|
||||
.. code-block:: python
|
||||
|
||||
selector = Selector(text=data["html"])
|
||||
selector = Selector(data["html"])
|
||||
|
||||
- If the response is JavaScript, or HTML with a ``<script/>`` element
|
||||
containing the desired data, see :ref:`topics-parsing-javascript`.
|
||||
|
|
|
|||
|
|
@ -12,8 +12,7 @@ Exceptions
|
|||
Built-in Exceptions reference
|
||||
=============================
|
||||
|
||||
Here's a list of all exceptions included in Scrapy and their usage, except for
|
||||
the :ref:`download handler exceptions <download-handlers-exceptions>`.
|
||||
Here's a list of all exceptions included in Scrapy and their usage.
|
||||
|
||||
|
||||
CloseSpider
|
||||
|
|
@ -32,7 +31,7 @@ For example:
|
|||
.. code-block:: python
|
||||
|
||||
def parse_page(self, response):
|
||||
if "Bandwidth exceeded" in response.text:
|
||||
if "Bandwidth exceeded" in response.body:
|
||||
raise CloseSpider("bandwidth_exceeded")
|
||||
|
||||
DontCloseSpider
|
||||
|
|
@ -72,8 +71,7 @@ remain disabled. Those components include:
|
|||
- Downloader middlewares
|
||||
- Spider middlewares
|
||||
|
||||
The exception must be raised in the component's ``__init__()`` or
|
||||
``from_crawler()`` method.
|
||||
The exception must be raised in the component's ``__init__`` method.
|
||||
|
||||
NotSupported
|
||||
------------
|
||||
|
|
|
|||
|
|
@ -93,25 +93,24 @@ described next.
|
|||
1. Declaring a serializer in the field
|
||||
--------------------------------------
|
||||
|
||||
Every :ref:`item type <item-types>` except :class:`dict` lets you declare a
|
||||
serializer in the :ref:`field metadata <topics-items-fields>`. The serializer
|
||||
must be a callable which receives a value and returns its serialized form.
|
||||
If you use :class:`~scrapy.Item` you can declare a serializer in the
|
||||
:ref:`field metadata <topics-items-fields>`. The serializer must be
|
||||
a callable which receives a value and returns its serialized form.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
import scrapy
|
||||
|
||||
|
||||
def serialize_price(value):
|
||||
return f"$ {str(value)}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Product:
|
||||
name: str
|
||||
price: float = field(metadata={"serializer": serialize_price})
|
||||
class Product(scrapy.Item):
|
||||
name = scrapy.Field()
|
||||
price = scrapy.Field(serializer=serialize_price)
|
||||
|
||||
|
||||
2. Overriding the serialize_field() method
|
||||
|
|
@ -153,7 +152,7 @@ output examples, which assume you're exporting these two items:
|
|||
BaseItemExporter
|
||||
----------------
|
||||
|
||||
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding=None, indent=None, dont_fail=False)
|
||||
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0, dont_fail=False)
|
||||
|
||||
This is the (abstract) base class for all Item Exporters. It provides
|
||||
support for common features used by all (concrete) Item Exporters, such as
|
||||
|
|
@ -211,17 +210,13 @@ BaseItemExporter
|
|||
|
||||
- ``None`` (all fields [2]_, default)
|
||||
|
||||
- A list of fields:
|
||||
- A list of fields::
|
||||
|
||||
.. code-block:: python
|
||||
['field1', 'field2']
|
||||
|
||||
["field1", "field2"]
|
||||
- A dict where keys are fields and values are output names::
|
||||
|
||||
- A dict where keys are fields and values are output names:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
{"field1": "Field 1", "field2": "Field 2"}
|
||||
{'field1': 'Field 1', 'field2': 'Field 2'}
|
||||
|
||||
.. [1] Not all exporters respect the specified field order.
|
||||
.. [2] When using :ref:`item objects <item-types>` that do not expose
|
||||
|
|
@ -243,7 +238,7 @@ BaseItemExporter
|
|||
|
||||
.. attribute:: indent
|
||||
|
||||
Amount of spaces used to indent the output on each level. Defaults to ``None``.
|
||||
Amount of spaces used to indent the output on each level. Defaults to ``0``.
|
||||
|
||||
* ``indent=None`` selects the most compact representation,
|
||||
all items in the same line with no indentation
|
||||
|
|
@ -277,9 +272,7 @@ XmlItemExporter
|
|||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method.
|
||||
|
||||
A typical output of this exporter would be:
|
||||
|
||||
.. code-block:: xml
|
||||
A typical output of this exporter would be::
|
||||
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<items>
|
||||
|
|
@ -297,17 +290,11 @@ XmlItemExporter
|
|||
exported by serializing each value inside a ``<value>`` element. This is for
|
||||
convenience, as multi-valued fields are very common.
|
||||
|
||||
For example, the item:
|
||||
For example, the item::
|
||||
|
||||
.. skip: next
|
||||
Item(name=['John', 'Doe'], age='23')
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
Item(name=["John", "Doe"], age="23")
|
||||
|
||||
Would be serialized as:
|
||||
|
||||
.. code-block:: xml
|
||||
Would be serialized as::
|
||||
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<items>
|
||||
|
|
@ -340,7 +327,7 @@ CsvItemExporter
|
|||
|
||||
:param join_multivalued: The char (or chars) that will be used for joining
|
||||
multi-valued fields, if found.
|
||||
:type join_multivalued: str
|
||||
:type include_headers_line: str
|
||||
|
||||
:param errors: The optional string that specifies how encoding and decoding
|
||||
errors are to be handled. For more information see
|
||||
|
|
@ -354,14 +341,14 @@ CsvItemExporter
|
|||
|
||||
A typical output of this exporter would be::
|
||||
|
||||
name,price
|
||||
product,price
|
||||
Color TV,1200
|
||||
DVD player,200
|
||||
|
||||
PickleItemExporter
|
||||
------------------
|
||||
|
||||
.. class:: PickleItemExporter(file, protocol=4, **kwargs)
|
||||
.. class:: PickleItemExporter(file, protocol=0, **kwargs)
|
||||
|
||||
Exports items in pickle format to the given file-like object.
|
||||
|
||||
|
|
@ -391,12 +378,10 @@ PprintItemExporter
|
|||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method.
|
||||
|
||||
A typical output of this exporter would be:
|
||||
A typical output of this exporter would be::
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
{"name": "Color TV", "price": "1200"}
|
||||
{"name": "DVD player", "price": "200"}
|
||||
{'name': 'Color TV', 'price': '1200'}
|
||||
{'name': 'DVD player', 'price': '200'}
|
||||
|
||||
Longer lines (when present) are pretty-formatted.
|
||||
|
||||
|
|
@ -414,9 +399,7 @@ JsonItemExporter
|
|||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
||||
A typical output of this exporter would be:
|
||||
|
||||
.. code-block:: json
|
||||
A typical output of this exporter would be::
|
||||
|
||||
[{"name": "Color TV", "price": "1200"},
|
||||
{"name": "DVD player", "price": "200"}]
|
||||
|
|
@ -445,9 +428,7 @@ JsonLinesItemExporter
|
|||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
||||
A typical output of this exporter would be:
|
||||
|
||||
.. code-block:: json
|
||||
A typical output of this exporter would be::
|
||||
|
||||
{"name": "Color TV", "price": "1200"}
|
||||
{"name": "DVD player", "price": "200"}
|
||||
|
|
|
|||
|
|
@ -136,18 +136,7 @@ Core Stats extension
|
|||
Enable the collection of core statistics, provided the stats collection is
|
||||
enabled (see :ref:`topics-stats`).
|
||||
|
||||
The following stats are collected:
|
||||
|
||||
* ``start_time``: start date/time of the crawl (:class:`~datetime.datetime`).
|
||||
* ``finish_time``: end date/time of the crawl (:class:`~datetime.datetime`).
|
||||
* ``elapsed_time_seconds``: total crawl duration in seconds (:class:`float`).
|
||||
* ``finish_reason``: the closing reason string (e.g. ``"finished"``,
|
||||
``"closespider_timeout"``).
|
||||
* ``item_scraped_count``: total number of items that passed all pipelines.
|
||||
* ``item_dropped_count``: total number of items dropped by a pipeline.
|
||||
* ``item_dropped_reasons_count/<ExceptionName>``: per-exception drop count
|
||||
(e.g. ``item_dropped_reasons_count/DropItem``).
|
||||
* ``response_received_count``: total number of HTTP responses received.
|
||||
.. _topics-extensions-ref-telnetconsole:
|
||||
|
||||
Log Count extension
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
|
|
@ -157,8 +146,6 @@ Log Count extension
|
|||
|
||||
.. autoclass:: LogCount
|
||||
|
||||
.. _topics-extensions-ref-telnetconsole:
|
||||
|
||||
Telnet console extension
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
@ -260,7 +247,6 @@ settings:
|
|||
* :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`
|
||||
* :setting:`CLOSESPIDER_ITEMCOUNT`
|
||||
* :setting:`CLOSESPIDER_PAGECOUNT`
|
||||
* :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM`
|
||||
* :setting:`CLOSESPIDER_ERRORCOUNT`
|
||||
|
||||
.. note::
|
||||
|
|
@ -274,11 +260,12 @@ settings:
|
|||
CLOSESPIDER_TIMEOUT
|
||||
"""""""""""""""""""
|
||||
|
||||
Default: ``0.0``
|
||||
Default: ``0``
|
||||
|
||||
If the spider remains open for more than this number of seconds, it will be
|
||||
automatically closed with the reason ``closespider_timeout``. If zero (or non
|
||||
set), spiders won't be closed by timeout.
|
||||
An integer which specifies a number of seconds. If the spider remains open for
|
||||
more than that number of seconds, it will be automatically closed with the
|
||||
reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by
|
||||
timeout.
|
||||
|
||||
.. setting:: CLOSESPIDER_TIMEOUT_NO_ITEM
|
||||
|
||||
|
|
@ -341,6 +328,9 @@ closing the spider. If the spider generates more than that number of errors,
|
|||
it will be closed with the reason ``closespider_errorcount``. If zero (or non
|
||||
set), spiders won't be closed by number of errors.
|
||||
|
||||
.. module:: scrapy.extensions.debug
|
||||
:synopsis: Extensions for debugging Scrapy
|
||||
|
||||
.. module:: scrapy.extensions.periodic_log
|
||||
:synopsis: Periodic stats logging
|
||||
|
||||
|
|
@ -415,7 +405,7 @@ Example extension configuration:
|
|||
custom_settings = {
|
||||
"LOG_LEVEL": "INFO",
|
||||
"PERIODIC_LOG_STATS": {
|
||||
"include": ["downloader/", "scheduler/", "log_count/", "item_scraped_count"],
|
||||
"include": ["downloader/", "scheduler/", "log_count/", "item_scraped_count/"],
|
||||
},
|
||||
"PERIODIC_LOG_DELTA": {"include": ["downloader/"]},
|
||||
"PERIODIC_LOG_TIMING_ENABLED": True,
|
||||
|
|
@ -460,9 +450,6 @@ Default: ``False``
|
|||
Debugging extensions
|
||||
--------------------
|
||||
|
||||
.. module:: scrapy.extensions.debug
|
||||
:synopsis: Extensions for debugging Scrapy
|
||||
|
||||
Stack trace dump extension
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
|
|||
|
|
@ -143,11 +143,6 @@ Here are some examples to illustrate:
|
|||
.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence
|
||||
they can also be used as storage URI parameters.
|
||||
|
||||
.. note:: Only ``%(...)s`` parameters are replaced. Any other percent
|
||||
character is kept as-is, so percent-encoded URIs (e.g. ``%20`` for a
|
||||
space or percent-encoded FTP credentials) and :class:`pathlib.Path`
|
||||
keys containing ``%(...)s`` parameters both work as expected.
|
||||
|
||||
|
||||
.. _topics-feed-storage-backends:
|
||||
|
||||
|
|
@ -166,7 +161,7 @@ The feeds are stored in the local filesystem.
|
|||
- Required external libraries: none
|
||||
|
||||
Note that for the local filesystem storage (only) you can omit the scheme if
|
||||
you specify a path (e.g. ``/tmp/export.csv``).
|
||||
you specify an absolute path like ``/tmp/export.csv`` (Unix systems only).
|
||||
Alternatively you can also use a :class:`pathlib.Path` object.
|
||||
|
||||
.. _topics-feed-storage-ftp:
|
||||
|
|
@ -434,37 +429,33 @@ This setting is required for enabling the feed export feature.
|
|||
|
||||
See :ref:`topics-feed-storage-backends` for supported URI schemes.
|
||||
|
||||
For instance:
|
||||
|
||||
.. skip: next
|
||||
|
||||
.. code-block:: python
|
||||
For instance::
|
||||
|
||||
{
|
||||
"items.json": {
|
||||
"format": "json",
|
||||
"encoding": "utf8",
|
||||
"store_empty": False,
|
||||
"item_classes": [MyItemClass1, "myproject.items.MyItemClass2"],
|
||||
"fields": None,
|
||||
"indent": 4,
|
||||
"item_export_kwargs": {
|
||||
"export_empty_fields": True,
|
||||
'items.json': {
|
||||
'format': 'json',
|
||||
'encoding': 'utf8',
|
||||
'store_empty': False,
|
||||
'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'],
|
||||
'fields': None,
|
||||
'indent': 4,
|
||||
'item_export_kwargs': {
|
||||
'export_empty_fields': True,
|
||||
},
|
||||
},
|
||||
"/home/user/documents/items.xml": {
|
||||
"format": "xml",
|
||||
"fields": ["name", "price"],
|
||||
"item_filter": MyCustomFilter1,
|
||||
"encoding": "latin1",
|
||||
"indent": 8,
|
||||
'/home/user/documents/items.xml': {
|
||||
'format': 'xml',
|
||||
'fields': ['name', 'price'],
|
||||
'item_filter': MyCustomFilter1,
|
||||
'encoding': 'latin1',
|
||||
'indent': 8,
|
||||
},
|
||||
pathlib.Path("items.csv.gz"): {
|
||||
"format": "csv",
|
||||
"fields": ["price", "name"],
|
||||
"item_filter": "myproject.filters.MyCustomFilter2",
|
||||
"postprocessing": [MyPlugin1, "scrapy.extensions.postprocessing.GzipPlugin"],
|
||||
"gzip_compresslevel": 5,
|
||||
pathlib.Path('items.csv.gz'): {
|
||||
'format': 'csv',
|
||||
'fields': ['price', 'name'],
|
||||
'item_filter': 'myproject.filters.MyCustomFilter2',
|
||||
'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],
|
||||
'gzip_compresslevel': 5,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -537,6 +528,10 @@ safe numeric encoding (``\uXXXX`` sequences) for historic reasons.
|
|||
|
||||
Use ``"utf-8"`` if you want UTF-8 for JSON too.
|
||||
|
||||
.. versionchanged:: 2.8
|
||||
The :command:`startproject` command now sets this setting to
|
||||
``"utf-8"`` in the generated ``settings.py`` file.
|
||||
|
||||
.. setting:: FEED_EXPORT_FIELDS
|
||||
|
||||
FEED_EXPORT_FIELDS
|
||||
|
|
@ -624,7 +619,6 @@ Default:
|
|||
"file": "scrapy.extensions.feedexport.FileFeedStorage",
|
||||
"stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",
|
||||
"s3": "scrapy.extensions.feedexport.S3FeedStorage",
|
||||
"gs": "scrapy.extensions.feedexport.GCSFeedStorage",
|
||||
"ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
|
||||
}
|
||||
|
||||
|
|
@ -766,8 +760,8 @@ The function signature should be as follows:
|
|||
:param spider: source spider of the feed items
|
||||
:type spider: scrapy.Spider
|
||||
|
||||
.. caution:: The function must return a new dictionary instead of modifying
|
||||
the received ``params`` in-place.
|
||||
.. caution:: The function should return a new dictionary, modifying
|
||||
the received ``params`` in-place is deprecated.
|
||||
|
||||
For example, to include the :attr:`name <scrapy.Spider.name>` of the
|
||||
source spider in the feed URI:
|
||||
|
|
|
|||
|
|
@ -57,8 +57,6 @@ Any of these methods may be defined as a coroutine function (``async def``).
|
|||
Item pipeline example
|
||||
=====================
|
||||
|
||||
.. _price-pipeline-example:
|
||||
|
||||
Price validation and dropping items with no prices
|
||||
--------------------------------------------------
|
||||
|
||||
|
|
@ -121,7 +119,7 @@ Write items to MongoDB
|
|||
|
||||
In this example we'll write items to MongoDB_ using pymongo_.
|
||||
MongoDB address and database name are specified in Scrapy settings;
|
||||
MongoDB collection is specified in a class attribute.
|
||||
MongoDB collection is named after item class.
|
||||
|
||||
The main point of this example is to show how to :ref:`get the crawler
|
||||
<from-crawler>` and how to clean up the resources properly.
|
||||
|
|
@ -248,8 +246,6 @@ returns multiples items with the same id:
|
|||
return item
|
||||
|
||||
|
||||
.. _activating-item-pipeline:
|
||||
|
||||
Activating an Item Pipeline component
|
||||
=====================================
|
||||
|
||||
|
|
@ -266,110 +262,3 @@ To activate an Item Pipeline component you must add its class to the
|
|||
The integer values you assign to classes in this setting determine the
|
||||
order in which they run: items go through from lower valued to higher
|
||||
valued classes. It's customary to define these numbers in the 0-1000 range.
|
||||
|
||||
A complete example
|
||||
==================
|
||||
|
||||
The examples above show item pipeline components on their own. In a project, a
|
||||
pipeline is one of four pieces that work together: the :ref:`item
|
||||
<topics-items>` your spider produces, the :ref:`spider <topics-spiders>` that
|
||||
yields it, the pipeline that processes it, and the :setting:`ITEM_PIPELINES`
|
||||
setting that enables the pipeline.
|
||||
|
||||
The following example wires those pieces together to validate the price of
|
||||
books scraped from `books.toscrape.com`_, reusing the ``PricePipeline`` from
|
||||
:ref:`price-pipeline-example` above.
|
||||
|
||||
Define the item in ``myproject/items.py``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class BookItem:
|
||||
title: str
|
||||
price: float
|
||||
|
||||
Yield instances of that item from your spider, e.g. in
|
||||
``myproject/spiders/books.py``:
|
||||
|
||||
.. skip: next
|
||||
.. code-block:: python
|
||||
|
||||
import scrapy
|
||||
|
||||
from myproject.items import BookItem
|
||||
|
||||
|
||||
class BooksSpider(scrapy.Spider):
|
||||
name = "books"
|
||||
start_urls = ["https://books.toscrape.com/"]
|
||||
|
||||
def parse(self, response):
|
||||
for book in response.css("article.product_pod"):
|
||||
yield BookItem(
|
||||
title=book.css("h3 a::attr(title)").get(),
|
||||
price=float(book.css("p.price_color::text").re_first(r"[\d.]+")),
|
||||
)
|
||||
|
||||
Put the ``PricePipeline`` shown earlier in ``myproject/pipelines.py``, and
|
||||
enable it in ``myproject/settings.py``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
ITEM_PIPELINES = {
|
||||
"myproject.pipelines.PricePipeline": 300,
|
||||
}
|
||||
|
||||
With these pieces in place, every ``BookItem`` that ``BooksSpider`` yields
|
||||
passes through ``PricePipeline`` before it reaches the :ref:`feed exports
|
||||
<topics-feed-exports>` or any other output.
|
||||
|
||||
.. _books.toscrape.com: https://books.toscrape.com/
|
||||
|
||||
|
||||
Common pitfalls
|
||||
===============
|
||||
|
||||
The pipeline does not run
|
||||
-------------------------
|
||||
|
||||
A pipeline component only runs if its class is listed in the
|
||||
:setting:`ITEM_PIPELINES` setting, normally in your project's
|
||||
:file:`settings.py` file (see :ref:`activating-item-pipeline`). Adding it to
|
||||
the spider or elsewhere has no effect.
|
||||
|
||||
To confirm that Scrapy loaded your pipeline, look for a line like this near the
|
||||
start of the crawl log::
|
||||
|
||||
[scrapy.middleware] INFO: Enabled item pipelines:
|
||||
['myproject.pipelines.PricePipeline']
|
||||
|
||||
If your pipeline is missing from that list, check that its import path matches
|
||||
the :setting:`ITEM_PIPELINES` entry, and that the setting is not being
|
||||
overridden, for example by :attr:`~scrapy.Spider.custom_settings` or by a
|
||||
redefinition of :setting:`ITEM_PIPELINES` in :file:`settings.py`.
|
||||
|
||||
The item is not returned
|
||||
------------------------
|
||||
|
||||
:meth:`process_item` must return the item (or raise
|
||||
:exc:`~scrapy.exceptions.DropItem`). A common mistake is to modify the item but
|
||||
forget to return it:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def process_item(self, item):
|
||||
ItemAdapter(item)["price"] *= 1.15
|
||||
# Bug: returns None, so the next component gets None instead of the item.
|
||||
|
||||
Return the item so that the next component, and the rest of Scrapy, can keep
|
||||
processing it:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def process_item(self, item):
|
||||
ItemAdapter(item)["price"] *= 1.15
|
||||
return item
|
||||
|
|
|
|||
|
|
@ -23,8 +23,7 @@ Item Types
|
|||
|
||||
Scrapy supports the following types of items, via the `itemadapter`_ library:
|
||||
:ref:`dictionaries <dict-items>`, :ref:`Item objects <item-objects>`,
|
||||
:ref:`dataclass objects <dataclass-items>`, :ref:`attrs objects <attrs-items>`
|
||||
and :ref:`Pydantic models <pydantic-items>`.
|
||||
:ref:`dataclass objects <dataclass-items>`, and :ref:`attrs objects <attrs-items>`.
|
||||
|
||||
.. _itemadapter: https://github.com/scrapy/itemadapter
|
||||
|
||||
|
|
@ -62,8 +61,8 @@ its ``__init__`` method.
|
|||
:class:`Item` also allows the defining of field metadata, which can be used to
|
||||
:ref:`customize serialization <topics-exporters-field-serialization>`.
|
||||
|
||||
:mod:`scrapy.utils.trackref` tracks :class:`Item` objects to help find memory
|
||||
leaks (see :ref:`topics-leaks-trackrefs`).
|
||||
:mod:`trackref` tracks :class:`Item` objects to help find memory leaks
|
||||
(see :ref:`topics-leaks-trackrefs`).
|
||||
|
||||
Example:
|
||||
|
||||
|
|
@ -263,7 +262,7 @@ Creating items
|
|||
|
||||
>>> product = Product(name="Desktop PC", price=1000)
|
||||
>>> print(product)
|
||||
{'name': 'Desktop PC', 'price': 1000}
|
||||
Product(name='Desktop PC', price=1000)
|
||||
|
||||
|
||||
Getting field values
|
||||
|
|
@ -377,12 +376,10 @@ Creating dicts from items:
|
|||
>>> dict(product) # create a dict from all populated values
|
||||
{'price': 1000, 'name': 'Desktop PC'}
|
||||
|
||||
Creating items from dicts:
|
||||
|
||||
.. code-block:: pycon
|
||||
Creating items from dicts:
|
||||
|
||||
>>> Product({"name": "Laptop PC", "price": 1500})
|
||||
{'name': 'Laptop PC', 'price': 1500}
|
||||
Product(price=1500, name='Laptop PC')
|
||||
|
||||
>>> Product({"name": "Laptop PC", "lala": 1500}) # warning: unknown field in dict
|
||||
Traceback (most recent call last):
|
||||
|
|
|
|||
|
|
@ -104,15 +104,6 @@ If you wish to log the requests that couldn't be serialized, you can set the
|
|||
:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
|
||||
It is ``False`` by default.
|
||||
|
||||
.. note:: Because requests are serialized with :mod:`pickle`, the objects you
|
||||
store on a request, such as the values of its
|
||||
:attr:`~scrapy.Request.cb_kwargs` and :attr:`~scrapy.Request.meta`
|
||||
dictionaries, are deep-copied when the request is written to and later read
|
||||
back from the job directory. As a result, the callback receives a *copy* of
|
||||
those objects rather than the original ones, and changes made to the copy are
|
||||
not reflected in the original object. Keep this in mind if you rely on
|
||||
sharing mutable state through ``cb_kwargs`` or ``meta``.
|
||||
|
||||
.. _job-dir-contents:
|
||||
|
||||
Job directory contents
|
||||
|
|
@ -151,8 +142,8 @@ Where:
|
|||
- :class:`~scrapy.pqueues.ScrapyPriorityQueue` creates the ``{priority}{s?}``
|
||||
directories.
|
||||
|
||||
- :class:`scrapy.squeues.PickleFifoDiskQueue`, a subclass of
|
||||
:class:`queuelib.FifoDiskQueue` that uses :mod:`pickle` to serialize
|
||||
- :class:`scrapy.squeues.PickleLifoDiskQueue`, a subclass of
|
||||
:class:`queuelib.LifoDiskQueue` that uses :mod:`pickle` to serialize
|
||||
:class:`dict` representations of :class:`scrapy.Request` objects, creates
|
||||
the ``info.json`` and ``q{00000}`` files.
|
||||
|
||||
|
|
|
|||
|
|
@ -62,9 +62,9 @@ Debugging memory leaks with ``trackref``
|
|||
|
||||
.. skip: start
|
||||
|
||||
:mod:`scrapy.utils.trackref` is a module provided by Scrapy to debug the most
|
||||
common cases of memory leaks. It basically tracks the references to all live
|
||||
Request, Response, Item, Spider and Selector objects.
|
||||
:mod:`trackref` is a module provided by Scrapy to debug the most common cases of
|
||||
memory leaks. It basically tracks the references to all live Request,
|
||||
Response, Item, Spider and Selector objects.
|
||||
|
||||
You can enter the telnet console and inspect how many objects (of the classes
|
||||
mentioned above) are currently alive using the ``prefs()`` function which is an
|
||||
|
|
@ -93,7 +93,7 @@ You can get the oldest object of each class using the
|
|||
Which objects are tracked?
|
||||
--------------------------
|
||||
|
||||
The objects tracked by ``trackref`` are all from these classes (and all its
|
||||
The objects tracked by ``trackrefs`` are all from these classes (and all its
|
||||
subclasses):
|
||||
|
||||
* :class:`scrapy.Request`
|
||||
|
|
@ -106,15 +106,10 @@ A real example
|
|||
--------------
|
||||
|
||||
Let's see a concrete example of a hypothetical case of memory leaks.
|
||||
Suppose we have some spider with a line similar to this one:
|
||||
Suppose we have some spider with a line similar to this one::
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
return Request(
|
||||
f"http://www.somenastyspider.com/product.php?pid={product_id}",
|
||||
callback=self.parse,
|
||||
cb_kwargs={"referer": response},
|
||||
)
|
||||
return Request(f"http://www.somenastyspider.com/product.php?pid={product_id}",
|
||||
callback=self.parse, cb_kwargs={'referer': response})
|
||||
|
||||
That line is passing a response reference inside a request which effectively
|
||||
ties the response lifetime to the requests' one, and that would definitely
|
||||
|
|
@ -169,7 +164,7 @@ Too many spiders?
|
|||
-----------------
|
||||
|
||||
If your project has too many spiders executed in parallel,
|
||||
the output of ``prefs()`` can be difficult to read.
|
||||
the output of :func:`prefs` can be difficult to read.
|
||||
For this reason, that function has a ``ignore`` argument which can be used to
|
||||
ignore a particular class (and all its subclasses). For
|
||||
example, this won't show any live references to spiders:
|
||||
|
|
@ -192,7 +187,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
|
|||
Inherit from this class if you want to track live
|
||||
instances with the ``trackref`` module.
|
||||
|
||||
.. function:: print_live_refs(ignore=NoneType)
|
||||
.. function:: print_live_refs(class_name, ignore=NoneType)
|
||||
|
||||
Print a report of live references, grouped by class name.
|
||||
|
||||
|
|
@ -208,9 +203,9 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
|
|||
|
||||
.. function:: iter_all(class_name)
|
||||
|
||||
Return an iterator over all objects alive with the given class name. Use
|
||||
:func:`print_live_refs` first to get a list of all tracked live objects
|
||||
per class name.
|
||||
Return an iterator over all objects alive with the given class name, or
|
||||
``None`` if none is found. Use :func:`print_live_refs` first to get a list
|
||||
of all tracked live objects per class name.
|
||||
|
||||
.. skip: end
|
||||
|
||||
|
|
|
|||
|
|
@ -36,9 +36,7 @@ Link extractor reference
|
|||
|
||||
The link extractor class is
|
||||
:class:`scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`. For convenience it
|
||||
can also be imported as ``scrapy.linkextractors.LinkExtractor``:
|
||||
|
||||
.. code-block:: python
|
||||
can also be imported as ``scrapy.linkextractors.LinkExtractor``::
|
||||
|
||||
from scrapy.linkextractors import LinkExtractor
|
||||
|
||||
|
|
@ -49,7 +47,108 @@ LxmlLinkExtractor
|
|||
:synopsis: lxml's HTMLParser-based link extractors
|
||||
|
||||
|
||||
.. autoclass:: LxmlLinkExtractor
|
||||
.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, strip=True)
|
||||
|
||||
LxmlLinkExtractor is the recommended link extractor with handy filtering
|
||||
options. It is implemented using lxml's robust HTMLParser.
|
||||
|
||||
:param allow: a single regular expression (or list of regular expressions)
|
||||
that the (absolute) urls must match in order to be extracted. If not
|
||||
given (or empty), it will match all links.
|
||||
:type allow: str or list
|
||||
|
||||
:param deny: a single regular expression (or list of regular expressions)
|
||||
that the (absolute) urls must match in order to be excluded (i.e. not
|
||||
extracted). It has precedence over the ``allow`` parameter. If not
|
||||
given (or empty) it won't exclude any links.
|
||||
:type deny: str or list
|
||||
|
||||
:param allow_domains: a single value or a list of string containing
|
||||
domains which will be considered for extracting the links
|
||||
:type allow_domains: str or list
|
||||
|
||||
:param deny_domains: a single value or a list of strings containing
|
||||
domains which won't be considered for extracting the links
|
||||
:type deny_domains: str or list
|
||||
|
||||
:param deny_extensions: a single value or list of strings containing
|
||||
extensions that should be ignored when extracting links.
|
||||
If not given, it will default to
|
||||
:data:`scrapy.linkextractors.IGNORED_EXTENSIONS`.
|
||||
|
||||
:type deny_extensions: list
|
||||
|
||||
:param restrict_xpaths: is an XPath (or list of XPath's) which defines
|
||||
regions inside the response where links should be extracted from.
|
||||
If given, only the text selected by those XPath will be scanned for
|
||||
links.
|
||||
:type restrict_xpaths: str or list
|
||||
|
||||
:param restrict_css: a CSS selector (or list of selectors) which defines
|
||||
regions inside the response where links should be extracted from.
|
||||
Has the same behaviour as ``restrict_xpaths``.
|
||||
:type restrict_css: str or list
|
||||
|
||||
:param restrict_text: a single regular expression (or list of regular expressions)
|
||||
that the link's text must match in order to be extracted. If not
|
||||
given (or empty), it will match all links. If a list of regular expressions is
|
||||
given, the link will be extracted if it matches at least one.
|
||||
:type restrict_text: str or list
|
||||
|
||||
:param tags: a tag or a list of tags to consider when extracting links.
|
||||
Defaults to ``('a', 'area')``.
|
||||
:type tags: str or list
|
||||
|
||||
:param attrs: an attribute or list of attributes which should be considered when looking
|
||||
for links to extract (only for those tags specified in the ``tags``
|
||||
parameter). Defaults to ``('href',)``
|
||||
:type attrs: list
|
||||
|
||||
:param canonicalize: canonicalize each extracted url (using
|
||||
w3lib.url.canonicalize_url). Defaults to ``False``.
|
||||
Note that canonicalize_url is meant for duplicate checking;
|
||||
it can change the URL visible at server side, so the response can be
|
||||
different for requests with canonicalized and raw URLs. If you're
|
||||
using LinkExtractor to follow links it is more robust to
|
||||
keep the default ``canonicalize=False``.
|
||||
:type canonicalize: bool
|
||||
|
||||
:param unique: whether duplicate filtering should be applied to extracted
|
||||
links.
|
||||
:type unique: bool
|
||||
|
||||
:param process_value: a function which receives each value extracted from
|
||||
the tag and attributes scanned and can modify the value and return a
|
||||
new one, or return ``None`` to ignore the link altogether. If not
|
||||
given, ``process_value`` defaults to ``lambda x: x``.
|
||||
|
||||
.. highlight:: html
|
||||
|
||||
For example, to extract links from this code::
|
||||
|
||||
<a href="javascript:goToPage('../other/page.html'); return false">Link text</a>
|
||||
|
||||
.. highlight:: python
|
||||
|
||||
You can use the following function in ``process_value``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def process_value(value):
|
||||
m = re.search(r"javascript:goToPage\('(.*?)'", value)
|
||||
if m:
|
||||
return m.group(1)
|
||||
|
||||
:type process_value: collections.abc.Callable
|
||||
|
||||
:param strip: whether to strip whitespaces from extracted attributes.
|
||||
According to HTML5 standard, leading and trailing whitespaces
|
||||
must be stripped from ``href`` attributes of ``<a>``, ``<area>``
|
||||
and many other elements, ``src`` attribute of ``<img>``, ``<iframe>``
|
||||
elements, etc., so LinkExtractor strips space chars by default.
|
||||
Set ``strip=False`` to turn it off (e.g. if you're extracting urls
|
||||
from elements or attributes which allow leading/trailing whitespaces).
|
||||
:type strip: bool
|
||||
|
||||
.. automethod:: extract_links
|
||||
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ data that will be assigned to the ``name`` field later.
|
|||
|
||||
Afterwards, similar calls are used for ``price`` and ``stock`` fields
|
||||
(the latter using a CSS selector with the :meth:`~ItemLoader.add_css` method),
|
||||
and finally the ``last_updated`` field is populated directly with a literal value
|
||||
and finally the ``last_update`` field is populated directly with a literal value
|
||||
(``today``) using a different method: :meth:`~ItemLoader.add_value`.
|
||||
|
||||
Finally, when all data is collected, the :meth:`ItemLoader.load_item` method is
|
||||
|
|
@ -102,13 +102,14 @@ One approach to overcome this is to define items using the
|
|||
.. code-block:: python
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class InventoryItem:
|
||||
name: str | None = field(default=None)
|
||||
price: float | None = field(default=None)
|
||||
stock: int | None = field(default=None)
|
||||
name: Optional[str] = field(default=None)
|
||||
price: Optional[float] = field(default=None)
|
||||
stock: Optional[int] = field(default=None)
|
||||
|
||||
|
||||
.. _topics-loaders-processors:
|
||||
|
|
@ -227,8 +228,7 @@ metadata. Here is an example:
|
|||
|
||||
.. code-block:: python
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import scrapy
|
||||
from itemloaders.processors import Join, MapCompose, TakeFirst
|
||||
from w3lib.html import remove_tags
|
||||
|
||||
|
|
@ -238,21 +238,14 @@ metadata. Here is an example:
|
|||
return value
|
||||
|
||||
|
||||
@dataclass
|
||||
class Product:
|
||||
name: str | None = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"input_processor": MapCompose(remove_tags),
|
||||
"output_processor": Join(),
|
||||
},
|
||||
class Product(scrapy.Item):
|
||||
name = scrapy.Field(
|
||||
input_processor=MapCompose(remove_tags),
|
||||
output_processor=Join(),
|
||||
)
|
||||
price: str | None = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"input_processor": MapCompose(remove_tags, filter_price),
|
||||
"output_processor": TakeFirst(),
|
||||
},
|
||||
price = scrapy.Field(
|
||||
input_processor=MapCompose(remove_tags, filter_price),
|
||||
output_processor=TakeFirst(),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -264,7 +257,7 @@ metadata. Here is an example:
|
|||
>>> il.add_value("name", ["Welcome to my", "<strong>website</strong>"])
|
||||
>>> il.add_value("price", ["€", "<span>1000</span>"])
|
||||
>>> il.load_item()
|
||||
Product(name='Welcome to my website', price='1000')
|
||||
{'name': 'Welcome to my website', 'price': '1000'}
|
||||
|
||||
.. skip: end
|
||||
|
||||
|
|
@ -273,8 +266,8 @@ The precedence order, for both input and output processors, is as follows:
|
|||
1. Item Loader field-specific attributes: ``field_in`` and ``field_out`` (most
|
||||
precedence)
|
||||
2. Field metadata (``input_processor`` and ``output_processor`` key)
|
||||
3. Item Loader defaults: :attr:`ItemLoader.default_input_processor` and
|
||||
:attr:`ItemLoader.default_output_processor` (least precedence)
|
||||
3. Item Loader defaults: :meth:`ItemLoader.default_input_processor` and
|
||||
:meth:`ItemLoader.default_output_processor` (least precedence)
|
||||
|
||||
See also: :ref:`topics-loaders-extending`.
|
||||
|
||||
|
|
@ -323,8 +316,8 @@ There are several ways to modify Item Loader context values:
|
|||
loader = ItemLoader(product, unit="cm")
|
||||
|
||||
3. On Item Loader declaration, for those input/output processors that support
|
||||
instantiating them with an Item Loader context.
|
||||
:class:`~itemloaders.processors.MapCompose` is one of them:
|
||||
instantiating them with an Item Loader context. :class:`~processor.MapCompose` is one of
|
||||
them:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
|
|
@ -350,9 +343,7 @@ When parsing related values from a subsection of a document, it can be
|
|||
useful to create nested loaders. Imagine you're extracting details from
|
||||
a footer of a page that looks something like:
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: html
|
||||
Example::
|
||||
|
||||
<footer>
|
||||
<a class="social" href="https://facebook.com/whatever">Like Us</a>
|
||||
|
|
|
|||
|
|
@ -4,6 +4,11 @@
|
|||
Logging
|
||||
=======
|
||||
|
||||
.. note::
|
||||
:mod:`scrapy.log` has been deprecated alongside its functions in favor of
|
||||
explicit calls to the Python standard logging. Keep reading to learn more
|
||||
about the new logging system.
|
||||
|
||||
Scrapy uses :mod:`logging` for event logging. We'll
|
||||
provide some simple examples to get you started, but for more advanced
|
||||
use-cases it's strongly suggested to read thoroughly its documentation.
|
||||
|
|
|
|||
|
|
@ -41,10 +41,11 @@ this:
|
|||
2. The item is returned from the spider and goes to the item pipeline.
|
||||
|
||||
3. When the item reaches the :class:`FilesPipeline`, the URLs in the
|
||||
``file_urls`` field are downloaded using the standard Scrapy downloader
|
||||
(which means the downloader middlewares are used, but the spider middlewares
|
||||
aren't). The item remains "locked" at that particular pipeline stage until
|
||||
the files have finished downloading (or failed for some reason).
|
||||
``file_urls`` field are scheduled for download using the standard
|
||||
Scrapy scheduler and downloader (which means the scheduler and downloader
|
||||
middlewares are reused), but with a higher priority, processing them before other
|
||||
pages are scraped. The item remains "locked" at that particular pipeline stage
|
||||
until the files have finish downloading (or fail for some reason).
|
||||
|
||||
4. When the files are downloaded, another field (``files``) will be populated
|
||||
with the results. This field will contain a list of dicts with information
|
||||
|
|
@ -80,6 +81,9 @@ thumbnailing and normalizing images to JPEG/RGB format.
|
|||
Enabling your Media Pipeline
|
||||
============================
|
||||
|
||||
.. setting:: IMAGES_STORE
|
||||
.. setting:: FILES_STORE
|
||||
|
||||
To enable your media pipeline you must first add it to your project
|
||||
:setting:`ITEM_PIPELINES` setting.
|
||||
|
||||
|
|
@ -98,8 +102,6 @@ For Files Pipeline, use:
|
|||
.. note::
|
||||
You can also use both the Files and Images Pipeline at the same time.
|
||||
|
||||
.. setting:: IMAGES_STORE
|
||||
.. setting:: FILES_STORE
|
||||
|
||||
Then, configure the target storage setting to a valid value that will be used
|
||||
for storing the downloaded images. Otherwise the pipeline will remain disabled,
|
||||
|
|
@ -335,18 +337,17 @@ respectively), the pipeline will put the results under the respective field
|
|||
When using :ref:`item types <item-types>` for which fields are defined beforehand,
|
||||
you must define both the URLs field and the results field. For example, when
|
||||
using the images pipeline, items must define both the ``image_urls`` and the
|
||||
``images`` field. For instance, using a dataclass:
|
||||
``images`` field. For instance, using the :class:`~scrapy.Item` class:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
import scrapy
|
||||
|
||||
|
||||
@dataclass
|
||||
class MyItem:
|
||||
class MyItem(scrapy.Item):
|
||||
# ... other item fields ...
|
||||
image_urls: list[str] = field(default_factory=list)
|
||||
images: list[dict] = field(default_factory=list)
|
||||
image_urls = scrapy.Field()
|
||||
images = scrapy.Field()
|
||||
|
||||
If you want to use another field name for the URLs key or for the results key,
|
||||
it is also possible to override it.
|
||||
|
|
@ -370,12 +371,11 @@ For the Images Pipeline, set :setting:`IMAGES_URLS_FIELD` and/or
|
|||
If you need something more complex and want to override the custom pipeline
|
||||
behaviour, see :ref:`topics-media-pipeline-override`.
|
||||
|
||||
If you have multiple image pipelines inheriting from :class:`ImagesPipeline`
|
||||
and you want to have different settings in different pipelines you can set
|
||||
setting keys preceded with uppercase name of your pipeline class. E.g. if your
|
||||
pipeline is called ``MyPipeline`` and you want to have custom
|
||||
:setting:`IMAGES_URLS_FIELD` you define setting
|
||||
``MYPIPELINE_IMAGES_URLS_FIELD`` and your custom settings will be used.
|
||||
If you have multiple image pipelines inheriting from ImagePipeline and you want
|
||||
to have different settings in different pipelines you can set setting keys
|
||||
preceded with uppercase name of your pipeline class. E.g. if your pipeline is
|
||||
called MyPipeline and you want to have custom IMAGES_URLS_FIELD you define
|
||||
setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used.
|
||||
|
||||
|
||||
Additional features
|
||||
|
|
@ -470,9 +470,7 @@ When using the Images Pipeline, you can drop images which are too small, by
|
|||
specifying the minimum allowed size in the :setting:`IMAGES_MIN_HEIGHT` and
|
||||
:setting:`IMAGES_MIN_WIDTH` settings.
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block:: python
|
||||
For example::
|
||||
|
||||
IMAGES_MIN_HEIGHT = 110
|
||||
IMAGES_MIN_WIDTH = 110
|
||||
|
|
@ -495,9 +493,7 @@ Allowing redirections
|
|||
By default media pipelines ignore redirects, i.e. an HTTP redirection
|
||||
to a media file URL request will mean the media download is considered failed.
|
||||
|
||||
To handle media redirections, set this setting to ``True``:
|
||||
|
||||
.. code-block:: python
|
||||
To handle media redirections, set this setting to ``True``::
|
||||
|
||||
MEDIA_ALLOW_REDIRECTS = True
|
||||
|
||||
|
|
@ -551,9 +547,10 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
|
||||
.. method:: FilesPipeline.get_media_requests(item, info)
|
||||
|
||||
As seen on the workflow, the pipeline will get the requests for the files
|
||||
to download from the item by calling this method. You can override it to
|
||||
change what requests are returned:
|
||||
As seen on the workflow, the pipeline will get the URLs of the images to
|
||||
download from the item. In order to do this, you can override the
|
||||
:meth:`~get_media_requests` method and return a Request for each
|
||||
file URL:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
|
|
@ -593,9 +590,8 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
* ``downloaded`` - file was downloaded.
|
||||
* ``uptodate`` - file was not downloaded, as it was downloaded recently,
|
||||
according to the file expiration policy.
|
||||
* ``cached`` - file was taken from a cache (the response has a
|
||||
``"cached"`` flag, e.g. from
|
||||
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
|
||||
* ``cached`` - file was already scheduled for download, by another item
|
||||
sharing the same file.
|
||||
|
||||
The list of tuples received by :meth:`~item_completed` is
|
||||
guaranteed to retain the same order of the requests returned from the
|
||||
|
|
@ -622,6 +618,9 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
(False, Failure(...)),
|
||||
]
|
||||
|
||||
By default the :meth:`get_media_requests` method returns ``None`` which
|
||||
means there are no files to download for the item.
|
||||
|
||||
.. method:: FilesPipeline.item_completed(results, item, info)
|
||||
|
||||
The :meth:`FilesPipeline.item_completed` method called when all file
|
||||
|
|
@ -775,28 +774,4 @@ To enable your custom media pipeline component you must add its class import pat
|
|||
|
||||
ITEM_PIPELINES = {"myproject.pipelines.MyImagesPipeline": 300}
|
||||
|
||||
Content-based image filtering pipeline
|
||||
--------------------------------------
|
||||
|
||||
This example overrides ``get_images()`` to filter images using a classifier,
|
||||
such as a TensorFlow_ model. Override ``is_valid_image()`` with your
|
||||
classification logic:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy.pipelines.images import ImagesPipeline, ImageException
|
||||
|
||||
|
||||
class ImageClassifierPipeline(ImagesPipeline):
|
||||
def is_valid_image(self, image):
|
||||
raise NotImplementedError
|
||||
|
||||
def get_images(self, response, request, info, *, item=None):
|
||||
for path, image, buf in super().get_images(response, request, info, item=item):
|
||||
if not self.is_valid_image(image):
|
||||
raise ImageException("Image does not match criteria")
|
||||
yield path, image, buf
|
||||
|
||||
|
||||
.. _MD5 hash: https://en.wikipedia.org/wiki/MD5
|
||||
.. _TensorFlow: https://tensorflow.org
|
||||
|
|
|
|||
|
|
@ -17,10 +17,8 @@ Run Scrapy from a script
|
|||
You can use the :ref:`API <topics-api>` to run Scrapy from a script, instead of
|
||||
the typical way of running Scrapy via ``scrapy crawl``.
|
||||
|
||||
Remember that Scrapy requires a Twisted reactor or (with
|
||||
:setting:`TWISTED_REACTOR_ENABLED` set to ``False``) an asyncio event loop, so
|
||||
you need to run one of those in your script for it to work (helpers described
|
||||
below can do it for you).
|
||||
Remember that Scrapy is built on top of the Twisted
|
||||
asynchronous networking library, so you need to run it inside the Twisted reactor.
|
||||
|
||||
The first utility you can use to run your spiders is
|
||||
:class:`scrapy.crawler.AsyncCrawlerProcess` or
|
||||
|
|
@ -247,110 +245,6 @@ Using :func:`asyncio.run` with :class:`~scrapy.crawler.AsyncCrawlerRunner`:
|
|||
|
||||
asyncio.run(main())
|
||||
|
||||
.. _run-spiders-in-apps:
|
||||
|
||||
Running spiders inside existing applications
|
||||
============================================
|
||||
|
||||
You may want to run Scrapy spiders inside an existing application. In simple
|
||||
cases (e.g. task queues that spawn a process for every task, or applications
|
||||
that can execute tasks synchronously in the same process) you can use the same
|
||||
approach as for standalone scripts (see :ref:`run-from-script`). More complex
|
||||
cases, e.g. asynchronous web applications, have additional caveats and
|
||||
limitations.
|
||||
|
||||
If the application runs its own Twisted reactor, you can use
|
||||
:class:`~scrapy.crawler.AsyncCrawlerRunner` or
|
||||
:class:`~scrapy.crawler.CrawlerRunner` to run spiders using this reactor, see
|
||||
:ref:`run-from-script` for examples.
|
||||
|
||||
If the application doesn't run a Twisted reactor or an asyncio event loop (for
|
||||
example, a Django web app deployed with a WSGI server such as uWSGI), you can
|
||||
use :class:`~scrapy.crawler.AsyncCrawlerProcess` with
|
||||
:setting:`TWISTED_REACTOR_ENABLED` set to ``False``, so that Scrapy starts and
|
||||
stops an asyncio event loop for every spider run:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import scrapy
|
||||
from django.http import HttpResponse
|
||||
from scrapy.crawler import AsyncCrawlerProcess
|
||||
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
# Your spider definition
|
||||
...
|
||||
|
||||
|
||||
def crawl_view(request):
|
||||
process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
|
||||
process.crawl(MySpider)
|
||||
process.start() # returns when the spider finishes
|
||||
return HttpResponse("Crawling finished")
|
||||
|
||||
If the application runs its own asyncio event loop (for example, a Django web
|
||||
app deployed with an ASGI server such as uvicorn), you can use
|
||||
:class:`~scrapy.crawler.AsyncCrawlerRunner` with
|
||||
:setting:`TWISTED_REACTOR_ENABLED` set to ``False``, so that Scrapy uses the
|
||||
existing event loop:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import scrapy
|
||||
from django.http import HttpResponse
|
||||
from scrapy.crawler import AsyncCrawlerRunner
|
||||
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
# Your spider definition
|
||||
...
|
||||
|
||||
|
||||
async def crawl_view(request):
|
||||
runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False})
|
||||
await runner.crawl(MySpider) # completes when the spider finishes
|
||||
return HttpResponse("Crawling finished")
|
||||
|
||||
.. note:: Running Scrapy without a Twisted reactor is experimental and has
|
||||
some limitations, described in :ref:`asyncio-without-reactor`.
|
||||
|
||||
.. _run-in-notebook:
|
||||
|
||||
Running spiders in Jupyter notebooks
|
||||
====================================
|
||||
|
||||
You can run Scrapy spiders in Jupyter notebooks. You need to use
|
||||
:class:`~scrapy.crawler.AsyncCrawlerRunner` with
|
||||
:setting:`TWISTED_REACTOR_ENABLED` set to ``False`` for this, so that Scrapy
|
||||
uses the event loop provided by the notebook kernel. As
|
||||
:class:`~scrapy.crawler.AsyncCrawlerRunner` doesn't configure logging, and you
|
||||
most likely want to see the spider log in the notebook, you should call
|
||||
:func:`scrapy.utils.log.configure_logging`. Here is a full example, which
|
||||
supports rerunning both as a single cell and as separate cells:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy import Spider
|
||||
from scrapy.crawler import AsyncCrawlerRunner
|
||||
from scrapy.utils.log import configure_logging
|
||||
|
||||
configure_logging()
|
||||
|
||||
|
||||
class BooksSpider(Spider):
|
||||
name = "books"
|
||||
start_urls = ["https://books.toscrape.com"]
|
||||
|
||||
def parse(self, response):
|
||||
for book in response.css("h3"):
|
||||
yield {"title": book.css("a::attr(title)").get()}
|
||||
|
||||
|
||||
runner = AsyncCrawlerRunner({"TWISTED_REACTOR_ENABLED": False})
|
||||
await runner.crawl(BooksSpider)
|
||||
|
||||
.. note:: Running Scrapy without a Twisted reactor is experimental and has
|
||||
some limitations, described in :ref:`asyncio-without-reactor`.
|
||||
|
||||
.. _run-multiple-spiders:
|
||||
|
||||
|
|
@ -453,10 +347,10 @@ finishes before starting the next one:
|
|||
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
|
||||
react(deferred_f_from_coro_f(crawl))
|
||||
|
||||
.. note:: When running multiple spiders in the same process, :ref:`logging
|
||||
settings <logging-settings>` and :ref:`reactor settings <reactor-settings>`
|
||||
should not have a different value per spider, and :ref:`pre-crawler
|
||||
settings <pre-crawler-settings>` cannot be defined per spider.
|
||||
.. note:: When running multiple spiders in the same process, :ref:`reactor
|
||||
settings <reactor-settings>` should not have a different value per spider.
|
||||
Also, :ref:`pre-crawler settings <pre-crawler-settings>` cannot be defined
|
||||
per spider.
|
||||
|
||||
.. seealso:: :ref:`run-from-script`.
|
||||
|
||||
|
|
@ -493,26 +387,6 @@ crawl::
|
|||
curl http://scrapy2.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=2
|
||||
curl http://scrapy3.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=3
|
||||
|
||||
.. _large-project-startup:
|
||||
|
||||
Reducing startup time in large projects
|
||||
=======================================
|
||||
|
||||
When running a spider with ``scrapy crawl``, Scrapy loads all modules listed in
|
||||
:setting:`SPIDER_MODULES` to find the target spider. In large projects with
|
||||
many spiders, this can noticeably increase startup time and memory usage.
|
||||
|
||||
To avoid loading every spider module, override :setting:`SPIDER_MODULES` on the
|
||||
command line to point only to the module that contains the spider you want to
|
||||
run:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
scrapy crawl myspider -s SPIDER_MODULES=myproject.spiders.myspider
|
||||
|
||||
Because :setting:`SPIDER_MODULES` is a list setting, you can include multiple
|
||||
modules by separating them with commas.
|
||||
|
||||
.. _bans:
|
||||
|
||||
Avoiding getting banned
|
||||
|
|
@ -536,9 +410,9 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
|
|||
services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
|
||||
super proxy that you can attach your own proxies to.
|
||||
* for HTTPS websites, if blocking appears related to TLS behavior, consider
|
||||
adjusting the :setting:`DOWNLOAD_TLS_MIN_VERSION` and
|
||||
:setting:`DOWNLOAD_TLS_MAX_VERSION` settings, since some websites may respond
|
||||
differently depending on the TLS method used by the client.
|
||||
adjusting the :setting:`DOWNLOADER_CLIENT_TLS_METHOD` setting, since some
|
||||
websites may respond differently depending on the TLS method used by the
|
||||
client.
|
||||
* use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy
|
||||
plugin <https://github.com/scrapy-plugins/scrapy-zyte-api>`__ and additional
|
||||
features, like `AI web scraping <https://www.zyte.com/ai-web-scraping/>`__
|
||||
|
|
@ -546,14 +420,6 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
|
|||
If you are still unable to prevent your bot getting banned, consider contacting
|
||||
`commercial support`_.
|
||||
|
||||
.. _static-analysis:
|
||||
|
||||
Static analysis
|
||||
===============
|
||||
|
||||
Consider using :doc:`scrapy-lint <scrapy-lint:index>`, a linter for Scrapy
|
||||
projects that detects common mistakes and anti-patterns.
|
||||
|
||||
.. _Tor project: https://www.torproject.org/
|
||||
.. _commercial support: https://www.scrapy.org/companies
|
||||
.. _ProxyMesh: https://proxymesh.com/
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ Request objects
|
|||
|
||||
.. invisible-code-block: python
|
||||
|
||||
from scrapy import Request
|
||||
from scrapy.http import Request
|
||||
|
||||
1. Using a dict:
|
||||
|
||||
|
|
@ -117,9 +117,6 @@ Request objects
|
|||
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
|
||||
This encoding will be used to percent-encode the URL and to convert the
|
||||
body to bytes (if given as a string).
|
||||
|
||||
To disable URL percent-encoding for a request, use the
|
||||
:reqmeta:`verbatim_url` request meta key.
|
||||
:type encoding: str
|
||||
|
||||
:param priority: sets :attr:`priority`, defaults to ``0``.
|
||||
|
|
@ -139,13 +136,9 @@ Request objects
|
|||
|
||||
.. attribute:: Request.url
|
||||
|
||||
A string containing the URL of this request.
|
||||
|
||||
Keep in mind that this attribute contains the escaped URL, so it can
|
||||
differ from the URL passed in the ``__init__()`` method.
|
||||
|
||||
If :reqmeta:`verbatim_url` is set to ``True``, the URL is kept as
|
||||
passed to ``__init__()``.
|
||||
A string containing the URL of this request. Keep in mind that this
|
||||
attribute contains the escaped URL, so it can differ from the URL passed in
|
||||
the ``__init__()`` method.
|
||||
|
||||
This attribute is read-only. To change the URL of a Request use
|
||||
:meth:`replace`.
|
||||
|
|
@ -188,13 +181,6 @@ Request objects
|
|||
``failure.request.cb_kwargs`` in the request's errback. For more information,
|
||||
see :ref:`errback-cb_kwargs`.
|
||||
|
||||
.. note:: When :setting:`JOBDIR` is set, requests are serialized to disk
|
||||
with :mod:`pickle` (see :ref:`request-serialization`). As a result,
|
||||
the callback receives a deep copy of any object stored in
|
||||
``cb_kwargs``, so mutating such an object in the callback does not
|
||||
affect the original. Avoid relying on shared mutable state passed
|
||||
through ``cb_kwargs`` in that case.
|
||||
|
||||
.. attribute:: Request.meta
|
||||
:value: {}
|
||||
|
||||
|
|
@ -247,7 +233,7 @@ Request objects
|
|||
Return a new Request which is a copy of this Request. See also:
|
||||
:ref:`topics-request-response-ref-request-callback-arguments`.
|
||||
|
||||
.. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs, cls])
|
||||
.. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs])
|
||||
|
||||
Return a Request object with the same members, except for those members
|
||||
given new values by whichever keyword arguments are specified. The
|
||||
|
|
@ -436,7 +422,7 @@ errors if needed:
|
|||
)
|
||||
|
||||
def parse_httpbin(self, response):
|
||||
self.logger.info(f"Got successful response from {response.url}")
|
||||
self.logger.info("Got successful response from {}".format(response.url))
|
||||
# do something useful here...
|
||||
|
||||
def errback_httpbin(self, failure):
|
||||
|
|
@ -555,11 +541,6 @@ in your :meth:`fingerprint` method implementation:
|
|||
|
||||
.. autofunction:: scrapy.utils.request.fingerprint
|
||||
|
||||
By default, request fingerprinting canonicalizes the request URL. If
|
||||
:reqmeta:`verbatim_url` is set to ``True``, fingerprinting does not
|
||||
canonicalize the URL, and the ``keep_fragments`` parameter is ignored (it is
|
||||
effectively true).
|
||||
|
||||
For example, to take the value of a request header named ``X-ID`` into
|
||||
account:
|
||||
|
||||
|
|
@ -717,24 +698,18 @@ Those are:
|
|||
* :reqmeta:`download_fail_on_dataloss`
|
||||
* :reqmeta:`download_latency`
|
||||
* :reqmeta:`download_maxsize`
|
||||
* :reqmeta:`download_slot`
|
||||
* :reqmeta:`download_warnsize`
|
||||
* :reqmeta:`download_timeout`
|
||||
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
|
||||
* ``ftp_user`` (See :setting:`FTP_USER` for more info)
|
||||
* :reqmeta:`give_up_log_level`
|
||||
* :reqmeta:`handle_httpstatus_all`
|
||||
* :reqmeta:`handle_httpstatus_list`
|
||||
* :reqmeta:`http_auth_domain`
|
||||
* :reqmeta:`http_pass`
|
||||
* :reqmeta:`http_user`
|
||||
* :reqmeta:`is_start_request`
|
||||
* :reqmeta:`max_retry_times`
|
||||
* :reqmeta:`proxy`
|
||||
* :reqmeta:`redirect_reasons`
|
||||
* :reqmeta:`redirect_urls`
|
||||
* :reqmeta:`referrer_policy`
|
||||
* :reqmeta:`verbatim_url`
|
||||
|
||||
.. reqmeta:: bindaddress
|
||||
|
||||
|
|
@ -802,68 +777,15 @@ download_fail_on_dataloss
|
|||
Whether or not to fail on broken responses. See:
|
||||
:setting:`DOWNLOAD_FAIL_ON_DATALOSS`.
|
||||
|
||||
.. reqmeta:: give_up_log_level
|
||||
|
||||
give_up_log_level
|
||||
-----------------
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
:ref:`Logging level <levels>` used for the message logged when a request
|
||||
exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details.
|
||||
|
||||
.. reqmeta:: http_auth_domain
|
||||
|
||||
http_auth_domain
|
||||
----------------
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Overrides :setting:`HTTPAUTH_DOMAIN` for this request.
|
||||
|
||||
.. reqmeta:: http_pass
|
||||
|
||||
http_pass
|
||||
---------
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Overrides :setting:`HTTPAUTH_PASS` for this request.
|
||||
|
||||
.. reqmeta:: http_user
|
||||
|
||||
http_user
|
||||
---------
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Overrides :setting:`HTTPAUTH_USER` for this request.
|
||||
|
||||
.. reqmeta:: max_retry_times
|
||||
|
||||
max_retry_times
|
||||
---------------
|
||||
|
||||
The meta key is used set retry times per request. When set, the
|
||||
The meta key is used set retry times per request. When initialized, the
|
||||
:reqmeta:`max_retry_times` meta key takes higher precedence over the
|
||||
:setting:`RETRY_TIMES` setting.
|
||||
|
||||
.. reqmeta:: verbatim_url
|
||||
|
||||
verbatim_url
|
||||
------------
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Set this key to ``True`` to keep the request URL as passed to
|
||||
:class:`~scrapy.Request`, without URL percent-encoding.
|
||||
|
||||
When this key is enabled, :func:`~scrapy.utils.request.fingerprint` does not
|
||||
canonicalize the request URL, so requests whose URLs differ only in
|
||||
characters that would otherwise be canonicalized get different fingerprints.
|
||||
|
||||
In this mode, the ``keep_fragments`` parameter is ignored, and it is
|
||||
effectively true.
|
||||
|
||||
.. _topics-stop-response-download:
|
||||
|
||||
|
|
@ -921,11 +843,6 @@ Request subclasses
|
|||
Here is the list of built-in :class:`~scrapy.Request` subclasses. You can also subclass
|
||||
it to implement your own custom functionality.
|
||||
|
||||
FormRequest
|
||||
-----------
|
||||
|
||||
.. autoclass:: scrapy.FormRequest
|
||||
|
||||
JsonRequest
|
||||
-----------
|
||||
|
||||
|
|
@ -1025,16 +942,12 @@ Response objects
|
|||
|
||||
A dictionary-like (:class:`scrapy.http.headers.Headers`) object which contains
|
||||
the response headers. Values can be accessed using
|
||||
:meth:`~scrapy.http.headers.Headers.get` to return the last header value with
|
||||
:meth:`~scrapy.http.headers.Headers.get` to return the first header value with
|
||||
the specified name or :meth:`~scrapy.http.headers.Headers.getlist` to return
|
||||
all header values with the specified name. For example, this call will give you
|
||||
all cookies in the headers:
|
||||
all cookies in the headers::
|
||||
|
||||
.. skip: next
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
response.headers.getlist("Set-Cookie")
|
||||
response.headers.getlist('Set-Cookie')
|
||||
|
||||
.. attribute:: Response.body
|
||||
|
||||
|
|
@ -1090,7 +1003,7 @@ Response objects
|
|||
.. attribute:: Response.flags
|
||||
|
||||
A list that contains flags for this response. Flags are labels used for
|
||||
tagging Responses. For example: ``'cached'``, ``'redirected'``', etc. And
|
||||
tagging Responses. For example: ``'cached'``, ``'redirected``', etc. And
|
||||
they're shown on the string representation of the Response (``__str__()``
|
||||
method) which is used by the engine for logging.
|
||||
|
||||
|
|
@ -1105,8 +1018,8 @@ Response objects
|
|||
|
||||
The IP address of the server from which the Response originated.
|
||||
|
||||
This attribute is currently only populated by the HTTP download
|
||||
handlers, i.e. for ``http(s)`` responses. For other handlers,
|
||||
This attribute is currently only populated by the HTTP 1.1 download
|
||||
handler, i.e. for ``http(s)`` responses. For other handlers,
|
||||
:attr:`ip_address` is always ``None``.
|
||||
|
||||
.. attribute:: Response.protocol
|
||||
|
|
@ -1124,7 +1037,7 @@ Response objects
|
|||
|
||||
Returns a new Response which is a copy of this Response.
|
||||
|
||||
.. method:: Response.replace([url, status, headers, body, request, flags, certificate, ip_address, protocol, cls])
|
||||
.. method:: Response.replace([url, status, headers, body, request, flags, cls])
|
||||
|
||||
Returns a Response object with the same members, except for those members
|
||||
given new values by whichever keyword arguments are specified. The
|
||||
|
|
@ -1136,11 +1049,7 @@ Response objects
|
|||
a possible relative url.
|
||||
|
||||
This is a wrapper over :func:`~urllib.parse.urljoin`, it's merely an alias for
|
||||
making this call:
|
||||
|
||||
.. skip: next
|
||||
|
||||
.. code-block:: python
|
||||
making this call::
|
||||
|
||||
urllib.parse.urljoin(response.url, url)
|
||||
|
||||
|
|
@ -1229,31 +1138,21 @@ TextResponse objects
|
|||
|
||||
.. method:: TextResponse.jmespath(query)
|
||||
|
||||
.. skip: start
|
||||
A shortcut to ``TextResponse.selector.jmespath(query)``::
|
||||
|
||||
A shortcut to ``TextResponse.selector.jmespath(query)``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
response.jmespath("object.[*]")
|
||||
response.jmespath('object.[*]')
|
||||
|
||||
.. method:: TextResponse.xpath(query)
|
||||
|
||||
A shortcut to ``TextResponse.selector.xpath(query)``:
|
||||
A shortcut to ``TextResponse.selector.xpath(query)``::
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
response.xpath("//p")
|
||||
response.xpath('//p')
|
||||
|
||||
.. method:: TextResponse.css(query)
|
||||
|
||||
A shortcut to ``TextResponse.selector.css(query)``:
|
||||
A shortcut to ``TextResponse.selector.css(query)``::
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
response.css("p")
|
||||
|
||||
.. skip: end
|
||||
response.css('p')
|
||||
|
||||
.. automethod:: TextResponse.follow
|
||||
|
||||
|
|
|
|||
|
|
@ -1,207 +0,0 @@
|
|||
.. _security:
|
||||
|
||||
========
|
||||
Security
|
||||
========
|
||||
|
||||
Scrapy defaults are optimized for web scraping, not for the security posture
|
||||
that you might expect from software that handles untrusted input or runs in a
|
||||
shared or exposed environment. Some common security practices are unnecessary
|
||||
for many scraping use cases, and a few can even prevent valid ones (for
|
||||
example, sites that you must scrape may use misconfigured TLS certificates or
|
||||
serve content over unencrypted protocols).
|
||||
|
||||
This page highlights the Scrapy defaults that have security implications, so
|
||||
that you can make an informed decision about whether to keep them, and explains
|
||||
how to harden them along with the trade-offs involved.
|
||||
|
||||
.. note::
|
||||
|
||||
None of the options below are silver bullets. Which of them make sense
|
||||
depends on your threat model: whether the URLs you crawl come from trusted
|
||||
sources, whether the machine running Scrapy is exposed to a network you do
|
||||
not control, whether the data you handle is sensitive, and so on.
|
||||
|
||||
.. _security-untrusted-responses:
|
||||
|
||||
Treat responses as untrusted input
|
||||
==================================
|
||||
|
||||
Regardless of any setting, remember that response data comes from servers you
|
||||
do not control, even when you trust the site you are crawling, as responses may
|
||||
be tampered with in transit or the server itself may be compromised.
|
||||
|
||||
Never pass response data to functions that can execute code or otherwise act on
|
||||
their input in an unsafe way, such as :func:`eval`, :func:`exec`, or
|
||||
:func:`pickle.loads`, and be careful when writing response data to paths
|
||||
derived from the response itself.
|
||||
|
||||
TLS connections
|
||||
===============
|
||||
|
||||
.. _security-certificate-verification:
|
||||
|
||||
Certificate verification
|
||||
------------------------
|
||||
|
||||
By default Scrapy does **not** verify the TLS certificate of HTTPS servers, as
|
||||
controlled by the :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting (default:
|
||||
``False``).
|
||||
|
||||
This default favors reach over security: many sites that are otherwise fine to
|
||||
scrape have expired, self-signed, or otherwise invalid certificates, and
|
||||
verifying certificates would make requests to them fail.
|
||||
|
||||
If the integrity of the connection matters to you (for example, to detect
|
||||
man-in-the-middle attacks), set:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
DOWNLOAD_VERIFY_CERTIFICATES = True
|
||||
|
||||
* **Pro:** requests to servers with invalid or untrusted certificates fail
|
||||
instead of silently succeeding, protecting you from some man-in-the-middle
|
||||
attacks.
|
||||
|
||||
* **Con:** you can no longer scrape sites with misconfigured certificates
|
||||
without re-disabling verification for them.
|
||||
|
||||
.. _security-tls-protocols-ciphers:
|
||||
|
||||
Protocol versions and ciphers
|
||||
-----------------------------
|
||||
|
||||
You can restrict the TLS protocol versions that Scrapy accepts through the
|
||||
:setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION`
|
||||
settings, e.g. to reject obsolete protocol versions.
|
||||
|
||||
By default Scrapy uses the OpenSSL ``DEFAULT`` cipher list
|
||||
(:setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`), which favors compatibility and still
|
||||
allows some older, weaker ciphers. Set it to ``None`` to instead use the curated
|
||||
cipher list of the underlying TLS implementation (Twisted), which excludes weak
|
||||
ciphers:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
DOWNLOADER_CLIENT_TLS_CIPHERS = None
|
||||
|
||||
* **Pro:** connections that would negotiate a weak cipher fail instead of
|
||||
succeeding.
|
||||
|
||||
* **Con:** you can no longer connect to servers that only support the excluded
|
||||
ciphers.
|
||||
|
||||
.. _security-unencrypted-protocols:
|
||||
|
||||
Unencrypted protocols
|
||||
=====================
|
||||
|
||||
By default Scrapy enables download handlers for unencrypted protocols, namely
|
||||
``http://`` and ``ftp://`` (see :setting:`DOWNLOAD_HANDLERS_BASE`). Data sent
|
||||
and received over these protocols, including any credentials, travels in plain
|
||||
text and can be read or modified by anyone on the network path.
|
||||
|
||||
If you only crawl over encrypted protocols, you can disable the unencrypted
|
||||
ones so that no request can accidentally be sent unencrypted:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
DOWNLOAD_HANDLERS = {
|
||||
"http": None,
|
||||
"ftp": None,
|
||||
}
|
||||
|
||||
* **Pro:** a misconfigured or maliciously-redirected request cannot leak data
|
||||
over an unencrypted connection, as such requests fail instead.
|
||||
|
||||
* **Con:** you can no longer crawl resources that are only available over those
|
||||
protocols.
|
||||
|
||||
Note that disabling the ``http`` handler also prevents plain-HTTP requests that
|
||||
result from following an ``http://`` redirect or link, which is often the point
|
||||
of disabling it.
|
||||
|
||||
.. _security-local-resources:
|
||||
|
||||
Local and non-network resources
|
||||
===============================
|
||||
|
||||
By default Scrapy enables download handlers for the ``file://`` and ``data:``
|
||||
schemes (see :setting:`DOWNLOAD_HANDLERS_BASE`). The ``file://`` handler reads
|
||||
arbitrary files from the local filesystem, limited only by the permissions of
|
||||
the process running Scrapy.
|
||||
|
||||
This is convenient (for example, to parse a local HTML file), but it is a risk
|
||||
if any of the URLs you schedule come from an untrusted source: a crafted
|
||||
``file:///etc/passwd`` URL could read local files.
|
||||
|
||||
If you do not need them, disable these handlers:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
DOWNLOAD_HANDLERS = {
|
||||
"file": None,
|
||||
"data": None,
|
||||
}
|
||||
|
||||
* **Pro:** crawled URLs cannot be used to read local files or inline data.
|
||||
|
||||
* **Con:** you can no longer fetch ``file://`` or ``data:`` URLs.
|
||||
|
||||
More generally, if you crawl URLs from untrusted sources, consider validating
|
||||
their schemes (and, where applicable, their hosts) before scheduling requests,
|
||||
to avoid server-side request forgery (SSRF) and similar issues.
|
||||
|
||||
.. _security-telnet:
|
||||
|
||||
Telnet console
|
||||
==============
|
||||
|
||||
Scrapy enables the :ref:`telnet console <topics-telnetconsole>` by default
|
||||
(:setting:`TELNETCONSOLE_ENABLED`). The telnet console is a Python shell
|
||||
running inside the Scrapy process, so anyone who can connect to it can run
|
||||
arbitrary code in that process.
|
||||
|
||||
By default the console binds to ``127.0.0.1`` (:setting:`TELNETCONSOLE_HOST`)
|
||||
and is protected by a username (:setting:`TELNETCONSOLE_USERNAME`, default
|
||||
``scrapy``) and an automatically generated password
|
||||
(:setting:`TELNETCONSOLE_PASSWORD`), so it is only reachable from the local
|
||||
machine.
|
||||
|
||||
.. warning::
|
||||
|
||||
Telnet does not provide any transport-layer security, so the
|
||||
username/password authentication does not protect the credentials or the
|
||||
session from anyone able to observe the traffic. Never expose the telnet
|
||||
console over an untrusted network by changing :setting:`TELNETCONSOLE_HOST`
|
||||
to a non-local address.
|
||||
|
||||
If you do not use the telnet console, disable it entirely:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
TELNETCONSOLE_ENABLED = False
|
||||
|
||||
* **Pro:** removes a local code-execution surface and one less listening port.
|
||||
|
||||
* **Con:** you can no longer :ref:`inspect and control a running crawler
|
||||
<topics-telnetconsole>` through it.
|
||||
|
||||
.. _security-credential-leakage:
|
||||
|
||||
Credential leakage across domains
|
||||
=================================
|
||||
|
||||
Some Scrapy features attach credentials or other sensitive headers to requests,
|
||||
and a crawl that spans multiple domains can leak them to unintended hosts:
|
||||
|
||||
* HTTP authentication credentials set through
|
||||
:class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` are only
|
||||
sent to the domain set in :setting:`HTTPAUTH_DOMAIN`. Leave this set to the
|
||||
intended domain rather than ``None`` so that credentials are not sent to
|
||||
every domain you crawl.
|
||||
|
||||
* The ``Referer`` header may disclose the URLs you crawl to other sites. The
|
||||
default :setting:`REFERRER_POLICY` already avoids sending the referrer from
|
||||
HTTPS to HTTP, but you can tighten it further (for example, to
|
||||
``same-origin`` or ``no-referrer``) if needed.
|
||||
|
|
@ -308,7 +308,7 @@ Examples:
|
|||
|
||||
* ``*::text`` selects all descendant text nodes of the current selector context:
|
||||
|
||||
.. skip: next
|
||||
..skip: next
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> response.css("#images *::text").getall()
|
||||
|
|
@ -634,7 +634,8 @@ Example:
|
|||
.. code-block:: pycon
|
||||
|
||||
>>> from scrapy import Selector
|
||||
>>> sel = Selector(text="""
|
||||
>>> sel = Selector(
|
||||
... text="""
|
||||
... <ul class="list">
|
||||
... <li>1</li>
|
||||
... <li>2</li>
|
||||
|
|
@ -644,8 +645,8 @@ Example:
|
|||
... <li>4</li>
|
||||
... <li>5</li>
|
||||
... <li>6</li>
|
||||
... </ul>""")
|
||||
...
|
||||
... </ul>"""
|
||||
... )
|
||||
>>> xp = lambda x: sel.xpath(x).getall()
|
||||
|
||||
This gets all first ``<li>`` elements under whatever it is its parent:
|
||||
|
|
@ -947,9 +948,11 @@ with groups of itemscopes and corresponding itemprops:
|
|||
>>> sel = Selector(text=doc, type="html")
|
||||
>>> for scope in sel.xpath("//div[@itemscope]"):
|
||||
... print("current scope:", scope.xpath("@itemtype").getall())
|
||||
... props = scope.xpath("""
|
||||
... props = scope.xpath(
|
||||
... """
|
||||
... set:difference(./descendant::*/@itemprop,
|
||||
... .//*[@itemscope]/*/@itemprop)""")
|
||||
... .//*[@itemscope]/*/@itemprop)"""
|
||||
... )
|
||||
... print(f" properties: {props.getall()}")
|
||||
... print("")
|
||||
...
|
||||
|
|
|
|||
|
|
@ -361,31 +361,6 @@ All of these settings, except for :setting:`ASYNCIO_EVENT_LOOP`, are only used
|
|||
when the Twisted reactor is used, i.e. when :setting:`TWISTED_REACTOR_ENABLED`
|
||||
is ``True``.
|
||||
|
||||
.. _logging-settings:
|
||||
|
||||
Logging settings
|
||||
----------------
|
||||
|
||||
**Logging settings** are settings that configure the global root logging
|
||||
handler installed by :func:`~scrapy.utils.log.configure_logging`.
|
||||
|
||||
These settings can be defined from a spider. However, because only 1 root
|
||||
logging handler is active per process, these settings cannot use a different
|
||||
value per spider when :ref:`running multiple spiders in the same process
|
||||
<run-multiple-spiders>`.
|
||||
|
||||
These settings are:
|
||||
|
||||
- :setting:`LOG_DATEFORMAT`
|
||||
- :setting:`LOG_ENABLED`
|
||||
- :setting:`LOG_ENCODING`
|
||||
- :setting:`LOG_FILE`
|
||||
- :setting:`LOG_FILE_APPEND`
|
||||
- :setting:`LOG_FORMAT`
|
||||
- :setting:`LOG_LEVEL`
|
||||
- :setting:`LOG_SHORT_NAMES`
|
||||
- :setting:`LOG_STDOUT`
|
||||
|
||||
.. _topics-settings-ref:
|
||||
|
||||
Built-in settings reference
|
||||
|
|
@ -409,36 +384,6 @@ Default: ``{}``
|
|||
A dict containing paths to the add-ons enabled in your project and their
|
||||
priorities. For more information, see :ref:`topics-addons`.
|
||||
|
||||
.. setting:: ASYNCIO_EVENT_LOOP
|
||||
|
||||
ASYNCIO_EVENT_LOOP
|
||||
------------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Import path of a given ``asyncio`` event loop class.
|
||||
|
||||
If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) or when
|
||||
:ref:`running Scrapy without a reactor <asyncio-without-reactor>` this setting
|
||||
can be used to specify the
|
||||
asyncio event loop to be used with it. Set the setting to the import path of the
|
||||
desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
|
||||
event loop will be used.
|
||||
|
||||
If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor`
|
||||
function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
|
||||
class to be used.
|
||||
|
||||
Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`.
|
||||
|
||||
.. caution:: Please be aware that, when using a non-default event loop
|
||||
(either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with
|
||||
:func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call
|
||||
:func:`asyncio.set_event_loop`, which will set the specified event loop
|
||||
as the current loop for the current OS thread.
|
||||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
|
||||
.. setting:: AWS_ACCESS_KEY_ID
|
||||
|
||||
AWS_ACCESS_KEY_ID
|
||||
|
|
@ -449,24 +394,6 @@ Default: ``None``
|
|||
The AWS access key used by code that requires access to `Amazon Web services`_,
|
||||
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
|
||||
|
||||
.. setting:: AWS_ENDPOINT_URL
|
||||
|
||||
AWS_ENDPOINT_URL
|
||||
----------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Endpoint URL used for S3-like storage, for example Minio or s3.scality.
|
||||
|
||||
.. setting:: AWS_REGION_NAME
|
||||
|
||||
AWS_REGION_NAME
|
||||
---------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The name of the region associated with the AWS client.
|
||||
|
||||
.. setting:: AWS_SECRET_ACCESS_KEY
|
||||
|
||||
AWS_SECRET_ACCESS_KEY
|
||||
|
|
@ -490,6 +417,15 @@ such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`, when using
|
|||
|
||||
.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html
|
||||
|
||||
.. setting:: AWS_ENDPOINT_URL
|
||||
|
||||
AWS_ENDPOINT_URL
|
||||
----------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Endpoint URL used for S3-like storage, for example Minio or s3.scality.
|
||||
|
||||
.. setting:: AWS_USE_SSL
|
||||
|
||||
AWS_USE_SSL
|
||||
|
|
@ -510,6 +446,41 @@ Default: ``None``
|
|||
Verify SSL connection between Scrapy and S3 or S3-like storage. By default
|
||||
SSL verification will occur.
|
||||
|
||||
.. setting:: AWS_REGION_NAME
|
||||
|
||||
AWS_REGION_NAME
|
||||
---------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The name of the region associated with the AWS client.
|
||||
|
||||
.. setting:: ASYNCIO_EVENT_LOOP
|
||||
|
||||
ASYNCIO_EVENT_LOOP
|
||||
------------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Import path of a given ``asyncio`` event loop class.
|
||||
|
||||
If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the
|
||||
asyncio event loop to be used with it. Set the setting to the import path of the
|
||||
desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
|
||||
event loop will be used.
|
||||
|
||||
If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor`
|
||||
function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
|
||||
class to be used.
|
||||
|
||||
Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`.
|
||||
|
||||
.. caution:: Please be aware that, when using a non-default event loop
|
||||
(either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with
|
||||
:func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call
|
||||
:func:`asyncio.set_event_loop`, which will set the specified event loop
|
||||
as the current loop for the current OS thread.
|
||||
|
||||
.. setting:: BOT_NAME
|
||||
|
||||
BOT_NAME
|
||||
|
|
@ -556,8 +527,6 @@ performed to any single domain.
|
|||
See also: :ref:`topics-autothrottle` and its
|
||||
:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` option.
|
||||
|
||||
It is possible to change this setting per domain by using
|
||||
:setting:`DOWNLOAD_SLOTS`.
|
||||
|
||||
.. setting:: DEFAULT_DROPITEM_LOG_LEVEL
|
||||
|
||||
|
|
@ -597,7 +566,7 @@ When writing an item pipeline, you can force a different log level by setting
|
|||
DEFAULT_ITEM_CLASS
|
||||
------------------
|
||||
|
||||
Default: ``'scrapy.item.Item'``
|
||||
Default: ``'scrapy.Item'``
|
||||
|
||||
The default class that will be used for instantiating items in the :ref:`the
|
||||
Scrapy shell <topics-shell>`.
|
||||
|
|
@ -691,9 +660,7 @@ Whether to enable DNS in-memory cache.
|
|||
:class:`~scrapy.resolver.CachingThreadedResolver` and
|
||||
:class:`~scrapy.resolver.CachingHostnameResolver`. It has no effect when
|
||||
:setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect
|
||||
either when :setting:`TWISTED_DNS_RESOLVER` is set to a different resolver.
|
||||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
either when :setting:`DNS_RESOLVER` is set to a different resolver.
|
||||
|
||||
.. setting:: DNSCACHE_SIZE
|
||||
|
||||
|
|
@ -704,7 +671,22 @@ Default: ``10000``
|
|||
|
||||
DNS in-memory cache size, see :setting:`DNSCACHE_ENABLED`.
|
||||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
.. setting:: TWISTED_DNS_RESOLVER
|
||||
|
||||
TWISTED_DNS_RESOLVER
|
||||
--------------------
|
||||
|
||||
Default: ``'scrapy.resolver.CachingThreadedResolver'``
|
||||
|
||||
The class to be used by Twisted to resolve DNS names. The default
|
||||
``scrapy.resolver.CachingThreadedResolver`` supports specifying a timeout for
|
||||
DNS requests via the :setting:`DNS_TIMEOUT` setting, but works only with IPv4
|
||||
addresses. Scrapy provides an alternative resolver,
|
||||
``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not
|
||||
take the :setting:`DNS_TIMEOUT` setting into account.
|
||||
|
||||
.. note::
|
||||
This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
||||
|
||||
.. setting:: DNS_TIMEOUT
|
||||
|
||||
|
|
@ -719,9 +701,7 @@ Timeout for processing of DNS queries in seconds. Float is supported.
|
|||
This setting is only used by
|
||||
:class:`~scrapy.resolver.CachingThreadedResolver`. It has no effect when
|
||||
:setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect
|
||||
either when :setting:`TWISTED_DNS_RESOLVER` is set to a different resolver.
|
||||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
either when :setting:`DNS_RESOLVER` is set to a different resolver.
|
||||
|
||||
.. setting:: DOWNLOADER
|
||||
|
||||
|
|
@ -748,74 +728,41 @@ necessary to access certain HTTPS websites: for example, you may need to use
|
|||
``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a
|
||||
specific cipher that is not included in ``DEFAULT`` if a website requires it.
|
||||
|
||||
Set this setting to ``None`` to use the default ciphers of the underlying TLS
|
||||
implementation.
|
||||
|
||||
.. versionchanged:: 2.17.0
|
||||
Added support for setting this to ``None``.
|
||||
|
||||
.. _OpenSSL cipher list format: https://docs.openssl.org/master/man1/openssl-ciphers/#cipher-list-format
|
||||
|
||||
.. note::
|
||||
|
||||
Handling of this setting needs to be implemented inside the :ref:`download
|
||||
handler <topics-download-handlers>`, so it's not guaranteed to be supported
|
||||
by all 3rd-party handlers.
|
||||
by all 3rd-party handlers. It's currently unsupported by
|
||||
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.
|
||||
|
||||
.. seealso:: :ref:`security-tls-protocols-ciphers`
|
||||
.. setting:: DOWNLOADER_CLIENT_TLS_METHOD
|
||||
|
||||
.. setting:: DOWNLOAD_TLS_MAX_VERSION
|
||||
DOWNLOADER_CLIENT_TLS_METHOD
|
||||
----------------------------
|
||||
|
||||
DOWNLOAD_TLS_MAX_VERSION
|
||||
------------------------
|
||||
Default: ``'TLS'``
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
Use this setting to customize the TLS/SSL method used by the HTTPS download
|
||||
handler.
|
||||
|
||||
Default: ``None``
|
||||
This setting must be one of these string values:
|
||||
|
||||
Use this setting to change the maximum version of the TLS protocol allowed to
|
||||
be used by Scrapy.
|
||||
|
||||
This setting must be either ``None``, in which case it doesn't affect the
|
||||
version selection, or one of these string values:
|
||||
|
||||
- ``'TLSv1.0'``
|
||||
- ``'TLSv1.1'``
|
||||
- ``'TLSv1.2'``
|
||||
- ``'TLSv1.3'``
|
||||
|
||||
The range of allowed TLS versions advertised by Scrapy when making TLS
|
||||
connections will depend on the TLS implementation defaults and the values of
|
||||
:setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION`.
|
||||
It's possible to re-enable versions that are supported by the TLS
|
||||
implementation but disabled by default by adjusting these settings, but it's
|
||||
impossible to enable unsupported ones, such as any versions below 1.2 in many
|
||||
modern environments.
|
||||
- ``'TLS'``: maps to OpenSSL's ``TLS_method()`` (a.k.a ``SSLv23_method()``),
|
||||
which allows protocol negotiation, starting from the highest supported
|
||||
by the platform; **default, recommended**
|
||||
- ``'TLSv1.0'``: this value forces HTTPS connections to use TLS version 1.0 ;
|
||||
set this if you want the behavior of Scrapy<1.1
|
||||
- ``'TLSv1.1'``: forces TLS version 1.1
|
||||
- ``'TLSv1.2'``: forces TLS version 1.2
|
||||
|
||||
.. note::
|
||||
|
||||
Handling of this setting needs to be implemented inside the :ref:`download
|
||||
handler <topics-download-handlers>`, so it's not guaranteed to be supported
|
||||
by all 3rd-party handlers. Additionally, the set of supported TLS versions
|
||||
depends on the TLS implementation being used by the handler.
|
||||
|
||||
.. seealso:: :ref:`security-tls-protocols-ciphers`
|
||||
|
||||
.. setting:: DOWNLOAD_TLS_MIN_VERSION
|
||||
|
||||
DOWNLOAD_TLS_MIN_VERSION
|
||||
------------------------
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Use this setting to change the minimum version of the TLS protocol allowed to
|
||||
be used by Scrapy.
|
||||
|
||||
See :setting:`DOWNLOAD_TLS_MAX_VERSION` for the details and limitations.
|
||||
|
||||
.. seealso:: :ref:`security-tls-protocols-ciphers`
|
||||
by all 3rd-party handlers. It's currently unsupported by
|
||||
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.
|
||||
|
||||
.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
|
||||
|
||||
|
|
@ -899,9 +846,7 @@ Use :setting:`DOWNLOAD_DELAY` to throttle your crawling speed, to avoid hitting
|
|||
servers too hard.
|
||||
|
||||
Decimal numbers are supported. For example, to send a maximum of 4 requests
|
||||
every 10 seconds:
|
||||
|
||||
.. code-block:: python
|
||||
every 10 seconds::
|
||||
|
||||
DOWNLOAD_DELAY = 2.5
|
||||
|
||||
|
|
@ -923,8 +868,9 @@ desired.
|
|||
|
||||
This delay can be set per spider using :attr:`download_delay` spider attribute.
|
||||
|
||||
It is possible to change this setting per domain by using
|
||||
:setting:`DOWNLOAD_SLOTS`.
|
||||
It is also possible to change this setting per domain, although it requires
|
||||
non-trivial code. See the implementation of the :ref:`AutoThrottle
|
||||
<topics-autothrottle>` extension for an example.
|
||||
|
||||
.. setting:: DOWNLOAD_BIND_ADDRESS
|
||||
|
||||
|
|
@ -978,9 +924,6 @@ enabled in your project.
|
|||
|
||||
See :setting:`DOWNLOAD_HANDLERS_BASE` for example format.
|
||||
|
||||
.. seealso:: :ref:`security-unencrypted-protocols` and
|
||||
:ref:`security-local-resources`
|
||||
|
||||
.. setting:: DOWNLOAD_HANDLERS_BASE
|
||||
|
||||
DOWNLOAD_HANDLERS_BASE
|
||||
|
|
@ -1028,9 +971,6 @@ handler (without replacement), place this in your ``settings.py``:
|
|||
"ftp": None,
|
||||
}
|
||||
|
||||
.. seealso:: :ref:`security-unencrypted-protocols` and
|
||||
:ref:`security-local-resources`
|
||||
|
||||
|
||||
.. setting:: DOWNLOAD_SLOTS
|
||||
|
||||
|
|
@ -1189,8 +1129,6 @@ when making a request and abort the request if the verification fails.
|
|||
certificate problems are logged when this setting is set to ``False``)
|
||||
depends on its implementation.
|
||||
|
||||
.. seealso:: :ref:`security-certificate-verification`
|
||||
|
||||
.. setting:: DUPEFILTER_CLASS
|
||||
|
||||
DUPEFILTER_CLASS
|
||||
|
|
@ -1233,9 +1171,7 @@ the ``dont_filter`` parameter to ``True`` on the ``__init__`` method of a
|
|||
specific :class:`~scrapy.Request` object that should not be filtered out.
|
||||
|
||||
A class assigned to :setting:`DUPEFILTER_CLASS` must implement the following
|
||||
interface:
|
||||
|
||||
.. code-block:: python
|
||||
interface::
|
||||
|
||||
class MyDupeFilter:
|
||||
|
||||
|
|
@ -1318,7 +1254,6 @@ Default:
|
|||
|
||||
{
|
||||
"scrapy.extensions.corestats.CoreStats": 0,
|
||||
"scrapy.extensions.logcount.LogCount": 0,
|
||||
"scrapy.extensions.telnet.TelnetConsole": 0,
|
||||
"scrapy.extensions.memusage.MemoryUsage": 0,
|
||||
"scrapy.extensions.memdebug.MemoryDebugger": 0,
|
||||
|
|
@ -1341,8 +1276,6 @@ and the :ref:`list of available extensions <topics-extensions-ref>`.
|
|||
FEED_TEMPDIR
|
||||
------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The Feed Temp dir allows you to set a custom folder to save crawler
|
||||
temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
|
||||
:ref:`Amazon S3 <topics-feed-storage-s3>`.
|
||||
|
|
@ -1352,8 +1285,6 @@ temporary files before uploading with :ref:`FTP feed storage <topics-feed-storag
|
|||
FEED_STORAGE_GCS_ACL
|
||||
--------------------
|
||||
|
||||
Default: ``""``
|
||||
|
||||
The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage <topics-feed-storage-gcs>`.
|
||||
For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation <https://docs.cloud.google.com/storage/docs/access-control/lists>`_.
|
||||
|
||||
|
|
@ -1365,19 +1296,14 @@ FORCE_CRAWLER_PROCESS
|
|||
Default: ``False``
|
||||
|
||||
If ``False``, :ref:`Scrapy commands that need a CrawlerProcess
|
||||
<topics-commands-crawlerprocess>`, when :setting:`TWISTED_REACTOR_ENABLED`
|
||||
is set to ``True``, will decide between using
|
||||
<topics-commands-crawlerprocess>` will decide between using
|
||||
:class:`scrapy.crawler.AsyncCrawlerProcess` and
|
||||
:class:`scrapy.crawler.CrawlerProcess` based on the value of the
|
||||
:setting:`TWISTED_REACTOR` setting, but ignoring its value in :ref:`per-spider
|
||||
settings <spider-settings>`.
|
||||
|
||||
If ``True``, these commands will always use
|
||||
:class:`~scrapy.crawler.CrawlerProcess` when :setting:`TWISTED_REACTOR_ENABLED`
|
||||
is set to ``True``.
|
||||
|
||||
When :setting:`TWISTED_REACTOR_ENABLED` is set to ``False``,
|
||||
:class:`~scrapy.crawler.AsyncCrawlerProcess` will be used in all cases.
|
||||
:class:`~scrapy.crawler.CrawlerProcess`.
|
||||
|
||||
Set this to ``True`` if you want to set :setting:`TWISTED_REACTOR` to a
|
||||
non-default value in :ref:`per-spider settings <spider-settings>`.
|
||||
|
|
@ -1499,8 +1425,6 @@ Default: ``True``
|
|||
|
||||
Whether to enable logging.
|
||||
|
||||
.. note:: This is a :ref:`logging setting <logging-settings>`.
|
||||
|
||||
.. setting:: LOG_ENCODING
|
||||
|
||||
LOG_ENCODING
|
||||
|
|
@ -1510,8 +1434,6 @@ Default: ``'utf-8'``
|
|||
|
||||
The encoding to use for logging.
|
||||
|
||||
.. note:: This is a :ref:`logging setting <logging-settings>`.
|
||||
|
||||
.. setting:: LOG_FILE
|
||||
|
||||
LOG_FILE
|
||||
|
|
@ -1521,8 +1443,6 @@ Default: ``None``
|
|||
|
||||
File name to use for logging output. If ``None``, standard error will be used.
|
||||
|
||||
.. note:: This is a :ref:`logging setting <logging-settings>`.
|
||||
|
||||
.. setting:: LOG_FILE_APPEND
|
||||
|
||||
LOG_FILE_APPEND
|
||||
|
|
@ -1533,8 +1453,6 @@ Default: ``True``
|
|||
If ``False``, the log file specified with :setting:`LOG_FILE` will be
|
||||
overwritten (discarding the output from previous runs, if any).
|
||||
|
||||
.. note:: This is a :ref:`logging setting <logging-settings>`.
|
||||
|
||||
.. setting:: LOG_FORMAT
|
||||
|
||||
LOG_FORMAT
|
||||
|
|
@ -1546,8 +1464,6 @@ String for formatting log messages. Refer to the
|
|||
:ref:`Python logging documentation <logrecord-attributes>` for the whole
|
||||
list of available placeholders.
|
||||
|
||||
.. note:: This is a :ref:`logging setting <logging-settings>`.
|
||||
|
||||
.. setting:: LOG_DATEFORMAT
|
||||
|
||||
LOG_DATEFORMAT
|
||||
|
|
@ -1560,8 +1476,6 @@ in :setting:`LOG_FORMAT`. Refer to the
|
|||
:ref:`Python datetime documentation <strftime-strptime-behavior>` for the
|
||||
whole list of available directives.
|
||||
|
||||
.. note:: This is a :ref:`logging setting <logging-settings>`.
|
||||
|
||||
.. setting:: LOG_FORMATTER
|
||||
|
||||
LOG_FORMATTER
|
||||
|
|
@ -1581,8 +1495,6 @@ Default: ``'DEBUG'``
|
|||
Minimum level to log. Available levels are: CRITICAL, ERROR, WARNING,
|
||||
INFO, DEBUG. For more info see :ref:`topics-logging`.
|
||||
|
||||
.. note:: This is a :ref:`logging setting <logging-settings>`.
|
||||
|
||||
.. setting:: LOG_STDOUT
|
||||
|
||||
LOG_STDOUT
|
||||
|
|
@ -1594,8 +1506,6 @@ If ``True``, all standard output (and error) of your process will be redirected
|
|||
to the log. For example if you ``print('hello')`` it will appear in the Scrapy
|
||||
log.
|
||||
|
||||
.. note:: This is a :ref:`logging setting <logging-settings>`.
|
||||
|
||||
.. setting:: LOG_SHORT_NAMES
|
||||
|
||||
LOG_SHORT_NAMES
|
||||
|
|
@ -1606,8 +1516,6 @@ Default: ``False``
|
|||
If ``True``, the logs will just contain the root path. If it is set to ``False``
|
||||
then it displays the component responsible for the log output
|
||||
|
||||
.. note:: This is a :ref:`logging setting <logging-settings>`.
|
||||
|
||||
.. setting:: LOG_VERSIONS
|
||||
|
||||
LOG_VERSIONS
|
||||
|
|
@ -1627,8 +1535,6 @@ The following special items are also supported:
|
|||
|
||||
- ``Python``
|
||||
|
||||
- ``pyOpenSSL``
|
||||
|
||||
.. setting:: LOGSTATS_INTERVAL
|
||||
|
||||
LOGSTATS_INTERVAL
|
||||
|
|
@ -1648,6 +1554,21 @@ Default: ``False``
|
|||
|
||||
Whether to enable memory debugging.
|
||||
|
||||
.. setting:: MEMDEBUG_NOTIFY
|
||||
|
||||
MEMDEBUG_NOTIFY
|
||||
---------------
|
||||
|
||||
Default: ``[]``
|
||||
|
||||
When memory debugging is enabled a memory report will be sent to the specified
|
||||
addresses if this setting is not empty, otherwise the report will be written to
|
||||
the log.
|
||||
|
||||
Example::
|
||||
|
||||
MEMDEBUG_NOTIFY = ['user@example.com']
|
||||
|
||||
.. setting:: MEMUSAGE_ENABLED
|
||||
|
||||
MEMUSAGE_ENABLED
|
||||
|
|
@ -1721,11 +1642,9 @@ Default: ``"<project name>.spiders"`` (:ref:`fallback <default-settings>`: ``""`
|
|||
|
||||
Module where to create new spiders using the :command:`genspider` command.
|
||||
|
||||
Example:
|
||||
Example::
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
NEWSPIDER_MODULE = "mybot.spiders_dev"
|
||||
NEWSPIDER_MODULE = 'mybot.spiders_dev'
|
||||
|
||||
.. setting:: RANDOMIZE_DOWNLOAD_DELAY
|
||||
|
||||
|
|
@ -1743,10 +1662,7 @@ significant similarities in the time between their requests.
|
|||
|
||||
The randomization policy is the same used by `wget`_ ``--random-wait`` option.
|
||||
|
||||
If :setting:`DOWNLOAD_DELAY` is zero this option has no effect.
|
||||
|
||||
It is possible to change this setting per domain by using
|
||||
:setting:`DOWNLOAD_SLOTS`.
|
||||
If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect.
|
||||
|
||||
.. _wget: https://www.gnu.org/software/wget/manual/wget.html
|
||||
|
||||
|
|
@ -1762,8 +1678,6 @@ multi-purpose thread pool used by various Scrapy components. Threaded
|
|||
DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase
|
||||
this value if you're experiencing problems with insufficient blocking IO.
|
||||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
|
||||
.. setting:: REDIRECT_PRIORITY_ADJUST
|
||||
|
||||
REDIRECT_PRIORITY_ADJUST
|
||||
|
|
@ -1807,7 +1721,7 @@ The parser backend to use for parsing ``robots.txt`` files. For more information
|
|||
.. setting:: ROBOTSTXT_USER_AGENT
|
||||
|
||||
ROBOTSTXT_USER_AGENT
|
||||
--------------------
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Default: ``None``
|
||||
|
||||
|
|
@ -1962,8 +1876,6 @@ Default:
|
|||
|
||||
{
|
||||
"scrapy.contracts.default.UrlContract": 1,
|
||||
"scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,
|
||||
"scrapy.contracts.default.MetadataContract": 1,
|
||||
"scrapy.contracts.default.ReturnsContract": 2,
|
||||
"scrapy.contracts.default.ScrapesContract": 3,
|
||||
}
|
||||
|
|
@ -1992,8 +1904,6 @@ Default: ``'scrapy.spiderloader.SpiderLoader'``
|
|||
The class that will be used for loading spiders, which must implement the
|
||||
:ref:`topics-api-spiderloader`.
|
||||
|
||||
.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
|
||||
|
||||
.. setting:: SPIDER_LOADER_WARN_ONLY
|
||||
|
||||
SPIDER_LOADER_WARN_ONLY
|
||||
|
|
@ -2006,8 +1916,6 @@ it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception
|
|||
But you can choose to silence this exception and turn it into a simple
|
||||
warning by setting ``SPIDER_LOADER_WARN_ONLY = True``.
|
||||
|
||||
.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
|
||||
|
||||
.. setting:: SPIDER_MIDDLEWARES
|
||||
|
||||
SPIDER_MIDDLEWARES
|
||||
|
|
@ -2028,7 +1936,6 @@ Default:
|
|||
.. code-block:: python
|
||||
|
||||
{
|
||||
"scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25,
|
||||
"scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50,
|
||||
"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
|
||||
"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
|
||||
|
|
@ -2054,8 +1961,6 @@ Example:
|
|||
|
||||
SPIDER_MODULES = ["mybot.spiders_prod", "mybot.spiders_dev"]
|
||||
|
||||
.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
|
||||
|
||||
.. setting:: STATS_CLASS
|
||||
|
||||
STATS_CLASS
|
||||
|
|
@ -2088,8 +1993,6 @@ Default: ``True`` (``False`` when :setting:`TWISTED_REACTOR_ENABLED` is ``False`
|
|||
A boolean which specifies if the :ref:`telnet console <topics-telnetconsole>`
|
||||
will be enabled (provided its extension is also enabled).
|
||||
|
||||
.. seealso:: :ref:`security-telnet`
|
||||
|
||||
.. setting:: TEMPLATES_DIR
|
||||
|
||||
TEMPLATES_DIR
|
||||
|
|
@ -2104,25 +2007,6 @@ command.
|
|||
The project name must not conflict with the name of custom files or directories
|
||||
in the ``project`` subdirectory.
|
||||
|
||||
.. setting:: TWISTED_DNS_RESOLVER
|
||||
|
||||
TWISTED_DNS_RESOLVER
|
||||
--------------------
|
||||
|
||||
Default: ``'scrapy.resolver.CachingThreadedResolver'``
|
||||
|
||||
The class to be used by Twisted to resolve DNS names. The default
|
||||
``scrapy.resolver.CachingThreadedResolver`` supports specifying a timeout for
|
||||
DNS requests via the :setting:`DNS_TIMEOUT` setting, but works only with IPv4
|
||||
addresses. Scrapy provides an alternative resolver,
|
||||
``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not
|
||||
take the :setting:`DNS_TIMEOUT` setting into account.
|
||||
|
||||
.. note::
|
||||
This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
||||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
|
||||
.. setting:: TWISTED_REACTOR_ENABLED
|
||||
|
||||
TWISTED_REACTOR_ENABLED
|
||||
|
|
@ -2148,7 +2032,7 @@ stopped) will not apply. This mode is currently experimental and may not be
|
|||
suitable for production use. It may also not be supported by 3rd-party code.
|
||||
See :ref:`asyncio-without-reactor` for more information about this mode.
|
||||
|
||||
.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
|
||||
.. note:: This setting can't be set :ref:`per-spider <spider-settings>`.
|
||||
|
||||
.. versionadded:: 2.15.0
|
||||
|
||||
|
|
@ -2161,9 +2045,6 @@ Default: ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``
|
|||
|
||||
Import path of a given :mod:`~twisted.internet.reactor`.
|
||||
|
||||
.. note::
|
||||
This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
||||
|
||||
Scrapy will install this reactor if no other reactor is installed yet, such as
|
||||
when the ``scrapy`` CLI program is invoked or when using the
|
||||
:class:`~scrapy.crawler.AsyncCrawlerProcess` class or the
|
||||
|
|
@ -2200,7 +2081,7 @@ In order to use the reactor installed by Scrapy:
|
|||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.timeout = int(kwargs.pop("timeout", "60"))
|
||||
super().__init__(*args, **kwargs)
|
||||
super(QuotesSpider, self).__init__(*args, **kwargs)
|
||||
|
||||
async def start(self):
|
||||
reactor.callLater(self.timeout, self.stop)
|
||||
|
|
@ -2229,7 +2110,7 @@ which raises an exception, becomes:
|
|||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.timeout = int(kwargs.pop("timeout", "60"))
|
||||
super().__init__(*args, **kwargs)
|
||||
super(QuotesSpider, self).__init__(*args, **kwargs)
|
||||
|
||||
async def start(self):
|
||||
from twisted.internet import reactor
|
||||
|
|
@ -2258,7 +2139,6 @@ current platform.
|
|||
|
||||
For additional information, see :doc:`core/howto/choosing-reactor`.
|
||||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
|
||||
.. setting:: URLLENGTH_LIMIT
|
||||
|
||||
|
|
@ -2267,7 +2147,7 @@ URLLENGTH_LIMIT
|
|||
|
||||
Default: ``2083``
|
||||
|
||||
Scope: ``scrapy.spidermiddlewares.urllength``
|
||||
Scope: ``spidermiddlewares.urllength``
|
||||
|
||||
The maximum URL length to allow for crawled URLs.
|
||||
|
||||
|
|
|
|||
|
|
@ -34,9 +34,7 @@ is unavailable.
|
|||
Through Scrapy's settings you can configure it to use any one of
|
||||
``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which
|
||||
are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment
|
||||
variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`:
|
||||
|
||||
.. code-block:: ini
|
||||
variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
|
||||
|
||||
[settings]
|
||||
shell = bpython
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ Here is a simple example showing how you can catch signals and perform some acti
|
|||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
spider = super().from_crawler(crawler, *args, **kwargs)
|
||||
spider = super(DmozSpider, cls).from_crawler(crawler, *args, **kwargs)
|
||||
crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
|
||||
return spider
|
||||
|
||||
|
|
@ -60,8 +60,6 @@ Let's take an example using :ref:`coroutines <topics-coroutines>`:
|
|||
.. skip: next
|
||||
.. code-block:: python
|
||||
|
||||
import json
|
||||
|
||||
import scrapy
|
||||
import treq
|
||||
|
||||
|
|
@ -72,7 +70,7 @@ Let's take an example using :ref:`coroutines <topics-coroutines>`:
|
|||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
spider = super().from_crawler(crawler, *args, **kwargs)
|
||||
spider = super(SignalSpider, cls).from_crawler(crawler, *args, **kwargs)
|
||||
crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
|
||||
return spider
|
||||
|
||||
|
|
@ -454,7 +452,7 @@ bytes_received
|
|||
.. signal:: bytes_received
|
||||
.. function:: bytes_received(data, request, spider)
|
||||
|
||||
Sent by some download handlers when a group of bytes is
|
||||
Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is
|
||||
received for a specific request. This signal might be fired multiple
|
||||
times for the same request, with partial data each time. For instance,
|
||||
a possible scenario for a 25 kb response would be two signals fired
|
||||
|
|
@ -482,7 +480,7 @@ headers_received
|
|||
.. signal:: headers_received
|
||||
.. function:: headers_received(headers, body_length, request, spider)
|
||||
|
||||
Sent by some download handlers when the response headers are
|
||||
Sent by the HTTP 1.1 and S3 download handlers when the response headers are
|
||||
available for a given request, before downloading any additional content.
|
||||
|
||||
Handlers for this signal can stop the download of a response while it
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ previous (or subsequent) middleware being applied.
|
|||
If you want to disable a builtin middleware (the ones defined in
|
||||
:setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
|
||||
in your project :setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its
|
||||
value. For example, if you want to disable the referer middleware:
|
||||
value. For example, if you want to disable the off-site middleware:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
|
|
@ -355,8 +355,6 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
|
|||
using the special ``"referrer_policy"`` :ref:`Request.meta <topics-request-meta>` key,
|
||||
with the same acceptable values as for the ``REFERRER_POLICY`` setting.
|
||||
|
||||
.. seealso:: :ref:`security-credential-leakage`
|
||||
|
||||
Acceptable values for REFERRER_POLICY
|
||||
*************************************
|
||||
|
||||
|
|
|
|||
|
|
@ -208,6 +208,12 @@ scrapy.Spider
|
|||
:param response: the response to parse
|
||||
:type response: :class:`~scrapy.http.Response`
|
||||
|
||||
.. method:: log(message, [level, component])
|
||||
|
||||
Wrapper that sends a log message through the Spider's :attr:`logger`,
|
||||
kept for backward compatibility. For more information see
|
||||
:ref:`topics-logging-from-spiders`.
|
||||
|
||||
.. method:: closed(reason)
|
||||
|
||||
Called when the spider closes. This method provides a shortcut to
|
||||
|
|
@ -308,7 +314,7 @@ Spiders can access arguments in their `__init__` methods:
|
|||
name = "myspider"
|
||||
|
||||
def __init__(self, category=None, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
super(MySpider, self).__init__(*args, **kwargs)
|
||||
self.start_urls = [f"http://www.example.com/categories/{category}"]
|
||||
# ...
|
||||
|
||||
|
|
@ -329,8 +335,8 @@ The above example can also be written as follows:
|
|||
|
||||
If you are :ref:`running Scrapy from a script <run-from-script>`, you can
|
||||
specify spider arguments when calling
|
||||
:meth:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
|
||||
:meth:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`:
|
||||
:class:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
|
||||
:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`:
|
||||
|
||||
.. skip: next
|
||||
.. code-block:: python
|
||||
|
|
@ -348,6 +354,11 @@ Otherwise, you would cause iteration over a ``start_urls`` string
|
|||
(a very common python pitfall)
|
||||
resulting in each character being seen as a separate url.
|
||||
|
||||
A valid use case is to set the http auth credentials
|
||||
used by :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`::
|
||||
|
||||
scrapy crawl myspider -a http_user=myuser -a http_pass=mypassword
|
||||
|
||||
Spider arguments can also be passed through the Scrapyd ``schedule.json`` API.
|
||||
See `Scrapyd documentation`_.
|
||||
|
||||
|
|
@ -446,14 +457,13 @@ with a ``TestItem`` declared in a ``myproject.items`` module:
|
|||
|
||||
.. code-block:: python
|
||||
|
||||
from dataclasses import dataclass
|
||||
import scrapy
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestItem:
|
||||
id: str | None = None
|
||||
name: str | None = None
|
||||
description: str | None = None
|
||||
class TestItem(scrapy.Item):
|
||||
id = scrapy.Field()
|
||||
name = scrapy.Field()
|
||||
description = scrapy.Field()
|
||||
|
||||
|
||||
.. currentmodule:: scrapy.spiders
|
||||
|
|
@ -546,6 +556,7 @@ Let's now take a look at an example CrawlSpider with rules:
|
|||
|
||||
.. code-block:: python
|
||||
|
||||
import scrapy
|
||||
from scrapy.spiders import CrawlSpider, Rule
|
||||
from scrapy.linkextractors import LinkExtractor
|
||||
|
||||
|
|
@ -565,7 +576,7 @@ Let's now take a look at an example CrawlSpider with rules:
|
|||
|
||||
def parse_item(self, response):
|
||||
self.logger.info("Hi, this is an item page! %s", response.url)
|
||||
item = {}
|
||||
item = scrapy.Item()
|
||||
item["id"] = response.xpath('//td[@id="item_id"]/text()').re(r"ID: (\d+)")
|
||||
item["name"] = response.xpath('//td[@id="item_name"]/text()').get()
|
||||
item["description"] = response.xpath(
|
||||
|
|
@ -587,7 +598,7 @@ Let's now take a look at an example CrawlSpider with rules:
|
|||
This spider would start crawling example.com's home page, collecting category
|
||||
links, and item links, parsing the latter with the ``parse_item`` method. For
|
||||
each item response, some data will be extracted from the HTML using XPath, and
|
||||
a dictionary will be filled with it.
|
||||
an :class:`~scrapy.Item` will be filled with it.
|
||||
|
||||
XMLFeedSpider
|
||||
-------------
|
||||
|
|
@ -608,7 +619,7 @@ XMLFeedSpider
|
|||
|
||||
A string which defines the iterator to use. It can be either:
|
||||
|
||||
- ``'iternodes'`` - a fast iterator based on ``lxml``
|
||||
- ``'iternodes'`` - a fast iterator based on regular expressions
|
||||
|
||||
- ``'html'`` - an iterator which uses :class:`~scrapy.Selector`.
|
||||
Keep in mind this uses DOM parsing and must load all DOM in memory
|
||||
|
|
@ -622,11 +633,9 @@ XMLFeedSpider
|
|||
|
||||
.. attribute:: itertag
|
||||
|
||||
A string with the name of the node (or element) to iterate in. Example:
|
||||
A string with the name of the node (or element) to iterate in. Example::
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
itertag = "product"
|
||||
itertag = 'product'
|
||||
|
||||
.. attribute:: namespaces
|
||||
|
||||
|
|
@ -639,17 +648,12 @@ XMLFeedSpider
|
|||
You can then specify nodes with namespaces in the :attr:`itertag`
|
||||
attribute.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy.spiders import XMLFeedSpider
|
||||
|
||||
Example::
|
||||
|
||||
class YourSpider(XMLFeedSpider):
|
||||
|
||||
namespaces = [("n", "http://www.sitemaps.org/schemas/sitemap/0.9")]
|
||||
itertag = "n:url"
|
||||
namespaces = [('n', 'http://www.sitemaps.org/schemas/sitemap/0.9')]
|
||||
itertag = 'n:url'
|
||||
# ...
|
||||
|
||||
Apart from these new attributes, this spider has the following overridable
|
||||
|
|
@ -710,9 +714,9 @@ These spiders are pretty easy to use, let's have a look at one example:
|
|||
)
|
||||
|
||||
item = TestItem()
|
||||
item.id = node.xpath("@id").get()
|
||||
item.name = node.xpath("name").get()
|
||||
item.description = node.xpath("description").get()
|
||||
item["id"] = node.xpath("@id").get()
|
||||
item["name"] = node.xpath("name").get()
|
||||
item["description"] = node.xpath("description").get()
|
||||
return item
|
||||
|
||||
Basically what we did up there was to create a spider that downloads a feed from
|
||||
|
|
@ -774,9 +778,9 @@ Let's see an example similar to the previous one, but using a
|
|||
self.logger.info("Hi, this is a row!: %r", row)
|
||||
|
||||
item = TestItem()
|
||||
item.id = row["id"]
|
||||
item.name = row["name"]
|
||||
item.description = row["description"]
|
||||
item["id"] = row["id"]
|
||||
item["name"] = row["name"]
|
||||
item["description"] = row["description"]
|
||||
return item
|
||||
|
||||
|
||||
|
|
@ -809,11 +813,9 @@ SitemapSpider
|
|||
the regular expression. ``callback`` can be a string (indicating the
|
||||
name of a spider method) or a callable.
|
||||
|
||||
For example:
|
||||
For example::
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
sitemap_rules = [("/product/", "parse_product")]
|
||||
sitemap_rules = [('/product/', 'parse_product')]
|
||||
|
||||
Rules are applied in order, and only the first one that matches will be
|
||||
used.
|
||||
|
|
@ -835,9 +837,7 @@ SitemapSpider
|
|||
are links for the same website in another language passed within
|
||||
the same ``url`` block.
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block:: xml
|
||||
For example::
|
||||
|
||||
<url>
|
||||
<loc>http://example.com/</loc>
|
||||
|
|
@ -855,9 +855,7 @@ SitemapSpider
|
|||
This is a filter function that could be overridden to select sitemap entries
|
||||
based on their attributes.
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block:: xml
|
||||
For example::
|
||||
|
||||
<url>
|
||||
<loc>http://example.com/</loc>
|
||||
|
|
@ -960,7 +958,6 @@ Combine SitemapSpider with other sources of urls:
|
|||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy import Request
|
||||
from scrapy.spiders import SitemapSpider
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ Collector, and can be accessed through the :attr:`~scrapy.crawler.Crawler.stats`
|
|||
attribute of the :ref:`topics-api-crawler`, as illustrated by the examples in
|
||||
the :ref:`topics-stats-usecases` section below.
|
||||
|
||||
The Stats Collector API is always available, so you can always use it (to
|
||||
increment or set new stat keys), regardless
|
||||
However, the Stats Collector is always available, so you can always import it
|
||||
in your module and use its API (to increment or set new stat keys), regardless
|
||||
of whether the stats collection is enabled or not. If it's disabled, the API
|
||||
will still work but it won't collect anything. This is aimed at simplifying the
|
||||
stats collector usage: you should spend no more than one line of code for
|
||||
|
|
@ -21,6 +21,9 @@ using the Stats Collector from.
|
|||
Another feature of the Stats Collector is that it's very efficient (when
|
||||
enabled) and extremely efficient (almost unnoticeable) when disabled.
|
||||
|
||||
The Stats Collector keeps a stats table per open spider which is automatically
|
||||
opened when the spider is opened, and closed when the spider is closed.
|
||||
|
||||
.. _topics-stats-usecases:
|
||||
|
||||
Common Stats Collector uses
|
||||
|
|
@ -84,13 +87,13 @@ Get all stats:
|
|||
Available Stats Collectors
|
||||
==========================
|
||||
|
||||
.. currentmodule:: scrapy.statscollectors
|
||||
|
||||
Besides the basic :class:`StatsCollector` there are other Stats Collectors
|
||||
available in Scrapy which extend the basic Stats Collector. You can select
|
||||
which Stats Collector to use through the :setting:`STATS_CLASS` setting. The
|
||||
default Stats Collector used is the :class:`MemoryStatsCollector`.
|
||||
|
||||
.. currentmodule:: scrapy.statscollectors
|
||||
|
||||
MemoryStatsCollector
|
||||
--------------------
|
||||
|
||||
|
|
@ -99,7 +102,7 @@ MemoryStatsCollector
|
|||
A simple stats collector that keeps the stats of the last scraping run (for
|
||||
each spider) in memory, after they're closed. The stats can be accessed
|
||||
through the :attr:`spider_stats` attribute, which is a dict keyed by spider
|
||||
name.
|
||||
domain name.
|
||||
|
||||
This is the default Stats Collector used in Scrapy.
|
||||
|
||||
|
|
|
|||
|
|
@ -29,16 +29,14 @@ disable it if you want. For more information about the extension itself see
|
|||
.. note::
|
||||
This feature is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
||||
|
||||
.. seealso:: :ref:`security-telnet`
|
||||
|
||||
.. highlight:: none
|
||||
|
||||
How to access the telnet console
|
||||
================================
|
||||
|
||||
The telnet console listens on the first available TCP port from the range
|
||||
defined in the :setting:`TELNETCONSOLE_PORT` setting, which defaults to
|
||||
``[6023, 6073]``. To access the console you need to type::
|
||||
The telnet console listens in the TCP port defined in the
|
||||
:setting:`TELNETCONSOLE_PORT` setting, which defaults to ``6023``. To access
|
||||
the console you need to type::
|
||||
|
||||
telnet localhost 6023
|
||||
Trying localhost...
|
||||
|
|
@ -96,6 +94,8 @@ convenience:
|
|||
+----------------+-------------------------------------------------------------------+
|
||||
| ``p`` | a shortcut to the :func:`pprint.pprint` function |
|
||||
+----------------+-------------------------------------------------------------------+
|
||||
| ``hpy`` | for memory debugging (see :ref:`topics-leaks`) |
|
||||
+----------------+-------------------------------------------------------------------+
|
||||
|
||||
Telnet console usage examples
|
||||
=============================
|
||||
|
|
@ -107,8 +107,8 @@ Here are some example tasks you can do with the telnet console:
|
|||
View engine status
|
||||
------------------
|
||||
|
||||
You can use the ``est()`` method provided by the console to quickly show the
|
||||
engine status::
|
||||
You can use the ``est()`` method of the Scrapy engine to quickly show its state
|
||||
using the telnet console::
|
||||
|
||||
telnet localhost 6023
|
||||
>>> est()
|
||||
|
|
@ -192,8 +192,6 @@ Default: ``'127.0.0.1'``
|
|||
|
||||
The interface the telnet console should listen on
|
||||
|
||||
.. seealso:: :ref:`security-telnet`
|
||||
|
||||
|
||||
.. setting:: TELNETCONSOLE_USERNAME
|
||||
|
||||
|
|
|
|||
|
|
@ -39,8 +39,8 @@ API stability
|
|||
|
||||
API stability was one of the major goals for the *1.0* release.
|
||||
|
||||
Methods or functions that start with a single underscore (``_``) are private
|
||||
and should never be relied upon as stable.
|
||||
Methods or functions that start with a single dash (``_``) are private and
|
||||
should never be relied as stable.
|
||||
|
||||
Also, keep in mind that stable doesn't mean complete: stable APIs could grow
|
||||
new methods or functionality but the existing methods should keep working the
|
||||
|
|
|
|||
|
|
@ -94,82 +94,7 @@ untyped_calls_exclude = [
|
|||
[[tool.mypy.overrides]]
|
||||
module = "tests.*"
|
||||
allow_untyped_defs = true
|
||||
allow_incomplete_defs = true # 59 errors
|
||||
|
||||
# TODO
|
||||
[[tool.mypy.overrides]]
|
||||
module = [
|
||||
"tests.mockserver.*",
|
||||
"tests.spiders",
|
||||
"tests.test_closespider",
|
||||
"tests.test_cmdline",
|
||||
"tests.test_contracts",
|
||||
"tests.test_core_downloader",
|
||||
"tests.test_downloader_handler_twisted_ftp",
|
||||
"tests.test_downloadermiddleware_cookies",
|
||||
"tests.test_downloadermiddleware_httpauth",
|
||||
"tests.test_downloadermiddleware_httpcache",
|
||||
"tests.test_downloadermiddleware_httpcompression",
|
||||
"tests.test_downloadermiddleware_httpproxy",
|
||||
"tests.test_downloadermiddleware_offsite",
|
||||
"tests.test_downloadermiddleware_redirect",
|
||||
"tests.test_downloadermiddleware_redirect_base",
|
||||
"tests.test_downloadermiddleware_redirect_metarefresh",
|
||||
"tests.test_downloadermiddleware_retry",
|
||||
"tests.test_downloadermiddleware_robotstxt",
|
||||
"tests.test_downloadermiddleware_stats",
|
||||
"tests.test_downloaderslotssettings",
|
||||
"tests.test_dupefilters",
|
||||
"tests.test_engine_loop",
|
||||
"tests.test_exporters",
|
||||
"tests.test_extension_statsmailer",
|
||||
"tests.test_extension_throttle",
|
||||
"tests.test_feedexport",
|
||||
"tests.test_feedexport_postprocess",
|
||||
"tests.test_feedexport_storages",
|
||||
"tests.test_feedexport_uri_params",
|
||||
"tests.test_http2_client_protocol",
|
||||
"tests.test_http_headers",
|
||||
"tests.test_http_request",
|
||||
"tests.test_http_request_form",
|
||||
"tests.test_http_response",
|
||||
"tests.test_http_response_text",
|
||||
"tests.test_item",
|
||||
"tests.test_link",
|
||||
"tests.test_linkextractors",
|
||||
"tests.test_loader",
|
||||
"tests.test_logformatter",
|
||||
"tests.test_logstats",
|
||||
"tests.test_mail",
|
||||
"tests.test_pipeline_crawl",
|
||||
"tests.test_pipeline_files",
|
||||
"tests.test_pipeline_images",
|
||||
"tests.test_pipeline_media",
|
||||
"tests.test_pipelines",
|
||||
"tests.test_pqueues",
|
||||
"tests.test_request_attribute_binding",
|
||||
"tests.test_request_cb_kwargs",
|
||||
"tests.test_request_dict",
|
||||
"tests.test_request_left",
|
||||
"tests.test_robotstxt_interface",
|
||||
"tests.test_scheduler_base",
|
||||
"tests.test_settings",
|
||||
"tests.test_spider",
|
||||
"tests.test_spider_crawl",
|
||||
"tests.test_spidermiddleware_output_chain",
|
||||
"tests.test_spidermiddleware_process_start",
|
||||
"tests.test_spider_sitemap",
|
||||
"tests.test_squeues",
|
||||
"tests.test_squeues_request",
|
||||
"tests.test_stats",
|
||||
"tests.test_utils_datatypes",
|
||||
"tests.test_utils_decorators",
|
||||
"tests.test_utils_defer",
|
||||
"tests.test_utils_deprecate",
|
||||
"tests.test_utils_misc.test_return_with_argument_inside_generator",
|
||||
"tests.test_utils_python",
|
||||
"tests.test_utils_request",
|
||||
]
|
||||
allow_incomplete_defs = true # 48 errors
|
||||
check_untyped_defs = false
|
||||
|
||||
# Interface classes are hard to support
|
||||
|
|
@ -212,7 +137,7 @@ module = [
|
|||
ignore_missing_imports = true
|
||||
|
||||
[tool.bumpversion]
|
||||
current_version = "2.17.0"
|
||||
current_version = "2.16.0"
|
||||
commit = true
|
||||
tag = true
|
||||
tag_name = "{new_version}"
|
||||
|
|
@ -232,8 +157,6 @@ parse = """(?P<major>0|[1-9]\\d*)\\.(?P<minor>0|[1-9]\\d*)"""
|
|||
serialize = ["{major}.{minor}"]
|
||||
|
||||
[tool.coverage.run]
|
||||
# sysmon, default on 3.14, is too slow: https://github.com/coveragepy/coveragepy/issues/2172
|
||||
core = "ctrace"
|
||||
branch = true
|
||||
include = ["scrapy/*"]
|
||||
omit = ["tests/*"]
|
||||
|
|
@ -302,10 +225,8 @@ disable = [
|
|||
"too-many-positional-arguments",
|
||||
"too-many-public-methods",
|
||||
"too-many-return-statements",
|
||||
"undefined-variable",
|
||||
"unused-argument",
|
||||
"unused-variable",
|
||||
"use-implicit-booleaness-not-comparison",
|
||||
"useless-import-alias", # used as a hint to mypy
|
||||
"useless-return", # https://github.com/pylint-dev/pylint/issues/6530
|
||||
"wrong-import-position",
|
||||
|
|
@ -344,13 +265,11 @@ markers = [
|
|||
"requires_uvloop: marks tests as only enabled when uvloop is known to be working",
|
||||
"requires_botocore: marks tests that need botocore (but not boto3)",
|
||||
"requires_boto3: marks tests that need botocore and boto3",
|
||||
"requires_mitmproxy: marks tests that need a mitmdump executable",
|
||||
"requires_mitmproxy: marks tests that need mitmproxy",
|
||||
"requires_internet: marks tests that need real Internet access",
|
||||
]
|
||||
filterwarnings = [
|
||||
"ignore::DeprecationWarning:twisted.web.static",
|
||||
# Twisted doesn't close failed sockets after CannotListenError: https://github.com/twisted/twisted/issues/6108
|
||||
"ignore:Exception ignored in. <socket\\.socket.*laddr=..0\\.0\\.0\\.0., 0.:pytest.PytestUnraisableExceptionWarning",
|
||||
]
|
||||
|
||||
[tool.ruff.lint]
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
2.17.0
|
||||
2.16.0
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
# pragma: no file cover
|
||||
from scrapy.cmdline import execute
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ class AddonManager:
|
|||
|
||||
:param settings: The :class:`~scrapy.settings.BaseSettings` object from \
|
||||
which to read the early add-on configuration
|
||||
:type settings: :class:`~scrapy.settings.BaseSettings`
|
||||
:type settings: :class:`~scrapy.settings.Settings`
|
||||
"""
|
||||
for clspath in build_component_list(settings["ADDONS"]):
|
||||
addoncls = load_object(clspath)
|
||||
|
|
|
|||
|
|
@ -16,8 +16,6 @@ from twisted.python import failure
|
|||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
|
||||
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
|
||||
from scrapy.utils.deprecate import method_is_overridden
|
||||
from scrapy.utils.python import global_object_name
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
|
|
@ -38,14 +36,6 @@ class ScrapyCommand(ABC):
|
|||
|
||||
def __init__(self) -> None:
|
||||
self.settings: Settings | None = None # set in scrapy.cmdline
|
||||
if method_is_overridden(self.__class__, ScrapyCommand, "help"):
|
||||
warnings.warn(
|
||||
"The ScrapyCommand.help() method is deprecated and overriding "
|
||||
f"it, as the {global_object_name(self.__class__)} class does, "
|
||||
"has no effect; override long_desc() instead.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
def set_crawler(self, crawler: Crawler) -> None: # pragma: no cover
|
||||
warnings.warn(
|
||||
|
|
@ -73,16 +63,15 @@ class ScrapyCommand(ABC):
|
|||
def long_desc(self) -> str:
|
||||
"""A long description of the command. Return short description when not
|
||||
available. It cannot contain newlines since contents will be formatted
|
||||
by argparse which removes newlines and wraps text.
|
||||
by optparser which removes newlines and wraps text.
|
||||
"""
|
||||
return self.short_desc()
|
||||
|
||||
def help(self) -> str:
|
||||
warnings.warn(
|
||||
"ScrapyCommand.help() is deprecated, use long_desc() instead.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
"""An extensive help for the command. It will be shown when using the
|
||||
"help" command. It can contain newlines since no post-formatting will
|
||||
be applied to its contents.
|
||||
"""
|
||||
return self.long_desc()
|
||||
|
||||
def add_options(self, parser: argparse.ArgumentParser) -> None:
|
||||
|
|
|
|||
|
|
@ -1,29 +1,12 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.spiderloader import get_spider_loader
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import argparse
|
||||
|
||||
|
||||
def _edit_file(editor: str, file_path: str | os.PathLike[str]) -> int:
|
||||
"""Open ``file_path`` with ``editor`` and return the editor exit code.
|
||||
|
||||
``editor`` may include arguments (e.g. ``"code -w"``); it is split with
|
||||
:func:`shlex.split` and the file is passed as a separate argument, so no
|
||||
shell is involved.
|
||||
"""
|
||||
return subprocess.call([*shlex.split(editor), os.fspath(file_path)]) # noqa: S603
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
requires_project = True
|
||||
|
|
@ -62,4 +45,4 @@ class Command(ScrapyCommand):
|
|||
sfile = sys.modules[spidercls.__module__].__file__
|
||||
assert sfile
|
||||
sfile = sfile.replace(".pyc", ".py")
|
||||
self.exitcode = _edit_file(editor, Path(sfile))
|
||||
self.exitcode = os.system(f'{editor} "{sfile}"') # noqa: S605
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import string
|
||||
from importlib import import_module
|
||||
|
|
@ -9,14 +10,12 @@ from urllib.parse import urlparse
|
|||
|
||||
import scrapy
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.commands.edit import _edit_file
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.spiderloader import get_spider_loader
|
||||
from scrapy.utils.template import render_templatefile, string_camelcase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import argparse
|
||||
import os
|
||||
|
||||
|
||||
def sanitize_module_name(module_name: str) -> str:
|
||||
|
|
@ -119,11 +118,9 @@ class Command(ScrapyCommand):
|
|||
|
||||
template_file = self._find_template(opts.template)
|
||||
if template_file:
|
||||
spider_file = self._genspider(
|
||||
module, name, url, opts.template, template_file
|
||||
)
|
||||
self._genspider(module, name, url, opts.template, template_file)
|
||||
if opts.edit:
|
||||
self.exitcode = _edit_file(self.settings["EDITOR"], spider_file)
|
||||
self.exitcode = os.system(f'scrapy edit "{name}"') # noqa: S605
|
||||
|
||||
def _generate_template_variables(
|
||||
self,
|
||||
|
|
@ -151,7 +148,7 @@ class Command(ScrapyCommand):
|
|||
url: str,
|
||||
template_name: str,
|
||||
template_file: str | os.PathLike[str],
|
||||
) -> Path:
|
||||
) -> None:
|
||||
"""Generate the spider module, based on the given template"""
|
||||
assert self.settings is not None
|
||||
tvars = self._generate_template_variables(module, name, url, template_name)
|
||||
|
|
@ -171,7 +168,6 @@ class Command(ScrapyCommand):
|
|||
)
|
||||
if spiders_module:
|
||||
print(f"in module:\n {spiders_module.__name__}.{module}")
|
||||
return Path(spider_file)
|
||||
|
||||
def _find_template(self, template: str) -> Path | None:
|
||||
template_file = Path(self.templates_dir, f"{template}.tmpl")
|
||||
|
|
|
|||
|
|
@ -51,8 +51,6 @@ class Contract:
|
|||
results.addSuccess(self.testcase_pre)
|
||||
cb_result = cb(response, **cb_kwargs)
|
||||
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
|
||||
if isinstance(cb_result, CoroutineType):
|
||||
cb_result.close()
|
||||
raise TypeError("Contracts don't support async callbacks")
|
||||
return list(cast("Iterable[Any]", iterate_spider_output(cb_result)))
|
||||
|
||||
|
|
@ -69,8 +67,6 @@ class Contract:
|
|||
def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]:
|
||||
cb_result = cb(response, **cb_kwargs)
|
||||
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
|
||||
if isinstance(cb_result, CoroutineType):
|
||||
cb_result.close()
|
||||
raise TypeError("Contracts don't support async callbacks")
|
||||
output = list(cast("Iterable[Any]", iterate_spider_output(cb_result)))
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,32 +1,34 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from OpenSSL import SSL
|
||||
from twisted.internet.ssl import (
|
||||
AcceptableCiphers,
|
||||
CertificateOptions,
|
||||
TLSVersion,
|
||||
optionsForClientTLS,
|
||||
)
|
||||
from twisted.web.client import BrowserLikePolicyForHTTPS
|
||||
from twisted.web.iweb import IPolicyForHTTPS
|
||||
from zope.interface.declarations import implementer
|
||||
from zope.interface.verify import verifyObject
|
||||
|
||||
from scrapy.core.downloader.tls import (
|
||||
_TWISTED_VERSION_MAP,
|
||||
_openssl_methods,
|
||||
DEFAULT_CIPHERS,
|
||||
_ScrapyClientTLSOptions,
|
||||
_ScrapyClientTLSOptions26,
|
||||
openssl_methods,
|
||||
)
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils._deps_compat import TWISTED_TLS_NEW_IMPL
|
||||
from scrapy.utils.deprecate import create_deprecated_class
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
from scrapy.utils.ssl import _get_cert_options_version_kwargs, _get_tls_version_limits
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
|
||||
from twisted.internet._sslverify import ClientTLSOptions
|
||||
|
||||
# typing.Self requires Python 3.11
|
||||
|
|
@ -36,53 +38,54 @@ if TYPE_CHECKING:
|
|||
from scrapy.settings import BaseSettings
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _filter_method_warning() -> Generator[None]:
|
||||
with warnings.catch_warnings():
|
||||
# Twisted deprecation, https://github.com/scrapy/scrapy/issues/3288
|
||||
warnings.filterwarnings(
|
||||
"ignore",
|
||||
message=r"Passing method to twisted\.internet\.ssl\.CertificateOptions",
|
||||
category=DeprecationWarning,
|
||||
)
|
||||
yield
|
||||
|
||||
|
||||
@implementer(IPolicyForHTTPS)
|
||||
class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
|
||||
"""Non-peer-certificate verifying HTTPS context factory.
|
||||
|
||||
Uses :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`,
|
||||
:setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION`
|
||||
to configure the :class:`~twisted.internet.ssl.CertificateOptions`
|
||||
instance.
|
||||
Default OpenSSL method is ``TLS_METHOD`` (also called ``SSLv23_METHOD``)
|
||||
which allows TLS protocol negotiation.
|
||||
|
||||
The purpose of this custom class is to provide a ``creatorForNetloc()``
|
||||
method that returns:
|
||||
|
||||
- a ``_ScrapyClientTLSOptions26`` or ``_ScrapyClientTLSOptions`` instance
|
||||
configured based on TLS settings provided to the factory (when the
|
||||
certificate verification is disabled);
|
||||
- a result of ``optionsForClientTLS()`` called with those TLS settings
|
||||
(when the certificate verification is enabled).
|
||||
method that returns a ``_ScrapyClientTLSOptions`` instance configured based
|
||||
on TLS settings provided to the factory.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
method: int | None = SSL.SSLv23_METHOD, # noqa: S503
|
||||
method: int = SSL.SSLv23_METHOD, # noqa: S503
|
||||
tls_verbose_logging: bool = False,
|
||||
tls_ciphers: str | None = None,
|
||||
*args: Any,
|
||||
verify_certificates: bool = False,
|
||||
tls_min_version: TLSVersion | None = None,
|
||||
tls_max_version: TLSVersion | None = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
super().__init__(*args, **kwargs) # type: ignore[no-untyped-call]
|
||||
self._ssl_method: int | None = method
|
||||
self.tls_min_version: TLSVersion | None = tls_min_version
|
||||
self.tls_max_version: TLSVersion | None = tls_max_version
|
||||
self._ssl_method: int = method
|
||||
self.tls_verbose_logging: bool = tls_verbose_logging # unused
|
||||
self.tls_ciphers: AcceptableCiphers | None = (
|
||||
AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers)
|
||||
if tls_ciphers
|
||||
else None
|
||||
)
|
||||
self.tls_ciphers: AcceptableCiphers
|
||||
if tls_ciphers:
|
||||
self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers)
|
||||
else:
|
||||
self.tls_ciphers = DEFAULT_CIPHERS
|
||||
self._verify_certificates = verify_certificates
|
||||
|
||||
@classmethod
|
||||
def from_crawler(
|
||||
cls,
|
||||
crawler: Crawler,
|
||||
method: int | None = SSL.SSLv23_METHOD, # noqa: S503
|
||||
method: int = SSL.SSLv23_METHOD, # noqa: S503
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Self:
|
||||
|
|
@ -90,21 +93,12 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
|
|||
"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
|
||||
)
|
||||
tls_ciphers: str | None = crawler.settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
|
||||
# DOWNLOADER_CLIENT_TLS_METHOD reading and handling should be also moved here
|
||||
# when the deprecated load_context_factory_from_settings() is removed
|
||||
tls_min_ver, tls_max_ver = _get_tls_version_limits(
|
||||
crawler.settings, _TWISTED_VERSION_MAP.__getitem__
|
||||
)
|
||||
if tls_min_ver or tls_max_ver:
|
||||
method = None
|
||||
verify_certificates = crawler.settings.getbool("DOWNLOAD_VERIFY_CERTIFICATES")
|
||||
return cls( # type: ignore[misc]
|
||||
*args,
|
||||
method=method,
|
||||
tls_verbose_logging=tls_verbose_logging,
|
||||
tls_ciphers=tls_ciphers,
|
||||
tls_min_version=tls_min_ver,
|
||||
tls_max_version=tls_max_ver,
|
||||
verify_certificates=verify_certificates,
|
||||
**kwargs,
|
||||
)
|
||||
|
|
@ -114,23 +108,12 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
|
|||
return self._get_cert_options()
|
||||
|
||||
def _get_cert_options(self) -> CertificateOptions:
|
||||
return _ScrapyCertificateOptions(**self._get_cert_options_kwargs())
|
||||
|
||||
def _get_cert_options_kwargs(self) -> dict[str, Any]:
|
||||
kwargs: dict[str, Any] = {
|
||||
"fixBrokenPeers": True,
|
||||
"acceptableCiphers": self.tls_ciphers,
|
||||
}
|
||||
if self.tls_min_version or self.tls_max_version:
|
||||
kwargs.update(
|
||||
_get_cert_options_version_kwargs(
|
||||
self.tls_min_version, self.tls_max_version
|
||||
)
|
||||
with _filter_method_warning():
|
||||
return _ScrapyCertificateOptions(
|
||||
method=self._ssl_method,
|
||||
fixBrokenPeers=True,
|
||||
acceptableCiphers=self.tls_ciphers,
|
||||
)
|
||||
# when ScrapyClientContextFactory is removed self._ssl_method can just be None by default
|
||||
elif self._ssl_method != SSL.SSLv23_METHOD:
|
||||
kwargs["method"] = self._ssl_method
|
||||
return kwargs
|
||||
|
||||
# should be removed together with ScrapyClientContextFactory
|
||||
def getContext(
|
||||
|
|
@ -154,10 +137,15 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
|
|||
self._get_context(), # type: ignore[arg-type]
|
||||
)
|
||||
# Otherwise use the normal Twisted function.
|
||||
return optionsForClientTLS( # type: ignore[no-any-return]
|
||||
hostname=hostname.decode("ascii"),
|
||||
extraCertificateOptions=self._get_cert_options_kwargs(),
|
||||
)
|
||||
# Note that this doesn't use self._get_context().
|
||||
with _filter_method_warning():
|
||||
return optionsForClientTLS( # type: ignore[no-any-return]
|
||||
hostname=hostname.decode("ascii"),
|
||||
extraCertificateOptions={
|
||||
"method": self._ssl_method,
|
||||
"acceptableCiphers": self.tls_ciphers,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
ScrapyClientContextFactory = create_deprecated_class(
|
||||
|
|
@ -182,6 +170,12 @@ class BrowserLikeContextFactory(_ScrapyClientContextFactory):
|
|||
:meth:`creatorForNetloc` is the same as
|
||||
:class:`~twisted.web.client.BrowserLikePolicyForHTTPS` except this context
|
||||
factory allows setting the TLS/SSL method to use.
|
||||
|
||||
The default OpenSSL method is ``TLS_METHOD`` (also called
|
||||
``SSLv23_METHOD``) which allows TLS protocol negotiation.
|
||||
|
||||
As this overrides the parent ``creatorForNetloc()`` method, only
|
||||
``self._ssl_method`` is used from the parent class.
|
||||
"""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any):
|
||||
|
|
@ -195,10 +189,11 @@ class BrowserLikeContextFactory(_ScrapyClientContextFactory):
|
|||
super().__init__(*args, **kwargs)
|
||||
|
||||
def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
|
||||
return optionsForClientTLS( # type: ignore[no-any-return]
|
||||
hostname=hostname.decode("ascii"),
|
||||
extraCertificateOptions=self._get_cert_options_kwargs(),
|
||||
)
|
||||
with _filter_method_warning():
|
||||
return optionsForClientTLS( # type: ignore[no-any-return]
|
||||
hostname=hostname.decode("ascii"),
|
||||
extraCertificateOptions={"method": self._ssl_method},
|
||||
)
|
||||
|
||||
|
||||
@implementer(IPolicyForHTTPS)
|
||||
|
|
@ -231,6 +226,7 @@ class _AcceptableProtocolsContextFactory:
|
|||
# all of this with _ScrapyClientContextFactory.acceptableProtocols.
|
||||
|
||||
def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]):
|
||||
verifyObject(IPolicyForHTTPS, context_factory)
|
||||
self._wrapped_context_factory: Any = context_factory
|
||||
self._acceptable_protocols: list[bytes] = acceptable_protocols
|
||||
|
||||
|
|
@ -272,16 +268,6 @@ def _load_context_factory_from_settings(crawler: Crawler) -> IPolicyForHTTPS:
|
|||
|
||||
Also passes values of other relevant settings to the factory class.
|
||||
"""
|
||||
tls_method_setting: str = crawler.settings["DOWNLOADER_CLIENT_TLS_METHOD"]
|
||||
if tls_method_setting != "TLS":
|
||||
warnings.warn(
|
||||
"Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is"
|
||||
" deprecated, please use DOWNLOAD_TLS_MIN_VERSION and/or"
|
||||
" DOWNLOAD_TLS_MAX_VERSION instead.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
tls_method = _openssl_methods[tls_method_setting]
|
||||
if crawler.settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] == "SENTINEL":
|
||||
context_factory_cls = _ScrapyClientContextFactory
|
||||
else: # pragma: no cover
|
||||
|
|
@ -293,12 +279,13 @@ def _load_context_factory_from_settings(crawler: Crawler) -> IPolicyForHTTPS:
|
|||
context_factory_cls = load_object(
|
||||
crawler.settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]
|
||||
)
|
||||
ssl_method = openssl_methods[crawler.settings.get("DOWNLOADER_CLIENT_TLS_METHOD")]
|
||||
return cast(
|
||||
"IPolicyForHTTPS",
|
||||
build_from_crawler(
|
||||
context_factory_cls,
|
||||
crawler,
|
||||
method=tls_method,
|
||||
method=ssl_method,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -241,16 +241,6 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon
|
|||
body=response_body.getvalue(),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _request_headers(request: Request) -> Headers:
|
||||
"""Get a prepared copy of the request headers.
|
||||
|
||||
This removes the Proxy-Authorization header.
|
||||
"""
|
||||
headers = request.headers.copy()
|
||||
headers.pop(b"Proxy-Authorization", None)
|
||||
return headers
|
||||
|
||||
def _get_bind_address_host(self) -> str | None:
|
||||
"""Return the host portion of the bind address.
|
||||
|
||||
|
|
@ -289,8 +279,10 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon
|
|||
if not proxy:
|
||||
return None, None
|
||||
proxy = add_http_if_no_scheme(proxy)
|
||||
auth_header: bytes | None = request.headers.get(b"Proxy-Authorization")
|
||||
return proxy, auth_header.decode("ascii") if auth_header else None
|
||||
auth_header: list[bytes] | None = request.headers.pop(
|
||||
b"Proxy-Authorization", None
|
||||
)
|
||||
return proxy, auth_header[0].decode("ascii") if auth_header else None
|
||||
|
||||
def _extract_proxy_url_with_creds(self, request: Request) -> str | None:
|
||||
"""Return the proxy URL with the userinfo added based on the
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ from __future__ import annotations
|
|||
import ipaddress
|
||||
import ssl
|
||||
from contextlib import asynccontextmanager
|
||||
from socket import gaierror
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from scrapy.exceptions import (
|
||||
|
|
@ -18,7 +17,6 @@ from scrapy.exceptions import (
|
|||
)
|
||||
from scrapy.http import Headers
|
||||
from scrapy.utils._download_handlers import NullCookieJar
|
||||
from scrapy.utils.python import _iter_exc_causes
|
||||
from scrapy.utils.ssl import (
|
||||
_log_sslobj_debug_info,
|
||||
_make_insecure_ssl_ctx,
|
||||
|
|
@ -36,35 +34,10 @@ if TYPE_CHECKING:
|
|||
from scrapy.crawler import Crawler
|
||||
|
||||
|
||||
HAS_SOCKS = HAS_HTTP2 = False
|
||||
|
||||
try:
|
||||
import httpx
|
||||
except ImportError:
|
||||
httpx = None # type: ignore[assignment]
|
||||
else:
|
||||
# a small hack to avoid importing these optional extras unconditionally
|
||||
|
||||
DOWNLOAD_FAILED_EXCEPTIONS: tuple[type[BaseException], ...] = (
|
||||
httpx.RequestError,
|
||||
httpx.InvalidURL,
|
||||
)
|
||||
|
||||
try:
|
||||
import h2.exceptions
|
||||
|
||||
HAS_HTTP2 = True
|
||||
DOWNLOAD_FAILED_EXCEPTIONS += (h2.exceptions.InvalidBodyLengthError,)
|
||||
except ImportError: # pragma: no cover
|
||||
pass
|
||||
|
||||
try:
|
||||
import socksio.exceptions
|
||||
|
||||
HAS_SOCKS = True
|
||||
DOWNLOAD_FAILED_EXCEPTIONS += (socksio.exceptions.ProtocolError,)
|
||||
except ImportError: # pragma: no cover
|
||||
pass
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -81,11 +54,6 @@ class HttpxDownloadHandler(_Base):
|
|||
self._verify_certificates: bool = crawler.settings.getbool(
|
||||
"DOWNLOAD_VERIFY_CERTIFICATES"
|
||||
)
|
||||
self._enable_h2: bool = crawler.settings.getbool("HTTPX_HTTP2_ENABLED")
|
||||
if self._enable_h2 and not HAS_HTTP2: # pragma: no cover
|
||||
raise NotConfigured(
|
||||
f"HTTP/2 support in {type(self).__name__} requires the 'httpx[http2]' extra to be installed."
|
||||
)
|
||||
self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)
|
||||
self._bind_host: str | None = self._get_bind_address_host()
|
||||
self._limits: httpx.Limits = httpx.Limits(
|
||||
|
|
@ -122,7 +90,6 @@ class HttpxDownloadHandler(_Base):
|
|||
transport=httpx.AsyncHTTPTransport(
|
||||
verify=self._ssl_context,
|
||||
local_address=self._bind_host,
|
||||
http2=self._enable_h2,
|
||||
limits=self._limits,
|
||||
trust_env=False,
|
||||
proxy=proxy,
|
||||
|
|
@ -146,20 +113,13 @@ class HttpxDownloadHandler(_Base):
|
|||
async def _make_request(
|
||||
self, request: Request, timeout: float
|
||||
) -> AsyncIterator[httpx.Response]:
|
||||
proxy = self._extract_proxy_url_with_creds(request)
|
||||
if proxy and proxy.startswith("socks") and not HAS_SOCKS: # pragma: no cover
|
||||
raise ValueError(
|
||||
f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed."
|
||||
)
|
||||
client = self._get_client(proxy)
|
||||
headers = self._request_headers(request).to_tuple_list()
|
||||
|
||||
client = self._get_client(self._extract_proxy_url_with_creds(request))
|
||||
try:
|
||||
async with client.stream(
|
||||
request.method,
|
||||
request.url,
|
||||
content=request.body,
|
||||
headers=headers,
|
||||
headers=request.headers.to_tuple_list(),
|
||||
timeout=timeout,
|
||||
) as response:
|
||||
yield response
|
||||
|
|
@ -170,12 +130,18 @@ class HttpxDownloadHandler(_Base):
|
|||
except httpx.UnsupportedProtocol as e:
|
||||
raise UnsupportedURLSchemeError(str(e)) from e
|
||||
except httpx.ConnectError as e:
|
||||
if any(isinstance(c, gaierror) for c in _iter_exc_causes(e)):
|
||||
raise CannotResolveHostError(str(e)) from e
|
||||
error_message = str(e)
|
||||
if (
|
||||
"Name or service not known" in error_message
|
||||
or "getaddrinfo failed" in error_message
|
||||
or "nodename nor servname" in error_message
|
||||
or "Temporary failure in name resolution" in error_message
|
||||
):
|
||||
raise CannotResolveHostError(error_message) from e
|
||||
raise DownloadConnectionRefusedError(str(e)) from e
|
||||
except httpx.ProxyError as e:
|
||||
raise DownloadConnectionRefusedError(str(e)) from e
|
||||
except DOWNLOAD_FAILED_EXCEPTIONS as e:
|
||||
except (httpx.NetworkError, httpx.RemoteProtocolError) as e:
|
||||
raise DownloadFailedError(str(e)) from e
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@
|
|||
An asynchronous FTP file download handler for scrapy which somehow emulates an http response.
|
||||
|
||||
FTP connection parameters are passed using the request meta field:
|
||||
- ftp_user (optional, falls back to FTP_USER)
|
||||
- ftp_password (optional, falls back to FTP_PASSWORD)
|
||||
- ftp_passive (optional, falls back to FTP_PASSIVE_MODE) sets FTP connection passive mode
|
||||
- ftp_user (required)
|
||||
- ftp_password (required)
|
||||
- ftp_passive (by default, enabled) sets FTP connection passive mode
|
||||
- ftp_local_filename
|
||||
- If not given, file data will come in the response.body, as a normal scrapy Response,
|
||||
which will imply that the entire file will be on memory.
|
||||
|
|
@ -119,10 +119,7 @@ class FTPDownloadHandler(BaseDownloadHandler):
|
|||
httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"])
|
||||
return Response(url=request.url, status=httpcode, body=message.encode())
|
||||
raise
|
||||
finally:
|
||||
protocol.close()
|
||||
assert client.transport
|
||||
client.transport.loseConnection()
|
||||
protocol.close()
|
||||
headers = {"local filename": protocol.filename or b"", "size": protocol.size}
|
||||
body = protocol.filename or protocol.body.read()
|
||||
respcls = responsetypes.from_args(url=request.url, body=body)
|
||||
|
|
|
|||
|
|
@ -104,6 +104,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
|
|||
self._disconnect_timeout: int = 1
|
||||
|
||||
async def download_request(self, request: Request) -> Response:
|
||||
"""Return a deferred for the HTTP download"""
|
||||
if hasattr(self._crawler.spider, "download_maxsize"): # pragma: no cover
|
||||
warn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE")
|
||||
if hasattr(self._crawler.spider, "download_warnsize"): # pragma: no cover
|
||||
|
|
@ -282,7 +283,7 @@ def _tunnel_request_data(
|
|||
|
||||
|
||||
class _TunnelingAgent(Agent):
|
||||
"""An agent that uses a ``_TunnelingTCP4ClientEndpoint`` to make HTTPS
|
||||
"""An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS
|
||||
downloads. It may look strange that we have chosen to subclass Agent and not
|
||||
ProxyAgent but consider that after the tunnel is opened the proxy is
|
||||
transparent to the client; thus the agent should behave like there is no
|
||||
|
|
@ -544,8 +545,7 @@ class _ScrapyAgent:
|
|||
expected_size, maxsize, request, expected=True
|
||||
)
|
||||
logger.warning(warning_msg)
|
||||
# Abort connection immediately.
|
||||
txresponse._transport._producer.abortConnection()
|
||||
txresponse._transport.loseConnection()
|
||||
raise DownloadCancelledError(warning_msg)
|
||||
|
||||
if warnsize and expected_size > warnsize:
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.boto import is_botocore_available
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
|
|
@ -50,16 +49,7 @@ class S3DownloadHandler(BaseDownloadHandler):
|
|||
|
||||
async def download_request(self, request: Request) -> Response:
|
||||
p = urlparse_cached(request)
|
||||
if request.meta.get("is_secure") is False:
|
||||
warnings.warn(
|
||||
"Passing is_secure=False for s3:// requests is deprecated."
|
||||
" In future Scrapy releases this flag will be ignored.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
scheme = "http"
|
||||
else:
|
||||
scheme = "https"
|
||||
scheme = "https" if request.meta.get("is_secure") else "http"
|
||||
bucket = p.hostname
|
||||
path = p.path + "?" + p.query if p.query else p.path
|
||||
url = f"{scheme}://{bucket}.s3.amazonaws.com{path}"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from OpenSSL import SSL
|
||||
|
|
@ -19,9 +18,8 @@ from service_identity.pyopenssl import (
|
|||
verify_ip_address,
|
||||
)
|
||||
from twisted.internet._sslverify import ClientTLSOptions
|
||||
from twisted.internet.ssl import AcceptableCiphers, TLSVersion
|
||||
from twisted.internet.ssl import AcceptableCiphers
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.deprecate import create_deprecated_class
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -34,44 +32,17 @@ if TYPE_CHECKING:
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_openssl_methods: dict[str, int] = {
|
||||
"TLS": SSL.SSLv23_METHOD, # protocol negotiation (recommended)
|
||||
"TLSv1.0": SSL.TLSv1_METHOD, # TLS 1.0 only
|
||||
"TLSv1.1": SSL.TLSv1_1_METHOD, # TLS 1.1 only
|
||||
"TLSv1.2": SSL.TLSv1_2_METHOD, # TLS 1.2 only
|
||||
}
|
||||
METHOD_TLS = "TLS"
|
||||
METHOD_TLSv10 = "TLSv1.0"
|
||||
METHOD_TLSv11 = "TLSv1.1"
|
||||
METHOD_TLSv12 = "TLSv1.2"
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
if name == "DEFAULT_CIPHERS":
|
||||
warnings.warn(
|
||||
"scrapy.core.downloader.tls.DEFAULT_CIPHERS is deprecated.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return AcceptableCiphers.fromOpenSSLCipherString("DEFAULT")
|
||||
deprecated = {
|
||||
"METHOD_TLS": "TLS",
|
||||
"METHOD_TLSv10": "TLSv1.0",
|
||||
"METHOD_TLSv11": "TLSv1.1",
|
||||
"METHOD_TLSv12": "TLSv1.2",
|
||||
"openssl_methods": _openssl_methods,
|
||||
}
|
||||
if name in deprecated:
|
||||
warnings.warn(
|
||||
f"scrapy.core.downloader.tls.{name} is deprecated.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return deprecated[name]
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
_TWISTED_VERSION_MAP: dict[str, TLSVersion] = {
|
||||
"TLSv1.0": TLSVersion.TLSv1_0,
|
||||
"TLSv1.1": TLSVersion.TLSv1_1,
|
||||
"TLSv1.2": TLSVersion.TLSv1_2,
|
||||
"TLSv1.3": TLSVersion.TLSv1_3,
|
||||
openssl_methods: dict[str, int] = {
|
||||
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
|
||||
METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only
|
||||
METHOD_TLSv11: SSL.TLSv1_1_METHOD, # TLS 1.1 only
|
||||
METHOD_TLSv12: SSL.TLSv1_2_METHOD, # TLS 1.2 only
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -184,3 +155,8 @@ class _ScrapyClientTLSOptions26(ClientTLSOptions):
|
|||
return True
|
||||
|
||||
return verifyCallback
|
||||
|
||||
|
||||
DEFAULT_CIPHERS: AcceptableCiphers = AcceptableCiphers.fromOpenSSLCipherString(
|
||||
"DEFAULT"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -114,7 +114,11 @@ class H2ConnectionPool:
|
|||
d.errback(ResponseFailed(errors))
|
||||
|
||||
def close_connections(self) -> None:
|
||||
"""Close all the HTTP/2 connections and remove them from pool."""
|
||||
"""Close all the HTTP/2 connections and remove them from pool
|
||||
|
||||
Returns:
|
||||
Deferred that fires when all connections have been closed
|
||||
"""
|
||||
for conn in self._connections.values():
|
||||
assert conn.transport is not None # typing
|
||||
conn.transport.abortConnection()
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
|
|||
uri is used to verify that incoming client requests have correct
|
||||
base URL.
|
||||
settings -- Scrapy project settings
|
||||
conn_lost_deferred -- Deferred that fires with the list of underlying exceptions to notify
|
||||
conn_lost_deferred -- Deferred fires with the reason: Failure to notify
|
||||
that connection was lost
|
||||
tls_verbose_logging -- Whether to log TLS details
|
||||
"""
|
||||
|
|
@ -375,7 +375,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
|
|||
|
||||
def _handle_events(self, events: list[Event]) -> None:
|
||||
"""Private method which acts as a bridge between the events
|
||||
received from the HTTP/2 data and the handlers in this class.
|
||||
received from the HTTP/2 data and IH2EventsHandler
|
||||
|
||||
Arguments:
|
||||
events -- A list of events that the remote peer triggered by sending data
|
||||
|
|
|
|||
|
|
@ -131,10 +131,10 @@ class Scheduler(BaseScheduler):
|
|||
(:setting:`SCHEDULER_PRIORITY_QUEUE`) that sort requests by
|
||||
:attr:`~scrapy.http.Request.priority`.
|
||||
|
||||
By default, memory-based priority queues are used for all requests.
|
||||
When using :setting:`JOBDIR`, disk-based priority queues are also created,
|
||||
By default, a single, memory-based priority queue is used for all requests.
|
||||
When using :setting:`JOBDIR`, a disk-based priority queue is also created,
|
||||
and only unserializable requests are stored in the memory-based priority
|
||||
queues. For a given priority value, requests in memory take precedence over
|
||||
queue. For a given priority value, requests in memory take precedence over
|
||||
requests in disk.
|
||||
|
||||
Each priority queue stores requests in separate internal queues, one per
|
||||
|
|
@ -209,8 +209,8 @@ class Scheduler(BaseScheduler):
|
|||
-------------------------
|
||||
|
||||
While pending requests are below the configured values of
|
||||
:setting:`CONCURRENT_REQUESTS` or
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, those requests are sent
|
||||
:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
|
||||
or :setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent
|
||||
concurrently.
|
||||
|
||||
As a result, the first few requests of a crawl may not follow the desired
|
||||
|
|
@ -289,11 +289,11 @@ class Scheduler(BaseScheduler):
|
|||
|
||||
:param dqclass: A class to be used as persistent request queue.
|
||||
The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default.
|
||||
:type dqclass: type
|
||||
:type dqclass: class
|
||||
|
||||
:param mqclass: A class to be used as non-persistent request queue.
|
||||
The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default.
|
||||
:type mqclass: type
|
||||
:type mqclass: class
|
||||
|
||||
:param logunser: A boolean that indicates whether or not unserializable requests should be logged.
|
||||
The value for the :setting:`SCHEDULER_DEBUG` setting is used by default.
|
||||
|
|
@ -306,7 +306,7 @@ class Scheduler(BaseScheduler):
|
|||
|
||||
:param pqclass: A class to be used as priority queue for requests.
|
||||
The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default.
|
||||
:type pqclass: type
|
||||
:type pqclass: class
|
||||
|
||||
:param crawler: The crawler object corresponding to the current crawl.
|
||||
:type crawler: :class:`scrapy.crawler.Crawler`
|
||||
|
|
@ -342,7 +342,7 @@ class Scheduler(BaseScheduler):
|
|||
def open(self, spider: Spider) -> Deferred[None] | None:
|
||||
"""
|
||||
(1) initialize the memory queue
|
||||
(2) initialize the disk queue if the ``jobdir`` argument wasn't empty
|
||||
(2) initialize the disk queue if the ``jobdir`` attribute is a valid directory
|
||||
(3) return the result of the dupefilter's ``open`` method
|
||||
"""
|
||||
self.spider: Spider = spider
|
||||
|
|
|
|||
|
|
@ -125,7 +125,7 @@ class Scraper:
|
|||
|
||||
def _check_deprecated_itemproc_method(self, method: str) -> None:
|
||||
itemproc_cls = type(self.itemproc)
|
||||
if not hasattr(self.itemproc, f"{method}_async"):
|
||||
if not hasattr(self.itemproc, "process_item_async"):
|
||||
warnings.warn(
|
||||
f"{global_object_name(itemproc_cls)} doesn't define a {method}_async() method,"
|
||||
f" this is deprecated and the method will be required in future Scrapy versions.",
|
||||
|
|
@ -441,7 +441,7 @@ class Scraper:
|
|||
self, output: Any, response: Response | Failure
|
||||
) -> Deferred[None]:
|
||||
"""Process each Request/Item (given in the output parameter) returned
|
||||
from the spider.
|
||||
from the given spider.
|
||||
|
||||
Items are sent to the item pipelines, requests are scheduled.
|
||||
"""
|
||||
|
|
@ -451,7 +451,7 @@ class Scraper:
|
|||
self, output: Any, response: Response | Failure
|
||||
) -> None:
|
||||
"""Process each Request/Item (given in the output parameter) returned
|
||||
from the spider.
|
||||
from the given spider.
|
||||
|
||||
Items are sent to the item pipelines, requests are scheduled.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -39,11 +39,7 @@ from scrapy.utils.reactor import (
|
|||
verify_installed_asyncio_event_loop,
|
||||
verify_installed_reactor,
|
||||
)
|
||||
from scrapy.utils.reactorless import (
|
||||
ReactorImportHook,
|
||||
install_reactor_import_hook,
|
||||
uninstall_reactor_import_hook,
|
||||
)
|
||||
from scrapy.utils.reactorless import install_reactor_import_hook
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Awaitable, Generator, Iterable
|
||||
|
|
@ -367,12 +363,9 @@ class CrawlerRunnerBase(ABC):
|
|||
"""
|
||||
Return a :class:`~scrapy.crawler.Crawler` object.
|
||||
|
||||
* If ``crawler_or_spidercls`` is a Crawler, the runner's settings are
|
||||
merged into it as defaults: for each setting, the runner's value
|
||||
is applied only if the Crawler does not already have that setting at
|
||||
an equal or higher priority. The Crawler is then returned.
|
||||
* If ``crawler_or_spidercls`` is a Crawler, it is returned as-is.
|
||||
* If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler
|
||||
is constructed for it using this runner's settings.
|
||||
is constructed for it.
|
||||
* If ``crawler_or_spidercls`` is a string, this function finds
|
||||
a spider with this name in a Scrapy project (using spider loader),
|
||||
then creates a Crawler instance for it.
|
||||
|
|
@ -383,7 +376,6 @@ class CrawlerRunnerBase(ABC):
|
|||
"it must be a spider class (or a Crawler object)"
|
||||
)
|
||||
if isinstance(crawler_or_spidercls, Crawler):
|
||||
crawler_or_spidercls.settings.update(self.settings)
|
||||
return crawler_or_spidercls
|
||||
return self._create_crawler(crawler_or_spidercls)
|
||||
|
||||
|
|
@ -535,8 +527,8 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):
|
|||
"""
|
||||
Run a crawler with the provided arguments.
|
||||
|
||||
It will call the given Crawler's :meth:`~Crawler.crawl_async` method,
|
||||
while keeping track of it so it can be stopped later.
|
||||
It will call the given Crawler's :meth:`~Crawler.crawl` method, while
|
||||
keeping track of it so it can be stopped later.
|
||||
|
||||
If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler`
|
||||
instance, this method will try to create one using this parameter as
|
||||
|
|
@ -777,7 +769,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner):
|
|||
"""
|
||||
This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
|
||||
size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS
|
||||
resolver based on :setting:`TWISTED_DNS_RESOLVER`.
|
||||
resolver based on :setting:`DNSCACHE_ENABLED`.
|
||||
|
||||
If ``stop_after_crawl`` is True, the reactor will be stopped after all
|
||||
crawlers have finished, using :meth:`join`.
|
||||
|
|
@ -841,7 +833,6 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
|
|||
super().__init__(settings, install_root_handler)
|
||||
logger.debug("Using AsyncCrawlerProcess")
|
||||
self._reactorless_loop: asyncio.AbstractEventLoop | None = None
|
||||
self._reactor_import_hook: ReactorImportHook | None = None
|
||||
# We want the asyncio event loop to be installed early, so that it's
|
||||
# always the correct one. And as we do that, we can also install the
|
||||
# reactor here.
|
||||
|
|
@ -854,7 +845,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
|
|||
"TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed."
|
||||
)
|
||||
self._reactorless_loop = set_asyncio_event_loop(loop_path)
|
||||
self._reactor_import_hook = install_reactor_import_hook()
|
||||
install_reactor_import_hook()
|
||||
elif is_reactor_installed():
|
||||
# The user could install a reactor before this class is instantiated.
|
||||
# We need to make sure the reactor is the correct one and the loop
|
||||
|
|
@ -880,10 +871,10 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
|
|||
|
||||
When using a reactor it adjusts its pool size to
|
||||
:setting:`REACTOR_THREADPOOL_MAXSIZE` and installs a DNS resolver based
|
||||
on :setting:`TWISTED_DNS_RESOLVER`.
|
||||
on :setting:`DNSCACHE_ENABLED`.
|
||||
|
||||
If ``stop_after_crawl`` is True, the reactor/event loop will be stopped
|
||||
after all crawlers have finished, using :meth:`join`.
|
||||
If ``stop_after_crawl`` is True, the reactor will be stopped after all
|
||||
crawlers have finished, using :meth:`join`.
|
||||
|
||||
:param bool stop_after_crawl: stop or not the reactor when all
|
||||
crawlers have finished
|
||||
|
|
@ -984,10 +975,6 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
|
|||
loop.run_until_complete(loop.shutdown_asyncgens())
|
||||
loop.run_until_complete(loop.shutdown_default_executor())
|
||||
finally:
|
||||
# loop.close() can raise, so we uninstall the hook first
|
||||
if self._reactor_import_hook: # pragma: no branch
|
||||
uninstall_reactor_import_hook(self._reactor_import_hook)
|
||||
self._reactor_import_hook = None
|
||||
self._reactorless_main_task = None
|
||||
asyncio.set_event_loop(None)
|
||||
loop.close()
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ from scrapy.http.cookies import CookieJar
|
|||
from scrapy.utils.decorators import _warn_spider_arg
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.python import to_unicode
|
||||
from scrapy.utils.request import _decode_cookie, _to_verbose_cookies
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable, Sequence
|
||||
|
|
@ -135,10 +134,29 @@ class CookiesMiddleware:
|
|||
Given a dict consisting of cookie components, return its string representation.
|
||||
Decode from bytes if necessary.
|
||||
"""
|
||||
decoded = _decode_cookie(cookie, request)
|
||||
if decoded is None:
|
||||
return None
|
||||
decoded = {}
|
||||
flags = set()
|
||||
for key in ("name", "value", "path", "domain"):
|
||||
value = cookie.get(key)
|
||||
if value is None:
|
||||
if key in {"name", "value"}:
|
||||
msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
|
||||
logger.warning(msg)
|
||||
return None
|
||||
continue
|
||||
if isinstance(value, (bool, float, int, str)):
|
||||
decoded[key] = str(value)
|
||||
else:
|
||||
assert isinstance(value, bytes)
|
||||
try:
|
||||
decoded[key] = value.decode("utf8")
|
||||
except UnicodeDecodeError:
|
||||
logger.warning(
|
||||
"Non UTF-8 encoded cookie found in request %s: %s",
|
||||
request,
|
||||
cookie,
|
||||
)
|
||||
decoded[key] = value.decode("latin1", errors="replace")
|
||||
for flag in ("secure",):
|
||||
value = cookie.get(flag, _UNSET)
|
||||
if value is _UNSET or not value:
|
||||
|
|
@ -159,7 +177,11 @@ class CookiesMiddleware:
|
|||
"""
|
||||
if not request.cookies:
|
||||
return ()
|
||||
cookies: Iterable[VerboseCookie] = _to_verbose_cookies(request.cookies)
|
||||
cookies: Iterable[VerboseCookie]
|
||||
if isinstance(request.cookies, dict):
|
||||
cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items())
|
||||
else:
|
||||
cookies = request.cookies
|
||||
for cookie in cookies:
|
||||
cookie.setdefault("secure", urlparse_cached(request).scheme == "https")
|
||||
formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
|
||||
|
|
|
|||
|
|
@ -6,14 +6,11 @@ See documentation in docs/topics/downloader-middleware.rst
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from w3lib.http import basic_auth_header
|
||||
|
||||
from scrapy import Request, Spider, signals
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.settings import SETTINGS_PRIORITIES
|
||||
from scrapy.utils.decorators import _warn_spider_arg
|
||||
from scrapy.utils.url import url_is_from_any_domain
|
||||
|
||||
|
|
@ -26,28 +23,12 @@ if TYPE_CHECKING:
|
|||
|
||||
|
||||
class HttpAuthMiddleware:
|
||||
"""Set Basic HTTP Authorization header."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._auth: bytes | None = None
|
||||
self._domain: str | None = None
|
||||
"""Set Basic HTTP Authorization header
|
||||
(http_user and http_pass spider class attributes)"""
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||
o = cls()
|
||||
usr = crawler.settings.get("HTTPAUTH_USER", "")
|
||||
pwd = crawler.settings.get("HTTPAUTH_PASS", "")
|
||||
if usr or pwd:
|
||||
domain_priority = crawler.settings.getpriority("HTTPAUTH_DOMAIN") or 0
|
||||
if domain_priority <= SETTINGS_PRIORITIES["default"]:
|
||||
raise ValueError(
|
||||
"HTTPAUTH_DOMAIN must be set when HTTPAUTH_USER or HTTPAUTH_PASS "
|
||||
"is configured. Set it to a domain (e.g. 'example.com') to restrict "
|
||||
"credentials to that domain, or set it to None to send credentials "
|
||||
"with all requests."
|
||||
)
|
||||
o._auth = basic_auth_header(usr, pwd)
|
||||
o._domain = crawler.settings.get("HTTPAUTH_DOMAIN")
|
||||
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
|
||||
return o
|
||||
|
||||
|
|
@ -55,34 +36,18 @@ class HttpAuthMiddleware:
|
|||
usr = getattr(spider, "http_user", "")
|
||||
pwd = getattr(spider, "http_pass", "")
|
||||
if usr or pwd:
|
||||
warnings.warn(
|
||||
"Use the HTTPAUTH_USER, HTTPAUTH_PASS, and HTTPAUTH_DOMAIN settings "
|
||||
"instead of the http_user, http_pass, and http_auth_domain spider "
|
||||
"attributes. Support for the spider attributes will be removed in a "
|
||||
"future version of Scrapy.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
self._auth = basic_auth_header(usr, pwd)
|
||||
self._domain = spider.http_auth_domain # type: ignore[attr-defined]
|
||||
self.auth = basic_auth_header(usr, pwd)
|
||||
self.domain = spider.http_auth_domain # type: ignore[attr-defined]
|
||||
|
||||
@_warn_spider_arg
|
||||
def process_request(
|
||||
self, request: Request, spider: Spider | None = None
|
||||
) -> Request | Response | None:
|
||||
if b"Authorization" in request.headers:
|
||||
return None
|
||||
# Per-request meta overrides
|
||||
usr = request.meta.get("http_user", "")
|
||||
pwd = request.meta.get("http_pass", "")
|
||||
if usr or pwd:
|
||||
domain = request.meta.get("http_auth_domain")
|
||||
if not domain or url_is_from_any_domain(request.url, [domain]):
|
||||
request.headers[b"Authorization"] = basic_auth_header(usr, pwd)
|
||||
return None
|
||||
# Middleware-level auth
|
||||
if self._auth and (
|
||||
not self._domain or url_is_from_any_domain(request.url, [self._domain])
|
||||
auth = getattr(self, "auth", None)
|
||||
if (
|
||||
auth
|
||||
and b"Authorization" not in request.headers
|
||||
and (not self.domain or url_is_from_any_domain(request.url, [self.domain]))
|
||||
):
|
||||
request.headers[b"Authorization"] = self._auth
|
||||
request.headers[b"Authorization"] = auth
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from logging import getLogger
|
|||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from scrapy import Request, Spider, signals
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
from scrapy.http import Response, TextResponse
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils._compression import (
|
||||
|
|
@ -60,7 +60,7 @@ else:
|
|||
|
||||
|
||||
class HttpCompressionMiddleware:
|
||||
"""This middleware allows compressed (gzip, deflate etc.) traffic to be
|
||||
"""This middleware allows compressed (gzip, deflate) traffic to be
|
||||
sent/received from websites"""
|
||||
|
||||
def __init__(
|
||||
|
|
@ -70,12 +70,6 @@ class HttpCompressionMiddleware:
|
|||
crawler: Crawler | None = None,
|
||||
):
|
||||
if not crawler:
|
||||
warnings.warn(
|
||||
"Instantiating HttpCompressionMiddleware without a 'crawler' "
|
||||
"argument is deprecated.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
self.stats = stats
|
||||
self._max_size = 1073741824
|
||||
self._warn_size = 33554432
|
||||
|
|
@ -114,49 +108,49 @@ class HttpCompressionMiddleware:
|
|||
) -> Request | Response:
|
||||
if request.method == "HEAD":
|
||||
return response
|
||||
content_encoding = response.headers.getlist("Content-Encoding")
|
||||
if content_encoding:
|
||||
max_size = request.meta.get("download_maxsize", self._max_size)
|
||||
warn_size = request.meta.get("download_warnsize", self._warn_size)
|
||||
try:
|
||||
decoded_body, content_encoding = self._handle_encoding(
|
||||
response.body, content_encoding, max_size
|
||||
)
|
||||
except _DecompressionMaxSizeExceeded as e:
|
||||
msg = (
|
||||
f"Ignored response {response} because its body "
|
||||
f"({len(response.body)} B compressed, "
|
||||
f"{e.decompressed_size} B decompressed so far) exceeded "
|
||||
f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."
|
||||
)
|
||||
logger.warning(msg)
|
||||
raise IgnoreRequest(msg) from e
|
||||
if len(response.body) < warn_size <= len(decoded_body):
|
||||
logger.warning(
|
||||
f"{response} body size after decompression "
|
||||
f"({len(decoded_body)} B) is larger than the "
|
||||
f"download warning size ({warn_size} B)."
|
||||
)
|
||||
if isinstance(response, Response):
|
||||
content_encoding = response.headers.getlist("Content-Encoding")
|
||||
if content_encoding:
|
||||
self._warn_unknown_encoding(response, content_encoding)
|
||||
response.headers["Content-Encoding"] = content_encoding
|
||||
if self.stats:
|
||||
self.stats.inc_value(
|
||||
"httpcompression/response_bytes",
|
||||
len(decoded_body),
|
||||
max_size = request.meta.get("download_maxsize", self._max_size)
|
||||
warn_size = request.meta.get("download_warnsize", self._warn_size)
|
||||
try:
|
||||
decoded_body, content_encoding = self._handle_encoding(
|
||||
response.body, content_encoding, max_size
|
||||
)
|
||||
except _DecompressionMaxSizeExceeded as e:
|
||||
raise IgnoreRequest(
|
||||
f"Ignored response {response} because its body "
|
||||
f"({len(response.body)} B compressed, "
|
||||
f"{e.decompressed_size} B decompressed so far) exceeded "
|
||||
f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."
|
||||
) from e
|
||||
if len(response.body) < warn_size <= len(decoded_body):
|
||||
logger.warning(
|
||||
f"{response} body size after decompression "
|
||||
f"({len(decoded_body)} B) is larger than the "
|
||||
f"download warning size ({warn_size} B)."
|
||||
)
|
||||
if content_encoding:
|
||||
self._warn_unknown_encoding(response, content_encoding)
|
||||
response.headers["Content-Encoding"] = content_encoding
|
||||
if self.stats:
|
||||
self.stats.inc_value(
|
||||
"httpcompression/response_bytes",
|
||||
len(decoded_body),
|
||||
)
|
||||
self.stats.inc_value("httpcompression/response_count")
|
||||
respcls = responsetypes.from_args(
|
||||
headers=response.headers, url=response.url, body=decoded_body
|
||||
)
|
||||
self.stats.inc_value("httpcompression/response_count")
|
||||
respcls = responsetypes.from_args(
|
||||
headers=response.headers, url=response.url, body=decoded_body
|
||||
)
|
||||
kwargs: dict[str, Any] = {"body": decoded_body}
|
||||
if issubclass(respcls, TextResponse):
|
||||
# force recalculating the encoding until we make sure the
|
||||
# responsetypes guessing is reliable
|
||||
kwargs["encoding"] = None
|
||||
response = response.replace(cls=respcls, **kwargs)
|
||||
if not content_encoding:
|
||||
del response.headers["Content-Encoding"]
|
||||
kwargs: dict[str, Any] = {"body": decoded_body}
|
||||
if issubclass(respcls, TextResponse):
|
||||
# force recalculating the encoding until we make sure the
|
||||
# responsetypes guessing is reliable
|
||||
kwargs["encoding"] = None
|
||||
response = response.replace(cls=respcls, **kwargs)
|
||||
if not content_encoding:
|
||||
del response.headers["Content-Encoding"]
|
||||
|
||||
return response
|
||||
|
||||
def _handle_encoding(
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import logging
|
||||
import re
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from scrapy import Request, Spider, signals
|
||||
|
|
@ -74,10 +75,25 @@ class OffsiteMiddleware:
|
|||
allowed_domains = getattr(spider, "allowed_domains", None)
|
||||
if not allowed_domains:
|
||||
return re.compile("") # allow all by default
|
||||
url_pattern = re.compile(r"^https?://.*$")
|
||||
port_pattern = re.compile(r":\d+$")
|
||||
domains = []
|
||||
for domain in allowed_domains:
|
||||
if domain is None:
|
||||
continue
|
||||
domains.append(re.escape(domain))
|
||||
if url_pattern.match(domain):
|
||||
message = (
|
||||
"allowed_domains accepts only domains, not URLs. "
|
||||
f"Ignoring URL entry {domain} in allowed_domains."
|
||||
)
|
||||
warnings.warn(message, stacklevel=2)
|
||||
elif port_pattern.search(domain):
|
||||
message = (
|
||||
"allowed_domains accepts only domains without ports. "
|
||||
f"Ignoring entry {domain} in allowed_domains."
|
||||
)
|
||||
warnings.warn(message, stacklevel=2)
|
||||
else:
|
||||
domains.append(re.escape(domain))
|
||||
regex = rf"^(.*\.)?({'|'.join(domains)})$"
|
||||
return re.compile(regex)
|
||||
|
|
|
|||
|
|
@ -196,7 +196,10 @@ class BaseRedirectMiddleware:
|
|||
|
||||
|
||||
class RedirectMiddleware(BaseRedirectMiddleware):
|
||||
"""Handle redirection of requests based on response status."""
|
||||
"""
|
||||
Handle redirection of requests based on response status
|
||||
and meta-refresh html tag.
|
||||
"""
|
||||
|
||||
@_warn_spider_arg
|
||||
def process_response(
|
||||
|
|
@ -248,8 +251,6 @@ class RedirectMiddleware(BaseRedirectMiddleware):
|
|||
|
||||
|
||||
class MetaRefreshMiddleware(BaseRedirectMiddleware):
|
||||
"""Handle redirection of requests based on meta-refresh html tag."""
|
||||
|
||||
enabled_setting = "METAREFRESH_ENABLED"
|
||||
|
||||
def __init__(self, settings: BaseSettings):
|
||||
|
|
|
|||
|
|
@ -5,11 +5,14 @@ problems such as a connection timeout or HTTP 500 error.
|
|||
You can change the behaviour of this middleware by modifying the scraping settings:
|
||||
RETRY_TIMES - how many times to retry a failed page
|
||||
RETRY_HTTP_CODES - which HTTP response codes to retry
|
||||
|
||||
Failed pages are collected on the scraping process and rescheduled at the end,
|
||||
once the spider has finished crawling all regular (non-failed) pages.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from logging import Logger, getLevelName, getLogger
|
||||
from logging import Logger, getLogger
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
|
|
@ -40,7 +43,6 @@ def get_retry_request(
|
|||
max_retry_times: int | None = None,
|
||||
priority_adjust: int | None = None,
|
||||
logger: Logger = retry_logger,
|
||||
give_up_log_level: int | str | None = None,
|
||||
stats_base_key: str = "retry",
|
||||
) -> Request | None:
|
||||
"""
|
||||
|
|
@ -49,16 +51,14 @@ def get_retry_request(
|
|||
exhausted.
|
||||
|
||||
For example, in a :class:`~scrapy.Spider` callback, you could use it as
|
||||
follows:
|
||||
|
||||
.. code-block:: python
|
||||
follows::
|
||||
|
||||
def parse(self, response):
|
||||
if not response.text:
|
||||
new_request_or_none = get_retry_request(
|
||||
response.request,
|
||||
spider=self,
|
||||
reason="empty",
|
||||
reason='empty',
|
||||
)
|
||||
return new_request_or_none
|
||||
|
||||
|
|
@ -67,9 +67,8 @@ def get_retry_request(
|
|||
and :ref:`stats <topics-stats>`, and to provide extra logging context (see
|
||||
:func:`logging.debug`).
|
||||
|
||||
*reason* is a string, an :class:`Exception` subclass or an
|
||||
:class:`Exception` object that indicates the reason why the request needs
|
||||
to be retried. It is used to name retry stats.
|
||||
*reason* is a string or an :class:`Exception` object that indicates the
|
||||
reason why the request needs to be retried. It is used to name retry stats.
|
||||
|
||||
*max_retry_times* is a number that determines the maximum number of times
|
||||
that *request* can be retried. If not specified or ``None``, the number is
|
||||
|
|
@ -83,13 +82,6 @@ def get_retry_request(
|
|||
|
||||
*logger* is the logging.Logger object to be used when logging messages
|
||||
|
||||
*give_up_log_level* is the :ref:`logging level <levels>` used for the
|
||||
message logged when a request exceeds its retries. See
|
||||
:setting:`RETRY_GIVE_UP_LOG_LEVEL` for details.
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
The *give_up_log_level* parameter.
|
||||
|
||||
*stats_base_key* is a string to be used as the base key for the
|
||||
retry-related job stats
|
||||
"""
|
||||
|
|
@ -122,16 +114,8 @@ def get_retry_request(
|
|||
stats.inc_value(f"{stats_base_key}/count")
|
||||
stats.inc_value(f"{stats_base_key}/reason_count/{reason}")
|
||||
return new_request
|
||||
if give_up_log_level is None:
|
||||
give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"]
|
||||
if isinstance(give_up_log_level, str):
|
||||
level = getLevelName(give_up_log_level)
|
||||
if not isinstance(level, int):
|
||||
raise ValueError(f"Invalid give-up log level: {give_up_log_level!r}")
|
||||
give_up_log_level = level
|
||||
stats.inc_value(f"{stats_base_key}/max_reached")
|
||||
logger.log(
|
||||
give_up_log_level,
|
||||
logger.error(
|
||||
"Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s",
|
||||
{"request": request, "retry_times": retry_times, "reason": reason},
|
||||
extra={"spider": spider},
|
||||
|
|
@ -148,7 +132,6 @@ class RetryMiddleware:
|
|||
self.max_retry_times = settings.getint("RETRY_TIMES")
|
||||
self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")}
|
||||
self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
|
||||
self.give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"]
|
||||
self.exceptions_to_retry = tuple(
|
||||
load_object(x) if isinstance(x, str) else x
|
||||
for x in settings.getlist("RETRY_EXCEPTIONS")
|
||||
|
|
@ -192,9 +175,6 @@ class RetryMiddleware:
|
|||
) -> Request | None:
|
||||
max_retry_times = request.meta.get("max_retry_times", self.max_retry_times)
|
||||
priority_adjust = request.meta.get("priority_adjust", self.priority_adjust)
|
||||
give_up_log_level = request.meta.get(
|
||||
"give_up_log_level", self.give_up_log_level
|
||||
)
|
||||
assert self.crawler.spider
|
||||
return get_retry_request(
|
||||
request,
|
||||
|
|
@ -202,5 +182,4 @@ class RetryMiddleware:
|
|||
spider=self.crawler.spider,
|
||||
max_retry_times=max_retry_times,
|
||||
priority_adjust=priority_adjust,
|
||||
give_up_log_level=give_up_log_level,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -115,7 +115,7 @@ class UsageError(Exception):
|
|||
|
||||
class ScrapyDeprecationWarning(Warning):
|
||||
"""Warning category for deprecated features, since the default
|
||||
:exc:`DeprecationWarning` is silenced.
|
||||
DeprecationWarning is silenced on Python 2.7+
|
||||
"""
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ Item Exporters are used to export/serialize items into different formats.
|
|||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import logging
|
||||
import marshal
|
||||
import pickle
|
||||
import pprint
|
||||
|
|
@ -25,8 +24,6 @@ from scrapy.utils.serialize import ScrapyJSONEncoder
|
|||
if TYPE_CHECKING:
|
||||
from json import JSONEncoder
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"BaseItemExporter",
|
||||
"CsvItemExporter",
|
||||
|
|
@ -174,15 +171,7 @@ class XmlItemExporter(BaseItemExporter):
|
|||
super().__init__(**kwargs)
|
||||
if not self.encoding:
|
||||
self.encoding = "utf-8"
|
||||
# copied from xml.sax.saxutils._gettextwriter()
|
||||
self.stream = TextIOWrapper(
|
||||
file,
|
||||
encoding=self.encoding,
|
||||
errors="xmlcharrefreplace",
|
||||
newline="\n",
|
||||
write_through=True,
|
||||
)
|
||||
self.xg = XMLGenerator(self.stream, encoding=self.encoding)
|
||||
self.xg = XMLGenerator(file, encoding=self.encoding)
|
||||
|
||||
def _beautify_newline(self, new_item: bool = False) -> None:
|
||||
if self.indent is not None and (self.indent > 0 or new_item):
|
||||
|
|
@ -210,7 +199,6 @@ class XmlItemExporter(BaseItemExporter):
|
|||
def finish_exporting(self) -> None:
|
||||
self.xg.endElement(self.root_element)
|
||||
self.xg.endDocument()
|
||||
self.stream.detach() # Avoid closing the wrapped file.
|
||||
|
||||
def _export_xml_field(self, name: str, serialized_value: Any, depth: int) -> None:
|
||||
self._beautify_indent(depth=depth)
|
||||
|
|
@ -257,8 +245,6 @@ class CsvItemExporter(BaseItemExporter):
|
|||
self.csv_writer = csv.writer(self.stream, **self._kwargs)
|
||||
self._headers_not_written = True
|
||||
self._join_multivalued = join_multivalued
|
||||
self._autodetected_fields = False
|
||||
self._data_loss_warned = False
|
||||
|
||||
def serialize_field(
|
||||
self, field: Mapping[str, Any] | Field, name: str, value: Any
|
||||
|
|
@ -279,22 +265,6 @@ class CsvItemExporter(BaseItemExporter):
|
|||
self._headers_not_written = False
|
||||
self._write_headers_and_set_fields_to_export(item)
|
||||
|
||||
if (
|
||||
self._autodetected_fields
|
||||
and self.fields_to_export is not None
|
||||
and not self._data_loss_warned
|
||||
):
|
||||
item_fields = ItemAdapter(item).field_names()
|
||||
dropped_fields = set(item_fields) - set(self.fields_to_export)
|
||||
|
||||
if dropped_fields:
|
||||
dropped_fields_display = sorted(dropped_fields)
|
||||
logger.warning(
|
||||
f"CSVExporter dropped fields {dropped_fields_display}. "
|
||||
f"To avoid this, fully configure your FEED_EXPORT_FIELDS setting. "
|
||||
f"See: https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-fields",
|
||||
)
|
||||
self._data_loss_warned = True
|
||||
fields = self._get_serialized_fields(item, default_value="", include_empty=True)
|
||||
values = list(self._build_row(x for _, x in fields))
|
||||
self.csv_writer.writerow(values)
|
||||
|
|
@ -314,7 +284,6 @@ class CsvItemExporter(BaseItemExporter):
|
|||
if not self.fields_to_export:
|
||||
# use declared field names, or keys if the item is a dict
|
||||
self.fields_to_export = ItemAdapter(item).field_names()
|
||||
self._autodetected_fields = True
|
||||
fields: Iterable[str]
|
||||
if isinstance(self.fields_to_export, Mapping):
|
||||
fields = self.fields_to_export.values()
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from urllib.parse import unquote, urlparse
|
|||
|
||||
from twisted.internet.defer import Deferred, DeferredList
|
||||
from w3lib.url import file_uri_to_path
|
||||
from zope.interface import Interface
|
||||
from zope.interface import Interface, implementer
|
||||
|
||||
from scrapy import Spider, signals
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
|
|
@ -47,33 +47,6 @@ if TYPE_CHECKING:
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Printf-style placeholders (e.g. %(time)s) used to build feed URIs. Any other
|
||||
# percent character in a URI (e.g. percent-encoding such as %20 or %23) must be
|
||||
# treated as a literal rather than as the start of a placeholder.
|
||||
_FEED_URI_PLACEHOLDER_RE = re.compile(
|
||||
r"%\([^)]+\)[-+ #0]*(?:\d+|\*)?(?:\.(?:\d+|\*))?[diouxXeEfFgGcrsa]"
|
||||
)
|
||||
|
||||
|
||||
def apply_uri_params(uri_template: str, uri_params: dict[str, Any]) -> str:
|
||||
"""Return *uri_template* with its ``%(...)s`` placeholders replaced using
|
||||
*uri_params*, leaving any other percent character untouched.
|
||||
|
||||
This allows feed URIs to contain percent-encoded characters (e.g. ``%20``
|
||||
in a path with spaces or ``%23`` in FTP credentials) without them being
|
||||
misinterpreted as printf-style formatting directives.
|
||||
"""
|
||||
parts: list[str] = []
|
||||
last = 0
|
||||
for match in _FEED_URI_PLACEHOLDER_RE.finditer(uri_template):
|
||||
parts.append(uri_template[last : match.start()].replace("%", "%%"))
|
||||
parts.append(match.group(0))
|
||||
last = match.end()
|
||||
parts.append(uri_template[last:].replace("%", "%%"))
|
||||
return "".join(parts) % uri_params
|
||||
|
||||
|
||||
UriParamsCallableT: TypeAlias = Callable[
|
||||
[dict[str, Any], Spider], dict[str, Any] | None
|
||||
]
|
||||
|
|
@ -115,18 +88,25 @@ class ItemFilter:
|
|||
return True # accept all items by default
|
||||
|
||||
|
||||
class _IFeedStorage(Interface): # type: ignore[misc] # pragma: no cover
|
||||
class IFeedStorage(Interface): # type: ignore[misc]
|
||||
"""Interface that all Feed Storages must implement"""
|
||||
|
||||
# pylint: disable=no-self-argument
|
||||
|
||||
def __init__(uri, *, feed_options=None): ... # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called
|
||||
def __init__(uri, *, feed_options=None): # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called
|
||||
"""Initialize the storage with the parameters given in the URI and the
|
||||
feed-specific options (see :setting:`FEEDS`)"""
|
||||
|
||||
def open(spider): ... # type: ignore[no-untyped-def]
|
||||
def open(spider): # type: ignore[no-untyped-def]
|
||||
"""Open the storage for the given spider. It must return a file-like
|
||||
object that will be used for the exporters"""
|
||||
|
||||
def store(file): ... # type: ignore[no-untyped-def]
|
||||
def store(file): # type: ignore[no-untyped-def]
|
||||
"""Store the given file stream"""
|
||||
|
||||
|
||||
class FeedStorageProtocol(Protocol):
|
||||
"""Protocol that all Feed Storages must follow."""
|
||||
"""Reimplementation of ``IFeedStorage`` that can be used in type hints."""
|
||||
|
||||
def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
|
||||
"""Initialize the storage with the parameters given in the URI and the
|
||||
|
|
@ -140,6 +120,7 @@ class FeedStorageProtocol(Protocol):
|
|||
"""Store the given file stream"""
|
||||
|
||||
|
||||
@implementer(IFeedStorage)
|
||||
class BlockingFeedStorage(ABC):
|
||||
def open(self, spider: Spider) -> IO[bytes]:
|
||||
path = spider.crawler.settings["FEED_TEMPDIR"]
|
||||
|
|
@ -156,6 +137,7 @@ class BlockingFeedStorage(ABC):
|
|||
raise NotImplementedError
|
||||
|
||||
|
||||
@implementer(IFeedStorage)
|
||||
class StdoutFeedStorage:
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -182,6 +164,7 @@ class StdoutFeedStorage:
|
|||
pass
|
||||
|
||||
|
||||
@implementer(IFeedStorage)
|
||||
class FileFeedStorage:
|
||||
def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
|
||||
self.path: str = file_uri_to_path(uri) if uri.startswith("file:") else uri
|
||||
|
|
@ -267,22 +250,20 @@ class S3FeedStorage(BlockingFeedStorage):
|
|||
|
||||
def _store_in_thread(self, file: IO[bytes]) -> None:
|
||||
file.seek(0)
|
||||
try:
|
||||
if self.acl:
|
||||
self.s3_client.upload_fileobj(
|
||||
Bucket=self.bucketname,
|
||||
Key=self.keyname,
|
||||
Fileobj=file,
|
||||
ExtraArgs={"ACL": self.acl},
|
||||
)
|
||||
else:
|
||||
self.s3_client.upload_fileobj(
|
||||
Bucket=self.bucketname,
|
||||
Key=self.keyname,
|
||||
Fileobj=file,
|
||||
)
|
||||
finally:
|
||||
file.close()
|
||||
if self.acl:
|
||||
self.s3_client.upload_fileobj(
|
||||
Bucket=self.bucketname,
|
||||
Key=self.keyname,
|
||||
Fileobj=file,
|
||||
ExtraArgs={"ACL": self.acl},
|
||||
)
|
||||
else:
|
||||
self.s3_client.upload_fileobj(
|
||||
Bucket=self.bucketname,
|
||||
Key=self.keyname,
|
||||
Fileobj=file,
|
||||
)
|
||||
file.close()
|
||||
|
||||
|
||||
class GCSFeedStorage(BlockingFeedStorage):
|
||||
|
|
@ -325,15 +306,12 @@ class GCSFeedStorage(BlockingFeedStorage):
|
|||
|
||||
def _store_in_thread(self, file: IO[bytes]) -> None:
|
||||
file.seek(0)
|
||||
try:
|
||||
from google.cloud.storage import Client # noqa: PLC0415
|
||||
from google.cloud.storage import Client # noqa: PLC0415
|
||||
|
||||
client = Client(project=self.project_id)
|
||||
bucket = client.get_bucket(self.bucket_name)
|
||||
blob = bucket.blob(self.blob_name)
|
||||
blob.upload_from_file(file, predefined_acl=self.acl)
|
||||
finally:
|
||||
file.close()
|
||||
client = Client(project=self.project_id)
|
||||
bucket = client.get_bucket(self.bucket_name)
|
||||
blob = bucket.blob(self.blob_name)
|
||||
blob.upload_from_file(file, predefined_acl=self.acl)
|
||||
|
||||
|
||||
class FTPFeedStorage(BlockingFeedStorage):
|
||||
|
|
@ -490,7 +468,7 @@ class FeedExporter:
|
|||
)
|
||||
uri = self.settings["FEED_URI"]
|
||||
# handle pathlib.Path objects
|
||||
uri = str(uri.absolute()) if isinstance(uri, Path) else str(uri)
|
||||
uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
|
||||
feed_options = {"format": self.settings["FEED_FORMAT"]}
|
||||
self.feeds[uri] = feed_complete_default_values_from_settings(
|
||||
feed_options, self.settings
|
||||
|
|
@ -502,9 +480,9 @@ class FeedExporter:
|
|||
for settings_uri, feed_options in self.settings.getdict("FEEDS").items():
|
||||
# handle pathlib.Path objects
|
||||
uri = (
|
||||
str(settings_uri.absolute())
|
||||
if isinstance(settings_uri, Path)
|
||||
else str(settings_uri)
|
||||
str(settings_uri)
|
||||
if not isinstance(settings_uri, Path)
|
||||
else settings_uri.absolute().as_uri()
|
||||
)
|
||||
self.feeds[uri] = feed_complete_default_values_from_settings(
|
||||
feed_options, self.settings
|
||||
|
|
@ -531,7 +509,7 @@ class FeedExporter:
|
|||
self.slots.append(
|
||||
self._start_new_batch(
|
||||
batch_id=1,
|
||||
uri=apply_uri_params(uri, uri_params),
|
||||
uri=uri % uri_params,
|
||||
feed_options=feed_options,
|
||||
spider=spider,
|
||||
uri_template=uri,
|
||||
|
|
@ -656,7 +634,7 @@ class FeedExporter:
|
|||
slots.append(
|
||||
self._start_new_batch(
|
||||
batch_id=slot.batch_id + 1,
|
||||
uri=apply_uri_params(slot.uri_template, uri_params),
|
||||
uri=slot.uri_template % uri_params,
|
||||
feed_options=self.feeds[slot.uri_template],
|
||||
spider=spider,
|
||||
uri_template=slot.uri_template,
|
||||
|
|
@ -748,14 +726,3 @@ class FeedExporter:
|
|||
feed_options.get("item_filter", ItemFilter)
|
||||
)
|
||||
return item_filter_class(feed_options)
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any: # pragma: no cover
|
||||
if name == "IFeedStorage":
|
||||
warnings.warn(
|
||||
"scrapy.extensions.feedexport.IFeedStorage is deprecated.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return _IFeedStorage
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
class LogStats:
|
||||
"""Log basic scraping stats periodically like:
|
||||
* RPM - Responses per Minute
|
||||
* RPM - Requests per Minute
|
||||
* IPM - Items per Minute
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -113,7 +113,7 @@ class MemoryUsage:
|
|||
{"memusage": mem},
|
||||
extra={"crawler": self.crawler},
|
||||
)
|
||||
if self.notify_mails: # pragma: no cover
|
||||
if self.notify_mails:
|
||||
subj = (
|
||||
f"{self.crawler.settings['BOT_NAME']} terminated: "
|
||||
f"memory usage exceeded {mem}MiB at {socket.gethostname()}"
|
||||
|
|
@ -146,7 +146,7 @@ class MemoryUsage:
|
|||
{"memusage": mem},
|
||||
extra={"crawler": self.crawler},
|
||||
)
|
||||
if self.notify_mails: # pragma: no cover
|
||||
if self.notify_mails:
|
||||
subj = (
|
||||
f"{self.crawler.settings['BOT_NAME']} warning: "
|
||||
f"memory usage reached {mem}MiB at {socket.gethostname()}"
|
||||
|
|
@ -155,7 +155,7 @@ class MemoryUsage:
|
|||
self.crawler.stats.set_value("memusage/warning_notified", 1)
|
||||
self.warned = True
|
||||
|
||||
def _send_report(self, rcpts: list[str], subject: str) -> None: # pragma: no cover
|
||||
def _send_report(self, rcpts: list[str], subject: str) -> None:
|
||||
"""send notification mail with some additional useful info"""
|
||||
assert self.crawler.engine
|
||||
assert self.crawler.stats
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ class AutoThrottle:
|
|||
# It works better with problematic sites.
|
||||
new_delay = max(target_delay, new_delay)
|
||||
|
||||
# Make sure self.mindelay <= new_delay <= self.maxdelay
|
||||
# Make sure self.mindelay <= new_delay <= self.max_delay
|
||||
new_delay = min(max(self.mindelay, new_delay), self.maxdelay)
|
||||
|
||||
# Dont adjust delay if response status != 200 and new delay is smaller
|
||||
|
|
|
|||
|
|
@ -5,9 +5,11 @@ Use this module (instead of the more specific ones) when importing Headers,
|
|||
Request and Response outside this module.
|
||||
"""
|
||||
|
||||
from warnings import catch_warnings, filterwarnings
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http.headers import Headers
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.http.request.form import FormRequest
|
||||
from scrapy.http.request.json_request import JsonRequest
|
||||
from scrapy.http.request.rpc import XmlRpcRequest
|
||||
from scrapy.http.response import Response
|
||||
|
|
@ -15,6 +17,19 @@ from scrapy.http.response.html import HtmlResponse
|
|||
from scrapy.http.response.json import JsonResponse
|
||||
from scrapy.http.response.text import TextResponse
|
||||
from scrapy.http.response.xml import XmlResponse
|
||||
from scrapy.utils.deprecate import create_deprecated_class
|
||||
|
||||
with catch_warnings():
|
||||
filterwarnings("ignore", category=ScrapyDeprecationWarning)
|
||||
|
||||
from scrapy.http.request.form import FormRequest as _FormRequest
|
||||
|
||||
FormRequest = create_deprecated_class(
|
||||
name="FormRequest",
|
||||
new_class=_FormRequest,
|
||||
subclass_warn_message="{cls} inherits from deprecated class {old}, use the form2request library instead.",
|
||||
instance_warn_message="{cls} is deprecated, use the form2request library instead.",
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"FormRequest",
|
||||
|
|
|
|||
|
|
@ -136,9 +136,9 @@ class _DummyLock:
|
|||
|
||||
|
||||
class WrappedRequest:
|
||||
"""Wraps a :class:`scrapy.Request` class with methods defined by
|
||||
the :class:`urllib.request.Request` class to interact with
|
||||
the :class:`http.cookiejar.CookieJar` class.
|
||||
"""Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class
|
||||
|
||||
see http://docs.python.org/library/urllib2.html#urllib2.Request
|
||||
"""
|
||||
|
||||
def __init__(self, request: Request):
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ class VerboseCookie(TypedDict):
|
|||
secure: NotRequired[bool]
|
||||
|
||||
|
||||
CookiesT: TypeAlias = dict[str | bytes, str | bytes] | list[VerboseCookie]
|
||||
CookiesT: TypeAlias = dict[str, str] | list[VerboseCookie]
|
||||
|
||||
|
||||
RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
|
||||
|
|
@ -138,7 +138,6 @@ class Request(object_ref):
|
|||
) -> None:
|
||||
self._encoding: str = encoding # this one has to be set first
|
||||
self.method: str = str(method).upper()
|
||||
self._meta: dict[str, Any] | None = dict(meta) if meta else None
|
||||
self._set_url(url)
|
||||
self._set_body(body)
|
||||
if not isinstance(priority, int):
|
||||
|
|
@ -233,6 +232,7 @@ class Request(object_ref):
|
|||
#: default. See :meth:`~scrapy.Spider.start`.
|
||||
self.dont_filter: bool = dont_filter
|
||||
|
||||
self._meta: dict[str, Any] | None = dict(meta) if meta else None
|
||||
self._cb_kwargs: dict[str, Any] | None = dict(cb_kwargs) if cb_kwargs else None
|
||||
self._flags: list[str] | None = list(flags) if flags else None
|
||||
|
||||
|
|
@ -252,17 +252,11 @@ class Request(object_ref):
|
|||
def url(self) -> str:
|
||||
return self._url
|
||||
|
||||
def _url_is_verbatim(self) -> bool:
|
||||
return bool(self._meta and self._meta.get("verbatim_url"))
|
||||
|
||||
def _set_url(self, url: str) -> None:
|
||||
if not isinstance(url, str):
|
||||
raise TypeError(f"Request url must be str, got {type(url).__name__}")
|
||||
|
||||
if self._url_is_verbatim():
|
||||
self._url = url
|
||||
else:
|
||||
self._url = safe_url_string(url, self.encoding)
|
||||
self._url = safe_url_string(url, self.encoding)
|
||||
|
||||
if (
|
||||
"://" not in self._url
|
||||
|
|
|
|||
|
|
@ -32,61 +32,19 @@ if TYPE_CHECKING:
|
|||
|
||||
from scrapy.http.response.text import TextResponse
|
||||
|
||||
warn(
|
||||
"The entire scrapy.http.request.form module is deprecated. Use the "
|
||||
"form2request library instead.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
FormdataVType: TypeAlias = str | Iterable[str]
|
||||
FormdataKVType: TypeAlias = tuple[str, FormdataVType]
|
||||
FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None
|
||||
|
||||
|
||||
class FormRequest(Request):
|
||||
"""A :class:`~scrapy.Request` subclass with a ``formdata`` parameter that
|
||||
url-encodes the given data and assigns it to the request, which makes it
|
||||
convenient to send arbitrary form data via HTTP POST or GET without an HTML
|
||||
``<form>`` element to parse.
|
||||
|
||||
.. note:: To build a request from an HTML ``<form>`` element found in a
|
||||
response, use :doc:`form2request <form2request:index>` instead. See
|
||||
:ref:`form`.
|
||||
|
||||
The remaining arguments are the same as for the :class:`~scrapy.Request`
|
||||
class and are not documented here.
|
||||
|
||||
:param formdata: a dictionary (or iterable of (key, value) tuples)
|
||||
containing HTML form data which will be url-encoded. If
|
||||
:attr:`~scrapy.Request.method` is not given and ``formdata`` is
|
||||
provided, the method is set to ``"POST"`` and the data is assigned to
|
||||
the request body; if the method is ``"GET"``, the data is added to the
|
||||
URL query string instead.
|
||||
:type formdata: dict or collections.abc.Iterable
|
||||
|
||||
To send data via HTTP POST, simulating an HTML form submission, return a
|
||||
:class:`~scrapy.FormRequest` object from your spider:
|
||||
|
||||
.. skip: next
|
||||
.. code-block:: python
|
||||
|
||||
return [
|
||||
FormRequest(
|
||||
url="http://www.example.com/post/action",
|
||||
formdata={"name": "John Doe", "age": "27"},
|
||||
callback=self.after_post,
|
||||
)
|
||||
]
|
||||
|
||||
To send the data in the URL query string instead, use the ``GET`` method:
|
||||
|
||||
.. skip: next
|
||||
.. code-block:: python
|
||||
|
||||
return [
|
||||
FormRequest(
|
||||
url="http://www.example.com/search",
|
||||
method="GET",
|
||||
formdata={"q": "keyword", "page": "1"},
|
||||
callback=self.parse_results,
|
||||
)
|
||||
]
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
valid_form_methods: ClassVar[list[str]] = ["GET", "POST"]
|
||||
|
|
@ -126,13 +84,6 @@ class FormRequest(Request):
|
|||
formcss: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Self:
|
||||
warn(
|
||||
"FormRequest.from_response() is deprecated. Use the form2request "
|
||||
"library instead.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
kwargs.setdefault("encoding", response.encoding)
|
||||
|
||||
if formcss is not None:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
"""
|
||||
This module implements the :class:`HtmlResponse` class which is used as a
|
||||
content type marker by :class:`~scrapy.selector.Selector` and can be used in
|
||||
``isinstance()`` checks.
|
||||
This module implements the HtmlResponse class which adds encoding
|
||||
discovering through HTML encoding declarations to the TextResponse class.
|
||||
|
||||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
"""
|
||||
This module implements the :class:`XmlResponse` class which is used as a
|
||||
content type marker by :class:`~scrapy.selector.Selector` and can be used in
|
||||
``isinstance()`` checks.
|
||||
This module implements the XmlResponse class which adds encoding
|
||||
discovering through XML encoding declarations to the TextResponse class.
|
||||
|
||||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,23 +1,19 @@
|
|||
# pragma: no file cover
|
||||
# pylint: disable=no-method-argument,no-self-argument
|
||||
import warnings
|
||||
|
||||
from zope.interface import Interface
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
|
||||
warnings.warn(
|
||||
"The scrapy.interfaces module is deprecated.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
|
||||
class ISpiderLoader(Interface):
|
||||
def from_settings(settings): ...
|
||||
def from_settings(settings):
|
||||
"""Return an instance of the class for the given settings"""
|
||||
|
||||
def load(spider_name): ...
|
||||
def load(spider_name):
|
||||
"""Return the Spider class for the given spider name. If the spider
|
||||
name is not found, it must raise a KeyError."""
|
||||
|
||||
def list(): ...
|
||||
def list():
|
||||
"""Return a list with the names of all spiders available in the
|
||||
project"""
|
||||
|
||||
def find_by_request(request): ...
|
||||
def find_by_request(request):
|
||||
"""Return the list of spiders names that can handle the given request"""
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""
|
||||
Scrapy Item
|
||||
|
||||
See documentation in docs/topics/items.rst
|
||||
See documentation in docs/topics/item.rst
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -25,25 +25,6 @@ class Field(dict[str, Any]):
|
|||
"""Container of field metadata"""
|
||||
|
||||
|
||||
def _ordered_field_names(cls: type) -> list[str]:
|
||||
"""Return the names of the :class:`Field` attributes of *cls* in definition
|
||||
order.
|
||||
|
||||
Fields declared in base classes come first, ordered from the topmost base
|
||||
to the most derived class. Within each class, fields keep their definition
|
||||
order. A field redefined in a subclass keeps the position of its first
|
||||
definition.
|
||||
"""
|
||||
names: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for base in reversed(cls.__mro__):
|
||||
for name, value in vars(base).items():
|
||||
if isinstance(value, Field) and name not in seen:
|
||||
seen.add(name)
|
||||
names.append(name)
|
||||
return names
|
||||
|
||||
|
||||
class ItemMeta(ABCMeta):
|
||||
"""Metaclass_ of :class:`Item` that handles field definitions.
|
||||
|
||||
|
|
@ -58,9 +39,13 @@ class ItemMeta(ABCMeta):
|
|||
_class = super().__new__(mcs, "x_" + class_name, new_bases, attrs)
|
||||
|
||||
fields = getattr(_class, "fields", {})
|
||||
for n in _ordered_field_names(_class):
|
||||
fields[n] = getattr(_class, n)
|
||||
new_attrs = {n: v for n, v in attrs.items() if not isinstance(v, Field)}
|
||||
new_attrs = {}
|
||||
for n in dir(_class):
|
||||
v = getattr(_class, n)
|
||||
if isinstance(v, Field):
|
||||
fields[n] = v
|
||||
elif n in attrs:
|
||||
new_attrs[n] = attrs[n]
|
||||
|
||||
new_attrs["fields"] = fields
|
||||
new_attrs["_class"] = _class
|
||||
|
|
@ -95,14 +80,6 @@ class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta):
|
|||
#: those populated. The keys are the field names and the values are the
|
||||
#: :class:`Field` objects used in the :ref:`Item declaration
|
||||
#: <topics-items-declaring>`.
|
||||
#:
|
||||
#: Fields are kept in definition order: fields declared in base classes
|
||||
#: come first, followed by fields declared in subclasses, and a field
|
||||
#: redefined in a subclass keeps the position of its first definition.
|
||||
#:
|
||||
#: .. versionchanged:: 2.17.0
|
||||
#: Fields are now returned in definition order rather than alphabetical
|
||||
#: order.
|
||||
fields: dict[str, Field]
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any):
|
||||
|
|
|
|||
|
|
@ -9,9 +9,7 @@ its documentation in: docs/topics/link-extractors.rst
|
|||
class Link:
|
||||
"""Link objects represent an extracted link by the LinkExtractor.
|
||||
|
||||
Using the anchor tag sample below to illustrate the parameters:
|
||||
|
||||
.. code-block:: html
|
||||
Using the anchor tag sample below to illustrate the parameters::
|
||||
|
||||
<a href="https://example.com/nofollow.html#foo" rel="nofollow">Dont follow this one</a>
|
||||
|
||||
|
|
@ -41,7 +39,7 @@ class Link:
|
|||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
if not isinstance(other, Link):
|
||||
return NotImplemented
|
||||
raise NotImplementedError
|
||||
return (
|
||||
self.url == other.url
|
||||
and self.text == other.text
|
||||
|
|
|
|||
|
|
@ -57,18 +57,6 @@ def _canonicalize_link_url(link: Link) -> str:
|
|||
return canonicalize_url(link.url, keep_fragments=True)
|
||||
|
||||
|
||||
def _name_matches(allowed: set[str], denied: set[str], name: str) -> bool:
|
||||
"""Return whether a tag or attribute *name* should be considered.
|
||||
|
||||
A name matches when it is allowed and not denied. A name is allowed if it is
|
||||
listed in *allowed*, or if *allowed* contains the ``"*"`` wildcard, which
|
||||
matches every name. *denied* has precedence over *allowed*.
|
||||
"""
|
||||
if name in denied:
|
||||
return False
|
||||
return "*" in allowed or name in allowed
|
||||
|
||||
|
||||
class LxmlParserLinkExtractor:
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -174,126 +162,6 @@ _RegexOrSeveral: TypeAlias = _Regex | Iterable[_Regex]
|
|||
|
||||
|
||||
class LxmlLinkExtractor:
|
||||
r"""LxmlLinkExtractor is the recommended link extractor with handy filtering
|
||||
options. It is implemented using lxml's robust HTMLParser.
|
||||
|
||||
:param allow: a single regular expression (or list of regular expressions)
|
||||
that the (absolute) urls must match in order to be extracted. If not
|
||||
given (or empty), it will match all links.
|
||||
:type allow: str or list
|
||||
|
||||
:param deny: a single regular expression (or list of regular expressions)
|
||||
that the (absolute) urls must match in order to be excluded (i.e. not
|
||||
extracted). It has precedence over the ``allow`` parameter. If not
|
||||
given (or empty) it won't exclude any links.
|
||||
:type deny: str or list
|
||||
|
||||
:param allow_domains: a single value or a list of string containing
|
||||
domains which will be considered for extracting the links
|
||||
:type allow_domains: str or list
|
||||
|
||||
:param deny_domains: a single value or a list of strings containing
|
||||
domains which won't be considered for extracting the links
|
||||
:type deny_domains: str or list
|
||||
|
||||
:param deny_extensions: a single value or list of strings containing
|
||||
extensions that should be ignored when extracting links.
|
||||
If not given, it will default to
|
||||
:data:`scrapy.linkextractors.IGNORED_EXTENSIONS`.
|
||||
:type deny_extensions: list
|
||||
|
||||
:param restrict_xpaths: is an XPath (or list of XPath's) which defines
|
||||
regions inside the response where links should be extracted from.
|
||||
If given, only the text selected by those XPath will be scanned for
|
||||
links.
|
||||
:type restrict_xpaths: str or list
|
||||
|
||||
:param restrict_css: a CSS selector (or list of selectors) which defines
|
||||
regions inside the response where links should be extracted from.
|
||||
Has the same behaviour as ``restrict_xpaths``.
|
||||
:type restrict_css: str or list
|
||||
|
||||
:param restrict_text: a single regular expression (or list of regular
|
||||
expressions) that the link's text must match in order to be extracted.
|
||||
If not given (or empty), it will match all links. If a list of regular
|
||||
expressions is given, the link will be extracted if it matches at least
|
||||
one.
|
||||
:type restrict_text: str or list
|
||||
|
||||
:param tags: a tag or a list of tags to consider when extracting links.
|
||||
Defaults to ``('a', 'area')``. Use ``'*'`` to consider every tag.
|
||||
:type tags: str or list
|
||||
|
||||
:param attrs: an attribute or list of attributes which should be considered
|
||||
when looking for links to extract (only for those tags specified in the
|
||||
``tags`` parameter). Defaults to ``('href',)``. Use ``'*'`` to consider
|
||||
every attribute.
|
||||
:type attrs: list
|
||||
|
||||
:param deny_tags: a tag or a list of tags that should not be considered when
|
||||
extracting links. It has precedence over the ``tags`` parameter, so it
|
||||
can be combined with ``tags='*'`` to consider every tag except a few.
|
||||
Defaults to ``()`` (no tag is excluded).
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
:type deny_tags: str or list
|
||||
|
||||
:param deny_attrs: an attribute or a list of attributes that should not be
|
||||
considered when looking for links to extract. It has precedence over the
|
||||
``attrs`` parameter, so it can be combined with ``attrs='*'`` to consider
|
||||
every attribute except a few. Defaults to ``()`` (no attribute is
|
||||
excluded).
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
:type deny_attrs: str or list
|
||||
|
||||
:param canonicalize: canonicalize each extracted url (using
|
||||
w3lib.url.canonicalize_url). Defaults to ``False``.
|
||||
Note that canonicalize_url is meant for duplicate checking;
|
||||
it can change the URL visible at server side, so the response can be
|
||||
different for requests with canonicalized and raw URLs. If you're
|
||||
using LinkExtractor to follow links it is more robust to
|
||||
keep the default ``canonicalize=False``.
|
||||
:type canonicalize: bool
|
||||
|
||||
:param unique: whether duplicate filtering should be applied to extracted
|
||||
links.
|
||||
:type unique: bool
|
||||
|
||||
:param process_value: a function which receives each value extracted from
|
||||
the tag and attributes scanned and can modify the value and return a
|
||||
new one, or return ``None`` to ignore the link altogether. If not
|
||||
given, ``process_value`` defaults to ``lambda x: x``.
|
||||
|
||||
.. highlight:: html
|
||||
|
||||
For example, to extract links from this code::
|
||||
|
||||
<a href="javascript:goToPage('../other/page.html'); return false">Link text</a>
|
||||
|
||||
.. highlight:: python
|
||||
|
||||
You can use the following function in ``process_value``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def process_value(value):
|
||||
m = re.search(r"javascript:goToPage\('(.*?)'", value)
|
||||
if m:
|
||||
return m.group(1)
|
||||
|
||||
:type process_value: collections.abc.Callable
|
||||
|
||||
:param strip: whether to strip whitespaces from extracted attributes.
|
||||
According to HTML5 standard, leading and trailing whitespaces
|
||||
must be stripped from ``href`` attributes of ``<a>``, ``<area>``
|
||||
and many other elements, ``src`` attribute of ``<img>``, ``<iframe>``
|
||||
elements, etc., so LinkExtractor strips space chars by default.
|
||||
Set ``strip=False`` to turn it off (e.g. if you're extracting urls
|
||||
from elements or attributes which allow leading/trailing whitespaces).
|
||||
:type strip: bool
|
||||
"""
|
||||
|
||||
_csstranslator = HTMLTranslator()
|
||||
|
||||
def __init__(
|
||||
|
|
@ -312,17 +180,11 @@ class LxmlLinkExtractor:
|
|||
restrict_css: str | Iterable[str] = (),
|
||||
strip: bool = True,
|
||||
restrict_text: _RegexOrSeveral | None = None,
|
||||
deny_tags: str | Iterable[str] = (),
|
||||
deny_attrs: str | Iterable[str] = (),
|
||||
):
|
||||
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
|
||||
deny_tags, deny_attrs = (
|
||||
set(arg_to_iter(deny_tags)),
|
||||
set(arg_to_iter(deny_attrs)),
|
||||
)
|
||||
self.link_extractor = LxmlParserLinkExtractor(
|
||||
tag=partial(_name_matches, tags, deny_tags),
|
||||
attr=partial(_name_matches, attrs, deny_attrs),
|
||||
tag=partial(operator.contains, tags),
|
||||
attr=partial(operator.contains, attrs),
|
||||
unique=unique,
|
||||
process=process_value,
|
||||
strip=strip,
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ class ItemLoader(itemloaders.ItemLoader):
|
|||
:param item: The item instance to populate using subsequent calls to
|
||||
:meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
|
||||
or :meth:`~ItemLoader.add_value`.
|
||||
:type item: :ref:`item object <item-types>`
|
||||
:type item: scrapy.item.Item
|
||||
|
||||
:param selector: The selector to extract data from, when using the
|
||||
:meth:`add_xpath`, :meth:`add_css`, :meth:`replace_xpath`, or
|
||||
|
|
|
|||
|
|
@ -58,20 +58,18 @@ class LogFormatter:
|
|||
logging an action the method must return ``None``.
|
||||
|
||||
Here is an example on how to create a custom log formatter to lower the severity level of
|
||||
the log message when an item is dropped from the pipeline:
|
||||
the log message when an item is dropped from the pipeline::
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
class PoliteLogFormatter(logformatter.LogFormatter):
|
||||
def dropped(self, item, exception, response, spider):
|
||||
return {
|
||||
"level": logging.INFO, # lowering the level from logging.WARNING
|
||||
"msg": "Dropped: %(exception)s" + os.linesep + "%(item)s",
|
||||
"args": {
|
||||
"exception": exception,
|
||||
"item": item,
|
||||
},
|
||||
}
|
||||
class PoliteLogFormatter(logformatter.LogFormatter):
|
||||
def dropped(self, item, exception, response, spider):
|
||||
return {
|
||||
'level': logging.INFO, # lowering the level from logging.WARNING
|
||||
'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s",
|
||||
'args': {
|
||||
'exception': exception,
|
||||
'item': item,
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
def crawled(
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
"""
|
||||
Mail sending helpers
|
||||
|
||||
See documentation in docs/topics/email.rst
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""
|
||||
Item pipeline
|
||||
|
||||
See documentation in docs/topics/item-pipeline.rst
|
||||
See documentation in docs/item-pipeline.rst
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
|
|||
|
|
@ -300,13 +300,13 @@ class GCSFilesStore:
|
|||
)
|
||||
if "storage.objects.get" not in permissions:
|
||||
logger.warning(
|
||||
"No 'storage.objects.get' permission for GCS bucket %(bucket)s. "
|
||||
"No 'storage.objects.get' permission for GSC bucket %(bucket)s. "
|
||||
"Checking if files are up to date will be impossible. Files will be downloaded every time.",
|
||||
{"bucket": bucket},
|
||||
)
|
||||
if "storage.objects.create" not in permissions:
|
||||
logger.error(
|
||||
"No 'storage.objects.create' permission for GCS bucket %(bucket)s. Saving files will be impossible!",
|
||||
"No 'storage.objects.create' permission for GSC bucket %(bucket)s. Saving files will be impossible!",
|
||||
{"bucket": bucket},
|
||||
)
|
||||
|
||||
|
|
@ -423,7 +423,7 @@ class FTPFilesStore:
|
|||
|
||||
|
||||
class FilesPipeline(MediaPipeline):
|
||||
"""Pipeline that implements file downloading.
|
||||
"""Abstract pipeline that implement the file downloading
|
||||
|
||||
This pipeline tries to minimize network transfers and file processing,
|
||||
doing stat of the files and determining if file is new, up-to-date or
|
||||
|
|
|
|||
|
|
@ -11,20 +11,14 @@ import hashlib
|
|||
import warnings
|
||||
from contextlib import suppress
|
||||
from io import BytesIO
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, cast
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
from itemadapter import ItemAdapter
|
||||
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.http.request import NO_CALLBACK
|
||||
from scrapy.pipelines.files import (
|
||||
FileException,
|
||||
FilesPipeline,
|
||||
GCSFilesStore,
|
||||
S3FilesStore,
|
||||
_md5sum,
|
||||
)
|
||||
from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum
|
||||
from scrapy.utils.defer import ensure_awaitable
|
||||
from scrapy.utils.python import to_bytes
|
||||
|
||||
|
|
@ -39,7 +33,6 @@ if TYPE_CHECKING:
|
|||
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.pipelines.media import FileInfoOrError, MediaPipeline
|
||||
from scrapy.settings import BaseSettings
|
||||
|
||||
|
||||
class ImageException(FileException):
|
||||
|
|
@ -47,7 +40,7 @@ class ImageException(FileException):
|
|||
|
||||
|
||||
class ImagesPipeline(FilesPipeline):
|
||||
"""Pipeline that implements the handling logic specific to images."""
|
||||
"""Abstract pipeline that implement the image thumbnail generation logic"""
|
||||
|
||||
MEDIA_NAME: str = "image"
|
||||
|
||||
|
|
@ -133,20 +126,6 @@ class ImagesPipeline(FilesPipeline):
|
|||
) -> str:
|
||||
return await self.image_downloaded(response, request, info, item=item)
|
||||
|
||||
@classmethod
|
||||
def _update_stores(cls, settings: BaseSettings) -> None:
|
||||
super()._update_stores(settings)
|
||||
|
||||
s3store: type[S3FilesStore] = cast(
|
||||
"type[S3FilesStore]", cls.STORE_SCHEMES["s3"]
|
||||
)
|
||||
s3store.POLICY = settings["IMAGES_STORE_S3_ACL"]
|
||||
|
||||
gcs_store: type[GCSFilesStore] = cast(
|
||||
"type[GCSFilesStore]", cls.STORE_SCHEMES["gs"]
|
||||
)
|
||||
gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None
|
||||
|
||||
async def image_downloaded(
|
||||
self,
|
||||
response: Response,
|
||||
|
|
|
|||
|
|
@ -92,10 +92,9 @@ class ScrapyPriorityQueue:
|
|||
- The :data:`~scrapy.Request.priority` of the request.
|
||||
|
||||
For each combination of the above seen, this class creates an instance of
|
||||
*downstream_queue_cls* (or *start_queue_cls* for start requests if it was
|
||||
passed) with *key* set to a subdirectory of the persistence directory,
|
||||
named as the negated request priority (e.g. ``-1``), with an ``s`` suffix
|
||||
in case of a start request (e.g. ``-1s``).
|
||||
*downstream_queue_cls* with *key* set to a subdirectory of the persistence
|
||||
directory, named as the request priority (e.g. ``1``), with an ``s`` suffix
|
||||
in case of a start request (e.g. ``1s``).
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
|
|
@ -142,14 +141,10 @@ class ScrapyPriorityQueue:
|
|||
q = self.qfactory(priority)
|
||||
if q:
|
||||
self.queues[priority] = q
|
||||
else:
|
||||
q.close()
|
||||
if self._start_queue_cls:
|
||||
q = self._sqfactory(priority)
|
||||
if q:
|
||||
self._start_queues[priority] = q
|
||||
else:
|
||||
q.close()
|
||||
|
||||
self.curprio = min(startprios)
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue