mirror of https://github.com/scrapy/scrapy.git
Merge remote-tracking branch 'origin/master' into feed-path
This commit is contained in:
Adrian Chaves
commit
87b90a66f8
|
|
@ -79,9 +79,6 @@ jobs:
|
|||
- python-version: "3.14"
|
||||
env:
|
||||
TOXENV: botocore
|
||||
- python-version: "3.14"
|
||||
env:
|
||||
TOXENV: mitmproxy
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
|
@ -97,6 +94,9 @@ jobs:
|
|||
sudo apt-get update
|
||||
sudo apt-get install libxml2-dev libxslt-dev
|
||||
|
||||
- name: Install mitmproxy
|
||||
run: pipx install mitmproxy
|
||||
|
||||
- name: Run tests
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ exclude: |
|
|||
)
|
||||
repos:
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.15.2
|
||||
rev: v0.15.20
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
args: [ --fix ]
|
||||
|
|
@ -16,7 +16,7 @@ repos:
|
|||
hooks:
|
||||
- id: blacken-docs
|
||||
additional_dependencies:
|
||||
- black==25.9.0
|
||||
- black==26.5.1
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v6.0.0
|
||||
hooks:
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@
|
|||
|
||||
| Version | Supported |
|
||||
| ------- | ------------------ |
|
||||
| 2.16.x | :white_check_mark: |
|
||||
| < 2.16.x | :x: |
|
||||
| 2.17.x | :white_check_mark: |
|
||||
| < 2.17.x | :x: |
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from scrapy.utils.reactor import set_asyncio_event_loop_policy
|
|||
from scrapy.utils.reactorless import install_reactor_import_hook
|
||||
from tests.keys import generate_keys
|
||||
from tests.mockserver.http import MockServer
|
||||
from tests.mockserver.mitm_proxy import MitmProxy
|
||||
from tests.mockserver.mitm_proxy import MitmProxy, mitmdump_cmd
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
|
|
@ -127,7 +127,6 @@ def pytest_runtest_setup(item):
|
|||
"uvloop",
|
||||
"botocore",
|
||||
"boto3",
|
||||
"mitmproxy",
|
||||
]
|
||||
|
||||
for module in optional_deps:
|
||||
|
|
@ -137,6 +136,9 @@ def pytest_runtest_setup(item):
|
|||
except ImportError:
|
||||
pytest.skip(f"{module} is not installed")
|
||||
|
||||
if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None:
|
||||
pytest.skip("mitmdump is not available")
|
||||
|
||||
|
||||
# Generate localhost certificate files, needed by some tests
|
||||
generate_keys()
|
||||
|
|
|
|||
|
|
@ -1,68 +0,0 @@
|
|||
:orphan:
|
||||
|
||||
======================================
|
||||
Scrapy documentation quick start guide
|
||||
======================================
|
||||
|
||||
This file provides a quick guide on how to compile the Scrapy documentation.
|
||||
|
||||
|
||||
Setup the environment
|
||||
---------------------
|
||||
|
||||
To compile the documentation you need Sphinx Python library. To install it
|
||||
and all its dependencies run the following command from this dir
|
||||
|
||||
::
|
||||
|
||||
pip install -r requirements.txt
|
||||
|
||||
|
||||
Compile the documentation
|
||||
-------------------------
|
||||
|
||||
To compile the documentation (to classic HTML output) run the following command
|
||||
from this dir::
|
||||
|
||||
make html
|
||||
|
||||
Documentation will be generated (in HTML format) inside the ``build/html`` dir.
|
||||
|
||||
|
||||
View the documentation
|
||||
----------------------
|
||||
|
||||
To view the documentation run the following command::
|
||||
|
||||
make htmlview
|
||||
|
||||
This command will fire up your default browser and open the main page of your
|
||||
(previously generated) HTML documentation.
|
||||
|
||||
|
||||
Start over
|
||||
----------
|
||||
|
||||
To clean up all generated documentation files and start from scratch run::
|
||||
|
||||
make clean
|
||||
|
||||
Keep in mind that this command won't touch any documentation source files.
|
||||
|
||||
|
||||
Recreating documentation on the fly
|
||||
-----------------------------------
|
||||
|
||||
There is a way to recreate the doc automatically when you make changes, you
|
||||
need to install watchdog (``pip install watchdog``) and then use::
|
||||
|
||||
make watch
|
||||
|
||||
Alternative method using tox
|
||||
----------------------------
|
||||
|
||||
To compile the documentation to HTML run the following command::
|
||||
|
||||
tox -e docs
|
||||
|
||||
Documentation will be generated inside the ``docs/_build/all`` dir.
|
||||
|
|
@ -323,9 +323,10 @@ deprecation removals are documented in the :ref:`release notes <news>`.
|
|||
Tests
|
||||
=====
|
||||
|
||||
Tests are implemented using the :doc:`Twisted unit-testing framework
|
||||
<twisted:development/test-standard>`. Running tests requires
|
||||
:doc:`tox <tox:index>`.
|
||||
Tests are implemented using pytest_. Running tests requires :doc:`tox
|
||||
<tox:index>`.
|
||||
|
||||
.. _pytest: https://pytest.org
|
||||
|
||||
.. _running-tests:
|
||||
|
||||
|
|
@ -371,6 +372,21 @@ To see coverage report install :doc:`coverage <coverage:index>`
|
|||
|
||||
see output of ``coverage --help`` for more options like html or xml report.
|
||||
|
||||
Some tests need a ``mitmdump`` executable (from mitmproxy_) to test against a
|
||||
fully featured proxy server; they are skipped when one cannot be found
|
||||
(``mitmproxy`` is intentionally not a test dependency that would be installed
|
||||
into test venvs, as that sometimes leads to various dependency conflicts).
|
||||
To run these tests, make ``mitmdump`` available in one of these ways:
|
||||
|
||||
* install ``mitmproxy`` so that ``mitmdump`` is on your ``PATH``, e.g. with
|
||||
pipx_ (``pipx install mitmproxy``) or uv_ (``uv tool install mitmproxy``);
|
||||
|
||||
* have uv_ installed, in which case the tests will run
|
||||
``uvx --from mitmproxy mitmdump``;
|
||||
|
||||
* set the ``MITMDUMP`` environment variable to the path of a ``mitmdump``
|
||||
executable.
|
||||
|
||||
Writing tests
|
||||
-------------
|
||||
|
||||
|
|
@ -398,3 +414,6 @@ And their unit-tests are in::
|
|||
.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
|
||||
.. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22
|
||||
.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy
|
||||
.. _mitmproxy: https://mitmproxy.org/
|
||||
.. _pipx: https://pipx.pypa.io/
|
||||
.. _uv: https://docs.astral.sh/uv/
|
||||
|
|
|
|||
|
|
@ -285,7 +285,8 @@ consume a lot of memory.
|
|||
In order to avoid parsing all the entire feed at once in memory, you can use
|
||||
the :func:`~scrapy.utils.iterators.xmliter_lxml` and
|
||||
:func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what
|
||||
:class:`~scrapy.spiders.XMLFeedSpider` uses.
|
||||
:class:`~scrapy.spiders.XMLFeedSpider` and
|
||||
:class:`~scrapy.spiders.CSVFeedSpider` use.
|
||||
|
||||
.. autofunction:: scrapy.utils.iterators.xmliter_lxml
|
||||
|
||||
|
|
@ -360,10 +361,11 @@ method for this purpose. For example:
|
|||
def process_spider_output(self, response, result):
|
||||
for item_or_request in result:
|
||||
if isinstance(item_or_request, Request):
|
||||
yield item_or_request
|
||||
continue
|
||||
adapter = ItemAdapter(item)
|
||||
adapter = ItemAdapter(item_or_request)
|
||||
for _ in range(adapter["multiply_by"]):
|
||||
yield deepcopy(item)
|
||||
yield deepcopy(item_or_request)
|
||||
|
||||
Does Scrapy support IPv6 addresses?
|
||||
-----------------------------------
|
||||
|
|
@ -409,7 +411,6 @@ How can I make a blank request?
|
|||
|
||||
from scrapy import Request
|
||||
|
||||
|
||||
blank_request = Request("data:,")
|
||||
|
||||
In this case, the URL is set to a data URI scheme. Data URLs allow you to include data
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ Having trouble? We'd like to help!
|
|||
* Ask or search questions in `StackOverflow using the scrapy tag`_.
|
||||
* Ask or search questions in the `Scrapy subreddit`_.
|
||||
* Search for questions on the archives of the `scrapy-users mailing list`_.
|
||||
* Ask a question in the `#scrapy IRC channel`_,
|
||||
* Ask a question in the `#scrapy IRC channel`_.
|
||||
* Report bugs with Scrapy in our `issue tracker`_.
|
||||
* Join the Discord community `Scrapy Discord`_.
|
||||
|
||||
|
|
@ -91,15 +91,15 @@ Basic concepts
|
|||
:doc:`topics/selectors`
|
||||
Extract the data from web pages using XPath.
|
||||
|
||||
:doc:`topics/shell`
|
||||
Test your extraction code in an interactive environment.
|
||||
|
||||
:doc:`topics/items`
|
||||
Define the data you want to scrape.
|
||||
|
||||
:doc:`topics/loaders`
|
||||
Populate your items with the extracted data.
|
||||
|
||||
:doc:`topics/shell`
|
||||
Test your extraction code in an interactive environment.
|
||||
|
||||
:doc:`topics/item-pipeline`
|
||||
Post-process and store your scraped data.
|
||||
|
||||
|
|
@ -151,6 +151,7 @@ Solving specific problems
|
|||
topics/debug
|
||||
topics/contracts
|
||||
topics/practices
|
||||
topics/security
|
||||
topics/broad-crawls
|
||||
topics/developer-tools
|
||||
topics/dynamic-content
|
||||
|
|
@ -175,6 +176,10 @@ Solving specific problems
|
|||
:doc:`topics/practices`
|
||||
Get familiar with some Scrapy common practices.
|
||||
|
||||
:doc:`topics/security`
|
||||
Understand the security implications of Scrapy defaults and how to harden
|
||||
them.
|
||||
|
||||
:doc:`topics/broad-crawls`
|
||||
Tune Scrapy for crawling a lot domains in parallel.
|
||||
|
||||
|
|
|
|||
|
|
@ -90,9 +90,10 @@ to figure these settings out automatically.
|
|||
.. note::
|
||||
|
||||
This is using :ref:`feed exports <topics-feed-exports>` to generate the
|
||||
JSON file, you can easily change the export format (XML or CSV, for example) or the
|
||||
storage backend (FTP or `Amazon S3`_, for example). You can also write an
|
||||
:ref:`item pipeline <topics-item-pipeline>` to store the items in a database.
|
||||
JSON Lines file, you can easily change the export format (XML or CSV, for
|
||||
example) or the storage backend (FTP or `Amazon S3`_, for example). You can
|
||||
also write an :ref:`item pipeline <topics-item-pipeline>` to store the
|
||||
items in a database.
|
||||
|
||||
|
||||
.. _topics-whatelse:
|
||||
|
|
|
|||
336
docs/news.rst
336
docs/news.rst
|
|
@ -3,6 +3,330 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
.. _release-2.17.0:
|
||||
|
||||
Scrapy 2.17.0 (2026-07-07)
|
||||
--------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
- Security bug fixes
|
||||
|
||||
- HTTP/2 and SOCKS proxy support for ``HttpxDownloadHandler``
|
||||
|
||||
- Improved settings for changing allowed TLS versions
|
||||
|
||||
Security bug fixes
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- ``s3://`` requests now use HTTPS by default, instead of plaintext HTTP.
|
||||
|
||||
Previously, :class:`~scrapy.core.downloader.handlers.s3.S3DownloadHandler`
|
||||
sent signed S3 requests over plaintext HTTP unless
|
||||
``request.meta["is_secure"]`` was set to a true value, exposing the request
|
||||
path, the AWS ``Authorization`` header, the ``X-Amz-Security-Token`` header
|
||||
(when using temporary credentials), and the response contents to network
|
||||
attackers, who could also tamper with responses. See the `76g3-c3x4-crvx`_
|
||||
security advisory for details.
|
||||
|
||||
To restore the previous behavior for a given request, set
|
||||
``request.meta["is_secure"]`` to ``False``.
|
||||
|
||||
.. _76g3-c3x4-crvx: https://github.com/scrapy/scrapy/security/advisories/GHSA-76g3-c3x4-crvx
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- The ``DOWNLOADER_CLIENT_TLS_METHOD`` setting is deprecated. You should use
|
||||
the :setting:`DOWNLOAD_TLS_MIN_VERSION` and/or
|
||||
:setting:`DOWNLOAD_TLS_MAX_VERSION` settings instead if you want to change
|
||||
the TLS method selection.
|
||||
(:issue:`3288`, :issue:`6546`)
|
||||
|
||||
- The following spider attributes are deprecated in favor of settings:
|
||||
|
||||
- ``http_user`` (use :setting:`HTTPAUTH_USER`)
|
||||
|
||||
- ``http_pass`` (use :setting:`HTTPAUTH_PASS`)
|
||||
|
||||
- ``http_auth_domain`` (use :setting:`HTTPAUTH_DOMAIN`)
|
||||
|
||||
(:issue:`7590`)
|
||||
|
||||
- The ``scrapy.commands.ScrapyCommand.help()`` method is deprecated. It was
|
||||
never called by Scrapy.
|
||||
(:issue:`7626`, :issue:`7633`)
|
||||
|
||||
- The following TLS-related functions and constants, intended for internal
|
||||
use, are deprecated:
|
||||
|
||||
- ``scrapy.core.downloader.tls.METHOD_TLS``
|
||||
|
||||
- ``scrapy.core.downloader.tls.METHOD_TLSv10``
|
||||
|
||||
- ``scrapy.core.downloader.tls.METHOD_TLSv11``
|
||||
|
||||
- ``scrapy.core.downloader.tls.METHOD_TLSv12``
|
||||
|
||||
- ``scrapy.core.downloader.tls.openssl_methods``
|
||||
|
||||
- ``scrapy.core.downloader.tls.DEFAULT_CIPHERS``
|
||||
|
||||
- ``scrapy.utils.ssl.ffi_buf_to_string()``
|
||||
|
||||
- ``scrapy.utils.ssl.get_temp_key_info()``
|
||||
|
||||
- ``scrapy.utils.ssl.x509name_to_string()``
|
||||
|
||||
(:issue:`6546`, :issue:`7619`, :issue:`7665`)
|
||||
|
||||
- The ``CRAWLSPIDER_FOLLOW_LINKS`` setting is deprecated. You can set
|
||||
``follow=False`` in your rules to achieve the same effect.
|
||||
(:issue:`7592`)
|
||||
|
||||
- Instantiating
|
||||
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
|
||||
without a ``crawler`` argument is deprecated.
|
||||
(:issue:`7655`)
|
||||
|
||||
- Instantiating
|
||||
:class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` without a
|
||||
``settings`` argument is deprecated.
|
||||
(:issue:`7664`)
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- Added support for HTTP/2 requests to
|
||||
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. It
|
||||
requires setting the new :setting:`HTTPX_HTTP2_ENABLED` setting to
|
||||
``True``.
|
||||
(:issue:`7575`)
|
||||
|
||||
- Added support for SOCKS proxies to
|
||||
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.
|
||||
(:issue:`747`, :issue:`7575`)
|
||||
|
||||
- Added :setting:`DOWNLOAD_TLS_MIN_VERSION` and
|
||||
:setting:`DOWNLOAD_TLS_MAX_VERSION` settings as replacements for the
|
||||
``DOWNLOADER_CLIENT_TLS_METHOD`` setting (which is now deprecated).
|
||||
Compared to the old setting, they support specifying a range of allowed
|
||||
versions and support newer TLS versions.
|
||||
(:issue:`4821`, :issue:`6546`)
|
||||
|
||||
- Added :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS` and
|
||||
:setting:`HTTPAUTH_DOMAIN` settings and :reqmeta:`http_user`,
|
||||
:reqmeta:`http_pass` and :reqmeta:`http_auth_domain` meta keys as more
|
||||
flexible ways to set HTTP authentication data.
|
||||
(:issue:`7590`)
|
||||
|
||||
- Added a :reqmeta:`verbatim_url` meta key that can be set to ``True`` to
|
||||
skip request URL canonicalization.
|
||||
(:issue:`7473`)
|
||||
|
||||
- Added ``deny_tags`` and ``deny_attrs`` arguments to :class:`LinkExtractor
|
||||
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`.
|
||||
(:issue:`6321`, :issue:`7679`)
|
||||
|
||||
- :attr:`scrapy.Item.fields` now returns the fields in the definition order
|
||||
instead of the alphabetical one.
|
||||
(:issue:`7015`, :issue:`7694`)
|
||||
|
||||
- Added a :setting:`RETRY_GIVE_UP_LOG_LEVEL` setting, a
|
||||
:reqmeta:`give_up_log_level` meta key and a ``give_up_log_level`` argument
|
||||
of the
|
||||
:func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function that
|
||||
allow changing the log level of the message logged when the retry limit has
|
||||
been reached.
|
||||
(:issue:`4622`, :issue:`5297`, :issue:`7567`)
|
||||
|
||||
- It's now possible to set :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` to
|
||||
``None`` to use the default ciphers of the underlying TLS implementation.
|
||||
(:issue:`7499`, :issue:`7665`)
|
||||
|
||||
Improvements
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- :class:`~scrapy.FormRequest` is no longer deprecated, only its
|
||||
``from_response()`` method is still deprecated.
|
||||
(:issue:`7561`, :issue:`7671`)
|
||||
|
||||
- Switched the item definition in the default project template from a
|
||||
:class:`scrapy.item.Item` to a dataclass.
|
||||
(:issue:`7493`, :issue:`7513`)
|
||||
|
||||
- Fixed deprecation warnings with pyOpenSSL 26.3.0.
|
||||
(:issue:`7619`)
|
||||
|
||||
- Removed the runtime warnings for :attr:`Spider.allowed_domains
|
||||
<scrapy.Spider.allowed_domains>` containing URLs or domains with ports
|
||||
instead of just domains and for spider classes having a ``start_url``
|
||||
attribute instead of :class:`~scrapy.spiders.Spider.start_urls`. Please use
|
||||
:doc:`scrapy-lint <scrapy-lint:index>` to find mistakes in your spider code
|
||||
instead.
|
||||
(:issue:`4421`, :issue:`7627`)
|
||||
|
||||
- :func:`scrapy.utils.test.get_crawler` now disables
|
||||
:setting:`TELNETCONSOLE_ENABLED` by default.
|
||||
(:issue:`7644`)
|
||||
|
||||
- Other code refactoring and improvements.
|
||||
(:issue:`7409`, :issue:`7593`, :issue:`7594`, :issue:`7611`, :issue:`7649`)
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
- :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` no
|
||||
longer ignores proxy credentials for redirected or retried requests.
|
||||
(:issue:`7601`, :issue:`7630`)
|
||||
|
||||
- :class:`~scrapy.extensions.feedexport.GCSFeedStorage` now closes the
|
||||
temporary file after the upload.
|
||||
(:issue:`7546`)
|
||||
|
||||
- Fixed ``scrapy shell <URL>`` running a full spider crawl when there is a
|
||||
spider for the requested URL. This bug was introduced in Scrapy 2.13.0.
|
||||
(:issue:`7552`, :issue:`7557`)
|
||||
|
||||
- The :setting:`IMAGES_STORE_S3_ACL` and :setting:`IMAGES_STORE_GCS_ACL`
|
||||
settings are no longer ignored. This bug was introduced in Scrapy 2.12.0.
|
||||
(:issue:`7597`, :issue:`7614`)
|
||||
|
||||
- :class:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler` now closes
|
||||
the connection after making the request.
|
||||
(:issue:`7602`, :issue:`7667`)
|
||||
|
||||
- Removed the deprecated ``spider`` argument from the pipeline defined in the
|
||||
default project template.
|
||||
(:issue:`7676`)
|
||||
|
||||
- Fixed ``scrapy genspider --edit`` not working.
|
||||
(:issue:`7260`, :issue:`7683`)
|
||||
|
||||
- When a :class:`~scrapy.crawler.Crawler` instance is passed to
|
||||
:meth:`AsyncCrawlerRunner.create_crawler()
|
||||
<scrapy.crawler.AsyncCrawlerRunner.create_crawler>` or
|
||||
:meth:`CrawlerRunner.create_crawler()
|
||||
<scrapy.crawler.CrawlerRunner.create_crawler>`, settings from both classes
|
||||
are now merged, previously only the settings from the
|
||||
:class:`~scrapy.crawler.Crawler` instance were used.
|
||||
(:issue:`1280`, :issue:`7647`)
|
||||
|
||||
- Fixed several issues with cookie handling in
|
||||
:func:`scrapy.utils.request.request_to_curl`.
|
||||
(:issue:`7603`, :issue:`7675`, :issue:`7684`)
|
||||
|
||||
- Fixed :class:`scrapy.resolver.CachingThreadedResolver` not disabling the
|
||||
cache when :setting:`DNSCACHE_ENABLED` is set to ``False``.
|
||||
(:issue:`7663`)
|
||||
|
||||
- Fixed :func:`scrapy.utils.response.open_in_browser` not removing comments
|
||||
when looking for the ``<base>`` tag.
|
||||
(:issue:`7506`)
|
||||
|
||||
- Fixed checking for deprecated methods in custom :setting:`ITEM_PROCESSOR`
|
||||
implementations.
|
||||
(:issue:`7589`)
|
||||
|
||||
- Fixed :func:`scrapy.utils.url.strip_url` corrupting some URLs with
|
||||
credentials.
|
||||
(:issue:`7604`, :issue:`7605`)
|
||||
|
||||
- :func:`scrapy.utils.misc.rel_has_nofollow` now ignores the case when
|
||||
looking for "nofollow" strings.
|
||||
(:issue:`7632`)
|
||||
|
||||
- Fixed an exception in :class:`scrapy.utils.sitemap.Sitemap` when parsing
|
||||
some malformed sitemaps.
|
||||
(:issue:`7686`, :issue:`7687`)
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
- Mentioned :doc:`scrapy-lint <scrapy-lint:index>` in the docs.
|
||||
(:issue:`4421`, :issue:`7627`)
|
||||
|
||||
- Added the docs about :ref:`security considerations <security>`.
|
||||
(:issue:`7389`, :issue:`7678`)
|
||||
|
||||
- Improved the :ref:`item pipeline docs <topics-item-pipeline>`.
|
||||
(:issue:`2350`, :issue:`7676`)
|
||||
|
||||
- Documented which stats are collected by
|
||||
:class:`~scrapy.extensions.corestats.CoreStats`.
|
||||
(:issue:`7421`)
|
||||
|
||||
- Switched documentation examples from using :class:`scrapy.item.Item` to
|
||||
using dataclasses.
|
||||
(:issue:`7493`, :issue:`7513`)
|
||||
|
||||
- Added feature comparison tables to the :ref:`download handler
|
||||
<download-handlers-ref>` docs.
|
||||
(:issue:`7575`)
|
||||
|
||||
- Improved the docs for :ref:`logging settings <logging-settings>`.
|
||||
(:issue:`6909`, :issue:`7668`)
|
||||
|
||||
- Documented a way to :ref:`improve startup time and memory usage
|
||||
<large-project-startup>` by using :setting:`SPIDER_MODULES`.
|
||||
(:issue:`7576`, :issue:`7600`)
|
||||
|
||||
- Clarified handling of the ``type`` argument of :class:`~scrapy.Selector`.
|
||||
(:issue:`7704`)
|
||||
|
||||
- Other documentation improvements and fixes.
|
||||
(:issue:`4954`,
|
||||
:issue:`6120`,
|
||||
:issue:`7286`,
|
||||
:issue:`7564`,
|
||||
:issue:`7573`,
|
||||
:issue:`7598`,
|
||||
:issue:`7599`,
|
||||
:issue:`7698`)
|
||||
|
||||
Quality assurance
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Fixed deprecation warnings with pytest 9.1.0.
|
||||
(:issue:`7621`)
|
||||
|
||||
- Type hints improvements and fixes.
|
||||
(:issue:`6958`, :issue:`7586`)
|
||||
|
||||
- CI and test improvements and fixes.
|
||||
(:issue:`5954`,
|
||||
:issue:`7002`,
|
||||
:issue:`7017`,
|
||||
:issue:`7247`,
|
||||
:issue:`7508`,
|
||||
:issue:`7545`,
|
||||
:issue:`7566`,
|
||||
:issue:`7574`,
|
||||
:issue:`7585`,
|
||||
:issue:`7595`,
|
||||
:issue:`7608`,
|
||||
:issue:`7610`,
|
||||
:issue:`7612`,
|
||||
:issue:`7616`,
|
||||
:issue:`7625`,
|
||||
:issue:`7637`,
|
||||
:issue:`7639`,
|
||||
:issue:`7640`,
|
||||
:issue:`7641`,
|
||||
:issue:`7642`,
|
||||
:issue:`7643`,
|
||||
:issue:`7644`,
|
||||
:issue:`7645`,
|
||||
:issue:`7646`,
|
||||
:issue:`7654`,
|
||||
:issue:`7655`,
|
||||
:issue:`7664`,
|
||||
:issue:`7672`,
|
||||
:issue:`7677`,
|
||||
:issue:`7680`,
|
||||
:issue:`7682`,
|
||||
:issue:`7692`)
|
||||
|
||||
.. _release-2.16.0:
|
||||
|
||||
Scrapy 2.16.0 (2026-05-19)
|
||||
|
|
@ -711,7 +1035,7 @@ Deprecations
|
|||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- Added a new setting, :setting:`REFERER_POLICIES`, to allow customizing
|
||||
- Added a new setting, :setting:`REFERRER_POLICIES`, to allow customizing
|
||||
supported referrer policies.
|
||||
|
||||
Bug fixes
|
||||
|
|
@ -5722,7 +6046,7 @@ The following changes may impact custom priority queue classes:
|
|||
|
||||
* A new keyword parameter has been added: ``key``. It is a string
|
||||
that is always an empty string for memory queues and indicates the
|
||||
:setting:`JOB_DIR` value for disk queues.
|
||||
:setting:`JOBDIR` value for disk queues.
|
||||
|
||||
* The parameter for disk queues that contains data from the previous
|
||||
crawl, ``startprios`` or ``slot_startprios``, is now passed as a
|
||||
|
|
@ -6243,8 +6567,8 @@ Backward-incompatible changes
|
|||
``429``, you must override :setting:`RETRY_HTTP_CODES` accordingly.
|
||||
|
||||
* :class:`~scrapy.crawler.Crawler`,
|
||||
:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>` and
|
||||
:class:`CrawlerRunner.create_crawler <scrapy.crawler.CrawlerRunner.create_crawler>`
|
||||
:meth:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>` and
|
||||
:meth:`CrawlerRunner.create_crawler <scrapy.crawler.CrawlerRunner.create_crawler>`
|
||||
no longer accept a :class:`~scrapy.spiders.Spider` subclass instance, they
|
||||
only accept a :class:`~scrapy.spiders.Spider` subclass now.
|
||||
|
||||
|
|
@ -6276,7 +6600,7 @@ New features
|
|||
``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
|
||||
:ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
|
||||
scheduling improvement on crawls targeting multiple web domains, at the
|
||||
cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
|
||||
cost of no ``CONCURRENT_REQUESTS_PER_IP`` support (:issue:`3520`)
|
||||
|
||||
* A new :attr:`.Request.cb_kwargs` attribute
|
||||
provides a cleaner way to pass keyword arguments to callback methods
|
||||
|
|
@ -8755,7 +9079,7 @@ New features and settings
|
|||
- In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`)
|
||||
- Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`)
|
||||
- ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by:
|
||||
- :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP`
|
||||
- :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, ``CONCURRENT_REQUESTS_PER_IP``
|
||||
- check the documentation for more details
|
||||
- Added builtin caching DNS resolver (:rev:`2728`)
|
||||
- Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`)
|
||||
|
|
|
|||
|
|
@ -168,7 +168,6 @@ Use a fallback component:
|
|||
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
|
||||
|
||||
FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -105,6 +105,9 @@ Scrapy API requires passing a Deferred to it) using the following helpers:
|
|||
|
||||
.. autofunction:: scrapy.utils.defer.deferred_from_coro
|
||||
.. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f
|
||||
|
||||
The following function helps with a reverse wrapping:
|
||||
|
||||
.. autofunction:: scrapy.utils.defer.ensure_awaitable
|
||||
|
||||
|
||||
|
|
@ -190,7 +193,7 @@ in future Scrapy versions. The following features are not available:
|
|||
:class:`~scrapy.crawler.CrawlerProcess`
|
||||
(:class:`~scrapy.crawler.AsyncCrawlerProcess` and
|
||||
:class:`~scrapy.crawler.AsyncCrawlerRunner` are available)
|
||||
* Twisted-specific DNS resolvers (the :setting:`DNS_RESOLVER` setting)
|
||||
* Twisted-specific DNS resolvers (the :setting:`TWISTED_DNS_RESOLVER` setting)
|
||||
* User and 3rd-party code that requires a reactor (see :ref:`below
|
||||
<asyncio-without-reactor-migrate>` for examples)
|
||||
|
||||
|
|
@ -315,8 +318,7 @@ implementations, :class:`~asyncio.ProactorEventLoop` (default) and
|
|||
:class:`~asyncio.SelectorEventLoop` works with Twisted.
|
||||
|
||||
Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop`
|
||||
automatically when you change the :setting:`TWISTED_REACTOR` setting or call
|
||||
:func:`~scrapy.utils.reactor.install_reactor`.
|
||||
automatically when installing the asyncio reactor.
|
||||
|
||||
.. note:: Other libraries you use may require
|
||||
:class:`~asyncio.ProactorEventLoop`, e.g. because it supports
|
||||
|
|
|
|||
|
|
@ -88,6 +88,8 @@ server) is, and this extension builds on that premise.
|
|||
Prevent specific requests from triggering slot delay adjustments
|
||||
================================================================
|
||||
|
||||
.. versionadded:: 2.12.0
|
||||
|
||||
AutoThrottle adjusts the delay of download slots based on the latencies of
|
||||
responses that belong to that download slot. The only exceptions are non-200
|
||||
responses, which are only taken into account to increase that delay, but
|
||||
|
|
|
|||
|
|
@ -199,6 +199,7 @@ Global commands:
|
|||
* :command:`fetch`
|
||||
* :command:`view`
|
||||
* :command:`version`
|
||||
* :command:`bench`
|
||||
|
||||
Project-only commands:
|
||||
|
||||
|
|
@ -207,7 +208,6 @@ Project-only commands:
|
|||
* :command:`list`
|
||||
* :command:`edit`
|
||||
* :command:`parse`
|
||||
* :command:`bench`
|
||||
|
||||
.. command:: startproject
|
||||
|
||||
|
|
@ -476,7 +476,7 @@ Supported options:
|
|||
|
||||
* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
|
||||
|
||||
* ``--a NAME=VALUE``: set spider argument (may be repeated)
|
||||
* ``-a NAME=VALUE``: set spider argument (may be repeated)
|
||||
|
||||
* ``--callback`` or ``-c``: spider method to use as callback for parsing the
|
||||
response
|
||||
|
|
|
|||
|
|
@ -268,5 +268,5 @@ You can also send multiple requests in parallel:
|
|||
yield {
|
||||
"h1": response.css("h1::text").get(),
|
||||
"price": responses[0].css(".price::text").get(),
|
||||
"price2": responses[1].css(".color::text").get(),
|
||||
"color": responses[1].css(".color::text").get(),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,6 +39,10 @@ for additional schemes and to replace or disable default ones:
|
|||
"sftp": "my.download_handlers.SftpHandler",
|
||||
}
|
||||
|
||||
.. seealso:: :ref:`security-unencrypted-protocols` and
|
||||
:ref:`security-local-resources`, for the security implications of the
|
||||
default ``http``, ``ftp``, ``file`` and ``data`` handlers.
|
||||
|
||||
Replacing HTTP(S) download handlers
|
||||
-----------------------------------
|
||||
|
||||
|
|
@ -325,13 +329,13 @@ Other limitations:
|
|||
HTTPX_HTTP2_ENABLED
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Default: ``False``
|
||||
|
||||
Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra
|
||||
needs to be installed if you want to enable this setting.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
Built-in non-HTTP download handlers reference
|
||||
=============================================
|
||||
|
||||
|
|
|
|||
|
|
@ -351,6 +351,8 @@ HttpAuthMiddleware
|
|||
HTTPAUTH_USER
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Default: ``""``
|
||||
|
||||
The username to use for HTTP basic authentication, applied to all requests
|
||||
|
|
@ -361,6 +363,8 @@ whose URL matches :setting:`HTTPAUTH_DOMAIN`.
|
|||
HTTPAUTH_PASS
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Default: ``""``
|
||||
|
||||
The password to use for HTTP basic authentication.
|
||||
|
|
@ -370,6 +374,8 @@ The password to use for HTTP basic authentication.
|
|||
HTTPAUTH_DOMAIN
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The domain (and its subdomains) to which HTTP basic authentication credentials
|
||||
|
|
@ -379,6 +385,8 @@ that this risks leaking credentials to unrelated domains.
|
|||
This setting must be explicitly configured whenever :setting:`HTTPAUTH_USER`
|
||||
or :setting:`HTTPAUTH_PASS` is set.
|
||||
|
||||
.. seealso:: :ref:`security-credential-leakage`
|
||||
|
||||
.. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication
|
||||
|
||||
|
||||
|
|
@ -497,7 +505,7 @@ Filesystem storage backend (default)
|
|||
|
||||
* ``response_body`` - the plain response body
|
||||
|
||||
* ``response_headers`` - the request headers (in raw HTTP format)
|
||||
* ``response_headers`` - the response headers (in raw HTTP format)
|
||||
|
||||
* ``meta`` - some metadata of this cache resource in Python ``repr()``
|
||||
format (grep-friendly format)
|
||||
|
|
@ -539,7 +547,7 @@ defines the methods described below.
|
|||
.. method:: open_spider(spider)
|
||||
|
||||
This method gets called after a spider has been opened for crawling. It handles
|
||||
the :signal:`open_spider <spider_opened>` signal.
|
||||
the :signal:`spider_opened` signal.
|
||||
|
||||
:param spider: the spider which has been opened
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
|
@ -547,7 +555,7 @@ defines the methods described below.
|
|||
.. method:: close_spider(spider)
|
||||
|
||||
This method gets called after a spider has been closed. It handles
|
||||
the :signal:`close_spider <spider_closed>` signal.
|
||||
the :signal:`spider_closed` signal.
|
||||
|
||||
:param spider: the spider which has been closed
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
|
@ -806,7 +814,6 @@ HttpProxyMiddleware settings
|
|||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. setting:: HTTPPROXY_ENABLED
|
||||
.. setting:: HTTPPROXY_AUTH_ENCODING
|
||||
|
||||
HTTPPROXY_ENABLED
|
||||
^^^^^^^^^^^^^^^^^
|
||||
|
|
@ -815,6 +822,8 @@ Default: ``True``
|
|||
|
||||
Whether or not to enable the :class:`HttpProxyMiddleware`.
|
||||
|
||||
.. setting:: HTTPPROXY_AUTH_ENCODING
|
||||
|
||||
HTTPPROXY_AUTH_ENCODING
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
|
|
@ -976,7 +985,7 @@ Whether the Meta Refresh middleware will be enabled.
|
|||
METAREFRESH_IGNORE_TAGS
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Default: ``[]``
|
||||
Default: ``["noscript"]``
|
||||
|
||||
Meta tags within these tags are ignored.
|
||||
|
||||
|
|
@ -1083,7 +1092,7 @@ Default::
|
|||
'twisted.internet.error.ConnectionDone',
|
||||
'twisted.internet.error.ConnectError',
|
||||
'twisted.internet.error.ConnectionLost',
|
||||
IOError,
|
||||
OSError,
|
||||
'scrapy.core.downloader.handlers.http11.TunnelError',
|
||||
]
|
||||
|
||||
|
|
@ -1102,6 +1111,8 @@ exception propagation, see
|
|||
RETRY_GIVE_UP_LOG_LEVEL
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Default: ``"ERROR"``
|
||||
|
||||
:ref:`Logging level <levels>` used for the message logged when a request
|
||||
|
|
|
|||
|
|
@ -133,7 +133,7 @@ data from it depends on the type of response:
|
|||
|
||||
.. code-block:: python
|
||||
|
||||
selector = Selector(data["html"])
|
||||
selector = Selector(text=data["html"])
|
||||
|
||||
- If the response is JavaScript, or HTML with a ``<script/>`` element
|
||||
containing the desired data, see :ref:`topics-parsing-javascript`.
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ For example:
|
|||
.. code-block:: python
|
||||
|
||||
def parse_page(self, response):
|
||||
if "Bandwidth exceeded" in response.body:
|
||||
if "Bandwidth exceeded" in response.text:
|
||||
raise CloseSpider("bandwidth_exceeded")
|
||||
|
||||
DontCloseSpider
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ output examples, which assume you're exporting these two items:
|
|||
BaseItemExporter
|
||||
----------------
|
||||
|
||||
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0, dont_fail=False)
|
||||
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding=None, indent=None, dont_fail=False)
|
||||
|
||||
This is the (abstract) base class for all Item Exporters. It provides
|
||||
support for common features used by all (concrete) Item Exporters, such as
|
||||
|
|
@ -239,7 +239,7 @@ BaseItemExporter
|
|||
|
||||
.. attribute:: indent
|
||||
|
||||
Amount of spaces used to indent the output on each level. Defaults to ``0``.
|
||||
Amount of spaces used to indent the output on each level. Defaults to ``None``.
|
||||
|
||||
* ``indent=None`` selects the most compact representation,
|
||||
all items in the same line with no indentation
|
||||
|
|
@ -328,7 +328,7 @@ CsvItemExporter
|
|||
|
||||
:param join_multivalued: The char (or chars) that will be used for joining
|
||||
multi-valued fields, if found.
|
||||
:type include_headers_line: str
|
||||
:type join_multivalued: str
|
||||
|
||||
:param errors: The optional string that specifies how encoding and decoding
|
||||
errors are to be handled. For more information see
|
||||
|
|
@ -342,14 +342,14 @@ CsvItemExporter
|
|||
|
||||
A typical output of this exporter would be::
|
||||
|
||||
product,price
|
||||
name,price
|
||||
Color TV,1200
|
||||
DVD player,200
|
||||
|
||||
PickleItemExporter
|
||||
------------------
|
||||
|
||||
.. class:: PickleItemExporter(file, protocol=0, **kwargs)
|
||||
.. class:: PickleItemExporter(file, protocol=4, **kwargs)
|
||||
|
||||
Exports items in pickle format to the given file-like object.
|
||||
|
||||
|
|
|
|||
|
|
@ -341,9 +341,6 @@ closing the spider. If the spider generates more than that number of errors,
|
|||
it will be closed with the reason ``closespider_errorcount``. If zero (or non
|
||||
set), spiders won't be closed by number of errors.
|
||||
|
||||
.. module:: scrapy.extensions.debug
|
||||
:synopsis: Extensions for debugging Scrapy
|
||||
|
||||
.. module:: scrapy.extensions.periodic_log
|
||||
:synopsis: Periodic stats logging
|
||||
|
||||
|
|
@ -418,7 +415,7 @@ Example extension configuration:
|
|||
custom_settings = {
|
||||
"LOG_LEVEL": "INFO",
|
||||
"PERIODIC_LOG_STATS": {
|
||||
"include": ["downloader/", "scheduler/", "log_count/", "item_scraped_count/"],
|
||||
"include": ["downloader/", "scheduler/", "log_count/", "item_scraped_count"],
|
||||
},
|
||||
"PERIODIC_LOG_DELTA": {"include": ["downloader/"]},
|
||||
"PERIODIC_LOG_TIMING_ENABLED": True,
|
||||
|
|
@ -463,6 +460,9 @@ Default: ``False``
|
|||
Debugging extensions
|
||||
--------------------
|
||||
|
||||
.. module:: scrapy.extensions.debug
|
||||
:synopsis: Extensions for debugging Scrapy
|
||||
|
||||
Stack trace dump extension
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
|
|||
|
|
@ -528,10 +528,6 @@ safe numeric encoding (``\uXXXX`` sequences) for historic reasons.
|
|||
|
||||
Use ``"utf-8"`` if you want UTF-8 for JSON too.
|
||||
|
||||
.. versionchanged:: 2.8
|
||||
The :command:`startproject` command now sets this setting to
|
||||
``"utf-8"`` in the generated ``settings.py`` file.
|
||||
|
||||
.. setting:: FEED_EXPORT_FIELDS
|
||||
|
||||
FEED_EXPORT_FIELDS
|
||||
|
|
@ -619,6 +615,7 @@ Default:
|
|||
"file": "scrapy.extensions.feedexport.FileFeedStorage",
|
||||
"stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",
|
||||
"s3": "scrapy.extensions.feedexport.S3FeedStorage",
|
||||
"gs": "scrapy.extensions.feedexport.GCSFeedStorage",
|
||||
"ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -57,6 +57,8 @@ Any of these methods may be defined as a coroutine function (``async def``).
|
|||
Item pipeline example
|
||||
=====================
|
||||
|
||||
.. _price-pipeline-example:
|
||||
|
||||
Price validation and dropping items with no prices
|
||||
--------------------------------------------------
|
||||
|
||||
|
|
@ -246,6 +248,8 @@ returns multiples items with the same id:
|
|||
return item
|
||||
|
||||
|
||||
.. _activating-item-pipeline:
|
||||
|
||||
Activating an Item Pipeline component
|
||||
=====================================
|
||||
|
||||
|
|
@ -262,3 +266,110 @@ To activate an Item Pipeline component you must add its class to the
|
|||
The integer values you assign to classes in this setting determine the
|
||||
order in which they run: items go through from lower valued to higher
|
||||
valued classes. It's customary to define these numbers in the 0-1000 range.
|
||||
|
||||
A complete example
|
||||
==================
|
||||
|
||||
The examples above show item pipeline components on their own. In a project, a
|
||||
pipeline is one of four pieces that work together: the :ref:`item
|
||||
<topics-items>` your spider produces, the :ref:`spider <topics-spiders>` that
|
||||
yields it, the pipeline that processes it, and the :setting:`ITEM_PIPELINES`
|
||||
setting that enables the pipeline.
|
||||
|
||||
The following example wires those pieces together to validate the price of
|
||||
books scraped from `books.toscrape.com`_, reusing the ``PricePipeline`` from
|
||||
:ref:`price-pipeline-example` above.
|
||||
|
||||
Define the item in ``myproject/items.py``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class BookItem:
|
||||
title: str
|
||||
price: float
|
||||
|
||||
Yield instances of that item from your spider, e.g. in
|
||||
``myproject/spiders/books.py``:
|
||||
|
||||
.. skip: next
|
||||
.. code-block:: python
|
||||
|
||||
import scrapy
|
||||
|
||||
from myproject.items import BookItem
|
||||
|
||||
|
||||
class BooksSpider(scrapy.Spider):
|
||||
name = "books"
|
||||
start_urls = ["https://books.toscrape.com/"]
|
||||
|
||||
def parse(self, response):
|
||||
for book in response.css("article.product_pod"):
|
||||
yield BookItem(
|
||||
title=book.css("h3 a::attr(title)").get(),
|
||||
price=float(book.css("p.price_color::text").re_first(r"[\d.]+")),
|
||||
)
|
||||
|
||||
Put the ``PricePipeline`` shown earlier in ``myproject/pipelines.py``, and
|
||||
enable it in ``myproject/settings.py``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
ITEM_PIPELINES = {
|
||||
"myproject.pipelines.PricePipeline": 300,
|
||||
}
|
||||
|
||||
With these pieces in place, every ``BookItem`` that ``BooksSpider`` yields
|
||||
passes through ``PricePipeline`` before it reaches the :ref:`feed exports
|
||||
<topics-feed-exports>` or any other output.
|
||||
|
||||
.. _books.toscrape.com: https://books.toscrape.com/
|
||||
|
||||
|
||||
Common pitfalls
|
||||
===============
|
||||
|
||||
The pipeline does not run
|
||||
-------------------------
|
||||
|
||||
A pipeline component only runs if its class is listed in the
|
||||
:setting:`ITEM_PIPELINES` setting, normally in your project's
|
||||
:file:`settings.py` file (see :ref:`activating-item-pipeline`). Adding it to
|
||||
the spider or elsewhere has no effect.
|
||||
|
||||
To confirm that Scrapy loaded your pipeline, look for a line like this near the
|
||||
start of the crawl log::
|
||||
|
||||
[scrapy.middleware] INFO: Enabled item pipelines:
|
||||
['myproject.pipelines.PricePipeline']
|
||||
|
||||
If your pipeline is missing from that list, check that its import path matches
|
||||
the :setting:`ITEM_PIPELINES` entry, and that the setting is not being
|
||||
overridden, for example by :attr:`~scrapy.Spider.custom_settings` or by a
|
||||
redefinition of :setting:`ITEM_PIPELINES` in :file:`settings.py`.
|
||||
|
||||
The item is not returned
|
||||
------------------------
|
||||
|
||||
:meth:`process_item` must return the item (or raise
|
||||
:exc:`~scrapy.exceptions.DropItem`). A common mistake is to modify the item but
|
||||
forget to return it:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def process_item(self, item):
|
||||
ItemAdapter(item)["price"] *= 1.15
|
||||
# Bug: returns None, so the next component gets None instead of the item.
|
||||
|
||||
Return the item so that the next component, and the rest of Scrapy, can keep
|
||||
processing it:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def process_item(self, item):
|
||||
ItemAdapter(item)["price"] *= 1.15
|
||||
return item
|
||||
|
|
|
|||
|
|
@ -376,7 +376,9 @@ Creating dicts from items:
|
|||
>>> dict(product) # create a dict from all populated values
|
||||
{'price': 1000, 'name': 'Desktop PC'}
|
||||
|
||||
Creating items from dicts:
|
||||
Creating items from dicts:
|
||||
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> Product({"name": "Laptop PC", "price": 1500})
|
||||
Product(price=1500, name='Laptop PC')
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ You can get the oldest object of each class using the
|
|||
Which objects are tracked?
|
||||
--------------------------
|
||||
|
||||
The objects tracked by ``trackrefs`` are all from these classes (and all its
|
||||
The objects tracked by ``trackref`` are all from these classes (and all its
|
||||
subclasses):
|
||||
|
||||
* :class:`scrapy.Request`
|
||||
|
|
@ -187,7 +187,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
|
|||
Inherit from this class if you want to track live
|
||||
instances with the ``trackref`` module.
|
||||
|
||||
.. function:: print_live_refs(class_name, ignore=NoneType)
|
||||
.. function:: print_live_refs(ignore=NoneType)
|
||||
|
||||
Print a report of live references, grouped by class name.
|
||||
|
||||
|
|
@ -203,9 +203,9 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
|
|||
|
||||
.. function:: iter_all(class_name)
|
||||
|
||||
Return an iterator over all objects alive with the given class name, or
|
||||
``None`` if none is found. Use :func:`print_live_refs` first to get a list
|
||||
of all tracked live objects per class name.
|
||||
Return an iterator over all objects alive with the given class name. Use
|
||||
:func:`print_live_refs` first to get a list of all tracked live objects
|
||||
per class name.
|
||||
|
||||
.. skip: end
|
||||
|
||||
|
|
|
|||
|
|
@ -47,108 +47,7 @@ LxmlLinkExtractor
|
|||
:synopsis: lxml's HTMLParser-based link extractors
|
||||
|
||||
|
||||
.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, strip=True)
|
||||
|
||||
LxmlLinkExtractor is the recommended link extractor with handy filtering
|
||||
options. It is implemented using lxml's robust HTMLParser.
|
||||
|
||||
:param allow: a single regular expression (or list of regular expressions)
|
||||
that the (absolute) urls must match in order to be extracted. If not
|
||||
given (or empty), it will match all links.
|
||||
:type allow: str or list
|
||||
|
||||
:param deny: a single regular expression (or list of regular expressions)
|
||||
that the (absolute) urls must match in order to be excluded (i.e. not
|
||||
extracted). It has precedence over the ``allow`` parameter. If not
|
||||
given (or empty) it won't exclude any links.
|
||||
:type deny: str or list
|
||||
|
||||
:param allow_domains: a single value or a list of string containing
|
||||
domains which will be considered for extracting the links
|
||||
:type allow_domains: str or list
|
||||
|
||||
:param deny_domains: a single value or a list of strings containing
|
||||
domains which won't be considered for extracting the links
|
||||
:type deny_domains: str or list
|
||||
|
||||
:param deny_extensions: a single value or list of strings containing
|
||||
extensions that should be ignored when extracting links.
|
||||
If not given, it will default to
|
||||
:data:`scrapy.linkextractors.IGNORED_EXTENSIONS`.
|
||||
|
||||
:type deny_extensions: list
|
||||
|
||||
:param restrict_xpaths: is an XPath (or list of XPath's) which defines
|
||||
regions inside the response where links should be extracted from.
|
||||
If given, only the text selected by those XPath will be scanned for
|
||||
links.
|
||||
:type restrict_xpaths: str or list
|
||||
|
||||
:param restrict_css: a CSS selector (or list of selectors) which defines
|
||||
regions inside the response where links should be extracted from.
|
||||
Has the same behaviour as ``restrict_xpaths``.
|
||||
:type restrict_css: str or list
|
||||
|
||||
:param restrict_text: a single regular expression (or list of regular expressions)
|
||||
that the link's text must match in order to be extracted. If not
|
||||
given (or empty), it will match all links. If a list of regular expressions is
|
||||
given, the link will be extracted if it matches at least one.
|
||||
:type restrict_text: str or list
|
||||
|
||||
:param tags: a tag or a list of tags to consider when extracting links.
|
||||
Defaults to ``('a', 'area')``.
|
||||
:type tags: str or list
|
||||
|
||||
:param attrs: an attribute or list of attributes which should be considered when looking
|
||||
for links to extract (only for those tags specified in the ``tags``
|
||||
parameter). Defaults to ``('href',)``
|
||||
:type attrs: list
|
||||
|
||||
:param canonicalize: canonicalize each extracted url (using
|
||||
w3lib.url.canonicalize_url). Defaults to ``False``.
|
||||
Note that canonicalize_url is meant for duplicate checking;
|
||||
it can change the URL visible at server side, so the response can be
|
||||
different for requests with canonicalized and raw URLs. If you're
|
||||
using LinkExtractor to follow links it is more robust to
|
||||
keep the default ``canonicalize=False``.
|
||||
:type canonicalize: bool
|
||||
|
||||
:param unique: whether duplicate filtering should be applied to extracted
|
||||
links.
|
||||
:type unique: bool
|
||||
|
||||
:param process_value: a function which receives each value extracted from
|
||||
the tag and attributes scanned and can modify the value and return a
|
||||
new one, or return ``None`` to ignore the link altogether. If not
|
||||
given, ``process_value`` defaults to ``lambda x: x``.
|
||||
|
||||
.. highlight:: html
|
||||
|
||||
For example, to extract links from this code::
|
||||
|
||||
<a href="javascript:goToPage('../other/page.html'); return false">Link text</a>
|
||||
|
||||
.. highlight:: python
|
||||
|
||||
You can use the following function in ``process_value``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def process_value(value):
|
||||
m = re.search(r"javascript:goToPage\('(.*?)'", value)
|
||||
if m:
|
||||
return m.group(1)
|
||||
|
||||
:type process_value: collections.abc.Callable
|
||||
|
||||
:param strip: whether to strip whitespaces from extracted attributes.
|
||||
According to HTML5 standard, leading and trailing whitespaces
|
||||
must be stripped from ``href`` attributes of ``<a>``, ``<area>``
|
||||
and many other elements, ``src`` attribute of ``<img>``, ``<iframe>``
|
||||
elements, etc., so LinkExtractor strips space chars by default.
|
||||
Set ``strip=False`` to turn it off (e.g. if you're extracting urls
|
||||
from elements or attributes which allow leading/trailing whitespaces).
|
||||
:type strip: bool
|
||||
.. autoclass:: LxmlLinkExtractor
|
||||
|
||||
.. automethod:: extract_links
|
||||
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ data that will be assigned to the ``name`` field later.
|
|||
|
||||
Afterwards, similar calls are used for ``price`` and ``stock`` fields
|
||||
(the latter using a CSS selector with the :meth:`~ItemLoader.add_css` method),
|
||||
and finally the ``last_update`` field is populated directly with a literal value
|
||||
and finally the ``last_updated`` field is populated directly with a literal value
|
||||
(``today``) using a different method: :meth:`~ItemLoader.add_value`.
|
||||
|
||||
Finally, when all data is collected, the :meth:`ItemLoader.load_item` method is
|
||||
|
|
@ -273,8 +273,8 @@ The precedence order, for both input and output processors, is as follows:
|
|||
1. Item Loader field-specific attributes: ``field_in`` and ``field_out`` (most
|
||||
precedence)
|
||||
2. Field metadata (``input_processor`` and ``output_processor`` key)
|
||||
3. Item Loader defaults: :meth:`ItemLoader.default_input_processor` and
|
||||
:meth:`ItemLoader.default_output_processor` (least precedence)
|
||||
3. Item Loader defaults: :attr:`ItemLoader.default_input_processor` and
|
||||
:attr:`ItemLoader.default_output_processor` (least precedence)
|
||||
|
||||
See also: :ref:`topics-loaders-extending`.
|
||||
|
||||
|
|
@ -323,8 +323,8 @@ There are several ways to modify Item Loader context values:
|
|||
loader = ItemLoader(product, unit="cm")
|
||||
|
||||
3. On Item Loader declaration, for those input/output processors that support
|
||||
instantiating them with an Item Loader context. :class:`~processor.MapCompose` is one of
|
||||
them:
|
||||
instantiating them with an Item Loader context.
|
||||
:class:`~itemloaders.processors.MapCompose` is one of them:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
|
|
|
|||
|
|
@ -4,11 +4,6 @@
|
|||
Logging
|
||||
=======
|
||||
|
||||
.. note::
|
||||
:mod:`scrapy.log` has been deprecated alongside its functions in favor of
|
||||
explicit calls to the Python standard logging. Keep reading to learn more
|
||||
about the new logging system.
|
||||
|
||||
Scrapy uses :mod:`logging` for event logging. We'll
|
||||
provide some simple examples to get you started, but for more advanced
|
||||
use-cases it's strongly suggested to read thoroughly its documentation.
|
||||
|
|
|
|||
|
|
@ -371,11 +371,12 @@ For the Images Pipeline, set :setting:`IMAGES_URLS_FIELD` and/or
|
|||
If you need something more complex and want to override the custom pipeline
|
||||
behaviour, see :ref:`topics-media-pipeline-override`.
|
||||
|
||||
If you have multiple image pipelines inheriting from ImagePipeline and you want
|
||||
to have different settings in different pipelines you can set setting keys
|
||||
preceded with uppercase name of your pipeline class. E.g. if your pipeline is
|
||||
called MyPipeline and you want to have custom IMAGES_URLS_FIELD you define
|
||||
setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used.
|
||||
If you have multiple image pipelines inheriting from :class:`ImagesPipeline`
|
||||
and you want to have different settings in different pipelines you can set
|
||||
setting keys preceded with uppercase name of your pipeline class. E.g. if your
|
||||
pipeline is called ``MyPipeline`` and you want to have custom
|
||||
:setting:`IMAGES_URLS_FIELD` you define setting
|
||||
``MYPIPELINE_IMAGES_URLS_FIELD`` and your custom settings will be used.
|
||||
|
||||
|
||||
Additional features
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ Request objects
|
|||
|
||||
.. invisible-code-block: python
|
||||
|
||||
from scrapy.http import Request
|
||||
from scrapy import Request
|
||||
|
||||
1. Using a dict:
|
||||
|
||||
|
|
@ -247,7 +247,7 @@ Request objects
|
|||
Return a new Request which is a copy of this Request. See also:
|
||||
:ref:`topics-request-response-ref-request-callback-arguments`.
|
||||
|
||||
.. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs])
|
||||
.. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs, cls])
|
||||
|
||||
Return a Request object with the same members, except for those members
|
||||
given new values by whichever keyword arguments are specified. The
|
||||
|
|
@ -717,6 +717,7 @@ Those are:
|
|||
* :reqmeta:`download_fail_on_dataloss`
|
||||
* :reqmeta:`download_latency`
|
||||
* :reqmeta:`download_maxsize`
|
||||
* :reqmeta:`download_slot`
|
||||
* :reqmeta:`download_warnsize`
|
||||
* :reqmeta:`download_timeout`
|
||||
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
|
||||
|
|
@ -806,6 +807,8 @@ Whether or not to fail on broken responses. See:
|
|||
give_up_log_level
|
||||
-----------------
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
:ref:`Logging level <levels>` used for the message logged when a request
|
||||
exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details.
|
||||
|
||||
|
|
@ -814,6 +817,8 @@ exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details.
|
|||
http_auth_domain
|
||||
----------------
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Overrides :setting:`HTTPAUTH_DOMAIN` for this request.
|
||||
|
||||
.. reqmeta:: http_pass
|
||||
|
|
@ -821,6 +826,8 @@ Overrides :setting:`HTTPAUTH_DOMAIN` for this request.
|
|||
http_pass
|
||||
---------
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Overrides :setting:`HTTPAUTH_PASS` for this request.
|
||||
|
||||
.. reqmeta:: http_user
|
||||
|
|
@ -828,6 +835,8 @@ Overrides :setting:`HTTPAUTH_PASS` for this request.
|
|||
http_user
|
||||
---------
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Overrides :setting:`HTTPAUTH_USER` for this request.
|
||||
|
||||
.. reqmeta:: max_retry_times
|
||||
|
|
@ -844,6 +853,8 @@ The meta key is used set retry times per request. When set, the
|
|||
verbatim_url
|
||||
------------
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Set this key to ``True`` to keep the request URL as passed to
|
||||
:class:`~scrapy.Request`, without URL percent-encoding.
|
||||
|
||||
|
|
@ -854,7 +865,6 @@ characters that would otherwise be canonicalized get different fingerprints.
|
|||
In this mode, the ``keep_fragments`` parameter is ignored, and it is
|
||||
effectively true.
|
||||
|
||||
|
||||
.. _topics-stop-response-download:
|
||||
|
||||
Stopping the download of a Response
|
||||
|
|
@ -911,6 +921,11 @@ Request subclasses
|
|||
Here is the list of built-in :class:`~scrapy.Request` subclasses. You can also subclass
|
||||
it to implement your own custom functionality.
|
||||
|
||||
FormRequest
|
||||
-----------
|
||||
|
||||
.. autoclass:: scrapy.FormRequest
|
||||
|
||||
JsonRequest
|
||||
-----------
|
||||
|
||||
|
|
@ -1071,7 +1086,7 @@ Response objects
|
|||
.. attribute:: Response.flags
|
||||
|
||||
A list that contains flags for this response. Flags are labels used for
|
||||
tagging Responses. For example: ``'cached'``, ``'redirected``', etc. And
|
||||
tagging Responses. For example: ``'cached'``, ``'redirected'``', etc. And
|
||||
they're shown on the string representation of the Response (``__str__()``
|
||||
method) which is used by the engine for logging.
|
||||
|
||||
|
|
@ -1105,7 +1120,7 @@ Response objects
|
|||
|
||||
Returns a new Response which is a copy of this Response.
|
||||
|
||||
.. method:: Response.replace([url, status, headers, body, request, flags, cls])
|
||||
.. method:: Response.replace([url, status, headers, body, request, flags, certificate, ip_address, protocol, cls])
|
||||
|
||||
Returns a Response object with the same members, except for those members
|
||||
given new values by whichever keyword arguments are specified. The
|
||||
|
|
|
|||
|
|
@ -0,0 +1,207 @@
|
|||
.. _security:
|
||||
|
||||
========
|
||||
Security
|
||||
========
|
||||
|
||||
Scrapy defaults are optimized for web scraping, not for the security posture
|
||||
that you might expect from software that handles untrusted input or runs in a
|
||||
shared or exposed environment. Some common security practices are unnecessary
|
||||
for many scraping use cases, and a few can even prevent valid ones (for
|
||||
example, sites that you must scrape may use misconfigured TLS certificates or
|
||||
serve content over unencrypted protocols).
|
||||
|
||||
This page highlights the Scrapy defaults that have security implications, so
|
||||
that you can make an informed decision about whether to keep them, and explains
|
||||
how to harden them along with the trade-offs involved.
|
||||
|
||||
.. note::
|
||||
|
||||
None of the options below are silver bullets. Which of them make sense
|
||||
depends on your threat model: whether the URLs you crawl come from trusted
|
||||
sources, whether the machine running Scrapy is exposed to a network you do
|
||||
not control, whether the data you handle is sensitive, and so on.
|
||||
|
||||
.. _security-untrusted-responses:
|
||||
|
||||
Treat responses as untrusted input
|
||||
==================================
|
||||
|
||||
Regardless of any setting, remember that response data comes from servers you
|
||||
do not control, even when you trust the site you are crawling, as responses may
|
||||
be tampered with in transit or the server itself may be compromised.
|
||||
|
||||
Never pass response data to functions that can execute code or otherwise act on
|
||||
their input in an unsafe way, such as :func:`eval`, :func:`exec`, or
|
||||
:func:`pickle.loads`, and be careful when writing response data to paths
|
||||
derived from the response itself.
|
||||
|
||||
TLS connections
|
||||
===============
|
||||
|
||||
.. _security-certificate-verification:
|
||||
|
||||
Certificate verification
|
||||
------------------------
|
||||
|
||||
By default Scrapy does **not** verify the TLS certificate of HTTPS servers, as
|
||||
controlled by the :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting (default:
|
||||
``False``).
|
||||
|
||||
This default favors reach over security: many sites that are otherwise fine to
|
||||
scrape have expired, self-signed, or otherwise invalid certificates, and
|
||||
verifying certificates would make requests to them fail.
|
||||
|
||||
If the integrity of the connection matters to you (for example, to detect
|
||||
man-in-the-middle attacks), set:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
DOWNLOAD_VERIFY_CERTIFICATES = True
|
||||
|
||||
* **Pro:** requests to servers with invalid or untrusted certificates fail
|
||||
instead of silently succeeding, protecting you from some man-in-the-middle
|
||||
attacks.
|
||||
|
||||
* **Con:** you can no longer scrape sites with misconfigured certificates
|
||||
without re-disabling verification for them.
|
||||
|
||||
.. _security-tls-protocols-ciphers:
|
||||
|
||||
Protocol versions and ciphers
|
||||
-----------------------------
|
||||
|
||||
You can restrict the TLS protocol versions that Scrapy accepts through the
|
||||
:setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION`
|
||||
settings, e.g. to reject obsolete protocol versions.
|
||||
|
||||
By default Scrapy uses the OpenSSL ``DEFAULT`` cipher list
|
||||
(:setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`), which favors compatibility and still
|
||||
allows some older, weaker ciphers. Set it to ``None`` to instead use the curated
|
||||
cipher list of the underlying TLS implementation (Twisted), which excludes weak
|
||||
ciphers:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
DOWNLOADER_CLIENT_TLS_CIPHERS = None
|
||||
|
||||
* **Pro:** connections that would negotiate a weak cipher fail instead of
|
||||
succeeding.
|
||||
|
||||
* **Con:** you can no longer connect to servers that only support the excluded
|
||||
ciphers.
|
||||
|
||||
.. _security-unencrypted-protocols:
|
||||
|
||||
Unencrypted protocols
|
||||
=====================
|
||||
|
||||
By default Scrapy enables download handlers for unencrypted protocols, namely
|
||||
``http://`` and ``ftp://`` (see :setting:`DOWNLOAD_HANDLERS_BASE`). Data sent
|
||||
and received over these protocols, including any credentials, travels in plain
|
||||
text and can be read or modified by anyone on the network path.
|
||||
|
||||
If you only crawl over encrypted protocols, you can disable the unencrypted
|
||||
ones so that no request can accidentally be sent unencrypted:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
DOWNLOAD_HANDLERS = {
|
||||
"http": None,
|
||||
"ftp": None,
|
||||
}
|
||||
|
||||
* **Pro:** a misconfigured or maliciously-redirected request cannot leak data
|
||||
over an unencrypted connection, as such requests fail instead.
|
||||
|
||||
* **Con:** you can no longer crawl resources that are only available over those
|
||||
protocols.
|
||||
|
||||
Note that disabling the ``http`` handler also prevents plain-HTTP requests that
|
||||
result from following an ``http://`` redirect or link, which is often the point
|
||||
of disabling it.
|
||||
|
||||
.. _security-local-resources:
|
||||
|
||||
Local and non-network resources
|
||||
===============================
|
||||
|
||||
By default Scrapy enables download handlers for the ``file://`` and ``data:``
|
||||
schemes (see :setting:`DOWNLOAD_HANDLERS_BASE`). The ``file://`` handler reads
|
||||
arbitrary files from the local filesystem, limited only by the permissions of
|
||||
the process running Scrapy.
|
||||
|
||||
This is convenient (for example, to parse a local HTML file), but it is a risk
|
||||
if any of the URLs you schedule come from an untrusted source: a crafted
|
||||
``file:///etc/passwd`` URL could read local files.
|
||||
|
||||
If you do not need them, disable these handlers:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
DOWNLOAD_HANDLERS = {
|
||||
"file": None,
|
||||
"data": None,
|
||||
}
|
||||
|
||||
* **Pro:** crawled URLs cannot be used to read local files or inline data.
|
||||
|
||||
* **Con:** you can no longer fetch ``file://`` or ``data:`` URLs.
|
||||
|
||||
More generally, if you crawl URLs from untrusted sources, consider validating
|
||||
their schemes (and, where applicable, their hosts) before scheduling requests,
|
||||
to avoid server-side request forgery (SSRF) and similar issues.
|
||||
|
||||
.. _security-telnet:
|
||||
|
||||
Telnet console
|
||||
==============
|
||||
|
||||
Scrapy enables the :ref:`telnet console <topics-telnetconsole>` by default
|
||||
(:setting:`TELNETCONSOLE_ENABLED`). The telnet console is a Python shell
|
||||
running inside the Scrapy process, so anyone who can connect to it can run
|
||||
arbitrary code in that process.
|
||||
|
||||
By default the console binds to ``127.0.0.1`` (:setting:`TELNETCONSOLE_HOST`)
|
||||
and is protected by a username (:setting:`TELNETCONSOLE_USERNAME`, default
|
||||
``scrapy``) and an automatically generated password
|
||||
(:setting:`TELNETCONSOLE_PASSWORD`), so it is only reachable from the local
|
||||
machine.
|
||||
|
||||
.. warning::
|
||||
|
||||
Telnet does not provide any transport-layer security, so the
|
||||
username/password authentication does not protect the credentials or the
|
||||
session from anyone able to observe the traffic. Never expose the telnet
|
||||
console over an untrusted network by changing :setting:`TELNETCONSOLE_HOST`
|
||||
to a non-local address.
|
||||
|
||||
If you do not use the telnet console, disable it entirely:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
TELNETCONSOLE_ENABLED = False
|
||||
|
||||
* **Pro:** removes a local code-execution surface and one less listening port.
|
||||
|
||||
* **Con:** you can no longer :ref:`inspect and control a running crawler
|
||||
<topics-telnetconsole>` through it.
|
||||
|
||||
.. _security-credential-leakage:
|
||||
|
||||
Credential leakage across domains
|
||||
=================================
|
||||
|
||||
Some Scrapy features attach credentials or other sensitive headers to requests,
|
||||
and a crawl that spans multiple domains can leak them to unintended hosts:
|
||||
|
||||
* HTTP authentication credentials set through
|
||||
:class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` are only
|
||||
sent to the domain set in :setting:`HTTPAUTH_DOMAIN`. Leave this set to the
|
||||
intended domain rather than ``None`` so that credentials are not sent to
|
||||
every domain you crawl.
|
||||
|
||||
* The ``Referer`` header may disclose the URLs you crawl to other sites. The
|
||||
default :setting:`REFERRER_POLICY` already avoids sending the referrer from
|
||||
HTTPS to HTTP, but you can tighten it further (for example, to
|
||||
``same-origin`` or ``no-referrer``) if needed.
|
||||
|
|
@ -308,7 +308,7 @@ Examples:
|
|||
|
||||
* ``*::text`` selects all descendant text nodes of the current selector context:
|
||||
|
||||
..skip: next
|
||||
.. skip: next
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> response.css("#images *::text").getall()
|
||||
|
|
@ -634,8 +634,7 @@ Example:
|
|||
.. code-block:: pycon
|
||||
|
||||
>>> from scrapy import Selector
|
||||
>>> sel = Selector(
|
||||
... text="""
|
||||
>>> sel = Selector(text="""
|
||||
... <ul class="list">
|
||||
... <li>1</li>
|
||||
... <li>2</li>
|
||||
|
|
@ -645,8 +644,8 @@ Example:
|
|||
... <li>4</li>
|
||||
... <li>5</li>
|
||||
... <li>6</li>
|
||||
... </ul>"""
|
||||
... )
|
||||
... </ul>""")
|
||||
...
|
||||
>>> xp = lambda x: sel.xpath(x).getall()
|
||||
|
||||
This gets all first ``<li>`` elements under whatever it is its parent:
|
||||
|
|
@ -948,11 +947,9 @@ with groups of itemscopes and corresponding itemprops:
|
|||
>>> sel = Selector(text=doc, type="html")
|
||||
>>> for scope in sel.xpath("//div[@itemscope]"):
|
||||
... print("current scope:", scope.xpath("@itemtype").getall())
|
||||
... props = scope.xpath(
|
||||
... """
|
||||
... props = scope.xpath("""
|
||||
... set:difference(./descendant::*/@itemprop,
|
||||
... .//*[@itemscope]/*/@itemprop)"""
|
||||
... )
|
||||
... .//*[@itemscope]/*/@itemprop)""")
|
||||
... print(f" properties: {props.getall()}")
|
||||
... print("")
|
||||
...
|
||||
|
|
|
|||
|
|
@ -409,77 +409,6 @@ Default: ``{}``
|
|||
A dict containing paths to the add-ons enabled in your project and their
|
||||
priorities. For more information, see :ref:`topics-addons`.
|
||||
|
||||
.. setting:: AWS_ACCESS_KEY_ID
|
||||
|
||||
AWS_ACCESS_KEY_ID
|
||||
-----------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The AWS access key used by code that requires access to `Amazon Web services`_,
|
||||
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
|
||||
|
||||
.. setting:: AWS_SECRET_ACCESS_KEY
|
||||
|
||||
AWS_SECRET_ACCESS_KEY
|
||||
---------------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The AWS secret key used by code that requires access to `Amazon Web services`_,
|
||||
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
|
||||
|
||||
.. setting:: AWS_SESSION_TOKEN
|
||||
|
||||
AWS_SESSION_TOKEN
|
||||
-----------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The AWS security token used by code that requires access to `Amazon Web services`_,
|
||||
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`, when using
|
||||
`temporary security credentials`_.
|
||||
|
||||
.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html
|
||||
|
||||
.. setting:: AWS_ENDPOINT_URL
|
||||
|
||||
AWS_ENDPOINT_URL
|
||||
----------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Endpoint URL used for S3-like storage, for example Minio or s3.scality.
|
||||
|
||||
.. setting:: AWS_USE_SSL
|
||||
|
||||
AWS_USE_SSL
|
||||
-----------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Use this option if you want to disable SSL connection for communication with
|
||||
S3 or S3-like storage. By default SSL will be used.
|
||||
|
||||
.. setting:: AWS_VERIFY
|
||||
|
||||
AWS_VERIFY
|
||||
----------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Verify SSL connection between Scrapy and S3 or S3-like storage. By default
|
||||
SSL verification will occur.
|
||||
|
||||
.. setting:: AWS_REGION_NAME
|
||||
|
||||
AWS_REGION_NAME
|
||||
---------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The name of the region associated with the AWS client.
|
||||
|
||||
.. setting:: ASYNCIO_EVENT_LOOP
|
||||
|
||||
ASYNCIO_EVENT_LOOP
|
||||
|
|
@ -508,6 +437,77 @@ Note that the event loop class must inherit from :class:`asyncio.AbstractEventLo
|
|||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
|
||||
.. setting:: AWS_ACCESS_KEY_ID
|
||||
|
||||
AWS_ACCESS_KEY_ID
|
||||
-----------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The AWS access key used by code that requires access to `Amazon Web services`_,
|
||||
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
|
||||
|
||||
.. setting:: AWS_ENDPOINT_URL
|
||||
|
||||
AWS_ENDPOINT_URL
|
||||
----------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Endpoint URL used for S3-like storage, for example Minio or s3.scality.
|
||||
|
||||
.. setting:: AWS_REGION_NAME
|
||||
|
||||
AWS_REGION_NAME
|
||||
---------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The name of the region associated with the AWS client.
|
||||
|
||||
.. setting:: AWS_SECRET_ACCESS_KEY
|
||||
|
||||
AWS_SECRET_ACCESS_KEY
|
||||
---------------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The AWS secret key used by code that requires access to `Amazon Web services`_,
|
||||
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
|
||||
|
||||
.. setting:: AWS_SESSION_TOKEN
|
||||
|
||||
AWS_SESSION_TOKEN
|
||||
-----------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The AWS security token used by code that requires access to `Amazon Web services`_,
|
||||
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`, when using
|
||||
`temporary security credentials`_.
|
||||
|
||||
.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html
|
||||
|
||||
.. setting:: AWS_USE_SSL
|
||||
|
||||
AWS_USE_SSL
|
||||
-----------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Use this option if you want to disable SSL connection for communication with
|
||||
S3 or S3-like storage. By default SSL will be used.
|
||||
|
||||
.. setting:: AWS_VERIFY
|
||||
|
||||
AWS_VERIFY
|
||||
----------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Verify SSL connection between Scrapy and S3 or S3-like storage. By default
|
||||
SSL verification will occur.
|
||||
|
||||
.. setting:: BOT_NAME
|
||||
|
||||
BOT_NAME
|
||||
|
|
@ -593,7 +593,7 @@ When writing an item pipeline, you can force a different log level by setting
|
|||
DEFAULT_ITEM_CLASS
|
||||
------------------
|
||||
|
||||
Default: ``'scrapy.Item'``
|
||||
Default: ``'scrapy.item.Item'``
|
||||
|
||||
The default class that will be used for instantiating items in the :ref:`the
|
||||
Scrapy shell <topics-shell>`.
|
||||
|
|
@ -687,7 +687,7 @@ Whether to enable DNS in-memory cache.
|
|||
:class:`~scrapy.resolver.CachingThreadedResolver` and
|
||||
:class:`~scrapy.resolver.CachingHostnameResolver`. It has no effect when
|
||||
:setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect
|
||||
either when :setting:`DNS_RESOLVER` is set to a different resolver.
|
||||
either when :setting:`TWISTED_DNS_RESOLVER` is set to a different resolver.
|
||||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
|
||||
|
|
@ -702,25 +702,6 @@ DNS in-memory cache size, see :setting:`DNSCACHE_ENABLED`.
|
|||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
|
||||
.. setting:: TWISTED_DNS_RESOLVER
|
||||
|
||||
TWISTED_DNS_RESOLVER
|
||||
--------------------
|
||||
|
||||
Default: ``'scrapy.resolver.CachingThreadedResolver'``
|
||||
|
||||
The class to be used by Twisted to resolve DNS names. The default
|
||||
``scrapy.resolver.CachingThreadedResolver`` supports specifying a timeout for
|
||||
DNS requests via the :setting:`DNS_TIMEOUT` setting, but works only with IPv4
|
||||
addresses. Scrapy provides an alternative resolver,
|
||||
``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not
|
||||
take the :setting:`DNS_TIMEOUT` setting into account.
|
||||
|
||||
.. note::
|
||||
This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
||||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
|
||||
.. setting:: DNS_TIMEOUT
|
||||
|
||||
DNS_TIMEOUT
|
||||
|
|
@ -734,7 +715,7 @@ Timeout for processing of DNS queries in seconds. Float is supported.
|
|||
This setting is only used by
|
||||
:class:`~scrapy.resolver.CachingThreadedResolver`. It has no effect when
|
||||
:setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect
|
||||
either when :setting:`DNS_RESOLVER` is set to a different resolver.
|
||||
either when :setting:`TWISTED_DNS_RESOLVER` is set to a different resolver.
|
||||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
|
||||
|
|
@ -766,6 +747,9 @@ specific cipher that is not included in ``DEFAULT`` if a website requires it.
|
|||
Set this setting to ``None`` to use the default ciphers of the underlying TLS
|
||||
implementation.
|
||||
|
||||
.. versionchanged:: 2.17.0
|
||||
Added support for setting this to ``None``.
|
||||
|
||||
.. _OpenSSL cipher list format: https://docs.openssl.org/master/man1/openssl-ciphers/#cipher-list-format
|
||||
|
||||
.. note::
|
||||
|
|
@ -774,11 +758,15 @@ implementation.
|
|||
handler <topics-download-handlers>`, so it's not guaranteed to be supported
|
||||
by all 3rd-party handlers.
|
||||
|
||||
.. seealso:: :ref:`security-tls-protocols-ciphers`
|
||||
|
||||
.. setting:: DOWNLOAD_TLS_MAX_VERSION
|
||||
|
||||
DOWNLOAD_TLS_MAX_VERSION
|
||||
------------------------
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Use this setting to change the maximum version of the TLS protocol allowed to
|
||||
|
|
@ -807,11 +795,15 @@ modern environments.
|
|||
by all 3rd-party handlers. Additionally, the set of supported TLS versions
|
||||
depends on the TLS implementation being used by the handler.
|
||||
|
||||
.. seealso:: :ref:`security-tls-protocols-ciphers`
|
||||
|
||||
.. setting:: DOWNLOAD_TLS_MIN_VERSION
|
||||
|
||||
DOWNLOAD_TLS_MIN_VERSION
|
||||
------------------------
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Use this setting to change the minimum version of the TLS protocol allowed to
|
||||
|
|
@ -819,6 +811,8 @@ be used by Scrapy.
|
|||
|
||||
See :setting:`DOWNLOAD_TLS_MAX_VERSION` for the details and limitations.
|
||||
|
||||
.. seealso:: :ref:`security-tls-protocols-ciphers`
|
||||
|
||||
.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
|
||||
|
||||
DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
|
||||
|
|
@ -979,6 +973,9 @@ enabled in your project.
|
|||
|
||||
See :setting:`DOWNLOAD_HANDLERS_BASE` for example format.
|
||||
|
||||
.. seealso:: :ref:`security-unencrypted-protocols` and
|
||||
:ref:`security-local-resources`
|
||||
|
||||
.. setting:: DOWNLOAD_HANDLERS_BASE
|
||||
|
||||
DOWNLOAD_HANDLERS_BASE
|
||||
|
|
@ -1026,6 +1023,9 @@ handler (without replacement), place this in your ``settings.py``:
|
|||
"ftp": None,
|
||||
}
|
||||
|
||||
.. seealso:: :ref:`security-unencrypted-protocols` and
|
||||
:ref:`security-local-resources`
|
||||
|
||||
|
||||
.. setting:: DOWNLOAD_SLOTS
|
||||
|
||||
|
|
@ -1184,6 +1184,8 @@ when making a request and abort the request if the verification fails.
|
|||
certificate problems are logged when this setting is set to ``False``)
|
||||
depends on its implementation.
|
||||
|
||||
.. seealso:: :ref:`security-certificate-verification`
|
||||
|
||||
.. setting:: DUPEFILTER_CLASS
|
||||
|
||||
DUPEFILTER_CLASS
|
||||
|
|
@ -1309,6 +1311,7 @@ Default:
|
|||
|
||||
{
|
||||
"scrapy.extensions.corestats.CoreStats": 0,
|
||||
"scrapy.extensions.logcount.LogCount": 0,
|
||||
"scrapy.extensions.telnet.TelnetConsole": 0,
|
||||
"scrapy.extensions.memusage.MemoryUsage": 0,
|
||||
"scrapy.extensions.memdebug.MemoryDebugger": 0,
|
||||
|
|
@ -1331,6 +1334,8 @@ and the :ref:`list of available extensions <topics-extensions-ref>`.
|
|||
FEED_TEMPDIR
|
||||
------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The Feed Temp dir allows you to set a custom folder to save crawler
|
||||
temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
|
||||
:ref:`Amazon S3 <topics-feed-storage-s3>`.
|
||||
|
|
@ -1340,6 +1345,8 @@ temporary files before uploading with :ref:`FTP feed storage <topics-feed-storag
|
|||
FEED_STORAGE_GCS_ACL
|
||||
--------------------
|
||||
|
||||
Default: ``""``
|
||||
|
||||
The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage <topics-feed-storage-gcs>`.
|
||||
For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation <https://docs.cloud.google.com/storage/docs/access-control/lists>`_.
|
||||
|
||||
|
|
@ -1608,6 +1615,8 @@ The following special items are also supported:
|
|||
|
||||
- ``Python``
|
||||
|
||||
- ``pyOpenSSL``
|
||||
|
||||
.. setting:: LOGSTATS_INTERVAL
|
||||
|
||||
LOGSTATS_INTERVAL
|
||||
|
|
@ -1735,7 +1744,7 @@ significant similarities in the time between their requests.
|
|||
|
||||
The randomization policy is the same used by `wget`_ ``--random-wait`` option.
|
||||
|
||||
If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect.
|
||||
If :setting:`DOWNLOAD_DELAY` is zero this option has no effect.
|
||||
|
||||
.. _wget: https://www.gnu.org/software/wget/manual/wget.html
|
||||
|
||||
|
|
@ -1796,7 +1805,7 @@ The parser backend to use for parsing ``robots.txt`` files. For more information
|
|||
.. setting:: ROBOTSTXT_USER_AGENT
|
||||
|
||||
ROBOTSTXT_USER_AGENT
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
--------------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
|
|
@ -1951,6 +1960,8 @@ Default:
|
|||
|
||||
{
|
||||
"scrapy.contracts.default.UrlContract": 1,
|
||||
"scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,
|
||||
"scrapy.contracts.default.MetadataContract": 1,
|
||||
"scrapy.contracts.default.ReturnsContract": 2,
|
||||
"scrapy.contracts.default.ScrapesContract": 3,
|
||||
}
|
||||
|
|
@ -2015,6 +2026,7 @@ Default:
|
|||
.. code-block:: python
|
||||
|
||||
{
|
||||
"scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25,
|
||||
"scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50,
|
||||
"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
|
||||
"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
|
||||
|
|
@ -2074,6 +2086,8 @@ Default: ``True`` (``False`` when :setting:`TWISTED_REACTOR_ENABLED` is ``False`
|
|||
A boolean which specifies if the :ref:`telnet console <topics-telnetconsole>`
|
||||
will be enabled (provided its extension is also enabled).
|
||||
|
||||
.. seealso:: :ref:`security-telnet`
|
||||
|
||||
.. setting:: TEMPLATES_DIR
|
||||
|
||||
TEMPLATES_DIR
|
||||
|
|
@ -2088,6 +2102,25 @@ command.
|
|||
The project name must not conflict with the name of custom files or directories
|
||||
in the ``project`` subdirectory.
|
||||
|
||||
.. setting:: TWISTED_DNS_RESOLVER
|
||||
|
||||
TWISTED_DNS_RESOLVER
|
||||
--------------------
|
||||
|
||||
Default: ``'scrapy.resolver.CachingThreadedResolver'``
|
||||
|
||||
The class to be used by Twisted to resolve DNS names. The default
|
||||
``scrapy.resolver.CachingThreadedResolver`` supports specifying a timeout for
|
||||
DNS requests via the :setting:`DNS_TIMEOUT` setting, but works only with IPv4
|
||||
addresses. Scrapy provides an alternative resolver,
|
||||
``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not
|
||||
take the :setting:`DNS_TIMEOUT` setting into account.
|
||||
|
||||
.. note::
|
||||
This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
||||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
|
||||
.. setting:: TWISTED_REACTOR_ENABLED
|
||||
|
||||
TWISTED_REACTOR_ENABLED
|
||||
|
|
@ -2229,7 +2262,7 @@ URLLENGTH_LIMIT
|
|||
|
||||
Default: ``2083``
|
||||
|
||||
Scope: ``spidermiddlewares.urllength``
|
||||
Scope: ``scrapy.spidermiddlewares.urllength``
|
||||
|
||||
The maximum URL length to allow for crawled URLs.
|
||||
|
||||
|
|
|
|||
|
|
@ -60,6 +60,8 @@ Let's take an example using :ref:`coroutines <topics-coroutines>`:
|
|||
.. skip: next
|
||||
.. code-block:: python
|
||||
|
||||
import json
|
||||
|
||||
import scrapy
|
||||
import treq
|
||||
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ previous (or subsequent) middleware being applied.
|
|||
If you want to disable a builtin middleware (the ones defined in
|
||||
:setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
|
||||
in your project :setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its
|
||||
value. For example, if you want to disable the off-site middleware:
|
||||
value. For example, if you want to disable the referer middleware:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
|
|
@ -355,6 +355,8 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
|
|||
using the special ``"referrer_policy"`` :ref:`Request.meta <topics-request-meta>` key,
|
||||
with the same acceptable values as for the ``REFERRER_POLICY`` setting.
|
||||
|
||||
.. seealso:: :ref:`security-credential-leakage`
|
||||
|
||||
Acceptable values for REFERRER_POLICY
|
||||
*************************************
|
||||
|
||||
|
|
|
|||
|
|
@ -208,7 +208,7 @@ scrapy.Spider
|
|||
:param response: the response to parse
|
||||
:type response: :class:`~scrapy.http.Response`
|
||||
|
||||
.. method:: log(message, [level, component])
|
||||
.. method:: log(message, [level])
|
||||
|
||||
Wrapper that sends a log message through the Spider's :attr:`logger`,
|
||||
kept for backward compatibility. For more information see
|
||||
|
|
@ -335,8 +335,8 @@ The above example can also be written as follows:
|
|||
|
||||
If you are :ref:`running Scrapy from a script <run-from-script>`, you can
|
||||
specify spider arguments when calling
|
||||
:class:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
|
||||
:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`:
|
||||
:meth:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
|
||||
:meth:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`:
|
||||
|
||||
.. skip: next
|
||||
.. code-block:: python
|
||||
|
|
@ -593,7 +593,7 @@ Let's now take a look at an example CrawlSpider with rules:
|
|||
This spider would start crawling example.com's home page, collecting category
|
||||
links, and item links, parsing the latter with the ``parse_item`` method. For
|
||||
each item response, some data will be extracted from the HTML using XPath, and
|
||||
an :class:`~scrapy.Item` will be filled with it.
|
||||
a dictionary will be filled with it.
|
||||
|
||||
XMLFeedSpider
|
||||
-------------
|
||||
|
|
@ -953,6 +953,7 @@ Combine SitemapSpider with other sources of urls:
|
|||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy import Request
|
||||
from scrapy.spiders import SitemapSpider
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -29,14 +29,16 @@ disable it if you want. For more information about the extension itself see
|
|||
.. note::
|
||||
This feature is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
||||
|
||||
.. seealso:: :ref:`security-telnet`
|
||||
|
||||
.. highlight:: none
|
||||
|
||||
How to access the telnet console
|
||||
================================
|
||||
|
||||
The telnet console listens in the TCP port defined in the
|
||||
:setting:`TELNETCONSOLE_PORT` setting, which defaults to ``6023``. To access
|
||||
the console you need to type::
|
||||
The telnet console listens on the first available TCP port from the range
|
||||
defined in the :setting:`TELNETCONSOLE_PORT` setting, which defaults to
|
||||
``[6023, 6073]``. To access the console you need to type::
|
||||
|
||||
telnet localhost 6023
|
||||
Trying localhost...
|
||||
|
|
@ -190,6 +192,8 @@ Default: ``'127.0.0.1'``
|
|||
|
||||
The interface the telnet console should listen on
|
||||
|
||||
.. seealso:: :ref:`security-telnet`
|
||||
|
||||
|
||||
.. setting:: TELNETCONSOLE_USERNAME
|
||||
|
||||
|
|
|
|||
|
|
@ -39,8 +39,8 @@ API stability
|
|||
|
||||
API stability was one of the major goals for the *1.0* release.
|
||||
|
||||
Methods or functions that start with a single dash (``_``) are private and
|
||||
should never be relied as stable.
|
||||
Methods or functions that start with a single underscore (``_``) are private
|
||||
and should never be relied upon as stable.
|
||||
|
||||
Also, keep in mind that stable doesn't mean complete: stable APIs could grow
|
||||
new methods or functionality but the existing methods should keep working the
|
||||
|
|
|
|||
|
|
@ -94,7 +94,82 @@ untyped_calls_exclude = [
|
|||
[[tool.mypy.overrides]]
|
||||
module = "tests.*"
|
||||
allow_untyped_defs = true
|
||||
allow_incomplete_defs = true # 48 errors
|
||||
allow_incomplete_defs = true # 59 errors
|
||||
|
||||
# TODO
|
||||
[[tool.mypy.overrides]]
|
||||
module = [
|
||||
"tests.mockserver.*",
|
||||
"tests.spiders",
|
||||
"tests.test_closespider",
|
||||
"tests.test_cmdline",
|
||||
"tests.test_contracts",
|
||||
"tests.test_core_downloader",
|
||||
"tests.test_downloader_handler_twisted_ftp",
|
||||
"tests.test_downloadermiddleware_cookies",
|
||||
"tests.test_downloadermiddleware_httpauth",
|
||||
"tests.test_downloadermiddleware_httpcache",
|
||||
"tests.test_downloadermiddleware_httpcompression",
|
||||
"tests.test_downloadermiddleware_httpproxy",
|
||||
"tests.test_downloadermiddleware_offsite",
|
||||
"tests.test_downloadermiddleware_redirect",
|
||||
"tests.test_downloadermiddleware_redirect_base",
|
||||
"tests.test_downloadermiddleware_redirect_metarefresh",
|
||||
"tests.test_downloadermiddleware_retry",
|
||||
"tests.test_downloadermiddleware_robotstxt",
|
||||
"tests.test_downloadermiddleware_stats",
|
||||
"tests.test_downloaderslotssettings",
|
||||
"tests.test_dupefilters",
|
||||
"tests.test_engine_loop",
|
||||
"tests.test_exporters",
|
||||
"tests.test_extension_statsmailer",
|
||||
"tests.test_extension_throttle",
|
||||
"tests.test_feedexport",
|
||||
"tests.test_feedexport_postprocess",
|
||||
"tests.test_feedexport_storages",
|
||||
"tests.test_feedexport_uri_params",
|
||||
"tests.test_http2_client_protocol",
|
||||
"tests.test_http_headers",
|
||||
"tests.test_http_request",
|
||||
"tests.test_http_request_form",
|
||||
"tests.test_http_response",
|
||||
"tests.test_http_response_text",
|
||||
"tests.test_item",
|
||||
"tests.test_link",
|
||||
"tests.test_linkextractors",
|
||||
"tests.test_loader",
|
||||
"tests.test_logformatter",
|
||||
"tests.test_logstats",
|
||||
"tests.test_mail",
|
||||
"tests.test_pipeline_crawl",
|
||||
"tests.test_pipeline_files",
|
||||
"tests.test_pipeline_images",
|
||||
"tests.test_pipeline_media",
|
||||
"tests.test_pipelines",
|
||||
"tests.test_pqueues",
|
||||
"tests.test_request_attribute_binding",
|
||||
"tests.test_request_cb_kwargs",
|
||||
"tests.test_request_dict",
|
||||
"tests.test_request_left",
|
||||
"tests.test_robotstxt_interface",
|
||||
"tests.test_scheduler_base",
|
||||
"tests.test_settings",
|
||||
"tests.test_spider",
|
||||
"tests.test_spider_crawl",
|
||||
"tests.test_spidermiddleware_output_chain",
|
||||
"tests.test_spidermiddleware_process_start",
|
||||
"tests.test_spider_sitemap",
|
||||
"tests.test_squeues",
|
||||
"tests.test_squeues_request",
|
||||
"tests.test_stats",
|
||||
"tests.test_utils_datatypes",
|
||||
"tests.test_utils_decorators",
|
||||
"tests.test_utils_defer",
|
||||
"tests.test_utils_deprecate",
|
||||
"tests.test_utils_misc.test_return_with_argument_inside_generator",
|
||||
"tests.test_utils_python",
|
||||
"tests.test_utils_request",
|
||||
]
|
||||
check_untyped_defs = false
|
||||
|
||||
# Interface classes are hard to support
|
||||
|
|
@ -137,7 +212,7 @@ module = [
|
|||
ignore_missing_imports = true
|
||||
|
||||
[tool.bumpversion]
|
||||
current_version = "2.16.0"
|
||||
current_version = "2.17.0"
|
||||
commit = true
|
||||
tag = true
|
||||
tag_name = "{new_version}"
|
||||
|
|
@ -269,11 +344,13 @@ markers = [
|
|||
"requires_uvloop: marks tests as only enabled when uvloop is known to be working",
|
||||
"requires_botocore: marks tests that need botocore (but not boto3)",
|
||||
"requires_boto3: marks tests that need botocore and boto3",
|
||||
"requires_mitmproxy: marks tests that need mitmproxy",
|
||||
"requires_mitmproxy: marks tests that need a mitmdump executable",
|
||||
"requires_internet: marks tests that need real Internet access",
|
||||
]
|
||||
filterwarnings = [
|
||||
"ignore::DeprecationWarning:twisted.web.static",
|
||||
# Twisted doesn't close failed sockets after CannotListenError: https://github.com/twisted/twisted/issues/6108
|
||||
"ignore:Exception ignored in. <socket\\.socket.*laddr=..0\\.0\\.0\\.0., 0.:pytest.PytestUnraisableExceptionWarning",
|
||||
]
|
||||
|
||||
[tool.ruff.lint]
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
2.16.0
|
||||
2.17.0
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ class AddonManager:
|
|||
|
||||
:param settings: The :class:`~scrapy.settings.BaseSettings` object from \
|
||||
which to read the early add-on configuration
|
||||
:type settings: :class:`~scrapy.settings.Settings`
|
||||
:type settings: :class:`~scrapy.settings.BaseSettings`
|
||||
"""
|
||||
for clspath in build_component_list(settings["ADDONS"]):
|
||||
addoncls = load_object(clspath)
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ class ScrapyCommand(ABC):
|
|||
def long_desc(self) -> str:
|
||||
"""A long description of the command. Return short description when not
|
||||
available. It cannot contain newlines since contents will be formatted
|
||||
by optparser which removes newlines and wraps text.
|
||||
by argparse which removes newlines and wraps text.
|
||||
"""
|
||||
return self.short_desc()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,29 @@
|
|||
import argparse
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any, ClassVar
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.spiderloader import get_spider_loader
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import argparse
|
||||
|
||||
|
||||
def _edit_file(editor: str, file_path: str | os.PathLike[str]) -> int:
|
||||
"""Open ``file_path`` with ``editor`` and return the editor exit code.
|
||||
|
||||
``editor`` may include arguments (e.g. ``"code -w"``); it is split with
|
||||
:func:`shlex.split` and the file is passed as a separate argument, so no
|
||||
shell is involved.
|
||||
"""
|
||||
return subprocess.call([*shlex.split(editor), os.fspath(file_path)]) # noqa: S603
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
requires_project = True
|
||||
|
|
@ -45,4 +62,4 @@ class Command(ScrapyCommand):
|
|||
sfile = sys.modules[spidercls.__module__].__file__
|
||||
assert sfile
|
||||
sfile = sfile.replace(".pyc", ".py")
|
||||
self.exitcode = os.system(f'{editor} "{sfile}"') # noqa: S605
|
||||
self.exitcode = _edit_file(editor, Path(sfile))
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import string
|
||||
from importlib import import_module
|
||||
|
|
@ -10,12 +9,14 @@ from urllib.parse import urlparse
|
|||
|
||||
import scrapy
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.commands.edit import _edit_file
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.spiderloader import get_spider_loader
|
||||
from scrapy.utils.template import render_templatefile, string_camelcase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import argparse
|
||||
import os
|
||||
|
||||
|
||||
def sanitize_module_name(module_name: str) -> str:
|
||||
|
|
@ -118,9 +119,11 @@ class Command(ScrapyCommand):
|
|||
|
||||
template_file = self._find_template(opts.template)
|
||||
if template_file:
|
||||
self._genspider(module, name, url, opts.template, template_file)
|
||||
spider_file = self._genspider(
|
||||
module, name, url, opts.template, template_file
|
||||
)
|
||||
if opts.edit:
|
||||
self.exitcode = os.system(f'scrapy edit "{name}"') # noqa: S605
|
||||
self.exitcode = _edit_file(self.settings["EDITOR"], spider_file)
|
||||
|
||||
def _generate_template_variables(
|
||||
self,
|
||||
|
|
@ -148,7 +151,7 @@ class Command(ScrapyCommand):
|
|||
url: str,
|
||||
template_name: str,
|
||||
template_file: str | os.PathLike[str],
|
||||
) -> None:
|
||||
) -> Path:
|
||||
"""Generate the spider module, based on the given template"""
|
||||
assert self.settings is not None
|
||||
tvars = self._generate_template_variables(module, name, url, template_name)
|
||||
|
|
@ -168,6 +171,7 @@ class Command(ScrapyCommand):
|
|||
)
|
||||
if spiders_module:
|
||||
print(f"in module:\n {spiders_module.__name__}.{module}")
|
||||
return Path(spider_file)
|
||||
|
||||
def _find_template(self, template: str) -> Path | None:
|
||||
template_file = Path(self.templates_dir, f"{template}.tmpl")
|
||||
|
|
|
|||
|
|
@ -47,8 +47,13 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
|
|||
instance.
|
||||
|
||||
The purpose of this custom class is to provide a ``creatorForNetloc()``
|
||||
method that returns a ``_ScrapyClientTLSOptions`` instance configured based
|
||||
on TLS settings provided to the factory.
|
||||
method that returns:
|
||||
|
||||
- a ``_ScrapyClientTLSOptions26`` or ``_ScrapyClientTLSOptions`` instance
|
||||
configured based on TLS settings provided to the factory (when the
|
||||
certificate verification is disabled);
|
||||
- a result of ``optionsForClientTLS()`` called with those TLS settings
|
||||
(when the certificate verification is enabled).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@
|
|||
An asynchronous FTP file download handler for scrapy which somehow emulates an http response.
|
||||
|
||||
FTP connection parameters are passed using the request meta field:
|
||||
- ftp_user (required)
|
||||
- ftp_password (required)
|
||||
- ftp_passive (by default, enabled) sets FTP connection passive mode
|
||||
- ftp_user (optional, falls back to FTP_USER)
|
||||
- ftp_password (optional, falls back to FTP_PASSWORD)
|
||||
- ftp_passive (optional, falls back to FTP_PASSIVE_MODE) sets FTP connection passive mode
|
||||
- ftp_local_filename
|
||||
- If not given, file data will come in the response.body, as a normal scrapy Response,
|
||||
which will imply that the entire file will be on memory.
|
||||
|
|
|
|||
|
|
@ -104,7 +104,6 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
|
|||
self._disconnect_timeout: int = 1
|
||||
|
||||
async def download_request(self, request: Request) -> Response:
|
||||
"""Return a deferred for the HTTP download"""
|
||||
if hasattr(self._crawler.spider, "download_maxsize"): # pragma: no cover
|
||||
warn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE")
|
||||
if hasattr(self._crawler.spider, "download_warnsize"): # pragma: no cover
|
||||
|
|
@ -283,7 +282,7 @@ def _tunnel_request_data(
|
|||
|
||||
|
||||
class _TunnelingAgent(Agent):
|
||||
"""An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS
|
||||
"""An agent that uses a ``_TunnelingTCP4ClientEndpoint`` to make HTTPS
|
||||
downloads. It may look strange that we have chosen to subclass Agent and not
|
||||
ProxyAgent but consider that after the tunnel is opened the proxy is
|
||||
transparent to the client; thus the agent should behave like there is no
|
||||
|
|
@ -545,7 +544,8 @@ class _ScrapyAgent:
|
|||
expected_size, maxsize, request, expected=True
|
||||
)
|
||||
logger.warning(warning_msg)
|
||||
txresponse._transport.loseConnection()
|
||||
# Abort connection immediately.
|
||||
txresponse._transport._producer.abortConnection()
|
||||
raise DownloadCancelledError(warning_msg)
|
||||
|
||||
if warnsize and expected_size > warnsize:
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ class S3DownloadHandler(BaseDownloadHandler):
|
|||
|
||||
async def download_request(self, request: Request) -> Response:
|
||||
p = urlparse_cached(request)
|
||||
scheme = "https" if request.meta.get("is_secure") else "http"
|
||||
scheme = "http" if request.meta.get("is_secure") is False else "https"
|
||||
bucket = p.hostname
|
||||
path = p.path + "?" + p.query if p.query else p.path
|
||||
url = f"{scheme}://{bucket}.s3.amazonaws.com{path}"
|
||||
|
|
|
|||
|
|
@ -114,11 +114,7 @@ class H2ConnectionPool:
|
|||
d.errback(ResponseFailed(errors))
|
||||
|
||||
def close_connections(self) -> None:
|
||||
"""Close all the HTTP/2 connections and remove them from pool
|
||||
|
||||
Returns:
|
||||
Deferred that fires when all connections have been closed
|
||||
"""
|
||||
"""Close all the HTTP/2 connections and remove them from pool."""
|
||||
for conn in self._connections.values():
|
||||
assert conn.transport is not None # typing
|
||||
conn.transport.abortConnection()
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
|
|||
uri is used to verify that incoming client requests have correct
|
||||
base URL.
|
||||
settings -- Scrapy project settings
|
||||
conn_lost_deferred -- Deferred fires with the reason: Failure to notify
|
||||
conn_lost_deferred -- Deferred that fires with the list of underlying exceptions to notify
|
||||
that connection was lost
|
||||
tls_verbose_logging -- Whether to log TLS details
|
||||
"""
|
||||
|
|
@ -375,7 +375,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
|
|||
|
||||
def _handle_events(self, events: list[Event]) -> None:
|
||||
"""Private method which acts as a bridge between the events
|
||||
received from the HTTP/2 data and IH2EventsHandler
|
||||
received from the HTTP/2 data and the handlers in this class.
|
||||
|
||||
Arguments:
|
||||
events -- A list of events that the remote peer triggered by sending data
|
||||
|
|
|
|||
|
|
@ -131,10 +131,10 @@ class Scheduler(BaseScheduler):
|
|||
(:setting:`SCHEDULER_PRIORITY_QUEUE`) that sort requests by
|
||||
:attr:`~scrapy.http.Request.priority`.
|
||||
|
||||
By default, a single, memory-based priority queue is used for all requests.
|
||||
When using :setting:`JOBDIR`, a disk-based priority queue is also created,
|
||||
By default, memory-based priority queues are used for all requests.
|
||||
When using :setting:`JOBDIR`, disk-based priority queues are also created,
|
||||
and only unserializable requests are stored in the memory-based priority
|
||||
queue. For a given priority value, requests in memory take precedence over
|
||||
queues. For a given priority value, requests in memory take precedence over
|
||||
requests in disk.
|
||||
|
||||
Each priority queue stores requests in separate internal queues, one per
|
||||
|
|
@ -209,8 +209,8 @@ class Scheduler(BaseScheduler):
|
|||
-------------------------
|
||||
|
||||
While pending requests are below the configured values of
|
||||
:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
|
||||
or :setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent
|
||||
:setting:`CONCURRENT_REQUESTS` or
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, those requests are sent
|
||||
concurrently.
|
||||
|
||||
As a result, the first few requests of a crawl may not follow the desired
|
||||
|
|
@ -342,7 +342,7 @@ class Scheduler(BaseScheduler):
|
|||
def open(self, spider: Spider) -> Deferred[None] | None:
|
||||
"""
|
||||
(1) initialize the memory queue
|
||||
(2) initialize the disk queue if the ``jobdir`` attribute is a valid directory
|
||||
(2) initialize the disk queue if the ``jobdir`` argument wasn't empty
|
||||
(3) return the result of the dupefilter's ``open`` method
|
||||
"""
|
||||
self.spider: Spider = spider
|
||||
|
|
|
|||
|
|
@ -441,7 +441,7 @@ class Scraper:
|
|||
self, output: Any, response: Response | Failure
|
||||
) -> Deferred[None]:
|
||||
"""Process each Request/Item (given in the output parameter) returned
|
||||
from the given spider.
|
||||
from the spider.
|
||||
|
||||
Items are sent to the item pipelines, requests are scheduled.
|
||||
"""
|
||||
|
|
@ -451,7 +451,7 @@ class Scraper:
|
|||
self, output: Any, response: Response | Failure
|
||||
) -> None:
|
||||
"""Process each Request/Item (given in the output parameter) returned
|
||||
from the given spider.
|
||||
from the spider.
|
||||
|
||||
Items are sent to the item pipelines, requests are scheduled.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -531,8 +531,8 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):
|
|||
"""
|
||||
Run a crawler with the provided arguments.
|
||||
|
||||
It will call the given Crawler's :meth:`~Crawler.crawl` method, while
|
||||
keeping track of it so it can be stopped later.
|
||||
It will call the given Crawler's :meth:`~Crawler.crawl_async` method,
|
||||
while keeping track of it so it can be stopped later.
|
||||
|
||||
If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler`
|
||||
instance, this method will try to create one using this parameter as
|
||||
|
|
@ -773,7 +773,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner):
|
|||
"""
|
||||
This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
|
||||
size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS
|
||||
resolver based on :setting:`DNSCACHE_ENABLED`.
|
||||
resolver based on :setting:`TWISTED_DNS_RESOLVER`.
|
||||
|
||||
If ``stop_after_crawl`` is True, the reactor will be stopped after all
|
||||
crawlers have finished, using :meth:`join`.
|
||||
|
|
@ -875,10 +875,10 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
|
|||
|
||||
When using a reactor it adjusts its pool size to
|
||||
:setting:`REACTOR_THREADPOOL_MAXSIZE` and installs a DNS resolver based
|
||||
on :setting:`DNSCACHE_ENABLED`.
|
||||
on :setting:`TWISTED_DNS_RESOLVER`.
|
||||
|
||||
If ``stop_after_crawl`` is True, the reactor will be stopped after all
|
||||
crawlers have finished, using :meth:`join`.
|
||||
If ``stop_after_crawl`` is True, the reactor/event loop will be stopped
|
||||
after all crawlers have finished, using :meth:`join`.
|
||||
|
||||
:param bool stop_after_crawl: stop or not the reactor when all
|
||||
crawlers have finished
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from scrapy.http.cookies import CookieJar
|
|||
from scrapy.utils.decorators import _warn_spider_arg
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.python import to_unicode
|
||||
from scrapy.utils.request import _decode_cookie, _to_verbose_cookies
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable, Sequence
|
||||
|
|
@ -134,29 +135,10 @@ class CookiesMiddleware:
|
|||
Given a dict consisting of cookie components, return its string representation.
|
||||
Decode from bytes if necessary.
|
||||
"""
|
||||
decoded = {}
|
||||
decoded = _decode_cookie(cookie, request)
|
||||
if decoded is None:
|
||||
return None
|
||||
flags = set()
|
||||
for key in ("name", "value", "path", "domain"):
|
||||
value = cookie.get(key)
|
||||
if value is None:
|
||||
if key in {"name", "value"}:
|
||||
msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
|
||||
logger.warning(msg)
|
||||
return None
|
||||
continue
|
||||
if isinstance(value, (bool, float, int, str)):
|
||||
decoded[key] = str(value)
|
||||
else:
|
||||
assert isinstance(value, bytes)
|
||||
try:
|
||||
decoded[key] = value.decode("utf8")
|
||||
except UnicodeDecodeError:
|
||||
logger.warning(
|
||||
"Non UTF-8 encoded cookie found in request %s: %s",
|
||||
request,
|
||||
cookie,
|
||||
)
|
||||
decoded[key] = value.decode("latin1", errors="replace")
|
||||
for flag in ("secure",):
|
||||
value = cookie.get(flag, _UNSET)
|
||||
if value is _UNSET or not value:
|
||||
|
|
@ -177,11 +159,7 @@ class CookiesMiddleware:
|
|||
"""
|
||||
if not request.cookies:
|
||||
return ()
|
||||
cookies: Iterable[VerboseCookie]
|
||||
if isinstance(request.cookies, dict):
|
||||
cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items())
|
||||
else:
|
||||
cookies = request.cookies
|
||||
cookies: Iterable[VerboseCookie] = _to_verbose_cookies(request.cookies)
|
||||
for cookie in cookies:
|
||||
cookie.setdefault("secure", urlparse_cached(request).scheme == "https")
|
||||
formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ else:
|
|||
|
||||
|
||||
class HttpCompressionMiddleware:
|
||||
"""This middleware allows compressed (gzip, deflate) traffic to be
|
||||
"""This middleware allows compressed (gzip, deflate etc.) traffic to be
|
||||
sent/received from websites"""
|
||||
|
||||
def __init__(
|
||||
|
|
|
|||
|
|
@ -196,10 +196,7 @@ class BaseRedirectMiddleware:
|
|||
|
||||
|
||||
class RedirectMiddleware(BaseRedirectMiddleware):
|
||||
"""
|
||||
Handle redirection of requests based on response status
|
||||
and meta-refresh html tag.
|
||||
"""
|
||||
"""Handle redirection of requests based on response status."""
|
||||
|
||||
@_warn_spider_arg
|
||||
def process_response(
|
||||
|
|
@ -251,6 +248,8 @@ class RedirectMiddleware(BaseRedirectMiddleware):
|
|||
|
||||
|
||||
class MetaRefreshMiddleware(BaseRedirectMiddleware):
|
||||
"""Handle redirection of requests based on meta-refresh html tag."""
|
||||
|
||||
enabled_setting = "METAREFRESH_ENABLED"
|
||||
|
||||
def __init__(self, settings: BaseSettings):
|
||||
|
|
|
|||
|
|
@ -5,9 +5,6 @@ problems such as a connection timeout or HTTP 500 error.
|
|||
You can change the behaviour of this middleware by modifying the scraping settings:
|
||||
RETRY_TIMES - how many times to retry a failed page
|
||||
RETRY_HTTP_CODES - which HTTP response codes to retry
|
||||
|
||||
Failed pages are collected on the scraping process and rescheduled at the end,
|
||||
once the spider has finished crawling all regular (non-failed) pages.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -70,8 +67,9 @@ def get_retry_request(
|
|||
and :ref:`stats <topics-stats>`, and to provide extra logging context (see
|
||||
:func:`logging.debug`).
|
||||
|
||||
*reason* is a string or an :class:`Exception` object that indicates the
|
||||
reason why the request needs to be retried. It is used to name retry stats.
|
||||
*reason* is a string, an :class:`Exception` subclass or an
|
||||
:class:`Exception` object that indicates the reason why the request needs
|
||||
to be retried. It is used to name retry stats.
|
||||
|
||||
*max_retry_times* is a number that determines the maximum number of times
|
||||
that *request* can be retried. If not specified or ``None``, the number is
|
||||
|
|
@ -89,6 +87,9 @@ def get_retry_request(
|
|||
message logged when a request exceeds its retries. See
|
||||
:setting:`RETRY_GIVE_UP_LOG_LEVEL` for details.
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
The *give_up_log_level* parameter.
|
||||
|
||||
*stats_base_key* is a string to be used as the base key for the
|
||||
retry-related job stats
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -115,7 +115,7 @@ class UsageError(Exception):
|
|||
|
||||
class ScrapyDeprecationWarning(Warning):
|
||||
"""Warning category for deprecated features, since the default
|
||||
DeprecationWarning is silenced on Python 2.7+
|
||||
:exc:`DeprecationWarning` is silenced.
|
||||
"""
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ Item Exporters are used to export/serialize items into different formats.
|
|||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import logging
|
||||
import marshal
|
||||
import pickle
|
||||
import pprint
|
||||
|
|
@ -24,6 +25,8 @@ from scrapy.utils.serialize import ScrapyJSONEncoder
|
|||
if TYPE_CHECKING:
|
||||
from json import JSONEncoder
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"BaseItemExporter",
|
||||
"CsvItemExporter",
|
||||
|
|
@ -171,7 +174,15 @@ class XmlItemExporter(BaseItemExporter):
|
|||
super().__init__(**kwargs)
|
||||
if not self.encoding:
|
||||
self.encoding = "utf-8"
|
||||
self.xg = XMLGenerator(file, encoding=self.encoding)
|
||||
# copied from xml.sax.saxutils._gettextwriter()
|
||||
self.stream = TextIOWrapper(
|
||||
file,
|
||||
encoding=self.encoding,
|
||||
errors="xmlcharrefreplace",
|
||||
newline="\n",
|
||||
write_through=True,
|
||||
)
|
||||
self.xg = XMLGenerator(self.stream, encoding=self.encoding)
|
||||
|
||||
def _beautify_newline(self, new_item: bool = False) -> None:
|
||||
if self.indent is not None and (self.indent > 0 or new_item):
|
||||
|
|
@ -199,6 +210,7 @@ class XmlItemExporter(BaseItemExporter):
|
|||
def finish_exporting(self) -> None:
|
||||
self.xg.endElement(self.root_element)
|
||||
self.xg.endDocument()
|
||||
self.stream.detach() # Avoid closing the wrapped file.
|
||||
|
||||
def _export_xml_field(self, name: str, serialized_value: Any, depth: int) -> None:
|
||||
self._beautify_indent(depth=depth)
|
||||
|
|
@ -245,6 +257,8 @@ class CsvItemExporter(BaseItemExporter):
|
|||
self.csv_writer = csv.writer(self.stream, **self._kwargs)
|
||||
self._headers_not_written = True
|
||||
self._join_multivalued = join_multivalued
|
||||
self._autodetected_fields = False
|
||||
self._data_loss_warned = False
|
||||
|
||||
def serialize_field(
|
||||
self, field: Mapping[str, Any] | Field, name: str, value: Any
|
||||
|
|
@ -265,6 +279,22 @@ class CsvItemExporter(BaseItemExporter):
|
|||
self._headers_not_written = False
|
||||
self._write_headers_and_set_fields_to_export(item)
|
||||
|
||||
if (
|
||||
self._autodetected_fields
|
||||
and self.fields_to_export is not None
|
||||
and not self._data_loss_warned
|
||||
):
|
||||
item_fields = ItemAdapter(item).field_names()
|
||||
dropped_fields = set(item_fields) - set(self.fields_to_export)
|
||||
|
||||
if dropped_fields:
|
||||
dropped_fields_display = sorted(dropped_fields)
|
||||
logger.warning(
|
||||
f"CSVExporter dropped fields {dropped_fields_display}. "
|
||||
f"To avoid this, fully configure your FEED_EXPORT_FIELDS setting. "
|
||||
f"See: https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-fields",
|
||||
)
|
||||
self._data_loss_warned = True
|
||||
fields = self._get_serialized_fields(item, default_value="", include_empty=True)
|
||||
values = list(self._build_row(x for _, x in fields))
|
||||
self.csv_writer.writerow(values)
|
||||
|
|
@ -284,6 +314,7 @@ class CsvItemExporter(BaseItemExporter):
|
|||
if not self.fields_to_export:
|
||||
# use declared field names, or keys if the item is a dict
|
||||
self.fields_to_export = ItemAdapter(item).field_names()
|
||||
self._autodetected_fields = True
|
||||
fields: Iterable[str]
|
||||
if isinstance(self.fields_to_export, Mapping):
|
||||
fields = self.fields_to_export.values()
|
||||
|
|
|
|||
|
|
@ -250,20 +250,22 @@ class S3FeedStorage(BlockingFeedStorage):
|
|||
|
||||
def _store_in_thread(self, file: IO[bytes]) -> None:
|
||||
file.seek(0)
|
||||
if self.acl:
|
||||
self.s3_client.upload_fileobj(
|
||||
Bucket=self.bucketname,
|
||||
Key=self.keyname,
|
||||
Fileobj=file,
|
||||
ExtraArgs={"ACL": self.acl},
|
||||
)
|
||||
else:
|
||||
self.s3_client.upload_fileobj(
|
||||
Bucket=self.bucketname,
|
||||
Key=self.keyname,
|
||||
Fileobj=file,
|
||||
)
|
||||
file.close()
|
||||
try:
|
||||
if self.acl:
|
||||
self.s3_client.upload_fileobj(
|
||||
Bucket=self.bucketname,
|
||||
Key=self.keyname,
|
||||
Fileobj=file,
|
||||
ExtraArgs={"ACL": self.acl},
|
||||
)
|
||||
else:
|
||||
self.s3_client.upload_fileobj(
|
||||
Bucket=self.bucketname,
|
||||
Key=self.keyname,
|
||||
Fileobj=file,
|
||||
)
|
||||
finally:
|
||||
file.close()
|
||||
|
||||
|
||||
class GCSFeedStorage(BlockingFeedStorage):
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
class LogStats:
|
||||
"""Log basic scraping stats periodically like:
|
||||
* RPM - Requests per Minute
|
||||
* RPM - Responses per Minute
|
||||
* IPM - Items per Minute
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ class AutoThrottle:
|
|||
# It works better with problematic sites.
|
||||
new_delay = max(target_delay, new_delay)
|
||||
|
||||
# Make sure self.mindelay <= new_delay <= self.max_delay
|
||||
# Make sure self.mindelay <= new_delay <= self.maxdelay
|
||||
new_delay = min(max(self.mindelay, new_delay), self.maxdelay)
|
||||
|
||||
# Dont adjust delay if response status != 200 and new delay is smaller
|
||||
|
|
|
|||
|
|
@ -5,11 +5,9 @@ Use this module (instead of the more specific ones) when importing Headers,
|
|||
Request and Response outside this module.
|
||||
"""
|
||||
|
||||
from warnings import catch_warnings, filterwarnings
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http.headers import Headers
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.http.request.form import FormRequest
|
||||
from scrapy.http.request.json_request import JsonRequest
|
||||
from scrapy.http.request.rpc import XmlRpcRequest
|
||||
from scrapy.http.response import Response
|
||||
|
|
@ -17,19 +15,6 @@ from scrapy.http.response.html import HtmlResponse
|
|||
from scrapy.http.response.json import JsonResponse
|
||||
from scrapy.http.response.text import TextResponse
|
||||
from scrapy.http.response.xml import XmlResponse
|
||||
from scrapy.utils.deprecate import create_deprecated_class
|
||||
|
||||
with catch_warnings():
|
||||
filterwarnings("ignore", category=ScrapyDeprecationWarning)
|
||||
|
||||
from scrapy.http.request.form import FormRequest as _FormRequest
|
||||
|
||||
FormRequest = create_deprecated_class(
|
||||
name="FormRequest",
|
||||
new_class=_FormRequest,
|
||||
subclass_warn_message="{cls} inherits from deprecated class {old}, use the form2request library instead.",
|
||||
instance_warn_message="{cls} is deprecated, use the form2request library instead.",
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"FormRequest",
|
||||
|
|
|
|||
|
|
@ -136,9 +136,9 @@ class _DummyLock:
|
|||
|
||||
|
||||
class WrappedRequest:
|
||||
"""Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class
|
||||
|
||||
see http://docs.python.org/library/urllib2.html#urllib2.Request
|
||||
"""Wraps a :class:`scrapy.Request` class with methods defined by
|
||||
the :class:`urllib.request.Request` class to interact with
|
||||
the :class:`http.cookiejar.CookieJar` class.
|
||||
"""
|
||||
|
||||
def __init__(self, request: Request):
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ class VerboseCookie(TypedDict):
|
|||
secure: NotRequired[bool]
|
||||
|
||||
|
||||
CookiesT: TypeAlias = dict[str, str] | list[VerboseCookie]
|
||||
CookiesT: TypeAlias = dict[str | bytes, str | bytes] | list[VerboseCookie]
|
||||
|
||||
|
||||
RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
|
||||
|
|
|
|||
|
|
@ -32,19 +32,61 @@ if TYPE_CHECKING:
|
|||
|
||||
from scrapy.http.response.text import TextResponse
|
||||
|
||||
warn(
|
||||
"The entire scrapy.http.request.form module is deprecated. Use the "
|
||||
"form2request library instead.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
FormdataVType: TypeAlias = str | Iterable[str]
|
||||
FormdataKVType: TypeAlias = tuple[str, FormdataVType]
|
||||
FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None
|
||||
|
||||
|
||||
class FormRequest(Request):
|
||||
"""A :class:`~scrapy.Request` subclass with a ``formdata`` parameter that
|
||||
url-encodes the given data and assigns it to the request, which makes it
|
||||
convenient to send arbitrary form data via HTTP POST or GET without an HTML
|
||||
``<form>`` element to parse.
|
||||
|
||||
.. note:: To build a request from an HTML ``<form>`` element found in a
|
||||
response, use :doc:`form2request <form2request:index>` instead. See
|
||||
:ref:`form`.
|
||||
|
||||
The remaining arguments are the same as for the :class:`~scrapy.Request`
|
||||
class and are not documented here.
|
||||
|
||||
:param formdata: a dictionary (or iterable of (key, value) tuples)
|
||||
containing HTML form data which will be url-encoded. If
|
||||
:attr:`~scrapy.Request.method` is not given and ``formdata`` is
|
||||
provided, the method is set to ``"POST"`` and the data is assigned to
|
||||
the request body; if the method is ``"GET"``, the data is added to the
|
||||
URL query string instead.
|
||||
:type formdata: dict or collections.abc.Iterable
|
||||
|
||||
To send data via HTTP POST, simulating an HTML form submission, return a
|
||||
:class:`~scrapy.FormRequest` object from your spider:
|
||||
|
||||
.. skip: next
|
||||
.. code-block:: python
|
||||
|
||||
return [
|
||||
FormRequest(
|
||||
url="http://www.example.com/post/action",
|
||||
formdata={"name": "John Doe", "age": "27"},
|
||||
callback=self.after_post,
|
||||
)
|
||||
]
|
||||
|
||||
To send the data in the URL query string instead, use the ``GET`` method:
|
||||
|
||||
.. skip: next
|
||||
.. code-block:: python
|
||||
|
||||
return [
|
||||
FormRequest(
|
||||
url="http://www.example.com/search",
|
||||
method="GET",
|
||||
formdata={"q": "keyword", "page": "1"},
|
||||
callback=self.parse_results,
|
||||
)
|
||||
]
|
||||
"""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
valid_form_methods: ClassVar[list[str]] = ["GET", "POST"]
|
||||
|
|
@ -84,6 +126,13 @@ class FormRequest(Request):
|
|||
formcss: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Self:
|
||||
warn(
|
||||
"FormRequest.from_response() is deprecated. Use the form2request "
|
||||
"library instead.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
kwargs.setdefault("encoding", response.encoding)
|
||||
|
||||
if formcss is not None:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""
|
||||
This module implements the HtmlResponse class which adds encoding
|
||||
discovering through HTML encoding declarations to the TextResponse class.
|
||||
This module implements the :class:`HtmlResponse` class which is used as a
|
||||
content type marker by :class:`~scrapy.selector.Selector` and can be used in
|
||||
``isinstance()`` checks.
|
||||
|
||||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""
|
||||
This module implements the XmlResponse class which adds encoding
|
||||
discovering through XML encoding declarations to the TextResponse class.
|
||||
This module implements the :class:`XmlResponse` class which is used as a
|
||||
content type marker by :class:`~scrapy.selector.Selector` and can be used in
|
||||
``isinstance()`` checks.
|
||||
|
||||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""
|
||||
Scrapy Item
|
||||
|
||||
See documentation in docs/topics/item.rst
|
||||
See documentation in docs/topics/items.rst
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -25,6 +25,25 @@ class Field(dict[str, Any]):
|
|||
"""Container of field metadata"""
|
||||
|
||||
|
||||
def _ordered_field_names(cls: type) -> list[str]:
|
||||
"""Return the names of the :class:`Field` attributes of *cls* in definition
|
||||
order.
|
||||
|
||||
Fields declared in base classes come first, ordered from the topmost base
|
||||
to the most derived class. Within each class, fields keep their definition
|
||||
order. A field redefined in a subclass keeps the position of its first
|
||||
definition.
|
||||
"""
|
||||
names: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for base in reversed(cls.__mro__):
|
||||
for name, value in vars(base).items():
|
||||
if isinstance(value, Field) and name not in seen:
|
||||
seen.add(name)
|
||||
names.append(name)
|
||||
return names
|
||||
|
||||
|
||||
class ItemMeta(ABCMeta):
|
||||
"""Metaclass_ of :class:`Item` that handles field definitions.
|
||||
|
||||
|
|
@ -39,13 +58,9 @@ class ItemMeta(ABCMeta):
|
|||
_class = super().__new__(mcs, "x_" + class_name, new_bases, attrs)
|
||||
|
||||
fields = getattr(_class, "fields", {})
|
||||
new_attrs = {}
|
||||
for n in dir(_class):
|
||||
v = getattr(_class, n)
|
||||
if isinstance(v, Field):
|
||||
fields[n] = v
|
||||
elif n in attrs:
|
||||
new_attrs[n] = attrs[n]
|
||||
for n in _ordered_field_names(_class):
|
||||
fields[n] = getattr(_class, n)
|
||||
new_attrs = {n: v for n, v in attrs.items() if not isinstance(v, Field)}
|
||||
|
||||
new_attrs["fields"] = fields
|
||||
new_attrs["_class"] = _class
|
||||
|
|
@ -80,6 +95,14 @@ class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta):
|
|||
#: those populated. The keys are the field names and the values are the
|
||||
#: :class:`Field` objects used in the :ref:`Item declaration
|
||||
#: <topics-items-declaring>`.
|
||||
#:
|
||||
#: Fields are kept in definition order: fields declared in base classes
|
||||
#: come first, followed by fields declared in subclasses, and a field
|
||||
#: redefined in a subclass keeps the position of its first definition.
|
||||
#:
|
||||
#: .. versionchanged:: 2.17.0
|
||||
#: Fields are now returned in definition order rather than alphabetical
|
||||
#: order.
|
||||
fields: dict[str, Field]
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any):
|
||||
|
|
|
|||
|
|
@ -9,7 +9,9 @@ its documentation in: docs/topics/link-extractors.rst
|
|||
class Link:
|
||||
"""Link objects represent an extracted link by the LinkExtractor.
|
||||
|
||||
Using the anchor tag sample below to illustrate the parameters::
|
||||
Using the anchor tag sample below to illustrate the parameters:
|
||||
|
||||
.. code-block:: html
|
||||
|
||||
<a href="https://example.com/nofollow.html#foo" rel="nofollow">Dont follow this one</a>
|
||||
|
||||
|
|
|
|||
|
|
@ -57,6 +57,18 @@ def _canonicalize_link_url(link: Link) -> str:
|
|||
return canonicalize_url(link.url, keep_fragments=True)
|
||||
|
||||
|
||||
def _name_matches(allowed: set[str], denied: set[str], name: str) -> bool:
|
||||
"""Return whether a tag or attribute *name* should be considered.
|
||||
|
||||
A name matches when it is allowed and not denied. A name is allowed if it is
|
||||
listed in *allowed*, or if *allowed* contains the ``"*"`` wildcard, which
|
||||
matches every name. *denied* has precedence over *allowed*.
|
||||
"""
|
||||
if name in denied:
|
||||
return False
|
||||
return "*" in allowed or name in allowed
|
||||
|
||||
|
||||
class LxmlParserLinkExtractor:
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -162,6 +174,126 @@ _RegexOrSeveral: TypeAlias = _Regex | Iterable[_Regex]
|
|||
|
||||
|
||||
class LxmlLinkExtractor:
|
||||
r"""LxmlLinkExtractor is the recommended link extractor with handy filtering
|
||||
options. It is implemented using lxml's robust HTMLParser.
|
||||
|
||||
:param allow: a single regular expression (or list of regular expressions)
|
||||
that the (absolute) urls must match in order to be extracted. If not
|
||||
given (or empty), it will match all links.
|
||||
:type allow: str or list
|
||||
|
||||
:param deny: a single regular expression (or list of regular expressions)
|
||||
that the (absolute) urls must match in order to be excluded (i.e. not
|
||||
extracted). It has precedence over the ``allow`` parameter. If not
|
||||
given (or empty) it won't exclude any links.
|
||||
:type deny: str or list
|
||||
|
||||
:param allow_domains: a single value or a list of string containing
|
||||
domains which will be considered for extracting the links
|
||||
:type allow_domains: str or list
|
||||
|
||||
:param deny_domains: a single value or a list of strings containing
|
||||
domains which won't be considered for extracting the links
|
||||
:type deny_domains: str or list
|
||||
|
||||
:param deny_extensions: a single value or list of strings containing
|
||||
extensions that should be ignored when extracting links.
|
||||
If not given, it will default to
|
||||
:data:`scrapy.linkextractors.IGNORED_EXTENSIONS`.
|
||||
:type deny_extensions: list
|
||||
|
||||
:param restrict_xpaths: is an XPath (or list of XPath's) which defines
|
||||
regions inside the response where links should be extracted from.
|
||||
If given, only the text selected by those XPath will be scanned for
|
||||
links.
|
||||
:type restrict_xpaths: str or list
|
||||
|
||||
:param restrict_css: a CSS selector (or list of selectors) which defines
|
||||
regions inside the response where links should be extracted from.
|
||||
Has the same behaviour as ``restrict_xpaths``.
|
||||
:type restrict_css: str or list
|
||||
|
||||
:param restrict_text: a single regular expression (or list of regular
|
||||
expressions) that the link's text must match in order to be extracted.
|
||||
If not given (or empty), it will match all links. If a list of regular
|
||||
expressions is given, the link will be extracted if it matches at least
|
||||
one.
|
||||
:type restrict_text: str or list
|
||||
|
||||
:param tags: a tag or a list of tags to consider when extracting links.
|
||||
Defaults to ``('a', 'area')``. Use ``'*'`` to consider every tag.
|
||||
:type tags: str or list
|
||||
|
||||
:param attrs: an attribute or list of attributes which should be considered
|
||||
when looking for links to extract (only for those tags specified in the
|
||||
``tags`` parameter). Defaults to ``('href',)``. Use ``'*'`` to consider
|
||||
every attribute.
|
||||
:type attrs: list
|
||||
|
||||
:param deny_tags: a tag or a list of tags that should not be considered when
|
||||
extracting links. It has precedence over the ``tags`` parameter, so it
|
||||
can be combined with ``tags='*'`` to consider every tag except a few.
|
||||
Defaults to ``()`` (no tag is excluded).
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
:type deny_tags: str or list
|
||||
|
||||
:param deny_attrs: an attribute or a list of attributes that should not be
|
||||
considered when looking for links to extract. It has precedence over the
|
||||
``attrs`` parameter, so it can be combined with ``attrs='*'`` to consider
|
||||
every attribute except a few. Defaults to ``()`` (no attribute is
|
||||
excluded).
|
||||
|
||||
.. versionadded:: 2.17.0
|
||||
:type deny_attrs: str or list
|
||||
|
||||
:param canonicalize: canonicalize each extracted url (using
|
||||
w3lib.url.canonicalize_url). Defaults to ``False``.
|
||||
Note that canonicalize_url is meant for duplicate checking;
|
||||
it can change the URL visible at server side, so the response can be
|
||||
different for requests with canonicalized and raw URLs. If you're
|
||||
using LinkExtractor to follow links it is more robust to
|
||||
keep the default ``canonicalize=False``.
|
||||
:type canonicalize: bool
|
||||
|
||||
:param unique: whether duplicate filtering should be applied to extracted
|
||||
links.
|
||||
:type unique: bool
|
||||
|
||||
:param process_value: a function which receives each value extracted from
|
||||
the tag and attributes scanned and can modify the value and return a
|
||||
new one, or return ``None`` to ignore the link altogether. If not
|
||||
given, ``process_value`` defaults to ``lambda x: x``.
|
||||
|
||||
.. highlight:: html
|
||||
|
||||
For example, to extract links from this code::
|
||||
|
||||
<a href="javascript:goToPage('../other/page.html'); return false">Link text</a>
|
||||
|
||||
.. highlight:: python
|
||||
|
||||
You can use the following function in ``process_value``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def process_value(value):
|
||||
m = re.search(r"javascript:goToPage\('(.*?)'", value)
|
||||
if m:
|
||||
return m.group(1)
|
||||
|
||||
:type process_value: collections.abc.Callable
|
||||
|
||||
:param strip: whether to strip whitespaces from extracted attributes.
|
||||
According to HTML5 standard, leading and trailing whitespaces
|
||||
must be stripped from ``href`` attributes of ``<a>``, ``<area>``
|
||||
and many other elements, ``src`` attribute of ``<img>``, ``<iframe>``
|
||||
elements, etc., so LinkExtractor strips space chars by default.
|
||||
Set ``strip=False`` to turn it off (e.g. if you're extracting urls
|
||||
from elements or attributes which allow leading/trailing whitespaces).
|
||||
:type strip: bool
|
||||
"""
|
||||
|
||||
_csstranslator = HTMLTranslator()
|
||||
|
||||
def __init__(
|
||||
|
|
@ -180,11 +312,17 @@ class LxmlLinkExtractor:
|
|||
restrict_css: str | Iterable[str] = (),
|
||||
strip: bool = True,
|
||||
restrict_text: _RegexOrSeveral | None = None,
|
||||
deny_tags: str | Iterable[str] = (),
|
||||
deny_attrs: str | Iterable[str] = (),
|
||||
):
|
||||
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
|
||||
deny_tags, deny_attrs = (
|
||||
set(arg_to_iter(deny_tags)),
|
||||
set(arg_to_iter(deny_attrs)),
|
||||
)
|
||||
self.link_extractor = LxmlParserLinkExtractor(
|
||||
tag=partial(operator.contains, tags),
|
||||
attr=partial(operator.contains, attrs),
|
||||
tag=partial(_name_matches, tags, deny_tags),
|
||||
attr=partial(_name_matches, attrs, deny_attrs),
|
||||
unique=unique,
|
||||
process=process_value,
|
||||
strip=strip,
|
||||
|
|
|
|||
|
|
@ -58,18 +58,20 @@ class LogFormatter:
|
|||
logging an action the method must return ``None``.
|
||||
|
||||
Here is an example on how to create a custom log formatter to lower the severity level of
|
||||
the log message when an item is dropped from the pipeline::
|
||||
the log message when an item is dropped from the pipeline:
|
||||
|
||||
class PoliteLogFormatter(logformatter.LogFormatter):
|
||||
def dropped(self, item, exception, response, spider):
|
||||
return {
|
||||
'level': logging.INFO, # lowering the level from logging.WARNING
|
||||
'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s",
|
||||
'args': {
|
||||
'exception': exception,
|
||||
'item': item,
|
||||
}
|
||||
}
|
||||
.. code-block:: python
|
||||
|
||||
class PoliteLogFormatter(logformatter.LogFormatter):
|
||||
def dropped(self, item, exception, response, spider):
|
||||
return {
|
||||
"level": logging.INFO, # lowering the level from logging.WARNING
|
||||
"msg": "Dropped: %(exception)s" + os.linesep + "%(item)s",
|
||||
"args": {
|
||||
"exception": exception,
|
||||
"item": item,
|
||||
},
|
||||
}
|
||||
"""
|
||||
|
||||
def crawled(
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
"""
|
||||
Mail sending helpers
|
||||
|
||||
See documentation in docs/topics/email.rst
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""
|
||||
Item pipeline
|
||||
|
||||
See documentation in docs/item-pipeline.rst
|
||||
See documentation in docs/topics/item-pipeline.rst
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
|
|||
|
|
@ -423,7 +423,7 @@ class FTPFilesStore:
|
|||
|
||||
|
||||
class FilesPipeline(MediaPipeline):
|
||||
"""Abstract pipeline that implement the file downloading
|
||||
"""Pipeline that implements file downloading.
|
||||
|
||||
This pipeline tries to minimize network transfers and file processing,
|
||||
doing stat of the files and determining if file is new, up-to-date or
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ class ImageException(FileException):
|
|||
|
||||
|
||||
class ImagesPipeline(FilesPipeline):
|
||||
"""Abstract pipeline that implement the image thumbnail generation logic"""
|
||||
"""Pipeline that implements the handling logic specific to images."""
|
||||
|
||||
MEDIA_NAME: str = "image"
|
||||
|
||||
|
|
|
|||
|
|
@ -92,9 +92,10 @@ class ScrapyPriorityQueue:
|
|||
- The :data:`~scrapy.Request.priority` of the request.
|
||||
|
||||
For each combination of the above seen, this class creates an instance of
|
||||
*downstream_queue_cls* with *key* set to a subdirectory of the persistence
|
||||
directory, named as the request priority (e.g. ``1``), with an ``s`` suffix
|
||||
in case of a start request (e.g. ``1s``).
|
||||
*downstream_queue_cls* (or *start_queue_cls* for start requests if it was
|
||||
passed) with *key* set to a subdirectory of the persistence directory,
|
||||
named as the negated request priority (e.g. ``-1``), with an ``s`` suffix
|
||||
in case of a start request (e.g. ``-1s``).
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
|
|
@ -141,10 +142,14 @@ class ScrapyPriorityQueue:
|
|||
q = self.qfactory(priority)
|
||||
if q:
|
||||
self.queues[priority] = q
|
||||
else:
|
||||
q.close()
|
||||
if self._start_queue_cls:
|
||||
q = self._sqfactory(priority)
|
||||
if q:
|
||||
self._start_queues[priority] = q
|
||||
else:
|
||||
q.close()
|
||||
|
||||
self.curprio = min(startprios)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,6 @@
|
|||
"""
|
||||
XPath selectors based on lxml
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from typing import Any, Literal
|
||||
|
||||
from parsel import Selector as _ParselSelector
|
||||
|
||||
|
|
@ -18,13 +14,10 @@ __all__ = ["Selector", "SelectorList"]
|
|||
_NOT_SET = object()
|
||||
|
||||
|
||||
def _st(response: TextResponse | None, st: str | None) -> str:
|
||||
if st is None:
|
||||
return "xml" if isinstance(response, XmlResponse) else "html"
|
||||
return st
|
||||
SelectorType = Literal["html", "xml", "json", "text"]
|
||||
|
||||
|
||||
def _response_from_text(text: str | bytes, st: str | None) -> TextResponse:
|
||||
def _response_from_text(text: str | bytes, st: SelectorType | None) -> TextResponse:
|
||||
rt: type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse
|
||||
return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8"))
|
||||
|
||||
|
|
@ -49,23 +42,16 @@ class Selector(_ParselSelector, object_ref):
|
|||
``response`` isn't available. Using ``text`` and ``response`` together is
|
||||
undefined behavior.
|
||||
|
||||
``type`` defines the selector type, it can be ``"html"``, ``"xml"``, ``"json"``
|
||||
or ``None`` (default).
|
||||
``type`` defines the selector type, it can be ``"html"``, ``"xml"``,
|
||||
``"json"``, ``"text"`` or ``None`` (default). It's passed to
|
||||
:class:`parsel.Selector` and its meaning is defined there. However, when
|
||||
``type`` is ``None``, it is set to ``"xml"`` for an
|
||||
:class:`~scrapy.http.XmlResponse` and to ``"html"`` otherwise before
|
||||
passing it to :class:`parsel.Selector`.
|
||||
|
||||
If ``type`` is ``None``, the selector automatically chooses the best type
|
||||
based on ``response`` type (see below), or defaults to ``"html"`` in case it
|
||||
is used together with ``text``.
|
||||
|
||||
If ``type`` is ``None`` and a ``response`` is passed, the selector type is
|
||||
inferred from the response type as follows:
|
||||
|
||||
* ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
|
||||
* ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
|
||||
* ``"json"`` for :class:`~scrapy.http.TextResponse` type
|
||||
* ``"html"`` for anything else
|
||||
|
||||
Otherwise, if ``type`` is set, the selector type will be forced and no
|
||||
detection will occur.
|
||||
.. note:: JSON selector support requires ``parsel`` 1.8.0 or higher. With
|
||||
older versions setting ``type`` to ``"json"`` or ``"text"`` is not
|
||||
supported.
|
||||
"""
|
||||
|
||||
__slots__ = ["response"]
|
||||
|
|
@ -75,7 +61,7 @@ class Selector(_ParselSelector, object_ref):
|
|||
self,
|
||||
response: TextResponse | None = None,
|
||||
text: str | None = None,
|
||||
type: str | None = None, # noqa: A002
|
||||
type: SelectorType | None = None, # noqa: A002
|
||||
root: Any | None = _NOT_SET,
|
||||
**kwargs: Any,
|
||||
):
|
||||
|
|
@ -84,10 +70,11 @@ class Selector(_ParselSelector, object_ref):
|
|||
f"{self.__class__.__name__}.__init__() received both response and text"
|
||||
)
|
||||
|
||||
st = _st(response, type)
|
||||
if type is None:
|
||||
type = "xml" if isinstance(response, XmlResponse) else "html" # noqa: A001
|
||||
|
||||
if text is not None:
|
||||
response = _response_from_text(text, st)
|
||||
response = _response_from_text(text, type)
|
||||
|
||||
if response is not None:
|
||||
text = response.text
|
||||
|
|
@ -98,4 +85,4 @@ class Selector(_ParselSelector, object_ref):
|
|||
if root is not _NOT_SET:
|
||||
kwargs["root"] = root
|
||||
|
||||
super().__init__(text=text, type=st, **kwargs)
|
||||
super().__init__(text=text, type=type, **kwargs)
|
||||
|
|
|
|||
|
|
@ -454,9 +454,10 @@ class BaseSettings(MutableMapping[str, Any]):
|
|||
"""
|
||||
Store a key/value attribute with a given priority.
|
||||
|
||||
Settings should be populated *before* configuring the Crawler object
|
||||
(through the :meth:`~scrapy.crawler.Crawler.configure` method),
|
||||
otherwise they won't have any effect.
|
||||
Settings should be populated *before* the Crawler object applies them
|
||||
(in the :meth:`~scrapy.crawler.Crawler.crawl_async` or
|
||||
:meth:`~scrapy.crawler.Crawler.crawl` method), otherwise they won't
|
||||
have any effect.
|
||||
|
||||
:param name: the setting name
|
||||
:type name: str
|
||||
|
|
@ -613,7 +614,7 @@ class BaseSettings(MutableMapping[str, Any]):
|
|||
"""
|
||||
Make a deep copy of current settings.
|
||||
|
||||
This method returns a new instance of the :class:`Settings` class,
|
||||
This method returns a new instance of this class,
|
||||
populated with the same values and their priorities.
|
||||
|
||||
Modifications to the new object won't be reflected on the original
|
||||
|
|
@ -658,7 +659,7 @@ class BaseSettings(MutableMapping[str, Any]):
|
|||
Make a copy of current settings and convert to a dict.
|
||||
|
||||
This method returns a new dict populated with the same values
|
||||
and their priorities as the current settings.
|
||||
as the current settings.
|
||||
|
||||
Modifications to the returned dict won't be reflected on the original
|
||||
settings.
|
||||
|
|
|
|||
|
|
@ -240,7 +240,7 @@ BOT_NAME = "scrapybot"
|
|||
CLOSESPIDER_ERRORCOUNT = 0
|
||||
CLOSESPIDER_ITEMCOUNT = 0
|
||||
CLOSESPIDER_PAGECOUNT = 0
|
||||
CLOSESPIDER_TIMEOUT = 0
|
||||
CLOSESPIDER_TIMEOUT = 0.0
|
||||
CLOSESPIDER_PAGECOUNT_NO_ITEM = 0
|
||||
CLOSESPIDER_TIMEOUT_NO_ITEM = 0
|
||||
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ if TYPE_CHECKING:
|
|||
# running event loop.
|
||||
#
|
||||
# Side note: it should be possible to remove _request_deferred() by using
|
||||
# engine.download_async() instead of engine.schedule(), losing the usual stuff
|
||||
# engine.download_async() instead of engine.crawl(), losing the usual stuff
|
||||
# like spider middlewares (none of which should be important).
|
||||
#
|
||||
# Other architecture problems:
|
||||
|
|
@ -188,7 +188,8 @@ class Shell:
|
|||
async def _schedule(self, request: Request, spider: Spider | None) -> Response:
|
||||
"""Send the request to the engine, wait for the result.
|
||||
|
||||
Runs in the reactor thread.
|
||||
Runs in the reactor thread when using the reactor, or in the asyncio
|
||||
event loop thread otherwise.
|
||||
"""
|
||||
if not self.spider:
|
||||
await self._open_spider(spider)
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ class BaseSpiderMiddleware:
|
|||
) -> Request | None:
|
||||
"""Return a processed request from the spider output.
|
||||
|
||||
This method is called with a single request from the start seeds or the
|
||||
This method is called with a single request from ``start()`` or the
|
||||
spider output. It should return the same or a different request, or
|
||||
``None`` to ignore it.
|
||||
|
||||
|
|
@ -84,7 +84,7 @@ class BaseSpiderMiddleware:
|
|||
|
||||
:param response: the response being processed
|
||||
:type response: :class:`~scrapy.http.Response` object or ``None`` for
|
||||
start seeds
|
||||
start requests
|
||||
|
||||
:return: the processed request or ``None``
|
||||
"""
|
||||
|
|
@ -93,7 +93,7 @@ class BaseSpiderMiddleware:
|
|||
def get_processed_item(self, item: Any, response: Response | None) -> Any:
|
||||
"""Return a processed item from the spider output.
|
||||
|
||||
This method is called with a single item from the start seeds or the
|
||||
This method is called with a single item from ``start()`` or the
|
||||
spider output. It should return the same or a different item, or
|
||||
``None`` to ignore it.
|
||||
|
||||
|
|
@ -102,7 +102,7 @@ class BaseSpiderMiddleware:
|
|||
|
||||
:param response: the response being processed
|
||||
:type response: :class:`~scrapy.http.Response` object or ``None`` for
|
||||
start seeds
|
||||
start items
|
||||
|
||||
:return: the processed item or ``None``
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
|
||||
class HttpError(IgnoreRequest):
|
||||
"""A non-200 response was filtered"""
|
||||
"""A non-2xx response was filtered"""
|
||||
|
||||
def __init__(self, response: Response, *args: Any, **kwargs: Any):
|
||||
self.response = response
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ class ReferrerPolicy(ABC):
|
|||
)
|
||||
|
||||
def origin(self, url: str) -> str | None:
|
||||
"""Return serialized origin (scheme, host, path) for a request or response URL."""
|
||||
"""Return serialized origin (scheme, host, port) for a request or response URL."""
|
||||
return self.strip_url(url, origin_only=True)
|
||||
|
||||
def potentially_trustworthy(self, url: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -54,17 +54,21 @@ class XMLFeedSpider(Spider):
|
|||
return response
|
||||
|
||||
def parse_node(self, response: Response, selector: Selector) -> Any:
|
||||
"""This method must be overridden with your custom spider functionality"""
|
||||
"""This method is called for the nodes matching the provided tag name
|
||||
(itertag). Receives the response and an Selector for each node.
|
||||
|
||||
This method must return either an item, a request, or a list
|
||||
containing any of them.
|
||||
|
||||
This method must be overridden with your custom spider functionality.
|
||||
"""
|
||||
if hasattr(self, "parse_item"): # backward compatibility
|
||||
return self.parse_item(response, selector)
|
||||
raise NotImplementedError
|
||||
|
||||
def parse_nodes(self, response: Response, nodes: Iterable[Selector]) -> Any:
|
||||
"""This method is called for the nodes matching the provided tag name
|
||||
(itertag). Receives the response and an Selector for each node.
|
||||
Overriding this method is mandatory. Otherwise, you spider won't work.
|
||||
This method must return either an item, a request, or a list
|
||||
containing any of them.
|
||||
(itertag). Receives the response and an iterable of Selectors.
|
||||
"""
|
||||
|
||||
for selector in nodes:
|
||||
|
|
@ -113,6 +117,9 @@ class CSVFeedSpider(Spider):
|
|||
It receives a CSV file in a response; iterates through each of its rows,
|
||||
and calls parse_row with a dict containing each field's data.
|
||||
|
||||
This spider also gives the opportunity to override adapt_response and
|
||||
process_results methods for pre and post-processing purposes.
|
||||
|
||||
You can set some options regarding the CSV file, such as the delimiter, quotechar
|
||||
and the file's headers.
|
||||
"""
|
||||
|
|
@ -136,16 +143,14 @@ class CSVFeedSpider(Spider):
|
|||
return response
|
||||
|
||||
def parse_row(self, response: Response, row: dict[str, str]) -> Any:
|
||||
"""This method must be overridden with your custom spider functionality"""
|
||||
"""Receives a response and a dict (representing each row) with a key for
|
||||
each provided (or detected) header of the CSV file.
|
||||
|
||||
This method must be overridden with your custom spider functionality.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def parse_rows(self, response: Response) -> Any:
|
||||
"""Receives a response and a dict (representing each row) with a key for
|
||||
each provided (or detected) header of the CSV file. This spider also
|
||||
gives the opportunity to override adapt_response and
|
||||
process_results methods for pre and post-processing purposes.
|
||||
"""
|
||||
|
||||
for row in csviter(
|
||||
response, self.delimiter, self.headers, quotechar=self.quotechar
|
||||
):
|
||||
|
|
|
|||
|
|
@ -9,5 +9,5 @@ from itemadapter import ItemAdapter
|
|||
|
||||
|
||||
class ${ProjectName}Pipeline:
|
||||
def process_item(self, item, spider):
|
||||
def process_item(self, item):
|
||||
return item
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
"""
|
||||
This module contains data types used by Scrapy which are not included in the
|
||||
Python Standard Library.
|
||||
|
||||
This module must not depend on any module outside the Standard Library.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -152,7 +150,9 @@ class LocalCache(OrderedDict[_KT, _VT]):
|
|||
self.limit: int | None = limit
|
||||
|
||||
def __setitem__(self, key: _KT, value: _VT) -> None:
|
||||
if self.limit:
|
||||
if self.limit is not None:
|
||||
if self.limit == 0:
|
||||
return
|
||||
while len(self) >= self.limit:
|
||||
self.popitem(last=False)
|
||||
super().__setitem__(key, value)
|
||||
|
|
|
|||
|
|
@ -103,7 +103,7 @@ async def _defer_sleep_async() -> None:
|
|||
def defer_result(result: Any) -> Deferred[Any]: # pragma: no cover
|
||||
warnings.warn(
|
||||
"scrapy.utils.defer.defer_result() is deprecated, use"
|
||||
" twisted.internet.defer.success() and twisted.internet.defer.fail(),"
|
||||
" twisted.internet.defer.succeed() and twisted.internet.defer.fail(),"
|
||||
" plus an explicit sleep if needed, or explicit reactor.callLater().",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
|
|
@ -469,22 +469,22 @@ def _maybeDeferred_coro(
|
|||
def deferred_to_future(d: Deferred[_T]) -> Future[_T]:
|
||||
"""Return an :class:`asyncio.Future` object that wraps *d*.
|
||||
|
||||
This function requires
|
||||
:class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be
|
||||
installed.
|
||||
This function requires an installed asyncio reactor or a running asyncio
|
||||
event loop, see :ref:`using-asyncio`.
|
||||
|
||||
When :ref:`using the asyncio reactor <install-asyncio>`, you cannot await
|
||||
on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy
|
||||
callables defined as coroutines <coroutine-support>`, you can only await on
|
||||
``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects
|
||||
allows you to wait on them::
|
||||
In this state you cannot await on :class:`~twisted.internet.defer.Deferred`
|
||||
objects from :ref:`Scrapy callables defined as coroutines
|
||||
<coroutine-support>`, you can only await on ``Future`` objects. Wrapping
|
||||
``Deferred`` objects into ``Future`` objects allows you to wait on them:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
class MySpider(Spider):
|
||||
...
|
||||
|
||||
async def parse(self, response):
|
||||
additional_request = scrapy.Request('https://example.org/price')
|
||||
deferred = self.crawler.engine.download(additional_request)
|
||||
additional_response = await deferred_to_future(deferred)
|
||||
deferred = some_dfd_helper()
|
||||
result = await deferred_to_future(deferred)
|
||||
|
||||
.. versionchanged:: 2.14
|
||||
This function no longer installs an asyncio loop if called before the
|
||||
|
|
@ -492,7 +492,10 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]:
|
|||
in this case.
|
||||
"""
|
||||
if not is_asyncio_available():
|
||||
raise RuntimeError("deferred_to_future() requires AsyncioSelectorReactor.")
|
||||
raise RuntimeError(
|
||||
"deferred_to_future() requires an installed asyncio reactor"
|
||||
" or a running asyncio event loop."
|
||||
)
|
||||
return d.asFuture(asyncio.get_event_loop())
|
||||
|
||||
|
||||
|
|
@ -501,23 +504,26 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]:
|
|||
defined as a coroutine <coroutine-support>`.
|
||||
|
||||
What you can await in Scrapy callables defined as coroutines depends on the
|
||||
value of :setting:`TWISTED_REACTOR`:
|
||||
value of :setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED`:
|
||||
|
||||
- When :ref:`using the asyncio reactor <install-asyncio>`, you can only
|
||||
await on :class:`asyncio.Future` objects.
|
||||
- When :ref:`using the asyncio reactor <install-asyncio>`, or :ref:`not
|
||||
using a reactor at all <asyncio-without-reactor>`, you can only await
|
||||
on :class:`asyncio.Future` objects.
|
||||
|
||||
- When not using the asyncio reactor, you can only await on
|
||||
:class:`~twisted.internet.defer.Deferred` objects.
|
||||
- When :ref:`using a non-asyncio reactor <disable-asyncio>`, you can only
|
||||
await on :class:`~twisted.internet.defer.Deferred` objects.
|
||||
|
||||
If you want to write code that uses ``Deferred`` objects but works with any
|
||||
reactor, use this function on all ``Deferred`` objects::
|
||||
If you want to write code that uses ``Deferred`` objects but works in both
|
||||
of these states, use this function on all ``Deferred`` objects:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
class MySpider(Spider):
|
||||
...
|
||||
|
||||
async def parse(self, response):
|
||||
additional_request = scrapy.Request('https://example.org/price')
|
||||
deferred = self.crawler.engine.download(additional_request)
|
||||
additional_response = await maybe_deferred_to_future(deferred)
|
||||
deferred = some_dfd_helper()
|
||||
result = await maybe_deferred_to_future(deferred)
|
||||
"""
|
||||
if not is_asyncio_available():
|
||||
return d
|
||||
|
|
|
|||
|
|
@ -43,15 +43,18 @@ def create_deprecated_class(
|
|||
It can be used to rename a base class in a library. For example, if we
|
||||
have
|
||||
|
||||
class OldName(SomeClass):
|
||||
# ...
|
||||
.. code-block:: python
|
||||
|
||||
and we want to rename it to NewName, we can do the following::
|
||||
class OldName(SomeClass): ...
|
||||
|
||||
class NewName(SomeClass):
|
||||
# ...
|
||||
and we want to rename it to NewName, we can do the following:
|
||||
|
||||
OldName = create_deprecated_class('OldName', NewName)
|
||||
.. code-block:: python
|
||||
|
||||
class NewName(SomeClass): ...
|
||||
|
||||
|
||||
OldName = create_deprecated_class("OldName", NewName)
|
||||
|
||||
Then, if user class inherits from OldName, warning is issued. Also, if
|
||||
some code uses ``issubclass(sub, OldName)`` or ``isinstance(sub(), OldName)``
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import posixpath
|
||||
from contextlib import closing
|
||||
from ftplib import FTP, error_perm
|
||||
from posixpath import dirname
|
||||
from typing import IO
|
||||
|
|
@ -32,7 +33,7 @@ def ftp_store_file(
|
|||
"""Opens a FTP connection with passed credentials,sets current directory
|
||||
to the directory extracted from given path, then uploads the file to server
|
||||
"""
|
||||
with FTP() as ftp:
|
||||
with FTP() as ftp, closing(file):
|
||||
ftp.connect(host, port)
|
||||
ftp.login(username, password)
|
||||
if use_active_mode:
|
||||
|
|
@ -42,4 +43,3 @@ def ftp_store_file(
|
|||
ftp_makedirs_cwd(ftp, dirname)
|
||||
command = "STOR" if overwrite else "APPE"
|
||||
ftp.storbinary(f"{command} {filename}", file)
|
||||
file.close()
|
||||
|
|
|
|||
|
|
@ -149,11 +149,12 @@ def install_scrapy_root_handler(settings: Settings) -> None:
|
|||
def _uninstall_scrapy_root_handler() -> None:
|
||||
global _scrapy_root_handler # noqa: PLW0603
|
||||
|
||||
if (
|
||||
_scrapy_root_handler is not None
|
||||
and _scrapy_root_handler in logging.root.handlers
|
||||
):
|
||||
if _scrapy_root_handler is None:
|
||||
return
|
||||
|
||||
if _scrapy_root_handler in logging.root.handlers:
|
||||
logging.root.removeHandler(_scrapy_root_handler)
|
||||
_scrapy_root_handler.close()
|
||||
_scrapy_root_handler = None
|
||||
|
||||
|
||||
|
|
@ -247,8 +248,7 @@ def logformatter_adapter(
|
|||
) -> tuple[int, str, dict[str, Any] | tuple[Any, ...]]:
|
||||
"""
|
||||
Helper that takes the dictionary output from the methods in LogFormatter
|
||||
and adapts it into a tuple of positional arguments for logger.log calls,
|
||||
handling backward compatibility as well.
|
||||
and adapts it into a tuple of positional arguments for logger.log calls.
|
||||
"""
|
||||
|
||||
level = logkws.get("level", logging.INFO)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from __future__ import annotations
|
|||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
from urllib.parse import urlunparse
|
||||
from weakref import WeakKeyDictionary
|
||||
|
|
@ -25,6 +26,9 @@ if TYPE_CHECKING:
|
|||
from typing_extensions import Self
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.http.request import CookiesT, VerboseCookie
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_fingerprint_cache: WeakKeyDictionary[
|
||||
|
|
@ -179,17 +183,52 @@ def _get_method(obj: Any, name: Any) -> Any:
|
|||
raise ValueError(f"Method {name!r} not found in: {obj}") from None
|
||||
|
||||
|
||||
def _cookie_value_to_unicode(value: str | bytes | float) -> str:
|
||||
if isinstance(value, bytes):
|
||||
return value.decode()
|
||||
return str(value)
|
||||
def _to_verbose_cookies(cookies: CookiesT) -> list[VerboseCookie]:
|
||||
"""Return a list of verbose cookies from ``request.cookies``.
|
||||
|
||||
The list of dicts form is returned as is, the dict one is converted first.
|
||||
"""
|
||||
if isinstance(cookies, dict):
|
||||
return [{"name": k, "value": v} for k, v in cookies.items()]
|
||||
return cookies
|
||||
|
||||
|
||||
def _decode_cookie(cookie: VerboseCookie, request: Request) -> dict[str, str] | None:
|
||||
"""Return a dict with non-flag verbose cookie values converted to strings.
|
||||
|
||||
``name``, ``value``, ``path``, ``domain`` are included, ``secure`` isn't.
|
||||
"""
|
||||
|
||||
decoded = {}
|
||||
for key in ("name", "value", "path", "domain"):
|
||||
value = cookie.get(key)
|
||||
if value is None:
|
||||
if key in {"name", "value"}:
|
||||
logger.warning(
|
||||
f"Invalid cookie found in request {request}:"
|
||||
f" {cookie} ('{key}' is missing)"
|
||||
)
|
||||
return None
|
||||
continue
|
||||
if isinstance(value, (bool, float, int, str)):
|
||||
decoded[key] = str(value)
|
||||
else:
|
||||
assert isinstance(value, bytes)
|
||||
try:
|
||||
decoded[key] = value.decode("utf8")
|
||||
except UnicodeDecodeError:
|
||||
logger.warning(
|
||||
f"Non UTF-8 encoded cookie found in request {request}: {cookie}",
|
||||
)
|
||||
decoded[key] = value.decode("latin1", errors="replace")
|
||||
return decoded
|
||||
|
||||
|
||||
def request_to_curl(request: Request) -> str:
|
||||
"""
|
||||
Converts a :class:`~scrapy.Request` object to a curl command.
|
||||
|
||||
:param :class:`~scrapy.Request`: Request object to be converted
|
||||
:param request: Request object to be converted
|
||||
:return: string containing the curl command
|
||||
"""
|
||||
method = request.method
|
||||
|
|
@ -201,19 +240,14 @@ def request_to_curl(request: Request) -> str:
|
|||
)
|
||||
|
||||
url = request.url
|
||||
cookies = ""
|
||||
if request.cookies:
|
||||
if isinstance(request.cookies, dict):
|
||||
cookie = "; ".join(f"{k}={v}" for k, v in request.cookies.items())
|
||||
cookies = f"--cookie '{cookie}'"
|
||||
elif isinstance(request.cookies, list):
|
||||
cookie = "; ".join(
|
||||
f"{_cookie_value_to_unicode(c['name'])}={_cookie_value_to_unicode(c['value'])}"
|
||||
if "name" in c and "value" in c
|
||||
else f"{next(iter(c.keys()))}={next(iter(c.values()))}"
|
||||
for c in request.cookies
|
||||
)
|
||||
cookies = f"--cookie '{cookie}'"
|
||||
|
||||
cookie_list: list[VerboseCookie] = _to_verbose_cookies(request.cookies)
|
||||
pairs = [
|
||||
f"{decoded['name']}={decoded['value']}"
|
||||
for c in cookie_list
|
||||
if (decoded := _decode_cookie(c, request)) is not None
|
||||
]
|
||||
cookies = f"--cookie '{'; '.join(pairs)}'" if pairs else ""
|
||||
|
||||
curl_cmd = f"curl -X {method} {url} {data} {headers} {cookies}".strip()
|
||||
return " ".join(curl_cmd.split())
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ def open_in_browser(
|
|||
|
||||
|
||||
def parse_details(self, response):
|
||||
if "item name" not in response.body:
|
||||
if "item name" not in response.text:
|
||||
open_in_browser(response)
|
||||
"""
|
||||
# circular imports
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ def send_catch_log(
|
|||
*arguments: TypingAny,
|
||||
**named: TypingAny,
|
||||
) -> list[tuple[TypingAny, TypingAny]]:
|
||||
"""Like ``pydispatcher.robust.sendRobust()`` but it also logs errors and returns
|
||||
"""Like ``pydispatch.robust.sendRobust()`` but it also logs errors and returns
|
||||
Failures instead of exceptions.
|
||||
"""
|
||||
dont_log = named.pop("dont_log", ())
|
||||
|
|
@ -172,9 +172,8 @@ async def _send_catch_log_asyncio(
|
|||
|
||||
Returns a coroutine that completes once all signal handlers have finished.
|
||||
|
||||
This function requires
|
||||
:class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be
|
||||
installed.
|
||||
This function requires an installed asyncio reactor or a running asyncio
|
||||
event loop.
|
||||
|
||||
.. versionadded:: 2.14
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -97,10 +97,11 @@ class Sitemap:
|
|||
|
||||
@staticmethod
|
||||
def _get_tag_name(elem: lxml.etree._Element) -> str:
|
||||
if TYPE_CHECKING:
|
||||
assert isinstance(elem.tag, str)
|
||||
_, _, localname = elem.tag.partition("}")
|
||||
return localname or elem.tag
|
||||
tag = elem.tag
|
||||
if not isinstance(tag, str):
|
||||
return ""
|
||||
_, _, localname = tag.partition("}")
|
||||
return localname or tag
|
||||
|
||||
|
||||
def sitemap_urls_from_robots(
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ def get_crawler(
|
|||
# When needed, useful settings can be added here, e.g. ones that prevent
|
||||
# deprecation warnings.
|
||||
settings: dict[str, Any] = {
|
||||
"TELNETCONSOLE_ENABLED": False,
|
||||
**get_reactor_settings(),
|
||||
**(settings_dict or {}),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,9 +4,7 @@ references to live object instances.
|
|||
If you want live objects for a particular class to be tracked, you only have to
|
||||
subclass from object_ref (instead of object).
|
||||
|
||||
About performance: This library has a minimal performance impact when enabled,
|
||||
and no performance penalty at all when disabled (as object_ref becomes just an
|
||||
alias to object in that case).
|
||||
This library has a minimal performance impact.
|
||||
|
||||
.. note:: PyPy uses a tracing garbage collector, so objects may
|
||||
remain in the ``live_refs`` longer than expected, even after they
|
||||
|
|
|
|||
|
|
@ -117,8 +117,8 @@ def strip_url(
|
|||
- ``strip_credentials`` removes "user:password@"
|
||||
- ``strip_default_port`` removes ":80" (resp. ":443", ":21")
|
||||
from http:// (resp. https://, ftp://) URLs
|
||||
- ``origin_only`` replaces path component with "/", also dropping
|
||||
query and fragment components ; it also strips credentials
|
||||
- ``origin_only`` replaces the path component with "/", also dropping
|
||||
the query component; it also strips credentials
|
||||
- ``strip_fragment`` drops any #fragment component
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from scrapy.utils.defer import deferred_from_coro
|
|||
|
||||
|
||||
class UppercasePipeline:
|
||||
async def _open_spider(self, spider):
|
||||
async def _open_spider(self, spider: Spider) -> None:
|
||||
spider.logger.info("async pipeline opened!")
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
import asyncio
|
||||
import sys
|
||||
|
||||
from twisted.internet import asyncioreactor
|
||||
|
||||
import scrapy
|
||||
from scrapy.crawler import AsyncCrawlerProcess
|
||||
|
||||
if sys.platform == "win32":
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
loop = asyncio.SelectorEventLoop()
|
||||
asyncio.set_event_loop(loop)
|
||||
asyncioreactor.install(loop)
|
||||
|
||||
|
||||
class NoRequestsSpider(scrapy.Spider):
|
||||
name = "no_request"
|
||||
|
||||
async def start(self):
|
||||
return
|
||||
yield
|
||||
|
||||
|
||||
process = AsyncCrawlerProcess(
|
||||
settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
"ASYNCIO_EVENT_LOOP": "asyncio.SelectorEventLoop",
|
||||
}
|
||||
)
|
||||
process.crawl(NoRequestsSpider)
|
||||
process.start()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue