mirror of https://github.com/scrapy/scrapy.git
Compare commits
No commits in common. "master" and "2.17.0" have entirely different histories.
|
|
@ -79,6 +79,9 @@ jobs:
|
|||
- python-version: "3.14"
|
||||
env:
|
||||
TOXENV: botocore
|
||||
- python-version: "3.14"
|
||||
env:
|
||||
TOXENV: mitmproxy
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
|
@ -94,9 +97,6 @@ jobs:
|
|||
sudo apt-get update
|
||||
sudo apt-get install libxml2-dev libxslt-dev
|
||||
|
||||
- name: Install mitmproxy
|
||||
run: pipx install mitmproxy
|
||||
|
||||
- name: Run tests
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ exclude: |
|
|||
)
|
||||
repos:
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.15.20
|
||||
rev: v0.15.2
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
args: [ --fix ]
|
||||
|
|
@ -16,7 +16,7 @@ repos:
|
|||
hooks:
|
||||
- id: blacken-docs
|
||||
additional_dependencies:
|
||||
- black==26.5.1
|
||||
- black==25.9.0
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v6.0.0
|
||||
hooks:
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from scrapy.utils.reactor import set_asyncio_event_loop_policy
|
|||
from scrapy.utils.reactorless import install_reactor_import_hook
|
||||
from tests.keys import generate_keys
|
||||
from tests.mockserver.http import MockServer
|
||||
from tests.mockserver.mitm_proxy import MitmProxy, mitmdump_cmd
|
||||
from tests.mockserver.mitm_proxy import MitmProxy
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
|
|
@ -127,6 +127,7 @@ def pytest_runtest_setup(item):
|
|||
"uvloop",
|
||||
"botocore",
|
||||
"boto3",
|
||||
"mitmproxy",
|
||||
]
|
||||
|
||||
for module in optional_deps:
|
||||
|
|
@ -136,9 +137,6 @@ def pytest_runtest_setup(item):
|
|||
except ImportError:
|
||||
pytest.skip(f"{module} is not installed")
|
||||
|
||||
if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None:
|
||||
pytest.skip("mitmdump is not available")
|
||||
|
||||
|
||||
# Generate localhost certificate files, needed by some tests
|
||||
generate_keys()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
:orphan:
|
||||
|
||||
======================================
|
||||
Scrapy documentation quick start guide
|
||||
======================================
|
||||
|
||||
This file provides a quick guide on how to compile the Scrapy documentation.
|
||||
|
||||
|
||||
Setup the environment
|
||||
---------------------
|
||||
|
||||
To compile the documentation you need Sphinx Python library. To install it
|
||||
and all its dependencies run the following command from this dir
|
||||
|
||||
::
|
||||
|
||||
pip install -r requirements.txt
|
||||
|
||||
|
||||
Compile the documentation
|
||||
-------------------------
|
||||
|
||||
To compile the documentation (to classic HTML output) run the following command
|
||||
from this dir::
|
||||
|
||||
make html
|
||||
|
||||
Documentation will be generated (in HTML format) inside the ``build/html`` dir.
|
||||
|
||||
|
||||
View the documentation
|
||||
----------------------
|
||||
|
||||
To view the documentation run the following command::
|
||||
|
||||
make htmlview
|
||||
|
||||
This command will fire up your default browser and open the main page of your
|
||||
(previously generated) HTML documentation.
|
||||
|
||||
|
||||
Start over
|
||||
----------
|
||||
|
||||
To clean up all generated documentation files and start from scratch run::
|
||||
|
||||
make clean
|
||||
|
||||
Keep in mind that this command won't touch any documentation source files.
|
||||
|
||||
|
||||
Recreating documentation on the fly
|
||||
-----------------------------------
|
||||
|
||||
There is a way to recreate the doc automatically when you make changes, you
|
||||
need to install watchdog (``pip install watchdog``) and then use::
|
||||
|
||||
make watch
|
||||
|
||||
Alternative method using tox
|
||||
----------------------------
|
||||
|
||||
To compile the documentation to HTML run the following command::
|
||||
|
||||
tox -e docs
|
||||
|
||||
Documentation will be generated inside the ``docs/_build/all`` dir.
|
||||
|
|
@ -323,10 +323,9 @@ deprecation removals are documented in the :ref:`release notes <news>`.
|
|||
Tests
|
||||
=====
|
||||
|
||||
Tests are implemented using pytest_. Running tests requires :doc:`tox
|
||||
<tox:index>`.
|
||||
|
||||
.. _pytest: https://pytest.org
|
||||
Tests are implemented using the :doc:`Twisted unit-testing framework
|
||||
<twisted:development/test-standard>`. Running tests requires
|
||||
:doc:`tox <tox:index>`.
|
||||
|
||||
.. _running-tests:
|
||||
|
||||
|
|
@ -372,21 +371,6 @@ To see coverage report install :doc:`coverage <coverage:index>`
|
|||
|
||||
see output of ``coverage --help`` for more options like html or xml report.
|
||||
|
||||
Some tests need a ``mitmdump`` executable (from mitmproxy_) to test against a
|
||||
fully featured proxy server; they are skipped when one cannot be found
|
||||
(``mitmproxy`` is intentionally not a test dependency that would be installed
|
||||
into test venvs, as that sometimes leads to various dependency conflicts).
|
||||
To run these tests, make ``mitmdump`` available in one of these ways:
|
||||
|
||||
* install ``mitmproxy`` so that ``mitmdump`` is on your ``PATH``, e.g. with
|
||||
pipx_ (``pipx install mitmproxy``) or uv_ (``uv tool install mitmproxy``);
|
||||
|
||||
* have uv_ installed, in which case the tests will run
|
||||
``uvx --from mitmproxy mitmdump``;
|
||||
|
||||
* set the ``MITMDUMP`` environment variable to the path of a ``mitmdump``
|
||||
executable.
|
||||
|
||||
Writing tests
|
||||
-------------
|
||||
|
||||
|
|
@ -414,6 +398,3 @@ And their unit-tests are in::
|
|||
.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
|
||||
.. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22
|
||||
.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy
|
||||
.. _mitmproxy: https://mitmproxy.org/
|
||||
.. _pipx: https://pipx.pypa.io/
|
||||
.. _uv: https://docs.astral.sh/uv/
|
||||
|
|
|
|||
14
docs/faq.rst
14
docs/faq.rst
|
|
@ -285,8 +285,7 @@ consume a lot of memory.
|
|||
In order to avoid parsing all the entire feed at once in memory, you can use
|
||||
the :func:`~scrapy.utils.iterators.xmliter_lxml` and
|
||||
:func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what
|
||||
:class:`~scrapy.spiders.XMLFeedSpider` and
|
||||
:class:`~scrapy.spiders.CSVFeedSpider` use.
|
||||
:class:`~scrapy.spiders.XMLFeedSpider` uses.
|
||||
|
||||
.. autofunction:: scrapy.utils.iterators.xmliter_lxml
|
||||
|
||||
|
|
@ -361,11 +360,10 @@ method for this purpose. For example:
|
|||
def process_spider_output(self, response, result):
|
||||
for item_or_request in result:
|
||||
if isinstance(item_or_request, Request):
|
||||
yield item_or_request
|
||||
continue
|
||||
adapter = ItemAdapter(item_or_request)
|
||||
adapter = ItemAdapter(item)
|
||||
for _ in range(adapter["multiply_by"]):
|
||||
yield deepcopy(item_or_request)
|
||||
yield deepcopy(item)
|
||||
|
||||
Does Scrapy support IPv6 addresses?
|
||||
-----------------------------------
|
||||
|
|
@ -384,9 +382,8 @@ How to deal with ``<class 'ValueError'>: filedescriptor out of range in select()
|
|||
----------------------------------------------------------------------------------------------
|
||||
|
||||
This issue `has been reported`_ to appear when running broad crawls in macOS, where the default
|
||||
Twisted reactor was :class:`twisted.internet.selectreactor.SelectReactor` at that time.
|
||||
If you have switched to this reactor using the :setting:`TWISTED_REACTOR` setting you can switch
|
||||
to a different one in the same way.
|
||||
Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a
|
||||
different reactor is possible by using the :setting:`TWISTED_REACTOR` setting.
|
||||
|
||||
|
||||
.. _faq-stop-response-download:
|
||||
|
|
@ -412,6 +409,7 @@ How can I make a blank request?
|
|||
|
||||
from scrapy import Request
|
||||
|
||||
|
||||
blank_request = Request("data:,")
|
||||
|
||||
In this case, the URL is set to a data URI scheme. Data URLs allow you to include data
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ Having trouble? We'd like to help!
|
|||
* Ask or search questions in `StackOverflow using the scrapy tag`_.
|
||||
* Ask or search questions in the `Scrapy subreddit`_.
|
||||
* Search for questions on the archives of the `scrapy-users mailing list`_.
|
||||
* Ask a question in the `#scrapy IRC channel`_.
|
||||
* Ask a question in the `#scrapy IRC channel`_,
|
||||
* Report bugs with Scrapy in our `issue tracker`_.
|
||||
* Join the Discord community `Scrapy Discord`_.
|
||||
|
||||
|
|
@ -91,15 +91,15 @@ Basic concepts
|
|||
:doc:`topics/selectors`
|
||||
Extract the data from web pages using XPath.
|
||||
|
||||
:doc:`topics/shell`
|
||||
Test your extraction code in an interactive environment.
|
||||
|
||||
:doc:`topics/items`
|
||||
Define the data you want to scrape.
|
||||
|
||||
:doc:`topics/loaders`
|
||||
Populate your items with the extracted data.
|
||||
|
||||
:doc:`topics/shell`
|
||||
Test your extraction code in an interactive environment.
|
||||
|
||||
:doc:`topics/item-pipeline`
|
||||
Post-process and store your scraped data.
|
||||
|
||||
|
|
|
|||
|
|
@ -230,8 +230,8 @@ Installing Scrapy with PyPy on Windows is not tested.
|
|||
You can check that Scrapy is installed correctly by running ``scrapy bench``.
|
||||
If this command gives errors such as
|
||||
``TypeError: ... got 2 unexpected keyword arguments``, this means
|
||||
that the ``PyPyDispatcher`` dependency wasn't installed. To fix this issue, run
|
||||
``pip install 'PyPyDispatcher>=2.1.0'``.
|
||||
that setuptools was unable to pick up one PyPy-specific dependency.
|
||||
To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``.
|
||||
|
||||
|
||||
.. _intro-install-troubleshooting:
|
||||
|
|
|
|||
|
|
@ -83,17 +83,16 @@ While this enables you to do very fast crawls (sending multiple concurrent
|
|||
requests at the same time, in a fault-tolerant way) Scrapy also gives you
|
||||
control over the politeness of the crawl through :ref:`a few settings
|
||||
<topics-settings-ref>`. You can do things like setting a download delay between
|
||||
each request, limiting the amount of concurrent requests per domain, and
|
||||
each request, limiting the amount of concurrent requests per domain or per IP, and
|
||||
even :ref:`using an auto-throttling extension <topics-autothrottle>` that tries
|
||||
to figure these settings out automatically.
|
||||
|
||||
.. note::
|
||||
|
||||
This is using :ref:`feed exports <topics-feed-exports>` to generate the
|
||||
JSON Lines file, you can easily change the export format (XML or CSV, for
|
||||
example) or the storage backend (FTP or `Amazon S3`_, for example). You can
|
||||
also write an :ref:`item pipeline <topics-item-pipeline>` to store the
|
||||
items in a database.
|
||||
JSON file, you can easily change the export format (XML or CSV, for example) or the
|
||||
storage backend (FTP or `Amazon S3`_, for example). You can also write an
|
||||
:ref:`item pipeline <topics-item-pipeline>` to store the items in a database.
|
||||
|
||||
|
||||
.. _topics-whatelse:
|
||||
|
|
|
|||
|
|
@ -3,28 +3,6 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
Scrapy VERSION (unreleased)
|
||||
---------------------------
|
||||
|
||||
Backward-incompatible changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- The following runtime usage of zope.interface_ interfaces is removed:
|
||||
|
||||
- :class:`~scrapy.spiderloader.SpiderLoader` and
|
||||
:class:`~scrapy.spiderloader.DummySpiderLoader` are no longer marked
|
||||
as implementing the ``ISpiderLoader`` interface.
|
||||
|
||||
- :func:`~scrapy.spiderloader.get_spider_loader` no longer checks that the
|
||||
configured spider loader implements the ``ISpiderLoader`` interface.
|
||||
|
||||
- :class:`~scrapy.extensions.feedexport.BlockingFeedStorage`,
|
||||
:class:`~scrapy.extensions.feedexport.FileFeedStorage` and
|
||||
:class:`~scrapy.extensions.feedexport.StdoutFeedStorage` are no longer
|
||||
marked as implementing the ``IFeedStorage`` interface.
|
||||
|
||||
(:issue:`6585`, :issue:`7731`)
|
||||
|
||||
.. _release-2.17.0:
|
||||
|
||||
Scrapy 2.17.0 (2026-07-07)
|
||||
|
|
@ -1057,7 +1035,7 @@ Deprecations
|
|||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- Added a new setting, :setting:`REFERRER_POLICIES`, to allow customizing
|
||||
- Added a new setting, :setting:`REFERER_POLICIES`, to allow customizing
|
||||
supported referrer policies.
|
||||
|
||||
Bug fixes
|
||||
|
|
@ -2665,7 +2643,7 @@ Deprecation removals
|
|||
(:issue:`6109`, :issue:`6116`)
|
||||
|
||||
- A custom class assigned to the :setting:`SPIDER_LOADER_CLASS` setting that
|
||||
does not implement the ``ISpiderLoader`` interface
|
||||
does not implement the :class:`~scrapy.interfaces.ISpiderLoader` interface
|
||||
will now raise a :exc:`zope.interface.verify.DoesNotImplement` exception at
|
||||
run time. Non-compliant classes have been triggering a deprecation warning
|
||||
since Scrapy 1.0.0.
|
||||
|
|
@ -6068,7 +6046,7 @@ The following changes may impact custom priority queue classes:
|
|||
|
||||
* A new keyword parameter has been added: ``key``. It is a string
|
||||
that is always an empty string for memory queues and indicates the
|
||||
:setting:`JOBDIR` value for disk queues.
|
||||
:setting:`JOB_DIR` value for disk queues.
|
||||
|
||||
* The parameter for disk queues that contains data from the previous
|
||||
crawl, ``startprios`` or ``slot_startprios``, is now passed as a
|
||||
|
|
@ -6622,7 +6600,7 @@ New features
|
|||
``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
|
||||
:ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
|
||||
scheduling improvement on crawls targeting multiple web domains, at the
|
||||
cost of no ``CONCURRENT_REQUESTS_PER_IP`` support (:issue:`3520`)
|
||||
cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
|
||||
|
||||
* A new :attr:`.Request.cb_kwargs` attribute
|
||||
provides a cleaner way to pass keyword arguments to callback methods
|
||||
|
|
@ -9101,7 +9079,7 @@ New features and settings
|
|||
- In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`)
|
||||
- Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`)
|
||||
- ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by:
|
||||
- :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, ``CONCURRENT_REQUESTS_PER_IP``
|
||||
- :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP`
|
||||
- check the documentation for more details
|
||||
- Added builtin caching DNS resolver (:rev:`2728`)
|
||||
- Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`)
|
||||
|
|
|
|||
|
|
@ -21,14 +21,10 @@ The ``ADDONS`` setting is a dict in which every key is an add-on class or its
|
|||
import path and the value is its priority.
|
||||
|
||||
This is an example where two add-ons are enabled in a project's
|
||||
``settings.py``:
|
||||
|
||||
.. skip: next
|
||||
|
||||
.. code-block:: python
|
||||
``settings.py``::
|
||||
|
||||
ADDONS = {
|
||||
"path.to.someaddon": 0,
|
||||
'path.to.someaddon': 0,
|
||||
SomeAddonClass: 1,
|
||||
}
|
||||
|
||||
|
|
@ -60,9 +56,7 @@ the following methods:
|
|||
:type settings: :class:`~scrapy.settings.BaseSettings`
|
||||
|
||||
The settings set by the add-on should use the ``addon`` priority (see
|
||||
:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):
|
||||
|
||||
.. code-block:: python
|
||||
:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`)::
|
||||
|
||||
class MyAddon:
|
||||
def update_settings(self, settings):
|
||||
|
|
@ -174,6 +168,7 @@ Use a fallback component:
|
|||
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
|
||||
|
||||
FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -172,15 +172,46 @@ SpiderLoader API
|
|||
.. module:: scrapy.spiderloader
|
||||
:synopsis: The spider loader
|
||||
|
||||
Custom spider loaders can be employed by specifying their path in the
|
||||
:setting:`SPIDER_LOADER_CLASS` project setting. They must implement
|
||||
:class:`SpiderLoaderProtocol`.
|
||||
.. class:: SpiderLoader
|
||||
|
||||
.. autoclass:: SpiderLoaderProtocol
|
||||
:members:
|
||||
This class is in charge of retrieving and handling the spider classes
|
||||
defined across the project.
|
||||
|
||||
.. autoclass:: SpiderLoader
|
||||
:members:
|
||||
Custom spider loaders can be employed by specifying their path in the
|
||||
:setting:`SPIDER_LOADER_CLASS` project setting. They must fully implement
|
||||
the :class:`scrapy.interfaces.ISpiderLoader` interface to guarantee an
|
||||
errorless execution.
|
||||
|
||||
.. method:: from_settings(settings)
|
||||
|
||||
This class method is used by Scrapy to create an instance of the class.
|
||||
It's called with the current project settings, and it loads the spiders
|
||||
found recursively in the modules of the :setting:`SPIDER_MODULES`
|
||||
setting.
|
||||
|
||||
:param settings: project settings
|
||||
:type settings: :class:`~scrapy.settings.Settings` instance
|
||||
|
||||
.. method:: load(spider_name)
|
||||
|
||||
Get the Spider class with the given name. It'll look into the previously
|
||||
loaded spiders for a spider class with name ``spider_name`` and will raise
|
||||
a KeyError if not found.
|
||||
|
||||
:param spider_name: spider class name
|
||||
:type spider_name: str
|
||||
|
||||
.. method:: list()
|
||||
|
||||
Get the names of the available spiders in the project.
|
||||
|
||||
.. method:: find_by_request(request)
|
||||
|
||||
List the spiders' names that can handle the given request. Will try to
|
||||
match the request's url against the domains of the spiders.
|
||||
|
||||
:param request: queried request
|
||||
:type request: :class:`~scrapy.Request` instance
|
||||
|
||||
.. autoclass:: DummySpiderLoader
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ asyncio
|
|||
=======
|
||||
|
||||
Scrapy supports :mod:`asyncio` natively. New projects created with
|
||||
:command:`startproject` have asyncio enabled by default, and you can use
|
||||
:command:`scrapy startproject` have asyncio enabled by default, and you can use
|
||||
:mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine
|
||||
<coroutines>`.
|
||||
|
||||
|
|
@ -18,7 +18,7 @@ no additional setup is needed.
|
|||
Configuring the asyncio reactor
|
||||
===============================
|
||||
|
||||
New projects generated with :command:`startproject` have the asyncio
|
||||
New projects generated with :command:`scrapy startproject` have the asyncio
|
||||
reactor configured by default. No manual setup is needed.
|
||||
|
||||
The :setting:`TWISTED_REACTOR` setting controls which Twisted reactor Scrapy
|
||||
|
|
@ -105,9 +105,6 @@ Scrapy API requires passing a Deferred to it) using the following helpers:
|
|||
|
||||
.. autofunction:: scrapy.utils.defer.deferred_from_coro
|
||||
.. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f
|
||||
|
||||
The following function helps with a reverse wrapping:
|
||||
|
||||
.. autofunction:: scrapy.utils.defer.ensure_awaitable
|
||||
|
||||
|
||||
|
|
@ -150,12 +147,6 @@ Using Scrapy without a Twisted reactor
|
|||
.. warning::
|
||||
This is currently experimental and may not be suitable for production use.
|
||||
|
||||
.. note:: As the Twisted download handlers cannot be used without a reactor,
|
||||
the default download handler in this mode is
|
||||
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. You
|
||||
will need to additionally install the ``httpx`` library to use it, unless
|
||||
you switch to some different handler.
|
||||
|
||||
It's possible to use Scrapy without installing a Twisted reactor at all, by
|
||||
setting the :setting:`TWISTED_REACTOR_ENABLED` setting to ``False``. In this
|
||||
mode Scrapy will use the asyncio event loop directly, and most of the Scrapy
|
||||
|
|
@ -199,7 +190,7 @@ in future Scrapy versions. The following features are not available:
|
|||
:class:`~scrapy.crawler.CrawlerProcess`
|
||||
(:class:`~scrapy.crawler.AsyncCrawlerProcess` and
|
||||
:class:`~scrapy.crawler.AsyncCrawlerRunner` are available)
|
||||
* Twisted-specific DNS resolvers (the :setting:`TWISTED_DNS_RESOLVER` setting)
|
||||
* Twisted-specific DNS resolvers (the :setting:`DNS_RESOLVER` setting)
|
||||
* User and 3rd-party code that requires a reactor (see :ref:`below
|
||||
<asyncio-without-reactor-migrate>` for examples)
|
||||
|
||||
|
|
@ -227,8 +218,7 @@ for its differences and limitations compared to
|
|||
|
||||
Additionally, :class:`~scrapy.crawler.AsyncCrawlerProcess` will install a
|
||||
:term:`meta path finder` that prevents :mod:`twisted.internet.reactor` from
|
||||
being imported. It will be uninstalled when :meth:`AsyncCrawlerProcess.start()
|
||||
<scrapy.crawler.AsyncCrawlerProcess.start>` exits.
|
||||
being imported.
|
||||
|
||||
.. _asyncio-without-reactor-migrate:
|
||||
|
||||
|
|
@ -325,7 +315,8 @@ implementations, :class:`~asyncio.ProactorEventLoop` (default) and
|
|||
:class:`~asyncio.SelectorEventLoop` works with Twisted.
|
||||
|
||||
Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop`
|
||||
automatically when installing the asyncio reactor.
|
||||
automatically when you change the :setting:`TWISTED_REACTOR` setting or call
|
||||
:func:`~scrapy.utils.reactor.install_reactor`.
|
||||
|
||||
.. note:: Other libraries you use may require
|
||||
:class:`~asyncio.ProactorEventLoop`, e.g. because it supports
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ AutoThrottle algorithm adjusts download delays based on the following rules:
|
|||
.. _download-latency:
|
||||
|
||||
In Scrapy, the download latency is measured as the time elapsed between
|
||||
sending the request and receiving the HTTP headers.
|
||||
establishing the TCP connection and receiving the HTTP headers.
|
||||
|
||||
Note that these latencies are very hard to measure accurately in a cooperative
|
||||
multitasking environment because Scrapy may be busy processing a spider
|
||||
|
|
|
|||
|
|
@ -199,7 +199,6 @@ Global commands:
|
|||
* :command:`fetch`
|
||||
* :command:`view`
|
||||
* :command:`version`
|
||||
* :command:`bench`
|
||||
|
||||
Project-only commands:
|
||||
|
||||
|
|
@ -208,6 +207,7 @@ Project-only commands:
|
|||
* :command:`list`
|
||||
* :command:`edit`
|
||||
* :command:`parse`
|
||||
* :command:`bench`
|
||||
|
||||
.. command:: startproject
|
||||
|
||||
|
|
@ -309,25 +309,11 @@ Usage examples::
|
|||
* parse_item
|
||||
|
||||
$ scrapy check
|
||||
F.F.
|
||||
======================================================================
|
||||
FAIL: [first_spider] parse (@returns post-hook)
|
||||
----------------------------------------------------------------------
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
scrapy.exceptions.ContractFail: Returned 92 requests, expected 0..4
|
||||
[FAILED] first_spider:parse_item
|
||||
>>> 'RetailPricex' field is missing
|
||||
|
||||
======================================================================
|
||||
FAIL: [first_spider] parse_item (@scrapes post-hook)
|
||||
----------------------------------------------------------------------
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
scrapy.exceptions.ContractFail: Missing fields: RetailPricex
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 4 contracts in 0.174s
|
||||
|
||||
FAILED (failures=2)
|
||||
[FAILED] first_spider:parse
|
||||
>>> Returned 92 requests, expected 0..4
|
||||
|
||||
.. skip: end
|
||||
|
||||
|
|
@ -391,7 +377,7 @@ Supported options:
|
|||
|
||||
* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
|
||||
|
||||
* ``--headers``: print the request's and response's HTTP headers instead of the response's body
|
||||
* ``--headers``: print the response's HTTP headers instead of the response's body
|
||||
|
||||
* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them)
|
||||
|
||||
|
|
@ -401,19 +387,15 @@ Usage examples::
|
|||
[ ... html content here ... ]
|
||||
|
||||
$ scrapy fetch --nolog --headers http://www.example.com/
|
||||
> Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
|
||||
> Accept-Language: en
|
||||
> User-Agent: Scrapy/2.16.0 (+https://scrapy.org)
|
||||
> Accept-Encoding: gzip, deflate, br
|
||||
>
|
||||
< Date: Wed, 08 Jul 2026 06:15:01 GMT
|
||||
< Content-Type: text/html
|
||||
< Server: cloudflare
|
||||
< Last-Modified: Wed, 01 Jul 2026 17:50:18 GMT
|
||||
< Allow: GET, HEAD
|
||||
< Cf-Cache-Status: HIT
|
||||
< Age: 8184
|
||||
< Cf-Ray: a17cf3b80eddf141-DME
|
||||
{'Accept-Ranges': ['bytes'],
|
||||
'Age': ['1263 '],
|
||||
'Connection': ['close '],
|
||||
'Content-Length': ['596'],
|
||||
'Content-Type': ['text/html; charset=UTF-8'],
|
||||
'Date': ['Wed, 18 Aug 2010 23:59:46 GMT'],
|
||||
'Etag': ['"573c1-254-48c9c87349680"'],
|
||||
'Last-Modified': ['Fri, 30 Jul 2010 15:30:18 GMT'],
|
||||
'Server': ['Apache/2.2.3 (CentOS)']}
|
||||
|
||||
.. command:: view
|
||||
|
||||
|
|
@ -494,7 +476,7 @@ Supported options:
|
|||
|
||||
* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
|
||||
|
||||
* ``-a NAME=VALUE``: set spider argument (may be repeated)
|
||||
* ``--a NAME=VALUE``: set spider argument (may be repeated)
|
||||
|
||||
* ``--callback`` or ``-c``: spider method to use as callback for parsing the
|
||||
response
|
||||
|
|
@ -623,10 +605,7 @@ shouldn't matter to the user running the command, but when the user :ref:`needs
|
|||
a non-default Twisted reactor <disable-asyncio>`, it may be important.
|
||||
|
||||
Scrapy decides which of these two classes to use based on the value of the
|
||||
:setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED` settings.
|
||||
With :setting:`TWISTED_REACTOR_ENABLED` set to ``False`` it will use
|
||||
:class:`~scrapy.crawler.AsyncCrawlerProcess`. Otherwise, if the
|
||||
:setting:`TWISTED_REACTOR` value is the default one
|
||||
:setting:`TWISTED_REACTOR` setting. If the setting value is the default one
|
||||
(``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``),
|
||||
:class:`~scrapy.crawler.AsyncCrawlerProcess` will be used, otherwise
|
||||
:class:`~scrapy.crawler.CrawlerProcess` will be used. The :ref:`spider settings
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ Supported callables
|
|||
The following callables may be defined as coroutines using ``async def``, and
|
||||
hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
|
||||
|
||||
- The :meth:`~scrapy.Spider.start` spider method, which *must* be
|
||||
- The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be
|
||||
defined as an :term:`asynchronous generator`.
|
||||
|
||||
.. versionadded:: 2.13
|
||||
|
|
@ -204,15 +204,13 @@ This means you can use many useful Python libraries providing such code:
|
|||
Common use cases for asynchronous code include:
|
||||
|
||||
* requesting data from websites, databases and other services (in
|
||||
:meth:`~scrapy.Spider.start`, callbacks, pipelines and
|
||||
:meth:`~scrapy.spiders.Spider.start`, callbacks, pipelines and
|
||||
middlewares);
|
||||
* storing data in databases (in pipelines and middlewares);
|
||||
* delaying the spider initialization until some external event (in the
|
||||
:signal:`spider_opened` handler);
|
||||
* calling asynchronous Scrapy methods like
|
||||
:meth:`ExecutionEngine.download_async()
|
||||
<scrapy.core.engine.ExecutionEngine.download_async>` (see :ref:`the
|
||||
screenshot pipeline example <ScreenshotPipeline>`).
|
||||
* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download`
|
||||
(see :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
|
||||
|
||||
.. _aio-libs: https://github.com/aio-libs
|
||||
|
||||
|
|
@ -270,5 +268,5 @@ You can also send multiple requests in parallel:
|
|||
yield {
|
||||
"h1": response.css("h1::text").get(),
|
||||
"price": responses[0].css(".price::text").get(),
|
||||
"color": responses[1].css(".color::text").get(),
|
||||
"price2": responses[1].css(".color::text").get(),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -246,6 +246,7 @@ also request each page to get every quote on the site:
|
|||
.. code-block:: python
|
||||
|
||||
import scrapy
|
||||
import json
|
||||
|
||||
|
||||
class QuoteSpider(scrapy.Spider):
|
||||
|
|
@ -255,7 +256,7 @@ also request each page to get every quote on the site:
|
|||
start_urls = ["https://quotes.toscrape.com/api/quotes?page=1"]
|
||||
|
||||
def parse(self, response):
|
||||
data = response.json()
|
||||
data = json.loads(response.text)
|
||||
for quote in data["quotes"]:
|
||||
yield {"quote": quote["text"]}
|
||||
if data["has_next"]:
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ the following API:
|
|||
If ``True``, the handler will only be instantiated when the first
|
||||
request handled by it needs to be downloaded.
|
||||
|
||||
.. method:: download_request(request: Request) -> Response
|
||||
.. method:: download_request(request: Request) -> Response:
|
||||
:async:
|
||||
|
||||
Download the given request and return a response.
|
||||
|
|
|
|||
|
|
@ -505,7 +505,7 @@ Filesystem storage backend (default)
|
|||
|
||||
* ``response_body`` - the plain response body
|
||||
|
||||
* ``response_headers`` - the response headers (in raw HTTP format)
|
||||
* ``response_headers`` - the request headers (in raw HTTP format)
|
||||
|
||||
* ``meta`` - some metadata of this cache resource in Python ``repr()``
|
||||
format (grep-friendly format)
|
||||
|
|
@ -547,7 +547,7 @@ defines the methods described below.
|
|||
.. method:: open_spider(spider)
|
||||
|
||||
This method gets called after a spider has been opened for crawling. It handles
|
||||
the :signal:`spider_opened` signal.
|
||||
the :signal:`open_spider <spider_opened>` signal.
|
||||
|
||||
:param spider: the spider which has been opened
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
|
@ -555,7 +555,7 @@ defines the methods described below.
|
|||
.. method:: close_spider(spider)
|
||||
|
||||
This method gets called after a spider has been closed. It handles
|
||||
the :signal:`spider_closed` signal.
|
||||
the :signal:`close_spider <spider_closed>` signal.
|
||||
|
||||
:param spider: the spider which has been closed
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
|
@ -591,8 +591,8 @@ In order to use your storage backend, set:
|
|||
HTTPCache middleware settings
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` can be
|
||||
configured through the following settings:
|
||||
The :class:`HttpCacheMiddleware` can be configured through the following
|
||||
settings:
|
||||
|
||||
.. setting:: HTTPCACHE_ENABLED
|
||||
|
||||
|
|
@ -814,6 +814,7 @@ HttpProxyMiddleware settings
|
|||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. setting:: HTTPPROXY_ENABLED
|
||||
.. setting:: HTTPPROXY_AUTH_ENCODING
|
||||
|
||||
HTTPPROXY_ENABLED
|
||||
^^^^^^^^^^^^^^^^^
|
||||
|
|
@ -822,8 +823,6 @@ Default: ``True``
|
|||
|
||||
Whether or not to enable the :class:`HttpProxyMiddleware`.
|
||||
|
||||
.. setting:: HTTPPROXY_AUTH_ENCODING
|
||||
|
||||
HTTPPROXY_AUTH_ENCODING
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
|
|
@ -868,9 +867,9 @@ OffsiteMiddleware
|
|||
.. reqmeta:: allow_offsite
|
||||
|
||||
If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to
|
||||
``True`` or :attr:`Request.meta <scrapy.Request.meta>` has ``allow_offsite``
|
||||
set to ``True``, then the OffsiteMiddleware will allow the request even if
|
||||
its domain is not listed in allowed domains.
|
||||
``True`` or :attr:`Request.meta` has ``allow_offsite`` set to ``True``, then
|
||||
the OffsiteMiddleware will allow the request even if its domain is not listed
|
||||
in allowed domains.
|
||||
|
||||
RedirectMiddleware
|
||||
------------------
|
||||
|
|
@ -985,7 +984,7 @@ Whether the Meta Refresh middleware will be enabled.
|
|||
METAREFRESH_IGNORE_TAGS
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Default: ``["noscript"]``
|
||||
Default: ``[]``
|
||||
|
||||
Meta tags within these tags are ignored.
|
||||
|
||||
|
|
@ -1015,6 +1014,17 @@ RetryMiddleware
|
|||
A middleware to retry failed requests that are potentially caused by
|
||||
temporary problems such as a connection timeout or HTTP 500 error.
|
||||
|
||||
Failed pages are collected on the scraping process and rescheduled at the
|
||||
end, once the spider has finished crawling all regular (non failed) pages.
|
||||
|
||||
The :class:`RetryMiddleware` can be configured through the following
|
||||
settings (see the settings documentation for more info):
|
||||
|
||||
* :setting:`RETRY_ENABLED`
|
||||
* :setting:`RETRY_TIMES`
|
||||
* :setting:`RETRY_HTTP_CODES`
|
||||
* :setting:`RETRY_EXCEPTIONS`
|
||||
|
||||
.. reqmeta:: dont_retry
|
||||
|
||||
If :attr:`Request.meta <scrapy.Request.meta>` has ``dont_retry`` key
|
||||
|
|
@ -1081,7 +1091,7 @@ Default::
|
|||
'twisted.internet.error.ConnectionDone',
|
||||
'twisted.internet.error.ConnectError',
|
||||
'twisted.internet.error.ConnectionLost',
|
||||
OSError,
|
||||
IOError,
|
||||
'scrapy.core.downloader.handlers.http11.TunnelError',
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -133,7 +133,7 @@ data from it depends on the type of response:
|
|||
|
||||
.. code-block:: python
|
||||
|
||||
selector = Selector(text=data["html"])
|
||||
selector = Selector(data["html"])
|
||||
|
||||
- If the response is JavaScript, or HTML with a ``<script/>`` element
|
||||
containing the desired data, see :ref:`topics-parsing-javascript`.
|
||||
|
|
|
|||
|
|
@ -12,8 +12,7 @@ Exceptions
|
|||
Built-in Exceptions reference
|
||||
=============================
|
||||
|
||||
Here's a list of all exceptions included in Scrapy and their usage, except for
|
||||
the :ref:`download handler exceptions <download-handlers-exceptions>`.
|
||||
Here's a list of all exceptions included in Scrapy and their usage.
|
||||
|
||||
|
||||
CloseSpider
|
||||
|
|
@ -32,7 +31,7 @@ For example:
|
|||
.. code-block:: python
|
||||
|
||||
def parse_page(self, response):
|
||||
if "Bandwidth exceeded" in response.text:
|
||||
if "Bandwidth exceeded" in response.body:
|
||||
raise CloseSpider("bandwidth_exceeded")
|
||||
|
||||
DontCloseSpider
|
||||
|
|
@ -72,8 +71,7 @@ remain disabled. Those components include:
|
|||
- Downloader middlewares
|
||||
- Spider middlewares
|
||||
|
||||
The exception must be raised in the component's ``__init__()`` or
|
||||
``from_crawler()`` method.
|
||||
The exception must be raised in the component's ``__init__`` method.
|
||||
|
||||
NotSupported
|
||||
------------
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ output examples, which assume you're exporting these two items:
|
|||
BaseItemExporter
|
||||
----------------
|
||||
|
||||
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding=None, indent=None, dont_fail=False)
|
||||
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0, dont_fail=False)
|
||||
|
||||
This is the (abstract) base class for all Item Exporters. It provides
|
||||
support for common features used by all (concrete) Item Exporters, such as
|
||||
|
|
@ -211,17 +211,13 @@ BaseItemExporter
|
|||
|
||||
- ``None`` (all fields [2]_, default)
|
||||
|
||||
- A list of fields:
|
||||
- A list of fields::
|
||||
|
||||
.. code-block:: python
|
||||
['field1', 'field2']
|
||||
|
||||
["field1", "field2"]
|
||||
- A dict where keys are fields and values are output names::
|
||||
|
||||
- A dict where keys are fields and values are output names:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
{"field1": "Field 1", "field2": "Field 2"}
|
||||
{'field1': 'Field 1', 'field2': 'Field 2'}
|
||||
|
||||
.. [1] Not all exporters respect the specified field order.
|
||||
.. [2] When using :ref:`item objects <item-types>` that do not expose
|
||||
|
|
@ -243,7 +239,7 @@ BaseItemExporter
|
|||
|
||||
.. attribute:: indent
|
||||
|
||||
Amount of spaces used to indent the output on each level. Defaults to ``None``.
|
||||
Amount of spaces used to indent the output on each level. Defaults to ``0``.
|
||||
|
||||
* ``indent=None`` selects the most compact representation,
|
||||
all items in the same line with no indentation
|
||||
|
|
@ -277,9 +273,7 @@ XmlItemExporter
|
|||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method.
|
||||
|
||||
A typical output of this exporter would be:
|
||||
|
||||
.. code-block:: xml
|
||||
A typical output of this exporter would be::
|
||||
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<items>
|
||||
|
|
@ -297,17 +291,11 @@ XmlItemExporter
|
|||
exported by serializing each value inside a ``<value>`` element. This is for
|
||||
convenience, as multi-valued fields are very common.
|
||||
|
||||
For example, the item:
|
||||
For example, the item::
|
||||
|
||||
.. skip: next
|
||||
Item(name=['John', 'Doe'], age='23')
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
Item(name=["John", "Doe"], age="23")
|
||||
|
||||
Would be serialized as:
|
||||
|
||||
.. code-block:: xml
|
||||
Would be serialized as::
|
||||
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<items>
|
||||
|
|
@ -340,7 +328,7 @@ CsvItemExporter
|
|||
|
||||
:param join_multivalued: The char (or chars) that will be used for joining
|
||||
multi-valued fields, if found.
|
||||
:type join_multivalued: str
|
||||
:type include_headers_line: str
|
||||
|
||||
:param errors: The optional string that specifies how encoding and decoding
|
||||
errors are to be handled. For more information see
|
||||
|
|
@ -354,14 +342,14 @@ CsvItemExporter
|
|||
|
||||
A typical output of this exporter would be::
|
||||
|
||||
name,price
|
||||
product,price
|
||||
Color TV,1200
|
||||
DVD player,200
|
||||
|
||||
PickleItemExporter
|
||||
------------------
|
||||
|
||||
.. class:: PickleItemExporter(file, protocol=4, **kwargs)
|
||||
.. class:: PickleItemExporter(file, protocol=0, **kwargs)
|
||||
|
||||
Exports items in pickle format to the given file-like object.
|
||||
|
||||
|
|
@ -391,12 +379,10 @@ PprintItemExporter
|
|||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method.
|
||||
|
||||
A typical output of this exporter would be:
|
||||
A typical output of this exporter would be::
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
{"name": "Color TV", "price": "1200"}
|
||||
{"name": "DVD player", "price": "200"}
|
||||
{'name': 'Color TV', 'price': '1200'}
|
||||
{'name': 'DVD player', 'price': '200'}
|
||||
|
||||
Longer lines (when present) are pretty-formatted.
|
||||
|
||||
|
|
@ -414,9 +400,7 @@ JsonItemExporter
|
|||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
||||
A typical output of this exporter would be:
|
||||
|
||||
.. code-block:: json
|
||||
A typical output of this exporter would be::
|
||||
|
||||
[{"name": "Color TV", "price": "1200"},
|
||||
{"name": "DVD player", "price": "200"}]
|
||||
|
|
@ -445,9 +429,7 @@ JsonLinesItemExporter
|
|||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
||||
A typical output of this exporter would be:
|
||||
|
||||
.. code-block:: json
|
||||
A typical output of this exporter would be::
|
||||
|
||||
{"name": "Color TV", "price": "1200"}
|
||||
{"name": "DVD player", "price": "200"}
|
||||
|
|
|
|||
|
|
@ -341,6 +341,9 @@ closing the spider. If the spider generates more than that number of errors,
|
|||
it will be closed with the reason ``closespider_errorcount``. If zero (or non
|
||||
set), spiders won't be closed by number of errors.
|
||||
|
||||
.. module:: scrapy.extensions.debug
|
||||
:synopsis: Extensions for debugging Scrapy
|
||||
|
||||
.. module:: scrapy.extensions.periodic_log
|
||||
:synopsis: Periodic stats logging
|
||||
|
||||
|
|
@ -415,7 +418,7 @@ Example extension configuration:
|
|||
custom_settings = {
|
||||
"LOG_LEVEL": "INFO",
|
||||
"PERIODIC_LOG_STATS": {
|
||||
"include": ["downloader/", "scheduler/", "log_count/", "item_scraped_count"],
|
||||
"include": ["downloader/", "scheduler/", "log_count/", "item_scraped_count/"],
|
||||
},
|
||||
"PERIODIC_LOG_DELTA": {"include": ["downloader/"]},
|
||||
"PERIODIC_LOG_TIMING_ENABLED": True,
|
||||
|
|
@ -460,9 +463,6 @@ Default: ``False``
|
|||
Debugging extensions
|
||||
--------------------
|
||||
|
||||
.. module:: scrapy.extensions.debug
|
||||
:synopsis: Extensions for debugging Scrapy
|
||||
|
||||
Stack trace dump extension
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
|
|||
|
|
@ -143,11 +143,6 @@ Here are some examples to illustrate:
|
|||
.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence
|
||||
they can also be used as storage URI parameters.
|
||||
|
||||
.. note:: Only ``%(...)s`` parameters are replaced. Any other percent
|
||||
character is kept as-is, so percent-encoded URIs (e.g. ``%20`` for a
|
||||
space or percent-encoded FTP credentials) and :class:`pathlib.Path`
|
||||
keys containing ``%(...)s`` parameters both work as expected.
|
||||
|
||||
|
||||
.. _topics-feed-storage-backends:
|
||||
|
||||
|
|
@ -166,7 +161,7 @@ The feeds are stored in the local filesystem.
|
|||
- Required external libraries: none
|
||||
|
||||
Note that for the local filesystem storage (only) you can omit the scheme if
|
||||
you specify a path (e.g. ``/tmp/export.csv``).
|
||||
you specify an absolute path like ``/tmp/export.csv`` (Unix systems only).
|
||||
Alternatively you can also use a :class:`pathlib.Path` object.
|
||||
|
||||
.. _topics-feed-storage-ftp:
|
||||
|
|
@ -434,37 +429,33 @@ This setting is required for enabling the feed export feature.
|
|||
|
||||
See :ref:`topics-feed-storage-backends` for supported URI schemes.
|
||||
|
||||
For instance:
|
||||
|
||||
.. skip: next
|
||||
|
||||
.. code-block:: python
|
||||
For instance::
|
||||
|
||||
{
|
||||
"items.json": {
|
||||
"format": "json",
|
||||
"encoding": "utf8",
|
||||
"store_empty": False,
|
||||
"item_classes": [MyItemClass1, "myproject.items.MyItemClass2"],
|
||||
"fields": None,
|
||||
"indent": 4,
|
||||
"item_export_kwargs": {
|
||||
"export_empty_fields": True,
|
||||
'items.json': {
|
||||
'format': 'json',
|
||||
'encoding': 'utf8',
|
||||
'store_empty': False,
|
||||
'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'],
|
||||
'fields': None,
|
||||
'indent': 4,
|
||||
'item_export_kwargs': {
|
||||
'export_empty_fields': True,
|
||||
},
|
||||
},
|
||||
"/home/user/documents/items.xml": {
|
||||
"format": "xml",
|
||||
"fields": ["name", "price"],
|
||||
"item_filter": MyCustomFilter1,
|
||||
"encoding": "latin1",
|
||||
"indent": 8,
|
||||
'/home/user/documents/items.xml': {
|
||||
'format': 'xml',
|
||||
'fields': ['name', 'price'],
|
||||
'item_filter': MyCustomFilter1,
|
||||
'encoding': 'latin1',
|
||||
'indent': 8,
|
||||
},
|
||||
pathlib.Path("items.csv.gz"): {
|
||||
"format": "csv",
|
||||
"fields": ["price", "name"],
|
||||
"item_filter": "myproject.filters.MyCustomFilter2",
|
||||
"postprocessing": [MyPlugin1, "scrapy.extensions.postprocessing.GzipPlugin"],
|
||||
"gzip_compresslevel": 5,
|
||||
pathlib.Path('items.csv.gz'): {
|
||||
'format': 'csv',
|
||||
'fields': ['price', 'name'],
|
||||
'item_filter': 'myproject.filters.MyCustomFilter2',
|
||||
'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],
|
||||
'gzip_compresslevel': 5,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -624,7 +615,6 @@ Default:
|
|||
"file": "scrapy.extensions.feedexport.FileFeedStorage",
|
||||
"stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",
|
||||
"s3": "scrapy.extensions.feedexport.S3FeedStorage",
|
||||
"gs": "scrapy.extensions.feedexport.GCSFeedStorage",
|
||||
"ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
|
||||
}
|
||||
|
||||
|
|
@ -766,8 +756,8 @@ The function signature should be as follows:
|
|||
:param spider: source spider of the feed items
|
||||
:type spider: scrapy.Spider
|
||||
|
||||
.. caution:: The function must return a new dictionary instead of modifying
|
||||
the received ``params`` in-place.
|
||||
.. caution:: The function should return a new dictionary, modifying
|
||||
the received ``params`` in-place is deprecated.
|
||||
|
||||
For example, to include the :attr:`name <scrapy.Spider.name>` of the
|
||||
source spider in the feed URI:
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ Write items to MongoDB
|
|||
|
||||
In this example we'll write items to MongoDB_ using pymongo_.
|
||||
MongoDB address and database name are specified in Scrapy settings;
|
||||
MongoDB collection is specified in a class attribute.
|
||||
MongoDB collection is named after item class.
|
||||
|
||||
The main point of this example is to show how to :ref:`get the crawler
|
||||
<from-crawler>` and how to clean up the resources properly.
|
||||
|
|
|
|||
|
|
@ -23,8 +23,7 @@ Item Types
|
|||
|
||||
Scrapy supports the following types of items, via the `itemadapter`_ library:
|
||||
:ref:`dictionaries <dict-items>`, :ref:`Item objects <item-objects>`,
|
||||
:ref:`dataclass objects <dataclass-items>`, :ref:`attrs objects <attrs-items>`
|
||||
and :ref:`Pydantic models <pydantic-items>`.
|
||||
:ref:`dataclass objects <dataclass-items>`, and :ref:`attrs objects <attrs-items>`.
|
||||
|
||||
.. _itemadapter: https://github.com/scrapy/itemadapter
|
||||
|
||||
|
|
@ -62,8 +61,8 @@ its ``__init__`` method.
|
|||
:class:`Item` also allows the defining of field metadata, which can be used to
|
||||
:ref:`customize serialization <topics-exporters-field-serialization>`.
|
||||
|
||||
:mod:`scrapy.utils.trackref` tracks :class:`Item` objects to help find memory
|
||||
leaks (see :ref:`topics-leaks-trackrefs`).
|
||||
:mod:`trackref` tracks :class:`Item` objects to help find memory leaks
|
||||
(see :ref:`topics-leaks-trackrefs`).
|
||||
|
||||
Example:
|
||||
|
||||
|
|
@ -263,7 +262,7 @@ Creating items
|
|||
|
||||
>>> product = Product(name="Desktop PC", price=1000)
|
||||
>>> print(product)
|
||||
{'name': 'Desktop PC', 'price': 1000}
|
||||
Product(name='Desktop PC', price=1000)
|
||||
|
||||
|
||||
Getting field values
|
||||
|
|
@ -377,12 +376,10 @@ Creating dicts from items:
|
|||
>>> dict(product) # create a dict from all populated values
|
||||
{'price': 1000, 'name': 'Desktop PC'}
|
||||
|
||||
Creating items from dicts:
|
||||
|
||||
.. code-block:: pycon
|
||||
Creating items from dicts:
|
||||
|
||||
>>> Product({"name": "Laptop PC", "price": 1500})
|
||||
{'name': 'Laptop PC', 'price': 1500}
|
||||
Product(price=1500, name='Laptop PC')
|
||||
|
||||
>>> Product({"name": "Laptop PC", "lala": 1500}) # warning: unknown field in dict
|
||||
Traceback (most recent call last):
|
||||
|
|
|
|||
|
|
@ -151,8 +151,8 @@ Where:
|
|||
- :class:`~scrapy.pqueues.ScrapyPriorityQueue` creates the ``{priority}{s?}``
|
||||
directories.
|
||||
|
||||
- :class:`scrapy.squeues.PickleFifoDiskQueue`, a subclass of
|
||||
:class:`queuelib.FifoDiskQueue` that uses :mod:`pickle` to serialize
|
||||
- :class:`scrapy.squeues.PickleLifoDiskQueue`, a subclass of
|
||||
:class:`queuelib.LifoDiskQueue` that uses :mod:`pickle` to serialize
|
||||
:class:`dict` representations of :class:`scrapy.Request` objects, creates
|
||||
the ``info.json`` and ``q{00000}`` files.
|
||||
|
||||
|
|
|
|||
|
|
@ -62,9 +62,9 @@ Debugging memory leaks with ``trackref``
|
|||
|
||||
.. skip: start
|
||||
|
||||
:mod:`scrapy.utils.trackref` is a module provided by Scrapy to debug the most
|
||||
common cases of memory leaks. It basically tracks the references to all live
|
||||
Request, Response, Item, Spider and Selector objects.
|
||||
:mod:`trackref` is a module provided by Scrapy to debug the most common cases of
|
||||
memory leaks. It basically tracks the references to all live Request,
|
||||
Response, Item, Spider and Selector objects.
|
||||
|
||||
You can enter the telnet console and inspect how many objects (of the classes
|
||||
mentioned above) are currently alive using the ``prefs()`` function which is an
|
||||
|
|
@ -93,7 +93,7 @@ You can get the oldest object of each class using the
|
|||
Which objects are tracked?
|
||||
--------------------------
|
||||
|
||||
The objects tracked by ``trackref`` are all from these classes (and all its
|
||||
The objects tracked by ``trackrefs`` are all from these classes (and all its
|
||||
subclasses):
|
||||
|
||||
* :class:`scrapy.Request`
|
||||
|
|
@ -106,15 +106,10 @@ A real example
|
|||
--------------
|
||||
|
||||
Let's see a concrete example of a hypothetical case of memory leaks.
|
||||
Suppose we have some spider with a line similar to this one:
|
||||
Suppose we have some spider with a line similar to this one::
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
return Request(
|
||||
f"http://www.somenastyspider.com/product.php?pid={product_id}",
|
||||
callback=self.parse,
|
||||
cb_kwargs={"referer": response},
|
||||
)
|
||||
return Request(f"http://www.somenastyspider.com/product.php?pid={product_id}",
|
||||
callback=self.parse, cb_kwargs={'referer': response})
|
||||
|
||||
That line is passing a response reference inside a request which effectively
|
||||
ties the response lifetime to the requests' one, and that would definitely
|
||||
|
|
@ -169,7 +164,7 @@ Too many spiders?
|
|||
-----------------
|
||||
|
||||
If your project has too many spiders executed in parallel,
|
||||
the output of ``prefs()`` can be difficult to read.
|
||||
the output of :func:`prefs` can be difficult to read.
|
||||
For this reason, that function has a ``ignore`` argument which can be used to
|
||||
ignore a particular class (and all its subclasses). For
|
||||
example, this won't show any live references to spiders:
|
||||
|
|
@ -192,7 +187,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
|
|||
Inherit from this class if you want to track live
|
||||
instances with the ``trackref`` module.
|
||||
|
||||
.. function:: print_live_refs(ignore=NoneType)
|
||||
.. function:: print_live_refs(class_name, ignore=NoneType)
|
||||
|
||||
Print a report of live references, grouped by class name.
|
||||
|
||||
|
|
@ -208,9 +203,9 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
|
|||
|
||||
.. function:: iter_all(class_name)
|
||||
|
||||
Return an iterator over all objects alive with the given class name. Use
|
||||
:func:`print_live_refs` first to get a list of all tracked live objects
|
||||
per class name.
|
||||
Return an iterator over all objects alive with the given class name, or
|
||||
``None`` if none is found. Use :func:`print_live_refs` first to get a list
|
||||
of all tracked live objects per class name.
|
||||
|
||||
.. skip: end
|
||||
|
||||
|
|
|
|||
|
|
@ -36,9 +36,7 @@ Link extractor reference
|
|||
|
||||
The link extractor class is
|
||||
:class:`scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`. For convenience it
|
||||
can also be imported as ``scrapy.linkextractors.LinkExtractor``:
|
||||
|
||||
.. code-block:: python
|
||||
can also be imported as ``scrapy.linkextractors.LinkExtractor``::
|
||||
|
||||
from scrapy.linkextractors import LinkExtractor
|
||||
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ data that will be assigned to the ``name`` field later.
|
|||
|
||||
Afterwards, similar calls are used for ``price`` and ``stock`` fields
|
||||
(the latter using a CSS selector with the :meth:`~ItemLoader.add_css` method),
|
||||
and finally the ``last_updated`` field is populated directly with a literal value
|
||||
and finally the ``last_update`` field is populated directly with a literal value
|
||||
(``today``) using a different method: :meth:`~ItemLoader.add_value`.
|
||||
|
||||
Finally, when all data is collected, the :meth:`ItemLoader.load_item` method is
|
||||
|
|
@ -264,7 +264,7 @@ metadata. Here is an example:
|
|||
>>> il.add_value("name", ["Welcome to my", "<strong>website</strong>"])
|
||||
>>> il.add_value("price", ["€", "<span>1000</span>"])
|
||||
>>> il.load_item()
|
||||
Product(name='Welcome to my website', price='1000')
|
||||
{'name': 'Welcome to my website', 'price': '1000'}
|
||||
|
||||
.. skip: end
|
||||
|
||||
|
|
@ -273,8 +273,8 @@ The precedence order, for both input and output processors, is as follows:
|
|||
1. Item Loader field-specific attributes: ``field_in`` and ``field_out`` (most
|
||||
precedence)
|
||||
2. Field metadata (``input_processor`` and ``output_processor`` key)
|
||||
3. Item Loader defaults: :attr:`ItemLoader.default_input_processor` and
|
||||
:attr:`ItemLoader.default_output_processor` (least precedence)
|
||||
3. Item Loader defaults: :meth:`ItemLoader.default_input_processor` and
|
||||
:meth:`ItemLoader.default_output_processor` (least precedence)
|
||||
|
||||
See also: :ref:`topics-loaders-extending`.
|
||||
|
||||
|
|
@ -323,8 +323,8 @@ There are several ways to modify Item Loader context values:
|
|||
loader = ItemLoader(product, unit="cm")
|
||||
|
||||
3. On Item Loader declaration, for those input/output processors that support
|
||||
instantiating them with an Item Loader context.
|
||||
:class:`~itemloaders.processors.MapCompose` is one of them:
|
||||
instantiating them with an Item Loader context. :class:`~processor.MapCompose` is one of
|
||||
them:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
|
|
@ -350,9 +350,7 @@ When parsing related values from a subsection of a document, it can be
|
|||
useful to create nested loaders. Imagine you're extracting details from
|
||||
a footer of a page that looks something like:
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: html
|
||||
Example::
|
||||
|
||||
<footer>
|
||||
<a class="social" href="https://facebook.com/whatever">Like Us</a>
|
||||
|
|
|
|||
|
|
@ -4,6 +4,11 @@
|
|||
Logging
|
||||
=======
|
||||
|
||||
.. note::
|
||||
:mod:`scrapy.log` has been deprecated alongside its functions in favor of
|
||||
explicit calls to the Python standard logging. Keep reading to learn more
|
||||
about the new logging system.
|
||||
|
||||
Scrapy uses :mod:`logging` for event logging. We'll
|
||||
provide some simple examples to get you started, but for more advanced
|
||||
use-cases it's strongly suggested to read thoroughly its documentation.
|
||||
|
|
|
|||
|
|
@ -41,10 +41,11 @@ this:
|
|||
2. The item is returned from the spider and goes to the item pipeline.
|
||||
|
||||
3. When the item reaches the :class:`FilesPipeline`, the URLs in the
|
||||
``file_urls`` field are downloaded using the standard Scrapy downloader
|
||||
(which means the downloader middlewares are used, but the spider middlewares
|
||||
aren't). The item remains "locked" at that particular pipeline stage until
|
||||
the files have finished downloading (or failed for some reason).
|
||||
``file_urls`` field are scheduled for download using the standard
|
||||
Scrapy scheduler and downloader (which means the scheduler and downloader
|
||||
middlewares are reused), but with a higher priority, processing them before other
|
||||
pages are scraped. The item remains "locked" at that particular pipeline stage
|
||||
until the files have finish downloading (or fail for some reason).
|
||||
|
||||
4. When the files are downloaded, another field (``files``) will be populated
|
||||
with the results. This field will contain a list of dicts with information
|
||||
|
|
@ -370,12 +371,11 @@ For the Images Pipeline, set :setting:`IMAGES_URLS_FIELD` and/or
|
|||
If you need something more complex and want to override the custom pipeline
|
||||
behaviour, see :ref:`topics-media-pipeline-override`.
|
||||
|
||||
If you have multiple image pipelines inheriting from :class:`ImagesPipeline`
|
||||
and you want to have different settings in different pipelines you can set
|
||||
setting keys preceded with uppercase name of your pipeline class. E.g. if your
|
||||
pipeline is called ``MyPipeline`` and you want to have custom
|
||||
:setting:`IMAGES_URLS_FIELD` you define setting
|
||||
``MYPIPELINE_IMAGES_URLS_FIELD`` and your custom settings will be used.
|
||||
If you have multiple image pipelines inheriting from ImagePipeline and you want
|
||||
to have different settings in different pipelines you can set setting keys
|
||||
preceded with uppercase name of your pipeline class. E.g. if your pipeline is
|
||||
called MyPipeline and you want to have custom IMAGES_URLS_FIELD you define
|
||||
setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used.
|
||||
|
||||
|
||||
Additional features
|
||||
|
|
@ -470,9 +470,7 @@ When using the Images Pipeline, you can drop images which are too small, by
|
|||
specifying the minimum allowed size in the :setting:`IMAGES_MIN_HEIGHT` and
|
||||
:setting:`IMAGES_MIN_WIDTH` settings.
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block:: python
|
||||
For example::
|
||||
|
||||
IMAGES_MIN_HEIGHT = 110
|
||||
IMAGES_MIN_WIDTH = 110
|
||||
|
|
@ -495,9 +493,7 @@ Allowing redirections
|
|||
By default media pipelines ignore redirects, i.e. an HTTP redirection
|
||||
to a media file URL request will mean the media download is considered failed.
|
||||
|
||||
To handle media redirections, set this setting to ``True``:
|
||||
|
||||
.. code-block:: python
|
||||
To handle media redirections, set this setting to ``True``::
|
||||
|
||||
MEDIA_ALLOW_REDIRECTS = True
|
||||
|
||||
|
|
@ -551,9 +547,10 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
|
||||
.. method:: FilesPipeline.get_media_requests(item, info)
|
||||
|
||||
As seen on the workflow, the pipeline will get the requests for the files
|
||||
to download from the item by calling this method. You can override it to
|
||||
change what requests are returned:
|
||||
As seen on the workflow, the pipeline will get the URLs of the images to
|
||||
download from the item. In order to do this, you can override the
|
||||
:meth:`~get_media_requests` method and return a Request for each
|
||||
file URL:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
|
|
@ -593,9 +590,8 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
* ``downloaded`` - file was downloaded.
|
||||
* ``uptodate`` - file was not downloaded, as it was downloaded recently,
|
||||
according to the file expiration policy.
|
||||
* ``cached`` - file was taken from a cache (the response has a
|
||||
``"cached"`` flag, e.g. from
|
||||
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
|
||||
* ``cached`` - file was already scheduled for download, by another item
|
||||
sharing the same file.
|
||||
|
||||
The list of tuples received by :meth:`~item_completed` is
|
||||
guaranteed to retain the same order of the requests returned from the
|
||||
|
|
@ -622,6 +618,9 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
(False, Failure(...)),
|
||||
]
|
||||
|
||||
By default the :meth:`get_media_requests` method returns ``None`` which
|
||||
means there are no files to download for the item.
|
||||
|
||||
.. method:: FilesPipeline.item_completed(results, item, info)
|
||||
|
||||
The :meth:`FilesPipeline.item_completed` method called when all file
|
||||
|
|
|
|||
|
|
@ -17,10 +17,8 @@ Run Scrapy from a script
|
|||
You can use the :ref:`API <topics-api>` to run Scrapy from a script, instead of
|
||||
the typical way of running Scrapy via ``scrapy crawl``.
|
||||
|
||||
Remember that Scrapy requires a Twisted reactor or (with
|
||||
:setting:`TWISTED_REACTOR_ENABLED` set to ``False``) an asyncio event loop, so
|
||||
you need to run one of those in your script for it to work (helpers described
|
||||
below can do it for you).
|
||||
Remember that Scrapy is built on top of the Twisted
|
||||
asynchronous networking library, so you need to run it inside the Twisted reactor.
|
||||
|
||||
The first utility you can use to run your spiders is
|
||||
:class:`scrapy.crawler.AsyncCrawlerProcess` or
|
||||
|
|
@ -247,110 +245,6 @@ Using :func:`asyncio.run` with :class:`~scrapy.crawler.AsyncCrawlerRunner`:
|
|||
|
||||
asyncio.run(main())
|
||||
|
||||
.. _run-spiders-in-apps:
|
||||
|
||||
Running spiders inside existing applications
|
||||
============================================
|
||||
|
||||
You may want to run Scrapy spiders inside an existing application. In simple
|
||||
cases (e.g. task queues that spawn a process for every task, or applications
|
||||
that can execute tasks synchronously in the same process) you can use the same
|
||||
approach as for standalone scripts (see :ref:`run-from-script`). More complex
|
||||
cases, e.g. asynchronous web applications, have additional caveats and
|
||||
limitations.
|
||||
|
||||
If the application runs its own Twisted reactor, you can use
|
||||
:class:`~scrapy.crawler.AsyncCrawlerRunner` or
|
||||
:class:`~scrapy.crawler.CrawlerRunner` to run spiders using this reactor, see
|
||||
:ref:`run-from-script` for examples.
|
||||
|
||||
If the application doesn't run a Twisted reactor or an asyncio event loop (for
|
||||
example, a Django web app deployed with a WSGI server such as uWSGI), you can
|
||||
use :class:`~scrapy.crawler.AsyncCrawlerProcess` with
|
||||
:setting:`TWISTED_REACTOR_ENABLED` set to ``False``, so that Scrapy starts and
|
||||
stops an asyncio event loop for every spider run:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import scrapy
|
||||
from django.http import HttpResponse
|
||||
from scrapy.crawler import AsyncCrawlerProcess
|
||||
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
# Your spider definition
|
||||
...
|
||||
|
||||
|
||||
def crawl_view(request):
|
||||
process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
|
||||
process.crawl(MySpider)
|
||||
process.start() # returns when the spider finishes
|
||||
return HttpResponse("Crawling finished")
|
||||
|
||||
If the application runs its own asyncio event loop (for example, a Django web
|
||||
app deployed with an ASGI server such as uvicorn), you can use
|
||||
:class:`~scrapy.crawler.AsyncCrawlerRunner` with
|
||||
:setting:`TWISTED_REACTOR_ENABLED` set to ``False``, so that Scrapy uses the
|
||||
existing event loop:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import scrapy
|
||||
from django.http import HttpResponse
|
||||
from scrapy.crawler import AsyncCrawlerRunner
|
||||
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
# Your spider definition
|
||||
...
|
||||
|
||||
|
||||
async def crawl_view(request):
|
||||
runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False})
|
||||
await runner.crawl(MySpider) # completes when the spider finishes
|
||||
return HttpResponse("Crawling finished")
|
||||
|
||||
.. note:: Running Scrapy without a Twisted reactor is experimental and has
|
||||
some limitations, described in :ref:`asyncio-without-reactor`.
|
||||
|
||||
.. _run-in-notebook:
|
||||
|
||||
Running spiders in Jupyter notebooks
|
||||
====================================
|
||||
|
||||
You can run Scrapy spiders in Jupyter notebooks. You need to use
|
||||
:class:`~scrapy.crawler.AsyncCrawlerRunner` with
|
||||
:setting:`TWISTED_REACTOR_ENABLED` set to ``False`` for this, so that Scrapy
|
||||
uses the event loop provided by the notebook kernel. As
|
||||
:class:`~scrapy.crawler.AsyncCrawlerRunner` doesn't configure logging, and you
|
||||
most likely want to see the spider log in the notebook, you should call
|
||||
:func:`scrapy.utils.log.configure_logging`. Here is a full example, which
|
||||
supports rerunning both as a single cell and as separate cells:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy import Spider
|
||||
from scrapy.crawler import AsyncCrawlerRunner
|
||||
from scrapy.utils.log import configure_logging
|
||||
|
||||
configure_logging()
|
||||
|
||||
|
||||
class BooksSpider(Spider):
|
||||
name = "books"
|
||||
start_urls = ["https://books.toscrape.com"]
|
||||
|
||||
def parse(self, response):
|
||||
for book in response.css("h3"):
|
||||
yield {"title": book.css("a::attr(title)").get()}
|
||||
|
||||
|
||||
runner = AsyncCrawlerRunner({"TWISTED_REACTOR_ENABLED": False})
|
||||
await runner.crawl(BooksSpider)
|
||||
|
||||
.. note:: Running Scrapy without a Twisted reactor is experimental and has
|
||||
some limitations, described in :ref:`asyncio-without-reactor`.
|
||||
|
||||
.. _run-multiple-spiders:
|
||||
|
||||
|
|
|
|||
|
|
@ -247,7 +247,7 @@ Request objects
|
|||
Return a new Request which is a copy of this Request. See also:
|
||||
:ref:`topics-request-response-ref-request-callback-arguments`.
|
||||
|
||||
.. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs, cls])
|
||||
.. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs])
|
||||
|
||||
Return a Request object with the same members, except for those members
|
||||
given new values by whichever keyword arguments are specified. The
|
||||
|
|
@ -436,7 +436,7 @@ errors if needed:
|
|||
)
|
||||
|
||||
def parse_httpbin(self, response):
|
||||
self.logger.info(f"Got successful response from {response.url}")
|
||||
self.logger.info("Got successful response from {}".format(response.url))
|
||||
# do something useful here...
|
||||
|
||||
def errback_httpbin(self, failure):
|
||||
|
|
@ -717,7 +717,6 @@ Those are:
|
|||
* :reqmeta:`download_fail_on_dataloss`
|
||||
* :reqmeta:`download_latency`
|
||||
* :reqmeta:`download_maxsize`
|
||||
* :reqmeta:`download_slot`
|
||||
* :reqmeta:`download_warnsize`
|
||||
* :reqmeta:`download_timeout`
|
||||
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
|
||||
|
|
@ -1028,13 +1027,9 @@ Response objects
|
|||
:meth:`~scrapy.http.headers.Headers.get` to return the last header value with
|
||||
the specified name or :meth:`~scrapy.http.headers.Headers.getlist` to return
|
||||
all header values with the specified name. For example, this call will give you
|
||||
all cookies in the headers:
|
||||
all cookies in the headers::
|
||||
|
||||
.. skip: next
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
response.headers.getlist("Set-Cookie")
|
||||
response.headers.getlist('Set-Cookie')
|
||||
|
||||
.. attribute:: Response.body
|
||||
|
||||
|
|
@ -1090,7 +1085,7 @@ Response objects
|
|||
.. attribute:: Response.flags
|
||||
|
||||
A list that contains flags for this response. Flags are labels used for
|
||||
tagging Responses. For example: ``'cached'``, ``'redirected'``', etc. And
|
||||
tagging Responses. For example: ``'cached'``, ``'redirected``', etc. And
|
||||
they're shown on the string representation of the Response (``__str__()``
|
||||
method) which is used by the engine for logging.
|
||||
|
||||
|
|
@ -1105,8 +1100,8 @@ Response objects
|
|||
|
||||
The IP address of the server from which the Response originated.
|
||||
|
||||
This attribute is currently only populated by the HTTP download
|
||||
handlers, i.e. for ``http(s)`` responses. For other handlers,
|
||||
This attribute is currently only populated by the HTTP 1.1 download
|
||||
handler, i.e. for ``http(s)`` responses. For other handlers,
|
||||
:attr:`ip_address` is always ``None``.
|
||||
|
||||
.. attribute:: Response.protocol
|
||||
|
|
@ -1124,7 +1119,7 @@ Response objects
|
|||
|
||||
Returns a new Response which is a copy of this Response.
|
||||
|
||||
.. method:: Response.replace([url, status, headers, body, request, flags, certificate, ip_address, protocol, cls])
|
||||
.. method:: Response.replace([url, status, headers, body, request, flags, cls])
|
||||
|
||||
Returns a Response object with the same members, except for those members
|
||||
given new values by whichever keyword arguments are specified. The
|
||||
|
|
@ -1136,11 +1131,7 @@ Response objects
|
|||
a possible relative url.
|
||||
|
||||
This is a wrapper over :func:`~urllib.parse.urljoin`, it's merely an alias for
|
||||
making this call:
|
||||
|
||||
.. skip: next
|
||||
|
||||
.. code-block:: python
|
||||
making this call::
|
||||
|
||||
urllib.parse.urljoin(response.url, url)
|
||||
|
||||
|
|
@ -1229,31 +1220,21 @@ TextResponse objects
|
|||
|
||||
.. method:: TextResponse.jmespath(query)
|
||||
|
||||
.. skip: start
|
||||
A shortcut to ``TextResponse.selector.jmespath(query)``::
|
||||
|
||||
A shortcut to ``TextResponse.selector.jmespath(query)``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
response.jmespath("object.[*]")
|
||||
response.jmespath('object.[*]')
|
||||
|
||||
.. method:: TextResponse.xpath(query)
|
||||
|
||||
A shortcut to ``TextResponse.selector.xpath(query)``:
|
||||
A shortcut to ``TextResponse.selector.xpath(query)``::
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
response.xpath("//p")
|
||||
response.xpath('//p')
|
||||
|
||||
.. method:: TextResponse.css(query)
|
||||
|
||||
A shortcut to ``TextResponse.selector.css(query)``:
|
||||
A shortcut to ``TextResponse.selector.css(query)``::
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
response.css("p")
|
||||
|
||||
.. skip: end
|
||||
response.css('p')
|
||||
|
||||
.. automethod:: TextResponse.follow
|
||||
|
||||
|
|
|
|||
|
|
@ -308,7 +308,7 @@ Examples:
|
|||
|
||||
* ``*::text`` selects all descendant text nodes of the current selector context:
|
||||
|
||||
.. skip: next
|
||||
..skip: next
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> response.css("#images *::text").getall()
|
||||
|
|
@ -634,7 +634,8 @@ Example:
|
|||
.. code-block:: pycon
|
||||
|
||||
>>> from scrapy import Selector
|
||||
>>> sel = Selector(text="""
|
||||
>>> sel = Selector(
|
||||
... text="""
|
||||
... <ul class="list">
|
||||
... <li>1</li>
|
||||
... <li>2</li>
|
||||
|
|
@ -644,8 +645,8 @@ Example:
|
|||
... <li>4</li>
|
||||
... <li>5</li>
|
||||
... <li>6</li>
|
||||
... </ul>""")
|
||||
...
|
||||
... </ul>"""
|
||||
... )
|
||||
>>> xp = lambda x: sel.xpath(x).getall()
|
||||
|
||||
This gets all first ``<li>`` elements under whatever it is its parent:
|
||||
|
|
@ -947,9 +948,11 @@ with groups of itemscopes and corresponding itemprops:
|
|||
>>> sel = Selector(text=doc, type="html")
|
||||
>>> for scope in sel.xpath("//div[@itemscope]"):
|
||||
... print("current scope:", scope.xpath("@itemtype").getall())
|
||||
... props = scope.xpath("""
|
||||
... props = scope.xpath(
|
||||
... """
|
||||
... set:difference(./descendant::*/@itemprop,
|
||||
... .//*[@itemscope]/*/@itemprop)""")
|
||||
... .//*[@itemscope]/*/@itemprop)"""
|
||||
... )
|
||||
... print(f" properties: {props.getall()}")
|
||||
... print("")
|
||||
...
|
||||
|
|
|
|||
|
|
@ -409,36 +409,6 @@ Default: ``{}``
|
|||
A dict containing paths to the add-ons enabled in your project and their
|
||||
priorities. For more information, see :ref:`topics-addons`.
|
||||
|
||||
.. setting:: ASYNCIO_EVENT_LOOP
|
||||
|
||||
ASYNCIO_EVENT_LOOP
|
||||
------------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Import path of a given ``asyncio`` event loop class.
|
||||
|
||||
If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) or when
|
||||
:ref:`running Scrapy without a reactor <asyncio-without-reactor>` this setting
|
||||
can be used to specify the
|
||||
asyncio event loop to be used with it. Set the setting to the import path of the
|
||||
desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
|
||||
event loop will be used.
|
||||
|
||||
If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor`
|
||||
function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
|
||||
class to be used.
|
||||
|
||||
Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`.
|
||||
|
||||
.. caution:: Please be aware that, when using a non-default event loop
|
||||
(either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with
|
||||
:func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call
|
||||
:func:`asyncio.set_event_loop`, which will set the specified event loop
|
||||
as the current loop for the current OS thread.
|
||||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
|
||||
.. setting:: AWS_ACCESS_KEY_ID
|
||||
|
||||
AWS_ACCESS_KEY_ID
|
||||
|
|
@ -449,24 +419,6 @@ Default: ``None``
|
|||
The AWS access key used by code that requires access to `Amazon Web services`_,
|
||||
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
|
||||
|
||||
.. setting:: AWS_ENDPOINT_URL
|
||||
|
||||
AWS_ENDPOINT_URL
|
||||
----------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Endpoint URL used for S3-like storage, for example Minio or s3.scality.
|
||||
|
||||
.. setting:: AWS_REGION_NAME
|
||||
|
||||
AWS_REGION_NAME
|
||||
---------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The name of the region associated with the AWS client.
|
||||
|
||||
.. setting:: AWS_SECRET_ACCESS_KEY
|
||||
|
||||
AWS_SECRET_ACCESS_KEY
|
||||
|
|
@ -490,6 +442,15 @@ such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`, when using
|
|||
|
||||
.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html
|
||||
|
||||
.. setting:: AWS_ENDPOINT_URL
|
||||
|
||||
AWS_ENDPOINT_URL
|
||||
----------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Endpoint URL used for S3-like storage, for example Minio or s3.scality.
|
||||
|
||||
.. setting:: AWS_USE_SSL
|
||||
|
||||
AWS_USE_SSL
|
||||
|
|
@ -510,6 +471,43 @@ Default: ``None``
|
|||
Verify SSL connection between Scrapy and S3 or S3-like storage. By default
|
||||
SSL verification will occur.
|
||||
|
||||
.. setting:: AWS_REGION_NAME
|
||||
|
||||
AWS_REGION_NAME
|
||||
---------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The name of the region associated with the AWS client.
|
||||
|
||||
.. setting:: ASYNCIO_EVENT_LOOP
|
||||
|
||||
ASYNCIO_EVENT_LOOP
|
||||
------------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Import path of a given ``asyncio`` event loop class.
|
||||
|
||||
If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the
|
||||
asyncio event loop to be used with it. Set the setting to the import path of the
|
||||
desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
|
||||
event loop will be used.
|
||||
|
||||
If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor`
|
||||
function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
|
||||
class to be used.
|
||||
|
||||
Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`.
|
||||
|
||||
.. caution:: Please be aware that, when using a non-default event loop
|
||||
(either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with
|
||||
:func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call
|
||||
:func:`asyncio.set_event_loop`, which will set the specified event loop
|
||||
as the current loop for the current OS thread.
|
||||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
|
||||
.. setting:: BOT_NAME
|
||||
|
||||
BOT_NAME
|
||||
|
|
@ -556,8 +554,6 @@ performed to any single domain.
|
|||
See also: :ref:`topics-autothrottle` and its
|
||||
:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` option.
|
||||
|
||||
It is possible to change this setting per domain by using
|
||||
:setting:`DOWNLOAD_SLOTS`.
|
||||
|
||||
.. setting:: DEFAULT_DROPITEM_LOG_LEVEL
|
||||
|
||||
|
|
@ -597,7 +593,7 @@ When writing an item pipeline, you can force a different log level by setting
|
|||
DEFAULT_ITEM_CLASS
|
||||
------------------
|
||||
|
||||
Default: ``'scrapy.item.Item'``
|
||||
Default: ``'scrapy.Item'``
|
||||
|
||||
The default class that will be used for instantiating items in the :ref:`the
|
||||
Scrapy shell <topics-shell>`.
|
||||
|
|
@ -691,7 +687,7 @@ Whether to enable DNS in-memory cache.
|
|||
:class:`~scrapy.resolver.CachingThreadedResolver` and
|
||||
:class:`~scrapy.resolver.CachingHostnameResolver`. It has no effect when
|
||||
:setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect
|
||||
either when :setting:`TWISTED_DNS_RESOLVER` is set to a different resolver.
|
||||
either when :setting:`DNS_RESOLVER` is set to a different resolver.
|
||||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
|
||||
|
|
@ -706,6 +702,25 @@ DNS in-memory cache size, see :setting:`DNSCACHE_ENABLED`.
|
|||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
|
||||
.. setting:: TWISTED_DNS_RESOLVER
|
||||
|
||||
TWISTED_DNS_RESOLVER
|
||||
--------------------
|
||||
|
||||
Default: ``'scrapy.resolver.CachingThreadedResolver'``
|
||||
|
||||
The class to be used by Twisted to resolve DNS names. The default
|
||||
``scrapy.resolver.CachingThreadedResolver`` supports specifying a timeout for
|
||||
DNS requests via the :setting:`DNS_TIMEOUT` setting, but works only with IPv4
|
||||
addresses. Scrapy provides an alternative resolver,
|
||||
``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not
|
||||
take the :setting:`DNS_TIMEOUT` setting into account.
|
||||
|
||||
.. note::
|
||||
This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
||||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
|
||||
.. setting:: DNS_TIMEOUT
|
||||
|
||||
DNS_TIMEOUT
|
||||
|
|
@ -719,7 +734,7 @@ Timeout for processing of DNS queries in seconds. Float is supported.
|
|||
This setting is only used by
|
||||
:class:`~scrapy.resolver.CachingThreadedResolver`. It has no effect when
|
||||
:setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect
|
||||
either when :setting:`TWISTED_DNS_RESOLVER` is set to a different resolver.
|
||||
either when :setting:`DNS_RESOLVER` is set to a different resolver.
|
||||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
|
||||
|
|
@ -899,9 +914,7 @@ Use :setting:`DOWNLOAD_DELAY` to throttle your crawling speed, to avoid hitting
|
|||
servers too hard.
|
||||
|
||||
Decimal numbers are supported. For example, to send a maximum of 4 requests
|
||||
every 10 seconds:
|
||||
|
||||
.. code-block:: python
|
||||
every 10 seconds::
|
||||
|
||||
DOWNLOAD_DELAY = 2.5
|
||||
|
||||
|
|
@ -923,8 +936,9 @@ desired.
|
|||
|
||||
This delay can be set per spider using :attr:`download_delay` spider attribute.
|
||||
|
||||
It is possible to change this setting per domain by using
|
||||
:setting:`DOWNLOAD_SLOTS`.
|
||||
It is also possible to change this setting per domain, although it requires
|
||||
non-trivial code. See the implementation of the :ref:`AutoThrottle
|
||||
<topics-autothrottle>` extension for an example.
|
||||
|
||||
.. setting:: DOWNLOAD_BIND_ADDRESS
|
||||
|
||||
|
|
@ -1233,9 +1247,7 @@ the ``dont_filter`` parameter to ``True`` on the ``__init__`` method of a
|
|||
specific :class:`~scrapy.Request` object that should not be filtered out.
|
||||
|
||||
A class assigned to :setting:`DUPEFILTER_CLASS` must implement the following
|
||||
interface:
|
||||
|
||||
.. code-block:: python
|
||||
interface::
|
||||
|
||||
class MyDupeFilter:
|
||||
|
||||
|
|
@ -1318,7 +1330,6 @@ Default:
|
|||
|
||||
{
|
||||
"scrapy.extensions.corestats.CoreStats": 0,
|
||||
"scrapy.extensions.logcount.LogCount": 0,
|
||||
"scrapy.extensions.telnet.TelnetConsole": 0,
|
||||
"scrapy.extensions.memusage.MemoryUsage": 0,
|
||||
"scrapy.extensions.memdebug.MemoryDebugger": 0,
|
||||
|
|
@ -1341,8 +1352,6 @@ and the :ref:`list of available extensions <topics-extensions-ref>`.
|
|||
FEED_TEMPDIR
|
||||
------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The Feed Temp dir allows you to set a custom folder to save crawler
|
||||
temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
|
||||
:ref:`Amazon S3 <topics-feed-storage-s3>`.
|
||||
|
|
@ -1352,8 +1361,6 @@ temporary files before uploading with :ref:`FTP feed storage <topics-feed-storag
|
|||
FEED_STORAGE_GCS_ACL
|
||||
--------------------
|
||||
|
||||
Default: ``""``
|
||||
|
||||
The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage <topics-feed-storage-gcs>`.
|
||||
For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation <https://docs.cloud.google.com/storage/docs/access-control/lists>`_.
|
||||
|
||||
|
|
@ -1365,19 +1372,14 @@ FORCE_CRAWLER_PROCESS
|
|||
Default: ``False``
|
||||
|
||||
If ``False``, :ref:`Scrapy commands that need a CrawlerProcess
|
||||
<topics-commands-crawlerprocess>`, when :setting:`TWISTED_REACTOR_ENABLED`
|
||||
is set to ``True``, will decide between using
|
||||
<topics-commands-crawlerprocess>` will decide between using
|
||||
:class:`scrapy.crawler.AsyncCrawlerProcess` and
|
||||
:class:`scrapy.crawler.CrawlerProcess` based on the value of the
|
||||
:setting:`TWISTED_REACTOR` setting, but ignoring its value in :ref:`per-spider
|
||||
settings <spider-settings>`.
|
||||
|
||||
If ``True``, these commands will always use
|
||||
:class:`~scrapy.crawler.CrawlerProcess` when :setting:`TWISTED_REACTOR_ENABLED`
|
||||
is set to ``True``.
|
||||
|
||||
When :setting:`TWISTED_REACTOR_ENABLED` is set to ``False``,
|
||||
:class:`~scrapy.crawler.AsyncCrawlerProcess` will be used in all cases.
|
||||
:class:`~scrapy.crawler.CrawlerProcess`.
|
||||
|
||||
Set this to ``True`` if you want to set :setting:`TWISTED_REACTOR` to a
|
||||
non-default value in :ref:`per-spider settings <spider-settings>`.
|
||||
|
|
@ -1627,8 +1629,6 @@ The following special items are also supported:
|
|||
|
||||
- ``Python``
|
||||
|
||||
- ``pyOpenSSL``
|
||||
|
||||
.. setting:: LOGSTATS_INTERVAL
|
||||
|
||||
LOGSTATS_INTERVAL
|
||||
|
|
@ -1648,6 +1648,21 @@ Default: ``False``
|
|||
|
||||
Whether to enable memory debugging.
|
||||
|
||||
.. setting:: MEMDEBUG_NOTIFY
|
||||
|
||||
MEMDEBUG_NOTIFY
|
||||
---------------
|
||||
|
||||
Default: ``[]``
|
||||
|
||||
When memory debugging is enabled a memory report will be sent to the specified
|
||||
addresses if this setting is not empty, otherwise the report will be written to
|
||||
the log.
|
||||
|
||||
Example::
|
||||
|
||||
MEMDEBUG_NOTIFY = ['user@example.com']
|
||||
|
||||
.. setting:: MEMUSAGE_ENABLED
|
||||
|
||||
MEMUSAGE_ENABLED
|
||||
|
|
@ -1721,11 +1736,9 @@ Default: ``"<project name>.spiders"`` (:ref:`fallback <default-settings>`: ``""`
|
|||
|
||||
Module where to create new spiders using the :command:`genspider` command.
|
||||
|
||||
Example:
|
||||
Example::
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
NEWSPIDER_MODULE = "mybot.spiders_dev"
|
||||
NEWSPIDER_MODULE = 'mybot.spiders_dev'
|
||||
|
||||
.. setting:: RANDOMIZE_DOWNLOAD_DELAY
|
||||
|
||||
|
|
@ -1743,10 +1756,7 @@ significant similarities in the time between their requests.
|
|||
|
||||
The randomization policy is the same used by `wget`_ ``--random-wait`` option.
|
||||
|
||||
If :setting:`DOWNLOAD_DELAY` is zero this option has no effect.
|
||||
|
||||
It is possible to change this setting per domain by using
|
||||
:setting:`DOWNLOAD_SLOTS`.
|
||||
If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect.
|
||||
|
||||
.. _wget: https://www.gnu.org/software/wget/manual/wget.html
|
||||
|
||||
|
|
@ -1807,7 +1817,7 @@ The parser backend to use for parsing ``robots.txt`` files. For more information
|
|||
.. setting:: ROBOTSTXT_USER_AGENT
|
||||
|
||||
ROBOTSTXT_USER_AGENT
|
||||
--------------------
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Default: ``None``
|
||||
|
||||
|
|
@ -1962,8 +1972,6 @@ Default:
|
|||
|
||||
{
|
||||
"scrapy.contracts.default.UrlContract": 1,
|
||||
"scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,
|
||||
"scrapy.contracts.default.MetadataContract": 1,
|
||||
"scrapy.contracts.default.ReturnsContract": 2,
|
||||
"scrapy.contracts.default.ScrapesContract": 3,
|
||||
}
|
||||
|
|
@ -2028,7 +2036,6 @@ Default:
|
|||
.. code-block:: python
|
||||
|
||||
{
|
||||
"scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25,
|
||||
"scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50,
|
||||
"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
|
||||
"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
|
||||
|
|
@ -2104,25 +2111,6 @@ command.
|
|||
The project name must not conflict with the name of custom files or directories
|
||||
in the ``project`` subdirectory.
|
||||
|
||||
.. setting:: TWISTED_DNS_RESOLVER
|
||||
|
||||
TWISTED_DNS_RESOLVER
|
||||
--------------------
|
||||
|
||||
Default: ``'scrapy.resolver.CachingThreadedResolver'``
|
||||
|
||||
The class to be used by Twisted to resolve DNS names. The default
|
||||
``scrapy.resolver.CachingThreadedResolver`` supports specifying a timeout for
|
||||
DNS requests via the :setting:`DNS_TIMEOUT` setting, but works only with IPv4
|
||||
addresses. Scrapy provides an alternative resolver,
|
||||
``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not
|
||||
take the :setting:`DNS_TIMEOUT` setting into account.
|
||||
|
||||
.. note::
|
||||
This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
||||
|
||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||
|
||||
.. setting:: TWISTED_REACTOR_ENABLED
|
||||
|
||||
TWISTED_REACTOR_ENABLED
|
||||
|
|
@ -2161,9 +2149,6 @@ Default: ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``
|
|||
|
||||
Import path of a given :mod:`~twisted.internet.reactor`.
|
||||
|
||||
.. note::
|
||||
This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
||||
|
||||
Scrapy will install this reactor if no other reactor is installed yet, such as
|
||||
when the ``scrapy`` CLI program is invoked or when using the
|
||||
:class:`~scrapy.crawler.AsyncCrawlerProcess` class or the
|
||||
|
|
@ -2200,7 +2185,7 @@ In order to use the reactor installed by Scrapy:
|
|||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.timeout = int(kwargs.pop("timeout", "60"))
|
||||
super().__init__(*args, **kwargs)
|
||||
super(QuotesSpider, self).__init__(*args, **kwargs)
|
||||
|
||||
async def start(self):
|
||||
reactor.callLater(self.timeout, self.stop)
|
||||
|
|
@ -2229,7 +2214,7 @@ which raises an exception, becomes:
|
|||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.timeout = int(kwargs.pop("timeout", "60"))
|
||||
super().__init__(*args, **kwargs)
|
||||
super(QuotesSpider, self).__init__(*args, **kwargs)
|
||||
|
||||
async def start(self):
|
||||
from twisted.internet import reactor
|
||||
|
|
@ -2267,7 +2252,7 @@ URLLENGTH_LIMIT
|
|||
|
||||
Default: ``2083``
|
||||
|
||||
Scope: ``scrapy.spidermiddlewares.urllength``
|
||||
Scope: ``spidermiddlewares.urllength``
|
||||
|
||||
The maximum URL length to allow for crawled URLs.
|
||||
|
||||
|
|
|
|||
|
|
@ -34,9 +34,7 @@ is unavailable.
|
|||
Through Scrapy's settings you can configure it to use any one of
|
||||
``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which
|
||||
are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment
|
||||
variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`:
|
||||
|
||||
.. code-block:: ini
|
||||
variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
|
||||
|
||||
[settings]
|
||||
shell = bpython
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ Here is a simple example showing how you can catch signals and perform some acti
|
|||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
spider = super().from_crawler(crawler, *args, **kwargs)
|
||||
spider = super(DmozSpider, cls).from_crawler(crawler, *args, **kwargs)
|
||||
crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
|
||||
return spider
|
||||
|
||||
|
|
@ -60,8 +60,6 @@ Let's take an example using :ref:`coroutines <topics-coroutines>`:
|
|||
.. skip: next
|
||||
.. code-block:: python
|
||||
|
||||
import json
|
||||
|
||||
import scrapy
|
||||
import treq
|
||||
|
||||
|
|
@ -72,7 +70,7 @@ Let's take an example using :ref:`coroutines <topics-coroutines>`:
|
|||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
spider = super().from_crawler(crawler, *args, **kwargs)
|
||||
spider = super(SignalSpider, cls).from_crawler(crawler, *args, **kwargs)
|
||||
crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
|
||||
return spider
|
||||
|
||||
|
|
@ -454,7 +452,7 @@ bytes_received
|
|||
.. signal:: bytes_received
|
||||
.. function:: bytes_received(data, request, spider)
|
||||
|
||||
Sent by some download handlers when a group of bytes is
|
||||
Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is
|
||||
received for a specific request. This signal might be fired multiple
|
||||
times for the same request, with partial data each time. For instance,
|
||||
a possible scenario for a 25 kb response would be two signals fired
|
||||
|
|
@ -482,7 +480,7 @@ headers_received
|
|||
.. signal:: headers_received
|
||||
.. function:: headers_received(headers, body_length, request, spider)
|
||||
|
||||
Sent by some download handlers when the response headers are
|
||||
Sent by the HTTP 1.1 and S3 download handlers when the response headers are
|
||||
available for a given request, before downloading any additional content.
|
||||
|
||||
Handlers for this signal can stop the download of a response while it
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ previous (or subsequent) middleware being applied.
|
|||
If you want to disable a builtin middleware (the ones defined in
|
||||
:setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
|
||||
in your project :setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its
|
||||
value. For example, if you want to disable the referer middleware:
|
||||
value. For example, if you want to disable the off-site middleware:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
|
|
|
|||
|
|
@ -208,6 +208,12 @@ scrapy.Spider
|
|||
:param response: the response to parse
|
||||
:type response: :class:`~scrapy.http.Response`
|
||||
|
||||
.. method:: log(message, [level, component])
|
||||
|
||||
Wrapper that sends a log message through the Spider's :attr:`logger`,
|
||||
kept for backward compatibility. For more information see
|
||||
:ref:`topics-logging-from-spiders`.
|
||||
|
||||
.. method:: closed(reason)
|
||||
|
||||
Called when the spider closes. This method provides a shortcut to
|
||||
|
|
@ -308,7 +314,7 @@ Spiders can access arguments in their `__init__` methods:
|
|||
name = "myspider"
|
||||
|
||||
def __init__(self, category=None, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
super(MySpider, self).__init__(*args, **kwargs)
|
||||
self.start_urls = [f"http://www.example.com/categories/{category}"]
|
||||
# ...
|
||||
|
||||
|
|
@ -329,8 +335,8 @@ The above example can also be written as follows:
|
|||
|
||||
If you are :ref:`running Scrapy from a script <run-from-script>`, you can
|
||||
specify spider arguments when calling
|
||||
:meth:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
|
||||
:meth:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`:
|
||||
:class:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
|
||||
:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`:
|
||||
|
||||
.. skip: next
|
||||
.. code-block:: python
|
||||
|
|
@ -587,7 +593,7 @@ Let's now take a look at an example CrawlSpider with rules:
|
|||
This spider would start crawling example.com's home page, collecting category
|
||||
links, and item links, parsing the latter with the ``parse_item`` method. For
|
||||
each item response, some data will be extracted from the HTML using XPath, and
|
||||
a dictionary will be filled with it.
|
||||
an :class:`~scrapy.Item` will be filled with it.
|
||||
|
||||
XMLFeedSpider
|
||||
-------------
|
||||
|
|
@ -608,7 +614,7 @@ XMLFeedSpider
|
|||
|
||||
A string which defines the iterator to use. It can be either:
|
||||
|
||||
- ``'iternodes'`` - a fast iterator based on ``lxml``
|
||||
- ``'iternodes'`` - a fast iterator based on regular expressions
|
||||
|
||||
- ``'html'`` - an iterator which uses :class:`~scrapy.Selector`.
|
||||
Keep in mind this uses DOM parsing and must load all DOM in memory
|
||||
|
|
@ -622,11 +628,9 @@ XMLFeedSpider
|
|||
|
||||
.. attribute:: itertag
|
||||
|
||||
A string with the name of the node (or element) to iterate in. Example:
|
||||
A string with the name of the node (or element) to iterate in. Example::
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
itertag = "product"
|
||||
itertag = 'product'
|
||||
|
||||
.. attribute:: namespaces
|
||||
|
||||
|
|
@ -639,17 +643,12 @@ XMLFeedSpider
|
|||
You can then specify nodes with namespaces in the :attr:`itertag`
|
||||
attribute.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy.spiders import XMLFeedSpider
|
||||
|
||||
Example::
|
||||
|
||||
class YourSpider(XMLFeedSpider):
|
||||
|
||||
namespaces = [("n", "http://www.sitemaps.org/schemas/sitemap/0.9")]
|
||||
itertag = "n:url"
|
||||
namespaces = [('n', 'http://www.sitemaps.org/schemas/sitemap/0.9')]
|
||||
itertag = 'n:url'
|
||||
# ...
|
||||
|
||||
Apart from these new attributes, this spider has the following overridable
|
||||
|
|
@ -809,11 +808,9 @@ SitemapSpider
|
|||
the regular expression. ``callback`` can be a string (indicating the
|
||||
name of a spider method) or a callable.
|
||||
|
||||
For example:
|
||||
For example::
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
sitemap_rules = [("/product/", "parse_product")]
|
||||
sitemap_rules = [('/product/', 'parse_product')]
|
||||
|
||||
Rules are applied in order, and only the first one that matches will be
|
||||
used.
|
||||
|
|
@ -835,9 +832,7 @@ SitemapSpider
|
|||
are links for the same website in another language passed within
|
||||
the same ``url`` block.
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block:: xml
|
||||
For example::
|
||||
|
||||
<url>
|
||||
<loc>http://example.com/</loc>
|
||||
|
|
@ -855,9 +850,7 @@ SitemapSpider
|
|||
This is a filter function that could be overridden to select sitemap entries
|
||||
based on their attributes.
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block:: xml
|
||||
For example::
|
||||
|
||||
<url>
|
||||
<loc>http://example.com/</loc>
|
||||
|
|
@ -960,7 +953,6 @@ Combine SitemapSpider with other sources of urls:
|
|||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy import Request
|
||||
from scrapy.spiders import SitemapSpider
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ Collector, and can be accessed through the :attr:`~scrapy.crawler.Crawler.stats`
|
|||
attribute of the :ref:`topics-api-crawler`, as illustrated by the examples in
|
||||
the :ref:`topics-stats-usecases` section below.
|
||||
|
||||
The Stats Collector API is always available, so you can always use it (to
|
||||
increment or set new stat keys), regardless
|
||||
However, the Stats Collector is always available, so you can always import it
|
||||
in your module and use its API (to increment or set new stat keys), regardless
|
||||
of whether the stats collection is enabled or not. If it's disabled, the API
|
||||
will still work but it won't collect anything. This is aimed at simplifying the
|
||||
stats collector usage: you should spend no more than one line of code for
|
||||
|
|
@ -21,6 +21,9 @@ using the Stats Collector from.
|
|||
Another feature of the Stats Collector is that it's very efficient (when
|
||||
enabled) and extremely efficient (almost unnoticeable) when disabled.
|
||||
|
||||
The Stats Collector keeps a stats table per open spider which is automatically
|
||||
opened when the spider is opened, and closed when the spider is closed.
|
||||
|
||||
.. _topics-stats-usecases:
|
||||
|
||||
Common Stats Collector uses
|
||||
|
|
@ -84,13 +87,13 @@ Get all stats:
|
|||
Available Stats Collectors
|
||||
==========================
|
||||
|
||||
.. currentmodule:: scrapy.statscollectors
|
||||
|
||||
Besides the basic :class:`StatsCollector` there are other Stats Collectors
|
||||
available in Scrapy which extend the basic Stats Collector. You can select
|
||||
which Stats Collector to use through the :setting:`STATS_CLASS` setting. The
|
||||
default Stats Collector used is the :class:`MemoryStatsCollector`.
|
||||
|
||||
.. currentmodule:: scrapy.statscollectors
|
||||
|
||||
MemoryStatsCollector
|
||||
--------------------
|
||||
|
||||
|
|
@ -99,7 +102,7 @@ MemoryStatsCollector
|
|||
A simple stats collector that keeps the stats of the last scraping run (for
|
||||
each spider) in memory, after they're closed. The stats can be accessed
|
||||
through the :attr:`spider_stats` attribute, which is a dict keyed by spider
|
||||
name.
|
||||
domain name.
|
||||
|
||||
This is the default Stats Collector used in Scrapy.
|
||||
|
||||
|
|
|
|||
|
|
@ -36,9 +36,9 @@ disable it if you want. For more information about the extension itself see
|
|||
How to access the telnet console
|
||||
================================
|
||||
|
||||
The telnet console listens on the first available TCP port from the range
|
||||
defined in the :setting:`TELNETCONSOLE_PORT` setting, which defaults to
|
||||
``[6023, 6073]``. To access the console you need to type::
|
||||
The telnet console listens in the TCP port defined in the
|
||||
:setting:`TELNETCONSOLE_PORT` setting, which defaults to ``6023``. To access
|
||||
the console you need to type::
|
||||
|
||||
telnet localhost 6023
|
||||
Trying localhost...
|
||||
|
|
@ -107,8 +107,8 @@ Here are some example tasks you can do with the telnet console:
|
|||
View engine status
|
||||
------------------
|
||||
|
||||
You can use the ``est()`` method provided by the console to quickly show the
|
||||
engine status::
|
||||
You can use the ``est()`` method of the Scrapy engine to quickly show its state
|
||||
using the telnet console::
|
||||
|
||||
telnet localhost 6023
|
||||
>>> est()
|
||||
|
|
|
|||
|
|
@ -39,8 +39,8 @@ API stability
|
|||
|
||||
API stability was one of the major goals for the *1.0* release.
|
||||
|
||||
Methods or functions that start with a single underscore (``_``) are private
|
||||
and should never be relied upon as stable.
|
||||
Methods or functions that start with a single dash (``_``) are private and
|
||||
should never be relied as stable.
|
||||
|
||||
Also, keep in mind that stable doesn't mean complete: stable APIs could grow
|
||||
new methods or functionality but the existing methods should keep working the
|
||||
|
|
|
|||
|
|
@ -94,82 +94,7 @@ untyped_calls_exclude = [
|
|||
[[tool.mypy.overrides]]
|
||||
module = "tests.*"
|
||||
allow_untyped_defs = true
|
||||
allow_incomplete_defs = true # 59 errors
|
||||
|
||||
# TODO
|
||||
[[tool.mypy.overrides]]
|
||||
module = [
|
||||
"tests.mockserver.*",
|
||||
"tests.spiders",
|
||||
"tests.test_closespider",
|
||||
"tests.test_cmdline",
|
||||
"tests.test_contracts",
|
||||
"tests.test_core_downloader",
|
||||
"tests.test_downloader_handler_twisted_ftp",
|
||||
"tests.test_downloadermiddleware_cookies",
|
||||
"tests.test_downloadermiddleware_httpauth",
|
||||
"tests.test_downloadermiddleware_httpcache",
|
||||
"tests.test_downloadermiddleware_httpcompression",
|
||||
"tests.test_downloadermiddleware_httpproxy",
|
||||
"tests.test_downloadermiddleware_offsite",
|
||||
"tests.test_downloadermiddleware_redirect",
|
||||
"tests.test_downloadermiddleware_redirect_base",
|
||||
"tests.test_downloadermiddleware_redirect_metarefresh",
|
||||
"tests.test_downloadermiddleware_retry",
|
||||
"tests.test_downloadermiddleware_robotstxt",
|
||||
"tests.test_downloadermiddleware_stats",
|
||||
"tests.test_downloaderslotssettings",
|
||||
"tests.test_dupefilters",
|
||||
"tests.test_engine_loop",
|
||||
"tests.test_exporters",
|
||||
"tests.test_extension_statsmailer",
|
||||
"tests.test_extension_throttle",
|
||||
"tests.test_feedexport",
|
||||
"tests.test_feedexport_postprocess",
|
||||
"tests.test_feedexport_storages",
|
||||
"tests.test_feedexport_uri_params",
|
||||
"tests.test_http2_client_protocol",
|
||||
"tests.test_http_headers",
|
||||
"tests.test_http_request",
|
||||
"tests.test_http_request_form",
|
||||
"tests.test_http_response",
|
||||
"tests.test_http_response_text",
|
||||
"tests.test_item",
|
||||
"tests.test_link",
|
||||
"tests.test_linkextractors",
|
||||
"tests.test_loader",
|
||||
"tests.test_logformatter",
|
||||
"tests.test_logstats",
|
||||
"tests.test_mail",
|
||||
"tests.test_pipeline_crawl",
|
||||
"tests.test_pipeline_files",
|
||||
"tests.test_pipeline_images",
|
||||
"tests.test_pipeline_media",
|
||||
"tests.test_pipelines",
|
||||
"tests.test_pqueues",
|
||||
"tests.test_request_attribute_binding",
|
||||
"tests.test_request_cb_kwargs",
|
||||
"tests.test_request_dict",
|
||||
"tests.test_request_left",
|
||||
"tests.test_robotstxt_interface",
|
||||
"tests.test_scheduler_base",
|
||||
"tests.test_settings",
|
||||
"tests.test_spider",
|
||||
"tests.test_spider_crawl",
|
||||
"tests.test_spidermiddleware_output_chain",
|
||||
"tests.test_spidermiddleware_process_start",
|
||||
"tests.test_spider_sitemap",
|
||||
"tests.test_squeues",
|
||||
"tests.test_squeues_request",
|
||||
"tests.test_stats",
|
||||
"tests.test_utils_datatypes",
|
||||
"tests.test_utils_decorators",
|
||||
"tests.test_utils_defer",
|
||||
"tests.test_utils_deprecate",
|
||||
"tests.test_utils_misc.test_return_with_argument_inside_generator",
|
||||
"tests.test_utils_python",
|
||||
"tests.test_utils_request",
|
||||
]
|
||||
allow_incomplete_defs = true # 48 errors
|
||||
check_untyped_defs = false
|
||||
|
||||
# Interface classes are hard to support
|
||||
|
|
@ -344,7 +269,7 @@ markers = [
|
|||
"requires_uvloop: marks tests as only enabled when uvloop is known to be working",
|
||||
"requires_botocore: marks tests that need botocore (but not boto3)",
|
||||
"requires_boto3: marks tests that need botocore and boto3",
|
||||
"requires_mitmproxy: marks tests that need a mitmdump executable",
|
||||
"requires_mitmproxy: marks tests that need mitmproxy",
|
||||
"requires_internet: marks tests that need real Internet access",
|
||||
]
|
||||
filterwarnings = [
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from twisted.internet.ssl import (
|
|||
from twisted.web.client import BrowserLikePolicyForHTTPS
|
||||
from twisted.web.iweb import IPolicyForHTTPS
|
||||
from zope.interface.declarations import implementer
|
||||
from zope.interface.verify import verifyObject
|
||||
|
||||
from scrapy.core.downloader.tls import (
|
||||
_TWISTED_VERSION_MAP,
|
||||
|
|
@ -231,6 +232,7 @@ class _AcceptableProtocolsContextFactory:
|
|||
# all of this with _ScrapyClientContextFactory.acceptableProtocols.
|
||||
|
||||
def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]):
|
||||
verifyObject(IPolicyForHTTPS, context_factory)
|
||||
self._wrapped_context_factory: Any = context_factory
|
||||
self._acceptable_protocols: list[bytes] = acceptable_protocols
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.boto import is_botocore_available
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
|
|
@ -50,16 +49,7 @@ class S3DownloadHandler(BaseDownloadHandler):
|
|||
|
||||
async def download_request(self, request: Request) -> Response:
|
||||
p = urlparse_cached(request)
|
||||
if request.meta.get("is_secure") is False:
|
||||
warnings.warn(
|
||||
"Passing is_secure=False for s3:// requests is deprecated."
|
||||
" In future Scrapy releases this flag will be ignored.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
scheme = "http"
|
||||
else:
|
||||
scheme = "https"
|
||||
scheme = "http" if request.meta.get("is_secure") is False else "https"
|
||||
bucket = p.hostname
|
||||
path = p.path + "?" + p.query if p.query else p.path
|
||||
url = f"{scheme}://{bucket}.s3.amazonaws.com{path}"
|
||||
|
|
|
|||
|
|
@ -289,11 +289,11 @@ class Scheduler(BaseScheduler):
|
|||
|
||||
:param dqclass: A class to be used as persistent request queue.
|
||||
The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default.
|
||||
:type dqclass: type
|
||||
:type dqclass: class
|
||||
|
||||
:param mqclass: A class to be used as non-persistent request queue.
|
||||
The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default.
|
||||
:type mqclass: type
|
||||
:type mqclass: class
|
||||
|
||||
:param logunser: A boolean that indicates whether or not unserializable requests should be logged.
|
||||
The value for the :setting:`SCHEDULER_DEBUG` setting is used by default.
|
||||
|
|
@ -306,7 +306,7 @@ class Scheduler(BaseScheduler):
|
|||
|
||||
:param pqclass: A class to be used as priority queue for requests.
|
||||
The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default.
|
||||
:type pqclass: type
|
||||
:type pqclass: class
|
||||
|
||||
:param crawler: The crawler object corresponding to the current crawl.
|
||||
:type crawler: :class:`scrapy.crawler.Crawler`
|
||||
|
|
|
|||
|
|
@ -39,11 +39,7 @@ from scrapy.utils.reactor import (
|
|||
verify_installed_asyncio_event_loop,
|
||||
verify_installed_reactor,
|
||||
)
|
||||
from scrapy.utils.reactorless import (
|
||||
ReactorImportHook,
|
||||
install_reactor_import_hook,
|
||||
uninstall_reactor_import_hook,
|
||||
)
|
||||
from scrapy.utils.reactorless import install_reactor_import_hook
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Awaitable, Generator, Iterable
|
||||
|
|
@ -841,7 +837,6 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
|
|||
super().__init__(settings, install_root_handler)
|
||||
logger.debug("Using AsyncCrawlerProcess")
|
||||
self._reactorless_loop: asyncio.AbstractEventLoop | None = None
|
||||
self._reactor_import_hook: ReactorImportHook | None = None
|
||||
# We want the asyncio event loop to be installed early, so that it's
|
||||
# always the correct one. And as we do that, we can also install the
|
||||
# reactor here.
|
||||
|
|
@ -854,7 +849,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
|
|||
"TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed."
|
||||
)
|
||||
self._reactorless_loop = set_asyncio_event_loop(loop_path)
|
||||
self._reactor_import_hook = install_reactor_import_hook()
|
||||
install_reactor_import_hook()
|
||||
elif is_reactor_installed():
|
||||
# The user could install a reactor before this class is instantiated.
|
||||
# We need to make sure the reactor is the correct one and the loop
|
||||
|
|
@ -984,10 +979,6 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
|
|||
loop.run_until_complete(loop.shutdown_asyncgens())
|
||||
loop.run_until_complete(loop.shutdown_default_executor())
|
||||
finally:
|
||||
# loop.close() can raise, so we uninstall the hook first
|
||||
if self._reactor_import_hook: # pragma: no branch
|
||||
uninstall_reactor_import_hook(self._reactor_import_hook)
|
||||
self._reactor_import_hook = None
|
||||
self._reactorless_main_task = None
|
||||
asyncio.set_event_loop(None)
|
||||
loop.close()
|
||||
|
|
|
|||
|
|
@ -123,14 +123,12 @@ class HttpCompressionMiddleware:
|
|||
response.body, content_encoding, max_size
|
||||
)
|
||||
except _DecompressionMaxSizeExceeded as e:
|
||||
msg = (
|
||||
raise IgnoreRequest(
|
||||
f"Ignored response {response} because its body "
|
||||
f"({len(response.body)} B compressed, "
|
||||
f"{e.decompressed_size} B decompressed so far) exceeded "
|
||||
f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."
|
||||
)
|
||||
logger.warning(msg)
|
||||
raise IgnoreRequest(msg) from e
|
||||
) from e
|
||||
if len(response.body) < warn_size <= len(decoded_body):
|
||||
logger.warning(
|
||||
f"{response} body size after decompression "
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ Item Exporters are used to export/serialize items into different formats.
|
|||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import logging
|
||||
import marshal
|
||||
import pickle
|
||||
import pprint
|
||||
|
|
@ -25,8 +24,6 @@ from scrapy.utils.serialize import ScrapyJSONEncoder
|
|||
if TYPE_CHECKING:
|
||||
from json import JSONEncoder
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"BaseItemExporter",
|
||||
"CsvItemExporter",
|
||||
|
|
@ -257,8 +254,6 @@ class CsvItemExporter(BaseItemExporter):
|
|||
self.csv_writer = csv.writer(self.stream, **self._kwargs)
|
||||
self._headers_not_written = True
|
||||
self._join_multivalued = join_multivalued
|
||||
self._autodetected_fields = False
|
||||
self._data_loss_warned = False
|
||||
|
||||
def serialize_field(
|
||||
self, field: Mapping[str, Any] | Field, name: str, value: Any
|
||||
|
|
@ -279,22 +274,6 @@ class CsvItemExporter(BaseItemExporter):
|
|||
self._headers_not_written = False
|
||||
self._write_headers_and_set_fields_to_export(item)
|
||||
|
||||
if (
|
||||
self._autodetected_fields
|
||||
and self.fields_to_export is not None
|
||||
and not self._data_loss_warned
|
||||
):
|
||||
item_fields = ItemAdapter(item).field_names()
|
||||
dropped_fields = set(item_fields) - set(self.fields_to_export)
|
||||
|
||||
if dropped_fields:
|
||||
dropped_fields_display = sorted(dropped_fields)
|
||||
logger.warning(
|
||||
f"CSVExporter dropped fields {dropped_fields_display}. "
|
||||
f"To avoid this, fully configure your FEED_EXPORT_FIELDS setting. "
|
||||
f"See: https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-fields",
|
||||
)
|
||||
self._data_loss_warned = True
|
||||
fields = self._get_serialized_fields(item, default_value="", include_empty=True)
|
||||
values = list(self._build_row(x for _, x in fields))
|
||||
self.csv_writer.writerow(values)
|
||||
|
|
@ -314,7 +293,6 @@ class CsvItemExporter(BaseItemExporter):
|
|||
if not self.fields_to_export:
|
||||
# use declared field names, or keys if the item is a dict
|
||||
self.fields_to_export = ItemAdapter(item).field_names()
|
||||
self._autodetected_fields = True
|
||||
fields: Iterable[str]
|
||||
if isinstance(self.fields_to_export, Mapping):
|
||||
fields = self.fields_to_export.values()
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from urllib.parse import unquote, urlparse
|
|||
|
||||
from twisted.internet.defer import Deferred, DeferredList
|
||||
from w3lib.url import file_uri_to_path
|
||||
from zope.interface import Interface
|
||||
from zope.interface import Interface, implementer
|
||||
|
||||
from scrapy import Spider, signals
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
|
|
@ -47,33 +47,6 @@ if TYPE_CHECKING:
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Printf-style placeholders (e.g. %(time)s) used to build feed URIs. Any other
|
||||
# percent character in a URI (e.g. percent-encoding such as %20 or %23) must be
|
||||
# treated as a literal rather than as the start of a placeholder.
|
||||
_FEED_URI_PLACEHOLDER_RE = re.compile(
|
||||
r"%\([^)]+\)[-+ #0]*(?:\d+|\*)?(?:\.(?:\d+|\*))?[diouxXeEfFgGcrsa]"
|
||||
)
|
||||
|
||||
|
||||
def apply_uri_params(uri_template: str, uri_params: dict[str, Any]) -> str:
|
||||
"""Return *uri_template* with its ``%(...)s`` placeholders replaced using
|
||||
*uri_params*, leaving any other percent character untouched.
|
||||
|
||||
This allows feed URIs to contain percent-encoded characters (e.g. ``%20``
|
||||
in a path with spaces or ``%23`` in FTP credentials) without them being
|
||||
misinterpreted as printf-style formatting directives.
|
||||
"""
|
||||
parts: list[str] = []
|
||||
last = 0
|
||||
for match in _FEED_URI_PLACEHOLDER_RE.finditer(uri_template):
|
||||
parts.append(uri_template[last : match.start()].replace("%", "%%"))
|
||||
parts.append(match.group(0))
|
||||
last = match.end()
|
||||
parts.append(uri_template[last:].replace("%", "%%"))
|
||||
return "".join(parts) % uri_params
|
||||
|
||||
|
||||
UriParamsCallableT: TypeAlias = Callable[
|
||||
[dict[str, Any], Spider], dict[str, Any] | None
|
||||
]
|
||||
|
|
@ -115,18 +88,25 @@ class ItemFilter:
|
|||
return True # accept all items by default
|
||||
|
||||
|
||||
class _IFeedStorage(Interface): # type: ignore[misc] # pragma: no cover
|
||||
class IFeedStorage(Interface): # type: ignore[misc]
|
||||
"""Interface that all Feed Storages must implement"""
|
||||
|
||||
# pylint: disable=no-self-argument
|
||||
|
||||
def __init__(uri, *, feed_options=None): ... # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called
|
||||
def __init__(uri, *, feed_options=None): # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called
|
||||
"""Initialize the storage with the parameters given in the URI and the
|
||||
feed-specific options (see :setting:`FEEDS`)"""
|
||||
|
||||
def open(spider): ... # type: ignore[no-untyped-def]
|
||||
def open(spider): # type: ignore[no-untyped-def]
|
||||
"""Open the storage for the given spider. It must return a file-like
|
||||
object that will be used for the exporters"""
|
||||
|
||||
def store(file): ... # type: ignore[no-untyped-def]
|
||||
def store(file): # type: ignore[no-untyped-def]
|
||||
"""Store the given file stream"""
|
||||
|
||||
|
||||
class FeedStorageProtocol(Protocol):
|
||||
"""Protocol that all Feed Storages must follow."""
|
||||
"""Reimplementation of ``IFeedStorage`` that can be used in type hints."""
|
||||
|
||||
def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
|
||||
"""Initialize the storage with the parameters given in the URI and the
|
||||
|
|
@ -140,6 +120,7 @@ class FeedStorageProtocol(Protocol):
|
|||
"""Store the given file stream"""
|
||||
|
||||
|
||||
@implementer(IFeedStorage)
|
||||
class BlockingFeedStorage(ABC):
|
||||
def open(self, spider: Spider) -> IO[bytes]:
|
||||
path = spider.crawler.settings["FEED_TEMPDIR"]
|
||||
|
|
@ -156,6 +137,7 @@ class BlockingFeedStorage(ABC):
|
|||
raise NotImplementedError
|
||||
|
||||
|
||||
@implementer(IFeedStorage)
|
||||
class StdoutFeedStorage:
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -182,6 +164,7 @@ class StdoutFeedStorage:
|
|||
pass
|
||||
|
||||
|
||||
@implementer(IFeedStorage)
|
||||
class FileFeedStorage:
|
||||
def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
|
||||
self.path: str = file_uri_to_path(uri) if uri.startswith("file:") else uri
|
||||
|
|
@ -490,7 +473,7 @@ class FeedExporter:
|
|||
)
|
||||
uri = self.settings["FEED_URI"]
|
||||
# handle pathlib.Path objects
|
||||
uri = str(uri.absolute()) if isinstance(uri, Path) else str(uri)
|
||||
uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
|
||||
feed_options = {"format": self.settings["FEED_FORMAT"]}
|
||||
self.feeds[uri] = feed_complete_default_values_from_settings(
|
||||
feed_options, self.settings
|
||||
|
|
@ -502,9 +485,9 @@ class FeedExporter:
|
|||
for settings_uri, feed_options in self.settings.getdict("FEEDS").items():
|
||||
# handle pathlib.Path objects
|
||||
uri = (
|
||||
str(settings_uri.absolute())
|
||||
if isinstance(settings_uri, Path)
|
||||
else str(settings_uri)
|
||||
str(settings_uri)
|
||||
if not isinstance(settings_uri, Path)
|
||||
else settings_uri.absolute().as_uri()
|
||||
)
|
||||
self.feeds[uri] = feed_complete_default_values_from_settings(
|
||||
feed_options, self.settings
|
||||
|
|
@ -531,7 +514,7 @@ class FeedExporter:
|
|||
self.slots.append(
|
||||
self._start_new_batch(
|
||||
batch_id=1,
|
||||
uri=apply_uri_params(uri, uri_params),
|
||||
uri=uri % uri_params,
|
||||
feed_options=feed_options,
|
||||
spider=spider,
|
||||
uri_template=uri,
|
||||
|
|
@ -656,7 +639,7 @@ class FeedExporter:
|
|||
slots.append(
|
||||
self._start_new_batch(
|
||||
batch_id=slot.batch_id + 1,
|
||||
uri=apply_uri_params(slot.uri_template, uri_params),
|
||||
uri=slot.uri_template % uri_params,
|
||||
feed_options=self.feeds[slot.uri_template],
|
||||
spider=spider,
|
||||
uri_template=slot.uri_template,
|
||||
|
|
@ -748,14 +731,3 @@ class FeedExporter:
|
|||
feed_options.get("item_filter", ItemFilter)
|
||||
)
|
||||
return item_filter_class(feed_options)
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any: # pragma: no cover
|
||||
if name == "IFeedStorage":
|
||||
warnings.warn(
|
||||
"scrapy.extensions.feedexport.IFeedStorage is deprecated.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return _IFeedStorage
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
|
|
|||
|
|
@ -1,23 +1,19 @@
|
|||
# pragma: no file cover
|
||||
# pylint: disable=no-method-argument,no-self-argument
|
||||
import warnings
|
||||
|
||||
from zope.interface import Interface
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
|
||||
warnings.warn(
|
||||
"The scrapy.interfaces module is deprecated.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
|
||||
class ISpiderLoader(Interface):
|
||||
def from_settings(settings): ...
|
||||
def from_settings(settings):
|
||||
"""Return an instance of the class for the given settings"""
|
||||
|
||||
def load(spider_name): ...
|
||||
def load(spider_name):
|
||||
"""Return the Spider class for the given spider name. If the spider
|
||||
name is not found, it must raise a KeyError."""
|
||||
|
||||
def list(): ...
|
||||
def list():
|
||||
"""Return a list with the names of all spiders available in the
|
||||
project"""
|
||||
|
||||
def find_by_request(request): ...
|
||||
def find_by_request(request):
|
||||
"""Return the list of spiders names that can handle the given request"""
|
||||
|
|
|
|||
|
|
@ -300,13 +300,13 @@ class GCSFilesStore:
|
|||
)
|
||||
if "storage.objects.get" not in permissions:
|
||||
logger.warning(
|
||||
"No 'storage.objects.get' permission for GCS bucket %(bucket)s. "
|
||||
"No 'storage.objects.get' permission for GSC bucket %(bucket)s. "
|
||||
"Checking if files are up to date will be impossible. Files will be downloaded every time.",
|
||||
{"bucket": bucket},
|
||||
)
|
||||
if "storage.objects.create" not in permissions:
|
||||
logger.error(
|
||||
"No 'storage.objects.create' permission for GCS bucket %(bucket)s. Saving files will be impossible!",
|
||||
"No 'storage.objects.create' permission for GSC bucket %(bucket)s. Saving files will be impossible!",
|
||||
{"bucket": bucket},
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -288,10 +288,10 @@ class BaseSettings(MutableMapping[str, Any]):
|
|||
- ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
|
||||
|
||||
:param name: the setting name
|
||||
:type name: str
|
||||
:type name: string
|
||||
|
||||
:param default: the value to return if no setting is found
|
||||
:type default: object
|
||||
:type default: any
|
||||
"""
|
||||
value = self.get(name, default)
|
||||
if value is None:
|
||||
|
|
|
|||
|
|
@ -154,6 +154,7 @@ __all__ = [
|
|||
"MAIL_TLS",
|
||||
"MAIL_USER",
|
||||
"MEMDEBUG_ENABLED",
|
||||
"MEMDEBUG_NOTIFY",
|
||||
"MEMUSAGE_CHECK_INTERVAL_SECONDS",
|
||||
"MEMUSAGE_ENABLED",
|
||||
"MEMUSAGE_LIMIT_MB",
|
||||
|
|
@ -239,7 +240,7 @@ BOT_NAME = "scrapybot"
|
|||
CLOSESPIDER_ERRORCOUNT = 0
|
||||
CLOSESPIDER_ITEMCOUNT = 0
|
||||
CLOSESPIDER_PAGECOUNT = 0
|
||||
CLOSESPIDER_TIMEOUT = 0.0
|
||||
CLOSESPIDER_TIMEOUT = 0
|
||||
CLOSESPIDER_PAGECOUNT_NO_ITEM = 0
|
||||
CLOSESPIDER_TIMEOUT_NO_ITEM = 0
|
||||
|
||||
|
|
@ -469,6 +470,7 @@ MAIL_SSL = False
|
|||
MAIL_TLS = False
|
||||
|
||||
MEMDEBUG_ENABLED = False # enable memory debugging
|
||||
MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown
|
||||
|
||||
MEMUSAGE_ENABLED = True
|
||||
MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0
|
||||
|
|
|
|||
|
|
@ -5,8 +5,10 @@ import warnings
|
|||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, Protocol, cast
|
||||
|
||||
# working around https://github.com/sphinx-doc/sphinx/issues/10400
|
||||
from scrapy import Request, Spider # noqa: TC001
|
||||
from zope.interface import implementer
|
||||
from zope.interface.verify import verifyClass
|
||||
|
||||
from scrapy.interfaces import ISpiderLoader
|
||||
from scrapy.utils.misc import load_object, walk_modules_iter
|
||||
from scrapy.utils.spider import iter_spider_classes
|
||||
|
||||
|
|
@ -16,6 +18,7 @@ if TYPE_CHECKING:
|
|||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
||||
from scrapy import Request, Spider
|
||||
from scrapy.settings import BaseSettings
|
||||
|
||||
|
||||
|
|
@ -23,31 +26,28 @@ def get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol:
|
|||
"""Get SpiderLoader instance from settings"""
|
||||
cls_path = settings.get("SPIDER_LOADER_CLASS")
|
||||
loader_cls = load_object(cls_path)
|
||||
verifyClass(ISpiderLoader, loader_cls)
|
||||
return cast("SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy()))
|
||||
|
||||
|
||||
class SpiderLoaderProtocol(Protocol):
|
||||
"""Protocol for spider loader implementations.
|
||||
|
||||
See :setting:`SPIDER_LOADER_CLASS`.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, settings: BaseSettings) -> Self:
|
||||
"""Return an instance of the class for the given settings."""
|
||||
"""Return an instance of the class for the given settings"""
|
||||
|
||||
def load(self, spider_name: str) -> type[Spider]:
|
||||
"""Return the spider class for the given spider name. If the spider
|
||||
name is not found, it must raise a :exc:`KeyError`."""
|
||||
"""Return the Spider class for the given spider name. If the spider
|
||||
name is not found, it must raise a KeyError."""
|
||||
|
||||
def list(self) -> list[str]:
|
||||
"""Return a list with the names of all spiders available in the
|
||||
project."""
|
||||
project"""
|
||||
|
||||
def find_by_request(self, request: Request) -> __builtins__.list[str]:
|
||||
"""Return the list of spiders names that can handle the given request."""
|
||||
"""Return the list of spiders names that can handle the given request"""
|
||||
|
||||
|
||||
@implementer(ISpiderLoader)
|
||||
class SpiderLoader:
|
||||
"""
|
||||
SpiderLoader is a class which locates and loads spiders
|
||||
|
|
@ -106,18 +106,12 @@ class SpiderLoader:
|
|||
|
||||
@classmethod
|
||||
def from_settings(cls, settings: BaseSettings) -> Self:
|
||||
"""Create an instance of the class.
|
||||
|
||||
It's called with the current project settings, and it loads the spiders
|
||||
found recursively in the modules of the :setting:`SPIDER_MODULES`
|
||||
setting.
|
||||
"""
|
||||
return cls(settings)
|
||||
|
||||
def load(self, spider_name: str) -> type[Spider]:
|
||||
"""Return the spider class for the given spider name.
|
||||
|
||||
If the spider name is not found, raise a :exc:`KeyError`.
|
||||
"""
|
||||
Return the Spider class for the given spider name. If the spider
|
||||
name is not found, raise a KeyError.
|
||||
"""
|
||||
try:
|
||||
return self._spiders[spider_name]
|
||||
|
|
@ -127,19 +121,19 @@ class SpiderLoader:
|
|||
def find_by_request(self, request: Request) -> list[str]:
|
||||
"""
|
||||
Return the list of spider names that can handle the given request.
|
||||
|
||||
It will try to match the request's url against the domains of
|
||||
the spiders.
|
||||
"""
|
||||
return [
|
||||
name for name, cls in self._spiders.items() if cls.handles_request(request)
|
||||
]
|
||||
|
||||
def list(self) -> list[str]:
|
||||
"""Return a list with the names of all spiders available in the project."""
|
||||
"""
|
||||
Return a list with the names of all spiders available in the project.
|
||||
"""
|
||||
return list(self._spiders.keys())
|
||||
|
||||
|
||||
@implementer(ISpiderLoader)
|
||||
class DummySpiderLoader:
|
||||
"""A dummy spider loader that does not load any spiders."""
|
||||
|
||||
|
|
|
|||
|
|
@ -7,11 +7,9 @@ See documentation in docs/topics/spiders.rst
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.utils.trackref import object_ref
|
||||
from scrapy.utils.url import url_is_from_spider
|
||||
|
|
@ -68,11 +66,6 @@ class Spider(object_ref):
|
|||
can use it directly (e.g. Spider.logger.info('msg')) or use any other
|
||||
Python logger too.
|
||||
"""
|
||||
warnings.warn(
|
||||
"Spider.log() is deprecated, use methods of Spider.logger instead.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
self.logger.log(level, message, **kw)
|
||||
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -23,11 +23,6 @@ class DataAction(argparse.Action):
|
|||
) -> None:
|
||||
value = str(values)
|
||||
value = value.removeprefix("$")
|
||||
# curl merges repeated -d/--data/--data-raw options into a single body
|
||||
# joined with "&"; mirror that instead of keeping only the last one.
|
||||
previous = getattr(namespace, self.dest, None)
|
||||
if previous is not None:
|
||||
value = f"{previous}&{value}"
|
||||
setattr(namespace, self.dest, value)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import sys
|
||||
from importlib.abc import MetaPathFinder
|
||||
from typing import TYPE_CHECKING
|
||||
|
|
@ -48,19 +47,7 @@ class ReactorImportHook(MetaPathFinder):
|
|||
return None
|
||||
|
||||
|
||||
def install_reactor_import_hook() -> ReactorImportHook:
|
||||
"""Prevent importing :mod:`twisted.internet.reactor`.
|
||||
def install_reactor_import_hook() -> None:
|
||||
"""Prevent importing :mod:`twisted.internet.reactor`."""
|
||||
|
||||
The hook is returned and can later be uninstalled with
|
||||
:func:`uninstall_reactor_import_hook()`.
|
||||
"""
|
||||
|
||||
hook = ReactorImportHook()
|
||||
sys.meta_path.insert(0, hook)
|
||||
return hook
|
||||
|
||||
|
||||
def uninstall_reactor_import_hook(hook: ReactorImportHook) -> None:
|
||||
"""Uninstall the hook installed with :func:`install_reactor_import_hook()`."""
|
||||
with contextlib.suppress(ValueError):
|
||||
sys.meta_path.remove(hook)
|
||||
sys.meta_path.insert(0, ReactorImportHook())
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from scrapy.utils.defer import deferred_from_coro
|
|||
|
||||
|
||||
class UppercasePipeline:
|
||||
async def _open_spider(self, spider: Spider) -> None:
|
||||
async def _open_spider(self, spider):
|
||||
spider.logger.info("async pipeline opened!")
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from scrapy.crawler import AsyncCrawlerProcess
|
|||
|
||||
if sys.platform == "win32":
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
asyncioreactor.install()
|
||||
asyncioreactor.install(asyncio.get_event_loop())
|
||||
|
||||
|
||||
class NoRequestsSpider(scrapy.Spider):
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ class CachingHostnameResolverSpider(scrapy.Spider):
|
|||
"""
|
||||
|
||||
name = "caching_hostname_resolver_spider"
|
||||
url: str
|
||||
|
||||
async def start(self):
|
||||
yield scrapy.Request(self.url)
|
||||
|
|
|
|||
|
|
@ -1,29 +0,0 @@
|
|||
import logging
|
||||
import sys
|
||||
|
||||
import scrapy
|
||||
from scrapy.crawler import AsyncCrawlerProcess
|
||||
from scrapy.utils.reactorless import ReactorImportHook
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def hook_count() -> int:
|
||||
return sum(1 for finder in sys.meta_path if isinstance(finder, ReactorImportHook))
|
||||
|
||||
|
||||
class NoRequestsSpider(scrapy.Spider):
|
||||
name = "no_request"
|
||||
|
||||
async def start(self):
|
||||
self.logger.info(f"Hooks during run: {hook_count()}")
|
||||
return
|
||||
yield
|
||||
|
||||
|
||||
for _ in range(2):
|
||||
process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
|
||||
process.crawl(NoRequestsSpider)
|
||||
process.start()
|
||||
|
||||
logger.info(f"Hooks after runs: {hook_count()}")
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
import logging
|
||||
import sys
|
||||
|
||||
import scrapy
|
||||
from scrapy.crawler import AsyncCrawlerProcess
|
||||
from scrapy.utils.reactorless import ReactorImportHook
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class NoRequestsSpider(scrapy.Spider):
|
||||
name = "no_request"
|
||||
|
||||
async def start(self):
|
||||
return
|
||||
yield
|
||||
|
||||
|
||||
process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
|
||||
|
||||
process.crawl(NoRequestsSpider)
|
||||
process.start()
|
||||
|
||||
hook_count = sum(1 for finder in sys.meta_path if isinstance(finder, ReactorImportHook))
|
||||
logger.info(f"Hooks in sys.meta_path after start(): {hook_count}")
|
||||
|
||||
import twisted.internet.reactor # noqa: E402,F401,TID253
|
||||
|
||||
logger.info("Reactor imported after start()")
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
import scrapy
|
||||
from scrapy.crawler import AsyncCrawlerProcess
|
||||
|
||||
|
||||
class NoRequestsSpider(scrapy.Spider):
|
||||
name = "no_request"
|
||||
|
||||
async def start(self):
|
||||
return
|
||||
yield
|
||||
|
||||
|
||||
process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
|
||||
process.crawl(NoRequestsSpider)
|
||||
process.start()
|
||||
|
||||
process2 = AsyncCrawlerProcess()
|
||||
process2.crawl(NoRequestsSpider)
|
||||
process2.start()
|
||||
|
|
@ -9,7 +9,7 @@ from scrapy.utils.defer import deferred_from_coro
|
|||
|
||||
|
||||
class UppercasePipeline:
|
||||
async def _open_spider(self, spider: Spider) -> None:
|
||||
async def _open_spider(self, spider):
|
||||
spider.logger.info("async pipeline opened!")
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ class CachingHostnameResolverSpider(scrapy.Spider):
|
|||
"""
|
||||
|
||||
name = "caching_hostname_resolver_spider"
|
||||
url: str
|
||||
|
||||
async def start(self):
|
||||
yield scrapy.Request(self.url)
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ class SleepingSpider(scrapy.Spider):
|
|||
async def parse(self, response):
|
||||
from twisted.internet import reactor
|
||||
|
||||
d: Deferred[None] = Deferred()
|
||||
d = Deferred()
|
||||
reactor.callLater(int(sys.argv[1]), d.callback, None)
|
||||
await maybe_deferred_to_future(d)
|
||||
|
||||
|
|
|
|||
|
|
@ -36,7 +36,6 @@ def createResolver(servers: list[tuple[str, int]]) -> ResolverBase:
|
|||
|
||||
class LocalhostSpider(Spider):
|
||||
name = "localhost_spider"
|
||||
url: str
|
||||
|
||||
async def start(self):
|
||||
yield Request(self.url)
|
||||
|
|
|
|||
|
|
@ -1,42 +1,11 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import functools
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import time
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from subprocess import DEVNULL, Popen
|
||||
from subprocess import PIPE, Popen
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
from .utils import _free_port
|
||||
|
||||
|
||||
@functools.cache
|
||||
def mitmdump_cmd() -> list[str] | None:
|
||||
"""Return the command prefix used to invoke ``mitmdump``, or ``None`` if it
|
||||
cannot be resolved.
|
||||
|
||||
We don't want to install ``mitmproxy`` into the test env (it has a lot of
|
||||
dependencies that can conflict with some of the Scrapy/test ones, and its
|
||||
newer versions may not support older Python versions). So we expect it
|
||||
installed externally. We look for the ``mitmdump`` binary in the following
|
||||
sources:
|
||||
|
||||
1. the ``MITMDUMP`` environment variable;
|
||||
2. a ``mitmdump`` binary on ``PATH``;
|
||||
3. using ``uvx --from mitmproxy mitmdump`` if ``uvx`` is available.
|
||||
"""
|
||||
if env := os.environ.get("MITMDUMP"):
|
||||
return [env]
|
||||
if path := shutil.which("mitmdump"):
|
||||
return [path]
|
||||
if uvx := shutil.which("uvx"):
|
||||
return [uvx, "--from", "mitmproxy", "mitmdump"]
|
||||
return None
|
||||
|
||||
|
||||
class MitmProxy:
|
||||
auth_user = "scrapy"
|
||||
|
|
@ -46,22 +15,18 @@ class MitmProxy:
|
|||
self.mode = mode
|
||||
|
||||
def start(self) -> str:
|
||||
cmd = mitmdump_cmd()
|
||||
if not cmd:
|
||||
raise RuntimeError(
|
||||
"mitmdump is not available. Please install mitmproxy or uv."
|
||||
)
|
||||
script = """
|
||||
import sys
|
||||
from mitmproxy.tools.main import mitmdump
|
||||
sys.argv[0] = "mitmdump"
|
||||
sys.exit(mitmdump())
|
||||
"""
|
||||
cert_path = Path(__file__).parent.parent.resolve() / "keys"
|
||||
# Choose a free port ourselves instead of reading the mitmdump output
|
||||
# as there is no easy way to disable stdout buffering for all kinds of
|
||||
# mitmdump installs that we support.
|
||||
host = "127.0.0.1"
|
||||
port = _free_port()
|
||||
args = [
|
||||
"--listen-host",
|
||||
host,
|
||||
"127.0.0.1",
|
||||
"--listen-port",
|
||||
str(port),
|
||||
"0",
|
||||
"--proxyauth",
|
||||
f"{self.auth_user}:{self.auth_pass}",
|
||||
"--set",
|
||||
|
|
@ -72,39 +37,30 @@ class MitmProxy:
|
|||
]
|
||||
if self.mode:
|
||||
args += ["--mode", self.mode]
|
||||
self.proc: Popen[bytes] = Popen(
|
||||
[*cmd, *args],
|
||||
stdout=DEVNULL,
|
||||
stderr=DEVNULL,
|
||||
start_new_session=True, # needed for killpg() to make sense
|
||||
self.proc: Popen[str] = Popen(
|
||||
[
|
||||
sys.executable,
|
||||
"-u",
|
||||
"-c",
|
||||
script,
|
||||
*args,
|
||||
],
|
||||
stdout=PIPE,
|
||||
text=True,
|
||||
)
|
||||
assert self.proc.stdout is not None
|
||||
scheme = "socks5" if self.mode == "socks5" else "http"
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
if self.proc.poll() is not None:
|
||||
raise RuntimeError(
|
||||
f"mitmdump exited with code {self.proc.returncode} before it "
|
||||
f"started listening"
|
||||
)
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=1):
|
||||
return f"{scheme}://{self.auth_user}:{self.auth_pass}@{host}:{port}"
|
||||
except OSError:
|
||||
if time.monotonic() >= deadline:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
line = ""
|
||||
for line in self.proc.stdout:
|
||||
m = re.search(r"listening at (?:\w+://)?([^:]+:\d+)", line)
|
||||
if m:
|
||||
host_port = m.group(1)
|
||||
return f"{scheme}://{self.auth_user}:{self.auth_pass}@{host_port}"
|
||||
self.stop()
|
||||
raise RuntimeError(f"mitmdump did not start listening on {host}:{port} in time")
|
||||
raise RuntimeError(f"Failed to parse mitmdump output: {line}")
|
||||
|
||||
def stop(self) -> None:
|
||||
if os.name == "posix":
|
||||
# SIGKILL doesn't propagate to the actual process (child of uvx)
|
||||
# https://github.com/astral-sh/uv/issues/11817#issuecomment-2688830077
|
||||
# https://stackoverflow.com/a/61980200/113586
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.killpg(os.getpgid(self.proc.pid), signal.SIGKILL)
|
||||
else:
|
||||
self.proc.kill()
|
||||
self.proc.kill()
|
||||
self.proc.communicate()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
|
|
@ -19,13 +18,6 @@ if TYPE_CHECKING:
|
|||
from twisted.internet.interfaces import IOpenSSLContextFactory
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
# racy but should be fine for tests
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
return cast("int", s.getsockname()[1])
|
||||
|
||||
|
||||
def ssl_context_factory(
|
||||
keyfile: str = "keys/localhost.key",
|
||||
certfile: str = "keys/localhost.crt",
|
||||
|
|
|
|||
|
|
@ -571,19 +571,3 @@ class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider):
|
|||
def headers_received(self, headers, body_length, request, spider):
|
||||
self.meta["headers_received"] = headers
|
||||
raise StopDownload(fail=True)
|
||||
|
||||
|
||||
class ExceptionSpider(Spider):
|
||||
name = "exception"
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
raise ValueError("Exception in from_crawler method")
|
||||
|
||||
|
||||
class NoRequestsSpider(Spider):
|
||||
name = "no_request"
|
||||
|
||||
async def start(self):
|
||||
return
|
||||
yield
|
||||
|
|
|
|||
|
|
@ -107,11 +107,11 @@ class TestAddonManager:
|
|||
addonlist = []
|
||||
for i in range(3):
|
||||
addon = get_addon_cls({"KEY1": i})
|
||||
addon.number = i # type: ignore[attr-defined]
|
||||
addon.number = i
|
||||
addonlist.append(addon)
|
||||
# Test for every possible ordering
|
||||
for ordered_addons in itertools.permutations(addonlist):
|
||||
expected_order = [a.number for a in ordered_addons] # type: ignore[attr-defined]
|
||||
expected_order = [a.number for a in ordered_addons]
|
||||
settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
manager = crawler.addons
|
||||
|
|
@ -177,7 +177,7 @@ class TestAddonManager:
|
|||
)
|
||||
settings["SCHEDULER"] = "AddonScheduler"
|
||||
|
||||
settings_dict: dict[str, Any] = {
|
||||
settings_dict = {
|
||||
"ADDONS": {AddonWithFallback: 1},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
|
|
|
|||
|
|
@ -1,24 +1,21 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from subprocess import PIPE, Popen
|
||||
|
||||
|
||||
def _execute(spname: str) -> int:
|
||||
args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname)
|
||||
cwd = Path(__file__).resolve().parent
|
||||
proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)
|
||||
proc.communicate()
|
||||
return proc.returncode
|
||||
class TestCmdlineCrawlPipeline:
|
||||
def _execute(self, spname):
|
||||
args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname)
|
||||
cwd = Path(__file__).resolve().parent
|
||||
proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)
|
||||
_, stderr = proc.communicate()
|
||||
return proc.returncode, stderr
|
||||
|
||||
def test_open_spider_normally_in_pipeline(self):
|
||||
returncode, _ = self._execute("normal")
|
||||
assert returncode == 0
|
||||
|
||||
def test_open_spider_normally_in_pipeline():
|
||||
returncode = _execute("normal")
|
||||
assert returncode == 0
|
||||
|
||||
|
||||
def test_exception_at_open_spider_in_pipeline():
|
||||
returncode = _execute("exception")
|
||||
# An exception in pipeline's open_spider should result in a non-zero exit code
|
||||
assert returncode == 1
|
||||
def test_exception_at_open_spider_in_pipeline(self):
|
||||
returncode, _ = self._execute("exception")
|
||||
# An exception in pipeline's open_spider should result in a non-zero exit code
|
||||
assert returncode == 1
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from unittest import TestCase
|
|||
from unittest.mock import MagicMock, Mock, PropertyMock, call, patch
|
||||
|
||||
from scrapy.commands.check import Command, TextTestResult
|
||||
from tests.utils.base_commands import TestProjectBase
|
||||
from tests.test_commands import TestProjectBase
|
||||
from tests.utils.cmdline import proc
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
|||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from tests.utils.base_commands import TestProjectBase
|
||||
from tests.test_commands import TestProjectBase
|
||||
from tests.utils.cmdline import proc
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
|
|||
|
|
@ -6,8 +6,15 @@ from pathlib import Path
|
|||
|
||||
import pytest
|
||||
|
||||
from tests.utils.base_commands import TestProjectBase
|
||||
from tests.utils.cmdline import call, proc, write_recording_editor
|
||||
from tests.test_commands import TestProjectBase
|
||||
from tests.utils.cmdline import call, proc
|
||||
|
||||
|
||||
def write_recording_editor(editor: Path) -> None:
|
||||
"""Create an executable editor script that writes the path it is asked to
|
||||
open (its last argument) into the file given as its first argument."""
|
||||
editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")
|
||||
editor.chmod(0o755)
|
||||
|
||||
|
||||
def find_in_file(filename: Path, regex: str) -> re.Match[str] | None:
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import pytest
|
|||
|
||||
from scrapy.commands import parse
|
||||
from scrapy.settings import Settings
|
||||
from tests.utils.base_commands import TestProjectBase
|
||||
from tests.test_commands import TestProjectBase
|
||||
from tests.utils.cmdline import call, proc
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from typing import TYPE_CHECKING
|
|||
|
||||
import pytest
|
||||
|
||||
from tests.spiders import ExceptionSpider, NoRequestsSpider
|
||||
from tests.test_crawler import ExceptionSpider, NoRequestsSpider
|
||||
from tests.utils.cmdline import proc
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -65,14 +65,13 @@ class BadSpider(scrapy.Spider):
|
|||
|
||||
def test_run_fail_spider(self, tmp_path: Path) -> None:
|
||||
ret, _, _ = self.runspider(
|
||||
tmp_path, "from scrapy import Spider\n" + inspect.getsource(ExceptionSpider)
|
||||
tmp_path, "import scrapy\n" + inspect.getsource(ExceptionSpider)
|
||||
)
|
||||
assert ret != 0
|
||||
|
||||
def test_run_good_spider(self, tmp_path: Path) -> None:
|
||||
ret, _, _ = self.runspider(
|
||||
tmp_path,
|
||||
"from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider),
|
||||
tmp_path, "import scrapy\n" + inspect.getsource(NoRequestsSpider)
|
||||
)
|
||||
assert ret == 0
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import argparse
|
|||
import json
|
||||
import sys
|
||||
from io import StringIO
|
||||
from shutil import copytree
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest import mock
|
||||
|
||||
|
|
@ -15,8 +16,7 @@ from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view
|
|||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.reactor import _asyncio_reactor_path
|
||||
from tests.utils.base_commands import TestProjectBase
|
||||
from tests.utils.cmdline import call, proc, write_recording_editor
|
||||
from tests.utils.cmdline import call, proc
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
|
@ -89,7 +89,6 @@ class TestCommandSettings:
|
|||
args=["-s", f"FEEDS={feeds_json}", "spider.py"]
|
||||
)
|
||||
self.command.process_options(args, opts)
|
||||
assert self.command.settings is not None
|
||||
assert isinstance(self.command.settings["FEEDS"], scrapy.settings.BaseSettings)
|
||||
assert dict(self.command.settings["FEEDS"]) == json.loads(feeds_json)
|
||||
|
||||
|
|
@ -109,6 +108,29 @@ class TestCommandSettings:
|
|||
)
|
||||
|
||||
|
||||
class TestProjectBase:
|
||||
"""A base class for tests that may need a Scrapy project."""
|
||||
|
||||
project_name = "testproject"
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def _proj_path_cached(self, tmp_path_factory: pytest.TempPathFactory) -> Path:
|
||||
"""Create a Scrapy project in a temporary directory and return its path.
|
||||
|
||||
Used as a cache for ``proj_path``.
|
||||
"""
|
||||
tmp_path = tmp_path_factory.mktemp("proj")
|
||||
call("startproject", self.project_name, cwd=tmp_path)
|
||||
return tmp_path / self.project_name
|
||||
|
||||
@pytest.fixture
|
||||
def proj_path(self, tmp_path: Path, _proj_path_cached: Path) -> Path:
|
||||
"""Copy a pre-generated Scrapy project into a temporary directory and return its path."""
|
||||
proj_path = tmp_path / self.project_name
|
||||
copytree(_proj_path_cached, proj_path)
|
||||
return proj_path
|
||||
|
||||
|
||||
class TestCommandCrawlerProcess(TestProjectBase):
|
||||
"""Test that the command uses the expected kind of *CrawlerProcess
|
||||
and produces expected errors when needed."""
|
||||
|
|
@ -407,7 +429,9 @@ class TestEditCommand(TestProjectBase):
|
|||
spider = proj_path / self.project_name / "spiders" / "example.py"
|
||||
edited = proj_path / "edited.txt"
|
||||
editor = proj_path / "fake-editor.sh"
|
||||
write_recording_editor(editor)
|
||||
# Records the file it is asked to open ($2) into the file given as $1.
|
||||
editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")
|
||||
editor.chmod(0o755)
|
||||
monkeypatch.setenv("EDITOR", f"{editor} {edited}")
|
||||
|
||||
assert call("genspider", "example", "example.com", cwd=proj_path) == 0
|
||||
|
|
|
|||
|
|
@ -14,13 +14,14 @@ from twisted.internet.ssl import Certificate
|
|||
from twisted.python.failure import Failure
|
||||
|
||||
from scrapy import Spider, signals
|
||||
from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunner
|
||||
from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload
|
||||
from scrapy.http import Request
|
||||
from scrapy.http.response import Response
|
||||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
|
||||
from scrapy.utils.engine import format_engine_status, get_engine_status
|
||||
from scrapy.utils.python import to_unicode
|
||||
from scrapy.utils.test import get_crawler
|
||||
from scrapy.utils.test import get_crawler, get_reactor_settings
|
||||
from tests import NON_EXISTING_RESOLVABLE
|
||||
from tests.spiders import (
|
||||
AsyncDefAsyncioGenComplexSpider,
|
||||
|
|
@ -355,10 +356,9 @@ with multiples lines
|
|||
|
||||
@coroutine_test
|
||||
async def test_engine_status(self, mockserver: MockServer) -> None:
|
||||
est: list[list[tuple[str, Any]]] = []
|
||||
est = []
|
||||
|
||||
def cb(response):
|
||||
assert crawler.engine
|
||||
est.append(get_engine_status(crawler.engine))
|
||||
|
||||
crawler = get_crawler(SingleRequestSpider)
|
||||
|
|
@ -373,10 +373,9 @@ with multiples lines
|
|||
|
||||
@coroutine_test
|
||||
async def test_format_engine_status(self, mockserver: MockServer) -> None:
|
||||
est: list[str] = []
|
||||
est = []
|
||||
|
||||
def cb(response):
|
||||
assert crawler.engine
|
||||
est.append(format_engine_status(crawler.engine))
|
||||
|
||||
crawler = get_crawler(SingleRequestSpider)
|
||||
|
|
@ -387,8 +386,8 @@ with multiples lines
|
|||
assert len(est) == 1, est
|
||||
est = est[0].split("\n")[2:-2] # remove header & footer
|
||||
# convert to dict
|
||||
est_split = [x.split(":") for x in est]
|
||||
est = [x for sublist in est_split for x in sublist] # flatten
|
||||
est = [x.split(":") for x in est]
|
||||
est = [x for sublist in est for x in sublist] # flatten
|
||||
est = [x.lstrip().rstrip() for x in est]
|
||||
it = iter(est)
|
||||
s = dict(zip(it, it, strict=True))
|
||||
|
|
@ -429,6 +428,50 @@ with multiples lines
|
|||
)
|
||||
assert not crawler.crawling
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawlerrunner_accepts_crawler(
|
||||
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
|
||||
) -> None:
|
||||
crawler = Crawler(SimpleSpider, get_reactor_settings())
|
||||
runner = CrawlerRunner()
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
await maybe_deferred_to_future(
|
||||
runner.crawl(
|
||||
crawler,
|
||||
mockserver.url("/status?n=200"),
|
||||
mockserver=mockserver,
|
||||
)
|
||||
)
|
||||
assert "Got response 200" in caplog.text
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawl_multiple(
|
||||
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
|
||||
) -> None:
|
||||
settings_dict = get_reactor_settings()
|
||||
runner_cls = (
|
||||
CrawlerRunner
|
||||
if settings_dict.get("TWISTED_REACTOR_ENABLED", True)
|
||||
else AsyncCrawlerRunner
|
||||
)
|
||||
runner = runner_cls(settings_dict)
|
||||
runner.crawl(
|
||||
SimpleSpider,
|
||||
mockserver.url("/status?n=200"),
|
||||
mockserver=mockserver,
|
||||
)
|
||||
runner.crawl(
|
||||
SimpleSpider,
|
||||
mockserver.url("/status?n=503"),
|
||||
mockserver=mockserver,
|
||||
)
|
||||
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
await ensure_awaitable(runner.join())
|
||||
|
||||
self._assert_retried(caplog.text)
|
||||
assert "Got response 200" in caplog.text
|
||||
|
||||
@coroutine_test
|
||||
async def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None:
|
||||
crawler = get_crawler(SimpleSpider)
|
||||
|
|
|
|||
|
|
@ -10,14 +10,22 @@ from typing import TYPE_CHECKING, Any, ClassVar
|
|||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from zope.interface.exceptions import MultipleInvalid
|
||||
|
||||
import scrapy
|
||||
from scrapy import Spider
|
||||
from scrapy.crawler import AsyncCrawlerProcess, Crawler, CrawlerProcess
|
||||
from scrapy.crawler import (
|
||||
AsyncCrawlerProcess,
|
||||
AsyncCrawlerRunner,
|
||||
Crawler,
|
||||
CrawlerProcess,
|
||||
CrawlerRunner,
|
||||
CrawlerRunnerBase,
|
||||
)
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.extensions.throttle import AutoThrottle
|
||||
from scrapy.settings import Settings, default_settings
|
||||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
|
||||
from scrapy.utils.log import (
|
||||
_uninstall_scrapy_root_handler,
|
||||
configure_logging,
|
||||
|
|
@ -25,8 +33,6 @@ from scrapy.utils.log import (
|
|||
)
|
||||
from scrapy.utils.spider import DefaultSpider
|
||||
from scrapy.utils.test import get_crawler, get_reactor_settings
|
||||
from tests.spiders import NoRequestsSpider
|
||||
from tests.utils import assert_option_is_default
|
||||
from tests.utils.decorators import coroutine_test
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -46,7 +52,14 @@ def get_raw_crawler(
|
|||
return Crawler(spidercls or DefaultSpider, settings)
|
||||
|
||||
|
||||
class TestCrawler:
|
||||
class TestBaseCrawler:
|
||||
@staticmethod
|
||||
def assertOptionIsDefault(settings: Settings, key: str) -> None:
|
||||
assert isinstance(settings, Settings)
|
||||
assert settings[key] == getattr(default_settings, key)
|
||||
|
||||
|
||||
class TestCrawler(TestBaseCrawler):
|
||||
def test_populate_spidercls_settings(self) -> None:
|
||||
spider_settings: dict[str, Any] = {
|
||||
"TEST1": "spider",
|
||||
|
|
@ -77,11 +90,11 @@ class TestCrawler:
|
|||
def test_crawler_accepts_dict(self) -> None:
|
||||
crawler = get_crawler(DefaultSpider, {"foo": "bar"})
|
||||
assert crawler.settings["foo"] == "bar"
|
||||
assert_option_is_default(crawler.settings, "RETRY_ENABLED")
|
||||
self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
|
||||
|
||||
def test_crawler_accepts_None(self) -> None:
|
||||
crawler = Crawler(DefaultSpider)
|
||||
assert_option_is_default(crawler.settings, "RETRY_ENABLED")
|
||||
self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
|
||||
|
||||
def test_crawler_rejects_spider_objects(self) -> None:
|
||||
with pytest.raises(ValueError, match="spidercls argument must be a class"):
|
||||
|
|
@ -510,7 +523,6 @@ class TestCrawlerLogging:
|
|||
}
|
||||
|
||||
async def start(self):
|
||||
assert crawler.stats
|
||||
info_count_start = crawler.stats.get_value("log_count/INFO")
|
||||
logging.debug("debug message") # noqa: LOG015
|
||||
logging.info("info message") # noqa: LOG015
|
||||
|
|
@ -573,6 +585,78 @@ class TestCrawlerLogging:
|
|||
assert "debug message" in logged
|
||||
|
||||
|
||||
class SpiderLoaderWithWrongInterface:
|
||||
def unneeded_method(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class TestCrawlerRunner(TestBaseCrawler):
|
||||
def test_spider_manager_verify_interface(self) -> None:
|
||||
settings = Settings(
|
||||
{
|
||||
"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
|
||||
}
|
||||
)
|
||||
with pytest.raises(MultipleInvalid):
|
||||
CrawlerRunner(settings)
|
||||
|
||||
def test_crawler_runner_accepts_dict(self) -> None:
|
||||
runner = CrawlerRunner({"foo": "bar"})
|
||||
assert runner.settings["foo"] == "bar"
|
||||
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
def test_crawler_runner_accepts_None(self) -> None:
|
||||
runner = CrawlerRunner()
|
||||
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
|
||||
class TestAsyncCrawlerRunner(TestBaseCrawler):
|
||||
def test_spider_manager_verify_interface(self) -> None:
|
||||
settings = Settings(
|
||||
{
|
||||
"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
|
||||
}
|
||||
)
|
||||
with pytest.raises(MultipleInvalid):
|
||||
AsyncCrawlerRunner(settings)
|
||||
|
||||
def test_crawler_runner_accepts_dict(self) -> None:
|
||||
runner = AsyncCrawlerRunner({"foo": "bar"})
|
||||
assert runner.settings["foo"] == "bar"
|
||||
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
def test_crawler_runner_accepts_None(self) -> None:
|
||||
runner = AsyncCrawlerRunner()
|
||||
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
|
||||
class TestCrawlerProcess(TestBaseCrawler):
|
||||
def test_crawler_process_accepts_dict(self) -> None:
|
||||
runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False)
|
||||
assert runner.settings["foo"] == "bar"
|
||||
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
def test_crawler_process_accepts_None(self) -> None:
|
||||
runner = CrawlerProcess(install_root_handler=False)
|
||||
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
class TestAsyncCrawlerProcess(TestBaseCrawler):
|
||||
def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None:
|
||||
runner = AsyncCrawlerProcess(
|
||||
{"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"},
|
||||
install_root_handler=False,
|
||||
)
|
||||
assert runner.settings["foo"] == "bar"
|
||||
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
@pytest.mark.requires_reactor # can't pass TWISTED_REACTOR_ENABLED=False
|
||||
def test_crawler_process_accepts_None(self) -> None:
|
||||
runner = AsyncCrawlerProcess(install_root_handler=False)
|
||||
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
|
||||
class TestAsyncCrawlerProcessReactorlessHelpers:
|
||||
"""Unit tests for the reactorless shutdown helpers of AsyncCrawlerProcess.
|
||||
|
||||
|
|
@ -712,6 +796,161 @@ class TestAsyncCrawlerProcessReactorlessHelpers:
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
|
||||
def test_runner_settings_applied_to_crawler_instance(
|
||||
runner_cls: type[CrawlerRunnerBase],
|
||||
) -> None:
|
||||
runner = runner_cls({"FOO": "runner"})
|
||||
crawler = Crawler(DefaultSpider)
|
||||
result = runner.create_crawler(crawler)
|
||||
assert result is crawler
|
||||
assert result.settings["FOO"] == "runner"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
|
||||
def test_spider_custom_settings_override_runner(
|
||||
runner_cls: type[CrawlerRunnerBase],
|
||||
) -> None:
|
||||
class MySpider(DefaultSpider):
|
||||
custom_settings = {"FOO": "spider"}
|
||||
|
||||
runner = runner_cls({"FOO": "runner"})
|
||||
crawler = Crawler(MySpider)
|
||||
runner.create_crawler(crawler)
|
||||
assert crawler.settings["FOO"] == "spider"
|
||||
|
||||
|
||||
def test_create_crawler_instance_consistent_with_spider_class() -> None:
|
||||
runner = AsyncCrawlerRunner({"FOO": "runner"})
|
||||
|
||||
crawler_from_class = runner.create_crawler(DefaultSpider)
|
||||
|
||||
pre_built = Crawler(DefaultSpider)
|
||||
runner.create_crawler(pre_built)
|
||||
|
||||
assert crawler_from_class.settings["FOO"] == "runner"
|
||||
assert pre_built.settings["FOO"] == "runner"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
|
||||
def test_create_crawler_rejects_spider_object(
|
||||
runner_cls: type[CrawlerRunnerBase],
|
||||
) -> None:
|
||||
runner = runner_cls()
|
||||
with pytest.raises(ValueError, match="cannot be a spider object"):
|
||||
runner.create_crawler(DefaultSpider()) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
|
||||
def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None:
|
||||
runner = runner_cls()
|
||||
with pytest.raises(ValueError, match="cannot be a spider object"):
|
||||
runner.crawl(DefaultSpider()) # type: ignore[arg-type]
|
||||
|
||||
|
||||
class ExceptionSpider(scrapy.Spider):
|
||||
name = "exception"
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
raise ValueError("Exception in from_crawler method")
|
||||
|
||||
|
||||
class NoRequestsSpider(scrapy.Spider):
|
||||
name = "no_request"
|
||||
|
||||
async def start(self):
|
||||
return
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.requires_reactor # CrawlerRunner requires a reactor
|
||||
class TestCrawlerRunnerHasSpider:
|
||||
@pytest.fixture
|
||||
def runner(self) -> CrawlerRunnerBase:
|
||||
return CrawlerRunner(get_reactor_settings())
|
||||
|
||||
@staticmethod
|
||||
async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None:
|
||||
await ensure_awaitable(runner.crawl(spider))
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_bootstrap_successful(
|
||||
self, runner: CrawlerRunnerBase
|
||||
) -> None:
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
assert not runner.bootstrap_failed
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_bootstrap_successful_for_several(
|
||||
self, runner: CrawlerRunnerBase
|
||||
) -> None:
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
assert not runner.bootstrap_failed
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_bootstrap_failed(
|
||||
self, runner: CrawlerRunnerBase
|
||||
) -> None:
|
||||
try:
|
||||
await self._crawl(runner, ExceptionSpider)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
pytest.fail("Exception should be raised from spider")
|
||||
|
||||
assert runner.bootstrap_failed
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_bootstrap_failed_for_several(
|
||||
self, runner: CrawlerRunnerBase
|
||||
) -> None:
|
||||
try:
|
||||
await self._crawl(runner, ExceptionSpider)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
pytest.fail("Exception should be raised from spider")
|
||||
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
|
||||
assert runner.bootstrap_failed
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_asyncio_enabled_true(
|
||||
self, reactor_pytest: str
|
||||
) -> None:
|
||||
if reactor_pytest != "asyncio":
|
||||
runner = CrawlerRunner(
|
||||
settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
}
|
||||
)
|
||||
with pytest.raises(
|
||||
Exception,
|
||||
match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",
|
||||
):
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
else:
|
||||
runner = CrawlerRunner(
|
||||
settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
}
|
||||
)
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):
|
||||
@pytest.fixture
|
||||
def runner(self) -> CrawlerRunnerBase:
|
||||
return AsyncCrawlerRunner(get_reactor_settings())
|
||||
|
||||
def test_crawler_runner_asyncio_enabled_true(self) -> None: # type: ignore[override]
|
||||
pytest.skip("This test is only for CrawlerRunner")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("settings", "items"),
|
||||
[
|
||||
|
|
|
|||
|
|
@ -1,264 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from scrapy.crawler import (
|
||||
AsyncCrawlerProcess,
|
||||
AsyncCrawlerRunner,
|
||||
Crawler,
|
||||
CrawlerProcess,
|
||||
CrawlerRunner,
|
||||
CrawlerRunnerBase,
|
||||
)
|
||||
from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
|
||||
from scrapy.utils.spider import DefaultSpider
|
||||
from scrapy.utils.test import get_reactor_settings
|
||||
from tests.spiders import ExceptionSpider, NoRequestsSpider, SimpleSpider
|
||||
from tests.utils import assert_option_is_default
|
||||
from tests.utils.decorators import coroutine_test
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy import Spider
|
||||
from tests.mockserver.http import MockServer
|
||||
|
||||
|
||||
class SpiderLoaderWithWrongInterface:
|
||||
def unneeded_method(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class TestCrawlerRunner:
|
||||
def test_crawler_runner_accepts_dict(self) -> None:
|
||||
runner = CrawlerRunner({"foo": "bar"})
|
||||
assert runner.settings["foo"] == "bar"
|
||||
assert_option_is_default(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
def test_crawler_runner_accepts_None(self) -> None:
|
||||
runner = CrawlerRunner()
|
||||
assert_option_is_default(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
|
||||
class TestAsyncCrawlerRunner:
|
||||
def test_crawler_runner_accepts_dict(self) -> None:
|
||||
runner = AsyncCrawlerRunner({"foo": "bar"})
|
||||
assert runner.settings["foo"] == "bar"
|
||||
assert_option_is_default(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
def test_crawler_runner_accepts_None(self) -> None:
|
||||
runner = AsyncCrawlerRunner()
|
||||
assert_option_is_default(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
|
||||
class TestCrawlerProcess:
|
||||
def test_crawler_process_accepts_dict(self) -> None:
|
||||
runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False)
|
||||
assert runner.settings["foo"] == "bar"
|
||||
assert_option_is_default(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
def test_crawler_process_accepts_None(self) -> None:
|
||||
runner = CrawlerProcess(install_root_handler=False)
|
||||
assert_option_is_default(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
class TestAsyncCrawlerProcess:
|
||||
def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None:
|
||||
runner = AsyncCrawlerProcess(
|
||||
{"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"},
|
||||
install_root_handler=False,
|
||||
)
|
||||
assert runner.settings["foo"] == "bar"
|
||||
assert_option_is_default(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
@pytest.mark.requires_reactor # can't pass TWISTED_REACTOR_ENABLED=False
|
||||
def test_crawler_process_accepts_None(self) -> None:
|
||||
runner = AsyncCrawlerProcess(install_root_handler=False)
|
||||
assert_option_is_default(runner.settings, "RETRY_ENABLED")
|
||||
|
||||
|
||||
@pytest.mark.requires_reactor # CrawlerRunner requires a reactor
|
||||
class TestCrawlerRunnerHasSpider:
|
||||
@pytest.fixture
|
||||
def runner(self) -> CrawlerRunnerBase:
|
||||
return CrawlerRunner(get_reactor_settings())
|
||||
|
||||
@staticmethod
|
||||
async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None:
|
||||
await ensure_awaitable(runner.crawl(spider))
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_bootstrap_successful(
|
||||
self, runner: CrawlerRunnerBase
|
||||
) -> None:
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
assert not runner.bootstrap_failed
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_bootstrap_successful_for_several(
|
||||
self, runner: CrawlerRunnerBase
|
||||
) -> None:
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
assert not runner.bootstrap_failed
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_bootstrap_failed(
|
||||
self, runner: CrawlerRunnerBase
|
||||
) -> None:
|
||||
try:
|
||||
await self._crawl(runner, ExceptionSpider)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
pytest.fail("Exception should be raised from spider")
|
||||
|
||||
assert runner.bootstrap_failed
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_bootstrap_failed_for_several(
|
||||
self, runner: CrawlerRunnerBase
|
||||
) -> None:
|
||||
try:
|
||||
await self._crawl(runner, ExceptionSpider)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
pytest.fail("Exception should be raised from spider")
|
||||
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
|
||||
assert runner.bootstrap_failed
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawler_runner_asyncio_enabled_true(
|
||||
self, reactor_pytest: str
|
||||
) -> None:
|
||||
if reactor_pytest != "asyncio":
|
||||
runner = CrawlerRunner(
|
||||
settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
}
|
||||
)
|
||||
with pytest.raises(
|
||||
Exception,
|
||||
match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",
|
||||
):
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
else:
|
||||
runner = CrawlerRunner(
|
||||
settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
}
|
||||
)
|
||||
await self._crawl(runner, NoRequestsSpider)
|
||||
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):
|
||||
@pytest.fixture
|
||||
def runner(self) -> CrawlerRunnerBase:
|
||||
return AsyncCrawlerRunner(get_reactor_settings())
|
||||
|
||||
def test_crawler_runner_asyncio_enabled_true(self) -> None: # type: ignore[override]
|
||||
pytest.skip("This test is only for CrawlerRunner")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
|
||||
def test_runner_settings_applied_to_crawler_instance(
|
||||
runner_cls: type[CrawlerRunnerBase],
|
||||
) -> None:
|
||||
runner = runner_cls({"FOO": "runner"})
|
||||
crawler = Crawler(DefaultSpider)
|
||||
result = runner.create_crawler(crawler)
|
||||
assert result is crawler
|
||||
assert result.settings["FOO"] == "runner"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
|
||||
def test_spider_custom_settings_override_runner(
|
||||
runner_cls: type[CrawlerRunnerBase],
|
||||
) -> None:
|
||||
class MySpider(DefaultSpider):
|
||||
custom_settings = {"FOO": "spider"}
|
||||
|
||||
runner = runner_cls({"FOO": "runner"})
|
||||
crawler = Crawler(MySpider)
|
||||
runner.create_crawler(crawler)
|
||||
assert crawler.settings["FOO"] == "spider"
|
||||
|
||||
|
||||
def test_create_crawler_instance_consistent_with_spider_class() -> None:
|
||||
runner = AsyncCrawlerRunner({"FOO": "runner"})
|
||||
|
||||
crawler_from_class = runner.create_crawler(DefaultSpider)
|
||||
|
||||
pre_built = Crawler(DefaultSpider)
|
||||
runner.create_crawler(pre_built)
|
||||
|
||||
assert crawler_from_class.settings["FOO"] == "runner"
|
||||
assert pre_built.settings["FOO"] == "runner"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
|
||||
def test_create_crawler_rejects_spider_object(
|
||||
runner_cls: type[CrawlerRunnerBase],
|
||||
) -> None:
|
||||
runner = runner_cls()
|
||||
with pytest.raises(ValueError, match="cannot be a spider object"):
|
||||
runner.create_crawler(DefaultSpider()) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
|
||||
def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None:
|
||||
runner = runner_cls()
|
||||
with pytest.raises(ValueError, match="cannot be a spider object"):
|
||||
runner.crawl(DefaultSpider()) # type: ignore[arg-type]
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawlerrunner_accepts_crawler(
|
||||
caplog: pytest.LogCaptureFixture, mockserver: MockServer
|
||||
) -> None:
|
||||
crawler = Crawler(SimpleSpider, get_reactor_settings())
|
||||
runner = CrawlerRunner()
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
await maybe_deferred_to_future(
|
||||
runner.crawl(
|
||||
crawler,
|
||||
mockserver.url("/status?n=200"),
|
||||
mockserver=mockserver,
|
||||
)
|
||||
)
|
||||
assert "Got response 200" in caplog.text
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawl_multiple(
|
||||
caplog: pytest.LogCaptureFixture, mockserver: MockServer
|
||||
) -> None:
|
||||
settings_dict = get_reactor_settings()
|
||||
runner_cls = (
|
||||
CrawlerRunner
|
||||
if settings_dict.get("TWISTED_REACTOR_ENABLED", True)
|
||||
else AsyncCrawlerRunner
|
||||
)
|
||||
runner = runner_cls(settings_dict)
|
||||
runner.crawl(
|
||||
SimpleSpider,
|
||||
mockserver.url("/status?n=200"),
|
||||
mockserver=mockserver,
|
||||
)
|
||||
runner.crawl(
|
||||
SimpleSpider,
|
||||
mockserver.url("/status?n=503"),
|
||||
mockserver=mockserver,
|
||||
)
|
||||
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
await ensure_awaitable(runner.join())
|
||||
|
||||
assert "Got response 200" in caplog.text
|
||||
assert "Gave up retrying" in caplog.text
|
||||
|
|
@ -352,7 +352,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
|
|||
) in log
|
||||
|
||||
@pytest.mark.requires_uvloop
|
||||
def test_asyncio_custom_loop_custom_settings_same(self) -> None:
|
||||
def test_asyncio_enabled_reactor_same_loop(self) -> None:
|
||||
log = self.run_script("asyncio_custom_loop_custom_settings_same.py")
|
||||
assert "Spider closed (finished)" in log
|
||||
assert (
|
||||
|
|
@ -362,7 +362,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
|
|||
assert "Using asyncio event loop: uvloop.Loop" in log
|
||||
|
||||
@pytest.mark.requires_uvloop
|
||||
def test_asyncio_custom_loop_custom_settings_different(self) -> None:
|
||||
def test_asyncio_enabled_reactor_different_loop(self) -> None:
|
||||
log = self.run_script("asyncio_custom_loop_custom_settings_different.py")
|
||||
assert "Spider closed (finished)" not in log
|
||||
assert (
|
||||
|
|
@ -407,38 +407,6 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
|
|||
assert "Spider closed (finished)" in log
|
||||
assert "ImportError: Import of twisted.internet.reactor is forbidden" in log
|
||||
|
||||
def test_reactorless_import_hook_uninstall(self) -> None:
|
||||
"""The import hook is removed when start() returns, so importing
|
||||
twisted.internet.reactor becomes possible again."""
|
||||
log = self.run_script("reactorless_import_hook_uninstall.py")
|
||||
assert "Not using a Twisted reactor" in log
|
||||
assert "Spider closed (finished)" in log
|
||||
assert "Hooks in sys.meta_path after start(): 0" in log
|
||||
assert "Reactor imported after start()" in log
|
||||
assert "ImportError" not in log
|
||||
|
||||
def test_reactorless_import_hook_multiple(self) -> None:
|
||||
"""Sequential AsyncCrawlerProcess instances don't accumulate import
|
||||
hooks: there is exactly one during each run and none afterwards."""
|
||||
log = self.run_script("reactorless_import_hook_multiple.py")
|
||||
assert log.count("Spider closed (finished)") == 2
|
||||
assert log.count("Hooks during run: 1") == 2
|
||||
assert "Hooks after runs: 0" in log
|
||||
assert "ERROR: " not in log
|
||||
|
||||
def test_reactorless_then_reactor(self) -> None:
|
||||
"""After a reactorless run finishes, a reactor-based run is possible
|
||||
in the same process."""
|
||||
log = self.run_script("reactorless_then_reactor.py")
|
||||
assert log.count("Spider closed (finished)") == 2
|
||||
assert "Not using a Twisted reactor" in log
|
||||
assert (
|
||||
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
|
||||
in log
|
||||
)
|
||||
assert "ImportError" not in log
|
||||
assert "ERROR: " not in log
|
||||
|
||||
def test_reactorless_telnetconsole_default(self) -> None:
|
||||
"""By default TWISTED_REACTOR_ENABLED=False silently sets TELNETCONSOLE_ENABLED=False."""
|
||||
log = self.run_script("reactorless_simple.py") # no need for a separate script
|
||||
|
|
@ -469,14 +437,14 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
|
|||
in log
|
||||
)
|
||||
|
||||
def test_reactorless_shutdown_graceful(self) -> None:
|
||||
def test_shutdown_graceful(self) -> None:
|
||||
self._test_shutdown_graceful("reactorless_sleeping.py")
|
||||
|
||||
@coroutine_test
|
||||
async def test_reactorless_shutdown_forced(self) -> None:
|
||||
async def test_shutdown_forced(self) -> None:
|
||||
await self._test_shutdown_forced("reactorless_sleeping.py")
|
||||
|
||||
def test_reactorless_shutdown_graceful_no_stop(self) -> None:
|
||||
def test_shutdown_graceful_reactorless_no_stop(self) -> None:
|
||||
self._test_shutdown_graceful("reactorless_sleeping.py", "--no-stop")
|
||||
|
||||
def test_asyncio_enabled_reactor_same_loop_default(self) -> None:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from configparser import ConfigParser
|
||||
|
|
@ -25,7 +23,6 @@ class TestScrapyUtils:
|
|||
config_parser.read(tox_config_file_path)
|
||||
pattern = r"Twisted==([\d.]+)"
|
||||
match = re.search(pattern, config_parser["min"]["deps"])
|
||||
assert match
|
||||
pinned_twisted_version_string = match[1]
|
||||
|
||||
assert twisted_version.short() == pinned_twisted_version_string
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler
|
|||
from scrapy.core.downloader.handlers.file import FileDownloadHandler
|
||||
from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.http import Request, TextResponse
|
||||
from scrapy.http import Request
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.boto import is_botocore_available
|
||||
from scrapy.utils.misc import build_from_crawler
|
||||
|
|
@ -175,11 +175,7 @@ class TestS3Anon:
|
|||
@coroutine_test
|
||||
async def test_anon_request_insecure(self):
|
||||
req = Request("s3://aws-publicdatasets/", meta={"is_secure": False})
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="Passing is_secure=False for s3:// requests is deprecated",
|
||||
):
|
||||
httpreq = await self.download_request(req)
|
||||
httpreq = await self.download_request(req)
|
||||
assert httpreq.url == "http://aws-publicdatasets.s3.amazonaws.com/"
|
||||
|
||||
|
||||
|
|
@ -219,11 +215,7 @@ class TestS3:
|
|||
@coroutine_test
|
||||
async def test_insecure_opt_out(self):
|
||||
req = Request("s3://johnsmith/photos/puppy.jpg", meta={"is_secure": False})
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="Passing is_secure=False for s3:// requests is deprecated",
|
||||
):
|
||||
httpreq = await self.download_request(req)
|
||||
httpreq = await self.download_request(req)
|
||||
assert httpreq.url == "http://johnsmith.s3.amazonaws.com/photos/puppy.jpg"
|
||||
|
||||
@coroutine_test
|
||||
|
|
@ -362,7 +354,6 @@ class TestDataURI:
|
|||
response = await self.download_request(request)
|
||||
assert response.text == "A brief note"
|
||||
assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
|
||||
assert isinstance(response, TextResponse)
|
||||
assert response.encoding == "US-ASCII"
|
||||
|
||||
@coroutine_test
|
||||
|
|
@ -371,7 +362,6 @@ class TestDataURI:
|
|||
response = await self.download_request(request)
|
||||
assert response.text == "\u038e\u03a3\u038e"
|
||||
assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
|
||||
assert isinstance(response, TextResponse)
|
||||
assert response.encoding == "iso-8859-7"
|
||||
|
||||
@coroutine_test
|
||||
|
|
@ -380,7 +370,6 @@ class TestDataURI:
|
|||
response = await self.download_request(request)
|
||||
assert response.text == "\u038e\u03a3\u038e"
|
||||
assert response.body == b"\xbe\xd3\xbe"
|
||||
assert isinstance(response, TextResponse)
|
||||
assert response.encoding == "iso-8859-7"
|
||||
|
||||
@coroutine_test
|
||||
|
|
@ -393,7 +382,6 @@ class TestDataURI:
|
|||
response = await self.download_request(request)
|
||||
assert response.text == "\u038e\u03a3\u038e"
|
||||
assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck
|
||||
assert isinstance(response, TextResponse)
|
||||
assert response.encoding == "utf-8"
|
||||
|
||||
@coroutine_test
|
||||
|
|
|
|||
|
|
@ -538,6 +538,13 @@ class TestHttpBase(ABC):
|
|||
else:
|
||||
assert latency > 0
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None:
|
||||
request = Request(mockserver.url("/text", is_secure=self.is_secure))
|
||||
async with self.get_dh() as download_handler:
|
||||
response = await download_handler.download_request(request)
|
||||
assert response.body == b"Works"
|
||||
|
||||
@coroutine_test
|
||||
async def test_response_class_choosing_request(
|
||||
self, mockserver: MockServer
|
||||
|
|
@ -553,13 +560,6 @@ class TestHttpBase(ABC):
|
|||
response = await download_handler.download_request(request)
|
||||
assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None:
|
||||
request = Request(mockserver.url("/text", is_secure=self.is_secure))
|
||||
async with self.get_dh({"DOWNLOAD_MAXSIZE": 0}) as download_handler:
|
||||
response = await download_handler.download_request(request)
|
||||
assert response.body == b"Works"
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_with_maxsize(
|
||||
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
|
||||
|
|
|
|||
|
|
@ -233,7 +233,7 @@ class TestMiddlewareUsingDeferreds(TestManagerBase):
|
|||
return result
|
||||
|
||||
def process_request(self, request):
|
||||
d: Deferred[Response] = Deferred()
|
||||
d = Deferred()
|
||||
d.addCallback(self.cb)
|
||||
d.callback(resp)
|
||||
return d
|
||||
|
|
@ -298,8 +298,6 @@ class TestDownloadDeprecated(TestManagerBase):
|
|||
return succeed(resp)
|
||||
|
||||
async with self.get_mwman() as mwman:
|
||||
assert mwman.crawler
|
||||
assert mwman.crawler.spider
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match=r"DownloaderMiddlewareManager.download\(\) is deprecated, use download_async\(\) instead",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
|
||||
from scrapy.http import Request
|
||||
from scrapy.spiders import Spider
|
||||
|
|
@ -7,29 +5,28 @@ from scrapy.utils.python import to_bytes
|
|||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
def get_defaults_mw() -> tuple[dict[bytes, list[bytes]], DefaultHeadersMiddleware]:
|
||||
crawler = get_crawler(Spider)
|
||||
defaults = {
|
||||
to_bytes(k): [to_bytes(v)]
|
||||
for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items()
|
||||
}
|
||||
return defaults, DefaultHeadersMiddleware.from_crawler(crawler)
|
||||
class TestDefaultHeadersMiddleware:
|
||||
def get_defaults_mw(self):
|
||||
crawler = get_crawler(Spider)
|
||||
defaults = {
|
||||
to_bytes(k): [to_bytes(v)]
|
||||
for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items()
|
||||
}
|
||||
return defaults, DefaultHeadersMiddleware.from_crawler(crawler)
|
||||
|
||||
def test_process_request(self):
|
||||
defaults, mw = self.get_defaults_mw()
|
||||
req = Request("http://www.scrapytest.org")
|
||||
mw.process_request(req)
|
||||
assert req.headers == defaults
|
||||
|
||||
def test_process_request():
|
||||
defaults, mw = get_defaults_mw()
|
||||
req = Request("http://www.scrapytest.org")
|
||||
mw.process_request(req)
|
||||
assert req.headers == defaults
|
||||
def test_update_headers(self):
|
||||
defaults, mw = self.get_defaults_mw()
|
||||
headers = {"Accept-Language": ["es"], "Test-Header": ["test"]}
|
||||
bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]}
|
||||
req = Request("http://www.scrapytest.org", headers=headers)
|
||||
assert req.headers == bytes_headers
|
||||
|
||||
|
||||
def test_update_headers():
|
||||
defaults, mw = get_defaults_mw()
|
||||
headers = {"Accept-Language": ["es"], "Test-Header": ["test"]}
|
||||
bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]}
|
||||
req = Request("http://www.scrapytest.org", headers=headers)
|
||||
assert req.headers == bytes_headers
|
||||
|
||||
mw.process_request(req)
|
||||
defaults.update(bytes_headers)
|
||||
assert req.headers == defaults
|
||||
mw.process_request(req)
|
||||
defaults.update(bytes_headers)
|
||||
assert req.headers == defaults
|
||||
|
|
|
|||
|
|
@ -1,51 +1,43 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware
|
||||
from scrapy.http import Request
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
def get_request_spider_mw(settings: dict[str, Any] | None = None):
|
||||
crawler = get_crawler(Spider, settings)
|
||||
spider = crawler._create_spider("foo")
|
||||
request = Request("http://scrapytest.org/")
|
||||
return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)
|
||||
class TestDownloadTimeoutMiddleware:
|
||||
def get_request_spider_mw(self, settings=None):
|
||||
crawler = get_crawler(Spider, settings)
|
||||
spider = crawler._create_spider("foo")
|
||||
request = Request("http://scrapytest.org/")
|
||||
return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)
|
||||
|
||||
def test_default_download_timeout(self):
|
||||
req, spider, mw = self.get_request_spider_mw()
|
||||
mw.spider_opened(spider)
|
||||
assert mw.process_request(req) is None
|
||||
assert req.meta.get("download_timeout") == 180
|
||||
|
||||
def test_default_download_timeout():
|
||||
req, spider, mw = get_request_spider_mw()
|
||||
mw.spider_opened(spider)
|
||||
assert mw.process_request(req) is None
|
||||
assert req.meta.get("download_timeout") == 180
|
||||
def test_string_download_timeout(self):
|
||||
req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"})
|
||||
mw.spider_opened(spider)
|
||||
assert mw.process_request(req) is None
|
||||
assert req.meta.get("download_timeout") == 20.1
|
||||
|
||||
def test_setting_has_download_timeout(self):
|
||||
req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})
|
||||
mw.spider_opened(spider)
|
||||
assert mw.process_request(req) is None
|
||||
assert req.meta.get("download_timeout") == 2
|
||||
|
||||
def test_string_download_timeout():
|
||||
req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"})
|
||||
mw.spider_opened(spider)
|
||||
assert mw.process_request(req) is None
|
||||
assert req.meta.get("download_timeout") == 20.1
|
||||
def test_request_has_download_timeout(self):
|
||||
req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})
|
||||
mw.spider_opened(spider)
|
||||
req.meta["download_timeout"] = 1
|
||||
assert mw.process_request(req) is None
|
||||
assert req.meta.get("download_timeout") == 1
|
||||
|
||||
|
||||
def test_setting_has_download_timeout():
|
||||
req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})
|
||||
mw.spider_opened(spider)
|
||||
assert mw.process_request(req) is None
|
||||
assert req.meta.get("download_timeout") == 2
|
||||
|
||||
|
||||
def test_request_has_download_timeout():
|
||||
req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})
|
||||
mw.spider_opened(spider)
|
||||
req.meta["download_timeout"] = 1
|
||||
assert mw.process_request(req) is None
|
||||
assert req.meta.get("download_timeout") == 1
|
||||
|
||||
|
||||
def test_zero_download_timeout():
|
||||
req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 0})
|
||||
mw.spider_opened(spider)
|
||||
assert mw.process_request(req) is None
|
||||
assert req.meta.get("download_timeout") is None
|
||||
def test_zero_download_timeout(self):
|
||||
req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 0})
|
||||
mw.spider_opened(spider)
|
||||
assert mw.process_request(req) is None
|
||||
assert req.meta.get("download_timeout") is None
|
||||
|
|
|
|||
|
|
@ -558,30 +558,6 @@ class TestHttpCompression:
|
|||
|
||||
self._test_compression_bomb_setting("zstd")
|
||||
|
||||
def test_compression_bomb_setting_logs_warning(self, caplog):
|
||||
settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
|
||||
crawler = get_crawler(Spider, settings_dict=settings)
|
||||
spider = crawler._create_spider("scrapytest.org")
|
||||
mw = HttpCompressionMiddleware.from_crawler(crawler)
|
||||
mw.open_spider(spider)
|
||||
|
||||
response = self._getresponse("bomb-gzip") # 11_511_612 B
|
||||
caplog.clear()
|
||||
with (
|
||||
caplog.at_level(
|
||||
WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
|
||||
),
|
||||
pytest.raises(IgnoreRequest) as exc_info,
|
||||
):
|
||||
mw.process_response(response.request, response)
|
||||
assert caplog.record_tuples == [
|
||||
(
|
||||
"scrapy.downloadermiddlewares.httpcompression",
|
||||
WARNING,
|
||||
str(exc_info.value),
|
||||
)
|
||||
]
|
||||
|
||||
def _test_compression_bomb_spider_attr(self, compression_id):
|
||||
class DownloadMaxSizeSpider(Spider):
|
||||
download_maxsize = 1_000_000
|
||||
|
|
|
|||
|
|
@ -16,8 +16,11 @@ from scrapy.spiders import Spider
|
|||
from scrapy.utils.misc import build_from_crawler
|
||||
from scrapy.utils.spider import DefaultSpider
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests.test_downloadermiddleware_redirect_base import Base
|
||||
from tests.utils.redirect import REDIRECT_SCHEME_CASES, SCHEME_PARAMS
|
||||
from tests.test_downloadermiddleware_redirect_base import (
|
||||
REDIRECT_SCHEME_CASES,
|
||||
SCHEME_PARAMS,
|
||||
Base,
|
||||
)
|
||||
|
||||
|
||||
class TestRedirectMiddleware(Base.Test):
|
||||
|
|
@ -33,7 +36,7 @@ class TestRedirectMiddleware(Base.Test):
|
|||
headers = {"Location": location}
|
||||
return Response(request.url, status=status, headers=headers)
|
||||
|
||||
def test_redirect_307_308_preserve_method(self):
|
||||
def test_redirect_3xx_permanent(self):
|
||||
def _test(method, status: int):
|
||||
url = f"http://www.example.com/{status}"
|
||||
url2 = "http://www.example.com/redirected"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from itertools import product
|
||||
|
||||
import pytest
|
||||
|
||||
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
|
||||
|
|
@ -8,6 +10,43 @@ from scrapy.http import Request, Response
|
|||
from scrapy.utils.misc import set_environ
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
SCHEME_PARAMS = ("url", "location", "target")
|
||||
HTTP_SCHEMES = ("http", "https")
|
||||
NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo")
|
||||
REDIRECT_SCHEME_CASES = (
|
||||
# http/https → http/https redirects
|
||||
*(
|
||||
(
|
||||
f"{input_scheme}://example.com/a",
|
||||
f"{output_scheme}://example.com/b",
|
||||
f"{output_scheme}://example.com/b",
|
||||
)
|
||||
for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2)
|
||||
),
|
||||
# http/https → data/file/ftp/s3/foo does not redirect
|
||||
*(
|
||||
(
|
||||
f"{input_scheme}://example.com/a",
|
||||
f"{output_scheme}://example.com/b",
|
||||
None,
|
||||
)
|
||||
for input_scheme in HTTP_SCHEMES
|
||||
for output_scheme in NON_HTTP_SCHEMES
|
||||
),
|
||||
# http/https → relative redirects
|
||||
*(
|
||||
(
|
||||
f"{scheme}://example.com/a",
|
||||
location,
|
||||
f"{scheme}://example.com/b",
|
||||
)
|
||||
for scheme in HTTP_SCHEMES
|
||||
for location in ("//example.com/b", "/b")
|
||||
),
|
||||
# Note: We do not test data/file/ftp/s3 schemes for the initial URL
|
||||
# because their download handlers cannot return a status code of 3xx.
|
||||
)
|
||||
|
||||
|
||||
class Base:
|
||||
class Test:
|
||||
|
|
|
|||
|
|
@ -12,12 +12,12 @@ from scrapy.http import HtmlResponse, Request, Response
|
|||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.misc import build_from_crawler
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests.test_downloadermiddleware_redirect_base import Base
|
||||
from tests.utils.redirect import (
|
||||
from tests.test_downloadermiddleware_redirect_base import (
|
||||
HTTP_SCHEMES,
|
||||
NON_HTTP_SCHEMES,
|
||||
REDIRECT_SCHEME_CASES,
|
||||
SCHEME_PARAMS,
|
||||
Base,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -42,18 +42,17 @@ class TestRetry:
|
|||
req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})
|
||||
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
|
||||
|
||||
# no retry
|
||||
# first retry
|
||||
r = self.mw.process_response(req, rsp)
|
||||
assert r is rsp
|
||||
|
||||
# Test retry when dont_retry set to False
|
||||
req = Request("http://www.scrapytest.org/503", meta={"dont_retry": False})
|
||||
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
|
||||
rsp = Response("http://www.scrapytest.org/503")
|
||||
|
||||
# first retry
|
||||
req = self.mw.process_response(req, rsp)
|
||||
assert isinstance(req, Request)
|
||||
assert req.meta["retry_times"] == 1
|
||||
r = self.mw.process_response(req, rsp)
|
||||
assert r is rsp
|
||||
|
||||
def test_dont_retry_exc(self):
|
||||
req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@ from scrapy.http.request import NO_CALLBACK
|
|||
from scrapy.settings import Settings
|
||||
from scrapy.utils.asyncio import call_later
|
||||
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
|
||||
from tests.test_robotstxt_interface import rerp_available
|
||||
from tests.utils.decorators import coroutine_test
|
||||
from tests.utils.robotstxt import rerp_available
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy.crawler import Crawler
|
||||
|
|
|
|||
|
|
@ -1,37 +1,33 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
|
||||
from scrapy.http import Request
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
def get_spider_and_mw(
|
||||
default_useragent: str | None,
|
||||
) -> tuple[Spider, UserAgentMiddleware]:
|
||||
crawler = get_crawler(Spider, {"USER_AGENT": default_useragent})
|
||||
spider = crawler._create_spider("foo")
|
||||
return spider, UserAgentMiddleware.from_crawler(crawler)
|
||||
class TestUserAgentMiddleware:
|
||||
def get_spider_and_mw(self, default_useragent):
|
||||
crawler = get_crawler(Spider, {"USER_AGENT": default_useragent})
|
||||
spider = crawler._create_spider("foo")
|
||||
return spider, UserAgentMiddleware.from_crawler(crawler)
|
||||
|
||||
def test_default_agent(self):
|
||||
_, mw = self.get_spider_and_mw("default_useragent")
|
||||
req = Request("http://scrapytest.org/")
|
||||
assert mw.process_request(req) is None
|
||||
assert req.headers["User-Agent"] == b"default_useragent"
|
||||
|
||||
def test_default_agent():
|
||||
_, mw = get_spider_and_mw("default_useragent")
|
||||
req = Request("http://scrapytest.org/")
|
||||
assert mw.process_request(req) is None
|
||||
assert req.headers["User-Agent"] == b"default_useragent"
|
||||
def test_header_agent(self):
|
||||
spider, mw = self.get_spider_and_mw("default_useragent")
|
||||
mw.spider_opened(spider)
|
||||
req = Request(
|
||||
"http://scrapytest.org/", headers={"User-Agent": "header_useragent"}
|
||||
)
|
||||
assert mw.process_request(req) is None
|
||||
assert req.headers["User-Agent"] == b"header_useragent"
|
||||
|
||||
|
||||
def test_header_agent():
|
||||
spider, mw = get_spider_and_mw("default_useragent")
|
||||
mw.spider_opened(spider)
|
||||
req = Request("http://scrapytest.org/", headers={"User-Agent": "header_useragent"})
|
||||
assert mw.process_request(req) is None
|
||||
assert req.headers["User-Agent"] == b"header_useragent"
|
||||
|
||||
|
||||
def test_no_agent():
|
||||
spider, mw = get_spider_and_mw(None)
|
||||
mw.spider_opened(spider)
|
||||
req = Request("http://scrapytest.org/")
|
||||
assert mw.process_request(req) is None
|
||||
assert "User-Agent" not in req.headers
|
||||
def test_no_agent(self):
|
||||
spider, mw = self.get_spider_and_mw(None)
|
||||
mw.spider_opened(spider)
|
||||
req = Request("http://scrapytest.org/")
|
||||
assert mw.process_request(req) is None
|
||||
assert "User-Agent" not in req.headers
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ import subprocess
|
|||
import sys
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from unittest.mock import Mock
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
from unittest.mock import Mock, call
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import attr
|
||||
|
|
@ -25,6 +25,7 @@ from scrapy.http import Headers, Request, Response
|
|||
from scrapy.item import Field, Item
|
||||
from scrapy.linkextractors import LinkExtractor
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.statscollectors import MemoryStatsCollector
|
||||
from scrapy.utils.defer import (
|
||||
_schedule_coro,
|
||||
deferred_from_coro,
|
||||
|
|
@ -37,10 +38,10 @@ from tests import get_testdata
|
|||
from tests.utils.decorators import coroutine_test, inline_callbacks_test
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from scrapy.core.scheduler import Scheduler
|
||||
from scrapy.crawler import Crawler
|
||||
from tests.mockserver.http import MockServer
|
||||
|
||||
|
||||
|
|
@ -512,11 +513,103 @@ class TestEngine(TestEngineBase):
|
|||
assert "AssertionError" not in stderr_str, stderr_str
|
||||
|
||||
|
||||
class TestEngineDownloadAsync:
|
||||
"""Test cases for ExecutionEngine.download_async()."""
|
||||
|
||||
@pytest.fixture
|
||||
def engine(self) -> ExecutionEngine:
|
||||
crawler = get_crawler(MySpider)
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
engine.downloader.close()
|
||||
engine.downloader = Mock()
|
||||
engine._slot = Mock()
|
||||
engine._slot.inprogress = set()
|
||||
return engine
|
||||
|
||||
@staticmethod
|
||||
async def _download(engine: ExecutionEngine, request: Request) -> Response:
|
||||
return await engine.download_async(request)
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_async_success(self, engine):
|
||||
"""Test basic successful async download of a request."""
|
||||
request = Request("http://example.com")
|
||||
response = Response("http://example.com", body=b"test body")
|
||||
engine.spider = Mock()
|
||||
engine.downloader.fetch.return_value = defer.succeed(response)
|
||||
engine._slot.add_request = Mock()
|
||||
engine._slot.remove_request = Mock()
|
||||
|
||||
result = await self._download(engine, request)
|
||||
assert result == response
|
||||
engine._slot.add_request.assert_called_once_with(request)
|
||||
engine._slot.remove_request.assert_called_once_with(request)
|
||||
engine.downloader.fetch.assert_called_once_with(request)
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_async_redirect(self, engine):
|
||||
"""Test async download with a redirect request."""
|
||||
original_request = Request("http://example.com")
|
||||
redirect_request = Request("http://example.com/redirect")
|
||||
final_response = Response("http://example.com/redirect", body=b"redirected")
|
||||
|
||||
# First call returns redirect request, second call returns final response
|
||||
engine.downloader.fetch.side_effect = [
|
||||
defer.succeed(redirect_request),
|
||||
defer.succeed(final_response),
|
||||
]
|
||||
engine.spider = Mock()
|
||||
engine._slot.add_request = Mock()
|
||||
engine._slot.remove_request = Mock()
|
||||
|
||||
result = await self._download(engine, original_request)
|
||||
assert result == final_response
|
||||
assert engine.downloader.fetch.call_count == 2
|
||||
engine._slot.add_request.assert_has_calls(
|
||||
[call(original_request), call(redirect_request)]
|
||||
)
|
||||
engine._slot.remove_request.assert_has_calls(
|
||||
[call(original_request), call(redirect_request)]
|
||||
)
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_async_no_spider(self, engine):
|
||||
"""Test async download attempt when no spider is available."""
|
||||
request = Request("http://example.com")
|
||||
engine.spider = None
|
||||
with pytest.raises(RuntimeError, match="No open spider to crawl:"):
|
||||
await self._download(engine, request)
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_async_failure(self, engine):
|
||||
"""Test async download when the downloader raises an exception."""
|
||||
request = Request("http://example.com")
|
||||
error = RuntimeError("Download failed")
|
||||
engine.spider = Mock()
|
||||
engine.downloader.fetch.return_value = defer.fail(error)
|
||||
engine._slot.add_request = Mock()
|
||||
engine._slot.remove_request = Mock()
|
||||
|
||||
with pytest.raises(RuntimeError, match="Download failed"):
|
||||
await self._download(engine, request)
|
||||
engine._slot.add_request.assert_called_once_with(request)
|
||||
engine._slot.remove_request.assert_called_once_with(request)
|
||||
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
class TestEngineDownload(TestEngineDownloadAsync):
|
||||
"""Test cases for ExecutionEngine.download()."""
|
||||
|
||||
@staticmethod
|
||||
async def _download(engine: ExecutionEngine, request: Request) -> Response:
|
||||
return await maybe_deferred_to_future(engine.download(request))
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_request_scheduled_signal():
|
||||
class TestScheduler(BaseScheduler):
|
||||
def __init__(self) -> None:
|
||||
self.enqueued: list[Request] = []
|
||||
def __init__(self):
|
||||
self.enqueued = []
|
||||
|
||||
def enqueue_request(self, request: Request) -> bool:
|
||||
self.enqueued.append(request)
|
||||
|
|
@ -528,9 +621,9 @@ async def test_request_scheduled_signal():
|
|||
|
||||
crawler = get_crawler(MySpider)
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
scheduler = TestScheduler() # type: ignore[abstract]
|
||||
scheduler = TestScheduler()
|
||||
|
||||
async def start() -> AsyncIterator[Any]:
|
||||
async def start():
|
||||
return
|
||||
yield
|
||||
|
||||
|
|
@ -545,3 +638,132 @@ async def test_request_scheduled_signal():
|
|||
f"{scheduler.enqueued!r} != [{keep_request!r}]"
|
||||
)
|
||||
crawler.signals.disconnect(signal_handler, signals.request_scheduled)
|
||||
|
||||
|
||||
class TestEngineCloseSpider:
|
||||
"""Tests for exception handling coverage during close_spider_async()."""
|
||||
|
||||
@pytest.fixture
|
||||
def crawler(self) -> Crawler:
|
||||
crawler = get_crawler(DefaultSpider)
|
||||
crawler.spider = crawler._create_spider()
|
||||
return crawler
|
||||
|
||||
@coroutine_test
|
||||
async def test_no_slot(self, crawler: Crawler) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
slot = engine._slot
|
||||
engine._slot = None
|
||||
with pytest.raises(RuntimeError, match="Engine slot not assigned"):
|
||||
await engine.close_spider_async()
|
||||
# close it correctly
|
||||
engine._slot = slot
|
||||
await engine.close_spider_async()
|
||||
|
||||
@coroutine_test
|
||||
async def test_no_spider(self, crawler: Crawler) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
with pytest.raises(RuntimeError, match="Spider not opened"):
|
||||
await engine.close_spider_async()
|
||||
engine.downloader.close() # cleanup
|
||||
|
||||
@coroutine_test
|
||||
async def test_exception_slot(
|
||||
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
assert engine._slot
|
||||
del engine._slot.heartbeat
|
||||
await engine.close_spider_async()
|
||||
assert "Slot close failure" in caplog.text
|
||||
|
||||
@coroutine_test
|
||||
async def test_exception_downloader(
|
||||
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
del engine.downloader.slots
|
||||
await engine.close_spider_async()
|
||||
assert "Downloader close failure" in caplog.text
|
||||
|
||||
@coroutine_test
|
||||
async def test_exception_scraper(
|
||||
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
engine.scraper.slot = None
|
||||
await engine.close_spider_async()
|
||||
assert "Scraper close failure" in caplog.text
|
||||
|
||||
@coroutine_test
|
||||
async def test_exception_scheduler(
|
||||
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
assert engine._slot
|
||||
del cast("Scheduler", engine._slot.scheduler).dqs
|
||||
await engine.close_spider_async()
|
||||
assert "Scheduler close failure" in caplog.text
|
||||
|
||||
@coroutine_test
|
||||
async def test_exception_signal(
|
||||
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
signal_manager = engine.signals
|
||||
del engine.signals
|
||||
await engine.close_spider_async()
|
||||
assert "Error while sending spider_close signal" in caplog.text
|
||||
# send the spider_closed signal to close various components
|
||||
await signal_manager.send_catch_log_async(
|
||||
signal=signals.spider_closed,
|
||||
spider=engine.spider,
|
||||
reason="cancelled",
|
||||
)
|
||||
|
||||
@coroutine_test
|
||||
async def test_exception_stats(
|
||||
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
assert isinstance(crawler.stats, MemoryStatsCollector)
|
||||
del crawler.stats.spider_stats
|
||||
await engine.close_spider_async()
|
||||
assert "Stats close failure" in caplog.text
|
||||
|
||||
@coroutine_test
|
||||
async def test_exception_callback(
|
||||
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError()))
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
await engine.close_spider_async()
|
||||
assert "Error running spider_closed_callback" in caplog.text
|
||||
|
||||
@coroutine_test
|
||||
async def test_exception_async_callback(
|
||||
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
async def cb(_):
|
||||
raise ValueError
|
||||
|
||||
engine = ExecutionEngine(crawler, cb)
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
await engine.close_spider_async()
|
||||
assert "Error running spider_closed_callback" in caplog.text
|
||||
|
|
|
|||
|
|
@ -1,153 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import pytest
|
||||
from twisted.internet import defer
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.core.engine import ExecutionEngine
|
||||
from scrapy.statscollectors import MemoryStatsCollector
|
||||
from scrapy.utils.spider import DefaultSpider
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests.utils.decorators import coroutine_test
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy.core.scheduler import Scheduler
|
||||
from scrapy.crawler import Crawler
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def crawler() -> Crawler:
|
||||
crawler = get_crawler(DefaultSpider)
|
||||
crawler.spider = crawler._create_spider()
|
||||
return crawler
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_no_slot(crawler: Crawler) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
slot = engine._slot
|
||||
engine._slot = None
|
||||
with pytest.raises(RuntimeError, match="Engine slot not assigned"):
|
||||
await engine.close_spider_async()
|
||||
# close it correctly
|
||||
engine._slot = slot
|
||||
await engine.close_spider_async()
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_no_spider(crawler: Crawler) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
with pytest.raises(RuntimeError, match="Spider not opened"):
|
||||
await engine.close_spider_async()
|
||||
engine.downloader.close() # cleanup
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_exception_slot(
|
||||
crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
assert engine._slot
|
||||
del engine._slot.heartbeat
|
||||
await engine.close_spider_async()
|
||||
assert "Slot close failure" in caplog.text
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_exception_downloader(
|
||||
crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
del engine.downloader.slots
|
||||
await engine.close_spider_async()
|
||||
assert "Downloader close failure" in caplog.text
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_exception_scraper(
|
||||
crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
engine.scraper.slot = None
|
||||
await engine.close_spider_async()
|
||||
assert "Scraper close failure" in caplog.text
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_exception_scheduler(
|
||||
crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
assert engine._slot
|
||||
del cast("Scheduler", engine._slot.scheduler).dqs
|
||||
await engine.close_spider_async()
|
||||
assert "Scheduler close failure" in caplog.text
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_exception_signal(
|
||||
crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
signal_manager = engine.signals
|
||||
del engine.signals
|
||||
await engine.close_spider_async()
|
||||
assert "Error while sending spider_close signal" in caplog.text
|
||||
# send the spider_closed signal to close various components
|
||||
await signal_manager.send_catch_log_async(
|
||||
signal=signals.spider_closed,
|
||||
spider=engine.spider,
|
||||
reason="cancelled",
|
||||
)
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_exception_stats(
|
||||
crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
assert isinstance(crawler.stats, MemoryStatsCollector)
|
||||
del crawler.stats.spider_stats
|
||||
await engine.close_spider_async()
|
||||
assert "Stats close failure" in caplog.text
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_exception_callback(
|
||||
crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError()))
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
await engine.close_spider_async()
|
||||
assert "Error running spider_closed_callback" in caplog.text
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_exception_async_callback(
|
||||
crawler: Crawler, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
async def cb(_):
|
||||
raise ValueError
|
||||
|
||||
engine = ExecutionEngine(crawler, cb)
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
await engine.close_spider_async()
|
||||
assert "Error running spider_closed_callback" in caplog.text
|
||||
|
|
@ -1,106 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import Mock, call
|
||||
|
||||
import pytest
|
||||
from twisted.internet import defer
|
||||
|
||||
from scrapy.core.engine import ExecutionEngine
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
from scrapy.utils.spider import DefaultSpider
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests.utils.decorators import coroutine_test
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def engine() -> ExecutionEngine:
|
||||
crawler = get_crawler(DefaultSpider)
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
engine.downloader.close()
|
||||
engine.downloader = Mock()
|
||||
engine._slot = Mock()
|
||||
engine._slot.inprogress = set()
|
||||
return engine
|
||||
|
||||
|
||||
class TestEngineDownloadAsync:
|
||||
"""Test cases for ExecutionEngine.download_async()."""
|
||||
|
||||
@staticmethod
|
||||
async def _download(engine: ExecutionEngine, request: Request) -> Response:
|
||||
return await engine.download_async(request)
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_async_success(self, engine):
|
||||
"""Test basic successful async download of a request."""
|
||||
request = Request("http://example.com")
|
||||
response = Response("http://example.com", body=b"test body")
|
||||
engine.spider = Mock()
|
||||
engine.downloader.fetch.return_value = defer.succeed(response)
|
||||
engine._slot.add_request = Mock()
|
||||
engine._slot.remove_request = Mock()
|
||||
|
||||
result = await self._download(engine, request)
|
||||
assert result == response
|
||||
engine._slot.add_request.assert_called_once_with(request)
|
||||
engine._slot.remove_request.assert_called_once_with(request)
|
||||
engine.downloader.fetch.assert_called_once_with(request)
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_async_redirect(self, engine):
|
||||
"""Test async download with a redirect request."""
|
||||
original_request = Request("http://example.com")
|
||||
redirect_request = Request("http://example.com/redirect")
|
||||
final_response = Response("http://example.com/redirect", body=b"redirected")
|
||||
|
||||
# First call returns redirect request, second call returns final response
|
||||
engine.downloader.fetch.side_effect = [
|
||||
defer.succeed(redirect_request),
|
||||
defer.succeed(final_response),
|
||||
]
|
||||
engine.spider = Mock()
|
||||
engine._slot.add_request = Mock()
|
||||
engine._slot.remove_request = Mock()
|
||||
|
||||
result = await self._download(engine, original_request)
|
||||
assert result == final_response
|
||||
assert engine.downloader.fetch.call_count == 2
|
||||
engine._slot.add_request.assert_has_calls(
|
||||
[call(original_request), call(redirect_request)]
|
||||
)
|
||||
engine._slot.remove_request.assert_has_calls(
|
||||
[call(original_request), call(redirect_request)]
|
||||
)
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_async_no_spider(self, engine):
|
||||
"""Test async download attempt when no spider is available."""
|
||||
request = Request("http://example.com")
|
||||
engine.spider = None
|
||||
with pytest.raises(RuntimeError, match="No open spider to crawl:"):
|
||||
await self._download(engine, request)
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_async_failure(self, engine):
|
||||
"""Test async download when the downloader raises an exception."""
|
||||
request = Request("http://example.com")
|
||||
error = RuntimeError("Download failed")
|
||||
engine.spider = Mock()
|
||||
engine.downloader.fetch.return_value = defer.fail(error)
|
||||
engine._slot.add_request = Mock()
|
||||
engine._slot.remove_request = Mock()
|
||||
|
||||
with pytest.raises(RuntimeError, match="Download failed"):
|
||||
await self._download(engine, request)
|
||||
engine._slot.add_request.assert_called_once_with(request)
|
||||
engine._slot.remove_request.assert_called_once_with(request)
|
||||
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
class TestEngineDownload(TestEngineDownloadAsync):
|
||||
"""Test cases for ExecutionEngine.download()."""
|
||||
|
||||
@staticmethod
|
||||
async def _download(engine: ExecutionEngine, request: Request) -> Response:
|
||||
return await maybe_deferred_to_future(engine.download(request))
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue