RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into asyncio-parse-asyncgen-proper-rebased

This commit is contained in:
Andrey Rakhmatullin 2021-04-14 17:34:58 +05:00
parent 7bd1d888d4 9bf9ab7291
commit ffc6f525ce
55 changed files with 1425 additions and 488 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.bumpversion.cfg
View File

@ -1,5 +1,5 @@
[bumpversion]
current_version = 2.4.1
current_version = 2.5.0
commit = True
tag = True
tag_name = {new_version}

1
.readthedocs.yml
View File

@ -12,6 +12,5 @@ python:
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
version: 3.8 # Keep in sync with .github/workflows/checks.yml
install:
- requirements: docs/pip.txt
- requirements: docs/requirements.txt
- path: .

10
docs/_static/custom.css vendored Normal file
View File

@ -0,0 +1,10 @@
/* Move lists closer to their introducing paragraph */
.rst-content .section ol p, .rst-content .section ul p {
margin-bottom: 0px;
}
.rst-content p + ol, .rst-content p + ul {
margin-top: -18px; /* Compensates margin-top: 24px of p */
}
.rst-content dl p + ol, .rst-content dl p + ul {
margin-top: -6px; /* Compensates margin-top: 12px of p */
}

7
docs/_templates/layout.html vendored
View File

@ -3,14 +3,9 @@
{% block footer %}
{{ super() }}
<script type="text/javascript">
!function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","page","once","off","on"];analytics.factory=function(t){return function(){var e=Array.prototype.slice.call(arguments);e.unshift(t);analytics.push(e);return analytics}};for(var t=0;t<analytics.methods.length;t++){var e=analytics.methods[t];analytics[e]=analytics.factory(e)}analytics.load=function(t){var e=document.createElement("script");e.type="text/javascript";e.async=!0;e.src=("https:"===document.location.protocol?"https://":"http://")+"cdn.segment.com/analytics.js/v1/"+t+"/analytics.min.js";var n=document.getElementsByTagName("script")[0];n.parentNode.insertBefore(e,n)};analytics.SNIPPET_VERSION="3.1.0";
analytics.load("8UDQfnf3cyFSTsM4YANnW5sXmgZVILbA");
analytics.page();
}}();
analytics.ready(function () {
ga('require', 'linker');
ga('linker:autoLink', ['scrapinghub.com', 'crawlera.com']);
ga('linker:autoLink', ['zyte.com']);
});
</script>
{% endblock %}

5
docs/conf.py
View File

@ -122,7 +122,6 @@ html_theme = 'sphinx_rtd_theme'
import sphinx_rtd_theme
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# The style sheet to use for HTML and HTML Help pages. A file of that name
# must exist either in Sphinx' static/ path, or in one of the custom paths
# given in html_static_path.
@ -183,6 +182,10 @@ html_copy_source = True
# Output file base name for HTML help builder.
htmlhelp_basename = 'Scrapydoc'
html_css_files = [
'custom.css',
]
# Options for LaTeX output
# ------------------------

30
docs/intro/install.rst
View File

@ -12,6 +12,7 @@ Supported Python versions
Scrapy requires Python 3.6+, either the CPython implementation (default) or
the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
.. _intro-install-scrapy:
Installing Scrapy
=================
@ -29,13 +30,13 @@ you can install Scrapy and its dependencies from PyPI with::
pip install Scrapy
We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
to avoid conflicting with your system packages.
Note that sometimes this may require solving compilation issues for some Scrapy
dependencies depending on your operating system, so be sure to check the
:ref:`intro-install-platform-notes`.
We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
to avoid conflicting with your system packages.
For more detailed and platform specifics instructions, as well as
troubleshooting information, read on.
@ -117,6 +118,27 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with::
conda install -c conda-forge scrapy
To install Scrapy on Windows using ``pip``:
.. warning::
This installation method requires “Microsoft Visual C++” for installing some
Scrapy dependencies, which demands significantly more disk space than Anaconda.
#. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer.
#. Run the Visual Studio Installer.
#. Under the Workloads section, select **C++ build tools**.
#. Check the installation details and make sure following packages are selected as optional components:
* **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) )
* **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0))
#. Install the Visual Studio Build Tools.
Now, you should be able to :ref:`install Scrapy <intro-install-scrapy>` using ``pip``.
.. _intro-install-ubuntu:
@ -268,4 +290,6 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
.. _zsh: https://www.zsh.org/
.. _Anaconda: https://docs.anaconda.com/anaconda/
.. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio
.. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/
.. _conda-forge: https://conda-forge.org/

193
docs/news.rst
View File

@ -3,6 +3,190 @@
Release notes
=============
.. _release-2.5.0:
Scrapy 2.5.0 (2021-04-06)
-------------------------
Highlights:
- Official Python 3.9 support
- Experimental :ref:`HTTP/2 support <http2>`
- New :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function
to retry requests from spider callbacks
- New :class:`~scrapy.signals.headers_received` signal that allows stopping
downloads early
- New :class:`Response.protocol <scrapy.http.Response.protocol>` attribute
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
- Removed all code that :ref:`was deprecated in 1.7.0 <1.7-deprecations>` and
had not :ref:`already been removed in 2.4.0 <2.4-deprecation-removals>`.
(:issue:`4901`)
- Removed support for the ``SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE`` environment
variable, :ref:`deprecated in 1.8.0 <1.8-deprecations>`. (:issue:`4912`)
Deprecations
~~~~~~~~~~~~
- The :mod:`scrapy.utils.py36` module is now deprecated in favor of
:mod:`scrapy.utils.asyncgen`. (:issue:`4900`)
New features
~~~~~~~~~~~~
- Experimental :ref:`HTTP/2 support <http2>` through a new download handler
that can be assigned to the ``https`` protocol in the
:setting:`DOWNLOAD_HANDLERS` setting.
(:issue:`1854`, :issue:`4769`, :issue:`5058`, :issue:`5059`, :issue:`5066`)
- The new :func:`scrapy.downloadermiddlewares.retry.get_retry_request`
function may be used from spider callbacks or middlewares to handle the
retrying of a request beyond the scenarios that
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` supports.
(:issue:`3590`, :issue:`3685`, :issue:`4902`)
- The new :class:`~scrapy.signals.headers_received` signal gives early access
to response headers and allows :ref:`stopping downloads
<topics-stop-response-download>`.
(:issue:`1772`, :issue:`4897`)
- The new :attr:`Response.protocol <scrapy.http.Response.protocol>`
attribute gives access to the string that identifies the protocol used to
download a response. (:issue:`4878`)
- :ref:`Stats <topics-stats>` now include the following entries that indicate
the number of successes and failures in storing
:ref:`feeds <topics-feed-exports>`::
feedexport/success_count/<storage type>
feedexport/failed_count/<storage type>
Where ``<storage type>`` is the feed storage backend class name, such as
:class:`~scrapy.extensions.feedexport.FileFeedStorage` or
:class:`~scrapy.extensions.feedexport.FTPFeedStorage`.
(:issue:`3947`, :issue:`4850`)
- The :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` spider
middleware now logs ignored URLs with ``INFO`` :ref:`logging level
<levels>` instead of ``DEBUG``, and it now includes the following entry
into :ref:`stats <topics-stats>` to keep track of the number of ignored
URLs::
urllength/request_ignored_count
(:issue:`5036`)
- The
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
downloader middleware now logs the number of decompressed responses and the
total count of resulting bytes::
httpcompression/response_bytes
httpcompression/response_count
(:issue:`4797`, :issue:`4799`)
Bug fixes
~~~~~~~~~
- Fixed installation on PyPy installing PyDispatcher in addition to
PyPyDispatcher, which could prevent Scrapy from working depending on which
package got imported. (:issue:`4710`, :issue:`4814`)
- When inspecting a callback to check if it is a generator that also returns
a value, an exception is no longer raised if the callback has a docstring
with lower indentation than the following code.
(:issue:`4477`, :issue:`4935`)
- The `Content-Length <https://tools.ietf.org/html/rfc2616#section-14.13>`_
header is no longer omitted from responses when using the default, HTTP/1.1
download handler (see :setting:`DOWNLOAD_HANDLERS`).
(:issue:`5009`, :issue:`5034`, :issue:`5045`, :issue:`5057`, :issue:`5062`)
- Setting the :reqmeta:`handle_httpstatus_all` request meta key to ``False``
now has the same effect as not setting it at all, instead of having the
same effect as setting it to ``True``.
(:issue:`3851`, :issue:`4694`)
Documentation
~~~~~~~~~~~~~
- Added instructions to :ref:`install Scrapy in Windows using pip
<intro-install-windows>`.
(:issue:`4715`, :issue:`4736`)
- Logging documentation now includes :ref:`additional ways to filter logs
<topics-logging-advanced-customization>`.
(:issue:`4216`, :issue:`4257`, :issue:`4965`)
- Covered how to deal with long lists of allowed domains in the :ref:`FAQ
<faq>`. (:issue:`2263`, :issue:`3667`)
- Covered scrapy-bench_ in :ref:`benchmarking`.
(:issue:`4996`, :issue:`5016`)
- Clarified that one :ref:`extension <topics-extensions>` instance is created
per crawler.
(:issue:`5014`)
- Fixed some errors in examples.
(:issue:`4829`, :issue:`4830`, :issue:`4907`, :issue:`4909`,
:issue:`5008`)
- Fixed some external links, typos, and so on.
(:issue:`4892`, :issue:`4899`, :issue:`4936`, :issue:`4942`, :issue:`5005`,
:issue:`5063`)
- The :ref:`list of Request.meta keys <topics-request-meta>` is now sorted
alphabetically.
(:issue:`5061`, :issue:`5065`)
- Updated references to Scrapinghub, which is now called Zyte.
(:issue:`4973`, :issue:`5072`)
- Added a mention to contributors in the README. (:issue:`4956`)
- Reduced the top margin of lists. (:issue:`4974`)
Quality Assurance
~~~~~~~~~~~~~~~~~
- Made Python 3.9 support official (:issue:`4757`, :issue:`4759`)
- Extended typing hints (:issue:`4895`)
- Fixed deprecated uses of the Twisted API.
(:issue:`4940`, :issue:`4950`, :issue:`5073`)
- Made our tests run with the new pip resolver.
(:issue:`4710`, :issue:`4814`)
- Added tests to ensure that :ref:`coroutine support <coroutine-support>`
is tested. (:issue:`4987`)
- Migrated from Travis CI to GitHub Actions. (:issue:`4924`)
- Fixed CI issues.
(:issue:`4986`, :issue:`5020`, :issue:`5022`, :issue:`5027`, :issue:`5052`,
:issue:`5053`)
- Implemented code refactorings, style fixes and cleanups.
(:issue:`4911`, :issue:`4982`, :issue:`5001`, :issue:`5002`, :issue:`5076`)
.. _release-2.4.1:
Scrapy 2.4.1 (2020-11-17)
@ -97,6 +281,8 @@ Backward-incompatible changes
(:issue:`4717`, :issue:`4823`)
.. _2.4-deprecation-removals:
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
@ -1433,6 +1619,8 @@ Deprecation removals
* ``scrapy.xlib`` has been removed (:issue:`4015`)
.. _1.8-deprecations:
Deprecations
~~~~~~~~~~~~
@ -1789,6 +1977,8 @@ The following deprecated settings have also been removed (:issue:`3578`):
* ``SPIDER_MANAGER_CLASS`` (use :setting:`SPIDER_LOADER_CLASS`)
.. _1.7-deprecations:
Deprecations
~~~~~~~~~~~~
@ -4184,7 +4374,7 @@ API changes
- ``url`` and ``body`` attributes of Request objects are now read-only (#230)
- ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231)
- Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default)
- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
- Offsite middleware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
- Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself.
- Changes to Scrapy Manager (now called "Crawler"):
- ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler``
@ -4331,6 +4521,7 @@ First release of Scrapy.
.. _resource: https://docs.python.org/2/library/resource.html
.. _robots.txt: https://www.robotstxt.org/
.. _scrapely: https://github.com/scrapy/scrapely
.. _scrapy-bench: https://github.com/scrapy/scrapy-bench
.. _service_identity: https://service-identity.readthedocs.io/en/stable/
.. _six: https://six.readthedocs.io/
.. _tox: https://pypi.org/project/tox/

3
docs/pip.txt
View File

@ -1,3 +0,0 @@
# In pip 20.3-21.0, the default dependency resolver causes the build in
# ReadTheDocs to fail due to memory exhaustion or timeout.
pip<20.3

2
docs/requirements.txt
View File

@ -1,4 +1,4 @@
Sphinx>=3.0
sphinx-hoverxref>=0.2b1
sphinx-notfound-page>=0.4
sphinx_rtd_theme>=0.4
sphinx-rtd-theme>=0.5.2

17
docs/topics/downloader-middleware.rst
View File

@ -892,6 +892,11 @@ settings (see the settings documentation for more info):
If :attr:`Request.meta <scrapy.http.Request.meta>` has ``dont_retry`` key
set to True, the request will be ignored by this middleware.
To retry requests from a spider callback, you can use the
:func:`get_retry_request` function:
.. autofunction:: get_retry_request
RetryMiddleware Settings
~~~~~~~~~~~~~~~~~~~~~~~~
@ -932,6 +937,18 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because
it is a common code used to indicate server overload. It is not included by
default because HTTP specs say so.
.. setting:: RETRY_PRIORITY_ADJUST
RETRY_PRIORITY_ADJUST
---------------------
Default: ``-1``
Adjust retry request priority relative to original request:
- a positive priority adjust means higher priority.
- **a negative priority adjust (default) means lower priority.**
.. _topics-dlmw-robots:

8
docs/topics/exceptions.rst
View File

@ -64,10 +64,10 @@ NotConfigured
This exception can be raised by some components to indicate that they will
remain disabled. Those components include:
* Extensions
* Item pipelines
* Downloader middlewares
* Spider middlewares
- Extensions
- Item pipelines
- Downloader middlewares
- Spider middlewares
The exception must be raised in the component's ``__init__`` method.

7
docs/topics/extensions.rst
View File

@ -7,8 +7,7 @@ Extensions
The extensions framework provides a mechanism for inserting your own
custom functionality into Scrapy.
Extensions are just regular classes that are instantiated at Scrapy startup,
when extensions are initialized.
Extensions are just regular classes.
Extension settings
==================
@ -27,8 +26,8 @@ Loading & activating extensions
===============================
Extensions are loaded and activated at startup by instantiating a single
instance of the extension class. Therefore, all the extension initialization
code must be performed in the class ``__init__`` method.
instance of the extension class per spider being run. All the extension
initialization code must be performed in the class ``__init__`` method.
To make an extension available, add it to the :setting:`EXTENSIONS` setting in
your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented

139
docs/topics/feed-exports.rst
View File

@ -21,10 +21,10 @@ Serialization formats
For serializing the scraped data, the feed exports use the :ref:`Item exporters
<topics-exporters>`. These formats are supported out of the box:
* :ref:`topics-feed-format-json`
* :ref:`topics-feed-format-jsonlines`
* :ref:`topics-feed-format-csv`
* :ref:`topics-feed-format-xml`
- :ref:`topics-feed-format-json`
- :ref:`topics-feed-format-jsonlines`
- :ref:`topics-feed-format-csv`
- :ref:`topics-feed-format-xml`
But you can also extend the supported format through the
:setting:`FEED_EXPORTERS` setting.
@ -34,54 +34,58 @@ But you can also extend the supported format through the
JSON
----
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``json``
* Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
* See :ref:`this warning <json-with-large-data>` if you're using JSON with
large feeds.
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``json``
- Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
- See :ref:`this warning <json-with-large-data>` if you're using JSON with
large feeds.
.. _topics-feed-format-jsonlines:
JSON lines
----------
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines``
* Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines``
- Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
.. _topics-feed-format-csv:
CSV
---
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
* Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
* To specify columns to export and their order use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
option, but it is important for CSV because unlike many other export
formats CSV uses a fixed header.
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
- Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
- To specify columns to export and their order use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
option, but it is important for CSV because unlike many other export
formats CSV uses a fixed header.
.. _topics-feed-format-xml:
XML
---
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml``
* Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml``
- Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
.. _topics-feed-format-pickle:
Pickle
------
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle``
* Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle``
- Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
.. _topics-feed-format-marshal:
Marshal
-------
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
* Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
- Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
.. _topics-feed-storage:
@ -95,11 +99,11 @@ storage backend types which are defined by the URI scheme.
The storages backends supported out of the box are:
* :ref:`topics-feed-storage-fs`
* :ref:`topics-feed-storage-ftp`
* :ref:`topics-feed-storage-s3` (requires botocore_)
* :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
* :ref:`topics-feed-storage-stdout`
- :ref:`topics-feed-storage-fs`
- :ref:`topics-feed-storage-ftp`
- :ref:`topics-feed-storage-s3` (requires botocore_)
- :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
- :ref:`topics-feed-storage-stdout`
Some storage backends may be unavailable if the required external libraries are
not available. For example, the S3 backend is only available if the botocore_
@ -114,8 +118,8 @@ Storage URI parameters
The storage URI can also contain parameters that get replaced when the feed is
being created. These parameters are:
* ``%(time)s`` - gets replaced by a timestamp when the feed is being created
* ``%(name)s`` - gets replaced by the spider name
- ``%(time)s`` - gets replaced by a timestamp when the feed is being created
- ``%(name)s`` - gets replaced by the spider name
Any other named parameter gets replaced by the spider attribute of the same
name. For example, ``%(site_id)s`` would get replaced by the ``spider.site_id``
@ -123,13 +127,13 @@ attribute the moment the feed is being created.
Here are some examples to illustrate:
* Store in FTP using one directory per spider:
- Store in FTP using one directory per spider:
* ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json``
- ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json``
* Store in S3 using one directory per spider:
- Store in S3 using one directory per spider:
* ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``
- ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``
.. _topics-feed-storage-backends:
@ -144,9 +148,9 @@ Local filesystem
The feeds are stored in the local filesystem.
* URI scheme: ``file``
* Example URI: ``file:///tmp/export.csv``
* Required external libraries: none
- URI scheme: ``file``
- Example URI: ``file:///tmp/export.csv``
- Required external libraries: none
Note that for the local filesystem storage (only) you can omit the scheme if
you specify an absolute path like ``/tmp/export.csv``. This only works on Unix
@ -159,9 +163,9 @@ FTP
The feeds are stored in a FTP server.
* URI scheme: ``ftp``
* Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
* Required external libraries: none
- URI scheme: ``ftp``
- Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
- Required external libraries: none
FTP supports two different connection modes: `active or passive
<https://stackoverflow.com/a/1699163>`_. Scrapy uses the passive connection
@ -178,23 +182,25 @@ S3
The feeds are stored on `Amazon S3`_.
* URI scheme: ``s3``
* Example URIs:
- URI scheme: ``s3``
* ``s3://mybucket/path/to/export.csv``
* ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
- Example URIs:
* Required external libraries: `botocore`_ >= 1.4.87
- ``s3://mybucket/path/to/export.csv``
- ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
- Required external libraries: `botocore`_ >= 1.4.87
The AWS credentials can be passed as user/password in the URI, or they can be
passed through the following settings:
* :setting:`AWS_ACCESS_KEY_ID`
* :setting:`AWS_SECRET_ACCESS_KEY`
- :setting:`AWS_ACCESS_KEY_ID`
- :setting:`AWS_SECRET_ACCESS_KEY`
You can also define a custom ACL for exported feeds using this setting:
* :setting:`FEED_STORAGE_S3_ACL`
- :setting:`FEED_STORAGE_S3_ACL`
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
@ -208,19 +214,20 @@ Google Cloud Storage (GCS)
The feeds are stored on `Google Cloud Storage`_.
* URI scheme: ``gs``
* Example URIs:
- URI scheme: ``gs``
* ``gs://mybucket/path/to/export.csv``
- Example URIs:
* Required external libraries: `google-cloud-storage`_.
- ``gs://mybucket/path/to/export.csv``
- Required external libraries: `google-cloud-storage`_.
For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
You can set a *Project ID* and *Access Control List (ACL)* through the following settings:
* :setting:`FEED_STORAGE_GCS_ACL`
* :setting:`GCS_PROJECT_ID`
- :setting:`FEED_STORAGE_GCS_ACL`
- :setting:`GCS_PROJECT_ID`
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
@ -234,9 +241,9 @@ Standard output
The feeds are written to the standard output of the Scrapy process.
* URI scheme: ``stdout``
* Example URI: ``stdout:``
* Required external libraries: none
- URI scheme: ``stdout``
- Example URI: ``stdout:``
- Required external libraries: none
.. _delayed-file-delivery:
@ -264,16 +271,16 @@ Settings
These are the settings used for configuring the feed exports:
* :setting:`FEEDS` (mandatory)
* :setting:`FEED_EXPORT_ENCODING`
* :setting:`FEED_STORE_EMPTY`
* :setting:`FEED_EXPORT_FIELDS`
* :setting:`FEED_EXPORT_INDENT`
* :setting:`FEED_STORAGES`
* :setting:`FEED_STORAGE_FTP_ACTIVE`
* :setting:`FEED_STORAGE_S3_ACL`
* :setting:`FEED_EXPORTERS`
* :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
- :setting:`FEEDS` (mandatory)
- :setting:`FEED_EXPORT_ENCODING`
- :setting:`FEED_STORE_EMPTY`
- :setting:`FEED_EXPORT_FIELDS`
- :setting:`FEED_EXPORT_INDENT`
- :setting:`FEED_STORAGES`
- :setting:`FEED_STORAGE_FTP_ACTIVE`
- :setting:`FEED_STORAGE_S3_ACL`
- :setting:`FEED_EXPORTERS`
- :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
.. currentmodule:: scrapy.extensions.feedexport

41
docs/topics/logging.rst
View File

@ -242,6 +242,47 @@ e.g. in the spider's ``__init__`` method::
If you run this spider again then INFO messages from
``scrapy.spidermiddlewares.httperror`` logger will be gone.
You can also filter log records by :class:`~logging.LogRecord` data. For
example, you can filter log records by message content using a substring or
a regular expression. Create a :class:`logging.Filter` subclass
and equip it with a regular expression pattern to
filter out unwanted messages::
import logging
import re
class ContentFilter(logging.Filter):
def filter(self, record):
match = re.search(r'\d{3} [Ee]rror, retrying', record.message)
if match:
return False
A project-level filter may be attached to the root
handler created by Scrapy, this is a wieldy way to
filter all loggers in different parts of the project
(middlewares, spider, etc.)::
import logging
import scrapy
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
for handler in logging.root.handlers:
handler.addFilter(ContentFilter())
Alternatively, you may choose a specific logger
and hide it without affecting other loggers::
import logging
import scrapy
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
logger = logging.getLogger('my_logger')
logger.addFilter(ContentFilter())
scrapy.utils.log module
=======================

4
docs/topics/request-response.rst
View File

@ -703,7 +703,7 @@ Response objects
.. versionadded:: 2.1.0
The ``ip_address`` parameter.
.. versionadded:: VERSION
.. versionadded:: 2.5.0
The ``protocol`` parameter.
.. attribute:: Response.url
@ -809,7 +809,7 @@ Response objects
.. attribute:: Response.protocol
.. versionadded:: VERSION
.. versionadded:: 2.5.0
The protocol that was used to download the response.
For instance: "HTTP/1.0", "HTTP/1.1"

14
docs/topics/selectors.rst
View File

@ -8,14 +8,14 @@ When you're scraping web pages, the most common task you need to perform is
to extract data from the HTML source. There are several libraries available to
achieve this, such as:
* `BeautifulSoup`_ is a very popular web scraping library among Python
programmers which constructs a Python object based on the structure of the
HTML code and also deals with bad markup reasonably well, but it has one
drawback: it's slow.
- `BeautifulSoup`_ is a very popular web scraping library among Python
programmers which constructs a Python object based on the structure of the
HTML code and also deals with bad markup reasonably well, but it has one
drawback: it's slow.
* `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python standard
library.)
- `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python
standard library.)
Scrapy comes with its own mechanism for extracting data. They're called
selectors because they "select" certain parts of the HTML document specified

20
docs/topics/settings.rst
View File

@ -657,6 +657,7 @@ DOWNLOAD_HANDLERS_BASE
Default::
{
'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler',
'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
@ -677,6 +678,8 @@ handler (without replacement), place this in your ``settings.py``::
'ftp': None,
}
.. _http2:
The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update
:setting:`DOWNLOAD_HANDLERS` as follows::
@ -703,7 +706,8 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update
- No support for `server pushes`_, which are ignored.
- No support for the :signal:`bytes_received` signal.
- No support for the :signal:`bytes_received` and
:signal:`headers_received` signals.
.. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2
.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
@ -1229,20 +1233,6 @@ Adjust redirect request priority relative to original request:
- **a positive priority adjust (default) means higher priority.**
- a negative priority adjust means lower priority.
.. setting:: RETRY_PRIORITY_ADJUST
RETRY_PRIORITY_ADJUST
---------------------
Default: ``-1``
Scope: ``scrapy.downloadermiddlewares.retry.RetryMiddleware``
Adjust retry request priority relative to original request:
- a positive priority adjust means higher priority.
- **a negative priority adjust (default) means lower priority.**
.. setting:: ROBOTSTXT_OBEY
ROBOTSTXT_OBEY

45
docs/topics/shell.rst
View File

@ -95,20 +95,21 @@ convenience.
Available Shortcuts
-------------------
* ``shelp()`` - print a help with the list of available objects and shortcuts
- ``shelp()`` - print a help with the list of available objects and
shortcuts
* ``fetch(url[, redirect=True])`` - fetch a new response from the given
URL and update all related objects accordingly. You can optionaly ask for
HTTP 3xx redirections to not be followed by passing ``redirect=False``
- ``fetch(url[, redirect=True])`` - fetch a new response from the given URL
and update all related objects accordingly. You can optionaly ask for HTTP
3xx redirections to not be followed by passing ``redirect=False``
* ``fetch(request)`` - fetch a new response from the given request and
update all related objects accordingly.
- ``fetch(request)`` - fetch a new response from the given request and update
all related objects accordingly.
* ``view(response)`` - open the given response in your local web browser, for
inspection. This will add a `\<base\> tag`_ to the response body in order
for external links (such as images and style sheets) to display properly.
Note, however, that this will create a temporary file in your computer,
which won't be removed automatically.
- ``view(response)`` - open the given response in your local web browser, for
inspection. This will add a `\<base\> tag`_ to the response body in order
for external links (such as images and style sheets) to display properly.
Note, however, that this will create a temporary file in your computer,
which won't be removed automatically.
.. _<base> tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/base
@ -122,21 +123,21 @@ content).
Those objects are:
* ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object.
- ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object.
* ``spider`` - the Spider which is known to handle the URL, or a
:class:`~scrapy.spiders.Spider` object if there is no spider found for
the current URL
- ``spider`` - the Spider which is known to handle the URL, or a
:class:`~scrapy.spiders.Spider` object if there is no spider found for the
current URL
* ``request`` - a :class:`~scrapy.http.Request` object of the last fetched
page. You can modify this request using :meth:`~scrapy.http.Request.replace`
or fetch a new request (without leaving the shell) using the ``fetch``
shortcut.
- ``request`` - a :class:`~scrapy.http.Request` object of the last fetched
page. You can modify this request using
:meth:`~scrapy.http.Request.replace` or fetch a new request (without
leaving the shell) using the ``fetch`` shortcut.
* ``response`` - a :class:`~scrapy.http.Response` object containing the last
fetched page
- ``response`` - a :class:`~scrapy.http.Response` object containing the last
fetched page
* ``settings`` - the current :ref:`Scrapy settings <topics-settings>`
- ``settings`` - the current :ref:`Scrapy settings <topics-settings>`
Example of shell session
========================

2
docs/topics/signals.rst
View File

@ -403,7 +403,7 @@ bytes_received
headers_received
~~~~~~~~~~~~~~~~
.. versionadded:: VERSION
.. versionadded:: 2.5
.. signal:: headers_received
.. function:: headers_received(headers, request, spider)

3
docs/topics/spider-middleware.rst
View File

@ -260,7 +260,8 @@ this::
The ``handle_httpstatus_list`` key of :attr:`Request.meta
<scrapy.http.Request.meta>` can also be used to specify which response codes to
allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all``
to ``True`` if you want to allow any response code for a request.
to ``True`` if you want to allow any response code for a request, and ``False`` to
disable the effects of the ``handle_httpstatus_all`` key.
Keep in mind, however, that it's usually a bad idea to handle non-200
responses, unless you really know what you're doing.

2
scrapy/VERSION
View File

@ -1 +1 @@
2.4.1
2.5.0

2
scrapy/__init__.py
View File

@ -22,7 +22,7 @@ __all__ = [
# Scrapy and Twisted versions
__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip()
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
twisted_version = (_txv.major, _txv.minor, _txv.micro)

4
scrapy/commands/__init__.py
View File

@ -3,6 +3,8 @@ Base class for Scrapy commands
"""
import os
from optparse import OptionGroup
from typing import Any, Dict
from twisted.python import failure
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
@ -15,7 +17,7 @@ class ScrapyCommand:
crawler_process = None
# default settings to be used for this command instead of global defaults
default_settings = {}
default_settings: Dict[str, Any] = {}
exitcode = 0

2
scrapy/commands/bench.py
View File

@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider):
def start_requests(self):
qargs = {'total': self.total, 'show': self.show}
url = f'{self.baseurl}?{urlencode(qargs, doseq=1)}'
url = f'{self.baseurl}?{urlencode(qargs, doseq=True)}'
return [scrapy.Request(url, dont_filter=True)]
def parse(self, response):

2
scrapy/commands/crawl.py
View File

@ -16,7 +16,7 @@ class Command(BaseRunSpiderCommand):
if len(args) < 1:
raise UsageError()
elif len(args) > 1:
raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
raise UsageError("running 'scrapy crawl' with more than one spider is not supported")
spname = args[0]
crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)

6
scrapy/commands/parse.py
View File

@ -1,5 +1,6 @@
import json
import logging
from typing import Dict
from itemadapter import is_item, ItemAdapter
from w3lib.url import is_url
@ -10,6 +11,7 @@ from scrapy.utils import display
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
from scrapy.exceptions import UsageError
logger = logging.getLogger(__name__)
@ -17,8 +19,8 @@ class Command(BaseRunSpiderCommand):
requires_project = True
spider = None
items = {}
requests = {}
items: Dict[int, list] = {}
requests: Dict[int, list] = {}
first_response = None

127
scrapy/contracts/__init__.py
View File

@ -1,16 +1,77 @@
import sys
import re
import sys
from functools import wraps
from inspect import getmembers
from typing import Dict
from unittest import TestCase
from scrapy.http import Request
from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.python import get_spec
from scrapy.utils.spider import iterate_spider_output
class Contract:
""" Abstract class for contracts """
request_cls = None
def __init__(self, method, *args):
self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
self.args = args
def add_pre_hook(self, request, results):
if hasattr(self, 'pre_process'):
cb = request.callback
@wraps(cb)
def wrapper(response, **cb_kwargs):
try:
results.startTest(self.testcase_pre)
self.pre_process(response)
results.stopTest(self.testcase_pre)
except AssertionError:
results.addFailure(self.testcase_pre, sys.exc_info())
except Exception:
results.addError(self.testcase_pre, sys.exc_info())
else:
results.addSuccess(self.testcase_pre)
finally:
return list(iterate_spider_output(cb(response, **cb_kwargs)))
request.callback = wrapper
return request
def add_post_hook(self, request, results):
if hasattr(self, 'post_process'):
cb = request.callback
@wraps(cb)
def wrapper(response, **cb_kwargs):
output = list(iterate_spider_output(cb(response, **cb_kwargs)))
try:
results.startTest(self.testcase_post)
self.post_process(output)
results.stopTest(self.testcase_post)
except AssertionError:
results.addFailure(self.testcase_post, sys.exc_info())
except Exception:
results.addError(self.testcase_post, sys.exc_info())
else:
results.addSuccess(self.testcase_post)
finally:
return output
request.callback = wrapper
return request
def adjust_request_args(self, args):
return args
class ContractsManager:
contracts = {}
contracts: Dict[str, Contract] = {}
def __init__(self, contracts):
for contract in contracts:
@ -107,66 +168,6 @@ class ContractsManager:
request.errback = eb_wrapper
class Contract:
""" Abstract class for contracts """
request_cls = None
def __init__(self, method, *args):
self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
self.args = args
def add_pre_hook(self, request, results):
if hasattr(self, 'pre_process'):
cb = request.callback
@wraps(cb)
def wrapper(response, **cb_kwargs):
try:
results.startTest(self.testcase_pre)
self.pre_process(response)
results.stopTest(self.testcase_pre)
except AssertionError:
results.addFailure(self.testcase_pre, sys.exc_info())
except Exception:
results.addError(self.testcase_pre, sys.exc_info())
else:
results.addSuccess(self.testcase_pre)
finally:
return list(iterate_spider_output(cb(response, **cb_kwargs)))
request.callback = wrapper
return request
def add_post_hook(self, request, results):
if hasattr(self, 'post_process'):
cb = request.callback
@wraps(cb)
def wrapper(response, **cb_kwargs):
output = list(iterate_spider_output(cb(response, **cb_kwargs)))
try:
results.startTest(self.testcase_post)
self.post_process(output)
results.stopTest(self.testcase_post)
except AssertionError:
results.addFailure(self.testcase_post, sys.exc_info())
except Exception:
results.addError(self.testcase_post, sys.exc_info())
else:
results.addSuccess(self.testcase_post)
finally:
return output
request.callback = wrapper
return request
def adjust_request_args(self, args):
return args
def _create_testcase(method, desc):
spider = method.__self__.name

23
scrapy/core/downloader/middleware.py
View File

@ -3,8 +3,12 @@ Downloader Middleware manager
See documentation in docs/topics/downloader-middleware.rst
"""
from twisted.internet import defer
from typing import Callable, Union
from twisted.internet import defer
from twisted.python.failure import Failure
from scrapy import Spider
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
from scrapy.middleware import MiddlewareManager
@ -29,15 +33,14 @@ class DownloaderMiddlewareManager(MiddlewareManager):
if hasattr(mw, 'process_exception'):
self.methods['process_exception'].appendleft(mw.process_exception)
def download(self, download_func, request, spider):
def download(self, download_func: Callable, request: Request, spider: Spider):
@defer.inlineCallbacks
def process_request(request):
def process_request(request: Request):
for method in self.methods['process_request']:
response = yield deferred_from_coro(method(request=request, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput(
f"Middleware {method.__self__.__class__.__name__}"
".process_request must return None, Response or "
f"Middleware {method.__qualname__} must return None, Response or "
f"Request, got {response.__class__.__name__}"
)
if response:
@ -45,7 +48,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
return (yield download_func(request=request, spider=spider))
@defer.inlineCallbacks
def process_response(response):
def process_response(response: Union[Response, Request]):
if response is None:
raise TypeError("Received None in process_response")
elif isinstance(response, Request):
@ -55,8 +58,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
if not isinstance(response, (Response, Request)):
raise _InvalidOutput(
f"Middleware {method.__self__.__class__.__name__}"
".process_response must return Response or Request, "
f"Middleware {method.__qualname__} must return Response or Request, "
f"got {type(response)}"
)
if isinstance(response, Request):
@ -64,14 +66,13 @@ class DownloaderMiddlewareManager(MiddlewareManager):
return response
@defer.inlineCallbacks
def process_exception(failure):
def process_exception(failure: Failure):
exception = failure.value
for method in self.methods['process_exception']:
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput(
f"Middleware {method.__self__.__class__.__name__}"
".process_exception must return None, Response or "
f"Middleware {method.__qualname__} must return None, Response or "
f"Request, got {type(response)}"
)
if response:

46
scrapy/core/scraper.py
View File

@ -3,12 +3,14 @@ extracts information from them"""
import collections
import logging
from collections import deque
from collections.abc import Iterable
from typing import Union
from itemadapter import is_item
from twisted.internet import defer
from twisted.python.failure import Failure
from scrapy import signals
from scrapy import signals, Spider
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
from scrapy.http import Request, Response
@ -90,28 +92,26 @@ class Scraper:
def close_spider(self, spider):
"""Close a spider being scraped and release its resources"""
slot = self.slot
slot.closing = defer.Deferred()
slot.closing.addCallback(self.itemproc.close_spider)
self._check_if_closing(spider, slot)
return slot.closing
self.slot.closing = defer.Deferred()
self.slot.closing.addCallback(self.itemproc.close_spider)
self._check_if_closing(spider)
return self.slot.closing
def is_idle(self):
"""Return True if there isn't any more spiders to process"""
return not self.slot
def _check_if_closing(self, spider, slot):
if slot.closing and slot.is_idle():
slot.closing.callback(spider)
def _check_if_closing(self, spider):
if self.slot.closing and self.slot.is_idle():
self.slot.closing.callback(spider)
def enqueue_scrape(self, response, request, spider):
slot = self.slot
dfd = slot.add_response_request(response, request)
dfd = self.slot.add_response_request(response, request)
def finish_scraping(_):
slot.finish_response(response, request)
self._check_if_closing(spider, slot)
self._scrape_next(spider, slot)
self.slot.finish_response(response, request)
self._check_if_closing(spider)
self._scrape_next(spider)
return _
dfd.addBoth(finish_scraping)
@ -120,15 +120,15 @@ class Scraper:
{'request': request},
exc_info=failure_to_exc_info(f),
extra={'spider': spider}))
self._scrape_next(spider, slot)
self._scrape_next(spider)
return dfd
def _scrape_next(self, spider, slot):
while slot.queue:
response, request, deferred = slot.next_response_request_deferred()
def _scrape_next(self, spider):
while self.slot.queue:
response, request, deferred = self.slot.next_response_request_deferred()
self._scrape(response, request, spider).chainDeferred(deferred)
def _scrape(self, result, request, spider):
def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider):
"""
Handle the downloaded response or failure through the spider callback/errback
"""
@ -139,7 +139,7 @@ class Scraper:
dfd.addCallback(self.handle_spider_output, request, result, spider)
return dfd
def _scrape2(self, result, request, spider):
def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider):
"""
Handle the different cases of request's result been a Response or a Failure
"""
@ -149,7 +149,7 @@ class Scraper:
dfd = self.call_spider(result, request, spider)
return dfd.addErrback(self._log_download_errors, result, request, spider)
def call_spider(self, result, request, spider):
def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider):
if isinstance(result, Response):
if getattr(result, "request", None) is None:
result.request = request
@ -164,7 +164,7 @@ class Scraper:
dfd.addErrback(request.errback)
return dfd.addCallback(iterate_spider_output)
def handle_spider_error(self, _failure, request, response, spider):
def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider):
exc = _failure.value
if isinstance(exc, CloseSpider):
self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
@ -185,7 +185,7 @@ class Scraper:
spider=spider
)
def handle_spider_output(self, result, request, response, spider):
def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider):
if not result:
return defer_succeed(None)
if isinstance(result, collections.abc.AsyncIterable):

34
scrapy/core/spidermw.py
View File

@ -5,10 +5,14 @@ See documentation in docs/topics/spider-middleware.rst
"""
import collections.abc
from itertools import islice
from typing import Any, Callable, Generator, Iterable, Union, AsyncIterable
from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
from scrapy import Request, Spider
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Response
from scrapy.middleware import MiddlewareManager
from scrapy.utils.asyncgen import _process_iterable_universal
from scrapy.utils.conf import build_component_list
@ -16,8 +20,11 @@ from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.python import MutableAsyncChain, MutableChain
def _isiterable(possible_iterator):
return hasattr(possible_iterator, '__iter__') or hasattr(possible_iterator, '__aiter__')
ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
def _isiterable(o) -> bool:
return isinstance(o, (Iterable, AsyncIterable))
class SpiderMiddlewareManager(MiddlewareManager):
@ -39,7 +46,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
process_spider_exception = getattr(mw, 'process_spider_exception', None)
self.methods['process_spider_exception'].appendleft(process_spider_exception)
def _process_spider_input(self, scrape_func, response, request, spider):
def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request,
spider: Spider) -> Any:
for method in self.methods['process_spider_input']:
try:
result = method(response=response, spider=spider)
@ -53,7 +61,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
return scrape_func(Failure(), request, spider)
return scrape_func(response, request, spider)
def _evaluate_iterable(self, response, spider, iterable, exception_processor_index, recover_to):
def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable,
exception_processor_index: int, recover_to: MutableChain) -> Generator:
@_process_iterable_universal
async def _evaluate_async_iterable(iterable):
try:
@ -67,7 +76,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
recover_to.extend(exception_result)
return _evaluate_async_iterable(iterable)
def _process_spider_exception(self, response, spider, _failure, start_index=0):
def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure,
start_index: int = 0) -> Union[Failure, MutableChain]:
exception = _failure.value
# don't handle _InvalidOutput exception
if isinstance(exception, _InvalidOutput):
@ -89,7 +99,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
raise _InvalidOutput(msg)
return _failure
def _process_spider_output(self, response, spider, result, start_index=0):
def _process_spider_output(self, response: Response, spider: Spider,
result: Iterable, start_index: int = 0) -> MutableChain:
# items in this iterable do not need to go through the process_spider_output
# chain, they went through it already from the process_spider_exception method
last_result_async = isinstance(result, collections.abc.AsyncIterator)
@ -125,7 +136,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
else:
return MutableChain(result, recovered)
def _process_callback_output(self, response, spider, result):
def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain:
if isinstance(result, collections.abc.AsyncIterator):
recovered = MutableAsyncChain()
else:
@ -137,16 +148,17 @@ class SpiderMiddlewareManager(MiddlewareManager):
else:
return MutableChain(result, recovered)
def scrape_response(self, scrape_func, response, request, spider):
def process_callback_output(result):
def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request,
spider: Spider) -> Deferred:
def process_callback_output(result: Iterable) -> MutableChain:
return self._process_callback_output(response, spider, result)
def process_spider_exception(_failure):
def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
return self._process_spider_exception(response, spider, _failure)
dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)
dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)
return dfd
def process_start_requests(self, start_requests, spider):
def process_start_requests(self, start_requests, spider: Spider) -> Deferred:
return self._process_chain('process_start_requests', start_requests, spider)

25
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -1,10 +1,12 @@
import io
import warnings
import zlib
from scrapy.utils.gz import gunzip
from scrapy.exceptions import NotConfigured
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.exceptions import NotConfigured
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.gz import gunzip
ACCEPTED_ENCODINGS = [b'gzip', b'deflate']
@ -25,11 +27,25 @@ except ImportError:
class HttpCompressionMiddleware:
"""This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites"""
def __init__(self, stats=None):
self.stats = stats
@classmethod
def from_crawler(cls, crawler):
if not crawler.settings.getbool('COMPRESSION_ENABLED'):
raise NotConfigured
return cls()
try:
return cls(stats=crawler.stats)
except TypeError:
warnings.warn(
"HttpCompressionMiddleware subclasses must either modify "
"their '__init__' method to support a 'stats' parameter or "
"reimplement the 'from_crawler' method.",
ScrapyDeprecationWarning,
)
result = cls()
result.stats = crawler.stats
return result
def process_request(self, request, spider):
request.headers.setdefault('Accept-Encoding',
@ -44,6 +60,9 @@ class HttpCompressionMiddleware:
if content_encoding:
encoding = content_encoding.pop()
decoded_body = self._decode(response.body, encoding.lower())
if self.stats:
self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
self.stats.inc_value('httpcompression/response_count', spider=spider)
respcls = responsetypes.from_args(
headers=response.headers, url=response.url, body=decoded_body
)

138
scrapy/downloadermiddlewares/retry.py
View File

@ -9,7 +9,8 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
Failed pages are collected on the scraping process and rescheduled at the end,
once the spider has finished crawling all regular (non failed) pages.
"""
import logging
from logging import getLogger, Logger
from typing import Optional, Union
from twisted.internet import defer
from twisted.internet.error import (
@ -23,12 +24,104 @@ from twisted.internet.error import (
)
from twisted.web.client import ResponseFailed
from scrapy.exceptions import NotConfigured
from scrapy.utils.response import response_status_message
from scrapy.core.downloader.handlers.http11 import TunnelError
from scrapy.exceptions import NotConfigured
from scrapy.http.request import Request
from scrapy.spiders import Spider
from scrapy.utils.python import global_object_name
from scrapy.utils.response import response_status_message
logger = logging.getLogger(__name__)
retry_logger = getLogger(__name__)
def get_retry_request(
request: Request,
*,
spider: Spider,
reason: Union[str, Exception] = 'unspecified',
max_retry_times: Optional[int] = None,
priority_adjust: Optional[int] = None,
logger: Logger = retry_logger,
stats_base_key: str = 'retry',
):
"""
Returns a new :class:`~scrapy.Request` object to retry the specified
request, or ``None`` if retries of the specified request have been
exhausted.
For example, in a :class:`~scrapy.Spider` callback, you could use it as
follows::
def parse(self, response):
if not response.text:
new_request_or_none = get_retry_request(
response.request,
spider=self,
reason='empty',
)
return new_request_or_none
*spider* is the :class:`~scrapy.Spider` instance which is asking for the
retry request. It is used to access the :ref:`settings <topics-settings>`
and :ref:`stats <topics-stats>`, and to provide extra logging context (see
:func:`logging.debug`).
*reason* is a string or an :class:`Exception` object that indicates the
reason why the request needs to be retried. It is used to name retry stats.
*max_retry_times* is a number that determines the maximum number of times
that *request* can be retried. If not specified or ``None``, the number is
read from the :reqmeta:`max_retry_times` meta key of the request. If the
:reqmeta:`max_retry_times` meta key is not defined or ``None``, the number
is read from the :setting:`RETRY_TIMES` setting.
*priority_adjust* is a number that determines how the priority of the new
request changes in relation to *request*. If not specified, the number is
read from the :setting:`RETRY_PRIORITY_ADJUST` setting.
*logger* is the logging.Logger object to be used when logging messages
*stats_base_key* is a string to be used as the base key for the
retry-related job stats
"""
settings = spider.crawler.settings
stats = spider.crawler.stats
retry_times = request.meta.get('retry_times', 0) + 1
if max_retry_times is None:
max_retry_times = request.meta.get('max_retry_times')
if max_retry_times is None:
max_retry_times = settings.getint('RETRY_TIMES')
if retry_times <= max_retry_times:
logger.debug(
"Retrying %(request)s (failed %(retry_times)d times): %(reason)s",
{'request': request, 'retry_times': retry_times, 'reason': reason},
extra={'spider': spider}
)
new_request = request.copy()
new_request.meta['retry_times'] = retry_times
new_request.dont_filter = True
if priority_adjust is None:
priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
new_request.priority = request.priority + priority_adjust
if callable(reason):
reason = reason()
if isinstance(reason, Exception):
reason = global_object_name(reason.__class__)
stats.inc_value(f'{stats_base_key}/count')
stats.inc_value(f'{stats_base_key}/reason_count/{reason}')
return new_request
else:
stats.inc_value(f'{stats_base_key}/max_reached')
logger.error(
"Gave up retrying %(request)s (failed %(retry_times)d times): "
"%(reason)s",
{'request': request, 'retry_times': retry_times, 'reason': reason},
extra={'spider': spider},
)
return None
class RetryMiddleware:
@ -67,31 +160,12 @@ class RetryMiddleware:
return self._retry(request, exception, spider)
def _retry(self, request, reason, spider):
retries = request.meta.get('retry_times', 0) + 1
retry_times = self.max_retry_times
if 'max_retry_times' in request.meta:
retry_times = request.meta['max_retry_times']
stats = spider.crawler.stats
if retries <= retry_times:
logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",
{'request': request, 'retries': retries, 'reason': reason},
extra={'spider': spider})
retryreq = request.copy()
retryreq.meta['retry_times'] = retries
retryreq.dont_filter = True
retryreq.priority = request.priority + self.priority_adjust
if isinstance(reason, Exception):
reason = global_object_name(reason.__class__)
stats.inc_value('retry/count')
stats.inc_value(f'retry/reason_count/{reason}')
return retryreq
else:
stats.inc_value('retry/max_reached')
logger.error("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",
{'request': request, 'retries': retries, 'reason': reason},
extra={'spider': spider})
max_retry_times = request.meta.get('max_retry_times', self.max_retry_times)
priority_adjust = request.meta.get('priority_adjust', self.priority_adjust)
return get_retry_request(
request,
reason=reason,
spider=spider,
max_retry_times=max_retry_times,
priority_adjust=priority_adjust,
)

8
scrapy/http/cookies.py
View File

@ -1,10 +1,16 @@
import re
import time
from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
# Defined in the http.cookiejar module, but undocumented:
# https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527
IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
class CookieJar:
def __init__(self, policy=None, check_expired_frequency=10000):
self.policy = policy or DefaultCookiePolicy()

2
scrapy/http/request/form.py
View File

@ -71,7 +71,7 @@ def _urlencode(seq, enc):
values = [(to_bytes(k, enc), to_bytes(v, enc))
for k, vs in seq
for v in (vs if is_listlike(vs) else [vs])]
return urlencode(values, doseq=1)
return urlencode(values, doseq=True)
def _get_form(response, formname, formid, formnumber, formxpath):

3
scrapy/item.py
View File

@ -8,6 +8,7 @@ from abc import ABCMeta
from collections.abc import MutableMapping
from copy import deepcopy
from pprint import pformat
from typing import Dict
from warnings import warn
from scrapy.utils.deprecate import ScrapyDeprecationWarning
@ -75,7 +76,7 @@ class ItemMeta(_BaseItemMeta):
class DictItem(MutableMapping, BaseItem):
fields = {}
fields: Dict[str, Field] = {}
def __new__(cls, *args, **kwargs):
if issubclass(cls, DictItem) and not issubclass(cls, Item):

7
scrapy/mail.py
View File

@ -9,7 +9,7 @@ from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
from email.mime.nonmultipart import MIMENonMultipart
from email.mime.text import MIMEText
from email.utils import COMMASPACE, formatdate
from email.utils import formatdate
from io import BytesIO
from twisted.internet import defer, ssl
@ -21,6 +21,11 @@ from scrapy.utils.python import to_bytes
logger = logging.getLogger(__name__)
# Defined in the email.utils module, but undocumented:
# https://github.com/python/cpython/blob/v3.9.0/Lib/email/utils.py#L42
COMMASPACE = ", "
def _to_bytes_or_none(text):
if text is None:
return None

25
scrapy/middleware.py
View File

@ -1,8 +1,13 @@
from collections import defaultdict, deque
import logging
import pprint
from collections import defaultdict, deque
from typing import Callable, Deque, Dict
from twisted.internet.defer import Deferred
from scrapy import Spider
from scrapy.exceptions import NotConfigured
from scrapy.settings import Settings
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.defer import process_parallel, process_chain, process_chain_both
@ -16,16 +21,16 @@ class MiddlewareManager:
def __init__(self, *middlewares):
self.middlewares = middlewares
self.methods = defaultdict(deque)
self.methods: Dict[str, Deque[Callable]] = defaultdict(deque)
for mw in middlewares:
self._add_middleware(mw)
@classmethod
def _get_mwlist_from_settings(cls, settings):
def _get_mwlist_from_settings(cls, settings: Settings) -> list:
raise NotImplementedError
@classmethod
def from_settings(cls, settings, crawler=None):
def from_settings(cls, settings: Settings, crawler=None):
mwlist = cls._get_mwlist_from_settings(settings)
middlewares = []
enabled = []
@ -52,24 +57,24 @@ class MiddlewareManager:
def from_crawler(cls, crawler):
return cls.from_settings(crawler.settings, crawler)
def _add_middleware(self, mw):
def _add_middleware(self, mw) -> None:
if hasattr(mw, 'open_spider'):
self.methods['open_spider'].append(mw.open_spider)
if hasattr(mw, 'close_spider'):
self.methods['close_spider'].appendleft(mw.close_spider)
def _process_parallel(self, methodname, obj, *args):
def _process_parallel(self, methodname: str, obj, *args) -> Deferred:
return process_parallel(self.methods[methodname], obj, *args)
def _process_chain(self, methodname, obj, *args):
def _process_chain(self, methodname: str, obj, *args) -> Deferred:
return process_chain(self.methods[methodname], obj, *args)
def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args):
def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred:
return process_chain_both(self.methods[cb_methodname],
self.methods[eb_methodname], obj, *args)
def open_spider(self, spider):
def open_spider(self, spider: Spider) -> Deferred:
return self._process_parallel('open_spider', spider)
def close_spider(self, spider):
def close_spider(self, spider: Spider) -> Deferred:
return self._process_parallel('close_spider', spider)

2
scrapy/spidermiddlewares/httperror.py
View File

@ -32,7 +32,7 @@ class HttpErrorMiddleware:
if 200 <= response.status < 300: # common case
return
meta = response.meta
if 'handle_httpstatus_all' in meta:
if meta.get('handle_httpstatus_all', False):
return
if 'handle_httpstatus_list' in meta:
allowed_statuses = meta['handle_httpstatus_list']

30
scrapy/spidermiddlewares/referer.py
View File

@ -3,16 +3,17 @@ RefererMiddleware: populates Request referer field, based on the Response which
originated it.
"""
import warnings
from typing import Tuple
from urllib.parse import urlparse
from w3lib.url import safe_url_string
from scrapy.http import Request, Response
from scrapy.exceptions import NotConfigured
from scrapy import signals
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.utils.asyncgen import _process_iterable_universal
from scrapy.utils.python import to_unicode
from scrapy.utils.misc import load_object
from scrapy.utils.python import to_unicode
from scrapy.utils.url import strip_url
@ -31,7 +32,8 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default"
class ReferrerPolicy:
NOREFERRER_SCHEMES = LOCAL_SCHEMES
NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES
name: str
def referrer(self, response_url, request_url):
raise NotImplementedError()
@ -89,7 +91,7 @@ class NoReferrerPolicy(ReferrerPolicy):
is to be sent along with requests made from a particular request client to any origin.
The header will be omitted entirely.
"""
name = POLICY_NO_REFERRER
name: str = POLICY_NO_REFERRER
def referrer(self, response_url, request_url):
return None
@ -109,7 +111,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
This is a user agent's default behavior, if no policy is otherwise specified.
"""
name = POLICY_NO_REFERRER_WHEN_DOWNGRADE
name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE
def referrer(self, response_url, request_url):
if not self.tls_protected(response_url) or self.tls_protected(request_url):
@ -126,7 +128,7 @@ class SameOriginPolicy(ReferrerPolicy):
Cross-origin requests, on the other hand, will contain no referrer information.
A Referer HTTP header will not be sent.
"""
name = POLICY_SAME_ORIGIN
name: str = POLICY_SAME_ORIGIN
def referrer(self, response_url, request_url):
if self.origin(response_url) == self.origin(request_url):
@ -142,7 +144,7 @@ class OriginPolicy(ReferrerPolicy):
when making both same-origin requests and cross-origin requests
from a particular request client.
"""
name = POLICY_ORIGIN
name: str = POLICY_ORIGIN
def referrer(self, response_url, request_url):
return self.origin_referrer(response_url)
@ -161,7 +163,7 @@ class StrictOriginPolicy(ReferrerPolicy):
on the other hand, will contain no referrer information.
A Referer HTTP header will not be sent.
"""
name = POLICY_STRICT_ORIGIN
name: str = POLICY_STRICT_ORIGIN
def referrer(self, response_url, request_url):
if (
@ -182,7 +184,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):
is sent as referrer information when making cross-origin requests
from a particular request client.
"""
name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
def referrer(self, response_url, request_url):
origin = self.origin(response_url)
@ -209,7 +211,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
on the other hand, will contain no referrer information.
A Referer HTTP header will not be sent.
"""
name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN
name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN
def referrer(self, response_url, request_url):
origin = self.origin(response_url)
@ -235,7 +237,7 @@ class UnsafeUrlPolicy(ReferrerPolicy):
to insecure origins.
Carefully consider the impact of setting such a policy for potentially sensitive documents.
"""
name = POLICY_UNSAFE_URL
name: str = POLICY_UNSAFE_URL
def referrer(self, response_url, request_url):
return self.stripped_referrer(response_url)
@ -247,8 +249,8 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
with the addition that "Referer" is not sent if the parent request was
using ``file://`` or ``s3://`` scheme.
"""
NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3')
name = POLICY_SCRAPY_DEFAULT
NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ('file', 's3')
name: str = POLICY_SCRAPY_DEFAULT
_policy_classes = {p.name: p for p in (

3
scrapy/utils/asyncgen.py
View File

@ -1,9 +1,10 @@
import collections
import functools
import inspect
from collections.abc import AsyncIterable
async def collect_asyncgen(result):
async def collect_asyncgen(result: AsyncIterable):
results = []
async for x in result:
results.append(x)

59
scrapy/utils/defer.py
View File

@ -3,16 +3,21 @@ Helper functions for dealing with Twisted deferreds
"""
import asyncio
import inspect
from collections.abc import Coroutine
from functools import wraps
from typing import Any, Callable, Generator, Iterable
from twisted.internet import defer, task
from twisted.internet import defer
from twisted.internet.defer import Deferred, DeferredList, ensureDeferred
from twisted.internet.task import Cooperator
from twisted.python import failure
from twisted.python.failure import Failure
from scrapy.exceptions import IgnoreRequest
from scrapy.utils.reactor import is_asyncio_reactor_installed
def defer_fail(_failure):
def defer_fail(_failure: Failure) -> Deferred:
"""Same as twisted.internet.defer.fail but delay calling errback until
next reactor loop
@ -20,12 +25,12 @@ def defer_fail(_failure):
before attending pending delayed calls, so do not set delay to zero.
"""
from twisted.internet import reactor
d = defer.Deferred()
d = Deferred()
reactor.callLater(0.1, d.errback, _failure)
return d
def defer_succeed(result):
def defer_succeed(result) -> Deferred:
"""Same as twisted.internet.defer.succeed but delay calling callback until
next reactor loop
@ -33,13 +38,13 @@ def defer_succeed(result):
before attending pending delayed calls, so do not set delay to zero.
"""
from twisted.internet import reactor
d = defer.Deferred()
d = Deferred()
reactor.callLater(0.1, d.callback, result)
return d
def defer_result(result):
if isinstance(result, defer.Deferred):
def defer_result(result) -> Deferred:
if isinstance(result, Deferred):
return result
elif isinstance(result, failure.Failure):
return defer_fail(result)
@ -47,7 +52,7 @@ def defer_result(result):
return defer_succeed(result)
def mustbe_deferred(f, *args, **kw):
def mustbe_deferred(f: Callable, *args, **kw) -> Deferred:
"""Same as twisted.internet.defer.maybeDeferred, but delay calling
callback/errback to next reactor loop
"""
@ -64,15 +69,15 @@ def mustbe_deferred(f, *args, **kw):
return defer_result(result)
def parallel(iterable, count, callable, *args, **named):
def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> DeferredList:
"""Execute a callable over the objects in the given iterable, in parallel,
using no more than ``count`` concurrent calls.
Taken from: https://jcalderone.livejournal.com/24285.html
"""
coop = task.Cooperator()
coop = Cooperator()
work = (callable(elem, *args, **named) for elem in iterable)
return defer.DeferredList([coop.coiterate(work) for _ in range(count)])
return DeferredList([coop.coiterate(work) for _ in range(count)])
class _AsyncCooperatorAdapter:
@ -172,24 +177,24 @@ class _AsyncCooperatorAdapter:
def parallel_async(async_iterable, count, callable, *args, **named):
""" Like parallel but for async iterables """
coop = task.Cooperator()
coop = Cooperator()
work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named)
dl = defer.DeferredList([coop.coiterate(work) for _ in range(count)])
return dl
def process_chain(callbacks, input, *a, **kw):
def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
"""Return a Deferred built by chaining the given callbacks"""
d = defer.Deferred()
d = Deferred()
for x in callbacks:
d.addCallback(x, *a, **kw)
d.callback(input)
return d
def process_chain_both(callbacks, errbacks, input, *a, **kw):
def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
"""Return a Deferred built by chaining the given callbacks and errbacks"""
d = defer.Deferred()
d = Deferred()
for cb, eb in zip(callbacks, errbacks):
d.addCallbacks(
callback=cb, errback=eb,
@ -203,17 +208,17 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw):
return d
def process_parallel(callbacks, input, *a, **kw):
def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
"""Return a Deferred with the output of all successful calls to the given
callbacks
"""
dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks]
d = defer.DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True)
d = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True)
d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure)
return d
def iter_errback(iterable, errback, *a, **kw):
def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator:
"""Wraps an iterable calling an errback if an error is caught while
iterating it.
"""
@ -241,22 +246,22 @@ async def aiter_errback(aiterable, errback, *a, **kw):
errback(failure.Failure(), *a, **kw)
def deferred_from_coro(o):
def deferred_from_coro(o) -> Any:
"""Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
if isinstance(o, defer.Deferred):
if isinstance(o, Deferred):
return o
if asyncio.isfuture(o) or inspect.isawaitable(o):
if not is_asyncio_reactor_installed():
# wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines
# that use asyncio, e.g. "await asyncio.sleep(1)"
return defer.ensureDeferred(o)
return ensureDeferred(o)
else:
# wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
return defer.Deferred.fromFuture(asyncio.ensure_future(o))
return Deferred.fromFuture(asyncio.ensure_future(o))
return o
def deferred_f_from_coro_f(coro_f):
def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable:
""" Converts a coroutine function into a function that returns a Deferred.
The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it.
@ -268,14 +273,14 @@ def deferred_f_from_coro_f(coro_f):
return f
def maybeDeferred_coro(f, *args, **kw):
def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred:
""" Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """
try:
result = f(*args, **kw)
except: # noqa: E722
return defer.fail(failure.Failure(captureVars=defer.Deferred.debug))
return defer.fail(failure.Failure(captureVars=Deferred.debug))
if isinstance(result, defer.Deferred):
if isinstance(result, Deferred):
return result
elif asyncio.isfuture(result) or inspect.isawaitable(result):
return deferred_from_coro(result)

11
scrapy/utils/httpobj.py
View File

@ -1,13 +1,16 @@
"""Helper functions for scrapy.http objects (Request, Response)"""
import weakref
from urllib.parse import urlparse
from typing import Union
from urllib.parse import urlparse, ParseResult
from weakref import WeakKeyDictionary
from scrapy.http import Request, Response
_urlparse_cache = weakref.WeakKeyDictionary()
_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = WeakKeyDictionary()
def urlparse_cached(request_or_response):
def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult:
"""Return urlparse.urlparse caching the result, where the argument can be a
Request or Response object
"""

7
scrapy/utils/python.py
View File

@ -8,6 +8,7 @@ import re
import sys
import warnings
import weakref
from collections.abc import Iterable
from functools import partial, wraps
from itertools import chain
@ -336,15 +337,15 @@ else:
gc.collect()
class MutableChain:
class MutableChain(Iterable):
"""
Thin wrapper around itertools.chain, allowing to add iterables "in-place"
"""
def __init__(self, *args):
def __init__(self, *args: Iterable):
self.data = chain.from_iterable(args)
def extend(self, *iterables):
def extend(self, *iterables: Iterable):
self.data = chain(self.data, chain.from_iterable(iterables))
def __iter__(self):

28
scrapy/utils/request.py
View File

@ -4,20 +4,27 @@ scrapy.http.Request objects
"""
import hashlib
import weakref
from typing import Dict, Iterable, Optional, Tuple, Union
from urllib.parse import urlunparse
from weakref import WeakKeyDictionary
from w3lib.http import basic_auth_header
from w3lib.url import canonicalize_url
from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes, to_unicode
_fingerprint_cache = weakref.WeakKeyDictionary()
_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
_fingerprint_cache = WeakKeyDictionary()
def request_fingerprint(request, include_headers=None, keep_fragments=False):
def request_fingerprint(
request: Request,
include_headers: Optional[Iterable[Union[bytes, str]]] = None,
keep_fragments: bool = False,
):
"""
Return the request fingerprint.
@ -49,17 +56,18 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
(for instance when handling requests with a headless browser).
"""
headers: Optional[Tuple[bytes, ...]] = None
if include_headers:
include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
cache = _fingerprint_cache.setdefault(request, {})
cache_key = (include_headers, keep_fragments)
cache_key = (headers, keep_fragments)
if cache_key not in cache:
fp = hashlib.sha1()
fp.update(to_bytes(request.method))
fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
fp.update(request.body or b'')
if include_headers:
for hdr in include_headers:
if headers:
for hdr in headers:
if hdr in request.headers:
fp.update(hdr)
for v in request.headers.getlist(hdr):
@ -68,14 +76,14 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
return cache[cache_key]
def request_authenticate(request, username, password):
def request_authenticate(request: Request, username: str, password: str) -> None:
"""Autenticate the given request (in place) using the HTTP basic access
authentication mechanism (RFC 2617) and the given username and password
"""
request.headers['Authorization'] = basic_auth_header(username, password)
def request_httprepr(request):
def request_httprepr(request: Request) -> bytes:
"""Return the raw HTTP representation (as bytes) of the given request.
This is provided only for reference since it's not the actual stream of
bytes that will be send when performing the request (that's controlled
@ -92,7 +100,7 @@ def request_httprepr(request):
return s
def referer_str(request):
def referer_str(request: Request) -> Optional[str]:
""" Return Referer HTTP header suitable for logging. """
referrer = request.headers.get('Referer')
if referrer is None:

31
scrapy/utils/response.py
View File

@ -3,19 +3,23 @@ This module provides some useful functions for working with
scrapy.http.Response objects
"""
import os
import weakref
import webbrowser
import tempfile
from typing import Any, Callable, Iterable, Optional, Tuple, Union
from weakref import WeakKeyDictionary
import scrapy
from scrapy.http.response import Response
from twisted.web import http
from scrapy.utils.python import to_bytes, to_unicode
from w3lib import html
_baseurl_cache = weakref.WeakKeyDictionary()
_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary()
def get_base_url(response):
def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str:
"""Return the base url of the given response, joined with the response url"""
if response not in _baseurl_cache:
text = response.text[0:4096]
@ -23,10 +27,13 @@ def get_base_url(response):
return _baseurl_cache[response]
_metaref_cache = weakref.WeakKeyDictionary()
_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = WeakKeyDictionary()
def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
def get_meta_refresh(
response: "scrapy.http.response.text.TextResponse",
ignore_tags: Optional[Iterable[str]] = ('script', 'noscript'),
) -> Union[Tuple[None, None], Tuple[float, str]]:
"""Parse the http-equiv refrsh parameter from the given response"""
if response not in _metaref_cache:
text = response.text[0:4096]
@ -35,14 +42,15 @@ def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
return _metaref_cache[response]
def response_status_message(status):
def response_status_message(status: Union[bytes, float, int, str]) -> str:
"""Return status code plus status text descriptive message
"""
message = http.RESPONSES.get(int(status), "Unknown Status")
return f'{status} {to_unicode(message)}'
status_int = int(status)
message = http.RESPONSES.get(status_int, "Unknown Status")
return f'{status_int} {to_unicode(message)}'
def response_httprepr(response):
def response_httprepr(response: Response) -> bytes:
"""Return raw HTTP representation (as bytes) of the given response. This
is provided only for reference, since it's not the exact stream of bytes
that was received (that's not exposed by Twisted).
@ -60,7 +68,10 @@ def response_httprepr(response):
return b"".join(values)
def open_in_browser(response, _openfunc=webbrowser.open):
def open_in_browser(
response: Union["scrapy.http.response.html.HtmlResponse", "scrapy.http.response.text.TextResponse"],
_openfunc: Callable[[str], Any] = webbrowser.open,
) -> Any:
"""Open the given response in a local web browser, populating the <base>
tag for external links to work
"""

9
scrapy/utils/trackref.py
View File

@ -9,14 +9,15 @@ and no performance penalty at all when disabled (as object_ref becomes just an
alias to object in that case).
"""
import weakref
from time import time
from operator import itemgetter
from collections import defaultdict
from operator import itemgetter
from time import time
from typing import DefaultDict
from weakref import WeakKeyDictionary
NoneType = type(None)
live_refs = defaultdict(weakref.WeakKeyDictionary)
live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary)
class object_ref:

39
setup.cfg
View File

@ -10,54 +10,18 @@ follow_imports = skip
# FIXME: remove the following sections once the issues are solved
[mypy-scrapy]
ignore_errors = True
[mypy-scrapy.commands]
ignore_errors = True
[mypy-scrapy.commands.parse]
ignore_errors = True
[mypy-scrapy.downloadermiddlewares.httpproxy]
ignore_errors = True
[mypy-scrapy.contracts]
ignore_errors = True
[mypy-scrapy.interfaces]
ignore_errors = True
[mypy-scrapy.item]
ignore_errors = True
[mypy-scrapy.http.cookies]
ignore_errors = True
[mypy-scrapy.mail]
ignore_errors = True
[mypy-scrapy.pipelines.images]
ignore_errors = True
[mypy-scrapy.settings.default_settings]
ignore_errors = True
[mypy-scrapy.spidermiddlewares.referer]
ignore_errors = True
[mypy-scrapy.utils.httpobj]
ignore_errors = True
[mypy-scrapy.utils.request]
ignore_errors = True
[mypy-scrapy.utils.response]
ignore_errors = True
[mypy-scrapy.utils.trackref]
ignore_errors = True
[mypy-tests.mocks.dummydbm]
ignore_errors = True
@ -79,9 +43,6 @@ ignore_errors = True
[mypy-tests.test_downloader_handlers]
ignore_errors = True
[mypy-tests.test_engine]
ignore_errors = True
[mypy-tests.test_exporters]
ignore_errors = True

2
setup.py
View File

@ -31,7 +31,7 @@ install_requires = [
'zope.interface>=4.1.3',
'protego>=0.1.15',
'itemadapter>=0.1.0',
'h2>=3.2.0',
'h2>=3.0,<4.0',
]
extras_require = {}
cpython_dependencies = [

4
tests/spiders.py
View File

@ -46,7 +46,7 @@ class FollowAllSpider(MetaSpider):
self.urls_visited = []
self.times = []
qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency}
url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}")
url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}")
self.start_urls = [url]
def parse(self, response):
@ -281,7 +281,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
for s in range(100):
qargs = {'total': 10, 'seed': s}
url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}")
url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}")
yield Request(url, meta={'seed': s})
if self.fail_yielding:
2 / 0

108
tests/test_downloadermiddleware_httpcompression.py
View File

@ -1,13 +1,16 @@
from io import BytesIO
from unittest import TestCase, SkipTest
from os.path import join
from gzip import GzipFile
from io import BytesIO
from os.path import join
from unittest import TestCase, SkipTest
from warnings import catch_warnings
from scrapy.spiders import Spider
from scrapy.http import Response, Request, HtmlResponse
from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.responsetypes import responsetypes
from scrapy.utils.gz import gunzip
from scrapy.utils.test import get_crawler
from tests import tests_datadir
from w3lib.encoding import resolve_encoding
@ -32,8 +35,10 @@ FORMAT = {
class HttpCompressionTest(TestCase):
def setUp(self):
self.spider = Spider('foo')
self.mw = HttpCompressionMiddleware()
self.crawler = get_crawler(Spider)
self.spider = self.crawler._create_spider('scrapytest.org')
self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
self.crawler.stats.open_spider(self.spider)
def _getresponse(self, coding):
if coding not in FORMAT:
@ -56,6 +61,34 @@ class HttpCompressionTest(TestCase):
response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'})
return response
def assertStatsEqual(self, key, value):
self.assertEqual(
self.crawler.stats.get_value(key, spider=self.spider),
value,
str(self.crawler.stats.get_stats(self.spider))
)
def test_setting_false_compression_enabled(self):
self.assertRaises(
NotConfigured,
HttpCompressionMiddleware.from_crawler,
get_crawler(settings_dict={'COMPRESSION_ENABLED': False})
)
def test_setting_default_compression_enabled(self):
self.assertIsInstance(
HttpCompressionMiddleware.from_crawler(get_crawler()),
HttpCompressionMiddleware
)
def test_setting_true_compression_enabled(self):
self.assertIsInstance(
HttpCompressionMiddleware.from_crawler(
get_crawler(settings_dict={'COMPRESSION_ENABLED': True})
),
HttpCompressionMiddleware
)
def test_process_request(self):
request = Request('http://scrapytest.org')
assert 'Accept-Encoding' not in request.headers
@ -72,6 +105,20 @@ class HttpCompressionTest(TestCase):
assert newresponse is not response
assert newresponse.body.startswith(b'<!DOCTYPE')
assert 'Content-Encoding' not in newresponse.headers
self.assertStatsEqual('httpcompression/response_count', 1)
self.assertStatsEqual('httpcompression/response_bytes', 74837)
def test_process_response_gzip_no_stats(self):
mw = HttpCompressionMiddleware()
response = self._getresponse('gzip')
request = response.request
self.assertEqual(response.headers['Content-Encoding'], b'gzip')
newresponse = mw.process_response(request, response, self.spider)
self.assertEqual(mw.stats, None)
assert newresponse is not response
assert newresponse.body.startswith(b'<!DOCTYPE')
assert 'Content-Encoding' not in newresponse.headers
def test_process_response_br(self):
try:
@ -85,6 +132,8 @@ class HttpCompressionTest(TestCase):
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert 'Content-Encoding' not in newresponse.headers
self.assertStatsEqual('httpcompression/response_count', 1)
self.assertStatsEqual('httpcompression/response_bytes', 74837)
def test_process_response_zstd(self):
try:
@ -116,6 +165,8 @@ class HttpCompressionTest(TestCase):
assert newresponse is not response
assert newresponse.body.startswith(b'<!DOCTYPE')
assert 'Content-Encoding' not in newresponse.headers
self.assertStatsEqual('httpcompression/response_count', 1)
self.assertStatsEqual('httpcompression/response_bytes', 74840)
def test_process_response_zlibdelate(self):
response = self._getresponse('zlibdeflate')
@ -126,6 +177,8 @@ class HttpCompressionTest(TestCase):
assert newresponse is not response
assert newresponse.body.startswith(b'<!DOCTYPE')
assert 'Content-Encoding' not in newresponse.headers
self.assertStatsEqual('httpcompression/response_count', 1)
self.assertStatsEqual('httpcompression/response_bytes', 74840)
def test_process_response_plain(self):
response = Response('http://scrapytest.org', body=b'<!DOCTYPE...')
@ -135,6 +188,8 @@ class HttpCompressionTest(TestCase):
newresponse = self.mw.process_response(request, response, self.spider)
assert newresponse is response
assert newresponse.body.startswith(b'<!DOCTYPE')
self.assertStatsEqual('httpcompression/response_count', None)
self.assertStatsEqual('httpcompression/response_bytes', None)
def test_multipleencodings(self):
response = self._getresponse('gzip')
@ -162,6 +217,8 @@ class HttpCompressionTest(TestCase):
assert isinstance(newresponse, HtmlResponse)
self.assertEqual(newresponse.body, plainbody)
self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
self.assertStatsEqual('httpcompression/response_count', 1)
self.assertStatsEqual('httpcompression/response_bytes', 104)
def test_process_response_force_recalculate_encoding(self):
headers = {
@ -181,6 +238,8 @@ class HttpCompressionTest(TestCase):
assert isinstance(newresponse, HtmlResponse)
self.assertEqual(newresponse.body, plainbody)
self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
self.assertStatsEqual('httpcompression/response_count', 1)
self.assertStatsEqual('httpcompression/response_bytes', 104)
def test_process_response_no_content_type_header(self):
headers = {
@ -196,6 +255,8 @@ class HttpCompressionTest(TestCase):
assert isinstance(newresponse, respcls)
self.assertEqual(newresponse.body, plainbody)
self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
self.assertStatsEqual('httpcompression/response_count', 1)
self.assertStatsEqual('httpcompression/response_bytes', 104)
def test_process_response_gzipped_contenttype(self):
response = self._getresponse('gzip')
@ -206,6 +267,8 @@ class HttpCompressionTest(TestCase):
self.assertIsNot(newresponse, response)
self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
self.assertNotIn('Content-Encoding', newresponse.headers)
self.assertStatsEqual('httpcompression/response_count', 1)
self.assertStatsEqual('httpcompression/response_bytes', 74837)
def test_process_response_gzip_app_octetstream_contenttype(self):
response = self._getresponse('gzip')
@ -216,6 +279,8 @@ class HttpCompressionTest(TestCase):
self.assertIsNot(newresponse, response)
self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
self.assertNotIn('Content-Encoding', newresponse.headers)
self.assertStatsEqual('httpcompression/response_count', 1)
self.assertStatsEqual('httpcompression/response_bytes', 74837)
def test_process_response_gzip_binary_octetstream_contenttype(self):
response = self._getresponse('x-gzip')
@ -226,6 +291,8 @@ class HttpCompressionTest(TestCase):
self.assertIsNot(newresponse, response)
self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
self.assertNotIn('Content-Encoding', newresponse.headers)
self.assertStatsEqual('httpcompression/response_count', 1)
self.assertStatsEqual('httpcompression/response_bytes', 74837)
def test_process_response_gzipped_gzip_file(self):
"""Test that a gzip Content-Encoded .gz file is gunzipped
@ -268,6 +335,8 @@ class HttpCompressionTest(TestCase):
newresponse = self.mw.process_response(request, response, self.spider)
self.assertEqual(gunzip(newresponse.body), plainbody)
self.assertStatsEqual('httpcompression/response_count', 1)
self.assertStatsEqual('httpcompression/response_bytes', 230)
def test_process_response_head_request_no_decode_required(self):
response = self._getresponse('gzip')
@ -278,3 +347,32 @@ class HttpCompressionTest(TestCase):
newresponse = self.mw.process_response(request, response, self.spider)
self.assertIs(newresponse, response)
self.assertEqual(response.body, b'')
self.assertStatsEqual('httpcompression/response_count', None)
self.assertStatsEqual('httpcompression/response_bytes', None)
class HttpCompressionSubclassTest(TestCase):
def test_init_missing_stats(self):
class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware):
def __init__(self):
super().__init__()
crawler = get_crawler(Spider)
with catch_warnings(record=True) as caught_warnings:
HttpCompressionMiddlewareSubclass.from_crawler(crawler)
messages = tuple(
str(warning.message) for warning in caught_warnings
if warning.category is ScrapyDeprecationWarning
)
self.assertEqual(
messages,
(
(
"HttpCompressionMiddleware subclasses must either modify "
"their '__init__' method to support a 'stats' parameter "
"or reimplement the 'from_crawler' method."
),
)
)

530
tests/test_downloadermiddleware_retry.py
View File

@ -1,19 +1,21 @@
import logging
import unittest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.error import (
ConnectError,
ConnectionDone,
ConnectionLost,
ConnectionRefusedError,
DNSLookupError,
TCPTimedOutError,
TimeoutError,
)
from twisted.web.client import ResponseFailed
from scrapy.downloadermiddlewares.retry import RetryMiddleware
from scrapy.spiders import Spider
from scrapy.downloadermiddlewares.retry import get_retry_request, RetryMiddleware
from scrapy.exceptions import IgnoreRequest
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
@ -119,82 +121,514 @@ class RetryTest(unittest.TestCase):
class MaxRetryTimesTest(unittest.TestCase):
def setUp(self):
self.crawler = get_crawler(Spider)
self.spider = self.crawler._create_spider('foo')
self.mw = RetryMiddleware.from_crawler(self.crawler)
self.mw.max_retry_times = 2
self.invalid_url = 'http://www.scrapytest.org/invalid_url'
invalid_url = 'http://www.scrapytest.org/invalid_url'
def get_spider_and_middleware(self, settings=None):
crawler = get_crawler(Spider, settings or {})
spider = crawler._create_spider('foo')
middleware = RetryMiddleware.from_crawler(crawler)
return spider, middleware
def test_with_settings_zero(self):
# SETTINGS: RETRY_TIMES = 0
self.mw.max_retry_times = 0
max_retry_times = 0
settings = {'RETRY_TIMES': max_retry_times}
spider, middleware = self.get_spider_and_middleware(settings)
req = Request(self.invalid_url)
self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times)
self._test_retry(
req,
DNSLookupError('foo'),
max_retry_times,
spider=spider,
middleware=middleware,
)
def test_with_metakey_zero(self):
# SETTINGS: meta(max_retry_times) = 0
meta_max_retry_times = 0
req = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
self._test_retry(req, DNSLookupError('foo'), meta_max_retry_times)
max_retry_times = 0
spider, middleware = self.get_spider_and_middleware()
meta = {'max_retry_times': max_retry_times}
req = Request(self.invalid_url, meta=meta)
self._test_retry(
req,
DNSLookupError('foo'),
max_retry_times,
spider=spider,
middleware=middleware,
)
def test_without_metakey(self):
# SETTINGS: RETRY_TIMES is NON-ZERO
self.mw.max_retry_times = 5
max_retry_times = 5
settings = {'RETRY_TIMES': max_retry_times}
spider, middleware = self.get_spider_and_middleware(settings)
req = Request(self.invalid_url)
self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times)
self._test_retry(
req,
DNSLookupError('foo'),
max_retry_times,
spider=spider,
middleware=middleware,
)
def test_with_metakey_greater(self):
# SETINGS: RETRY_TIMES < meta(max_retry_times)
self.mw.max_retry_times = 2
meta_max_retry_times = 3
middleware_max_retry_times = 2
req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
req2 = Request(self.invalid_url)
self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times)
self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times)
settings = {'RETRY_TIMES': middleware_max_retry_times}
spider, middleware = self.get_spider_and_middleware(settings)
self._test_retry(
req1,
DNSLookupError('foo'),
meta_max_retry_times,
spider=spider,
middleware=middleware,
)
self._test_retry(
req2,
DNSLookupError('foo'),
middleware_max_retry_times,
spider=spider,
middleware=middleware,
)
def test_with_metakey_lesser(self):
# SETINGS: RETRY_TIMES > meta(max_retry_times)
self.mw.max_retry_times = 5
meta_max_retry_times = 4
middleware_max_retry_times = 5
req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
req2 = Request(self.invalid_url)
self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times)
self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times)
settings = {'RETRY_TIMES': middleware_max_retry_times}
spider, middleware = self.get_spider_and_middleware(settings)
self._test_retry(
req1,
DNSLookupError('foo'),
meta_max_retry_times,
spider=spider,
middleware=middleware,
)
self._test_retry(
req2,
DNSLookupError('foo'),
middleware_max_retry_times,
spider=spider,
middleware=middleware,
)
def test_with_dont_retry(self):
max_retry_times = 4
spider, middleware = self.get_spider_and_middleware()
meta = {
'max_retry_times': max_retry_times,
'dont_retry': True,
}
req = Request(self.invalid_url, meta=meta)
self._test_retry(
req,
DNSLookupError('foo'),
0,
spider=spider,
middleware=middleware,
)
# SETTINGS: meta(max_retry_times) = 4
meta_max_retry_times = 4
req = Request(self.invalid_url, meta={
'max_retry_times': meta_max_retry_times, 'dont_retry': True
})
self._test_retry(req, DNSLookupError('foo'), 0)
def _test_retry(self, req, exception, max_retry_times):
def _test_retry(
self,
req,
exception,
max_retry_times,
spider=None,
middleware=None,
):
spider = spider or self.spider
middleware = middleware or self.mw
for i in range(0, max_retry_times):
req = self.mw.process_exception(req, exception, self.spider)
req = middleware.process_exception(req, exception, spider)
assert isinstance(req, Request)
# discard it
req = self.mw.process_exception(req, exception, self.spider)
req = middleware.process_exception(req, exception, spider)
self.assertEqual(req, None)
class GetRetryRequestTest(unittest.TestCase):
def get_spider(self, settings=None):
crawler = get_crawler(Spider, settings or {})
return crawler._create_spider('foo')
def test_basic_usage(self):
request = Request('https://example.com')
spider = self.get_spider()
with LogCapture() as log:
new_request = get_retry_request(
request,
spider=spider,
)
self.assertIsInstance(new_request, Request)
self.assertNotEqual(new_request, request)
self.assertEqual(new_request.dont_filter, True)
expected_retry_times = 1
self.assertEqual(new_request.meta['retry_times'], expected_retry_times)
self.assertEqual(new_request.priority, -1)
expected_reason = "unspecified"
for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
self.assertEqual(spider.crawler.stats.get_value(stat), 1)
log.check_present(
(
"scrapy.downloadermiddlewares.retry",
"DEBUG",
f"Retrying {request} (failed {expected_retry_times} times): "
f"{expected_reason}",
)
)
def test_max_retries_reached(self):
request = Request('https://example.com')
spider = self.get_spider()
max_retry_times = 0
with LogCapture() as log:
new_request = get_retry_request(
request,
spider=spider,
max_retry_times=max_retry_times,
)
self.assertEqual(new_request, None)
self.assertEqual(
spider.crawler.stats.get_value('retry/max_reached'),
1
)
failure_count = max_retry_times + 1
expected_reason = "unspecified"
log.check_present(
(
"scrapy.downloadermiddlewares.retry",
"ERROR",
f"Gave up retrying {request} (failed {failure_count} times): "
f"{expected_reason}",
)
)
def test_one_retry(self):
request = Request('https://example.com')
spider = self.get_spider()
with LogCapture() as log:
new_request = get_retry_request(
request,
spider=spider,
max_retry_times=1,
)
self.assertIsInstance(new_request, Request)
self.assertNotEqual(new_request, request)
self.assertEqual(new_request.dont_filter, True)
expected_retry_times = 1
self.assertEqual(new_request.meta['retry_times'], expected_retry_times)
self.assertEqual(new_request.priority, -1)
expected_reason = "unspecified"
for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
self.assertEqual(spider.crawler.stats.get_value(stat), 1)
log.check_present(
(
"scrapy.downloadermiddlewares.retry",
"DEBUG",
f"Retrying {request} (failed {expected_retry_times} times): "
f"{expected_reason}",
)
)
def test_two_retries(self):
spider = self.get_spider()
request = Request('https://example.com')
new_request = request
max_retry_times = 2
for index in range(max_retry_times):
with LogCapture() as log:
new_request = get_retry_request(
new_request,
spider=spider,
max_retry_times=max_retry_times,
)
self.assertIsInstance(new_request, Request)
self.assertNotEqual(new_request, request)
self.assertEqual(new_request.dont_filter, True)
expected_retry_times = index + 1
self.assertEqual(new_request.meta['retry_times'], expected_retry_times)
self.assertEqual(new_request.priority, -expected_retry_times)
expected_reason = "unspecified"
for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
value = spider.crawler.stats.get_value(stat)
self.assertEqual(value, expected_retry_times)
log.check_present(
(
"scrapy.downloadermiddlewares.retry",
"DEBUG",
f"Retrying {request} (failed {expected_retry_times} times): "
f"{expected_reason}",
)
)
with LogCapture() as log:
new_request = get_retry_request(
new_request,
spider=spider,
max_retry_times=max_retry_times,
)
self.assertEqual(new_request, None)
self.assertEqual(
spider.crawler.stats.get_value('retry/max_reached'),
1
)
failure_count = max_retry_times + 1
expected_reason = "unspecified"
log.check_present(
(
"scrapy.downloadermiddlewares.retry",
"ERROR",
f"Gave up retrying {request} (failed {failure_count} times): "
f"{expected_reason}",
)
)
def test_no_spider(self):
request = Request('https://example.com')
with self.assertRaises(TypeError):
get_retry_request(request) # pylint: disable=missing-kwoa
def test_max_retry_times_setting(self):
max_retry_times = 0
spider = self.get_spider({'RETRY_TIMES': max_retry_times})
request = Request('https://example.com')
new_request = get_retry_request(
request,
spider=spider,
)
self.assertEqual(new_request, None)
def test_max_retry_times_meta(self):
max_retry_times = 0
spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1})
meta = {'max_retry_times': max_retry_times}
request = Request('https://example.com', meta=meta)
new_request = get_retry_request(
request,
spider=spider,
)
self.assertEqual(new_request, None)
def test_max_retry_times_argument(self):
max_retry_times = 0
spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1})
meta = {'max_retry_times': max_retry_times + 1}
request = Request('https://example.com', meta=meta)
new_request = get_retry_request(
request,
spider=spider,
max_retry_times=max_retry_times,
)
self.assertEqual(new_request, None)
def test_priority_adjust_setting(self):
priority_adjust = 1
spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust})
request = Request('https://example.com')
new_request = get_retry_request(
request,
spider=spider,
)
self.assertEqual(new_request.priority, priority_adjust)
def test_priority_adjust_argument(self):
priority_adjust = 1
spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust + 1})
request = Request('https://example.com')
new_request = get_retry_request(
request,
spider=spider,
priority_adjust=priority_adjust,
)
self.assertEqual(new_request.priority, priority_adjust)
def test_log_extra_retry_success(self):
request = Request('https://example.com')
spider = self.get_spider()
with LogCapture(attributes=('spider',)) as log:
get_retry_request(
request,
spider=spider,
)
log.check_present(spider)
def test_log_extra_retries_exceeded(self):
request = Request('https://example.com')
spider = self.get_spider()
with LogCapture(attributes=('spider',)) as log:
get_retry_request(
request,
spider=spider,
max_retry_times=0,
)
log.check_present(spider)
def test_reason_string(self):
request = Request('https://example.com')
spider = self.get_spider()
expected_reason = 'because'
with LogCapture() as log:
get_retry_request(
request,
spider=spider,
reason=expected_reason,
)
expected_retry_times = 1
for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
self.assertEqual(spider.crawler.stats.get_value(stat), 1)
log.check_present(
(
"scrapy.downloadermiddlewares.retry",
"DEBUG",
f"Retrying {request} (failed {expected_retry_times} times): "
f"{expected_reason}",
)
)
def test_reason_builtin_exception(self):
request = Request('https://example.com')
spider = self.get_spider()
expected_reason = NotImplementedError()
expected_reason_string = 'builtins.NotImplementedError'
with LogCapture() as log:
get_retry_request(
request,
spider=spider,
reason=expected_reason,
)
expected_retry_times = 1
stat = spider.crawler.stats.get_value(
f'retry/reason_count/{expected_reason_string}'
)
self.assertEqual(stat, 1)
log.check_present(
(
"scrapy.downloadermiddlewares.retry",
"DEBUG",
f"Retrying {request} (failed {expected_retry_times} times): "
f"{expected_reason}",
)
)
def test_reason_builtin_exception_class(self):
request = Request('https://example.com')
spider = self.get_spider()
expected_reason = NotImplementedError
expected_reason_string = 'builtins.NotImplementedError'
with LogCapture() as log:
get_retry_request(
request,
spider=spider,
reason=expected_reason,
)
expected_retry_times = 1
stat = spider.crawler.stats.get_value(
f'retry/reason_count/{expected_reason_string}'
)
self.assertEqual(stat, 1)
log.check_present(
(
"scrapy.downloadermiddlewares.retry",
"DEBUG",
f"Retrying {request} (failed {expected_retry_times} times): "
f"{expected_reason}",
)
)
def test_reason_custom_exception(self):
request = Request('https://example.com')
spider = self.get_spider()
expected_reason = IgnoreRequest()
expected_reason_string = 'scrapy.exceptions.IgnoreRequest'
with LogCapture() as log:
get_retry_request(
request,
spider=spider,
reason=expected_reason,
)
expected_retry_times = 1
stat = spider.crawler.stats.get_value(
f'retry/reason_count/{expected_reason_string}'
)
self.assertEqual(stat, 1)
log.check_present(
(
"scrapy.downloadermiddlewares.retry",
"DEBUG",
f"Retrying {request} (failed {expected_retry_times} times): "
f"{expected_reason}",
)
)
def test_reason_custom_exception_class(self):
request = Request('https://example.com')
spider = self.get_spider()
expected_reason = IgnoreRequest
expected_reason_string = 'scrapy.exceptions.IgnoreRequest'
with LogCapture() as log:
get_retry_request(
request,
spider=spider,
reason=expected_reason,
)
expected_retry_times = 1
stat = spider.crawler.stats.get_value(
f'retry/reason_count/{expected_reason_string}'
)
self.assertEqual(stat, 1)
log.check_present(
(
"scrapy.downloadermiddlewares.retry",
"DEBUG",
f"Retrying {request} (failed {expected_retry_times} times): "
f"{expected_reason}",
)
)
def test_custom_logger(self):
logger = logging.getLogger("custom-logger")
request = Request("https://example.com")
spider = self.get_spider()
expected_reason = "because"
with LogCapture() as log:
get_retry_request(
request,
spider=spider,
reason=expected_reason,
logger=logger,
)
log.check_present(
(
"custom-logger",
"DEBUG",
f"Retrying {request} (failed 1 times): {expected_reason}",
)
)
def test_custom_stats_key(self):
request = Request("https://example.com")
spider = self.get_spider()
expected_reason = "because"
stats_key = "custom_retry"
get_retry_request(
request,
spider=spider,
reason=expected_reason,
stats_base_key=stats_key,
)
for stat in (f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}"):
self.assertEqual(spider.crawler.stats.get_value(stat), 1)
if __name__ == "__main__":
unittest.main()

22
tests/test_engine.py
View File

@ -56,7 +56,7 @@ class TestSpider(Spider):
name_re = re.compile(r"<h1>(.*?)</h1>", re.M)
price_re = re.compile(r">Price: \$(.*?)<", re.M)
item_cls = TestItem
item_cls: type = TestItem
def parse(self, response):
xlink = LinkExtractor()
@ -66,15 +66,15 @@ class TestSpider(Spider):
yield Request(url=link.url, callback=self.parse_item)
def parse_item(self, response):
item = self.item_cls()
adapter = ItemAdapter(self.item_cls())
m = self.name_re.search(response.text)
if m:
item['name'] = m.group(1)
item['url'] = response.url
adapter['name'] = m.group(1)
adapter['url'] = response.url
m = self.price_re.search(response.text)
if m:
item['price'] = m.group(1)
return item
adapter['price'] = m.group(1)
return adapter.item
class TestDupeFilterSpider(TestSpider):
@ -87,7 +87,7 @@ class DictItemsSpider(TestSpider):
class AttrsItemsSpider(TestSpider):
item_class = AttrsItem
item_cls = AttrsItem
try:
@ -97,14 +97,10 @@ except ImportError:
else:
TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)])
class DataClassItemsSpider(DictItemsSpider):
class DataClassItemsSpider(DictItemsSpider): # type: ignore[no-redef]
def parse_item(self, response):
item = super().parse_item(response)
return TestDataClass(
name=item.get('name'),
url=item.get('url'),
price=item.get('price'),
)
return TestDataClass(**item)
class ItemZeroDivisionErrorSpider(TestSpider):

13
tests/test_spidermiddleware_httperror.py
View File

@ -139,6 +139,19 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
def test_httperror_allow_all_false(self):
crawler = get_crawler(_HttpErrorSpider)
mw = HttpErrorMiddleware.from_crawler(crawler)
request_httpstatus_false = Request('http://scrapytest.org', meta={'handle_httpstatus_all': False})
request_httpstatus_true = Request('http://scrapytest.org', meta={'handle_httpstatus_all': True})
res404 = self.res404.copy()
res404.request = request_httpstatus_false
res402 = self.res402.copy()
res402.request = request_httpstatus_true
self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider)
self.assertIsNone(mw.process_spider_input(res402, self.spider))
class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
def setUp(self):

2
tests/test_webclient.py
View File

@ -418,7 +418,7 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
def testPayloadDisabledCipher(self):
if sys.implementation.name == "pypy" and parse_version(cryptography.__version__) <= parse_version("2.3.1"):
self.skipTest("This does work in PyPy with cryptography<=2.3.1")
self.skipTest("This test expects a failure, but the code does work in PyPy with cryptography<=2.3.1")
s = "0123456789" * 10
settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'})
client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)

4
tox.ini
View File

@ -37,7 +37,7 @@ basepython = python3
deps =
mypy==0.780
commands =
mypy {posargs: scrapy tests}
mypy --show-error-codes {posargs: scrapy tests}
[testenv:security]
basepython = python3
@ -71,7 +71,7 @@ commands =
deps =
cryptography==2.0
cssselect==0.9.1
h2==3.2.0
h2==3.0
itemadapter==0.1.0
parsel==1.5.0
Protego==0.1.15