mirror of https://github.com/scrapy/scrapy.git
Merge branch 'scrapy:master' into fix/unique-list-link-extractors
This commit is contained in:
silviopavanetto
GitHub
commit
a6c339edf5
|
|
@ -19,7 +19,7 @@ jobs:
|
|||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: pylint
|
||||
- python-version: 3.6
|
||||
- python-version: 3.7
|
||||
env:
|
||||
TOXENV: typing
|
||||
- python-version: "3.10" # Keep in sync with .readthedocs.yml
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
|
||||
python-version: ["3.7", "3.8", "3.9", "3.10"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
|
|||
|
|
@ -8,9 +8,6 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- python-version: 3.7
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: py
|
||||
|
|
@ -26,19 +23,19 @@ jobs:
|
|||
- python-version: pypy3
|
||||
env:
|
||||
TOXENV: pypy3
|
||||
PYPY_VERSION: 3.6-v7.3.3
|
||||
PYPY_VERSION: 3.9-v7.3.9
|
||||
|
||||
# pinned deps
|
||||
- python-version: 3.6.12
|
||||
- python-version: 3.7.13
|
||||
env:
|
||||
TOXENV: pinned
|
||||
- python-version: 3.6.12
|
||||
- python-version: 3.7.13
|
||||
env:
|
||||
TOXENV: asyncio-pinned
|
||||
- python-version: pypy3
|
||||
env:
|
||||
TOXENV: pypy3-pinned
|
||||
PYPY_VERSION: 3.6-v7.2.0
|
||||
PYPY_VERSION: 3.7-v7.3.5
|
||||
|
||||
# extras
|
||||
# extra-deps includes reppy, which does not support Python 3.9
|
||||
|
|
|
|||
|
|
@ -8,12 +8,9 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- python-version: 3.6
|
||||
env:
|
||||
TOXENV: windows-pinned
|
||||
- python-version: 3.7
|
||||
env:
|
||||
TOXENV: py
|
||||
TOXENV: windows-pinned
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: py
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ including a list of features.
|
|||
Requirements
|
||||
============
|
||||
|
||||
* Python 3.6+
|
||||
* Python 3.7+
|
||||
* Works on Linux, Windows, macOS, BSD
|
||||
|
||||
Install
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ This command will fire up your default browser and open the main page of your
|
|||
Start over
|
||||
----------
|
||||
|
||||
To cleanup all generated documentation files and start from scratch run::
|
||||
To clean up all generated documentation files and start from scratch run::
|
||||
|
||||
make clean
|
||||
|
||||
|
|
|
|||
|
|
@ -1,16 +1,17 @@
|
|||
<html>
|
||||
<head>
|
||||
<base href='http://example.com/' />
|
||||
<title>Example website</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id='images'>
|
||||
<a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' /></a>
|
||||
<a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' /></a>
|
||||
<a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' /></a>
|
||||
<a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' /></a>
|
||||
<a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' /></a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<base href='http://example.com/' />
|
||||
<title>Example website</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id='images'>
|
||||
<a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' alt='image1'/></a>
|
||||
<a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' alt='image2'/></a>
|
||||
<a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' alt='image3'/></a>
|
||||
<a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' alt='image4'/></a>
|
||||
<a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' alt='image5'/></a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -294,7 +294,9 @@ intersphinx_mapping = {
|
|||
'tox': ('https://tox.readthedocs.io/en/latest', None),
|
||||
'twisted': ('https://twistedmatrix.com/documents/current', None),
|
||||
'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
|
||||
'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
|
||||
}
|
||||
intersphinx_disabled_reftypes = []
|
||||
|
||||
|
||||
# Options for sphinx-hoverxref options
|
||||
|
|
|
|||
|
|
@ -232,15 +232,15 @@ To run a specific test (say ``tests/test_loader.py``) use:
|
|||
|
||||
To run the tests on a specific :doc:`tox <tox:index>` environment, use
|
||||
``-e <name>`` with an environment name from ``tox.ini``. For example, to run
|
||||
the tests with Python 3.6 use::
|
||||
the tests with Python 3.7 use::
|
||||
|
||||
tox -e py36
|
||||
tox -e py37
|
||||
|
||||
You can also specify a comma-separated list of environments, and use :ref:`tox’s
|
||||
parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
|
||||
parallel::
|
||||
|
||||
tox -e py36,py38 -p auto
|
||||
tox -e py37,py38 -p auto
|
||||
|
||||
To pass command-line options to :doc:`pytest <pytest:index>`, add them after
|
||||
``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
|
||||
|
|
@ -250,9 +250,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
|
|||
tox -- scrapy tests -x # stop after first failure
|
||||
|
||||
You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
|
||||
the Python 3.6 :doc:`tox <tox:index>` environment using all your CPU cores::
|
||||
the Python 3.7 :doc:`tox <tox:index>` environment using all your CPU cores::
|
||||
|
||||
tox -e py36 -- scrapy tests -n auto
|
||||
tox -e py37 -- scrapy tests -n auto
|
||||
|
||||
To see coverage report install :doc:`coverage <coverage:index>`
|
||||
(``pip install coverage``) and run:
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ Installation guide
|
|||
Supported Python versions
|
||||
=========================
|
||||
|
||||
Scrapy requires Python 3.6+, either the CPython implementation (default) or
|
||||
the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
|
||||
Scrapy requires Python 3.7+, either the CPython implementation (default) or
|
||||
the PyPy 7.3.5+ implementation (see :ref:`python:implementations`).
|
||||
|
||||
.. _intro-install-scrapy:
|
||||
|
||||
|
|
@ -52,16 +52,6 @@ Scrapy is written in pure Python and depends on a few key Python packages (among
|
|||
* `twisted`_, an asynchronous networking framework
|
||||
* `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs
|
||||
|
||||
The minimal versions which Scrapy is tested against are:
|
||||
|
||||
* Twisted 14.0
|
||||
* lxml 3.4
|
||||
* pyOpenSSL 0.14
|
||||
|
||||
Scrapy may work with older versions of these packages
|
||||
but it is not guaranteed it will continue working
|
||||
because it’s not being tested against them.
|
||||
|
||||
Some of these packages themselves depends on non-Python packages
|
||||
that might require additional installation steps depending on your platform.
|
||||
Please check :ref:`platform-specific guides below <intro-install-platform-notes>`.
|
||||
|
|
|
|||
|
|
@ -45,9 +45,9 @@ https://quotes.toscrape.com, following the pagination::
|
|||
Put this in a text file, name it to something like ``quotes_spider.py``
|
||||
and run the spider using the :command:`runspider` command::
|
||||
|
||||
scrapy runspider quotes_spider.py -o quotes.jl
|
||||
scrapy runspider quotes_spider.py -o quotes.jsonl
|
||||
|
||||
When this finishes you will have in the ``quotes.jl`` file a list of the
|
||||
When this finishes you will have in the ``quotes.jsonl`` file a list of the
|
||||
quotes in JSON Lines format, containing text and author, looking like this::
|
||||
|
||||
{"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}
|
||||
|
|
|
|||
|
|
@ -482,7 +482,7 @@ to append new content to any existing file. However, appending to a JSON file
|
|||
makes the file contents invalid JSON. When appending to a file, consider
|
||||
using a different serialization format, such as `JSON Lines`_::
|
||||
|
||||
scrapy crawl quotes -o quotes.jl
|
||||
scrapy crawl quotes -o quotes.jsonl
|
||||
|
||||
The `JSON Lines`_ format is useful because it's stream-like, you can easily
|
||||
append new records to it. It doesn't have the same problem of JSON when you run
|
||||
|
|
|
|||
|
|
@ -1643,7 +1643,7 @@ New features
|
|||
:issue:`4370`)
|
||||
|
||||
* A new ``keep_fragments`` parameter of
|
||||
:func:`scrapy.utils.request.request_fingerprint` allows to generate
|
||||
``scrapy.utils.request.request_fingerprint`` allows to generate
|
||||
different fingerprints for requests with different fragments in their URL
|
||||
(:issue:`4104`)
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,13 @@ how you :ref:`configure the downloader middlewares
|
|||
:class:`scrapy.Spider` subclass and a
|
||||
:class:`scrapy.settings.Settings` object.
|
||||
|
||||
.. attribute:: request_fingerprinter
|
||||
|
||||
The request fingerprint builder of this crawler.
|
||||
|
||||
This is used from extensions and middlewares to build short, unique
|
||||
identifiers for requests. See :ref:`request-fingerprints`.
|
||||
|
||||
.. attribute:: settings
|
||||
|
||||
The settings manager of this crawler.
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
.. _topics-coroutines:
|
||||
|
||||
==========
|
||||
Coroutines
|
||||
==========
|
||||
|
|
|
|||
|
|
@ -195,17 +195,25 @@ BaseItemExporter
|
|||
|
||||
.. attribute:: fields_to_export
|
||||
|
||||
A list with the name of the fields that will be exported, or ``None`` if
|
||||
you want to export all fields. Defaults to ``None``.
|
||||
Fields to export, their order [1]_ and their output names.
|
||||
|
||||
Some exporters (like :class:`CsvItemExporter`) respect the order of the
|
||||
fields defined in this attribute.
|
||||
Possible values are:
|
||||
|
||||
When using :ref:`item objects <item-types>` that do not expose all their
|
||||
possible fields, exporters that do not support exporting a different
|
||||
subset of fields per item will only export the fields found in the first
|
||||
item exported. Use ``fields_to_export`` to define all the fields to be
|
||||
exported.
|
||||
- ``None`` (all fields [2]_, default)
|
||||
|
||||
- A list of fields::
|
||||
|
||||
['field1', 'field2']
|
||||
|
||||
- A dict where keys are fields and values are output names::
|
||||
|
||||
{'field1': 'Field 1', 'field2': 'Field 2'}
|
||||
|
||||
.. [1] Not all exporters respect the specified field order.
|
||||
.. [2] When using :ref:`item objects <item-types>` that do not expose
|
||||
all their possible fields, exporters that do not support exporting
|
||||
a different subset of fields per item will only export the fields
|
||||
found in the first item exported.
|
||||
|
||||
.. attribute:: export_empty_fields
|
||||
|
||||
|
|
@ -297,8 +305,8 @@ CsvItemExporter
|
|||
|
||||
Exports items in CSV format to the given file-like object. If the
|
||||
:attr:`fields_to_export` attribute is set, it will be used to define the
|
||||
CSV columns and their order. The :attr:`export_empty_fields` attribute has
|
||||
no effect on this exporter.
|
||||
CSV columns, their order and their column names. The
|
||||
:attr:`export_empty_fields` attribute has no effect on this exporter.
|
||||
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ CSV
|
|||
|
||||
- Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
|
||||
|
||||
- To specify columns to export and their order use
|
||||
- To specify columns to export, their order and their column names, use
|
||||
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
|
||||
option, but it is important for CSV because unlike many other export
|
||||
formats CSV uses a fixed header.
|
||||
|
|
@ -522,18 +522,9 @@ FEED_EXPORT_FIELDS
|
|||
|
||||
Default: ``None``
|
||||
|
||||
A list of fields to export, optional.
|
||||
Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
|
||||
|
||||
Use FEED_EXPORT_FIELDS option to define fields to export and their order.
|
||||
|
||||
When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses the fields
|
||||
defined in :ref:`item objects <topics-items>` yielded by your spider.
|
||||
|
||||
If an exporter requires a fixed set of fields (this is the case for
|
||||
:ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS
|
||||
is empty or None, then Scrapy tries to infer field names from the
|
||||
exported data - currently it uses field names from the first item.
|
||||
Use the ``FEED_EXPORT_FIELDS`` setting to define the fields to export, their
|
||||
order and their output names. See :attr:`BaseItemExporter.fields_to_export
|
||||
<scrapy.exporters.BaseItemExporter.fields_to_export>` for more information.
|
||||
|
||||
.. setting:: FEED_EXPORT_INDENT
|
||||
|
||||
|
|
@ -638,6 +629,7 @@ Default::
|
|||
{
|
||||
'json': 'scrapy.exporters.JsonItemExporter',
|
||||
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
|
||||
'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
|
||||
'jl': 'scrapy.exporters.JsonLinesItemExporter',
|
||||
'csv': 'scrapy.exporters.CsvItemExporter',
|
||||
'xml': 'scrapy.exporters.XmlItemExporter',
|
||||
|
|
@ -763,7 +755,7 @@ source spider in the feed URI:
|
|||
|
||||
#. Use ``%(spider_name)s`` in your feed URI::
|
||||
|
||||
scrapy crawl <spider_name> -o "%(spider_name)s.jl"
|
||||
scrapy crawl <spider_name> -o "%(spider_name)s.jsonl"
|
||||
|
||||
|
||||
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
|
||||
|
|
|
|||
|
|
@ -60,9 +60,9 @@ Additionally, they may also implement the following methods:
|
|||
:param spider: the spider which was closed
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
.. method:: from_crawler(cls, crawler)
|
||||
.. classmethod:: from_crawler(cls, crawler)
|
||||
|
||||
If present, this classmethod is called to create a pipeline instance
|
||||
If present, this class method is called to create a pipeline instance
|
||||
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
|
||||
of the pipeline. Crawler object provides access to all Scrapy core
|
||||
components like settings and signals; it is a way for pipeline to
|
||||
|
|
@ -99,11 +99,11 @@ contain a price::
|
|||
raise DropItem(f"Missing price in {item}")
|
||||
|
||||
|
||||
Write items to a JSON file
|
||||
--------------------------
|
||||
Write items to a JSON lines file
|
||||
--------------------------------
|
||||
|
||||
The following pipeline stores all scraped items (from all spiders) into a
|
||||
single ``items.jl`` file, containing one item per line serialized in JSON
|
||||
single ``items.jsonl`` file, containing one item per line serialized in JSON
|
||||
format::
|
||||
|
||||
import json
|
||||
|
|
@ -113,7 +113,7 @@ format::
|
|||
class JsonWriterPipeline:
|
||||
|
||||
def open_spider(self, spider):
|
||||
self.file = open('items.jl', 'w')
|
||||
self.file = open('items.jsonl', 'w')
|
||||
|
||||
def close_spider(self, spider):
|
||||
self.file.close()
|
||||
|
|
|
|||
|
|
@ -102,11 +102,6 @@ Additionally, ``dataclass`` items also allow to:
|
|||
* define custom field metadata through :func:`dataclasses.field`, which can be used to
|
||||
:ref:`customize serialization <topics-exporters-field-serialization>`.
|
||||
|
||||
They work natively in Python 3.7 or later, or using the `dataclasses
|
||||
backport`_ in Python 3.6.
|
||||
|
||||
.. _dataclasses backport: https://pypi.org/project/dataclasses/
|
||||
|
||||
Example::
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
|
|||
can configure some extra functions like generating thumbnails and filtering
|
||||
the images based on their size.
|
||||
|
||||
The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for
|
||||
The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for
|
||||
thumbnailing and normalizing images to JPEG/RGB format.
|
||||
|
||||
.. _Pillow: https://github.com/python-pillow/Pillow
|
||||
|
|
@ -656,6 +656,26 @@ See here the methods that you can override in your custom Images Pipeline:
|
|||
.. versionadded:: 2.4
|
||||
The *item* parameter.
|
||||
|
||||
.. method:: ImagesPipeline.thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)
|
||||
|
||||
This method is called for every item of :setting:`IMAGES_THUMBS` per downloaded item. It returns the
|
||||
thumbnail download path of the image originating from the specified
|
||||
:class:`response <scrapy.http.Response>`.
|
||||
|
||||
In addition to ``response``, this method receives the original
|
||||
:class:`request <scrapy.Request>`,
|
||||
``thumb_id``,
|
||||
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
|
||||
:class:`item <scrapy.Item>`.
|
||||
|
||||
You can override this method to customize the thumbnail download path of each image.
|
||||
You can use the ``item`` to determine the file path based on some item
|
||||
property.
|
||||
|
||||
By default the :meth:`thumb_path` method returns
|
||||
``thumbs/<size name>/<request URL hash>.<extension>``.
|
||||
|
||||
|
||||
.. method:: ImagesPipeline.get_media_requests(item, info)
|
||||
|
||||
Works the same way as :meth:`FilesPipeline.get_media_requests` method,
|
||||
|
|
|
|||
|
|
@ -180,8 +180,8 @@ Same example but running the spiders sequentially by chaining the deferreds:
|
|||
# Your second spider definition
|
||||
...
|
||||
|
||||
configure_logging()
|
||||
settings = get_project_settings()
|
||||
configure_logging(settings)
|
||||
runner = CrawlerRunner(settings)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
|
|
|||
|
|
@ -339,6 +339,7 @@ errors if needed::
|
|||
request = failure.request
|
||||
self.logger.error('TimeoutError on %s', request.url)
|
||||
|
||||
|
||||
.. _errback-cb_kwargs:
|
||||
|
||||
Accessing additional data in errback functions
|
||||
|
|
@ -364,6 +365,273 @@ achieve this by using ``Failure.request.cb_kwargs``::
|
|||
main_url=failure.request.cb_kwargs['main_url'],
|
||||
)
|
||||
|
||||
|
||||
.. _request-fingerprints:
|
||||
|
||||
Request fingerprints
|
||||
--------------------
|
||||
|
||||
There are some aspects of scraping, such as filtering out duplicate requests
|
||||
(see :setting:`DUPEFILTER_CLASS`) or caching responses (see
|
||||
:setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short,
|
||||
unique identifier from a :class:`~scrapy.http.Request` object: a request
|
||||
fingerprint.
|
||||
|
||||
You often do not need to worry about request fingerprints, the default request
|
||||
fingerprinter works for most projects.
|
||||
|
||||
However, there is no universal way to generate a unique identifier from a
|
||||
request, because different situations require comparing requests differently.
|
||||
For example, sometimes you may need to compare URLs case-insensitively, include
|
||||
URL fragments, exclude certain URL query parameters, include some or all
|
||||
headers, etc.
|
||||
|
||||
To change how request fingerprints are built for your requests, use the
|
||||
:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
|
||||
|
||||
.. setting:: REQUEST_FINGERPRINTER_CLASS
|
||||
|
||||
REQUEST_FINGERPRINTER_CLASS
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
Default: :class:`scrapy.utils.request.RequestFingerprinter`
|
||||
|
||||
A :ref:`request fingerprinter class <custom-request-fingerprinter>` or its
|
||||
import path.
|
||||
|
||||
.. autoclass:: scrapy.utils.request.RequestFingerprinter
|
||||
|
||||
|
||||
.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
|
||||
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
Default: ``'PREVIOUS_VERSION'``
|
||||
|
||||
Determines which request fingerprinting algorithm is used by the default
|
||||
request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
|
||||
|
||||
Possible values are:
|
||||
|
||||
- ``'PREVIOUS_VERSION'`` (default)
|
||||
|
||||
This implementation uses the same request fingerprinting algorithm as
|
||||
Scrapy PREVIOUS_VERSION and earlier versions.
|
||||
|
||||
Even though this is the default value for backward compatibility reasons,
|
||||
it is a deprecated value.
|
||||
|
||||
- ``'VERSION'``
|
||||
|
||||
This implementation was introduced in Scrapy VERSION to fix an issue of the
|
||||
previous implementation.
|
||||
|
||||
New projects should use this value. The :command:`startproject` command
|
||||
sets this value in the generated ``settings.py`` file.
|
||||
|
||||
If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are
|
||||
using Scrapy components where changing the request fingerprinting algorithm
|
||||
would cause undesired results, you need to carefully decide when to change the
|
||||
value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
|
||||
setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request
|
||||
fingerprinting algorithm and does not log this warning (
|
||||
:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a
|
||||
class).
|
||||
|
||||
Scenarios where changing the request fingerprinting algorithm may cause
|
||||
undesired results include, for example, using the HTTP cache middleware (see
|
||||
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
|
||||
Changing the request fingerprinting algorithm would invalidade the current
|
||||
cache, requiring you to redownload all requests again.
|
||||
|
||||
Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in
|
||||
your settings to switch already to the request fingerprinting implementation
|
||||
that will be the only request fingerprinting implementation available in a
|
||||
future version of Scrapy, and remove the deprecation warning triggered by using
|
||||
the default value (``'PREVIOUS_VERSION'``).
|
||||
|
||||
|
||||
.. _PREVIOUS_VERSION-request-fingerprinter:
|
||||
.. _custom-request-fingerprinter:
|
||||
|
||||
Writing your own request fingerprinter
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
A request fingerprinter is a class that must implement the following method:
|
||||
|
||||
.. method:: fingerprint(self, request)
|
||||
|
||||
Return a :class:`bytes` object that uniquely identifies *request*.
|
||||
|
||||
See also :ref:`request-fingerprint-restrictions`.
|
||||
|
||||
:param request: request to fingerprint
|
||||
:type request: scrapy.http.Request
|
||||
|
||||
Additionally, it may also implement the following methods:
|
||||
|
||||
.. classmethod:: from_crawler(cls, crawler)
|
||||
|
||||
If present, this class method is called to create a request fingerprinter
|
||||
instance from a :class:`~scrapy.crawler.Crawler` object. It must return a
|
||||
new instance of the request fingerprinter.
|
||||
|
||||
*crawler* provides access to all Scrapy core components like settings and
|
||||
signals; it is a way for the request fingerprinter to access them and hook
|
||||
its functionality into Scrapy.
|
||||
|
||||
:param crawler: crawler that uses this request fingerprinter
|
||||
:type crawler: :class:`~scrapy.crawler.Crawler` object
|
||||
|
||||
.. classmethod:: from_settings(cls, settings)
|
||||
|
||||
If present, and ``from_crawler`` is not defined, this class method is called
|
||||
to create a request fingerprinter instance from a
|
||||
:class:`~scrapy.settings.Settings` object. It must return a new instance of
|
||||
the request fingerprinter.
|
||||
|
||||
The ``fingerprint`` method of the default request fingerprinter,
|
||||
:class:`scrapy.utils.request.RequestFingerprinter`, uses
|
||||
:func:`scrapy.utils.request.fingerprint` with its default parameters. For some
|
||||
common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well
|
||||
in your ``fingerprint`` method implementation:
|
||||
|
||||
.. autofunction:: scrapy.utils.request.fingerprint
|
||||
|
||||
For example, to take the value of a request header named ``X-ID`` into
|
||||
account::
|
||||
|
||||
# my_project/settings.py
|
||||
REQUEST_FINGERPRINTER_CLASS = 'my_project.utils.RequestFingerprinter'
|
||||
|
||||
# my_project/utils.py
|
||||
from scrapy.utils.request import fingerprint
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
def fingerprint(self, request):
|
||||
return fingerprint(request, include_headers=['X-ID'])
|
||||
|
||||
You can also write your own fingerprinting logic from scratch.
|
||||
|
||||
However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure
|
||||
you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
|
||||
|
||||
- Caching saves CPU by ensuring that fingerprints are calculated only once
|
||||
per request, and not once per Scrapy component that needs the fingerprint
|
||||
of a request.
|
||||
|
||||
- Using :class:`~weakref.WeakKeyDictionary` saves memory by ensuring that
|
||||
request objects do not stay in memory forever just because you have
|
||||
references to them in your cache dictionary.
|
||||
|
||||
For example, to take into account only the URL of a request, without any prior
|
||||
URL canonicalization or taking the request method or body into account::
|
||||
|
||||
from hashlib import sha1
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
from scrapy.utils.python import to_bytes
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
cache = WeakKeyDictionary()
|
||||
|
||||
def fingerprint(self, request):
|
||||
if request not in self.cache:
|
||||
fp = sha1()
|
||||
fp.update(to_bytes(request.url))
|
||||
self.cache[request] = fp.digest()
|
||||
return self.cache[request]
|
||||
|
||||
If you need to be able to override the request fingerprinting for arbitrary
|
||||
requests from your spider callbacks, you may implement a request fingerprinter
|
||||
that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>`
|
||||
when available, and then falls back to
|
||||
:func:`~scrapy.utils.request.fingerprint`. For example::
|
||||
|
||||
from scrapy.utils.request import fingerprint
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
def fingerprint(self, request):
|
||||
if 'fingerprint' in request.meta:
|
||||
return request.meta['fingerprint']
|
||||
return fingerprint(request)
|
||||
|
||||
If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION
|
||||
without using the deprecated ``'PREVIOUS_VERSION'`` value of the
|
||||
:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
|
||||
request fingerprinter::
|
||||
|
||||
from hashlib import sha1
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
from scrapy.utils.python import to_bytes
|
||||
from w3lib.url import canonicalize_url
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
cache = WeakKeyDictionary()
|
||||
|
||||
def fingerprint(self, request):
|
||||
if request not in self.cache:
|
||||
fp = sha1()
|
||||
fp.update(to_bytes(request.method))
|
||||
fp.update(to_bytes(canonicalize_url(request.url)))
|
||||
fp.update(request.body or b'')
|
||||
self.cache[request] = fp.digest()
|
||||
return self.cache[request]
|
||||
|
||||
|
||||
.. _request-fingerprint-restrictions:
|
||||
|
||||
Request fingerprint restrictions
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Scrapy components that use request fingerprints may impose additional
|
||||
restrictions on the format of the fingerprints that your :ref:`request
|
||||
fingerprinter <custom-request-fingerprinter>` generates.
|
||||
|
||||
The following built-in Scrapy components have such restrictions:
|
||||
|
||||
- :class:`scrapy.extensions.httpcache.FilesystemCacheStorage` (default
|
||||
value of :setting:`HTTPCACHE_STORAGE`)
|
||||
|
||||
Request fingerprints must be at least 1 byte long.
|
||||
|
||||
Path and filename length limits of the file system of
|
||||
:setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`,
|
||||
the following directory structure is created:
|
||||
|
||||
- :attr:`Spider.name <scrapy.spiders.Spider.name>`
|
||||
|
||||
- first byte of a request fingerprint as hexadecimal
|
||||
|
||||
- fingerprint as hexadecimal
|
||||
|
||||
- filenames up to 16 characters long
|
||||
|
||||
For example, if a request fingerprint is made of 20 bytes (default),
|
||||
:setting:`HTTPCACHE_DIR` is ``'/home/user/project/.scrapy/httpcache'``,
|
||||
and the name of your spider is ``'my_spider'`` your file system must
|
||||
support a file path like::
|
||||
|
||||
/home/user/project/.scrapy/httpcache/my_spider/01/0123456789abcdef0123456789abcdef01234567/response_headers
|
||||
|
||||
- :class:`scrapy.extensions.httpcache.DbmCacheStorage`
|
||||
|
||||
The underlying DBM implementation must support keys as long as twice
|
||||
the number of bytes of a request fingerprint, plus 5. For example,
|
||||
if a request fingerprint is made of 20 bytes (default),
|
||||
45-character-long keys must be supported.
|
||||
|
||||
|
||||
.. _topics-request-meta:
|
||||
|
||||
Request.meta special keys
|
||||
|
|
|
|||
|
|
@ -825,12 +825,8 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'``
|
|||
|
||||
The class used to detect and filter duplicate requests.
|
||||
|
||||
The default (``RFPDupeFilter``) filters based on request fingerprint using
|
||||
the ``scrapy.utils.request.request_fingerprint`` function. In order to change
|
||||
the way duplicates are checked you could subclass ``RFPDupeFilter`` and
|
||||
override its ``request_fingerprint`` method. This method should accept
|
||||
scrapy :class:`~scrapy.Request` object and return its fingerprint
|
||||
(a string).
|
||||
The default (``RFPDupeFilter``) filters based on the
|
||||
:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
|
||||
|
||||
You can disable filtering of duplicate requests by setting
|
||||
:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
|
||||
|
|
|
|||
|
|
@ -51,12 +51,12 @@ Deferred signal handlers
|
|||
========================
|
||||
|
||||
Some signals support returning :class:`~twisted.internet.defer.Deferred`
|
||||
objects from their handlers, allowing you to run asynchronous code that
|
||||
does not block Scrapy. If a signal handler returns a
|
||||
:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that
|
||||
:class:`~twisted.internet.defer.Deferred` to fire.
|
||||
or :term:`awaitable objects <awaitable>` from their handlers, allowing
|
||||
you to run asynchronous code that does not block Scrapy. If a signal
|
||||
handler returns one of these objects, Scrapy waits for that asynchronous
|
||||
operation to finish.
|
||||
|
||||
Let's take an example::
|
||||
Let's take an example using :ref:`coroutines <topics-coroutines>`::
|
||||
|
||||
class SignalSpider(scrapy.Spider):
|
||||
name = 'signals'
|
||||
|
|
@ -68,17 +68,15 @@ Let's take an example::
|
|||
crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
|
||||
return spider
|
||||
|
||||
def item_scraped(self, item):
|
||||
async def item_scraped(self, item):
|
||||
# Send the scraped item to the server
|
||||
d = treq.post(
|
||||
response = await treq.post(
|
||||
'http://example.com/post',
|
||||
json.dumps(item).encode('ascii'),
|
||||
headers={b'Content-Type': [b'application/json']}
|
||||
)
|
||||
|
||||
# The next item will be scraped only after
|
||||
# deferred (d) is fired
|
||||
return d
|
||||
return response
|
||||
|
||||
def parse(self, response):
|
||||
for quote in response.css('div.quote'):
|
||||
|
|
@ -89,7 +87,7 @@ Let's take an example::
|
|||
}
|
||||
|
||||
See the :ref:`topics-signals-ref` below to know which signals support
|
||||
:class:`~twisted.internet.defer.Deferred`.
|
||||
:class:`~twisted.internet.defer.Deferred` and :term:`awaitable objects <awaitable>`.
|
||||
|
||||
.. _topics-signals-ref:
|
||||
|
||||
|
|
|
|||
|
|
@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
|
|||
|
||||
|
||||
# Check minimum required Python version
|
||||
if sys.version_info < (3, 6):
|
||||
print(f"Scrapy {__version__} requires Python 3.6+")
|
||||
if sys.version_info < (3, 7):
|
||||
print(f"Scrapy {__version__} requires Python 3.7+")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -146,7 +146,8 @@ class Command(BaseRunSpiderCommand):
|
|||
|
||||
def _start_requests(spider):
|
||||
yield self.prepare_request(spider, Request(url), opts)
|
||||
self.spidercls.start_requests = _start_requests
|
||||
if self.spidercls:
|
||||
self.spidercls.start_requests = _start_requests
|
||||
|
||||
def start_parsing(self, url, opts):
|
||||
self.crawler_process.crawl(self.spidercls, **opts.spargs)
|
||||
|
|
|
|||
|
|
@ -102,11 +102,11 @@ class FTPDownloadHandler:
|
|||
|
||||
def _build_response(self, result, request, protocol):
|
||||
self.result = result
|
||||
respcls = responsetypes.from_args(url=request.url)
|
||||
protocol.close()
|
||||
body = protocol.filename or protocol.body.read()
|
||||
headers = {"local filename": protocol.filename or '', "size": protocol.size}
|
||||
return respcls(url=request.url, status=200, body=to_bytes(body), headers=headers)
|
||||
body = to_bytes(protocol.filename or protocol.body.read())
|
||||
respcls = responsetypes.from_args(url=request.url, body=body)
|
||||
return respcls(url=request.url, status=200, body=body, headers=headers)
|
||||
|
||||
def _failed(self, result, request):
|
||||
message = result.getErrorMessage()
|
||||
|
|
|
|||
|
|
@ -112,7 +112,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
|
|||
request.meta['download_latency'] = self.headers_time - self.start_time
|
||||
status = int(self.status)
|
||||
headers = Headers(self.response_headers)
|
||||
respcls = responsetypes.from_args(headers=headers, url=self._url)
|
||||
respcls = responsetypes.from_args(headers=headers, url=self._url, body=body)
|
||||
return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))
|
||||
|
||||
def _set_connection_attributes(self, request):
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ class Crawler:
|
|||
self.spidercls.update_settings(self.settings)
|
||||
|
||||
self.signals = SignalManager(self)
|
||||
|
||||
self.stats = load_object(self.settings['STATS_CLASS'])(self)
|
||||
|
||||
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
|
||||
|
|
@ -71,6 +72,12 @@ class Crawler:
|
|||
lf_cls = load_object(self.settings['LOG_FORMATTER'])
|
||||
self.logformatter = lf_cls.from_crawler(self)
|
||||
|
||||
self.request_fingerprinter = create_instance(
|
||||
load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
|
||||
settings=self.settings,
|
||||
crawler=self,
|
||||
)
|
||||
|
||||
reactor_class = self.settings.get("TWISTED_REACTOR")
|
||||
if init_reactor:
|
||||
# this needs to be done after the spider settings are merged,
|
||||
|
|
|
|||
|
|
@ -104,8 +104,8 @@ class CookiesMiddleware:
|
|||
for key in ("name", "value", "path", "domain"):
|
||||
if cookie.get(key) is None:
|
||||
if key in ("name", "value"):
|
||||
msg = "Invalid cookie found in request {}: {} ('{}' is missing)"
|
||||
logger.warning(msg.format(request, cookie, key))
|
||||
msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
|
||||
logger.warning(msg)
|
||||
return
|
||||
continue
|
||||
if isinstance(cookie[key], (bool, float, int, str)):
|
||||
|
|
|
|||
|
|
@ -9,10 +9,19 @@ import tarfile
|
|||
import zipfile
|
||||
from io import BytesIO
|
||||
from tempfile import mktemp
|
||||
from warnings import warn
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.responsetypes import responsetypes
|
||||
|
||||
|
||||
warn(
|
||||
'scrapy.downloadermiddlewares.decompression is deprecated',
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,16 @@
|
|||
import logging
|
||||
import os
|
||||
from typing import Optional, Set, Type, TypeVar
|
||||
from warnings import warn
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.settings import BaseSettings
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.job import job_dir
|
||||
from scrapy.utils.request import referer_str, request_fingerprint
|
||||
from scrapy.utils.request import referer_str, RequestFingerprinter
|
||||
|
||||
|
||||
BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")
|
||||
|
|
@ -39,8 +41,15 @@ RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter")
|
|||
class RFPDupeFilter(BaseDupeFilter):
|
||||
"""Request Fingerprint duplicates filter"""
|
||||
|
||||
def __init__(self, path: Optional[str] = None, debug: bool = False) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
path: Optional[str] = None,
|
||||
debug: bool = False,
|
||||
*,
|
||||
fingerprinter=None,
|
||||
) -> None:
|
||||
self.file = None
|
||||
self.fingerprinter = fingerprinter or RequestFingerprinter()
|
||||
self.fingerprints: Set[str] = set()
|
||||
self.logdupes = True
|
||||
self.debug = debug
|
||||
|
|
@ -51,9 +60,39 @@ class RFPDupeFilter(BaseDupeFilter):
|
|||
self.fingerprints.update(x.rstrip() for x in self.file)
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings) -> RFPDupeFilterTV:
|
||||
def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None) -> RFPDupeFilterTV:
|
||||
debug = settings.getbool('DUPEFILTER_DEBUG')
|
||||
return cls(job_dir(settings), debug)
|
||||
try:
|
||||
return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
|
||||
except TypeError:
|
||||
warn(
|
||||
"RFPDupeFilter subclasses must either modify their '__init__' "
|
||||
"method to support a 'fingerprinter' parameter or reimplement "
|
||||
"the 'from_settings' class method.",
|
||||
ScrapyDeprecationWarning,
|
||||
)
|
||||
result = cls(job_dir(settings), debug)
|
||||
result.fingerprinter = fingerprinter
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
try:
|
||||
return cls.from_settings(
|
||||
crawler.settings,
|
||||
fingerprinter=crawler.request_fingerprinter,
|
||||
)
|
||||
except TypeError:
|
||||
warn(
|
||||
"RFPDupeFilter subclasses must either modify their overridden "
|
||||
"'__init__' method and 'from_settings' class method to "
|
||||
"support a 'fingerprinter' parameter, or reimplement the "
|
||||
"'from_crawler' class method.",
|
||||
ScrapyDeprecationWarning,
|
||||
)
|
||||
result = cls.from_settings(crawler.settings)
|
||||
result.fingerprinter = crawler.request_fingerprinter
|
||||
return result
|
||||
|
||||
def request_seen(self, request: Request) -> bool:
|
||||
fp = self.request_fingerprint(request)
|
||||
|
|
@ -65,7 +104,7 @@ class RFPDupeFilter(BaseDupeFilter):
|
|||
return False
|
||||
|
||||
def request_fingerprint(self, request: Request) -> str:
|
||||
return request_fingerprint(request)
|
||||
return self.fingerprinter.fingerprint(request).hex()
|
||||
|
||||
def close(self, reason: str) -> None:
|
||||
if self.file:
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import marshal
|
|||
import pickle
|
||||
import pprint
|
||||
import warnings
|
||||
from collections.abc import Mapping
|
||||
from xml.sax.saxutils import XMLGenerator
|
||||
|
||||
from itemadapter import is_item, ItemAdapter
|
||||
|
|
@ -68,6 +69,14 @@ class BaseItemExporter:
|
|||
field_iter = item.field_names()
|
||||
else:
|
||||
field_iter = item.keys()
|
||||
elif isinstance(self.fields_to_export, Mapping):
|
||||
if include_empty:
|
||||
field_iter = self.fields_to_export.items()
|
||||
else:
|
||||
field_iter = (
|
||||
(x, y) for x, y in self.fields_to_export.items()
|
||||
if x in item
|
||||
)
|
||||
else:
|
||||
if include_empty:
|
||||
field_iter = self.fields_to_export
|
||||
|
|
@ -75,13 +84,17 @@ class BaseItemExporter:
|
|||
field_iter = (x for x in self.fields_to_export if x in item)
|
||||
|
||||
for field_name in field_iter:
|
||||
if field_name in item:
|
||||
field_meta = item.get_field_meta(field_name)
|
||||
value = self.serialize_field(field_meta, field_name, item[field_name])
|
||||
if isinstance(field_name, str):
|
||||
item_field, output_field = field_name, field_name
|
||||
else:
|
||||
item_field, output_field = field_name
|
||||
if item_field in item:
|
||||
field_meta = item.get_field_meta(item_field)
|
||||
value = self.serialize_field(field_meta, output_field, item[item_field])
|
||||
else:
|
||||
value = default_value
|
||||
|
||||
yield field_name, value
|
||||
yield output_field, value
|
||||
|
||||
|
||||
class JsonLinesItemExporter(BaseItemExporter):
|
||||
|
|
@ -246,7 +259,11 @@ class CsvItemExporter(BaseItemExporter):
|
|||
if not self.fields_to_export:
|
||||
# use declared field names, or keys if the item is a dict
|
||||
self.fields_to_export = ItemAdapter(item).field_names()
|
||||
row = list(self._build_row(self.fields_to_export))
|
||||
if isinstance(self.fields_to_export, Mapping):
|
||||
fields = self.fields_to_export.values()
|
||||
else:
|
||||
fields = self.fields_to_export
|
||||
row = list(self._build_row(fields))
|
||||
self.csv_writer.writerow(row)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ from scrapy.responsetypes import responsetypes
|
|||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.project import data_path
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -228,6 +227,8 @@ class DbmCacheStorage:
|
|||
|
||||
logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
|
||||
|
||||
self._fingerprinter = spider.crawler.request_fingerprinter
|
||||
|
||||
def close_spider(self, spider):
|
||||
self.db.close()
|
||||
|
||||
|
|
@ -239,12 +240,12 @@ class DbmCacheStorage:
|
|||
status = data['status']
|
||||
headers = Headers(data['headers'])
|
||||
body = data['body']
|
||||
respcls = responsetypes.from_args(headers=headers, url=url)
|
||||
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
|
||||
response = respcls(url=url, headers=headers, status=status, body=body)
|
||||
return response
|
||||
|
||||
def store_response(self, spider, request, response):
|
||||
key = self._request_key(request)
|
||||
key = self._fingerprinter.fingerprint(request).hex()
|
||||
data = {
|
||||
'status': response.status,
|
||||
'url': response.url,
|
||||
|
|
@ -255,7 +256,7 @@ class DbmCacheStorage:
|
|||
self.db[f'{key}_time'] = str(time())
|
||||
|
||||
def _read_data(self, spider, request):
|
||||
key = self._request_key(request)
|
||||
key = self._fingerprinter.fingerprint(request).hex()
|
||||
db = self.db
|
||||
tkey = f'{key}_time'
|
||||
if tkey not in db:
|
||||
|
|
@ -267,9 +268,6 @@ class DbmCacheStorage:
|
|||
|
||||
return pickle.loads(db[f'{key}_data'])
|
||||
|
||||
def _request_key(self, request):
|
||||
return request_fingerprint(request)
|
||||
|
||||
|
||||
class FilesystemCacheStorage:
|
||||
|
||||
|
|
@ -283,6 +281,8 @@ class FilesystemCacheStorage:
|
|||
logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
|
||||
extra={'spider': spider})
|
||||
|
||||
self._fingerprinter = spider.crawler.request_fingerprinter
|
||||
|
||||
def close_spider(self, spider):
|
||||
pass
|
||||
|
||||
|
|
@ -299,7 +299,7 @@ class FilesystemCacheStorage:
|
|||
url = metadata.get('response_url')
|
||||
status = metadata['status']
|
||||
headers = Headers(headers_raw_to_dict(rawheaders))
|
||||
respcls = responsetypes.from_args(headers=headers, url=url)
|
||||
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
|
||||
response = respcls(url=url, headers=headers, status=status, body=body)
|
||||
return response
|
||||
|
||||
|
|
@ -329,7 +329,7 @@ class FilesystemCacheStorage:
|
|||
f.write(request.body)
|
||||
|
||||
def _get_request_path(self, spider, request):
|
||||
key = request_fingerprint(request)
|
||||
key = self._fingerprinter.fingerprint(request).hex()
|
||||
return os.path.join(self.cachedir, spider.name, key[0:2], key)
|
||||
|
||||
def _read_meta(self, spider, request):
|
||||
|
|
|
|||
|
|
@ -33,8 +33,8 @@ class MemoryUsage:
|
|||
self.crawler = crawler
|
||||
self.warned = False
|
||||
self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')
|
||||
self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
|
||||
self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
|
||||
self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024
|
||||
self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024
|
||||
self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
|
||||
self.mail = MailSender.from_settings(crawler.settings)
|
||||
crawler.signals.connect(self.engine_started, signal=signals.engine_started)
|
||||
|
|
@ -77,7 +77,7 @@ class MemoryUsage:
|
|||
def _check_limit(self):
|
||||
if self.get_virtual_size() > self.limit:
|
||||
self.crawler.stats.set_value('memusage/limit_reached', 1)
|
||||
mem = self.limit/1024/1024
|
||||
mem = self.limit / 1024 / 1024
|
||||
logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
|
||||
{'memusage': mem}, extra={'crawler': self.crawler})
|
||||
if self.notify_mails:
|
||||
|
|
@ -94,11 +94,11 @@ class MemoryUsage:
|
|||
self.crawler.stop()
|
||||
|
||||
def _check_warning(self):
|
||||
if self.warned: # warn only once
|
||||
if self.warned: # warn only once
|
||||
return
|
||||
if self.get_virtual_size() > self.warning:
|
||||
self.crawler.stats.set_value('memusage/warning_reached', 1)
|
||||
mem = self.warning/1024/1024
|
||||
mem = self.warning / 1024 / 1024
|
||||
logger.warning("Memory usage reached %(memusage)dM",
|
||||
{'memusage': mem}, extra={'crawler': self.crawler})
|
||||
if self.notify_mails:
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from scrapy import signals
|
|||
from scrapy.mail import MailSender
|
||||
from scrapy.exceptions import NotConfigured
|
||||
|
||||
|
||||
class StatsMailer:
|
||||
|
||||
def __init__(self, stats, recipients, mail):
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
from collections.abc import Mapping
|
||||
|
||||
from w3lib.http import headers_dict_to_raw
|
||||
from scrapy.utils.datatypes import CaselessDict
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
|
@ -10,6 +12,13 @@ class Headers(CaselessDict):
|
|||
self.encoding = encoding
|
||||
super().__init__(seq)
|
||||
|
||||
def update(self, seq):
|
||||
seq = seq.items() if isinstance(seq, Mapping) else seq
|
||||
iseq = {}
|
||||
for k, v in seq:
|
||||
iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
|
||||
super().update(iseq)
|
||||
|
||||
def normkey(self, key):
|
||||
"""Normalize key to bytes"""
|
||||
return self._tobytes(key.title())
|
||||
|
|
@ -86,4 +95,5 @@ class Headers(CaselessDict):
|
|||
|
||||
def __copy__(self):
|
||||
return self.__class__(self)
|
||||
|
||||
copy = __copy__
|
||||
|
|
|
|||
|
|
@ -222,8 +222,8 @@ class GCSFilesStore:
|
|||
return {'checksum': checksum, 'last_modified': last_modified}
|
||||
else:
|
||||
return {}
|
||||
|
||||
return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess)
|
||||
blob_path = self._get_blob_path(path)
|
||||
return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess)
|
||||
|
||||
def _get_content_type(self, headers):
|
||||
if headers and 'Content-Type' in headers:
|
||||
|
|
@ -231,8 +231,12 @@ class GCSFilesStore:
|
|||
else:
|
||||
return 'application/octet-stream'
|
||||
|
||||
def _get_blob_path(self, path):
|
||||
return self.prefix + path
|
||||
|
||||
def persist_file(self, path, buf, info, meta=None, headers=None):
|
||||
blob = self.bucket.blob(self.prefix + path)
|
||||
blob_path = self._get_blob_path(path)
|
||||
blob = self.bucket.blob(blob_path)
|
||||
blob.cache_control = self.CACHE_CONTROL
|
||||
blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
|
||||
return threads.deferToThread(
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ class ImagesPipeline(FilesPipeline):
|
|||
yield path, image, buf
|
||||
|
||||
for thumb_id, size in self.thumbs.items():
|
||||
thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
|
||||
thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item)
|
||||
thumb_image, thumb_buf = self.convert_image(image, size)
|
||||
yield thumb_path, thumb_image, thumb_buf
|
||||
|
||||
|
|
@ -179,6 +179,6 @@ class ImagesPipeline(FilesPipeline):
|
|||
image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
|
||||
return f'full/{image_guid}.jpg'
|
||||
|
||||
def thumb_path(self, request, thumb_id, response=None, info=None):
|
||||
def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
|
||||
thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
|
||||
return f'thumbs/{thumb_id}/{thumb_guid}.jpg'
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ from scrapy.settings import Settings
|
|||
from scrapy.utils.datatypes import SequenceExclude
|
||||
from scrapy.utils.defer import mustbe_deferred, defer_result
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
|
||||
|
|
@ -77,6 +76,7 @@ class MediaPipeline:
|
|||
except AttributeError:
|
||||
pipe = cls()
|
||||
pipe.crawler = crawler
|
||||
pipe._fingerprinter = crawler.request_fingerprinter
|
||||
return pipe
|
||||
|
||||
def open_spider(self, spider):
|
||||
|
|
@ -90,7 +90,7 @@ class MediaPipeline:
|
|||
return dfd.addCallback(self.item_completed, item, info)
|
||||
|
||||
def _process_request(self, request, info, item):
|
||||
fp = request_fingerprint(request)
|
||||
fp = self._fingerprinter.fingerprint(request)
|
||||
cb = request.callback or (lambda _: _)
|
||||
eb = request.errback
|
||||
request.callback = None
|
||||
|
|
@ -121,7 +121,7 @@ class MediaPipeline:
|
|||
def _make_compatible(self):
|
||||
"""Make overridable methods of MediaPipeline and subclasses backwards compatible"""
|
||||
methods = [
|
||||
"file_path", "media_to_download", "media_downloaded",
|
||||
"file_path", "thumb_path", "media_to_download", "media_downloaded",
|
||||
"file_downloaded", "image_downloaded", "get_images"
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -95,12 +95,14 @@ class ResponseTypes:
|
|||
chunk = to_bytes(chunk)
|
||||
if not binary_is_text(chunk):
|
||||
return self.from_mimetype('application/octet-stream')
|
||||
elif b"<html>" in chunk.lower():
|
||||
lowercase_chunk = chunk.lower()
|
||||
if b"<html>" in lowercase_chunk:
|
||||
return self.from_mimetype('text/html')
|
||||
elif b"<?xml" in chunk.lower():
|
||||
if b"<?xml" in lowercase_chunk:
|
||||
return self.from_mimetype('text/xml')
|
||||
else:
|
||||
return self.from_mimetype('text')
|
||||
if b'<!doctype html>' in lowercase_chunk:
|
||||
return self.from_mimetype('text/html')
|
||||
return self.from_mimetype('text')
|
||||
|
||||
def from_args(self, headers=None, url=None, filename=None, body=None):
|
||||
"""Guess the most appropriate Response class based on
|
||||
|
|
|
|||
|
|
@ -197,6 +197,38 @@ class BaseSettings(MutableMapping):
|
|||
value = json.loads(value)
|
||||
return dict(value)
|
||||
|
||||
def getdictorlist(self, name, default=None):
|
||||
"""Get a setting value as either a :class:`dict` or a :class:`list`.
|
||||
|
||||
If the setting is already a dict or a list, a copy of it will be
|
||||
returned.
|
||||
|
||||
If it is a string it will be evaluated as JSON, or as a comma-separated
|
||||
list of strings as a fallback.
|
||||
|
||||
For example, settings populated from the command line will return:
|
||||
|
||||
- ``{'key1': 'value1', 'key2': 'value2'}`` if set to
|
||||
``'{"key1": "value1", "key2": "value2"}'``
|
||||
|
||||
- ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
|
||||
|
||||
:param name: the setting name
|
||||
:type name: string
|
||||
|
||||
:param default: the value to return if no setting is found
|
||||
:type default: any
|
||||
"""
|
||||
value = self.get(name, default)
|
||||
if value is None:
|
||||
return {}
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return json.loads(value)
|
||||
except ValueError:
|
||||
return value.split(',')
|
||||
return copy.deepcopy(value)
|
||||
|
||||
def getwithbase(self, name):
|
||||
"""Get a composition of a dictionary-like setting and its `_BASE`
|
||||
counterpart.
|
||||
|
|
|
|||
|
|
@ -154,6 +154,7 @@ FEED_EXPORTERS = {}
|
|||
FEED_EXPORTERS_BASE = {
|
||||
'json': 'scrapy.exporters.JsonItemExporter',
|
||||
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
|
||||
'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
|
||||
'jl': 'scrapy.exporters.JsonLinesItemExporter',
|
||||
'csv': 'scrapy.exporters.CsvItemExporter',
|
||||
'xml': 'scrapy.exporters.XmlItemExporter',
|
||||
|
|
@ -246,6 +247,9 @@ REDIRECT_PRIORITY_ADJUST = +2
|
|||
REFERER_ENABLED = True
|
||||
REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
|
||||
|
||||
REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter'
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION'
|
||||
|
||||
RETRY_ENABLED = True
|
||||
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
|
||||
RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
|
||||
|
|
|
|||
|
|
@ -86,3 +86,6 @@ ROBOTSTXT_OBEY = True
|
|||
#HTTPCACHE_DIR = 'httpcache'
|
||||
#HTTPCACHE_IGNORE_HTTP_CODES = []
|
||||
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
|
||||
|
||||
# Set settings whose default value is deprecated to a future-proof value
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION'
|
||||
|
|
|
|||
|
|
@ -118,7 +118,7 @@ def feed_complete_default_values_from_settings(feed, settings):
|
|||
out = feed.copy()
|
||||
out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))
|
||||
out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
|
||||
out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
|
||||
out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None)
|
||||
out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
|
||||
out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
|
||||
out.setdefault("item_export_kwargs", {})
|
||||
|
|
|
|||
|
|
@ -1,11 +0,0 @@
|
|||
import warnings
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.asyncgen import collect_asyncgen # noqa: F401
|
||||
|
||||
|
||||
warnings.warn(
|
||||
"Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.asyncgen` instead.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
|
@ -4,7 +4,9 @@ scrapy.http.Request objects
|
|||
"""
|
||||
|
||||
import hashlib
|
||||
from typing import Dict, Iterable, Optional, Tuple, Union
|
||||
import json
|
||||
import warnings
|
||||
from typing import Dict, Iterable, List, Optional, Tuple, Union
|
||||
from urllib.parse import urlunparse
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
|
|
@ -12,13 +14,22 @@ from w3lib.http import basic_auth_header
|
|||
from w3lib.url import canonicalize_url
|
||||
|
||||
from scrapy import Request, Spider
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
|
||||
|
||||
_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
|
||||
_fingerprint_cache = WeakKeyDictionary()
|
||||
_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
|
||||
_deprecated_fingerprint_cache = WeakKeyDictionary()
|
||||
|
||||
|
||||
def _serialize_headers(headers, request):
|
||||
for header in headers:
|
||||
if header in request.headers:
|
||||
yield header
|
||||
for value in request.headers.getlist(header):
|
||||
yield value
|
||||
|
||||
|
||||
def request_fingerprint(
|
||||
|
|
@ -26,6 +37,123 @@ def request_fingerprint(
|
|||
include_headers: Optional[Iterable[Union[bytes, str]]] = None,
|
||||
keep_fragments: bool = False,
|
||||
) -> str:
|
||||
"""
|
||||
Return the request fingerprint as an hexadecimal string.
|
||||
|
||||
The request fingerprint is a hash that uniquely identifies the resource the
|
||||
request points to. For example, take the following two urls:
|
||||
|
||||
http://www.example.com/query?id=111&cat=222
|
||||
http://www.example.com/query?cat=222&id=111
|
||||
|
||||
Even though those are two different URLs both point to the same resource
|
||||
and are equivalent (i.e. they should return the same response).
|
||||
|
||||
Another example are cookies used to store session ids. Suppose the
|
||||
following page is only accessible to authenticated users:
|
||||
|
||||
http://www.example.com/members/offers.html
|
||||
|
||||
Lots of sites use a cookie to store the session id, which adds a random
|
||||
component to the HTTP Request and thus should be ignored when calculating
|
||||
the fingerprint.
|
||||
|
||||
For this reason, request headers are ignored by default when calculating
|
||||
the fingerprint. If you want to include specific headers use the
|
||||
include_headers argument, which is a list of Request headers to include.
|
||||
|
||||
Also, servers usually ignore fragments in urls when handling requests,
|
||||
so they are also ignored by default when calculating the fingerprint.
|
||||
If you want to include them, set the keep_fragments argument to True
|
||||
(for instance when handling requests with a headless browser).
|
||||
"""
|
||||
if include_headers or keep_fragments:
|
||||
message = (
|
||||
'Call to deprecated function '
|
||||
'scrapy.utils.request.request_fingerprint().\n'
|
||||
'\n'
|
||||
'If you are using this function in a Scrapy component because you '
|
||||
'need a non-default fingerprinting algorithm, and you are OK '
|
||||
'with that non-default fingerprinting algorithm being used by '
|
||||
'all Scrapy components and not just the one calling this '
|
||||
'function, use crawler.request_fingerprinter.fingerprint() '
|
||||
'instead in your Scrapy component (you can get the crawler '
|
||||
'object from the \'from_crawler\' class method), and use the '
|
||||
'\'REQUEST_FINGERPRINTER_CLASS\' setting to configure your '
|
||||
'non-default fingerprinting algorithm.\n'
|
||||
'\n'
|
||||
'Otherwise, consider using the '
|
||||
'scrapy.utils.request.fingerprint() function instead.\n'
|
||||
'\n'
|
||||
'If you switch to \'fingerprint()\', or assign the '
|
||||
'\'REQUEST_FINGERPRINTER_CLASS\' setting a class that uses '
|
||||
'\'fingerprint()\', the generated fingerprints will not only be '
|
||||
'bytes instead of a string, but they will also be different from '
|
||||
'those generated by \'request_fingerprint()\'. Before you switch, '
|
||||
'make sure that you understand the consequences of this (e.g. '
|
||||
'cache invalidation) and are OK with them; otherwise, consider '
|
||||
'implementing your own function which returns the same '
|
||||
'fingerprints as the deprecated \'request_fingerprint()\' function.'
|
||||
)
|
||||
else:
|
||||
message = (
|
||||
'Call to deprecated function '
|
||||
'scrapy.utils.request.request_fingerprint().\n'
|
||||
'\n'
|
||||
'If you are using this function in a Scrapy component, and you '
|
||||
'are OK with users of your component changing the fingerprinting '
|
||||
'algorithm through settings, use '
|
||||
'crawler.request_fingerprinter.fingerprint() instead in your '
|
||||
'Scrapy component (you can get the crawler object from the '
|
||||
'\'from_crawler\' class method).\n'
|
||||
'\n'
|
||||
'Otherwise, consider using the '
|
||||
'scrapy.utils.request.fingerprint() function instead.\n'
|
||||
'\n'
|
||||
'Either way, the resulting fingerprints will be returned as '
|
||||
'bytes, not as a string, and they will also be different from '
|
||||
'those generated by \'request_fingerprint()\'. Before you switch, '
|
||||
'make sure that you understand the consequences of this (e.g. '
|
||||
'cache invalidation) and are OK with them; otherwise, consider '
|
||||
'implementing your own function which returns the same '
|
||||
'fingerprints as the deprecated \'request_fingerprint()\' function.'
|
||||
)
|
||||
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
|
||||
processed_include_headers: Optional[Tuple[bytes, ...]] = None
|
||||
if include_headers:
|
||||
processed_include_headers = tuple(
|
||||
to_bytes(h.lower()) for h in sorted(include_headers)
|
||||
)
|
||||
cache = _deprecated_fingerprint_cache.setdefault(request, {})
|
||||
cache_key = (processed_include_headers, keep_fragments)
|
||||
if cache_key not in cache:
|
||||
fp = hashlib.sha1()
|
||||
fp.update(to_bytes(request.method))
|
||||
fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
|
||||
fp.update(request.body or b'')
|
||||
if processed_include_headers:
|
||||
for part in _serialize_headers(processed_include_headers, request):
|
||||
fp.update(part)
|
||||
cache[cache_key] = fp.hexdigest()
|
||||
return cache[cache_key]
|
||||
|
||||
|
||||
def _request_fingerprint_as_bytes(*args, **kwargs):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
return bytes.fromhex(request_fingerprint(*args, **kwargs))
|
||||
|
||||
|
||||
_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
|
||||
_fingerprint_cache = WeakKeyDictionary()
|
||||
|
||||
|
||||
def fingerprint(
|
||||
request: Request,
|
||||
*,
|
||||
include_headers: Optional[Iterable[Union[bytes, str]]] = None,
|
||||
keep_fragments: bool = False,
|
||||
) -> bytes:
|
||||
"""
|
||||
Return the request fingerprint.
|
||||
|
||||
|
|
@ -43,7 +171,7 @@ def request_fingerprint(
|
|||
|
||||
http://www.example.com/members/offers.html
|
||||
|
||||
Lot of sites use a cookie to store the session id, which adds a random
|
||||
Lots of sites use a cookie to store the session id, which adds a random
|
||||
component to the HTTP Request and thus should be ignored when calculating
|
||||
the fingerprint.
|
||||
|
||||
|
|
@ -55,29 +183,96 @@ def request_fingerprint(
|
|||
so they are also ignored by default when calculating the fingerprint.
|
||||
If you want to include them, set the keep_fragments argument to True
|
||||
(for instance when handling requests with a headless browser).
|
||||
|
||||
"""
|
||||
headers: Optional[Tuple[bytes, ...]] = None
|
||||
processed_include_headers: Optional[Tuple[bytes, ...]] = None
|
||||
if include_headers:
|
||||
headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
|
||||
processed_include_headers = tuple(
|
||||
to_bytes(h.lower()) for h in sorted(include_headers)
|
||||
)
|
||||
cache = _fingerprint_cache.setdefault(request, {})
|
||||
cache_key = (headers, keep_fragments)
|
||||
cache_key = (processed_include_headers, keep_fragments)
|
||||
if cache_key not in cache:
|
||||
fp = hashlib.sha1()
|
||||
fp.update(to_bytes(request.method))
|
||||
fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
|
||||
fp.update(request.body or b'')
|
||||
if headers:
|
||||
for hdr in headers:
|
||||
if hdr in request.headers:
|
||||
fp.update(hdr)
|
||||
for v in request.headers.getlist(hdr):
|
||||
fp.update(v)
|
||||
cache[cache_key] = fp.hexdigest()
|
||||
# To decode bytes reliably (JSON does not support bytes), regardless of
|
||||
# character encoding, we use bytes.hex()
|
||||
headers: Dict[str, List[str]] = {}
|
||||
if processed_include_headers:
|
||||
for header in processed_include_headers:
|
||||
if header in request.headers:
|
||||
headers[header.hex()] = [
|
||||
header_value.hex()
|
||||
for header_value in request.headers.getlist(header)
|
||||
]
|
||||
fingerprint_data = {
|
||||
'method': to_unicode(request.method),
|
||||
'url': canonicalize_url(request.url, keep_fragments=keep_fragments),
|
||||
'body': (request.body or b'').hex(),
|
||||
'headers': headers,
|
||||
}
|
||||
fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)
|
||||
cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest()
|
||||
return cache[cache_key]
|
||||
|
||||
|
||||
def request_authenticate(request: Request, username: str, password: str) -> None:
|
||||
class RequestFingerprinter:
|
||||
"""Default fingerprinter.
|
||||
|
||||
It takes into account a canonical version
|
||||
(:func:`w3lib.url.canonicalize_url`) of :attr:`request.url
|
||||
<scrapy.http.Request.url>` and the values of :attr:`request.method
|
||||
<scrapy.http.Request.method>` and :attr:`request.body
|
||||
<scrapy.http.Request.body>`. It then generates an `SHA1
|
||||
<https://en.wikipedia.org/wiki/SHA-1>`_ hash.
|
||||
|
||||
.. seealso:: :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION`.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(crawler)
|
||||
|
||||
def __init__(self, crawler=None):
|
||||
if crawler:
|
||||
implementation = crawler.settings.get(
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION'
|
||||
)
|
||||
else:
|
||||
implementation = 'PREVIOUS_VERSION'
|
||||
if implementation == 'PREVIOUS_VERSION':
|
||||
message = (
|
||||
'\'PREVIOUS_VERSION\' is a deprecated value for the '
|
||||
'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n'
|
||||
'\n'
|
||||
'It is also the default value. In other words, it is normal '
|
||||
'to get this warning if you have not defined a value for the '
|
||||
'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting. This is so '
|
||||
'for backward compatibility reasons, but it will change in a '
|
||||
'future version of Scrapy.\n'
|
||||
'\n'
|
||||
'See the documentation of the '
|
||||
'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting for '
|
||||
'information on how to handle this deprecation.'
|
||||
)
|
||||
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
|
||||
self._fingerprint = _request_fingerprint_as_bytes
|
||||
elif implementation == 'VERSION':
|
||||
self._fingerprint = fingerprint
|
||||
else:
|
||||
raise ValueError(
|
||||
f'Got an invalid value on setting '
|
||||
f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': '
|
||||
f'{implementation!r}. Valid values are \'PREVIOUS_VERSION\' (deprecated) '
|
||||
f'and \'VERSION\'.'
|
||||
)
|
||||
|
||||
def fingerprint(self, request):
|
||||
return self._fingerprint(request)
|
||||
|
||||
|
||||
def request_authenticate(
|
||||
request: Request,
|
||||
username: str,
|
||||
password: str,
|
||||
) -> None:
|
||||
"""Authenticate the given request (in place) using the HTTP basic access
|
||||
authentication mechanism (RFC 2617) and the given username and password
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ def get_ftp_content_and_delete(
|
|||
return "".join(ftp_data)
|
||||
|
||||
|
||||
def get_crawler(spidercls=None, settings_dict=None):
|
||||
def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):
|
||||
"""Return an unconfigured Crawler object. If settings_dict is given, it
|
||||
will be used to populate the crawler settings with a project level
|
||||
priority.
|
||||
|
|
@ -62,7 +62,12 @@ def get_crawler(spidercls=None, settings_dict=None):
|
|||
from scrapy.crawler import CrawlerRunner
|
||||
from scrapy.spiders import Spider
|
||||
|
||||
runner = CrawlerRunner(settings_dict)
|
||||
# Set by default settings that prevent deprecation warnings.
|
||||
settings = {}
|
||||
if prevent_warnings:
|
||||
settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = 'VERSION'
|
||||
settings.update(settings_dict or {})
|
||||
runner = CrawlerRunner(settings)
|
||||
return runner.create_crawler(spidercls or Spider)
|
||||
|
||||
|
||||
|
|
|
|||
19
setup.py
19
setup.py
|
|
@ -19,35 +19,29 @@ def has_environment_marker_platform_impl_support():
|
|||
|
||||
|
||||
install_requires = [
|
||||
'Twisted>=17.9.0',
|
||||
'cryptography>=2.0',
|
||||
'Twisted>=18.9.0',
|
||||
'cryptography>=2.8',
|
||||
'cssselect>=0.9.1',
|
||||
'itemloaders>=1.0.1',
|
||||
'parsel>=1.5.0',
|
||||
'pyOpenSSL>=16.2.0',
|
||||
'pyOpenSSL>=19.1.0',
|
||||
'queuelib>=1.4.2',
|
||||
'service_identity>=16.0.0',
|
||||
'w3lib>=1.17.0',
|
||||
'zope.interface>=4.1.3',
|
||||
'zope.interface>=5.1.0',
|
||||
'protego>=0.1.15',
|
||||
'itemadapter>=0.1.0',
|
||||
'setuptools',
|
||||
'tldextract',
|
||||
'lxml>=4.3.0',
|
||||
]
|
||||
extras_require = {}
|
||||
cpython_dependencies = [
|
||||
'lxml>=3.5.0',
|
||||
'PyDispatcher>=2.0.5',
|
||||
]
|
||||
if has_environment_marker_platform_impl_support():
|
||||
extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies
|
||||
extras_require[':platform_python_implementation == "PyPy"'] = [
|
||||
# Earlier lxml versions are affected by
|
||||
# https://foss.heptapod.net/pypy/pypy/-/issues/2498,
|
||||
# which was fixed in Cython 0.26, released on 2017-06-19, and used to
|
||||
# generate the C headers of lxml release tarballs published since then, the
|
||||
# first of which was:
|
||||
'lxml>=4.0.0',
|
||||
'PyPyDispatcher>=2.1.0',
|
||||
]
|
||||
else:
|
||||
|
|
@ -84,7 +78,6 @@ setup(
|
|||
'Operating System :: OS Independent',
|
||||
'Programming Language :: Python',
|
||||
'Programming Language :: Python :: 3',
|
||||
'Programming Language :: Python :: 3.6',
|
||||
'Programming Language :: Python :: 3.7',
|
||||
'Programming Language :: Python :: 3.8',
|
||||
'Programming Language :: Python :: 3.9',
|
||||
|
|
@ -95,7 +88,7 @@ setup(
|
|||
'Topic :: Software Development :: Libraries :: Application Frameworks',
|
||||
'Topic :: Software Development :: Libraries :: Python Modules',
|
||||
],
|
||||
python_requires='>=3.6',
|
||||
python_requires='>=3.7',
|
||||
install_requires=install_requires,
|
||||
extras_require=extras_require,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ class AsyncioReactorSpider1(scrapy.Spider):
|
|||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
}
|
||||
|
||||
|
||||
class AsyncioReactorSpider2(scrapy.Spider):
|
||||
name = 'asyncio_reactor2'
|
||||
custom_settings = {
|
||||
|
|
|
|||
|
|
@ -1,14 +1,12 @@
|
|||
# Tests requirements
|
||||
attrs
|
||||
dataclasses; python_version == '3.6'
|
||||
pyftpdlib
|
||||
pytest
|
||||
pytest-cov==3.0.0
|
||||
pytest-xdist
|
||||
sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
|
||||
testfixtures
|
||||
uvloop < 0.15.0; platform_system != "Windows" and python_version == '3.6'
|
||||
uvloop; platform_system != "Windows" and python_version > '3.6'
|
||||
uvloop; platform_system != "Windows"
|
||||
|
||||
# optional for shell wrapper tests
|
||||
bpython
|
||||
|
|
|
|||
|
|
@ -1,20 +1,22 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<base href='http://example.com' />
|
||||
<title>Sample page with links for testing LinkExtractor</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id='wrapper'>
|
||||
<div id='subwrapper'>
|
||||
<area href='sample1.html' />
|
||||
<a href='sample2.html'>sample 2<img src='sample2.jpg'/></a>
|
||||
</div>
|
||||
<a href='http://example.com/sample3.html' title='sample 3'>sample 3 text</a>
|
||||
<a href='sample3.html'>sample 3 repetition</a>
|
||||
<a href='sample3.html#foo'>sample 3 repetition with fragment</a>
|
||||
<a href='http://www.google.com/something'></a>
|
||||
<a href='http://example.com/innertag.html'><b>inner</b> tag</a>
|
||||
<a href=' page 4.html '>href with whitespaces</a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
<head>
|
||||
<base href='http://example.com' />
|
||||
<title>Sample page with links for testing LinkExtractor</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id='wrapper'>
|
||||
<div id='subwrapper'>
|
||||
<area href='sample1.html' alt='sample1'/>
|
||||
<a href='sample2.html'>sample 2<img src='sample2.jpg' alt='sample2'/></a>
|
||||
</div>
|
||||
<a href='http://example.com/sample3.html' title='sample 3'>sample 3 text</a>
|
||||
<a href='sample3.html'>sample 3 repetition</a>
|
||||
<a href='sample3.html#foo'>sample 3 repetition with fragment</a>
|
||||
<a href='http://www.google.com/something'></a>
|
||||
<a href='http://example.com/innertag.html'><strong>inner</strong> tag</a>
|
||||
<a href='page 4.html'>href with whitespaces</a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=latin-1">
|
||||
|
|
@ -7,11 +9,11 @@
|
|||
<body>
|
||||
<div id='wrapper'>
|
||||
<div id='subwrapper'>
|
||||
<a href='sample_ń.html'><img src='sample2.jpg'/></a>
|
||||
<a href='sample_ñ.html'><img src='sample2.jpg' alt='sample2'/></a>
|
||||
</div>
|
||||
<a href='sample_á.html' title='sample á'>sample á text</a>
|
||||
<a href='sample_á.html' title='sample á'>sample á text</a>
|
||||
<div id='subwrapper2'>
|
||||
<a href='sample_ö.html?price=Ł32&ľ=unit'><img src='sample3.jpg'/></a>
|
||||
<a href='sample_ö.html?price=£32&µ=unit'><img src='sample3.jpg' alt='sample3'/></a>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<base href='http://example.com' />
|
||||
|
|
@ -21,5 +23,4 @@
|
|||
</div>
|
||||
</div>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -1,14 +1,17 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<base href='http://example.com' />
|
||||
<title>Sample page without encoding for testing LinkExtractor</title>
|
||||
</head>
|
||||
<head>
|
||||
<base href='http://example.com' />
|
||||
<title>Sample page without encoding for testing LinkExtractor</title>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div id='wrapper'>
|
||||
<div id='subwrapper'>
|
||||
<a href='sample_ñ.html'><img src='sample2.jpg'/></a>
|
||||
</div>
|
||||
<a href='sample_€.html' title='sample €'>sample € text</a>
|
||||
</div>
|
||||
<div id='wrapper'>
|
||||
<div id='subwrapper'>
|
||||
<a href='sample_ñ.html'><img src='sample2.jpg' alt='sample2'/></a>
|
||||
</div>
|
||||
<a href='sample_€.html' title='sample €'>sample € text</a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -1,18 +1,15 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Scrapy test site</title>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
<h1>Scrapy test site</h1>
|
||||
|
||||
<ul>
|
||||
<li><a href="item1.html">Item 1</li>
|
||||
<li><a href="item2.html">Item 2</li>
|
||||
<li><a href="item999.html">Item 999 (not found)</li>
|
||||
</ul>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
<head>
|
||||
<title>Scrapy test site</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Scrapy test site</h1>
|
||||
<ul>
|
||||
<li><a href="item1.html">Item 1</a></li>
|
||||
<li><a href="item2.html">Item 2</a></li>
|
||||
<li><a href="item999.html">Item 999 (not found)</a></li>
|
||||
</ul>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1,17 +1,14 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Item 1 - Scrapy test site</title>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
<h1>Item 1 name</h1>
|
||||
|
||||
<ul>
|
||||
<li>Price: $100</li>
|
||||
<li>Stock: 12</li>
|
||||
</ul>
|
||||
|
||||
</body>
|
||||
<head>
|
||||
<title>Item 1 - Scrapy test site</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Item 1 name</h1>
|
||||
<ul>
|
||||
<li>Price: $100</li>
|
||||
<li>Stock: 12</li>
|
||||
</ul>
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -1,17 +1,14 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Item 2 - Scrapy test site</title>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
<h1>Item 2 name</h1>
|
||||
|
||||
<ul>
|
||||
<li>Price: $200</li>
|
||||
<li>Stock: 5</li>
|
||||
</ul>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
<head>
|
||||
<title>Item 2 - Scrapy test site</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Item 2 name</h1>
|
||||
<ul>
|
||||
<li>Price: $200</li>
|
||||
<li>Stock: 5</li>
|
||||
</ul>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
import os
|
||||
import argparse
|
||||
from os.path import join, abspath, isfile, exists
|
||||
|
||||
from twisted.internet import defer
|
||||
from scrapy.commands import parse
|
||||
from scrapy.settings import Settings
|
||||
|
|
@ -222,6 +223,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
|
|||
self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""")
|
||||
self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_crawlspider_not_exists_with_not_matched_url(self):
|
||||
status, out, stderr = yield self.execute([self.url('/invalid_url')])
|
||||
self.assertEqual(status, 0)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_output_flag(self):
|
||||
"""Checks if a file was created successfully having
|
||||
|
|
|
|||
|
|
@ -104,7 +104,8 @@ class CrawlerLoggingTestCase(unittest.TestCase):
|
|||
custom_settings = {
|
||||
'LOG_LEVEL': 'INFO',
|
||||
'LOG_FILE': log_file,
|
||||
# disable telnet if not available to avoid an extra warning
|
||||
# settings to avoid extra warnings
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
|
||||
'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
|
|||
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
|
||||
from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.http import Headers, Request
|
||||
from scrapy.http import Headers, HtmlResponse, Request
|
||||
from scrapy.http.response.text import TextResponse
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.spiders import Spider
|
||||
|
|
@ -389,6 +389,23 @@ class HttpTestCase(unittest.TestCase):
|
|||
d.addCallback(self.assertEqual, b'159')
|
||||
return d
|
||||
|
||||
def _test_response_class(self, filename, body, response_class):
|
||||
def _test(response):
|
||||
self.assertEqual(type(response), response_class)
|
||||
|
||||
request = Request(self.getURL(filename), body=body)
|
||||
return self.download_request(request, Spider('foo')).addCallback(_test)
|
||||
|
||||
def test_response_class_from_url(self):
|
||||
return self._test_response_class('foo.html', b'', HtmlResponse)
|
||||
|
||||
def test_response_class_from_body(self):
|
||||
return self._test_response_class(
|
||||
'foo',
|
||||
b"<!DOCTYPE html>\n<title>.</title>",
|
||||
HtmlResponse,
|
||||
)
|
||||
|
||||
|
||||
class Http10TestCase(HttpTestCase):
|
||||
"""HTTP 1.0 test case"""
|
||||
|
|
@ -971,6 +988,12 @@ class BaseFTPTestCase(unittest.TestCase):
|
|||
password = "passwd"
|
||||
req_meta = {"ftp_user": username, "ftp_password": password}
|
||||
|
||||
test_files = (
|
||||
('file.txt', b"I have the power!"),
|
||||
('file with spaces.txt', b"Moooooooooo power!"),
|
||||
('html-file-without-extension', b"<!DOCTYPE html>\n<title>.</title>"),
|
||||
)
|
||||
|
||||
def setUp(self):
|
||||
from twisted.protocols.ftp import FTPRealm, FTPFactory
|
||||
from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
|
||||
|
|
@ -981,8 +1004,8 @@ class BaseFTPTestCase(unittest.TestCase):
|
|||
userdir = os.path.join(self.directory, self.username)
|
||||
os.mkdir(userdir)
|
||||
fp = FilePath(userdir)
|
||||
fp.child('file.txt').setContent(b"I have the power!")
|
||||
fp.child('file with spaces.txt').setContent(b"Moooooooooo power!")
|
||||
for filename, content in self.test_files:
|
||||
fp.child(filename).setContent(content)
|
||||
|
||||
# setup server
|
||||
realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory)
|
||||
|
|
@ -1069,6 +1092,27 @@ class BaseFTPTestCase(unittest.TestCase):
|
|||
|
||||
return self._add_test_callbacks(d, _test)
|
||||
|
||||
def _test_response_class(self, filename, response_class):
|
||||
f, local_fname = tempfile.mkstemp()
|
||||
local_fname = to_bytes(local_fname)
|
||||
os.close(f)
|
||||
meta = {}
|
||||
meta.update(self.req_meta)
|
||||
request = Request(url=f"ftp://127.0.0.1:{self.portNum}/{filename}",
|
||||
meta=meta)
|
||||
d = self.download_handler.download_request(request, None)
|
||||
|
||||
def _test(r):
|
||||
self.assertEqual(type(r), response_class)
|
||||
os.remove(local_fname)
|
||||
return self._add_test_callbacks(d, _test)
|
||||
|
||||
def test_response_class_from_url(self):
|
||||
return self._test_response_class('file.txt', TextResponse)
|
||||
|
||||
def test_response_class_from_body(self):
|
||||
return self._test_response_class('html-file-without-extension', HtmlResponse)
|
||||
|
||||
|
||||
class FTPTestCase(BaseFTPTestCase):
|
||||
|
||||
|
|
@ -1104,8 +1148,8 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
|
|||
os.mkdir(self.directory)
|
||||
|
||||
fp = FilePath(self.directory)
|
||||
fp.child('file.txt').setContent(b"I have the power!")
|
||||
fp.child('file with spaces.txt').setContent(b"Moooooooooo power!")
|
||||
for filename, content in self.test_files:
|
||||
fp.child(filename).setContent(content)
|
||||
|
||||
# setup server for anonymous access
|
||||
realm = FTPRealm(anonymousRoot=self.directory)
|
||||
|
|
|
|||
|
|
@ -1,13 +1,11 @@
|
|||
import asyncio
|
||||
from unittest import mock, SkipTest
|
||||
from unittest import mock
|
||||
|
||||
from pytest import mark
|
||||
from twisted import version as twisted_version
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.trial.unittest import TestCase
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.python.versions import Version
|
||||
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.spiders import Spider
|
||||
|
|
@ -218,16 +216,6 @@ class MiddlewareUsingCoro(ManagerTestCase):
|
|||
"""Middlewares using asyncio coroutines should work"""
|
||||
|
||||
def test_asyncdef(self):
|
||||
if (
|
||||
self.reactor_pytest == 'asyncio'
|
||||
and twisted_version < Version('twisted', 18, 4, 0)
|
||||
):
|
||||
raise SkipTest(
|
||||
'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
|
||||
'hangs when using AsyncIO and Twisted versions lower than '
|
||||
'18.4.0'
|
||||
)
|
||||
|
||||
resp = Response('http://example.com/index.html')
|
||||
|
||||
class CoroMiddleware:
|
||||
|
|
@ -248,12 +236,6 @@ class MiddlewareUsingCoro(ManagerTestCase):
|
|||
|
||||
@mark.only_asyncio()
|
||||
def test_asyncdef_asyncio(self):
|
||||
if twisted_version < Version('twisted', 18, 4, 0):
|
||||
raise SkipTest(
|
||||
'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
|
||||
'hangs when using Twisted versions lower than 18.4.0'
|
||||
)
|
||||
|
||||
resp = Response('http://example.com/index.html')
|
||||
|
||||
class CoroMiddleware:
|
||||
|
|
|
|||
|
|
@ -122,6 +122,21 @@ class DefaultStorageTest(_BaseTest):
|
|||
time.sleep(0.5) # give the chance to expire
|
||||
assert storage.retrieve_response(self.spider, self.request)
|
||||
|
||||
def test_storage_no_content_type_header(self):
|
||||
"""Test that the response body is used to get the right response class
|
||||
even if there is no Content-Type header"""
|
||||
with self._storage() as storage:
|
||||
assert storage.retrieve_response(self.spider, self.request) is None
|
||||
response = Response(
|
||||
'http://www.example.com',
|
||||
body=b'<!DOCTYPE html>\n<title>.</title>',
|
||||
status=202,
|
||||
)
|
||||
storage.store_response(self.spider, self.request, response)
|
||||
cached_response = storage.retrieve_response(self.spider, self.request)
|
||||
self.assertIsInstance(cached_response, HtmlResponse)
|
||||
self.assertEqualResponse(response, cached_response)
|
||||
|
||||
|
||||
class DbmStorageTest(DefaultStorageTest):
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,16 @@ from scrapy.utils.test import get_crawler
|
|||
from tests.spiders import SimpleSpider
|
||||
|
||||
|
||||
def _get_dupefilter(*, crawler=None, settings=None, open=True):
|
||||
if crawler is None:
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
dupefilter = scheduler.df
|
||||
if open:
|
||||
dupefilter.open()
|
||||
return dupefilter
|
||||
|
||||
|
||||
class FromCrawlerRFPDupeFilter(RFPDupeFilter):
|
||||
|
||||
@classmethod
|
||||
|
|
@ -64,9 +74,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
self.assertEqual(scheduler.df.method, 'n/a')
|
||||
|
||||
def test_filter(self):
|
||||
dupefilter = RFPDupeFilter()
|
||||
dupefilter.open()
|
||||
|
||||
dupefilter = _get_dupefilter()
|
||||
r1 = Request('http://scrapytest.org/1')
|
||||
r2 = Request('http://scrapytest.org/2')
|
||||
r3 = Request('http://scrapytest.org/2')
|
||||
|
|
@ -85,7 +93,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
|
||||
path = tempfile.mkdtemp()
|
||||
try:
|
||||
df = RFPDupeFilter(path)
|
||||
df = _get_dupefilter(settings={'JOBDIR': path}, open=False)
|
||||
try:
|
||||
df.open()
|
||||
assert not df.request_seen(r1)
|
||||
|
|
@ -93,7 +101,8 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
finally:
|
||||
df.close('finished')
|
||||
|
||||
df2 = RFPDupeFilter(path)
|
||||
df2 = _get_dupefilter(settings={'JOBDIR': path}, open=False)
|
||||
assert df != df2
|
||||
try:
|
||||
df2.open()
|
||||
assert df2.request_seen(r1)
|
||||
|
|
@ -109,26 +118,24 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
output of request_seen.
|
||||
|
||||
"""
|
||||
dupefilter = _get_dupefilter()
|
||||
r1 = Request('http://scrapytest.org/index.html')
|
||||
r2 = Request('http://scrapytest.org/INDEX.html')
|
||||
|
||||
dupefilter = RFPDupeFilter()
|
||||
dupefilter.open()
|
||||
|
||||
assert not dupefilter.request_seen(r1)
|
||||
assert not dupefilter.request_seen(r2)
|
||||
|
||||
dupefilter.close('finished')
|
||||
|
||||
class CaseInsensitiveRFPDupeFilter(RFPDupeFilter):
|
||||
class RequestFingerprinter:
|
||||
|
||||
def request_fingerprint(self, request):
|
||||
def fingerprint(self, request):
|
||||
fp = hashlib.sha1()
|
||||
fp.update(to_bytes(request.url.lower()))
|
||||
return fp.hexdigest()
|
||||
return fp.digest()
|
||||
|
||||
case_insensitive_dupefilter = CaseInsensitiveRFPDupeFilter()
|
||||
case_insensitive_dupefilter.open()
|
||||
settings = {'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter}
|
||||
case_insensitive_dupefilter = _get_dupefilter(settings=settings)
|
||||
|
||||
assert not case_insensitive_dupefilter.request_seen(r1)
|
||||
assert case_insensitive_dupefilter.request_seen(r2)
|
||||
|
|
@ -142,8 +149,10 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
r1 = Request('http://scrapytest.org/1')
|
||||
|
||||
path = tempfile.mkdtemp()
|
||||
crawler = get_crawler(settings_dict={'JOBDIR': path})
|
||||
try:
|
||||
df = RFPDupeFilter(path)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
df = scheduler.df
|
||||
df.open()
|
||||
df.request_seen(r1)
|
||||
df.close('finished')
|
||||
|
|
@ -164,11 +173,8 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
settings = {'DUPEFILTER_DEBUG': False,
|
||||
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
|
||||
crawler = get_crawler(SimpleSpider, settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
spider = SimpleSpider.from_crawler(crawler)
|
||||
|
||||
dupefilter = scheduler.df
|
||||
dupefilter.open()
|
||||
dupefilter = _get_dupefilter(crawler=crawler)
|
||||
|
||||
r1 = Request('http://scrapytest.org/index.html')
|
||||
r2 = Request('http://scrapytest.org/index.html')
|
||||
|
|
@ -193,11 +199,41 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
settings = {'DUPEFILTER_DEBUG': True,
|
||||
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
|
||||
crawler = get_crawler(SimpleSpider, settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
spider = SimpleSpider.from_crawler(crawler)
|
||||
|
||||
dupefilter = scheduler.df
|
||||
dupefilter.open()
|
||||
dupefilter = _get_dupefilter(crawler=crawler)
|
||||
|
||||
r1 = Request('http://scrapytest.org/index.html')
|
||||
r2 = Request('http://scrapytest.org/index.html',
|
||||
headers={'Referer': 'http://scrapytest.org/INDEX.html'})
|
||||
|
||||
dupefilter.log(r1, spider)
|
||||
dupefilter.log(r2, spider)
|
||||
|
||||
assert crawler.stats.get_value('dupefilter/filtered') == 2
|
||||
log.check_present(
|
||||
(
|
||||
'scrapy.dupefilters',
|
||||
'DEBUG',
|
||||
'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)'
|
||||
)
|
||||
)
|
||||
log.check_present(
|
||||
(
|
||||
'scrapy.dupefilters',
|
||||
'DEBUG',
|
||||
'Filtered duplicate request: <GET http://scrapytest.org/index.html>'
|
||||
' (referer: http://scrapytest.org/INDEX.html)'
|
||||
)
|
||||
)
|
||||
|
||||
dupefilter.close('finished')
|
||||
|
||||
def test_log_debug_default_dupefilter(self):
|
||||
with LogCapture() as log:
|
||||
settings = {'DUPEFILTER_DEBUG': True}
|
||||
crawler = get_crawler(SimpleSpider, settings_dict=settings)
|
||||
spider = SimpleSpider.from_crawler(crawler)
|
||||
dupefilter = _get_dupefilter(crawler=crawler)
|
||||
|
||||
r1 = Request('http://scrapytest.org/index.html')
|
||||
r2 = Request('http://scrapytest.org/index.html',
|
||||
|
|
|
|||
|
|
@ -112,6 +112,14 @@ class BaseItemExporterTest(unittest.TestCase):
|
|||
assert isinstance(name, str)
|
||||
self.assertEqual(name, 'John\xa3')
|
||||
|
||||
ie = self._get_exporter(
|
||||
fields_to_export={'name': '名稱'}
|
||||
)
|
||||
self.assertEqual(
|
||||
list(ie._get_serialized_fields(self.i)),
|
||||
[('名稱', 'John\xa3')]
|
||||
)
|
||||
|
||||
def test_field_custom_serializer(self):
|
||||
i = self.custom_field_item_class(name='John\xa3', age='22')
|
||||
a = ItemAdapter(i)
|
||||
|
|
@ -272,6 +280,7 @@ class MarshalItemExporterDataclassTest(MarshalItemExporterTest):
|
|||
|
||||
class CsvItemExporterTest(BaseItemExporterTest):
|
||||
def _get_exporter(self, **kwargs):
|
||||
self.output = tempfile.TemporaryFile()
|
||||
return CsvItemExporter(self.output, **kwargs)
|
||||
|
||||
def assertCsvEqual(self, first, second, msg=None):
|
||||
|
|
@ -283,7 +292,8 @@ class CsvItemExporterTest(BaseItemExporterTest):
|
|||
return self.assertEqual(split_csv(first), split_csv(second), msg=msg)
|
||||
|
||||
def _check_output(self):
|
||||
self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n')
|
||||
self.output.seek(0)
|
||||
self.assertCsvEqual(to_unicode(self.output.read()), 'age,name\r\n22,John\xa3\r\n')
|
||||
|
||||
def assertExportResult(self, item, expected, **kwargs):
|
||||
fp = BytesIO()
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import os
|
|||
import random
|
||||
import shutil
|
||||
import string
|
||||
import sys
|
||||
import tempfile
|
||||
import warnings
|
||||
from abc import ABC, abstractmethod
|
||||
|
|
@ -655,8 +656,8 @@ class FeedExportTestBase(ABC, unittest.TestCase):
|
|||
return data
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def assertExported(self, items, header, rows, settings=None, ordered=True):
|
||||
yield self.assertExportedCsv(items, header, rows, settings, ordered)
|
||||
def assertExported(self, items, header, rows, settings=None):
|
||||
yield self.assertExportedCsv(items, header, rows, settings)
|
||||
yield self.assertExportedJsonLines(items, rows, settings)
|
||||
yield self.assertExportedXml(items, rows, settings)
|
||||
yield self.assertExportedPickle(items, rows, settings)
|
||||
|
|
@ -717,7 +718,7 @@ class FeedExportTest(FeedExportTestBase):
|
|||
return content
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
|
||||
def assertExportedCsv(self, items, header, rows, settings=None):
|
||||
settings = settings or {}
|
||||
settings.update({
|
||||
'FEEDS': {
|
||||
|
|
@ -725,15 +726,9 @@ class FeedExportTest(FeedExportTestBase):
|
|||
},
|
||||
})
|
||||
data = yield self.exported_data(items, settings)
|
||||
|
||||
reader = csv.DictReader(to_unicode(data['csv']).splitlines())
|
||||
got_rows = list(reader)
|
||||
if ordered:
|
||||
self.assertEqual(reader.fieldnames, header)
|
||||
else:
|
||||
self.assertEqual(set(reader.fieldnames), set(header))
|
||||
|
||||
self.assertEqual(rows, got_rows)
|
||||
self.assertEqual(reader.fieldnames, list(header))
|
||||
self.assertEqual(rows, list(reader))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def assertExportedJsonLines(self, items, rows, settings=None):
|
||||
|
|
@ -884,7 +879,7 @@ class FeedExportTest(FeedExportTestBase):
|
|||
{'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
|
||||
]
|
||||
header = self.MyItem.fields.keys()
|
||||
yield self.assertExported(items, header, rows, ordered=False)
|
||||
yield self.assertExported(items, header, rows)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_no_items_not_store_empty(self):
|
||||
|
|
@ -956,25 +951,72 @@ class FeedExportTest(FeedExportTestBase):
|
|||
{'egg': 'spam4', 'foo': '', 'baz': ''},
|
||||
]
|
||||
rows_jl = [dict(row) for row in items]
|
||||
yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
|
||||
yield self.assertExportedCsv(items, header, rows_csv)
|
||||
yield self.assertExportedJsonLines(items, rows_jl)
|
||||
|
||||
# edge case: FEED_EXPORT_FIELDS==[] means the same as default None
|
||||
@defer.inlineCallbacks
|
||||
def test_export_items_empty_field_list(self):
|
||||
# FEED_EXPORT_FIELDS==[] means the same as default None
|
||||
items = [{'foo': 'bar'}]
|
||||
header = ["foo"]
|
||||
rows = [{'foo': 'bar'}]
|
||||
settings = {'FEED_EXPORT_FIELDS': []}
|
||||
yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
|
||||
yield self.assertExportedJsonLines(items, rows_jl, settings)
|
||||
yield self.assertExportedCsv(items, header, rows)
|
||||
yield self.assertExportedJsonLines(items, rows, settings)
|
||||
|
||||
# it is possible to override fields using FEED_EXPORT_FIELDS
|
||||
header = ["foo", "baz", "hello"]
|
||||
@defer.inlineCallbacks
|
||||
def test_export_items_field_list(self):
|
||||
items = [{'foo': 'bar'}]
|
||||
header = ["foo", "baz"]
|
||||
rows = [{'foo': 'bar', 'baz': ''}]
|
||||
settings = {'FEED_EXPORT_FIELDS': header}
|
||||
rows = [
|
||||
{'foo': 'bar1', 'baz': '', 'hello': ''},
|
||||
{'foo': 'bar2', 'baz': '', 'hello': 'world2'},
|
||||
{'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
|
||||
{'foo': '', 'baz': '', 'hello': 'world4'},
|
||||
]
|
||||
yield self.assertExported(items, header, rows,
|
||||
settings=settings, ordered=True)
|
||||
yield self.assertExported(items, header, rows, settings=settings)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_items_comma_separated_field_list(self):
|
||||
items = [{'foo': 'bar'}]
|
||||
header = ["foo", "baz"]
|
||||
rows = [{'foo': 'bar', 'baz': ''}]
|
||||
settings = {'FEED_EXPORT_FIELDS': ",".join(header)}
|
||||
yield self.assertExported(items, header, rows, settings=settings)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_items_json_field_list(self):
|
||||
items = [{'foo': 'bar'}]
|
||||
header = ["foo", "baz"]
|
||||
rows = [{'foo': 'bar', 'baz': ''}]
|
||||
settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
|
||||
yield self.assertExported(items, header, rows, settings=settings)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_items_field_names(self):
|
||||
items = [{'foo': 'bar'}]
|
||||
header = {'foo': 'Foo'}
|
||||
rows = [{'Foo': 'bar'}]
|
||||
settings = {'FEED_EXPORT_FIELDS': header}
|
||||
yield self.assertExported(items, list(header.values()), rows,
|
||||
settings=settings)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_items_dict_field_names(self):
|
||||
items = [{'foo': 'bar'}]
|
||||
header = {
|
||||
'baz': 'Baz',
|
||||
'foo': 'Foo',
|
||||
}
|
||||
rows = [{'Baz': '', 'Foo': 'bar'}]
|
||||
settings = {'FEED_EXPORT_FIELDS': header}
|
||||
yield self.assertExported(items, ['Baz', 'Foo'], rows,
|
||||
settings=settings)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_items_json_field_names(self):
|
||||
items = [{'foo': 'bar'}]
|
||||
header = {'foo': 'Foo'}
|
||||
rows = [{'Foo': 'bar'}]
|
||||
settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
|
||||
yield self.assertExported(items, list(header.values()), rows,
|
||||
settings=settings)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_based_on_item_classes(self):
|
||||
|
|
@ -1097,7 +1139,7 @@ class FeedExportTest(FeedExportTestBase):
|
|||
{'egg': 'spam', 'foo': 'bar'}
|
||||
]
|
||||
rows_jl = items
|
||||
yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False)
|
||||
yield self.assertExportedCsv(items, ['foo', 'egg'], rows_csv)
|
||||
yield self.assertExportedJsonLines(items, rows_jl)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
|
@ -1118,7 +1160,7 @@ class FeedExportTest(FeedExportTestBase):
|
|||
{'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
|
||||
]
|
||||
yield self.assertExported(items, ['foo', 'baz', 'egg'], rows,
|
||||
settings=settings, ordered=True)
|
||||
settings=settings)
|
||||
|
||||
# export a subset of columns
|
||||
settings = {'FEED_EXPORT_FIELDS': 'egg,baz'}
|
||||
|
|
@ -1127,7 +1169,7 @@ class FeedExportTest(FeedExportTestBase):
|
|||
{'egg': 'spam2', 'baz': 'quux2'}
|
||||
]
|
||||
yield self.assertExported(items, ['egg', 'baz'], rows,
|
||||
settings=settings, ordered=True)
|
||||
settings=settings)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_encoding(self):
|
||||
|
|
@ -1769,7 +1811,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def test_lzma_plugin_filters(self):
|
||||
import sys
|
||||
if "PyPy" in sys.version:
|
||||
# https://foss.heptapod.net/pypy/pypy/-/issues/3527
|
||||
raise unittest.SkipTest("lzma filters doesn't work in PyPy")
|
||||
|
|
@ -2016,7 +2057,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
|
|||
self.assertEqual(expected_batch, got_batch)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
|
||||
def assertExportedCsv(self, items, header, rows, settings=None):
|
||||
settings = settings or {}
|
||||
settings.update({
|
||||
'FEEDS': {
|
||||
|
|
|
|||
|
|
@ -38,6 +38,12 @@ class HeadersTest(unittest.TestCase):
|
|||
self.assertEqual(h.getlist('X-Forwarded-For'), [b'ip1', b'ip2'])
|
||||
assert h.getlist('X-Forwarded-For') is not hlist
|
||||
|
||||
def test_multivalue_for_one_header(self):
|
||||
h = Headers((("a", "b"), ("a", "c")))
|
||||
self.assertEqual(h["a"], b"c")
|
||||
self.assertEqual(h.get("a"), b"c")
|
||||
self.assertEqual(h.getlist("a"), [b"b", b"c"])
|
||||
|
||||
def test_encode_utf8(self):
|
||||
h = Headers({'key': '\xa3'}, encoding='utf-8')
|
||||
key, val = dict(h).popitem()
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from scrapy.pipelines.files import (
|
|||
from scrapy.settings import Settings
|
||||
from scrapy.utils.test import (
|
||||
assert_gcs_environ,
|
||||
get_crawler,
|
||||
get_ftp_content_and_delete,
|
||||
get_gcs_content_and_delete,
|
||||
skip_if_no_boto,
|
||||
|
|
@ -47,7 +48,9 @@ class FilesPipelineTestCase(unittest.TestCase):
|
|||
|
||||
def setUp(self):
|
||||
self.tempdir = mkdtemp()
|
||||
self.pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir}))
|
||||
settings_dict = {'FILES_STORE': self.tempdir}
|
||||
crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
|
||||
self.pipeline = FilesPipeline.from_crawler(crawler)
|
||||
self.pipeline.download_func = _mocked_download_func
|
||||
self.pipeline.open_spider(None)
|
||||
|
||||
|
|
@ -525,6 +528,29 @@ class TestGCSFilesStore(unittest.TestCase):
|
|||
self.assertEqual(blob.content_type, 'application/octet-stream')
|
||||
self.assertIn(expected_policy, acl)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_blob_path_consistency(self):
|
||||
"""Test to make sure that paths used to store files is the same as the one used to get
|
||||
already uploaded files.
|
||||
"""
|
||||
assert_gcs_environ()
|
||||
try:
|
||||
import google.cloud.storage # noqa
|
||||
except ModuleNotFoundError:
|
||||
raise unittest.SkipTest("google-cloud-storage is not installed")
|
||||
else:
|
||||
with mock.patch('google.cloud.storage') as _:
|
||||
with mock.patch('scrapy.pipelines.files.time') as _:
|
||||
uri = 'gs://my_bucket/my_prefix/'
|
||||
store = GCSFilesStore(uri)
|
||||
store.bucket = mock.Mock()
|
||||
path = 'full/my_data.txt'
|
||||
yield store.persist_file(path, mock.Mock(), info=None, meta=None, headers=None)
|
||||
yield store.stat_file(path, info=None)
|
||||
expected_blob_path = store.prefix + path
|
||||
store.bucket.blob.assert_called_with(expected_blob_path)
|
||||
store.bucket.get_blob.assert_called_with(expected_blob_path)
|
||||
|
||||
|
||||
class TestFTPFileStore(unittest.TestCase):
|
||||
@defer.inlineCallbacks
|
||||
|
|
|
|||
|
|
@ -93,6 +93,22 @@ class ImagesPipelineTestCase(unittest.TestCase):
|
|||
info=object()),
|
||||
'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg')
|
||||
|
||||
def test_thumbnail_name_from_item(self):
|
||||
"""
|
||||
Custom thumbnail name based on item data, overriding default implementation
|
||||
"""
|
||||
|
||||
class CustomImagesPipeline(ImagesPipeline):
|
||||
def thumb_path(self, request, thumb_id, response=None, info=None, item=None):
|
||||
return f"thumb/{thumb_id}/{item.get('path')}"
|
||||
|
||||
thumb_path = CustomImagesPipeline.from_settings(Settings(
|
||||
{'IMAGES_STORE': self.tempdir}
|
||||
)).thumb_path
|
||||
item = dict(path='path-to-store-file')
|
||||
request = Request("http://example.com")
|
||||
self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file')
|
||||
|
||||
def test_convert_image(self):
|
||||
SIZE = (100, 100)
|
||||
# straigh forward case: RGB and JPEG
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from typing import Optional
|
||||
import io
|
||||
|
||||
from testfixtures import LogCapture
|
||||
from twisted.trial import unittest
|
||||
|
|
@ -6,17 +7,17 @@ from twisted.python.failure import Failure
|
|||
from twisted.internet import reactor
|
||||
from twisted.internet.defer import Deferred, inlineCallbacks
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.pipelines.files import FileException
|
||||
from scrapy.pipelines.images import ImagesPipeline
|
||||
from scrapy.pipelines.media import MediaPipeline
|
||||
from scrapy.pipelines.files import FileException
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
from scrapy.utils.signal import disconnect_all
|
||||
from scrapy import signals
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
try:
|
||||
|
|
@ -38,11 +39,14 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
|
|||
settings = None
|
||||
|
||||
def setUp(self):
|
||||
self.spider = Spider('media.com')
|
||||
self.pipe = self.pipeline_class(download_func=_mocked_download_func,
|
||||
settings=Settings(self.settings))
|
||||
spider_cls = Spider
|
||||
self.spider = spider_cls('media.com')
|
||||
crawler = get_crawler(spider_cls, self.settings)
|
||||
self.pipe = self.pipeline_class.from_crawler(crawler)
|
||||
self.pipe.download_func = _mocked_download_func
|
||||
self.pipe.open_spider(self.spider)
|
||||
self.info = self.pipe.spiderinfo
|
||||
self.fingerprint = crawler.request_fingerprinter.fingerprint
|
||||
|
||||
def tearDown(self):
|
||||
for name, signal in vars(signals).items():
|
||||
|
|
@ -155,7 +159,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
|
|||
self.assertEqual(failure.value.__context__, def_gen_return_exc)
|
||||
|
||||
# Let's calculate the request fingerprint and fake some runtime data...
|
||||
fp = request_fingerprint(request)
|
||||
fp = self.fingerprint(request)
|
||||
info = self.pipe.spiderinfo
|
||||
info.downloading.add(fp)
|
||||
info.waiting[fp] = []
|
||||
|
|
@ -272,7 +276,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
item = dict(requests=req) # pass a single item
|
||||
new_item = yield self.pipe.process_item(item, self.spider)
|
||||
assert new_item is item
|
||||
assert request_fingerprint(req) in self.info.downloaded
|
||||
self.assertIn(self.fingerprint(req), self.info.downloaded)
|
||||
|
||||
# returns iterable of Requests
|
||||
req1 = Request('http://url1')
|
||||
|
|
@ -280,8 +284,8 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
item = dict(requests=iter([req1, req2]))
|
||||
new_item = yield self.pipe.process_item(item, self.spider)
|
||||
assert new_item is item
|
||||
assert request_fingerprint(req1) in self.info.downloaded
|
||||
assert request_fingerprint(req2) in self.info.downloaded
|
||||
assert self.fingerprint(req1) in self.info.downloaded
|
||||
assert self.fingerprint(req2) in self.info.downloaded
|
||||
|
||||
@inlineCallbacks
|
||||
def test_results_are_cached_across_multiple_items(self):
|
||||
|
|
@ -297,7 +301,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
item = dict(requests=req2)
|
||||
new_item = yield self.pipe.process_item(item, self.spider)
|
||||
self.assertTrue(new_item is item)
|
||||
self.assertEqual(request_fingerprint(req1), request_fingerprint(req2))
|
||||
self.assertEqual(self.fingerprint(req1), self.fingerprint(req2))
|
||||
self.assertEqual(new_item['results'], [(True, rsp1)])
|
||||
|
||||
@inlineCallbacks
|
||||
|
|
@ -313,7 +317,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
@inlineCallbacks
|
||||
def test_wait_if_request_is_downloading(self):
|
||||
def _check_downloading(response):
|
||||
fp = request_fingerprint(req1)
|
||||
fp = self.fingerprint(req1)
|
||||
self.assertTrue(fp in self.info.downloading)
|
||||
self.assertTrue(fp in self.info.waiting)
|
||||
self.assertTrue(fp not in self.info.downloaded)
|
||||
|
|
@ -350,14 +354,17 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(MockedMediaPipelineDeprecatedMethods, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
self._mockcalled = []
|
||||
|
||||
def get_media_requests(self, item, info):
|
||||
item_url = item['image_urls'][0]
|
||||
output_img = io.BytesIO()
|
||||
img = Image.new('RGB', (60, 30), color='red')
|
||||
img.save(output_img, format='JPEG')
|
||||
return Request(
|
||||
item_url,
|
||||
meta={'response': Response(item_url, status=200, body=b'data')}
|
||||
meta={'response': Response(item_url, status=200, body=output_img.getvalue())}
|
||||
)
|
||||
|
||||
def inc_stats(self, *args, **kwargs):
|
||||
|
|
@ -365,34 +372,44 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
|
|||
|
||||
def media_to_download(self, request, info):
|
||||
self._mockcalled.append('media_to_download')
|
||||
return super(MockedMediaPipelineDeprecatedMethods, self).media_to_download(request, info)
|
||||
return super().media_to_download(request, info)
|
||||
|
||||
def media_downloaded(self, response, request, info):
|
||||
self._mockcalled.append('media_downloaded')
|
||||
return super(MockedMediaPipelineDeprecatedMethods, self).media_downloaded(response, request, info)
|
||||
return super().media_downloaded(response, request, info)
|
||||
|
||||
def file_downloaded(self, response, request, info):
|
||||
self._mockcalled.append('file_downloaded')
|
||||
return super(MockedMediaPipelineDeprecatedMethods, self).file_downloaded(response, request, info)
|
||||
return super().file_downloaded(response, request, info)
|
||||
|
||||
def file_path(self, request, response=None, info=None):
|
||||
self._mockcalled.append('file_path')
|
||||
return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info)
|
||||
return super().file_path(request, response, info)
|
||||
|
||||
def thumb_path(self, request, thumb_id, response=None, info=None):
|
||||
self._mockcalled.append('thumb_path')
|
||||
return super(MockedMediaPipelineDeprecatedMethods, self).thumb_path(request, thumb_id, response, info)
|
||||
|
||||
def get_images(self, response, request, info):
|
||||
self._mockcalled.append('get_images')
|
||||
return []
|
||||
return super(MockedMediaPipelineDeprecatedMethods, self).get_images(response, request, info)
|
||||
|
||||
def image_downloaded(self, response, request, info):
|
||||
self._mockcalled.append('image_downloaded')
|
||||
return super(MockedMediaPipelineDeprecatedMethods, self).image_downloaded(response, request, info)
|
||||
return super().image_downloaded(response, request, info)
|
||||
|
||||
|
||||
class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
|
||||
skip = skip_pillow
|
||||
|
||||
def setUp(self):
|
||||
self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func)
|
||||
settings_dict = {
|
||||
'IMAGES_STORE': 'store-uri',
|
||||
'IMAGES_THUMBS': {'small': (50, 50)},
|
||||
}
|
||||
crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
|
||||
self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler)
|
||||
self.pipe.download_func = _mocked_download_func
|
||||
self.pipe.open_spider(None)
|
||||
self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[])
|
||||
|
||||
|
|
@ -444,6 +461,16 @@ class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
|
|||
)
|
||||
self._assert_method_called_with_warnings('file_path', message, warnings)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_thumb_path_called(self):
|
||||
yield self.pipe.process_item(self.item, None)
|
||||
warnings = self.flushWarnings([MediaPipeline._compatible])
|
||||
message = (
|
||||
'thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, '
|
||||
'please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)'
|
||||
)
|
||||
self._assert_method_called_with_warnings('thumb_path', message, warnings)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_get_images_called(self):
|
||||
yield self.pipe.process_item(self.item, None)
|
||||
|
|
|
|||
|
|
@ -54,6 +54,8 @@ class ResponseTypesTest(unittest.TestCase):
|
|||
(b'\x03\x02\xdf\xdd\x23', Response),
|
||||
(b'Some plain text\ndata with tabs\t and null bytes\0', TextResponse),
|
||||
(b'<html><head><title>Hello</title></head>', HtmlResponse),
|
||||
# https://codersblock.com/blog/the-smallest-valid-html5-page/
|
||||
(b'<!DOCTYPE html>\n<title>.</title>', HtmlResponse),
|
||||
(b'<?xml version="1.0" encoding="utf-8"', XmlResponse),
|
||||
]
|
||||
for source, cls in mappings:
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import gc
|
|||
import operator
|
||||
import platform
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
from itertools import count
|
||||
from warnings import catch_warnings, filterwarnings
|
||||
|
||||
|
|
@ -224,12 +223,7 @@ class UtilsPythonTestCase(unittest.TestCase):
|
|||
elif platform.python_implementation() == 'PyPy':
|
||||
self.assertEqual(get_func_args(str.split, stripself=True), ['sep', 'maxsplit'])
|
||||
self.assertEqual(get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
|
||||
|
||||
build_date = datetime.strptime(platform.python_build()[1], '%b %d %Y')
|
||||
if build_date >= datetime(2020, 4, 7): # PyPy 3.6-v7.3.1
|
||||
self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])
|
||||
else:
|
||||
self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
|
||||
self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])
|
||||
|
||||
def test_without_none_values(self):
|
||||
self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])
|
||||
|
|
|
|||
|
|
@ -1,73 +1,29 @@
|
|||
import unittest
|
||||
import warnings
|
||||
from hashlib import sha1
|
||||
from typing import Dict, Mapping, Optional, Tuple, Union
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
import pytest
|
||||
from w3lib.url import canonicalize_url
|
||||
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.python import to_bytes
|
||||
from scrapy.utils.request import (
|
||||
_deprecated_fingerprint_cache,
|
||||
_fingerprint_cache,
|
||||
_request_fingerprint_as_bytes,
|
||||
fingerprint,
|
||||
request_authenticate,
|
||||
request_fingerprint,
|
||||
request_httprepr,
|
||||
)
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class UtilsRequestTest(unittest.TestCase):
|
||||
|
||||
def test_request_fingerprint(self):
|
||||
r1 = Request("http://www.example.com/query?id=111&cat=222")
|
||||
r2 = Request("http://www.example.com/query?cat=222&id=111")
|
||||
self.assertEqual(request_fingerprint(r1), request_fingerprint(r1))
|
||||
self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
|
||||
|
||||
r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199')
|
||||
r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
|
||||
self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
|
||||
|
||||
# make sure caching is working
|
||||
self.assertEqual(request_fingerprint(r1), _fingerprint_cache[r1][(None, False)])
|
||||
|
||||
r1 = Request("http://www.example.com/members/offers.html")
|
||||
r2 = Request("http://www.example.com/members/offers.html")
|
||||
r2.headers['SESSIONID'] = b"somehash"
|
||||
self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
|
||||
|
||||
r1 = Request("http://www.example.com/")
|
||||
r2 = Request("http://www.example.com/")
|
||||
r2.headers['Accept-Language'] = b'en'
|
||||
r3 = Request("http://www.example.com/")
|
||||
r3.headers['Accept-Language'] = b'en'
|
||||
r3.headers['SESSIONID'] = b"somehash"
|
||||
|
||||
self.assertEqual(request_fingerprint(r1), request_fingerprint(r2), request_fingerprint(r3))
|
||||
|
||||
self.assertEqual(request_fingerprint(r1),
|
||||
request_fingerprint(r1, include_headers=['Accept-Language']))
|
||||
|
||||
self.assertNotEqual(
|
||||
request_fingerprint(r1),
|
||||
request_fingerprint(r2, include_headers=['Accept-Language']))
|
||||
|
||||
self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']),
|
||||
request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language']))
|
||||
|
||||
r1 = Request("http://www.example.com/test.html")
|
||||
r2 = Request("http://www.example.com/test.html#fragment")
|
||||
self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
|
||||
self.assertEqual(request_fingerprint(r1), request_fingerprint(r1, keep_fragments=True))
|
||||
self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r2, keep_fragments=True))
|
||||
self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2, keep_fragments=True))
|
||||
|
||||
r1 = Request("http://www.example.com")
|
||||
r2 = Request("http://www.example.com", method='POST')
|
||||
r3 = Request("http://www.example.com", method='POST', body=b'request body')
|
||||
|
||||
self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
|
||||
self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r3))
|
||||
|
||||
# cached fingerprint must be cleared on request copy
|
||||
r1 = Request("http://www.example.com")
|
||||
fp1 = request_fingerprint(r1)
|
||||
r2 = r1.replace(url="http://www.example.com/other")
|
||||
fp2 = request_fingerprint(r2)
|
||||
self.assertNotEqual(fp1, fp2)
|
||||
|
||||
def test_request_authenticate(self):
|
||||
r = Request("http://www.example.com")
|
||||
request_authenticate(r, 'someuser', 'somepass')
|
||||
|
|
@ -93,5 +49,632 @@ class UtilsRequestTest(unittest.TestCase):
|
|||
request_httprepr(Request("ftp://localhost/tmp/foo.txt"))
|
||||
|
||||
|
||||
class FingerprintTest(unittest.TestCase):
|
||||
maxDiff = None
|
||||
|
||||
function = staticmethod(fingerprint)
|
||||
cache: Union[
|
||||
"WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]",
|
||||
"WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]",
|
||||
] = _fingerprint_cache
|
||||
default_cache_key = (None, False)
|
||||
known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
|
||||
(
|
||||
Request("http://example.org"),
|
||||
b'xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org"),
|
||||
b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org?a"),
|
||||
b'G\xad\xb8Ck\x19\x1c\xed\x838,\x01\xc4\xde;\xee\xa5\x94a\x0c',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org?a=b"),
|
||||
b'\x024MYb\x8a\xc2\x1e\xbc>\xd6\xac*\xda\x9cF\xc1r\x7f\x17',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org?a=b&a"),
|
||||
b't+\xe8*\xfb\x84\xe3v\x1a}\x88p\xc0\xccB\xd7\x9d\xfez\x96',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org?a=b&a=c"),
|
||||
b'\xda\x1ec\xd0\x9c\x08s`\xb4\x9b\xe2\xb6R\xf8k\xef\xeaQG\xef',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org", method='POST'),
|
||||
b'\x9d\xcdA\x0fT\x02:\xca\xa0}\x90\xda\x05B\xded\x8aN7\x1d',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org", body=b'a'),
|
||||
b'\xc34z>\xd8\x99\x8b\xda7\x05r\x99I\xa8\xa0x;\xa41_',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org", method='POST', body=b'a'),
|
||||
b'5`\xe2y4\xd0\x9d\xee\xe0\xbatw\x87Q\xe8O\xd78\xfc\xe7',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org#a", headers={'A': b'B'}),
|
||||
b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org#a", headers={'A': b'B'}),
|
||||
b']\xc7\x1f\xf2\xafG2\xbc\xa4\xfa\x99\n33\xda\x18\x94\x81U.',
|
||||
{'include_headers': ['A']},
|
||||
),
|
||||
(
|
||||
Request("https://example.org#a", headers={'A': b'B'}),
|
||||
b'<\x1a\xeb\x85y\xdeW\xfb\xdcq\x88\xee\xaf\x17\xdd\x0c\xbfH\x18\x1f',
|
||||
{'keep_fragments': True},
|
||||
),
|
||||
(
|
||||
Request("https://example.org#a", headers={'A': b'B'}),
|
||||
b'\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11',
|
||||
{'include_headers': ['A'], 'keep_fragments': True},
|
||||
),
|
||||
(
|
||||
Request("https://example.org/ab"),
|
||||
b'N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org/a", body=b'b'),
|
||||
b'_NOv\xbco$6\xfcW\x9f\xb24g\x9f\xbb\xdd\xa82\xc5',
|
||||
{},
|
||||
),
|
||||
)
|
||||
|
||||
def test_query_string_key_order(self):
|
||||
r1 = Request("http://www.example.com/query?id=111&cat=222")
|
||||
r2 = Request("http://www.example.com/query?cat=222&id=111")
|
||||
self.assertEqual(self.function(r1), self.function(r1))
|
||||
self.assertEqual(self.function(r1), self.function(r2))
|
||||
|
||||
def test_query_string_key_without_value(self):
|
||||
r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199')
|
||||
r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
|
||||
self.assertNotEqual(self.function(r1), self.function(r2))
|
||||
|
||||
def test_caching(self):
|
||||
r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
|
||||
self.assertEqual(
|
||||
self.function(r1),
|
||||
self.cache[r1][self.default_cache_key]
|
||||
)
|
||||
|
||||
def test_header(self):
|
||||
r1 = Request("http://www.example.com/members/offers.html")
|
||||
r2 = Request("http://www.example.com/members/offers.html")
|
||||
r2.headers['SESSIONID'] = b"somehash"
|
||||
self.assertEqual(self.function(r1), self.function(r2))
|
||||
|
||||
def test_headers(self):
|
||||
r1 = Request("http://www.example.com/")
|
||||
r2 = Request("http://www.example.com/")
|
||||
r2.headers['Accept-Language'] = b'en'
|
||||
r3 = Request("http://www.example.com/")
|
||||
r3.headers['Accept-Language'] = b'en'
|
||||
r3.headers['SESSIONID'] = b"somehash"
|
||||
|
||||
self.assertEqual(self.function(r1), self.function(r2), self.function(r3))
|
||||
|
||||
self.assertEqual(self.function(r1),
|
||||
self.function(r1, include_headers=['Accept-Language']))
|
||||
|
||||
self.assertNotEqual(
|
||||
self.function(r1),
|
||||
self.function(r2, include_headers=['Accept-Language']))
|
||||
|
||||
self.assertEqual(self.function(r3, include_headers=['accept-language', 'sessionid']),
|
||||
self.function(r3, include_headers=['SESSIONID', 'Accept-Language']))
|
||||
|
||||
def test_fragment(self):
|
||||
r1 = Request("http://www.example.com/test.html")
|
||||
r2 = Request("http://www.example.com/test.html#fragment")
|
||||
self.assertEqual(self.function(r1), self.function(r2))
|
||||
self.assertEqual(self.function(r1), self.function(r1, keep_fragments=True))
|
||||
self.assertNotEqual(self.function(r2), self.function(r2, keep_fragments=True))
|
||||
self.assertNotEqual(self.function(r1), self.function(r2, keep_fragments=True))
|
||||
|
||||
def test_method_and_body(self):
|
||||
r1 = Request("http://www.example.com")
|
||||
r2 = Request("http://www.example.com", method='POST')
|
||||
r3 = Request("http://www.example.com", method='POST', body=b'request body')
|
||||
|
||||
self.assertNotEqual(self.function(r1), self.function(r2))
|
||||
self.assertNotEqual(self.function(r2), self.function(r3))
|
||||
|
||||
def test_request_replace(self):
|
||||
# cached fingerprint must be cleared on request copy
|
||||
r1 = Request("http://www.example.com")
|
||||
fp1 = self.function(r1)
|
||||
r2 = r1.replace(url="http://www.example.com/other")
|
||||
fp2 = self.function(r2)
|
||||
self.assertNotEqual(fp1, fp2)
|
||||
|
||||
def test_part_separation(self):
|
||||
# An old implementation used to serialize request data in a way that
|
||||
# would put the body right after the URL.
|
||||
r1 = Request("http://www.example.com/foo")
|
||||
fp1 = self.function(r1)
|
||||
r2 = Request("http://www.example.com/f", body=b'oo')
|
||||
fp2 = self.function(r2)
|
||||
self.assertNotEqual(fp1, fp2)
|
||||
|
||||
def test_hashes(self):
|
||||
"""Test hardcoded hashes, to make sure future changes to not introduce
|
||||
backward incompatibilities."""
|
||||
actual = [
|
||||
self.function(request, **kwargs)
|
||||
for request, _, kwargs in self.known_hashes
|
||||
]
|
||||
expected = [
|
||||
_fingerprint
|
||||
for _, _fingerprint, _ in self.known_hashes
|
||||
]
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
|
||||
class RequestFingerprintTest(FingerprintTest):
|
||||
function = staticmethod(request_fingerprint)
|
||||
cache = _deprecated_fingerprint_cache
|
||||
known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
|
||||
(
|
||||
Request("http://example.org"),
|
||||
'b2e5245ef826fd9576c93bd6e392fce3133fab62',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org"),
|
||||
'bd10a0a89ea32cdee77917320f1309b0da87e892',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org?a"),
|
||||
'2fb7d48ae02f04b749f40caa969c0bc3c43204ce',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org?a=b"),
|
||||
'42e5fe149b147476e3f67ad0670c57b4cc57856a',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org?a=b&a"),
|
||||
'd23a9787cb56c6375c2cae4453c5a8c634526942',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org?a=b&a=c"),
|
||||
'9a18a7a8552a9182b7f1e05d33876409e421e5c5',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org", method='POST'),
|
||||
'ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org", body=b'a'),
|
||||
'4bb136e54e715a4ea7a9dd1101831765d33f2d60',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org", method='POST', body=b'a'),
|
||||
'6c6595374a304b293be762f7b7be3f54e9947c65',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org#a", headers={'A': b'B'}),
|
||||
'bd10a0a89ea32cdee77917320f1309b0da87e892',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org#a", headers={'A': b'B'}),
|
||||
'515b633cb3ca502a33a9d8c890e889ec1e425e65',
|
||||
{'include_headers': ['A']},
|
||||
),
|
||||
(
|
||||
Request("https://example.org#a", headers={'A': b'B'}),
|
||||
'505c96e7da675920dfef58725e8c957dfdb38f47',
|
||||
{'keep_fragments': True},
|
||||
),
|
||||
(
|
||||
Request("https://example.org#a", headers={'A': b'B'}),
|
||||
'd6f673cdcb661b7970c2b9a00ee63e87d1e2e5da',
|
||||
{'include_headers': ['A'], 'keep_fragments': True},
|
||||
),
|
||||
(
|
||||
Request("https://example.org/ab"),
|
||||
'4e2870fee58582d6f81755e9b8fdefe3cba0c951',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org/a", body=b'b'),
|
||||
'4e2870fee58582d6f81755e9b8fdefe3cba0c951',
|
||||
{},
|
||||
),
|
||||
)
|
||||
|
||||
@pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
|
||||
def test_part_separation(self):
|
||||
super().test_part_separation()
|
||||
|
||||
def test_deprecation_default_parameters(self):
|
||||
with pytest.warns(ScrapyDeprecationWarning) as warnings:
|
||||
self.function(Request("http://www.example.com"))
|
||||
messages = [str(warning.message) for warning in warnings]
|
||||
self.assertTrue(
|
||||
any(
|
||||
'Call to deprecated function' in message
|
||||
for message in messages
|
||||
)
|
||||
)
|
||||
self.assertFalse(any('non-default' in message for message in messages))
|
||||
|
||||
def test_deprecation_non_default_parameters(self):
|
||||
with pytest.warns(ScrapyDeprecationWarning) as warnings:
|
||||
self.function(Request("http://www.example.com"), keep_fragments=True)
|
||||
messages = [str(warning.message) for warning in warnings]
|
||||
self.assertTrue(
|
||||
any(
|
||||
'Call to deprecated function' in message
|
||||
for message in messages
|
||||
)
|
||||
)
|
||||
self.assertTrue(any('non-default' in message for message in messages))
|
||||
|
||||
|
||||
class RequestFingerprintAsBytesTest(FingerprintTest):
|
||||
function = staticmethod(_request_fingerprint_as_bytes)
|
||||
cache = _deprecated_fingerprint_cache
|
||||
known_hashes = RequestFingerprintTest.known_hashes
|
||||
|
||||
def test_caching(self):
|
||||
r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
|
||||
self.assertEqual(
|
||||
self.function(r1),
|
||||
bytes.fromhex(self.cache[r1][self.default_cache_key])
|
||||
)
|
||||
|
||||
@pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
|
||||
def test_part_separation(self):
|
||||
super().test_part_separation()
|
||||
|
||||
def test_hashes(self):
|
||||
actual = [
|
||||
self.function(request, **kwargs)
|
||||
for request, _, kwargs in self.known_hashes
|
||||
]
|
||||
expected = [
|
||||
bytes.fromhex(_fingerprint)
|
||||
for _, _fingerprint, _ in self.known_hashes
|
||||
]
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
|
||||
_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary()
|
||||
|
||||
|
||||
def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False):
|
||||
if include_headers:
|
||||
include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
|
||||
cache = _fingerprint_cache_2_6.setdefault(request, {})
|
||||
cache_key = (include_headers, keep_fragments)
|
||||
if cache_key not in cache:
|
||||
fp = sha1()
|
||||
fp.update(to_bytes(request.method))
|
||||
fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
|
||||
fp.update(request.body or b'')
|
||||
if include_headers:
|
||||
for hdr in include_headers:
|
||||
if hdr in request.headers:
|
||||
fp.update(hdr)
|
||||
for v in request.headers.getlist(hdr):
|
||||
fp.update(v)
|
||||
cache[cache_key] = fp.hexdigest()
|
||||
return cache[cache_key]
|
||||
|
||||
|
||||
REQUEST_OBJECTS_TO_TEST = (
|
||||
Request("http://www.example.com/"),
|
||||
Request("http://www.example.com/query?id=111&cat=222"),
|
||||
Request("http://www.example.com/query?cat=222&id=111"),
|
||||
Request('http://www.example.com/hnnoticiaj1.aspx?78132,199'),
|
||||
Request('http://www.example.com/hnnoticiaj1.aspx?78160,199'),
|
||||
Request("http://www.example.com/members/offers.html"),
|
||||
Request(
|
||||
"http://www.example.com/members/offers.html",
|
||||
headers={'SESSIONID': b"somehash"},
|
||||
),
|
||||
Request(
|
||||
"http://www.example.com/",
|
||||
headers={'Accept-Language': b"en"},
|
||||
),
|
||||
Request(
|
||||
"http://www.example.com/",
|
||||
headers={
|
||||
'Accept-Language': b"en",
|
||||
'SESSIONID': b"somehash",
|
||||
},
|
||||
),
|
||||
Request("http://www.example.com/test.html"),
|
||||
Request("http://www.example.com/test.html#fragment"),
|
||||
Request("http://www.example.com", method='POST'),
|
||||
Request("http://www.example.com", method='POST', body=b'request body'),
|
||||
)
|
||||
|
||||
|
||||
class BackwardCompatibilityTestCase(unittest.TestCase):
|
||||
|
||||
def test_function_backward_compatibility(self):
|
||||
include_headers_to_test = (
|
||||
None,
|
||||
['Accept-Language'],
|
||||
['accept-language', 'sessionid'],
|
||||
['SESSIONID', 'Accept-Language'],
|
||||
)
|
||||
for request_object in REQUEST_OBJECTS_TO_TEST:
|
||||
for include_headers in include_headers_to_test:
|
||||
for keep_fragments in (False, True):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
fp = request_fingerprint(
|
||||
request_object,
|
||||
include_headers=include_headers,
|
||||
keep_fragments=keep_fragments,
|
||||
)
|
||||
old_fp = request_fingerprint_2_6(
|
||||
request_object,
|
||||
include_headers=include_headers,
|
||||
keep_fragments=keep_fragments,
|
||||
)
|
||||
self.assertEqual(fp, old_fp)
|
||||
|
||||
def test_component_backward_compatibility(self):
|
||||
for request_object in REQUEST_OBJECTS_TO_TEST:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
crawler = get_crawler(prevent_warnings=False)
|
||||
fp = crawler.request_fingerprinter.fingerprint(request_object)
|
||||
old_fp = request_fingerprint_2_6(request_object)
|
||||
self.assertEqual(fp.hex(), old_fp)
|
||||
|
||||
def test_custom_component_backward_compatibility(self):
|
||||
"""Tests that the backward-compatible request fingerprinting class featured
|
||||
in the documentation is indeed backward compatible and does not cause a
|
||||
warning to be logged."""
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
cache = WeakKeyDictionary()
|
||||
|
||||
def fingerprint(self, request):
|
||||
if request not in self.cache:
|
||||
fp = sha1()
|
||||
fp.update(to_bytes(request.method))
|
||||
fp.update(to_bytes(canonicalize_url(request.url)))
|
||||
fp.update(request.body or b'')
|
||||
self.cache[request] = fp.digest()
|
||||
return self.cache[request]
|
||||
|
||||
for request_object in REQUEST_OBJECTS_TO_TEST:
|
||||
with warnings.catch_warnings() as logged_warnings:
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
fp = crawler.request_fingerprinter.fingerprint(request_object)
|
||||
old_fp = request_fingerprint_2_6(request_object)
|
||||
self.assertEqual(fp.hex(), old_fp)
|
||||
self.assertFalse(logged_warnings)
|
||||
|
||||
|
||||
class RequestFingerprinterTestCase(unittest.TestCase):
|
||||
|
||||
def test_default_implementation(self):
|
||||
with warnings.catch_warnings(record=True) as logged_warnings:
|
||||
crawler = get_crawler(prevent_warnings=False)
|
||||
request = Request('https://example.com')
|
||||
self.assertEqual(
|
||||
crawler.request_fingerprinter.fingerprint(request),
|
||||
_request_fingerprint_as_bytes(request),
|
||||
)
|
||||
self.assertTrue(logged_warnings)
|
||||
|
||||
def test_deprecated_implementation(self):
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'PREVIOUS_VERSION',
|
||||
}
|
||||
with warnings.catch_warnings(record=True) as logged_warnings:
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
request = Request('https://example.com')
|
||||
self.assertEqual(
|
||||
crawler.request_fingerprinter.fingerprint(request),
|
||||
_request_fingerprint_as_bytes(request),
|
||||
)
|
||||
self.assertTrue(logged_warnings)
|
||||
|
||||
def test_recommended_implementation(self):
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
|
||||
}
|
||||
with warnings.catch_warnings(record=True) as logged_warnings:
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
request = Request('https://example.com')
|
||||
self.assertEqual(
|
||||
crawler.request_fingerprinter.fingerprint(request),
|
||||
fingerprint(request),
|
||||
)
|
||||
self.assertFalse(logged_warnings)
|
||||
|
||||
def test_unknown_implementation(self):
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.5',
|
||||
}
|
||||
with self.assertRaises(ValueError):
|
||||
get_crawler(settings_dict=settings)
|
||||
|
||||
|
||||
class CustomRequestFingerprinterTestCase(unittest.TestCase):
|
||||
|
||||
def test_include_headers(self):
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
def fingerprint(self, request):
|
||||
return fingerprint(request, include_headers=['X-ID'])
|
||||
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
|
||||
r1 = Request("http://www.example.com", headers={'X-ID': '1'})
|
||||
fp1 = crawler.request_fingerprinter.fingerprint(r1)
|
||||
r2 = Request("http://www.example.com", headers={'X-ID': '2'})
|
||||
fp2 = crawler.request_fingerprinter.fingerprint(r2)
|
||||
self.assertNotEqual(fp1, fp2)
|
||||
|
||||
def test_dont_canonicalize(self):
|
||||
|
||||
class RequestFingerprinter:
|
||||
cache = WeakKeyDictionary()
|
||||
|
||||
def fingerprint(self, request):
|
||||
if request not in self.cache:
|
||||
fp = sha1()
|
||||
fp.update(to_bytes(request.url))
|
||||
self.cache[request] = fp.digest()
|
||||
return self.cache[request]
|
||||
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
|
||||
r1 = Request("http://www.example.com?a=1&a=2")
|
||||
fp1 = crawler.request_fingerprinter.fingerprint(r1)
|
||||
r2 = Request("http://www.example.com?a=2&a=1")
|
||||
fp2 = crawler.request_fingerprinter.fingerprint(r2)
|
||||
self.assertNotEqual(fp1, fp2)
|
||||
|
||||
def test_meta(self):
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
def fingerprint(self, request):
|
||||
if 'fingerprint' in request.meta:
|
||||
return request.meta['fingerprint']
|
||||
return fingerprint(request)
|
||||
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
|
||||
r1 = Request("http://www.example.com")
|
||||
fp1 = crawler.request_fingerprinter.fingerprint(r1)
|
||||
r2 = Request("http://www.example.com", meta={'fingerprint': 'a'})
|
||||
fp2 = crawler.request_fingerprinter.fingerprint(r2)
|
||||
r3 = Request("http://www.example.com", meta={'fingerprint': 'a'})
|
||||
fp3 = crawler.request_fingerprinter.fingerprint(r3)
|
||||
r4 = Request("http://www.example.com", meta={'fingerprint': 'b'})
|
||||
fp4 = crawler.request_fingerprinter.fingerprint(r4)
|
||||
self.assertNotEqual(fp1, fp2)
|
||||
self.assertNotEqual(fp1, fp4)
|
||||
self.assertNotEqual(fp2, fp4)
|
||||
self.assertEqual(fp2, fp3)
|
||||
|
||||
def test_from_crawler(self):
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(crawler)
|
||||
|
||||
def __init__(self, crawler):
|
||||
self._fingerprint = crawler.settings['FINGERPRINT']
|
||||
|
||||
def fingerprint(self, request):
|
||||
return self._fingerprint
|
||||
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
|
||||
'FINGERPRINT': b'fingerprint',
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
|
||||
request = Request("http://www.example.com")
|
||||
fingerprint = crawler.request_fingerprinter.fingerprint(request)
|
||||
self.assertEqual(fingerprint, settings['FINGERPRINT'])
|
||||
|
||||
def test_from_settings(self):
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, settings):
|
||||
return cls(settings)
|
||||
|
||||
def __init__(self, settings):
|
||||
self._fingerprint = settings['FINGERPRINT']
|
||||
|
||||
def fingerprint(self, request):
|
||||
return self._fingerprint
|
||||
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
|
||||
'FINGERPRINT': b'fingerprint',
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
|
||||
request = Request("http://www.example.com")
|
||||
fingerprint = crawler.request_fingerprinter.fingerprint(request)
|
||||
self.assertEqual(fingerprint, settings['FINGERPRINT'])
|
||||
|
||||
def test_from_crawler_and_settings(self):
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
# This method is ignored due to the presence of from_crawler
|
||||
@classmethod
|
||||
def from_settings(cls, settings):
|
||||
return cls(settings)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(crawler)
|
||||
|
||||
def __init__(self, crawler):
|
||||
self._fingerprint = crawler.settings['FINGERPRINT']
|
||||
|
||||
def fingerprint(self, request):
|
||||
return self._fingerprint
|
||||
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
|
||||
'FINGERPRINT': b'fingerprint',
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
|
||||
request = Request("http://www.example.com")
|
||||
fingerprint = crawler.request_fingerprinter.fingerprint(request)
|
||||
self.assertEqual(fingerprint, settings['FINGERPRINT'])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -1,13 +1,10 @@
|
|||
import asyncio
|
||||
from unittest import SkipTest
|
||||
|
||||
from pydispatch import dispatcher
|
||||
from pytest import mark
|
||||
from testfixtures import LogCapture
|
||||
from twisted import version as twisted_version
|
||||
from twisted.internet import defer, reactor
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.python.versions import Version
|
||||
from twisted.trial import unittest
|
||||
|
||||
from scrapy.utils.signal import send_catch_log, send_catch_log_deferred
|
||||
|
|
@ -81,16 +78,6 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
|
|||
return "OK"
|
||||
|
||||
def test_send_catch_log(self):
|
||||
if (
|
||||
self.reactor_pytest == 'asyncio'
|
||||
and twisted_version < Version('twisted', 18, 4, 0)
|
||||
):
|
||||
raise SkipTest(
|
||||
'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
|
||||
'fails due to a timeout when using AsyncIO and Twisted '
|
||||
'versions lower than 18.4.0'
|
||||
)
|
||||
|
||||
return super().test_send_catch_log()
|
||||
|
||||
|
||||
|
|
@ -104,13 +91,6 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
|
|||
return await get_from_asyncio_queue("OK")
|
||||
|
||||
def test_send_catch_log(self):
|
||||
if twisted_version < Version('twisted', 18, 4, 0):
|
||||
raise SkipTest(
|
||||
'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
|
||||
'fails due to a timeout when using Twisted versions lower '
|
||||
'than 18.4.0'
|
||||
)
|
||||
|
||||
return super().test_send_catch_log()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,10 +4,7 @@ Tests borrowed from the twisted.web.client tests.
|
|||
"""
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from pkg_resources import parse_version
|
||||
|
||||
import cryptography
|
||||
import OpenSSL.SSL
|
||||
from twisted.trial import unittest
|
||||
from twisted.web import server, static, util, resource
|
||||
|
|
@ -417,8 +414,6 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
|
|||
).addCallback(self.assertEqual, to_bytes(s))
|
||||
|
||||
def testPayloadDisabledCipher(self):
|
||||
if sys.implementation.name == "pypy" and parse_version(cryptography.__version__) <= parse_version("2.3.1"):
|
||||
self.skipTest("This test expects a failure, but the code does work in PyPy with cryptography<=2.3.1")
|
||||
s = "0123456789" * 10
|
||||
settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'})
|
||||
client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)
|
||||
|
|
|
|||
33
tox.ini
33
tox.ini
|
|
@ -11,14 +11,13 @@ minversion = 1.7.0
|
|||
deps =
|
||||
-rtests/requirements.txt
|
||||
# mitmproxy does not support PyPy
|
||||
# mitmproxy does not support Windows when running Python < 3.7
|
||||
# Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe
|
||||
# Python 3.9+ requires mitmproxy >= 5.3.0
|
||||
# mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0
|
||||
#mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
|
||||
mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy'
|
||||
mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
|
||||
# The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
|
||||
mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
|
||||
# newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower)
|
||||
markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy'
|
||||
markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy'
|
||||
# Extras
|
||||
botocore>=1.4.87
|
||||
passenv =
|
||||
|
|
@ -43,7 +42,6 @@ deps =
|
|||
types-pyOpenSSL==20.0.3
|
||||
types-setuptools==57.0.0
|
||||
commands =
|
||||
pip install types-dataclasses # remove once py36 support is dropped
|
||||
mypy --show-error-codes {posargs: scrapy tests}
|
||||
|
||||
[testenv:security]
|
||||
|
|
@ -74,18 +72,19 @@ commands =
|
|||
|
||||
[pinned]
|
||||
deps =
|
||||
cryptography==2.0
|
||||
cryptography==2.8
|
||||
cssselect==0.9.1
|
||||
h2==3.0
|
||||
itemadapter==0.1.0
|
||||
parsel==1.5.0
|
||||
Protego==0.1.15
|
||||
pyOpenSSL==16.2.0
|
||||
pyOpenSSL==19.1.0
|
||||
queuelib==1.4.2
|
||||
service_identity==16.0.0
|
||||
Twisted[http2]==17.9.0
|
||||
Twisted[http2]==18.9.0
|
||||
w3lib==1.17.0
|
||||
zope.interface==4.1.3
|
||||
zope.interface==5.1.0
|
||||
lxml==4.3.0
|
||||
-rtests/requirements.txt
|
||||
|
||||
# mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
|
||||
|
|
@ -94,7 +93,7 @@ deps =
|
|||
# Extras
|
||||
botocore==1.4.87
|
||||
google-cloud-storage==1.29.0
|
||||
Pillow==4.0.0
|
||||
Pillow==7.1.0
|
||||
setenv =
|
||||
_SCRAPY_PINNED=true
|
||||
install_command =
|
||||
|
|
@ -103,7 +102,6 @@ install_command =
|
|||
[testenv:pinned]
|
||||
deps =
|
||||
{[pinned]deps}
|
||||
lxml==3.5.0
|
||||
PyDispatcher==2.0.5
|
||||
install_command = {[pinned]install_command}
|
||||
setenv =
|
||||
|
|
@ -113,9 +111,6 @@ setenv =
|
|||
basepython = python3
|
||||
deps =
|
||||
{[pinned]deps}
|
||||
# First lxml version that includes a Windows wheel for Python 3.6, so we do
|
||||
# not need to build lxml from sources in a CI Windows job:
|
||||
lxml==3.8.0
|
||||
PyDispatcher==2.0.5
|
||||
install_command = {[pinned]install_command}
|
||||
setenv =
|
||||
|
|
@ -125,13 +120,14 @@ setenv =
|
|||
deps =
|
||||
{[testenv]deps}
|
||||
boto
|
||||
google-cloud-storage
|
||||
# Twisted[http2] currently forces old mitmproxy because of h2 version
|
||||
# restrictions in their deps, so we need to pin old markupsafe here too.
|
||||
markupsafe < 2.1.0
|
||||
reppy
|
||||
robotexclusionrulesparser
|
||||
Pillow>=4.0.0
|
||||
Twisted[http2]>=17.9.0
|
||||
# Twisted[http2] currently forces old mitmproxy because of h2 version restrictions in their deps,
|
||||
# so we need to pin old markupsafe here too
|
||||
markupsafe < 2.1.0
|
||||
|
||||
[testenv:asyncio]
|
||||
commands =
|
||||
|
|
@ -153,7 +149,6 @@ commands =
|
|||
basepython = {[testenv:pypy3]basepython}
|
||||
deps =
|
||||
{[pinned]deps}
|
||||
lxml==4.0.0
|
||||
PyPyDispatcher==2.1.0
|
||||
commands = {[testenv:pypy3]commands}
|
||||
install_command = {[pinned]install_command}
|
||||
|
|
|
|||
Loading…
Reference in New Issue