diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 98fa44c7f..b26f344ff 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -19,7 +19,7 @@ jobs:
- python-version: 3.8
env:
TOXENV: pylint
- - python-version: 3.6
+ - python-version: 3.7
env:
TOXENV: typing
- python-version: "3.10" # Keep in sync with .readthedocs.yml
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 3aaf688c7..7819a4e12 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -7,7 +7,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+ python-version: ["3.7", "3.8", "3.9", "3.10"]
steps:
- uses: actions/checkout@v2
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 1fc8d914b..be40c7c71 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -8,9 +8,6 @@ jobs:
fail-fast: false
matrix:
include:
- - python-version: 3.7
- env:
- TOXENV: py
- python-version: 3.8
env:
TOXENV: py
@@ -26,19 +23,19 @@ jobs:
- python-version: pypy3
env:
TOXENV: pypy3
- PYPY_VERSION: 3.6-v7.3.3
+ PYPY_VERSION: 3.9-v7.3.9
# pinned deps
- - python-version: 3.6.12
+ - python-version: 3.7.13
env:
TOXENV: pinned
- - python-version: 3.6.12
+ - python-version: 3.7.13
env:
TOXENV: asyncio-pinned
- python-version: pypy3
env:
TOXENV: pypy3-pinned
- PYPY_VERSION: 3.6-v7.2.0
+ PYPY_VERSION: 3.7-v7.3.5
# extras
# extra-deps includes reppy, which does not support Python 3.9
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index ab7385118..955b9b449 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -8,12 +8,9 @@ jobs:
fail-fast: false
matrix:
include:
- - python-version: 3.6
- env:
- TOXENV: windows-pinned
- python-version: 3.7
env:
- TOXENV: py
+ TOXENV: windows-pinned
- python-version: 3.8
env:
TOXENV: py
diff --git a/README.rst b/README.rst
index 6b563d638..b543a30f4 100644
--- a/README.rst
+++ b/README.rst
@@ -57,7 +57,7 @@ including a list of features.
Requirements
============
-* Python 3.6+
+* Python 3.7+
* Works on Linux, Windows, macOS, BSD
Install
diff --git a/docs/README.rst b/docs/README.rst
index 0b7afa548..36dd5aea4 100644
--- a/docs/README.rst
+++ b/docs/README.rst
@@ -43,7 +43,7 @@ This command will fire up your default browser and open the main page of your
Start over
----------
-To cleanup all generated documentation files and start from scratch run::
+To clean up all generated documentation files and start from scratch run::
make clean
diff --git a/docs/_static/selectors-sample1.html b/docs/_static/selectors-sample1.html
index 8a79a3381..915718832 100644
--- a/docs/_static/selectors-sample1.html
+++ b/docs/_static/selectors-sample1.html
@@ -1,16 +1,17 @@
-
-
-
- Example website
-
-
-
-
-
+
+
+
+
+ Example website
+
+
+
+
+
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
index 55aa72d5a..378b01804 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -294,7 +294,9 @@ intersphinx_mapping = {
'tox': ('https://tox.readthedocs.io/en/latest', None),
'twisted': ('https://twistedmatrix.com/documents/current', None),
'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
+ 'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
}
+intersphinx_disabled_reftypes = []
# Options for sphinx-hoverxref options
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 4d2580a6c..946bdc23e 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -232,15 +232,15 @@ To run a specific test (say ``tests/test_loader.py``) use:
To run the tests on a specific :doc:`tox ` environment, use
``-e `` with an environment name from ``tox.ini``. For example, to run
-the tests with Python 3.6 use::
+the tests with Python 3.7 use::
- tox -e py36
+ tox -e py37
You can also specify a comma-separated list of environments, and use :ref:`tox’s
parallel mode ` to run the tests on multiple environments in
parallel::
- tox -e py36,py38 -p auto
+ tox -e py37,py38 -p auto
To pass command-line options to :doc:`pytest `, add them after
``--`` in your call to :doc:`tox `. Using ``--`` overrides the
@@ -250,9 +250,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
tox -- scrapy tests -x # stop after first failure
You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
-the Python 3.6 :doc:`tox ` environment using all your CPU cores::
+the Python 3.7 :doc:`tox ` environment using all your CPU cores::
- tox -e py36 -- scrapy tests -n auto
+ tox -e py37 -- scrapy tests -n auto
To see coverage report install :doc:`coverage `
(``pip install coverage``) and run:
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index b8d3a16bc..80a9c16d6 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -9,8 +9,8 @@ Installation guide
Supported Python versions
=========================
-Scrapy requires Python 3.6+, either the CPython implementation (default) or
-the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
+Scrapy requires Python 3.7+, either the CPython implementation (default) or
+the PyPy 7.3.5+ implementation (see :ref:`python:implementations`).
.. _intro-install-scrapy:
@@ -52,16 +52,6 @@ Scrapy is written in pure Python and depends on a few key Python packages (among
* `twisted`_, an asynchronous networking framework
* `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs
-The minimal versions which Scrapy is tested against are:
-
-* Twisted 14.0
-* lxml 3.4
-* pyOpenSSL 0.14
-
-Scrapy may work with older versions of these packages
-but it is not guaranteed it will continue working
-because it’s not being tested against them.
-
Some of these packages themselves depends on non-Python packages
that might require additional installation steps depending on your platform.
Please check :ref:`platform-specific guides below `.
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index f3d652621..cfa6bfa83 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -45,9 +45,9 @@ https://quotes.toscrape.com, following the pagination::
Put this in a text file, name it to something like ``quotes_spider.py``
and run the spider using the :command:`runspider` command::
- scrapy runspider quotes_spider.py -o quotes.jl
+ scrapy runspider quotes_spider.py -o quotes.jsonl
-When this finishes you will have in the ``quotes.jl`` file a list of the
+When this finishes you will have in the ``quotes.jsonl`` file a list of the
quotes in JSON Lines format, containing text and author, looking like this::
{"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index cde1b1ef4..75928077e 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -482,7 +482,7 @@ to append new content to any existing file. However, appending to a JSON file
makes the file contents invalid JSON. When appending to a file, consider
using a different serialization format, such as `JSON Lines`_::
- scrapy crawl quotes -o quotes.jl
+ scrapy crawl quotes -o quotes.jsonl
The `JSON Lines`_ format is useful because it's stream-like, you can easily
append new records to it. It doesn't have the same problem of JSON when you run
diff --git a/docs/news.rst b/docs/news.rst
index 5d92067b5..2d0ab485e 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -1643,7 +1643,7 @@ New features
:issue:`4370`)
* A new ``keep_fragments`` parameter of
- :func:`scrapy.utils.request.request_fingerprint` allows to generate
+ ``scrapy.utils.request.request_fingerprint`` allows to generate
different fingerprints for requests with different fragments in their URL
(:issue:`4104`)
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 900b19c7a..60b5acd10 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -32,6 +32,13 @@ how you :ref:`configure the downloader middlewares
:class:`scrapy.Spider` subclass and a
:class:`scrapy.settings.Settings` object.
+ .. attribute:: request_fingerprinter
+
+ The request fingerprint builder of this crawler.
+
+ This is used from extensions and middlewares to build short, unique
+ identifiers for requests. See :ref:`request-fingerprints`.
+
.. attribute:: settings
The settings manager of this crawler.
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 2aef755c7..549552bd1 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -1,3 +1,5 @@
+.. _topics-coroutines:
+
==========
Coroutines
==========
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 923336769..3c36ef002 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -195,17 +195,25 @@ BaseItemExporter
.. attribute:: fields_to_export
- A list with the name of the fields that will be exported, or ``None`` if
- you want to export all fields. Defaults to ``None``.
+ Fields to export, their order [1]_ and their output names.
- Some exporters (like :class:`CsvItemExporter`) respect the order of the
- fields defined in this attribute.
+ Possible values are:
- When using :ref:`item objects ` that do not expose all their
- possible fields, exporters that do not support exporting a different
- subset of fields per item will only export the fields found in the first
- item exported. Use ``fields_to_export`` to define all the fields to be
- exported.
+ - ``None`` (all fields [2]_, default)
+
+ - A list of fields::
+
+ ['field1', 'field2']
+
+ - A dict where keys are fields and values are output names::
+
+ {'field1': 'Field 1', 'field2': 'Field 2'}
+
+ .. [1] Not all exporters respect the specified field order.
+ .. [2] When using :ref:`item objects ` that do not expose
+ all their possible fields, exporters that do not support exporting
+ a different subset of fields per item will only export the fields
+ found in the first item exported.
.. attribute:: export_empty_fields
@@ -297,8 +305,8 @@ CsvItemExporter
Exports items in CSV format to the given file-like object. If the
:attr:`fields_to_export` attribute is set, it will be used to define the
- CSV columns and their order. The :attr:`export_empty_fields` attribute has
- no effect on this exporter.
+ CSV columns, their order and their column names. The
+ :attr:`export_empty_fields` attribute has no effect on this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 9a13eb82f..a620e2c04 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -58,7 +58,7 @@ CSV
- Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
-- To specify columns to export and their order use
+- To specify columns to export, their order and their column names, use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
option, but it is important for CSV because unlike many other export
formats CSV uses a fixed header.
@@ -522,18 +522,9 @@ FEED_EXPORT_FIELDS
Default: ``None``
-A list of fields to export, optional.
-Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
-
-Use FEED_EXPORT_FIELDS option to define fields to export and their order.
-
-When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses the fields
-defined in :ref:`item objects ` yielded by your spider.
-
-If an exporter requires a fixed set of fields (this is the case for
-:ref:`CSV ` export format) and FEED_EXPORT_FIELDS
-is empty or None, then Scrapy tries to infer field names from the
-exported data - currently it uses field names from the first item.
+Use the ``FEED_EXPORT_FIELDS`` setting to define the fields to export, their
+order and their output names. See :attr:`BaseItemExporter.fields_to_export
+` for more information.
.. setting:: FEED_EXPORT_INDENT
@@ -638,6 +629,7 @@ Default::
{
'json': 'scrapy.exporters.JsonItemExporter',
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
+ 'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
'jl': 'scrapy.exporters.JsonLinesItemExporter',
'csv': 'scrapy.exporters.CsvItemExporter',
'xml': 'scrapy.exporters.XmlItemExporter',
@@ -763,7 +755,7 @@ source spider in the feed URI:
#. Use ``%(spider_name)s`` in your feed URI::
- scrapy crawl -o "%(spider_name)s.jl"
+ scrapy crawl -o "%(spider_name)s.jsonl"
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 391751364..af294f52c 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -60,9 +60,9 @@ Additionally, they may also implement the following methods:
:param spider: the spider which was closed
:type spider: :class:`~scrapy.Spider` object
-.. method:: from_crawler(cls, crawler)
+.. classmethod:: from_crawler(cls, crawler)
- If present, this classmethod is called to create a pipeline instance
+ If present, this class method is called to create a pipeline instance
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
of the pipeline. Crawler object provides access to all Scrapy core
components like settings and signals; it is a way for pipeline to
@@ -99,11 +99,11 @@ contain a price::
raise DropItem(f"Missing price in {item}")
-Write items to a JSON file
---------------------------
+Write items to a JSON lines file
+--------------------------------
The following pipeline stores all scraped items (from all spiders) into a
-single ``items.jl`` file, containing one item per line serialized in JSON
+single ``items.jsonl`` file, containing one item per line serialized in JSON
format::
import json
@@ -113,7 +113,7 @@ format::
class JsonWriterPipeline:
def open_spider(self, spider):
- self.file = open('items.jl', 'w')
+ self.file = open('items.jsonl', 'w')
def close_spider(self, spider):
self.file.close()
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 7cd482d07..167014381 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -102,11 +102,6 @@ Additionally, ``dataclass`` items also allow to:
* define custom field metadata through :func:`dataclasses.field`, which can be used to
:ref:`customize serialization `.
-They work natively in Python 3.7 or later, or using the `dataclasses
-backport`_ in Python 3.6.
-
-.. _dataclasses backport: https://pypi.org/project/dataclasses/
-
Example::
from dataclasses import dataclass
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 7dff78390..0925e6bb5 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
can configure some extra functions like generating thumbnails and filtering
the images based on their size.
-The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for
+The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for
thumbnailing and normalizing images to JPEG/RGB format.
.. _Pillow: https://github.com/python-pillow/Pillow
@@ -656,6 +656,26 @@ See here the methods that you can override in your custom Images Pipeline:
.. versionadded:: 2.4
The *item* parameter.
+ .. method:: ImagesPipeline.thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)
+
+ This method is called for every item of :setting:`IMAGES_THUMBS` per downloaded item. It returns the
+ thumbnail download path of the image originating from the specified
+ :class:`response `.
+
+ In addition to ``response``, this method receives the original
+ :class:`request `,
+ ``thumb_id``,
+ :class:`info ` and
+ :class:`item `.
+
+ You can override this method to customize the thumbnail download path of each image.
+ You can use the ``item`` to determine the file path based on some item
+ property.
+
+ By default the :meth:`thumb_path` method returns
+ ``thumbs//.``.
+
+
.. method:: ImagesPipeline.get_media_requests(item, info)
Works the same way as :meth:`FilesPipeline.get_media_requests` method,
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index d0207fd18..7313c9246 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -180,8 +180,8 @@ Same example but running the spiders sequentially by chaining the deferreds:
# Your second spider definition
...
- configure_logging()
settings = get_project_settings()
+ configure_logging(settings)
runner = CrawlerRunner(settings)
@defer.inlineCallbacks
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 92a471faf..49cb69f67 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -339,6 +339,7 @@ errors if needed::
request = failure.request
self.logger.error('TimeoutError on %s', request.url)
+
.. _errback-cb_kwargs:
Accessing additional data in errback functions
@@ -364,6 +365,273 @@ achieve this by using ``Failure.request.cb_kwargs``::
main_url=failure.request.cb_kwargs['main_url'],
)
+
+.. _request-fingerprints:
+
+Request fingerprints
+--------------------
+
+There are some aspects of scraping, such as filtering out duplicate requests
+(see :setting:`DUPEFILTER_CLASS`) or caching responses (see
+:setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short,
+unique identifier from a :class:`~scrapy.http.Request` object: a request
+fingerprint.
+
+You often do not need to worry about request fingerprints, the default request
+fingerprinter works for most projects.
+
+However, there is no universal way to generate a unique identifier from a
+request, because different situations require comparing requests differently.
+For example, sometimes you may need to compare URLs case-insensitively, include
+URL fragments, exclude certain URL query parameters, include some or all
+headers, etc.
+
+To change how request fingerprints are built for your requests, use the
+:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
+
+.. setting:: REQUEST_FINGERPRINTER_CLASS
+
+REQUEST_FINGERPRINTER_CLASS
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: VERSION
+
+Default: :class:`scrapy.utils.request.RequestFingerprinter`
+
+A :ref:`request fingerprinter class ` or its
+import path.
+
+.. autoclass:: scrapy.utils.request.RequestFingerprinter
+
+
+.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
+
+REQUEST_FINGERPRINTER_IMPLEMENTATION
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: VERSION
+
+Default: ``'PREVIOUS_VERSION'``
+
+Determines which request fingerprinting algorithm is used by the default
+request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
+
+Possible values are:
+
+- ``'PREVIOUS_VERSION'`` (default)
+
+ This implementation uses the same request fingerprinting algorithm as
+ Scrapy PREVIOUS_VERSION and earlier versions.
+
+ Even though this is the default value for backward compatibility reasons,
+ it is a deprecated value.
+
+- ``'VERSION'``
+
+ This implementation was introduced in Scrapy VERSION to fix an issue of the
+ previous implementation.
+
+ New projects should use this value. The :command:`startproject` command
+ sets this value in the generated ``settings.py`` file.
+
+If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are
+using Scrapy components where changing the request fingerprinting algorithm
+would cause undesired results, you need to carefully decide when to change the
+value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
+setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request
+fingerprinting algorithm and does not log this warning (
+:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a
+class).
+
+Scenarios where changing the request fingerprinting algorithm may cause
+undesired results include, for example, using the HTTP cache middleware (see
+:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
+Changing the request fingerprinting algorithm would invalidade the current
+cache, requiring you to redownload all requests again.
+
+Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in
+your settings to switch already to the request fingerprinting implementation
+that will be the only request fingerprinting implementation available in a
+future version of Scrapy, and remove the deprecation warning triggered by using
+the default value (``'PREVIOUS_VERSION'``).
+
+
+.. _PREVIOUS_VERSION-request-fingerprinter:
+.. _custom-request-fingerprinter:
+
+Writing your own request fingerprinter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A request fingerprinter is a class that must implement the following method:
+
+.. method:: fingerprint(self, request)
+
+ Return a :class:`bytes` object that uniquely identifies *request*.
+
+ See also :ref:`request-fingerprint-restrictions`.
+
+ :param request: request to fingerprint
+ :type request: scrapy.http.Request
+
+Additionally, it may also implement the following methods:
+
+.. classmethod:: from_crawler(cls, crawler)
+
+ If present, this class method is called to create a request fingerprinter
+ instance from a :class:`~scrapy.crawler.Crawler` object. It must return a
+ new instance of the request fingerprinter.
+
+ *crawler* provides access to all Scrapy core components like settings and
+ signals; it is a way for the request fingerprinter to access them and hook
+ its functionality into Scrapy.
+
+ :param crawler: crawler that uses this request fingerprinter
+ :type crawler: :class:`~scrapy.crawler.Crawler` object
+
+.. classmethod:: from_settings(cls, settings)
+
+ If present, and ``from_crawler`` is not defined, this class method is called
+ to create a request fingerprinter instance from a
+ :class:`~scrapy.settings.Settings` object. It must return a new instance of
+ the request fingerprinter.
+
+The ``fingerprint`` method of the default request fingerprinter,
+:class:`scrapy.utils.request.RequestFingerprinter`, uses
+:func:`scrapy.utils.request.fingerprint` with its default parameters. For some
+common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well
+in your ``fingerprint`` method implementation:
+
+.. autofunction:: scrapy.utils.request.fingerprint
+
+For example, to take the value of a request header named ``X-ID`` into
+account::
+
+ # my_project/settings.py
+ REQUEST_FINGERPRINTER_CLASS = 'my_project.utils.RequestFingerprinter'
+
+ # my_project/utils.py
+ from scrapy.utils.request import fingerprint
+
+ class RequestFingerprinter:
+
+ def fingerprint(self, request):
+ return fingerprint(request, include_headers=['X-ID'])
+
+You can also write your own fingerprinting logic from scratch.
+
+However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure
+you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
+
+- Caching saves CPU by ensuring that fingerprints are calculated only once
+ per request, and not once per Scrapy component that needs the fingerprint
+ of a request.
+
+- Using :class:`~weakref.WeakKeyDictionary` saves memory by ensuring that
+ request objects do not stay in memory forever just because you have
+ references to them in your cache dictionary.
+
+For example, to take into account only the URL of a request, without any prior
+URL canonicalization or taking the request method or body into account::
+
+ from hashlib import sha1
+ from weakref import WeakKeyDictionary
+
+ from scrapy.utils.python import to_bytes
+
+ class RequestFingerprinter:
+
+ cache = WeakKeyDictionary()
+
+ def fingerprint(self, request):
+ if request not in self.cache:
+ fp = sha1()
+ fp.update(to_bytes(request.url))
+ self.cache[request] = fp.digest()
+ return self.cache[request]
+
+If you need to be able to override the request fingerprinting for arbitrary
+requests from your spider callbacks, you may implement a request fingerprinter
+that reads fingerprints from :attr:`request.meta `
+when available, and then falls back to
+:func:`~scrapy.utils.request.fingerprint`. For example::
+
+ from scrapy.utils.request import fingerprint
+
+ class RequestFingerprinter:
+
+ def fingerprint(self, request):
+ if 'fingerprint' in request.meta:
+ return request.meta['fingerprint']
+ return fingerprint(request)
+
+If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION
+without using the deprecated ``'PREVIOUS_VERSION'`` value of the
+:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
+request fingerprinter::
+
+ from hashlib import sha1
+ from weakref import WeakKeyDictionary
+
+ from scrapy.utils.python import to_bytes
+ from w3lib.url import canonicalize_url
+
+ class RequestFingerprinter:
+
+ cache = WeakKeyDictionary()
+
+ def fingerprint(self, request):
+ if request not in self.cache:
+ fp = sha1()
+ fp.update(to_bytes(request.method))
+ fp.update(to_bytes(canonicalize_url(request.url)))
+ fp.update(request.body or b'')
+ self.cache[request] = fp.digest()
+ return self.cache[request]
+
+
+.. _request-fingerprint-restrictions:
+
+Request fingerprint restrictions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Scrapy components that use request fingerprints may impose additional
+restrictions on the format of the fingerprints that your :ref:`request
+fingerprinter ` generates.
+
+The following built-in Scrapy components have such restrictions:
+
+- :class:`scrapy.extensions.httpcache.FilesystemCacheStorage` (default
+ value of :setting:`HTTPCACHE_STORAGE`)
+
+ Request fingerprints must be at least 1 byte long.
+
+ Path and filename length limits of the file system of
+ :setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`,
+ the following directory structure is created:
+
+ - :attr:`Spider.name `
+
+ - first byte of a request fingerprint as hexadecimal
+
+ - fingerprint as hexadecimal
+
+ - filenames up to 16 characters long
+
+ For example, if a request fingerprint is made of 20 bytes (default),
+ :setting:`HTTPCACHE_DIR` is ``'/home/user/project/.scrapy/httpcache'``,
+ and the name of your spider is ``'my_spider'`` your file system must
+ support a file path like::
+
+ /home/user/project/.scrapy/httpcache/my_spider/01/0123456789abcdef0123456789abcdef01234567/response_headers
+
+- :class:`scrapy.extensions.httpcache.DbmCacheStorage`
+
+ The underlying DBM implementation must support keys as long as twice
+ the number of bytes of a request fingerprint, plus 5. For example,
+ if a request fingerprint is made of 20 bytes (default),
+ 45-character-long keys must be supported.
+
+
.. _topics-request-meta:
Request.meta special keys
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 4e105642d..2046c6446 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -825,12 +825,8 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'``
The class used to detect and filter duplicate requests.
-The default (``RFPDupeFilter``) filters based on request fingerprint using
-the ``scrapy.utils.request.request_fingerprint`` function. In order to change
-the way duplicates are checked you could subclass ``RFPDupeFilter`` and
-override its ``request_fingerprint`` method. This method should accept
-scrapy :class:`~scrapy.Request` object and return its fingerprint
-(a string).
+The default (``RFPDupeFilter``) filters based on the
+:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
You can disable filtering of duplicate requests by setting
:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 2fbd0b51c..17bd16156 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -51,12 +51,12 @@ Deferred signal handlers
========================
Some signals support returning :class:`~twisted.internet.defer.Deferred`
-objects from their handlers, allowing you to run asynchronous code that
-does not block Scrapy. If a signal handler returns a
-:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that
-:class:`~twisted.internet.defer.Deferred` to fire.
+or :term:`awaitable objects ` from their handlers, allowing
+you to run asynchronous code that does not block Scrapy. If a signal
+handler returns one of these objects, Scrapy waits for that asynchronous
+operation to finish.
-Let's take an example::
+Let's take an example using :ref:`coroutines `::
class SignalSpider(scrapy.Spider):
name = 'signals'
@@ -68,17 +68,15 @@ Let's take an example::
crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
return spider
- def item_scraped(self, item):
+ async def item_scraped(self, item):
# Send the scraped item to the server
- d = treq.post(
+ response = await treq.post(
'http://example.com/post',
json.dumps(item).encode('ascii'),
headers={b'Content-Type': [b'application/json']}
)
- # The next item will be scraped only after
- # deferred (d) is fired
- return d
+ return response
def parse(self, response):
for quote in response.css('div.quote'):
@@ -89,7 +87,7 @@ Let's take an example::
}
See the :ref:`topics-signals-ref` below to know which signals support
-:class:`~twisted.internet.defer.Deferred`.
+:class:`~twisted.internet.defer.Deferred` and :term:`awaitable objects `.
.. _topics-signals-ref:
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 396f98219..86e584396 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Check minimum required Python version
-if sys.version_info < (3, 6):
- print(f"Scrapy {__version__} requires Python 3.6+")
+if sys.version_info < (3, 7):
+ print(f"Scrapy {__version__} requires Python 3.7+")
sys.exit(1)
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index a3f6b96f4..99fc8f955 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -146,7 +146,8 @@ class Command(BaseRunSpiderCommand):
def _start_requests(spider):
yield self.prepare_request(spider, Request(url), opts)
- self.spidercls.start_requests = _start_requests
+ if self.spidercls:
+ self.spidercls.start_requests = _start_requests
def start_parsing(self, url, opts):
self.crawler_process.crawl(self.spidercls, **opts.spargs)
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 3ef129587..a495874bd 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -102,11 +102,11 @@ class FTPDownloadHandler:
def _build_response(self, result, request, protocol):
self.result = result
- respcls = responsetypes.from_args(url=request.url)
protocol.close()
- body = protocol.filename or protocol.body.read()
headers = {"local filename": protocol.filename or '', "size": protocol.size}
- return respcls(url=request.url, status=200, body=to_bytes(body), headers=headers)
+ body = to_bytes(protocol.filename or protocol.body.read())
+ respcls = responsetypes.from_args(url=request.url, body=body)
+ return respcls(url=request.url, status=200, body=body, headers=headers)
def _failed(self, result, request):
message = result.getErrorMessage()
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 06cb96489..7d048c1e4 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -112,7 +112,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
request.meta['download_latency'] = self.headers_time - self.start_time
status = int(self.status)
headers = Headers(self.response_headers)
- respcls = responsetypes.from_args(headers=headers, url=self._url)
+ respcls = responsetypes.from_args(headers=headers, url=self._url, body=body)
return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))
def _set_connection_attributes(self, request):
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index a638254f1..fdca7b335 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -51,6 +51,7 @@ class Crawler:
self.spidercls.update_settings(self.settings)
self.signals = SignalManager(self)
+
self.stats = load_object(self.settings['STATS_CLASS'])(self)
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
@@ -71,6 +72,12 @@ class Crawler:
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
+ self.request_fingerprinter = create_instance(
+ load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
+ settings=self.settings,
+ crawler=self,
+ )
+
reactor_class = self.settings.get("TWISTED_REACTOR")
if init_reactor:
# this needs to be done after the spider settings are merged,
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 3afa06077..c592acb57 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -104,8 +104,8 @@ class CookiesMiddleware:
for key in ("name", "value", "path", "domain"):
if cookie.get(key) is None:
if key in ("name", "value"):
- msg = "Invalid cookie found in request {}: {} ('{}' is missing)"
- logger.warning(msg.format(request, cookie, key))
+ msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
+ logger.warning(msg)
return
continue
if isinstance(cookie[key], (bool, float, int, str)):
diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index 0fcf8fb8c..e01e9cc76 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -9,10 +9,19 @@ import tarfile
import zipfile
from io import BytesIO
from tempfile import mktemp
+from warnings import warn
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.responsetypes import responsetypes
+warn(
+ 'scrapy.downloadermiddlewares.decompression is deprecated',
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+)
+
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 292c68099..d1b0559ef 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -1,14 +1,16 @@
import logging
import os
from typing import Optional, Set, Type, TypeVar
+from warnings import warn
from twisted.internet.defer import Deferred
from scrapy.http.request import Request
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.job import job_dir
-from scrapy.utils.request import referer_str, request_fingerprint
+from scrapy.utils.request import referer_str, RequestFingerprinter
BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")
@@ -39,8 +41,15 @@ RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter")
class RFPDupeFilter(BaseDupeFilter):
"""Request Fingerprint duplicates filter"""
- def __init__(self, path: Optional[str] = None, debug: bool = False) -> None:
+ def __init__(
+ self,
+ path: Optional[str] = None,
+ debug: bool = False,
+ *,
+ fingerprinter=None,
+ ) -> None:
self.file = None
+ self.fingerprinter = fingerprinter or RequestFingerprinter()
self.fingerprints: Set[str] = set()
self.logdupes = True
self.debug = debug
@@ -51,9 +60,39 @@ class RFPDupeFilter(BaseDupeFilter):
self.fingerprints.update(x.rstrip() for x in self.file)
@classmethod
- def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings) -> RFPDupeFilterTV:
+ def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None) -> RFPDupeFilterTV:
debug = settings.getbool('DUPEFILTER_DEBUG')
- return cls(job_dir(settings), debug)
+ try:
+ return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
+ except TypeError:
+ warn(
+ "RFPDupeFilter subclasses must either modify their '__init__' "
+ "method to support a 'fingerprinter' parameter or reimplement "
+ "the 'from_settings' class method.",
+ ScrapyDeprecationWarning,
+ )
+ result = cls(job_dir(settings), debug)
+ result.fingerprinter = fingerprinter
+ return result
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ try:
+ return cls.from_settings(
+ crawler.settings,
+ fingerprinter=crawler.request_fingerprinter,
+ )
+ except TypeError:
+ warn(
+ "RFPDupeFilter subclasses must either modify their overridden "
+ "'__init__' method and 'from_settings' class method to "
+ "support a 'fingerprinter' parameter, or reimplement the "
+ "'from_crawler' class method.",
+ ScrapyDeprecationWarning,
+ )
+ result = cls.from_settings(crawler.settings)
+ result.fingerprinter = crawler.request_fingerprinter
+ return result
def request_seen(self, request: Request) -> bool:
fp = self.request_fingerprint(request)
@@ -65,7 +104,7 @@ class RFPDupeFilter(BaseDupeFilter):
return False
def request_fingerprint(self, request: Request) -> str:
- return request_fingerprint(request)
+ return self.fingerprinter.fingerprint(request).hex()
def close(self, reason: str) -> None:
if self.file:
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 1c26e81db..76cbe4d4b 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -8,6 +8,7 @@ import marshal
import pickle
import pprint
import warnings
+from collections.abc import Mapping
from xml.sax.saxutils import XMLGenerator
from itemadapter import is_item, ItemAdapter
@@ -68,6 +69,14 @@ class BaseItemExporter:
field_iter = item.field_names()
else:
field_iter = item.keys()
+ elif isinstance(self.fields_to_export, Mapping):
+ if include_empty:
+ field_iter = self.fields_to_export.items()
+ else:
+ field_iter = (
+ (x, y) for x, y in self.fields_to_export.items()
+ if x in item
+ )
else:
if include_empty:
field_iter = self.fields_to_export
@@ -75,13 +84,17 @@ class BaseItemExporter:
field_iter = (x for x in self.fields_to_export if x in item)
for field_name in field_iter:
- if field_name in item:
- field_meta = item.get_field_meta(field_name)
- value = self.serialize_field(field_meta, field_name, item[field_name])
+ if isinstance(field_name, str):
+ item_field, output_field = field_name, field_name
+ else:
+ item_field, output_field = field_name
+ if item_field in item:
+ field_meta = item.get_field_meta(item_field)
+ value = self.serialize_field(field_meta, output_field, item[item_field])
else:
value = default_value
- yield field_name, value
+ yield output_field, value
class JsonLinesItemExporter(BaseItemExporter):
@@ -246,7 +259,11 @@ class CsvItemExporter(BaseItemExporter):
if not self.fields_to_export:
# use declared field names, or keys if the item is a dict
self.fields_to_export = ItemAdapter(item).field_names()
- row = list(self._build_row(self.fields_to_export))
+ if isinstance(self.fields_to_export, Mapping):
+ fields = self.fields_to_export.values()
+ else:
+ fields = self.fields_to_export
+ row = list(self._build_row(fields))
self.csv_writer.writerow(row)
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index d0ae29b90..843e14812 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -14,7 +14,6 @@ from scrapy.responsetypes import responsetypes
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.project import data_path
from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.request import request_fingerprint
logger = logging.getLogger(__name__)
@@ -228,6 +227,8 @@ class DbmCacheStorage:
logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
+ self._fingerprinter = spider.crawler.request_fingerprinter
+
def close_spider(self, spider):
self.db.close()
@@ -239,12 +240,12 @@ class DbmCacheStorage:
status = data['status']
headers = Headers(data['headers'])
body = data['body']
- respcls = responsetypes.from_args(headers=headers, url=url)
+ respcls = responsetypes.from_args(headers=headers, url=url, body=body)
response = respcls(url=url, headers=headers, status=status, body=body)
return response
def store_response(self, spider, request, response):
- key = self._request_key(request)
+ key = self._fingerprinter.fingerprint(request).hex()
data = {
'status': response.status,
'url': response.url,
@@ -255,7 +256,7 @@ class DbmCacheStorage:
self.db[f'{key}_time'] = str(time())
def _read_data(self, spider, request):
- key = self._request_key(request)
+ key = self._fingerprinter.fingerprint(request).hex()
db = self.db
tkey = f'{key}_time'
if tkey not in db:
@@ -267,9 +268,6 @@ class DbmCacheStorage:
return pickle.loads(db[f'{key}_data'])
- def _request_key(self, request):
- return request_fingerprint(request)
-
class FilesystemCacheStorage:
@@ -283,6 +281,8 @@ class FilesystemCacheStorage:
logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
extra={'spider': spider})
+ self._fingerprinter = spider.crawler.request_fingerprinter
+
def close_spider(self, spider):
pass
@@ -299,7 +299,7 @@ class FilesystemCacheStorage:
url = metadata.get('response_url')
status = metadata['status']
headers = Headers(headers_raw_to_dict(rawheaders))
- respcls = responsetypes.from_args(headers=headers, url=url)
+ respcls = responsetypes.from_args(headers=headers, url=url, body=body)
response = respcls(url=url, headers=headers, status=status, body=body)
return response
@@ -329,7 +329,7 @@ class FilesystemCacheStorage:
f.write(request.body)
def _get_request_path(self, spider, request):
- key = request_fingerprint(request)
+ key = self._fingerprinter.fingerprint(request).hex()
return os.path.join(self.cachedir, spider.name, key[0:2], key)
def _read_meta(self, spider, request):
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 9de119a10..f5081a7d7 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -33,8 +33,8 @@ class MemoryUsage:
self.crawler = crawler
self.warned = False
self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')
- self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
- self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
+ self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024
+ self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024
self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
self.mail = MailSender.from_settings(crawler.settings)
crawler.signals.connect(self.engine_started, signal=signals.engine_started)
@@ -77,7 +77,7 @@ class MemoryUsage:
def _check_limit(self):
if self.get_virtual_size() > self.limit:
self.crawler.stats.set_value('memusage/limit_reached', 1)
- mem = self.limit/1024/1024
+ mem = self.limit / 1024 / 1024
logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
{'memusage': mem}, extra={'crawler': self.crawler})
if self.notify_mails:
@@ -94,11 +94,11 @@ class MemoryUsage:
self.crawler.stop()
def _check_warning(self):
- if self.warned: # warn only once
+ if self.warned: # warn only once
return
if self.get_virtual_size() > self.warning:
self.crawler.stats.set_value('memusage/warning_reached', 1)
- mem = self.warning/1024/1024
+ mem = self.warning / 1024 / 1024
logger.warning("Memory usage reached %(memusage)dM",
{'memusage': mem}, extra={'crawler': self.crawler})
if self.notify_mails:
diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py
index bcdbaff24..739e6b958 100644
--- a/scrapy/extensions/statsmailer.py
+++ b/scrapy/extensions/statsmailer.py
@@ -8,6 +8,7 @@ from scrapy import signals
from scrapy.mail import MailSender
from scrapy.exceptions import NotConfigured
+
class StatsMailer:
def __init__(self, stats, recipients, mail):
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 1a2b99b0a..9c03fe54f 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,3 +1,5 @@
+from collections.abc import Mapping
+
from w3lib.http import headers_dict_to_raw
from scrapy.utils.datatypes import CaselessDict
from scrapy.utils.python import to_unicode
@@ -10,6 +12,13 @@ class Headers(CaselessDict):
self.encoding = encoding
super().__init__(seq)
+ def update(self, seq):
+ seq = seq.items() if isinstance(seq, Mapping) else seq
+ iseq = {}
+ for k, v in seq:
+ iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
+ super().update(iseq)
+
def normkey(self, key):
"""Normalize key to bytes"""
return self._tobytes(key.title())
@@ -86,4 +95,5 @@ class Headers(CaselessDict):
def __copy__(self):
return self.__class__(self)
+
copy = __copy__
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 5c52c6c28..906e7eb24 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -222,8 +222,8 @@ class GCSFilesStore:
return {'checksum': checksum, 'last_modified': last_modified}
else:
return {}
-
- return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess)
+ blob_path = self._get_blob_path(path)
+ return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess)
def _get_content_type(self, headers):
if headers and 'Content-Type' in headers:
@@ -231,8 +231,12 @@ class GCSFilesStore:
else:
return 'application/octet-stream'
+ def _get_blob_path(self, path):
+ return self.prefix + path
+
def persist_file(self, path, buf, info, meta=None, headers=None):
- blob = self.bucket.blob(self.prefix + path)
+ blob_path = self._get_blob_path(path)
+ blob = self.bucket.blob(blob_path)
blob.cache_control = self.CACHE_CONTROL
blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
return threads.deferToThread(
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 9c99dc69e..6b97190ee 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -141,7 +141,7 @@ class ImagesPipeline(FilesPipeline):
yield path, image, buf
for thumb_id, size in self.thumbs.items():
- thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
+ thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item)
thumb_image, thumb_buf = self.convert_image(image, size)
yield thumb_path, thumb_image, thumb_buf
@@ -179,6 +179,6 @@ class ImagesPipeline(FilesPipeline):
image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
return f'full/{image_guid}.jpg'
- def thumb_path(self, request, thumb_id, response=None, info=None):
+ def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
return f'thumbs/{thumb_id}/{thumb_guid}.jpg'
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index d1bccf323..5308a9793 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -11,7 +11,6 @@ from scrapy.settings import Settings
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.defer import mustbe_deferred, defer_result
from scrapy.utils.deprecate import ScrapyDeprecationWarning
-from scrapy.utils.request import request_fingerprint
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.log import failure_to_exc_info
@@ -77,6 +76,7 @@ class MediaPipeline:
except AttributeError:
pipe = cls()
pipe.crawler = crawler
+ pipe._fingerprinter = crawler.request_fingerprinter
return pipe
def open_spider(self, spider):
@@ -90,7 +90,7 @@ class MediaPipeline:
return dfd.addCallback(self.item_completed, item, info)
def _process_request(self, request, info, item):
- fp = request_fingerprint(request)
+ fp = self._fingerprinter.fingerprint(request)
cb = request.callback or (lambda _: _)
eb = request.errback
request.callback = None
@@ -121,7 +121,7 @@ class MediaPipeline:
def _make_compatible(self):
"""Make overridable methods of MediaPipeline and subclasses backwards compatible"""
methods = [
- "file_path", "media_to_download", "media_downloaded",
+ "file_path", "thumb_path", "media_to_download", "media_downloaded",
"file_downloaded", "image_downloaded", "get_images"
]
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 6ed9f8b8f..3efd4d2fd 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -95,12 +95,14 @@ class ResponseTypes:
chunk = to_bytes(chunk)
if not binary_is_text(chunk):
return self.from_mimetype('application/octet-stream')
- elif b"" in chunk.lower():
+ lowercase_chunk = chunk.lower()
+ if b"" in lowercase_chunk:
return self.from_mimetype('text/html')
- elif b"' in lowercase_chunk:
+ return self.from_mimetype('text/html')
+ return self.from_mimetype('text')
def from_args(self, headers=None, url=None, filename=None, body=None):
"""Guess the most appropriate Response class based on
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 6b1ad0828..6cacc63e1 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -197,6 +197,38 @@ class BaseSettings(MutableMapping):
value = json.loads(value)
return dict(value)
+ def getdictorlist(self, name, default=None):
+ """Get a setting value as either a :class:`dict` or a :class:`list`.
+
+ If the setting is already a dict or a list, a copy of it will be
+ returned.
+
+ If it is a string it will be evaluated as JSON, or as a comma-separated
+ list of strings as a fallback.
+
+ For example, settings populated from the command line will return:
+
+ - ``{'key1': 'value1', 'key2': 'value2'}`` if set to
+ ``'{"key1": "value1", "key2": "value2"}'``
+
+ - ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
+
+ :param name: the setting name
+ :type name: string
+
+ :param default: the value to return if no setting is found
+ :type default: any
+ """
+ value = self.get(name, default)
+ if value is None:
+ return {}
+ if isinstance(value, str):
+ try:
+ return json.loads(value)
+ except ValueError:
+ return value.split(',')
+ return copy.deepcopy(value)
+
def getwithbase(self, name):
"""Get a composition of a dictionary-like setting and its `_BASE`
counterpart.
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 8389a70cb..ff86af125 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -154,6 +154,7 @@ FEED_EXPORTERS = {}
FEED_EXPORTERS_BASE = {
'json': 'scrapy.exporters.JsonItemExporter',
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
+ 'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
'jl': 'scrapy.exporters.JsonLinesItemExporter',
'csv': 'scrapy.exporters.CsvItemExporter',
'xml': 'scrapy.exporters.XmlItemExporter',
@@ -246,6 +247,9 @@ REDIRECT_PRIORITY_ADJUST = +2
REFERER_ENABLED = True
REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
+REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter'
+REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION'
+
RETRY_ENABLED = True
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index a414b5fde..5e541e2c0 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -86,3 +86,6 @@ ROBOTSTXT_OBEY = True
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+
+# Set settings whose default value is deprecated to a future-proof value
+REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION'
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 24a6187b9..00cc53725 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -118,7 +118,7 @@ def feed_complete_default_values_from_settings(feed, settings):
out = feed.copy()
out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))
out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
- out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
+ out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None)
out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
out.setdefault("item_export_kwargs", {})
diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py
deleted file mode 100644
index 653e2bbbb..000000000
--- a/scrapy/utils/py36.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import warnings
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.asyncgen import collect_asyncgen # noqa: F401
-
-
-warnings.warn(
- "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.asyncgen` instead.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
-)
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 70ef3ba2b..cf33317ce 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -4,7 +4,9 @@ scrapy.http.Request objects
"""
import hashlib
-from typing import Dict, Iterable, Optional, Tuple, Union
+import json
+import warnings
+from typing import Dict, Iterable, List, Optional, Tuple, Union
from urllib.parse import urlunparse
from weakref import WeakKeyDictionary
@@ -12,13 +14,22 @@ from w3lib.http import basic_auth_header
from w3lib.url import canonicalize_url
from scrapy import Request, Spider
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
from scrapy.utils.python import to_bytes, to_unicode
-_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
-_fingerprint_cache = WeakKeyDictionary()
+_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
+_deprecated_fingerprint_cache = WeakKeyDictionary()
+
+
+def _serialize_headers(headers, request):
+ for header in headers:
+ if header in request.headers:
+ yield header
+ for value in request.headers.getlist(header):
+ yield value
def request_fingerprint(
@@ -26,6 +37,123 @@ def request_fingerprint(
include_headers: Optional[Iterable[Union[bytes, str]]] = None,
keep_fragments: bool = False,
) -> str:
+ """
+ Return the request fingerprint as an hexadecimal string.
+
+ The request fingerprint is a hash that uniquely identifies the resource the
+ request points to. For example, take the following two urls:
+
+ http://www.example.com/query?id=111&cat=222
+ http://www.example.com/query?cat=222&id=111
+
+ Even though those are two different URLs both point to the same resource
+ and are equivalent (i.e. they should return the same response).
+
+ Another example are cookies used to store session ids. Suppose the
+ following page is only accessible to authenticated users:
+
+ http://www.example.com/members/offers.html
+
+ Lots of sites use a cookie to store the session id, which adds a random
+ component to the HTTP Request and thus should be ignored when calculating
+ the fingerprint.
+
+ For this reason, request headers are ignored by default when calculating
+ the fingerprint. If you want to include specific headers use the
+ include_headers argument, which is a list of Request headers to include.
+
+ Also, servers usually ignore fragments in urls when handling requests,
+ so they are also ignored by default when calculating the fingerprint.
+ If you want to include them, set the keep_fragments argument to True
+ (for instance when handling requests with a headless browser).
+ """
+ if include_headers or keep_fragments:
+ message = (
+ 'Call to deprecated function '
+ 'scrapy.utils.request.request_fingerprint().\n'
+ '\n'
+ 'If you are using this function in a Scrapy component because you '
+ 'need a non-default fingerprinting algorithm, and you are OK '
+ 'with that non-default fingerprinting algorithm being used by '
+ 'all Scrapy components and not just the one calling this '
+ 'function, use crawler.request_fingerprinter.fingerprint() '
+ 'instead in your Scrapy component (you can get the crawler '
+ 'object from the \'from_crawler\' class method), and use the '
+ '\'REQUEST_FINGERPRINTER_CLASS\' setting to configure your '
+ 'non-default fingerprinting algorithm.\n'
+ '\n'
+ 'Otherwise, consider using the '
+ 'scrapy.utils.request.fingerprint() function instead.\n'
+ '\n'
+ 'If you switch to \'fingerprint()\', or assign the '
+ '\'REQUEST_FINGERPRINTER_CLASS\' setting a class that uses '
+ '\'fingerprint()\', the generated fingerprints will not only be '
+ 'bytes instead of a string, but they will also be different from '
+ 'those generated by \'request_fingerprint()\'. Before you switch, '
+ 'make sure that you understand the consequences of this (e.g. '
+ 'cache invalidation) and are OK with them; otherwise, consider '
+ 'implementing your own function which returns the same '
+ 'fingerprints as the deprecated \'request_fingerprint()\' function.'
+ )
+ else:
+ message = (
+ 'Call to deprecated function '
+ 'scrapy.utils.request.request_fingerprint().\n'
+ '\n'
+ 'If you are using this function in a Scrapy component, and you '
+ 'are OK with users of your component changing the fingerprinting '
+ 'algorithm through settings, use '
+ 'crawler.request_fingerprinter.fingerprint() instead in your '
+ 'Scrapy component (you can get the crawler object from the '
+ '\'from_crawler\' class method).\n'
+ '\n'
+ 'Otherwise, consider using the '
+ 'scrapy.utils.request.fingerprint() function instead.\n'
+ '\n'
+ 'Either way, the resulting fingerprints will be returned as '
+ 'bytes, not as a string, and they will also be different from '
+ 'those generated by \'request_fingerprint()\'. Before you switch, '
+ 'make sure that you understand the consequences of this (e.g. '
+ 'cache invalidation) and are OK with them; otherwise, consider '
+ 'implementing your own function which returns the same '
+ 'fingerprints as the deprecated \'request_fingerprint()\' function.'
+ )
+ warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
+ processed_include_headers: Optional[Tuple[bytes, ...]] = None
+ if include_headers:
+ processed_include_headers = tuple(
+ to_bytes(h.lower()) for h in sorted(include_headers)
+ )
+ cache = _deprecated_fingerprint_cache.setdefault(request, {})
+ cache_key = (processed_include_headers, keep_fragments)
+ if cache_key not in cache:
+ fp = hashlib.sha1()
+ fp.update(to_bytes(request.method))
+ fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
+ fp.update(request.body or b'')
+ if processed_include_headers:
+ for part in _serialize_headers(processed_include_headers, request):
+ fp.update(part)
+ cache[cache_key] = fp.hexdigest()
+ return cache[cache_key]
+
+
+def _request_fingerprint_as_bytes(*args, **kwargs):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ return bytes.fromhex(request_fingerprint(*args, **kwargs))
+
+
+_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
+_fingerprint_cache = WeakKeyDictionary()
+
+
+def fingerprint(
+ request: Request,
+ *,
+ include_headers: Optional[Iterable[Union[bytes, str]]] = None,
+ keep_fragments: bool = False,
+) -> bytes:
"""
Return the request fingerprint.
@@ -43,7 +171,7 @@ def request_fingerprint(
http://www.example.com/members/offers.html
- Lot of sites use a cookie to store the session id, which adds a random
+ Lots of sites use a cookie to store the session id, which adds a random
component to the HTTP Request and thus should be ignored when calculating
the fingerprint.
@@ -55,29 +183,96 @@ def request_fingerprint(
so they are also ignored by default when calculating the fingerprint.
If you want to include them, set the keep_fragments argument to True
(for instance when handling requests with a headless browser).
-
"""
- headers: Optional[Tuple[bytes, ...]] = None
+ processed_include_headers: Optional[Tuple[bytes, ...]] = None
if include_headers:
- headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
+ processed_include_headers = tuple(
+ to_bytes(h.lower()) for h in sorted(include_headers)
+ )
cache = _fingerprint_cache.setdefault(request, {})
- cache_key = (headers, keep_fragments)
+ cache_key = (processed_include_headers, keep_fragments)
if cache_key not in cache:
- fp = hashlib.sha1()
- fp.update(to_bytes(request.method))
- fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
- fp.update(request.body or b'')
- if headers:
- for hdr in headers:
- if hdr in request.headers:
- fp.update(hdr)
- for v in request.headers.getlist(hdr):
- fp.update(v)
- cache[cache_key] = fp.hexdigest()
+ # To decode bytes reliably (JSON does not support bytes), regardless of
+ # character encoding, we use bytes.hex()
+ headers: Dict[str, List[str]] = {}
+ if processed_include_headers:
+ for header in processed_include_headers:
+ if header in request.headers:
+ headers[header.hex()] = [
+ header_value.hex()
+ for header_value in request.headers.getlist(header)
+ ]
+ fingerprint_data = {
+ 'method': to_unicode(request.method),
+ 'url': canonicalize_url(request.url, keep_fragments=keep_fragments),
+ 'body': (request.body or b'').hex(),
+ 'headers': headers,
+ }
+ fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)
+ cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest()
return cache[cache_key]
-def request_authenticate(request: Request, username: str, password: str) -> None:
+class RequestFingerprinter:
+ """Default fingerprinter.
+
+ It takes into account a canonical version
+ (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url
+ ` and the values of :attr:`request.method
+ ` and :attr:`request.body
+ `. It then generates an `SHA1
+ `_ hash.
+
+ .. seealso:: :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION`.
+ """
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
+
+ def __init__(self, crawler=None):
+ if crawler:
+ implementation = crawler.settings.get(
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION'
+ )
+ else:
+ implementation = 'PREVIOUS_VERSION'
+ if implementation == 'PREVIOUS_VERSION':
+ message = (
+ '\'PREVIOUS_VERSION\' is a deprecated value for the '
+ '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n'
+ '\n'
+ 'It is also the default value. In other words, it is normal '
+ 'to get this warning if you have not defined a value for the '
+ '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting. This is so '
+ 'for backward compatibility reasons, but it will change in a '
+ 'future version of Scrapy.\n'
+ '\n'
+ 'See the documentation of the '
+ '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting for '
+ 'information on how to handle this deprecation.'
+ )
+ warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
+ self._fingerprint = _request_fingerprint_as_bytes
+ elif implementation == 'VERSION':
+ self._fingerprint = fingerprint
+ else:
+ raise ValueError(
+ f'Got an invalid value on setting '
+ f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': '
+ f'{implementation!r}. Valid values are \'PREVIOUS_VERSION\' (deprecated) '
+ f'and \'VERSION\'.'
+ )
+
+ def fingerprint(self, request):
+ return self._fingerprint(request)
+
+
+def request_authenticate(
+ request: Request,
+ username: str,
+ password: str,
+) -> None:
"""Authenticate the given request (in place) using the HTTP basic access
authentication mechanism (RFC 2617) and the given username and password
"""
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 24c38283a..b90ea5009 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -54,7 +54,7 @@ def get_ftp_content_and_delete(
return "".join(ftp_data)
-def get_crawler(spidercls=None, settings_dict=None):
+def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):
"""Return an unconfigured Crawler object. If settings_dict is given, it
will be used to populate the crawler settings with a project level
priority.
@@ -62,7 +62,12 @@ def get_crawler(spidercls=None, settings_dict=None):
from scrapy.crawler import CrawlerRunner
from scrapy.spiders import Spider
- runner = CrawlerRunner(settings_dict)
+ # Set by default settings that prevent deprecation warnings.
+ settings = {}
+ if prevent_warnings:
+ settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = 'VERSION'
+ settings.update(settings_dict or {})
+ runner = CrawlerRunner(settings)
return runner.create_crawler(spidercls or Spider)
diff --git a/setup.py b/setup.py
index d86c0f285..ed197273f 100644
--- a/setup.py
+++ b/setup.py
@@ -19,35 +19,29 @@ def has_environment_marker_platform_impl_support():
install_requires = [
- 'Twisted>=17.9.0',
- 'cryptography>=2.0',
+ 'Twisted>=18.9.0',
+ 'cryptography>=2.8',
'cssselect>=0.9.1',
'itemloaders>=1.0.1',
'parsel>=1.5.0',
- 'pyOpenSSL>=16.2.0',
+ 'pyOpenSSL>=19.1.0',
'queuelib>=1.4.2',
'service_identity>=16.0.0',
'w3lib>=1.17.0',
- 'zope.interface>=4.1.3',
+ 'zope.interface>=5.1.0',
'protego>=0.1.15',
'itemadapter>=0.1.0',
'setuptools',
'tldextract',
+ 'lxml>=4.3.0',
]
extras_require = {}
cpython_dependencies = [
- 'lxml>=3.5.0',
'PyDispatcher>=2.0.5',
]
if has_environment_marker_platform_impl_support():
extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies
extras_require[':platform_python_implementation == "PyPy"'] = [
- # Earlier lxml versions are affected by
- # https://foss.heptapod.net/pypy/pypy/-/issues/2498,
- # which was fixed in Cython 0.26, released on 2017-06-19, and used to
- # generate the C headers of lxml release tarballs published since then, the
- # first of which was:
- 'lxml>=4.0.0',
'PyPyDispatcher>=2.1.0',
]
else:
@@ -84,7 +78,6 @@ setup(
'Operating System :: OS Independent',
'Programming Language :: Python',
'Programming Language :: Python :: 3',
- 'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
@@ -95,7 +88,7 @@ setup(
'Topic :: Software Development :: Libraries :: Application Frameworks',
'Topic :: Software Development :: Libraries :: Python Modules',
],
- python_requires='>=3.6',
+ python_requires='>=3.7',
install_requires=install_requires,
extras_require=extras_require,
)
diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
index 1f5a44010..72bb986bc 100644
--- a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
@@ -8,6 +8,7 @@ class AsyncioReactorSpider1(scrapy.Spider):
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
}
+
class AsyncioReactorSpider2(scrapy.Spider):
name = 'asyncio_reactor2'
custom_settings = {
diff --git a/tests/requirements.txt b/tests/requirements.txt
index d2a8aae1b..d9373dfa8 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,14 +1,12 @@
# Tests requirements
attrs
-dataclasses; python_version == '3.6'
pyftpdlib
pytest
pytest-cov==3.0.0
pytest-xdist
sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
testfixtures
-uvloop < 0.15.0; platform_system != "Windows" and python_version == '3.6'
-uvloop; platform_system != "Windows" and python_version > '3.6'
+uvloop; platform_system != "Windows"
# optional for shell wrapper tests
bpython
diff --git a/tests/sample_data/link_extractor/linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html
index 2307ea865..e3a2a4145 100644
--- a/tests/sample_data/link_extractor/linkextractor.html
+++ b/tests/sample_data/link_extractor/linkextractor.html
@@ -1,20 +1,22 @@
+
+
-
-
-Sample page with links for testing LinkExtractor
-
-
-
-
-
+
+
+ Sample page with links for testing LinkExtractor
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/sample_data/link_extractor/linkextractor_latin1.html b/tests/sample_data/link_extractor/linkextractor_latin1.html
index e7eee18de..1e05bf0f0 100644
--- a/tests/sample_data/link_extractor/linkextractor_latin1.html
+++ b/tests/sample_data/link_extractor/linkextractor_latin1.html
@@ -1,3 +1,5 @@
+
+
@@ -7,11 +9,11 @@
diff --git a/tests/sample_data/link_extractor/linkextractor_no_href.html b/tests/sample_data/link_extractor/linkextractor_no_href.html
index 0b01cede8..2d67ec6ff 100644
--- a/tests/sample_data/link_extractor/linkextractor_no_href.html
+++ b/tests/sample_data/link_extractor/linkextractor_no_href.html
@@ -1,3 +1,5 @@
+
+
@@ -21,5 +23,4 @@
-
\ No newline at end of file
diff --git a/tests/sample_data/link_extractor/linkextractor_noenc.html b/tests/sample_data/link_extractor/linkextractor_noenc.html
index f9166adbe..6fa137cd9 100644
--- a/tests/sample_data/link_extractor/linkextractor_noenc.html
+++ b/tests/sample_data/link_extractor/linkextractor_noenc.html
@@ -1,14 +1,17 @@
+
+
-
-
-Sample page without encoding for testing LinkExtractor
-
+
+
+ Sample page without encoding for testing LinkExtractor
+
+
-
+
diff --git a/tests/sample_data/test_site/index.html b/tests/sample_data/test_site/index.html
index d268c846a..afe17d8e2 100644
--- a/tests/sample_data/test_site/index.html
+++ b/tests/sample_data/test_site/index.html
@@ -1,18 +1,15 @@
+
+
-
-
-Scrapy test site
-
-
-
-
-Scrapy test site
-
-
-
-
-
+
+ Scrapy test site
+
+
+ Scrapy test site
+
+
+
\ No newline at end of file
diff --git a/tests/sample_data/test_site/item1.html b/tests/sample_data/test_site/item1.html
index ceeb6dc87..ee39f16f3 100644
--- a/tests/sample_data/test_site/item1.html
+++ b/tests/sample_data/test_site/item1.html
@@ -1,17 +1,14 @@
+
+
-
-
-Item 1 - Scrapy test site
-
-
-
-
-Item 1 name
-
-
-- Price: $100
-- Stock: 12
-
-
-
+
+ Item 1 - Scrapy test site
+
+
+ Item 1 name
+
+ - Price: $100
+ - Stock: 12
+
+
diff --git a/tests/sample_data/test_site/item2.html b/tests/sample_data/test_site/item2.html
index a64c92810..f40f70750 100644
--- a/tests/sample_data/test_site/item2.html
+++ b/tests/sample_data/test_site/item2.html
@@ -1,17 +1,14 @@
+
+
-
-
-Item 2 - Scrapy test site
-
-
-
-
-Item 2 name
-
-
-
-
-
+
+ Item 2 - Scrapy test site
+
+
+ Item 2 name
+
+ - Price: $200
+ - Stock: 5
+
+
+
\ No newline at end of file
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index f21ee971d..0d992be56 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -1,6 +1,7 @@
import os
import argparse
from os.path import join, abspath, isfile, exists
+
from twisted.internet import defer
from scrapy.commands import parse
from scrapy.settings import Settings
@@ -222,6 +223,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""")
self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))
+ @defer.inlineCallbacks
+ def test_crawlspider_not_exists_with_not_matched_url(self):
+ status, out, stderr = yield self.execute([self.url('/invalid_url')])
+ self.assertEqual(status, 0)
+
@defer.inlineCallbacks
def test_output_flag(self):
"""Checks if a file was created successfully having
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 8f6227109..f7aa769e4 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -104,7 +104,8 @@ class CrawlerLoggingTestCase(unittest.TestCase):
custom_settings = {
'LOG_LEVEL': 'INFO',
'LOG_FILE': log_file,
- # disable telnet if not available to avoid an extra warning
+ # settings to avoid extra warnings
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
}
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 2bb53950d..72f52121e 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -25,7 +25,7 @@ from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.http import Headers, Request
+from scrapy.http import Headers, HtmlResponse, Request
from scrapy.http.response.text import TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
@@ -389,6 +389,23 @@ class HttpTestCase(unittest.TestCase):
d.addCallback(self.assertEqual, b'159')
return d
+ def _test_response_class(self, filename, body, response_class):
+ def _test(response):
+ self.assertEqual(type(response), response_class)
+
+ request = Request(self.getURL(filename), body=body)
+ return self.download_request(request, Spider('foo')).addCallback(_test)
+
+ def test_response_class_from_url(self):
+ return self._test_response_class('foo.html', b'', HtmlResponse)
+
+ def test_response_class_from_body(self):
+ return self._test_response_class(
+ 'foo',
+ b"\n.",
+ HtmlResponse,
+ )
+
class Http10TestCase(HttpTestCase):
"""HTTP 1.0 test case"""
@@ -971,6 +988,12 @@ class BaseFTPTestCase(unittest.TestCase):
password = "passwd"
req_meta = {"ftp_user": username, "ftp_password": password}
+ test_files = (
+ ('file.txt', b"I have the power!"),
+ ('file with spaces.txt', b"Moooooooooo power!"),
+ ('html-file-without-extension', b"\n."),
+ )
+
def setUp(self):
from twisted.protocols.ftp import FTPRealm, FTPFactory
from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
@@ -981,8 +1004,8 @@ class BaseFTPTestCase(unittest.TestCase):
userdir = os.path.join(self.directory, self.username)
os.mkdir(userdir)
fp = FilePath(userdir)
- fp.child('file.txt').setContent(b"I have the power!")
- fp.child('file with spaces.txt').setContent(b"Moooooooooo power!")
+ for filename, content in self.test_files:
+ fp.child(filename).setContent(content)
# setup server
realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory)
@@ -1069,6 +1092,27 @@ class BaseFTPTestCase(unittest.TestCase):
return self._add_test_callbacks(d, _test)
+ def _test_response_class(self, filename, response_class):
+ f, local_fname = tempfile.mkstemp()
+ local_fname = to_bytes(local_fname)
+ os.close(f)
+ meta = {}
+ meta.update(self.req_meta)
+ request = Request(url=f"ftp://127.0.0.1:{self.portNum}/{filename}",
+ meta=meta)
+ d = self.download_handler.download_request(request, None)
+
+ def _test(r):
+ self.assertEqual(type(r), response_class)
+ os.remove(local_fname)
+ return self._add_test_callbacks(d, _test)
+
+ def test_response_class_from_url(self):
+ return self._test_response_class('file.txt', TextResponse)
+
+ def test_response_class_from_body(self):
+ return self._test_response_class('html-file-without-extension', HtmlResponse)
+
class FTPTestCase(BaseFTPTestCase):
@@ -1104,8 +1148,8 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
os.mkdir(self.directory)
fp = FilePath(self.directory)
- fp.child('file.txt').setContent(b"I have the power!")
- fp.child('file with spaces.txt').setContent(b"Moooooooooo power!")
+ for filename, content in self.test_files:
+ fp.child(filename).setContent(content)
# setup server for anonymous access
realm = FTPRealm(anonymousRoot=self.directory)
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index b538a0ed3..38be915f2 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -1,13 +1,11 @@
import asyncio
-from unittest import mock, SkipTest
+from unittest import mock
from pytest import mark
-from twisted import version as twisted_version
from twisted.internet import defer
from twisted.internet.defer import Deferred
from twisted.trial.unittest import TestCase
from twisted.python.failure import Failure
-from twisted.python.versions import Version
from scrapy.http import Request, Response
from scrapy.spiders import Spider
@@ -218,16 +216,6 @@ class MiddlewareUsingCoro(ManagerTestCase):
"""Middlewares using asyncio coroutines should work"""
def test_asyncdef(self):
- if (
- self.reactor_pytest == 'asyncio'
- and twisted_version < Version('twisted', 18, 4, 0)
- ):
- raise SkipTest(
- 'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
- 'hangs when using AsyncIO and Twisted versions lower than '
- '18.4.0'
- )
-
resp = Response('http://example.com/index.html')
class CoroMiddleware:
@@ -248,12 +236,6 @@ class MiddlewareUsingCoro(ManagerTestCase):
@mark.only_asyncio()
def test_asyncdef_asyncio(self):
- if twisted_version < Version('twisted', 18, 4, 0):
- raise SkipTest(
- 'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
- 'hangs when using Twisted versions lower than 18.4.0'
- )
-
resp = Response('http://example.com/index.html')
class CoroMiddleware:
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 0c6dcf2aa..928c007f5 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -122,6 +122,21 @@ class DefaultStorageTest(_BaseTest):
time.sleep(0.5) # give the chance to expire
assert storage.retrieve_response(self.spider, self.request)
+ def test_storage_no_content_type_header(self):
+ """Test that the response body is used to get the right response class
+ even if there is no Content-Type header"""
+ with self._storage() as storage:
+ assert storage.retrieve_response(self.spider, self.request) is None
+ response = Response(
+ 'http://www.example.com',
+ body=b'\n.',
+ status=202,
+ )
+ storage.store_response(self.spider, self.request, response)
+ cached_response = storage.retrieve_response(self.spider, self.request)
+ self.assertIsInstance(cached_response, HtmlResponse)
+ self.assertEqualResponse(response, cached_response)
+
class DbmStorageTest(DefaultStorageTest):
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 680bb6dc8..b7df2554a 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -15,6 +15,16 @@ from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
+def _get_dupefilter(*, crawler=None, settings=None, open=True):
+ if crawler is None:
+ crawler = get_crawler(settings_dict=settings)
+ scheduler = Scheduler.from_crawler(crawler)
+ dupefilter = scheduler.df
+ if open:
+ dupefilter.open()
+ return dupefilter
+
+
class FromCrawlerRFPDupeFilter(RFPDupeFilter):
@classmethod
@@ -64,9 +74,7 @@ class RFPDupeFilterTest(unittest.TestCase):
self.assertEqual(scheduler.df.method, 'n/a')
def test_filter(self):
- dupefilter = RFPDupeFilter()
- dupefilter.open()
-
+ dupefilter = _get_dupefilter()
r1 = Request('http://scrapytest.org/1')
r2 = Request('http://scrapytest.org/2')
r3 = Request('http://scrapytest.org/2')
@@ -85,7 +93,7 @@ class RFPDupeFilterTest(unittest.TestCase):
path = tempfile.mkdtemp()
try:
- df = RFPDupeFilter(path)
+ df = _get_dupefilter(settings={'JOBDIR': path}, open=False)
try:
df.open()
assert not df.request_seen(r1)
@@ -93,7 +101,8 @@ class RFPDupeFilterTest(unittest.TestCase):
finally:
df.close('finished')
- df2 = RFPDupeFilter(path)
+ df2 = _get_dupefilter(settings={'JOBDIR': path}, open=False)
+ assert df != df2
try:
df2.open()
assert df2.request_seen(r1)
@@ -109,26 +118,24 @@ class RFPDupeFilterTest(unittest.TestCase):
output of request_seen.
"""
+ dupefilter = _get_dupefilter()
r1 = Request('http://scrapytest.org/index.html')
r2 = Request('http://scrapytest.org/INDEX.html')
- dupefilter = RFPDupeFilter()
- dupefilter.open()
-
assert not dupefilter.request_seen(r1)
assert not dupefilter.request_seen(r2)
dupefilter.close('finished')
- class CaseInsensitiveRFPDupeFilter(RFPDupeFilter):
+ class RequestFingerprinter:
- def request_fingerprint(self, request):
+ def fingerprint(self, request):
fp = hashlib.sha1()
fp.update(to_bytes(request.url.lower()))
- return fp.hexdigest()
+ return fp.digest()
- case_insensitive_dupefilter = CaseInsensitiveRFPDupeFilter()
- case_insensitive_dupefilter.open()
+ settings = {'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter}
+ case_insensitive_dupefilter = _get_dupefilter(settings=settings)
assert not case_insensitive_dupefilter.request_seen(r1)
assert case_insensitive_dupefilter.request_seen(r2)
@@ -142,8 +149,10 @@ class RFPDupeFilterTest(unittest.TestCase):
r1 = Request('http://scrapytest.org/1')
path = tempfile.mkdtemp()
+ crawler = get_crawler(settings_dict={'JOBDIR': path})
try:
- df = RFPDupeFilter(path)
+ scheduler = Scheduler.from_crawler(crawler)
+ df = scheduler.df
df.open()
df.request_seen(r1)
df.close('finished')
@@ -164,11 +173,8 @@ class RFPDupeFilterTest(unittest.TestCase):
settings = {'DUPEFILTER_DEBUG': False,
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
- scheduler = Scheduler.from_crawler(crawler)
spider = SimpleSpider.from_crawler(crawler)
-
- dupefilter = scheduler.df
- dupefilter.open()
+ dupefilter = _get_dupefilter(crawler=crawler)
r1 = Request('http://scrapytest.org/index.html')
r2 = Request('http://scrapytest.org/index.html')
@@ -193,11 +199,41 @@ class RFPDupeFilterTest(unittest.TestCase):
settings = {'DUPEFILTER_DEBUG': True,
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
- scheduler = Scheduler.from_crawler(crawler)
spider = SimpleSpider.from_crawler(crawler)
-
- dupefilter = scheduler.df
- dupefilter.open()
+ dupefilter = _get_dupefilter(crawler=crawler)
+
+ r1 = Request('http://scrapytest.org/index.html')
+ r2 = Request('http://scrapytest.org/index.html',
+ headers={'Referer': 'http://scrapytest.org/INDEX.html'})
+
+ dupefilter.log(r1, spider)
+ dupefilter.log(r2, spider)
+
+ assert crawler.stats.get_value('dupefilter/filtered') == 2
+ log.check_present(
+ (
+ 'scrapy.dupefilters',
+ 'DEBUG',
+ 'Filtered duplicate request: (referer: None)'
+ )
+ )
+ log.check_present(
+ (
+ 'scrapy.dupefilters',
+ 'DEBUG',
+ 'Filtered duplicate request: '
+ ' (referer: http://scrapytest.org/INDEX.html)'
+ )
+ )
+
+ dupefilter.close('finished')
+
+ def test_log_debug_default_dupefilter(self):
+ with LogCapture() as log:
+ settings = {'DUPEFILTER_DEBUG': True}
+ crawler = get_crawler(SimpleSpider, settings_dict=settings)
+ spider = SimpleSpider.from_crawler(crawler)
+ dupefilter = _get_dupefilter(crawler=crawler)
r1 = Request('http://scrapytest.org/index.html')
r2 = Request('http://scrapytest.org/index.html',
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index b263b3475..096cd3116 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -112,6 +112,14 @@ class BaseItemExporterTest(unittest.TestCase):
assert isinstance(name, str)
self.assertEqual(name, 'John\xa3')
+ ie = self._get_exporter(
+ fields_to_export={'name': '名稱'}
+ )
+ self.assertEqual(
+ list(ie._get_serialized_fields(self.i)),
+ [('名稱', 'John\xa3')]
+ )
+
def test_field_custom_serializer(self):
i = self.custom_field_item_class(name='John\xa3', age='22')
a = ItemAdapter(i)
@@ -272,6 +280,7 @@ class MarshalItemExporterDataclassTest(MarshalItemExporterTest):
class CsvItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
+ self.output = tempfile.TemporaryFile()
return CsvItemExporter(self.output, **kwargs)
def assertCsvEqual(self, first, second, msg=None):
@@ -283,7 +292,8 @@ class CsvItemExporterTest(BaseItemExporterTest):
return self.assertEqual(split_csv(first), split_csv(second), msg=msg)
def _check_output(self):
- self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n')
+ self.output.seek(0)
+ self.assertCsvEqual(to_unicode(self.output.read()), 'age,name\r\n22,John\xa3\r\n')
def assertExportResult(self, item, expected, **kwargs):
fp = BytesIO()
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index f0acf1941..ec48f8d4a 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -7,6 +7,7 @@ import os
import random
import shutil
import string
+import sys
import tempfile
import warnings
from abc import ABC, abstractmethod
@@ -655,8 +656,8 @@ class FeedExportTestBase(ABC, unittest.TestCase):
return data
@defer.inlineCallbacks
- def assertExported(self, items, header, rows, settings=None, ordered=True):
- yield self.assertExportedCsv(items, header, rows, settings, ordered)
+ def assertExported(self, items, header, rows, settings=None):
+ yield self.assertExportedCsv(items, header, rows, settings)
yield self.assertExportedJsonLines(items, rows, settings)
yield self.assertExportedXml(items, rows, settings)
yield self.assertExportedPickle(items, rows, settings)
@@ -717,7 +718,7 @@ class FeedExportTest(FeedExportTestBase):
return content
@defer.inlineCallbacks
- def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+ def assertExportedCsv(self, items, header, rows, settings=None):
settings = settings or {}
settings.update({
'FEEDS': {
@@ -725,15 +726,9 @@ class FeedExportTest(FeedExportTestBase):
},
})
data = yield self.exported_data(items, settings)
-
reader = csv.DictReader(to_unicode(data['csv']).splitlines())
- got_rows = list(reader)
- if ordered:
- self.assertEqual(reader.fieldnames, header)
- else:
- self.assertEqual(set(reader.fieldnames), set(header))
-
- self.assertEqual(rows, got_rows)
+ self.assertEqual(reader.fieldnames, list(header))
+ self.assertEqual(rows, list(reader))
@defer.inlineCallbacks
def assertExportedJsonLines(self, items, rows, settings=None):
@@ -884,7 +879,7 @@ class FeedExportTest(FeedExportTestBase):
{'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
]
header = self.MyItem.fields.keys()
- yield self.assertExported(items, header, rows, ordered=False)
+ yield self.assertExported(items, header, rows)
@defer.inlineCallbacks
def test_export_no_items_not_store_empty(self):
@@ -956,25 +951,72 @@ class FeedExportTest(FeedExportTestBase):
{'egg': 'spam4', 'foo': '', 'baz': ''},
]
rows_jl = [dict(row) for row in items]
- yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
+ yield self.assertExportedCsv(items, header, rows_csv)
yield self.assertExportedJsonLines(items, rows_jl)
- # edge case: FEED_EXPORT_FIELDS==[] means the same as default None
+ @defer.inlineCallbacks
+ def test_export_items_empty_field_list(self):
+ # FEED_EXPORT_FIELDS==[] means the same as default None
+ items = [{'foo': 'bar'}]
+ header = ["foo"]
+ rows = [{'foo': 'bar'}]
settings = {'FEED_EXPORT_FIELDS': []}
- yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
- yield self.assertExportedJsonLines(items, rows_jl, settings)
+ yield self.assertExportedCsv(items, header, rows)
+ yield self.assertExportedJsonLines(items, rows, settings)
- # it is possible to override fields using FEED_EXPORT_FIELDS
- header = ["foo", "baz", "hello"]
+ @defer.inlineCallbacks
+ def test_export_items_field_list(self):
+ items = [{'foo': 'bar'}]
+ header = ["foo", "baz"]
+ rows = [{'foo': 'bar', 'baz': ''}]
settings = {'FEED_EXPORT_FIELDS': header}
- rows = [
- {'foo': 'bar1', 'baz': '', 'hello': ''},
- {'foo': 'bar2', 'baz': '', 'hello': 'world2'},
- {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
- {'foo': '', 'baz': '', 'hello': 'world4'},
- ]
- yield self.assertExported(items, header, rows,
- settings=settings, ordered=True)
+ yield self.assertExported(items, header, rows, settings=settings)
+
+ @defer.inlineCallbacks
+ def test_export_items_comma_separated_field_list(self):
+ items = [{'foo': 'bar'}]
+ header = ["foo", "baz"]
+ rows = [{'foo': 'bar', 'baz': ''}]
+ settings = {'FEED_EXPORT_FIELDS': ",".join(header)}
+ yield self.assertExported(items, header, rows, settings=settings)
+
+ @defer.inlineCallbacks
+ def test_export_items_json_field_list(self):
+ items = [{'foo': 'bar'}]
+ header = ["foo", "baz"]
+ rows = [{'foo': 'bar', 'baz': ''}]
+ settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
+ yield self.assertExported(items, header, rows, settings=settings)
+
+ @defer.inlineCallbacks
+ def test_export_items_field_names(self):
+ items = [{'foo': 'bar'}]
+ header = {'foo': 'Foo'}
+ rows = [{'Foo': 'bar'}]
+ settings = {'FEED_EXPORT_FIELDS': header}
+ yield self.assertExported(items, list(header.values()), rows,
+ settings=settings)
+
+ @defer.inlineCallbacks
+ def test_export_items_dict_field_names(self):
+ items = [{'foo': 'bar'}]
+ header = {
+ 'baz': 'Baz',
+ 'foo': 'Foo',
+ }
+ rows = [{'Baz': '', 'Foo': 'bar'}]
+ settings = {'FEED_EXPORT_FIELDS': header}
+ yield self.assertExported(items, ['Baz', 'Foo'], rows,
+ settings=settings)
+
+ @defer.inlineCallbacks
+ def test_export_items_json_field_names(self):
+ items = [{'foo': 'bar'}]
+ header = {'foo': 'Foo'}
+ rows = [{'Foo': 'bar'}]
+ settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
+ yield self.assertExported(items, list(header.values()), rows,
+ settings=settings)
@defer.inlineCallbacks
def test_export_based_on_item_classes(self):
@@ -1097,7 +1139,7 @@ class FeedExportTest(FeedExportTestBase):
{'egg': 'spam', 'foo': 'bar'}
]
rows_jl = items
- yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False)
+ yield self.assertExportedCsv(items, ['foo', 'egg'], rows_csv)
yield self.assertExportedJsonLines(items, rows_jl)
@defer.inlineCallbacks
@@ -1118,7 +1160,7 @@ class FeedExportTest(FeedExportTestBase):
{'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
]
yield self.assertExported(items, ['foo', 'baz', 'egg'], rows,
- settings=settings, ordered=True)
+ settings=settings)
# export a subset of columns
settings = {'FEED_EXPORT_FIELDS': 'egg,baz'}
@@ -1127,7 +1169,7 @@ class FeedExportTest(FeedExportTestBase):
{'egg': 'spam2', 'baz': 'quux2'}
]
yield self.assertExported(items, ['egg', 'baz'], rows,
- settings=settings, ordered=True)
+ settings=settings)
@defer.inlineCallbacks
def test_export_encoding(self):
@@ -1769,7 +1811,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_lzma_plugin_filters(self):
- import sys
if "PyPy" in sys.version:
# https://foss.heptapod.net/pypy/pypy/-/issues/3527
raise unittest.SkipTest("lzma filters doesn't work in PyPy")
@@ -2016,7 +2057,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
self.assertEqual(expected_batch, got_batch)
@defer.inlineCallbacks
- def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+ def assertExportedCsv(self, items, header, rows, settings=None):
settings = settings or {}
settings.update({
'FEEDS': {
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index 64ff7a73d..1ca936247 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -38,6 +38,12 @@ class HeadersTest(unittest.TestCase):
self.assertEqual(h.getlist('X-Forwarded-For'), [b'ip1', b'ip2'])
assert h.getlist('X-Forwarded-For') is not hlist
+ def test_multivalue_for_one_header(self):
+ h = Headers((("a", "b"), ("a", "c")))
+ self.assertEqual(h["a"], b"c")
+ self.assertEqual(h.get("a"), b"c")
+ self.assertEqual(h.getlist("a"), [b"b", b"c"])
+
def test_encode_utf8(self):
h = Headers({'key': '\xa3'}, encoding='utf-8')
key, val = dict(h).popitem()
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 4e1b90787..4228173ed 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -25,6 +25,7 @@ from scrapy.pipelines.files import (
from scrapy.settings import Settings
from scrapy.utils.test import (
assert_gcs_environ,
+ get_crawler,
get_ftp_content_and_delete,
get_gcs_content_and_delete,
skip_if_no_boto,
@@ -47,7 +48,9 @@ class FilesPipelineTestCase(unittest.TestCase):
def setUp(self):
self.tempdir = mkdtemp()
- self.pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir}))
+ settings_dict = {'FILES_STORE': self.tempdir}
+ crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
+ self.pipeline = FilesPipeline.from_crawler(crawler)
self.pipeline.download_func = _mocked_download_func
self.pipeline.open_spider(None)
@@ -525,6 +528,29 @@ class TestGCSFilesStore(unittest.TestCase):
self.assertEqual(blob.content_type, 'application/octet-stream')
self.assertIn(expected_policy, acl)
+ @defer.inlineCallbacks
+ def test_blob_path_consistency(self):
+ """Test to make sure that paths used to store files is the same as the one used to get
+ already uploaded files.
+ """
+ assert_gcs_environ()
+ try:
+ import google.cloud.storage # noqa
+ except ModuleNotFoundError:
+ raise unittest.SkipTest("google-cloud-storage is not installed")
+ else:
+ with mock.patch('google.cloud.storage') as _:
+ with mock.patch('scrapy.pipelines.files.time') as _:
+ uri = 'gs://my_bucket/my_prefix/'
+ store = GCSFilesStore(uri)
+ store.bucket = mock.Mock()
+ path = 'full/my_data.txt'
+ yield store.persist_file(path, mock.Mock(), info=None, meta=None, headers=None)
+ yield store.stat_file(path, info=None)
+ expected_blob_path = store.prefix + path
+ store.bucket.blob.assert_called_with(expected_blob_path)
+ store.bucket.get_blob.assert_called_with(expected_blob_path)
+
class TestFTPFileStore(unittest.TestCase):
@defer.inlineCallbacks
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index c69cd0e4a..dd94d296b 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -93,6 +93,22 @@ class ImagesPipelineTestCase(unittest.TestCase):
info=object()),
'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg')
+ def test_thumbnail_name_from_item(self):
+ """
+ Custom thumbnail name based on item data, overriding default implementation
+ """
+
+ class CustomImagesPipeline(ImagesPipeline):
+ def thumb_path(self, request, thumb_id, response=None, info=None, item=None):
+ return f"thumb/{thumb_id}/{item.get('path')}"
+
+ thumb_path = CustomImagesPipeline.from_settings(Settings(
+ {'IMAGES_STORE': self.tempdir}
+ )).thumb_path
+ item = dict(path='path-to-store-file')
+ request = Request("http://example.com")
+ self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file')
+
def test_convert_image(self):
SIZE = (100, 100)
# straigh forward case: RGB and JPEG
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 893d43052..84e867660 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,4 +1,5 @@
from typing import Optional
+import io
from testfixtures import LogCapture
from twisted.trial import unittest
@@ -6,17 +7,17 @@ from twisted.python.failure import Failure
from twisted.internet import reactor
from twisted.internet.defer import Deferred, inlineCallbacks
+from scrapy import signals
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.spiders import Spider
-from scrapy.utils.deprecate import ScrapyDeprecationWarning
-from scrapy.utils.request import request_fingerprint
+from scrapy.pipelines.files import FileException
from scrapy.pipelines.images import ImagesPipeline
from scrapy.pipelines.media import MediaPipeline
-from scrapy.pipelines.files import FileException
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.signal import disconnect_all
-from scrapy import signals
+from scrapy.utils.test import get_crawler
try:
@@ -38,11 +39,14 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
settings = None
def setUp(self):
- self.spider = Spider('media.com')
- self.pipe = self.pipeline_class(download_func=_mocked_download_func,
- settings=Settings(self.settings))
+ spider_cls = Spider
+ self.spider = spider_cls('media.com')
+ crawler = get_crawler(spider_cls, self.settings)
+ self.pipe = self.pipeline_class.from_crawler(crawler)
+ self.pipe.download_func = _mocked_download_func
self.pipe.open_spider(self.spider)
self.info = self.pipe.spiderinfo
+ self.fingerprint = crawler.request_fingerprinter.fingerprint
def tearDown(self):
for name, signal in vars(signals).items():
@@ -155,7 +159,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
self.assertEqual(failure.value.__context__, def_gen_return_exc)
# Let's calculate the request fingerprint and fake some runtime data...
- fp = request_fingerprint(request)
+ fp = self.fingerprint(request)
info = self.pipe.spiderinfo
info.downloading.add(fp)
info.waiting[fp] = []
@@ -272,7 +276,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
item = dict(requests=req) # pass a single item
new_item = yield self.pipe.process_item(item, self.spider)
assert new_item is item
- assert request_fingerprint(req) in self.info.downloaded
+ self.assertIn(self.fingerprint(req), self.info.downloaded)
# returns iterable of Requests
req1 = Request('http://url1')
@@ -280,8 +284,8 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
item = dict(requests=iter([req1, req2]))
new_item = yield self.pipe.process_item(item, self.spider)
assert new_item is item
- assert request_fingerprint(req1) in self.info.downloaded
- assert request_fingerprint(req2) in self.info.downloaded
+ assert self.fingerprint(req1) in self.info.downloaded
+ assert self.fingerprint(req2) in self.info.downloaded
@inlineCallbacks
def test_results_are_cached_across_multiple_items(self):
@@ -297,7 +301,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
item = dict(requests=req2)
new_item = yield self.pipe.process_item(item, self.spider)
self.assertTrue(new_item is item)
- self.assertEqual(request_fingerprint(req1), request_fingerprint(req2))
+ self.assertEqual(self.fingerprint(req1), self.fingerprint(req2))
self.assertEqual(new_item['results'], [(True, rsp1)])
@inlineCallbacks
@@ -313,7 +317,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
@inlineCallbacks
def test_wait_if_request_is_downloading(self):
def _check_downloading(response):
- fp = request_fingerprint(req1)
+ fp = self.fingerprint(req1)
self.assertTrue(fp in self.info.downloading)
self.assertTrue(fp in self.info.waiting)
self.assertTrue(fp not in self.info.downloaded)
@@ -350,14 +354,17 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
def __init__(self, *args, **kwargs):
- super(MockedMediaPipelineDeprecatedMethods, self).__init__(*args, **kwargs)
+ super().__init__(*args, **kwargs)
self._mockcalled = []
def get_media_requests(self, item, info):
item_url = item['image_urls'][0]
+ output_img = io.BytesIO()
+ img = Image.new('RGB', (60, 30), color='red')
+ img.save(output_img, format='JPEG')
return Request(
item_url,
- meta={'response': Response(item_url, status=200, body=b'data')}
+ meta={'response': Response(item_url, status=200, body=output_img.getvalue())}
)
def inc_stats(self, *args, **kwargs):
@@ -365,34 +372,44 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
def media_to_download(self, request, info):
self._mockcalled.append('media_to_download')
- return super(MockedMediaPipelineDeprecatedMethods, self).media_to_download(request, info)
+ return super().media_to_download(request, info)
def media_downloaded(self, response, request, info):
self._mockcalled.append('media_downloaded')
- return super(MockedMediaPipelineDeprecatedMethods, self).media_downloaded(response, request, info)
+ return super().media_downloaded(response, request, info)
def file_downloaded(self, response, request, info):
self._mockcalled.append('file_downloaded')
- return super(MockedMediaPipelineDeprecatedMethods, self).file_downloaded(response, request, info)
+ return super().file_downloaded(response, request, info)
def file_path(self, request, response=None, info=None):
self._mockcalled.append('file_path')
- return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info)
+ return super().file_path(request, response, info)
+
+ def thumb_path(self, request, thumb_id, response=None, info=None):
+ self._mockcalled.append('thumb_path')
+ return super(MockedMediaPipelineDeprecatedMethods, self).thumb_path(request, thumb_id, response, info)
def get_images(self, response, request, info):
self._mockcalled.append('get_images')
- return []
+ return super(MockedMediaPipelineDeprecatedMethods, self).get_images(response, request, info)
def image_downloaded(self, response, request, info):
self._mockcalled.append('image_downloaded')
- return super(MockedMediaPipelineDeprecatedMethods, self).image_downloaded(response, request, info)
+ return super().image_downloaded(response, request, info)
class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
skip = skip_pillow
def setUp(self):
- self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func)
+ settings_dict = {
+ 'IMAGES_STORE': 'store-uri',
+ 'IMAGES_THUMBS': {'small': (50, 50)},
+ }
+ crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
+ self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler)
+ self.pipe.download_func = _mocked_download_func
self.pipe.open_spider(None)
self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[])
@@ -444,6 +461,16 @@ class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
)
self._assert_method_called_with_warnings('file_path', message, warnings)
+ @inlineCallbacks
+ def test_thumb_path_called(self):
+ yield self.pipe.process_item(self.item, None)
+ warnings = self.flushWarnings([MediaPipeline._compatible])
+ message = (
+ 'thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, '
+ 'please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)'
+ )
+ self._assert_method_called_with_warnings('thumb_path', message, warnings)
+
@inlineCallbacks
def test_get_images_called(self):
yield self.pipe.process_item(self.item, None)
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index c07d3a99c..4b4095fb0 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -54,6 +54,8 @@ class ResponseTypesTest(unittest.TestCase):
(b'\x03\x02\xdf\xdd\x23', Response),
(b'Some plain text\ndata with tabs\t and null bytes\0', TextResponse),
(b'Hello', HtmlResponse),
+ # https://codersblock.com/blog/the-smallest-valid-html5-page/
+ (b'\n.', HtmlResponse),
(b'= datetime(2020, 4, 7): # PyPy 3.6-v7.3.1
- self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])
- else:
- self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
+ self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])
def test_without_none_values(self):
self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 7e0049b1d..e9edfee98 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -1,73 +1,29 @@
import unittest
+import warnings
+from hashlib import sha1
+from typing import Dict, Mapping, Optional, Tuple, Union
+from weakref import WeakKeyDictionary
+
+import pytest
+from w3lib.url import canonicalize_url
+
from scrapy.http import Request
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
+from scrapy.utils.python import to_bytes
from scrapy.utils.request import (
+ _deprecated_fingerprint_cache,
_fingerprint_cache,
+ _request_fingerprint_as_bytes,
+ fingerprint,
request_authenticate,
request_fingerprint,
request_httprepr,
)
+from scrapy.utils.test import get_crawler
class UtilsRequestTest(unittest.TestCase):
- def test_request_fingerprint(self):
- r1 = Request("http://www.example.com/query?id=111&cat=222")
- r2 = Request("http://www.example.com/query?cat=222&id=111")
- self.assertEqual(request_fingerprint(r1), request_fingerprint(r1))
- self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
-
- r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199')
- r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
- self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
-
- # make sure caching is working
- self.assertEqual(request_fingerprint(r1), _fingerprint_cache[r1][(None, False)])
-
- r1 = Request("http://www.example.com/members/offers.html")
- r2 = Request("http://www.example.com/members/offers.html")
- r2.headers['SESSIONID'] = b"somehash"
- self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
-
- r1 = Request("http://www.example.com/")
- r2 = Request("http://www.example.com/")
- r2.headers['Accept-Language'] = b'en'
- r3 = Request("http://www.example.com/")
- r3.headers['Accept-Language'] = b'en'
- r3.headers['SESSIONID'] = b"somehash"
-
- self.assertEqual(request_fingerprint(r1), request_fingerprint(r2), request_fingerprint(r3))
-
- self.assertEqual(request_fingerprint(r1),
- request_fingerprint(r1, include_headers=['Accept-Language']))
-
- self.assertNotEqual(
- request_fingerprint(r1),
- request_fingerprint(r2, include_headers=['Accept-Language']))
-
- self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']),
- request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language']))
-
- r1 = Request("http://www.example.com/test.html")
- r2 = Request("http://www.example.com/test.html#fragment")
- self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
- self.assertEqual(request_fingerprint(r1), request_fingerprint(r1, keep_fragments=True))
- self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r2, keep_fragments=True))
- self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2, keep_fragments=True))
-
- r1 = Request("http://www.example.com")
- r2 = Request("http://www.example.com", method='POST')
- r3 = Request("http://www.example.com", method='POST', body=b'request body')
-
- self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
- self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r3))
-
- # cached fingerprint must be cleared on request copy
- r1 = Request("http://www.example.com")
- fp1 = request_fingerprint(r1)
- r2 = r1.replace(url="http://www.example.com/other")
- fp2 = request_fingerprint(r2)
- self.assertNotEqual(fp1, fp2)
-
def test_request_authenticate(self):
r = Request("http://www.example.com")
request_authenticate(r, 'someuser', 'somepass')
@@ -93,5 +49,632 @@ class UtilsRequestTest(unittest.TestCase):
request_httprepr(Request("ftp://localhost/tmp/foo.txt"))
+class FingerprintTest(unittest.TestCase):
+ maxDiff = None
+
+ function = staticmethod(fingerprint)
+ cache: Union[
+ "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]",
+ "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]",
+ ] = _fingerprint_cache
+ default_cache_key = (None, False)
+ known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
+ (
+ Request("http://example.org"),
+ b'xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD',
+ {},
+ ),
+ (
+ Request("https://example.org"),
+ b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',
+ {},
+ ),
+ (
+ Request("https://example.org?a"),
+ b'G\xad\xb8Ck\x19\x1c\xed\x838,\x01\xc4\xde;\xee\xa5\x94a\x0c',
+ {},
+ ),
+ (
+ Request("https://example.org?a=b"),
+ b'\x024MYb\x8a\xc2\x1e\xbc>\xd6\xac*\xda\x9cF\xc1r\x7f\x17',
+ {},
+ ),
+ (
+ Request("https://example.org?a=b&a"),
+ b't+\xe8*\xfb\x84\xe3v\x1a}\x88p\xc0\xccB\xd7\x9d\xfez\x96',
+ {},
+ ),
+ (
+ Request("https://example.org?a=b&a=c"),
+ b'\xda\x1ec\xd0\x9c\x08s`\xb4\x9b\xe2\xb6R\xf8k\xef\xeaQG\xef',
+ {},
+ ),
+ (
+ Request("https://example.org", method='POST'),
+ b'\x9d\xcdA\x0fT\x02:\xca\xa0}\x90\xda\x05B\xded\x8aN7\x1d',
+ {},
+ ),
+ (
+ Request("https://example.org", body=b'a'),
+ b'\xc34z>\xd8\x99\x8b\xda7\x05r\x99I\xa8\xa0x;\xa41_',
+ {},
+ ),
+ (
+ Request("https://example.org", method='POST', body=b'a'),
+ b'5`\xe2y4\xd0\x9d\xee\xe0\xbatw\x87Q\xe8O\xd78\xfc\xe7',
+ {},
+ ),
+ (
+ Request("https://example.org#a", headers={'A': b'B'}),
+ b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',
+ {},
+ ),
+ (
+ Request("https://example.org#a", headers={'A': b'B'}),
+ b']\xc7\x1f\xf2\xafG2\xbc\xa4\xfa\x99\n33\xda\x18\x94\x81U.',
+ {'include_headers': ['A']},
+ ),
+ (
+ Request("https://example.org#a", headers={'A': b'B'}),
+ b'<\x1a\xeb\x85y\xdeW\xfb\xdcq\x88\xee\xaf\x17\xdd\x0c\xbfH\x18\x1f',
+ {'keep_fragments': True},
+ ),
+ (
+ Request("https://example.org#a", headers={'A': b'B'}),
+ b'\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11',
+ {'include_headers': ['A'], 'keep_fragments': True},
+ ),
+ (
+ Request("https://example.org/ab"),
+ b'N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6',
+ {},
+ ),
+ (
+ Request("https://example.org/a", body=b'b'),
+ b'_NOv\xbco$6\xfcW\x9f\xb24g\x9f\xbb\xdd\xa82\xc5',
+ {},
+ ),
+ )
+
+ def test_query_string_key_order(self):
+ r1 = Request("http://www.example.com/query?id=111&cat=222")
+ r2 = Request("http://www.example.com/query?cat=222&id=111")
+ self.assertEqual(self.function(r1), self.function(r1))
+ self.assertEqual(self.function(r1), self.function(r2))
+
+ def test_query_string_key_without_value(self):
+ r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199')
+ r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
+ self.assertNotEqual(self.function(r1), self.function(r2))
+
+ def test_caching(self):
+ r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
+ self.assertEqual(
+ self.function(r1),
+ self.cache[r1][self.default_cache_key]
+ )
+
+ def test_header(self):
+ r1 = Request("http://www.example.com/members/offers.html")
+ r2 = Request("http://www.example.com/members/offers.html")
+ r2.headers['SESSIONID'] = b"somehash"
+ self.assertEqual(self.function(r1), self.function(r2))
+
+ def test_headers(self):
+ r1 = Request("http://www.example.com/")
+ r2 = Request("http://www.example.com/")
+ r2.headers['Accept-Language'] = b'en'
+ r3 = Request("http://www.example.com/")
+ r3.headers['Accept-Language'] = b'en'
+ r3.headers['SESSIONID'] = b"somehash"
+
+ self.assertEqual(self.function(r1), self.function(r2), self.function(r3))
+
+ self.assertEqual(self.function(r1),
+ self.function(r1, include_headers=['Accept-Language']))
+
+ self.assertNotEqual(
+ self.function(r1),
+ self.function(r2, include_headers=['Accept-Language']))
+
+ self.assertEqual(self.function(r3, include_headers=['accept-language', 'sessionid']),
+ self.function(r3, include_headers=['SESSIONID', 'Accept-Language']))
+
+ def test_fragment(self):
+ r1 = Request("http://www.example.com/test.html")
+ r2 = Request("http://www.example.com/test.html#fragment")
+ self.assertEqual(self.function(r1), self.function(r2))
+ self.assertEqual(self.function(r1), self.function(r1, keep_fragments=True))
+ self.assertNotEqual(self.function(r2), self.function(r2, keep_fragments=True))
+ self.assertNotEqual(self.function(r1), self.function(r2, keep_fragments=True))
+
+ def test_method_and_body(self):
+ r1 = Request("http://www.example.com")
+ r2 = Request("http://www.example.com", method='POST')
+ r3 = Request("http://www.example.com", method='POST', body=b'request body')
+
+ self.assertNotEqual(self.function(r1), self.function(r2))
+ self.assertNotEqual(self.function(r2), self.function(r3))
+
+ def test_request_replace(self):
+ # cached fingerprint must be cleared on request copy
+ r1 = Request("http://www.example.com")
+ fp1 = self.function(r1)
+ r2 = r1.replace(url="http://www.example.com/other")
+ fp2 = self.function(r2)
+ self.assertNotEqual(fp1, fp2)
+
+ def test_part_separation(self):
+ # An old implementation used to serialize request data in a way that
+ # would put the body right after the URL.
+ r1 = Request("http://www.example.com/foo")
+ fp1 = self.function(r1)
+ r2 = Request("http://www.example.com/f", body=b'oo')
+ fp2 = self.function(r2)
+ self.assertNotEqual(fp1, fp2)
+
+ def test_hashes(self):
+ """Test hardcoded hashes, to make sure future changes to not introduce
+ backward incompatibilities."""
+ actual = [
+ self.function(request, **kwargs)
+ for request, _, kwargs in self.known_hashes
+ ]
+ expected = [
+ _fingerprint
+ for _, _fingerprint, _ in self.known_hashes
+ ]
+ self.assertEqual(actual, expected)
+
+
+class RequestFingerprintTest(FingerprintTest):
+ function = staticmethod(request_fingerprint)
+ cache = _deprecated_fingerprint_cache
+ known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
+ (
+ Request("http://example.org"),
+ 'b2e5245ef826fd9576c93bd6e392fce3133fab62',
+ {},
+ ),
+ (
+ Request("https://example.org"),
+ 'bd10a0a89ea32cdee77917320f1309b0da87e892',
+ {},
+ ),
+ (
+ Request("https://example.org?a"),
+ '2fb7d48ae02f04b749f40caa969c0bc3c43204ce',
+ {},
+ ),
+ (
+ Request("https://example.org?a=b"),
+ '42e5fe149b147476e3f67ad0670c57b4cc57856a',
+ {},
+ ),
+ (
+ Request("https://example.org?a=b&a"),
+ 'd23a9787cb56c6375c2cae4453c5a8c634526942',
+ {},
+ ),
+ (
+ Request("https://example.org?a=b&a=c"),
+ '9a18a7a8552a9182b7f1e05d33876409e421e5c5',
+ {},
+ ),
+ (
+ Request("https://example.org", method='POST'),
+ 'ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6',
+ {},
+ ),
+ (
+ Request("https://example.org", body=b'a'),
+ '4bb136e54e715a4ea7a9dd1101831765d33f2d60',
+ {},
+ ),
+ (
+ Request("https://example.org", method='POST', body=b'a'),
+ '6c6595374a304b293be762f7b7be3f54e9947c65',
+ {},
+ ),
+ (
+ Request("https://example.org#a", headers={'A': b'B'}),
+ 'bd10a0a89ea32cdee77917320f1309b0da87e892',
+ {},
+ ),
+ (
+ Request("https://example.org#a", headers={'A': b'B'}),
+ '515b633cb3ca502a33a9d8c890e889ec1e425e65',
+ {'include_headers': ['A']},
+ ),
+ (
+ Request("https://example.org#a", headers={'A': b'B'}),
+ '505c96e7da675920dfef58725e8c957dfdb38f47',
+ {'keep_fragments': True},
+ ),
+ (
+ Request("https://example.org#a", headers={'A': b'B'}),
+ 'd6f673cdcb661b7970c2b9a00ee63e87d1e2e5da',
+ {'include_headers': ['A'], 'keep_fragments': True},
+ ),
+ (
+ Request("https://example.org/ab"),
+ '4e2870fee58582d6f81755e9b8fdefe3cba0c951',
+ {},
+ ),
+ (
+ Request("https://example.org/a", body=b'b'),
+ '4e2870fee58582d6f81755e9b8fdefe3cba0c951',
+ {},
+ ),
+ )
+
+ @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
+ def test_part_separation(self):
+ super().test_part_separation()
+
+ def test_deprecation_default_parameters(self):
+ with pytest.warns(ScrapyDeprecationWarning) as warnings:
+ self.function(Request("http://www.example.com"))
+ messages = [str(warning.message) for warning in warnings]
+ self.assertTrue(
+ any(
+ 'Call to deprecated function' in message
+ for message in messages
+ )
+ )
+ self.assertFalse(any('non-default' in message for message in messages))
+
+ def test_deprecation_non_default_parameters(self):
+ with pytest.warns(ScrapyDeprecationWarning) as warnings:
+ self.function(Request("http://www.example.com"), keep_fragments=True)
+ messages = [str(warning.message) for warning in warnings]
+ self.assertTrue(
+ any(
+ 'Call to deprecated function' in message
+ for message in messages
+ )
+ )
+ self.assertTrue(any('non-default' in message for message in messages))
+
+
+class RequestFingerprintAsBytesTest(FingerprintTest):
+ function = staticmethod(_request_fingerprint_as_bytes)
+ cache = _deprecated_fingerprint_cache
+ known_hashes = RequestFingerprintTest.known_hashes
+
+ def test_caching(self):
+ r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
+ self.assertEqual(
+ self.function(r1),
+ bytes.fromhex(self.cache[r1][self.default_cache_key])
+ )
+
+ @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
+ def test_part_separation(self):
+ super().test_part_separation()
+
+ def test_hashes(self):
+ actual = [
+ self.function(request, **kwargs)
+ for request, _, kwargs in self.known_hashes
+ ]
+ expected = [
+ bytes.fromhex(_fingerprint)
+ for _, _fingerprint, _ in self.known_hashes
+ ]
+ self.assertEqual(actual, expected)
+
+
+_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary()
+
+
+def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False):
+ if include_headers:
+ include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
+ cache = _fingerprint_cache_2_6.setdefault(request, {})
+ cache_key = (include_headers, keep_fragments)
+ if cache_key not in cache:
+ fp = sha1()
+ fp.update(to_bytes(request.method))
+ fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
+ fp.update(request.body or b'')
+ if include_headers:
+ for hdr in include_headers:
+ if hdr in request.headers:
+ fp.update(hdr)
+ for v in request.headers.getlist(hdr):
+ fp.update(v)
+ cache[cache_key] = fp.hexdigest()
+ return cache[cache_key]
+
+
+REQUEST_OBJECTS_TO_TEST = (
+ Request("http://www.example.com/"),
+ Request("http://www.example.com/query?id=111&cat=222"),
+ Request("http://www.example.com/query?cat=222&id=111"),
+ Request('http://www.example.com/hnnoticiaj1.aspx?78132,199'),
+ Request('http://www.example.com/hnnoticiaj1.aspx?78160,199'),
+ Request("http://www.example.com/members/offers.html"),
+ Request(
+ "http://www.example.com/members/offers.html",
+ headers={'SESSIONID': b"somehash"},
+ ),
+ Request(
+ "http://www.example.com/",
+ headers={'Accept-Language': b"en"},
+ ),
+ Request(
+ "http://www.example.com/",
+ headers={
+ 'Accept-Language': b"en",
+ 'SESSIONID': b"somehash",
+ },
+ ),
+ Request("http://www.example.com/test.html"),
+ Request("http://www.example.com/test.html#fragment"),
+ Request("http://www.example.com", method='POST'),
+ Request("http://www.example.com", method='POST', body=b'request body'),
+)
+
+
+class BackwardCompatibilityTestCase(unittest.TestCase):
+
+ def test_function_backward_compatibility(self):
+ include_headers_to_test = (
+ None,
+ ['Accept-Language'],
+ ['accept-language', 'sessionid'],
+ ['SESSIONID', 'Accept-Language'],
+ )
+ for request_object in REQUEST_OBJECTS_TO_TEST:
+ for include_headers in include_headers_to_test:
+ for keep_fragments in (False, True):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ fp = request_fingerprint(
+ request_object,
+ include_headers=include_headers,
+ keep_fragments=keep_fragments,
+ )
+ old_fp = request_fingerprint_2_6(
+ request_object,
+ include_headers=include_headers,
+ keep_fragments=keep_fragments,
+ )
+ self.assertEqual(fp, old_fp)
+
+ def test_component_backward_compatibility(self):
+ for request_object in REQUEST_OBJECTS_TO_TEST:
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ crawler = get_crawler(prevent_warnings=False)
+ fp = crawler.request_fingerprinter.fingerprint(request_object)
+ old_fp = request_fingerprint_2_6(request_object)
+ self.assertEqual(fp.hex(), old_fp)
+
+ def test_custom_component_backward_compatibility(self):
+ """Tests that the backward-compatible request fingerprinting class featured
+ in the documentation is indeed backward compatible and does not cause a
+ warning to be logged."""
+
+ class RequestFingerprinter:
+
+ cache = WeakKeyDictionary()
+
+ def fingerprint(self, request):
+ if request not in self.cache:
+ fp = sha1()
+ fp.update(to_bytes(request.method))
+ fp.update(to_bytes(canonicalize_url(request.url)))
+ fp.update(request.body or b'')
+ self.cache[request] = fp.digest()
+ return self.cache[request]
+
+ for request_object in REQUEST_OBJECTS_TO_TEST:
+ with warnings.catch_warnings() as logged_warnings:
+ settings = {
+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ }
+ crawler = get_crawler(settings_dict=settings)
+ fp = crawler.request_fingerprinter.fingerprint(request_object)
+ old_fp = request_fingerprint_2_6(request_object)
+ self.assertEqual(fp.hex(), old_fp)
+ self.assertFalse(logged_warnings)
+
+
+class RequestFingerprinterTestCase(unittest.TestCase):
+
+ def test_default_implementation(self):
+ with warnings.catch_warnings(record=True) as logged_warnings:
+ crawler = get_crawler(prevent_warnings=False)
+ request = Request('https://example.com')
+ self.assertEqual(
+ crawler.request_fingerprinter.fingerprint(request),
+ _request_fingerprint_as_bytes(request),
+ )
+ self.assertTrue(logged_warnings)
+
+ def test_deprecated_implementation(self):
+ settings = {
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'PREVIOUS_VERSION',
+ }
+ with warnings.catch_warnings(record=True) as logged_warnings:
+ crawler = get_crawler(settings_dict=settings)
+ request = Request('https://example.com')
+ self.assertEqual(
+ crawler.request_fingerprinter.fingerprint(request),
+ _request_fingerprint_as_bytes(request),
+ )
+ self.assertTrue(logged_warnings)
+
+ def test_recommended_implementation(self):
+ settings = {
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
+ }
+ with warnings.catch_warnings(record=True) as logged_warnings:
+ crawler = get_crawler(settings_dict=settings)
+ request = Request('https://example.com')
+ self.assertEqual(
+ crawler.request_fingerprinter.fingerprint(request),
+ fingerprint(request),
+ )
+ self.assertFalse(logged_warnings)
+
+ def test_unknown_implementation(self):
+ settings = {
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.5',
+ }
+ with self.assertRaises(ValueError):
+ get_crawler(settings_dict=settings)
+
+
+class CustomRequestFingerprinterTestCase(unittest.TestCase):
+
+ def test_include_headers(self):
+
+ class RequestFingerprinter:
+
+ def fingerprint(self, request):
+ return fingerprint(request, include_headers=['X-ID'])
+
+ settings = {
+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ }
+ crawler = get_crawler(settings_dict=settings)
+
+ r1 = Request("http://www.example.com", headers={'X-ID': '1'})
+ fp1 = crawler.request_fingerprinter.fingerprint(r1)
+ r2 = Request("http://www.example.com", headers={'X-ID': '2'})
+ fp2 = crawler.request_fingerprinter.fingerprint(r2)
+ self.assertNotEqual(fp1, fp2)
+
+ def test_dont_canonicalize(self):
+
+ class RequestFingerprinter:
+ cache = WeakKeyDictionary()
+
+ def fingerprint(self, request):
+ if request not in self.cache:
+ fp = sha1()
+ fp.update(to_bytes(request.url))
+ self.cache[request] = fp.digest()
+ return self.cache[request]
+
+ settings = {
+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ }
+ crawler = get_crawler(settings_dict=settings)
+
+ r1 = Request("http://www.example.com?a=1&a=2")
+ fp1 = crawler.request_fingerprinter.fingerprint(r1)
+ r2 = Request("http://www.example.com?a=2&a=1")
+ fp2 = crawler.request_fingerprinter.fingerprint(r2)
+ self.assertNotEqual(fp1, fp2)
+
+ def test_meta(self):
+
+ class RequestFingerprinter:
+
+ def fingerprint(self, request):
+ if 'fingerprint' in request.meta:
+ return request.meta['fingerprint']
+ return fingerprint(request)
+
+ settings = {
+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ }
+ crawler = get_crawler(settings_dict=settings)
+
+ r1 = Request("http://www.example.com")
+ fp1 = crawler.request_fingerprinter.fingerprint(r1)
+ r2 = Request("http://www.example.com", meta={'fingerprint': 'a'})
+ fp2 = crawler.request_fingerprinter.fingerprint(r2)
+ r3 = Request("http://www.example.com", meta={'fingerprint': 'a'})
+ fp3 = crawler.request_fingerprinter.fingerprint(r3)
+ r4 = Request("http://www.example.com", meta={'fingerprint': 'b'})
+ fp4 = crawler.request_fingerprinter.fingerprint(r4)
+ self.assertNotEqual(fp1, fp2)
+ self.assertNotEqual(fp1, fp4)
+ self.assertNotEqual(fp2, fp4)
+ self.assertEqual(fp2, fp3)
+
+ def test_from_crawler(self):
+
+ class RequestFingerprinter:
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
+
+ def __init__(self, crawler):
+ self._fingerprint = crawler.settings['FINGERPRINT']
+
+ def fingerprint(self, request):
+ return self._fingerprint
+
+ settings = {
+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ 'FINGERPRINT': b'fingerprint',
+ }
+ crawler = get_crawler(settings_dict=settings)
+
+ request = Request("http://www.example.com")
+ fingerprint = crawler.request_fingerprinter.fingerprint(request)
+ self.assertEqual(fingerprint, settings['FINGERPRINT'])
+
+ def test_from_settings(self):
+
+ class RequestFingerprinter:
+
+ @classmethod
+ def from_settings(cls, settings):
+ return cls(settings)
+
+ def __init__(self, settings):
+ self._fingerprint = settings['FINGERPRINT']
+
+ def fingerprint(self, request):
+ return self._fingerprint
+
+ settings = {
+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ 'FINGERPRINT': b'fingerprint',
+ }
+ crawler = get_crawler(settings_dict=settings)
+
+ request = Request("http://www.example.com")
+ fingerprint = crawler.request_fingerprinter.fingerprint(request)
+ self.assertEqual(fingerprint, settings['FINGERPRINT'])
+
+ def test_from_crawler_and_settings(self):
+
+ class RequestFingerprinter:
+
+ # This method is ignored due to the presence of from_crawler
+ @classmethod
+ def from_settings(cls, settings):
+ return cls(settings)
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
+
+ def __init__(self, crawler):
+ self._fingerprint = crawler.settings['FINGERPRINT']
+
+ def fingerprint(self, request):
+ return self._fingerprint
+
+ settings = {
+ 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ 'FINGERPRINT': b'fingerprint',
+ }
+ crawler = get_crawler(settings_dict=settings)
+
+ request = Request("http://www.example.com")
+ fingerprint = crawler.request_fingerprinter.fingerprint(request)
+ self.assertEqual(fingerprint, settings['FINGERPRINT'])
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index ad7394232..a36e7bc97 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -1,13 +1,10 @@
import asyncio
-from unittest import SkipTest
from pydispatch import dispatcher
from pytest import mark
from testfixtures import LogCapture
-from twisted import version as twisted_version
from twisted.internet import defer, reactor
from twisted.python.failure import Failure
-from twisted.python.versions import Version
from twisted.trial import unittest
from scrapy.utils.signal import send_catch_log, send_catch_log_deferred
@@ -81,16 +78,6 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
return "OK"
def test_send_catch_log(self):
- if (
- self.reactor_pytest == 'asyncio'
- and twisted_version < Version('twisted', 18, 4, 0)
- ):
- raise SkipTest(
- 'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
- 'fails due to a timeout when using AsyncIO and Twisted '
- 'versions lower than 18.4.0'
- )
-
return super().test_send_catch_log()
@@ -104,13 +91,6 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
return await get_from_asyncio_queue("OK")
def test_send_catch_log(self):
- if twisted_version < Version('twisted', 18, 4, 0):
- raise SkipTest(
- 'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
- 'fails due to a timeout when using Twisted versions lower '
- 'than 18.4.0'
- )
-
return super().test_send_catch_log()
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index a6d55cb38..0d5827339 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -4,10 +4,7 @@ Tests borrowed from the twisted.web.client tests.
"""
import os
import shutil
-import sys
-from pkg_resources import parse_version
-import cryptography
import OpenSSL.SSL
from twisted.trial import unittest
from twisted.web import server, static, util, resource
@@ -417,8 +414,6 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
).addCallback(self.assertEqual, to_bytes(s))
def testPayloadDisabledCipher(self):
- if sys.implementation.name == "pypy" and parse_version(cryptography.__version__) <= parse_version("2.3.1"):
- self.skipTest("This test expects a failure, but the code does work in PyPy with cryptography<=2.3.1")
s = "0123456789" * 10
settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'})
client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)
diff --git a/tox.ini b/tox.ini
index db151f215..ab8a715c2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,14 +11,13 @@ minversion = 1.7.0
deps =
-rtests/requirements.txt
# mitmproxy does not support PyPy
- # mitmproxy does not support Windows when running Python < 3.7
- # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe
+ # Python 3.9+ requires mitmproxy >= 5.3.0
# mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0
#mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
- mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy'
- mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
+ # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
+ mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
# newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower)
- markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy'
+ markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy'
# Extras
botocore>=1.4.87
passenv =
@@ -43,7 +42,6 @@ deps =
types-pyOpenSSL==20.0.3
types-setuptools==57.0.0
commands =
- pip install types-dataclasses # remove once py36 support is dropped
mypy --show-error-codes {posargs: scrapy tests}
[testenv:security]
@@ -74,18 +72,19 @@ commands =
[pinned]
deps =
- cryptography==2.0
+ cryptography==2.8
cssselect==0.9.1
h2==3.0
itemadapter==0.1.0
parsel==1.5.0
Protego==0.1.15
- pyOpenSSL==16.2.0
+ pyOpenSSL==19.1.0
queuelib==1.4.2
service_identity==16.0.0
- Twisted[http2]==17.9.0
+ Twisted[http2]==18.9.0
w3lib==1.17.0
- zope.interface==4.1.3
+ zope.interface==5.1.0
+ lxml==4.3.0
-rtests/requirements.txt
# mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
@@ -94,7 +93,7 @@ deps =
# Extras
botocore==1.4.87
google-cloud-storage==1.29.0
- Pillow==4.0.0
+ Pillow==7.1.0
setenv =
_SCRAPY_PINNED=true
install_command =
@@ -103,7 +102,6 @@ install_command =
[testenv:pinned]
deps =
{[pinned]deps}
- lxml==3.5.0
PyDispatcher==2.0.5
install_command = {[pinned]install_command}
setenv =
@@ -113,9 +111,6 @@ setenv =
basepython = python3
deps =
{[pinned]deps}
- # First lxml version that includes a Windows wheel for Python 3.6, so we do
- # not need to build lxml from sources in a CI Windows job:
- lxml==3.8.0
PyDispatcher==2.0.5
install_command = {[pinned]install_command}
setenv =
@@ -125,13 +120,14 @@ setenv =
deps =
{[testenv]deps}
boto
+ google-cloud-storage
+ # Twisted[http2] currently forces old mitmproxy because of h2 version
+ # restrictions in their deps, so we need to pin old markupsafe here too.
+ markupsafe < 2.1.0
reppy
robotexclusionrulesparser
Pillow>=4.0.0
Twisted[http2]>=17.9.0
- # Twisted[http2] currently forces old mitmproxy because of h2 version restrictions in their deps,
- # so we need to pin old markupsafe here too
- markupsafe < 2.1.0
[testenv:asyncio]
commands =
@@ -153,7 +149,6 @@ commands =
basepython = {[testenv:pypy3]basepython}
deps =
{[pinned]deps}
- lxml==4.0.0
PyPyDispatcher==2.1.0
commands = {[testenv:pypy3]commands}
install_command = {[pinned]install_command}
- Name: My image 2 
- Name: My image 3 
- Name: My image 4 
- Name: My image 5 
- 
-