diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 98fa44c7f..b26f344ff 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -19,7 +19,7 @@ jobs:
- python-version: 3.8
env:
TOXENV: pylint
- - python-version: 3.6
+ - python-version: 3.7
env:
TOXENV: typing
- python-version: "3.10" # Keep in sync with .readthedocs.yml
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 3aaf688c7..7819a4e12 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -7,7 +7,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+ python-version: ["3.7", "3.8", "3.9", "3.10"]
steps:
- uses: actions/checkout@v2
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 1fc8d914b..be40c7c71 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -8,9 +8,6 @@ jobs:
fail-fast: false
matrix:
include:
- - python-version: 3.7
- env:
- TOXENV: py
- python-version: 3.8
env:
TOXENV: py
@@ -26,19 +23,19 @@ jobs:
- python-version: pypy3
env:
TOXENV: pypy3
- PYPY_VERSION: 3.6-v7.3.3
+ PYPY_VERSION: 3.9-v7.3.9
# pinned deps
- - python-version: 3.6.12
+ - python-version: 3.7.13
env:
TOXENV: pinned
- - python-version: 3.6.12
+ - python-version: 3.7.13
env:
TOXENV: asyncio-pinned
- python-version: pypy3
env:
TOXENV: pypy3-pinned
- PYPY_VERSION: 3.6-v7.2.0
+ PYPY_VERSION: 3.7-v7.3.5
# extras
# extra-deps includes reppy, which does not support Python 3.9
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index ab7385118..955b9b449 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -8,12 +8,9 @@ jobs:
fail-fast: false
matrix:
include:
- - python-version: 3.6
- env:
- TOXENV: windows-pinned
- python-version: 3.7
env:
- TOXENV: py
+ TOXENV: windows-pinned
- python-version: 3.8
env:
TOXENV: py
diff --git a/README.rst b/README.rst
index 6b563d638..b543a30f4 100644
--- a/README.rst
+++ b/README.rst
@@ -57,7 +57,7 @@ including a list of features.
Requirements
============
-* Python 3.6+
+* Python 3.7+
* Works on Linux, Windows, macOS, BSD
Install
diff --git a/docs/README.rst b/docs/README.rst
index 0b7afa548..36dd5aea4 100644
--- a/docs/README.rst
+++ b/docs/README.rst
@@ -43,7 +43,7 @@ This command will fire up your default browser and open the main page of your
Start over
----------
-To cleanup all generated documentation files and start from scratch run::
+To clean up all generated documentation files and start from scratch run::
make clean
diff --git a/docs/_static/selectors-sample1.html b/docs/_static/selectors-sample1.html
index 8a79a3381..915718832 100644
--- a/docs/_static/selectors-sample1.html
+++ b/docs/_static/selectors-sample1.html
@@ -1,16 +1,17 @@
-
-
-
- Example website
-
-
-
-
-
+
+
+
+
+ Example website
+
+
+
+
+
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
index 55aa72d5a..3241295af 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -291,10 +291,12 @@ intersphinx_mapping = {
'pytest': ('https://docs.pytest.org/en/latest', None),
'python': ('https://docs.python.org/3', None),
'sphinx': ('https://www.sphinx-doc.org/en/master', None),
- 'tox': ('https://tox.readthedocs.io/en/latest', None),
- 'twisted': ('https://twistedmatrix.com/documents/current', None),
- 'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
+ 'tox': ('https://tox.wiki/en/latest/', None),
+ 'twisted': ('https://docs.twisted.org/en/stable/', None),
+ 'twistedapi': ('https://docs.twisted.org/en/stable/api/', None),
+ 'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
}
+intersphinx_disabled_reftypes = []
# Options for sphinx-hoverxref options
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 4d2580a6c..946bdc23e 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -232,15 +232,15 @@ To run a specific test (say ``tests/test_loader.py``) use:
To run the tests on a specific :doc:`tox ` environment, use
``-e `` with an environment name from ``tox.ini``. For example, to run
-the tests with Python 3.6 use::
+the tests with Python 3.7 use::
- tox -e py36
+ tox -e py37
You can also specify a comma-separated list of environments, and use :ref:`tox’s
parallel mode ` to run the tests on multiple environments in
parallel::
- tox -e py36,py38 -p auto
+ tox -e py37,py38 -p auto
To pass command-line options to :doc:`pytest `, add them after
``--`` in your call to :doc:`tox `. Using ``--`` overrides the
@@ -250,9 +250,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
tox -- scrapy tests -x # stop after first failure
You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
-the Python 3.6 :doc:`tox ` environment using all your CPU cores::
+the Python 3.7 :doc:`tox ` environment using all your CPU cores::
- tox -e py36 -- scrapy tests -n auto
+ tox -e py37 -- scrapy tests -n auto
To see coverage report install :doc:`coverage `
(``pip install coverage``) and run:
diff --git a/docs/index.rst b/docs/index.rst
index 6e22db884..ea4950e4c 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -130,7 +130,6 @@ Built-in services
topics/stats
topics/email
topics/telnetconsole
- topics/webservice
:doc:`topics/logging`
Learn how to use Python's builtin logging on Scrapy.
@@ -144,9 +143,6 @@ Built-in services
:doc:`topics/telnetconsole`
Inspect a running crawler using a built-in Python console.
-:doc:`topics/webservice`
- Monitor and control a crawler using a web service.
-
Solving specific problems
=========================
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index b8d3a16bc..80a9c16d6 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -9,8 +9,8 @@ Installation guide
Supported Python versions
=========================
-Scrapy requires Python 3.6+, either the CPython implementation (default) or
-the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
+Scrapy requires Python 3.7+, either the CPython implementation (default) or
+the PyPy 7.3.5+ implementation (see :ref:`python:implementations`).
.. _intro-install-scrapy:
@@ -52,16 +52,6 @@ Scrapy is written in pure Python and depends on a few key Python packages (among
* `twisted`_, an asynchronous networking framework
* `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs
-The minimal versions which Scrapy is tested against are:
-
-* Twisted 14.0
-* lxml 3.4
-* pyOpenSSL 0.14
-
-Scrapy may work with older versions of these packages
-but it is not guaranteed it will continue working
-because it’s not being tested against them.
-
Some of these packages themselves depends on non-Python packages
that might require additional installation steps depending on your platform.
Please check :ref:`platform-specific guides below `.
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index f3d652621..cfa6bfa83 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -45,9 +45,9 @@ https://quotes.toscrape.com, following the pagination::
Put this in a text file, name it to something like ``quotes_spider.py``
and run the spider using the :command:`runspider` command::
- scrapy runspider quotes_spider.py -o quotes.jl
+ scrapy runspider quotes_spider.py -o quotes.jsonl
-When this finishes you will have in the ``quotes.jl`` file a list of the
+When this finishes you will have in the ``quotes.jsonl`` file a list of the
quotes in JSON Lines format, containing text and author, looking like this::
{"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index cde1b1ef4..75928077e 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -482,7 +482,7 @@ to append new content to any existing file. However, appending to a JSON file
makes the file contents invalid JSON. When appending to a file, consider
using a different serialization format, such as `JSON Lines`_::
- scrapy crawl quotes -o quotes.jl
+ scrapy crawl quotes -o quotes.jsonl
The `JSON Lines`_ format is useful because it's stream-like, you can easily
append new records to it. It doesn't have the same problem of JSON when you run
diff --git a/docs/news.rst b/docs/news.rst
index 5d92067b5..2d0ab485e 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -1643,7 +1643,7 @@ New features
:issue:`4370`)
* A new ``keep_fragments`` parameter of
- :func:`scrapy.utils.request.request_fingerprint` allows to generate
+ ``scrapy.utils.request.request_fingerprint`` allows to generate
different fingerprints for requests with different fragments in their URL
(:issue:`4104`)
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 900b19c7a..60b5acd10 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -32,6 +32,13 @@ how you :ref:`configure the downloader middlewares
:class:`scrapy.Spider` subclass and a
:class:`scrapy.settings.Settings` object.
+ .. attribute:: request_fingerprinter
+
+ The request fingerprint builder of this crawler.
+
+ This is used from extensions and middlewares to build short, unique
+ identifiers for requests. See :ref:`request-fingerprints`.
+
.. attribute:: settings
The settings manager of this crawler.
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 55d013c06..750263385 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -1,3 +1,5 @@
+.. _topics-coroutines:
+
==========
Coroutines
==========
diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index 96475899f..9bf97c628 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -19,14 +19,14 @@ Caveats with inspecting the live browser DOM
Since Developer Tools operate on a live browser DOM, what you'll actually see
when inspecting the page source is not the original HTML, but a modified one
-after applying some browser clean up and executing Javascript code. Firefox,
+after applying some browser clean up and executing JavaScript code. Firefox,
in particular, is known for adding ```` elements to tables. Scrapy, on
the other hand, does not modify the original page HTML, so you won't be able to
extract any data if you use ```` in your XPath expressions.
Therefore, you should keep in mind the following things:
-* Disable Javascript while inspecting the DOM looking for XPaths to be
+* Disable JavaScript while inspecting the DOM looking for XPaths to be
used in Scrapy (in the Developer Tools settings click `Disable JavaScript`)
* Never use full XPath paths, use relative and clever ones based on attributes
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index a15637ed6..986da0476 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -366,7 +366,7 @@ HttpCacheMiddleware
This middleware provides low-level cache to all HTTP requests and responses.
It has to be combined with a cache storage backend as well as a cache policy.
- Scrapy ships with three HTTP cache storage backends:
+ Scrapy ships with the following HTTP cache storage backends:
* :ref:`httpcache-storage-fs`
* :ref:`httpcache-storage-dbm`
@@ -704,14 +704,15 @@ HttpCompressionMiddleware
sent/received from web sites.
This middleware also supports decoding `brotli-compressed`_ as well as
- `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is
+ `zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is
installed, respectively.
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
-.. _brotlipy: https://pypi.org/project/brotlipy/
+.. _brotli: https://pypi.org/project/Brotli/
.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
.. _zstandard: https://pypi.org/project/zstandard/
+
HttpCompressionMiddleware Settings
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -954,7 +955,7 @@ default because HTTP specs say so.
.. setting:: RETRY_PRIORITY_ADJUST
RETRY_PRIORITY_ADJUST
----------------------
+^^^^^^^^^^^^^^^^^^^^^
Default: ``-1``
@@ -1118,7 +1119,7 @@ In order to use this parser:
.. _support-for-new-robots-parser:
Implementing support for a new parser
--------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
You can implement support for a new robots.txt_ parser by subclassing
the abstract base class :class:`~scrapy.robotstxt.RobotParser` and
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 923336769..9360ecf37 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -117,7 +117,7 @@ after your custom code.
Example::
- from scrapy.exporter import XmlItemExporter
+ from scrapy.exporters import XmlItemExporter
class ProductXmlExporter(XmlItemExporter):
@@ -195,17 +195,25 @@ BaseItemExporter
.. attribute:: fields_to_export
- A list with the name of the fields that will be exported, or ``None`` if
- you want to export all fields. Defaults to ``None``.
+ Fields to export, their order [1]_ and their output names.
- Some exporters (like :class:`CsvItemExporter`) respect the order of the
- fields defined in this attribute.
+ Possible values are:
- When using :ref:`item objects ` that do not expose all their
- possible fields, exporters that do not support exporting a different
- subset of fields per item will only export the fields found in the first
- item exported. Use ``fields_to_export`` to define all the fields to be
- exported.
+ - ``None`` (all fields [2]_, default)
+
+ - A list of fields::
+
+ ['field1', 'field2']
+
+ - A dict where keys are fields and values are output names::
+
+ {'field1': 'Field 1', 'field2': 'Field 2'}
+
+ .. [1] Not all exporters respect the specified field order.
+ .. [2] When using :ref:`item objects ` that do not expose
+ all their possible fields, exporters that do not support exporting
+ a different subset of fields per item will only export the fields
+ found in the first item exported.
.. attribute:: export_empty_fields
@@ -297,8 +305,8 @@ CsvItemExporter
Exports items in CSV format to the given file-like object. If the
:attr:`fields_to_export` attribute is set, it will be used to define the
- CSV columns and their order. The :attr:`export_empty_fields` attribute has
- no effect on this exporter.
+ CSV columns, their order and their column names. The
+ :attr:`export_empty_fields` attribute has no effect on this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 9a13eb82f..a620e2c04 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -58,7 +58,7 @@ CSV
- Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
-- To specify columns to export and their order use
+- To specify columns to export, their order and their column names, use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
option, but it is important for CSV because unlike many other export
formats CSV uses a fixed header.
@@ -522,18 +522,9 @@ FEED_EXPORT_FIELDS
Default: ``None``
-A list of fields to export, optional.
-Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
-
-Use FEED_EXPORT_FIELDS option to define fields to export and their order.
-
-When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses the fields
-defined in :ref:`item objects ` yielded by your spider.
-
-If an exporter requires a fixed set of fields (this is the case for
-:ref:`CSV ` export format) and FEED_EXPORT_FIELDS
-is empty or None, then Scrapy tries to infer field names from the
-exported data - currently it uses field names from the first item.
+Use the ``FEED_EXPORT_FIELDS`` setting to define the fields to export, their
+order and their output names. See :attr:`BaseItemExporter.fields_to_export
+` for more information.
.. setting:: FEED_EXPORT_INDENT
@@ -638,6 +629,7 @@ Default::
{
'json': 'scrapy.exporters.JsonItemExporter',
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
+ 'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
'jl': 'scrapy.exporters.JsonLinesItemExporter',
'csv': 'scrapy.exporters.CsvItemExporter',
'xml': 'scrapy.exporters.XmlItemExporter',
@@ -763,7 +755,7 @@ source spider in the feed URI:
#. Use ``%(spider_name)s`` in your feed URI::
- scrapy crawl -o "%(spider_name)s.jl"
+ scrapy crawl -o "%(spider_name)s.jsonl"
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 391751364..af294f52c 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -60,9 +60,9 @@ Additionally, they may also implement the following methods:
:param spider: the spider which was closed
:type spider: :class:`~scrapy.Spider` object
-.. method:: from_crawler(cls, crawler)
+.. classmethod:: from_crawler(cls, crawler)
- If present, this classmethod is called to create a pipeline instance
+ If present, this class method is called to create a pipeline instance
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
of the pipeline. Crawler object provides access to all Scrapy core
components like settings and signals; it is a way for pipeline to
@@ -99,11 +99,11 @@ contain a price::
raise DropItem(f"Missing price in {item}")
-Write items to a JSON file
---------------------------
+Write items to a JSON lines file
+--------------------------------
The following pipeline stores all scraped items (from all spiders) into a
-single ``items.jl`` file, containing one item per line serialized in JSON
+single ``items.jsonl`` file, containing one item per line serialized in JSON
format::
import json
@@ -113,7 +113,7 @@ format::
class JsonWriterPipeline:
def open_spider(self, spider):
- self.file = open('items.jl', 'w')
+ self.file = open('items.jsonl', 'w')
def close_spider(self, spider):
self.file.close()
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 7cd482d07..167014381 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -102,11 +102,6 @@ Additionally, ``dataclass`` items also allow to:
* define custom field metadata through :func:`dataclasses.field`, which can be used to
:ref:`customize serialization `.
-They work natively in Python 3.7 or later, or using the `dataclasses
-backport`_ in Python 3.6.
-
-.. _dataclasses backport: https://pypi.org/project/dataclasses/
-
Example::
from dataclasses import dataclass
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 7dff78390..0925e6bb5 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
can configure some extra functions like generating thumbnails and filtering
the images based on their size.
-The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for
+The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for
thumbnailing and normalizing images to JPEG/RGB format.
.. _Pillow: https://github.com/python-pillow/Pillow
@@ -656,6 +656,26 @@ See here the methods that you can override in your custom Images Pipeline:
.. versionadded:: 2.4
The *item* parameter.
+ .. method:: ImagesPipeline.thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)
+
+ This method is called for every item of :setting:`IMAGES_THUMBS` per downloaded item. It returns the
+ thumbnail download path of the image originating from the specified
+ :class:`response `.
+
+ In addition to ``response``, this method receives the original
+ :class:`request `,
+ ``thumb_id``,
+ :class:`info ` and
+ :class:`item `.
+
+ You can override this method to customize the thumbnail download path of each image.
+ You can use the ``item`` to determine the file path based on some item
+ property.
+
+ By default the :meth:`thumb_path` method returns
+ ``thumbs//.``.
+
+
.. method:: ImagesPipeline.get_media_requests(item, info)
Works the same way as :meth:`FilesPipeline.get_media_requests` method,
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index d0207fd18..7313c9246 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -180,8 +180,8 @@ Same example but running the spiders sequentially by chaining the deferreds:
# Your second spider definition
...
- configure_logging()
settings = get_project_settings()
+ configure_logging(settings)
runner = CrawlerRunner(settings)
@defer.inlineCallbacks
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 92a471faf..49cb69f67 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -339,6 +339,7 @@ errors if needed::
request = failure.request
self.logger.error('TimeoutError on %s', request.url)
+
.. _errback-cb_kwargs:
Accessing additional data in errback functions
@@ -364,6 +365,273 @@ achieve this by using ``Failure.request.cb_kwargs``::
main_url=failure.request.cb_kwargs['main_url'],
)
+
+.. _request-fingerprints:
+
+Request fingerprints
+--------------------
+
+There are some aspects of scraping, such as filtering out duplicate requests
+(see :setting:`DUPEFILTER_CLASS`) or caching responses (see
+:setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short,
+unique identifier from a :class:`~scrapy.http.Request` object: a request
+fingerprint.
+
+You often do not need to worry about request fingerprints, the default request
+fingerprinter works for most projects.
+
+However, there is no universal way to generate a unique identifier from a
+request, because different situations require comparing requests differently.
+For example, sometimes you may need to compare URLs case-insensitively, include
+URL fragments, exclude certain URL query parameters, include some or all
+headers, etc.
+
+To change how request fingerprints are built for your requests, use the
+:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
+
+.. setting:: REQUEST_FINGERPRINTER_CLASS
+
+REQUEST_FINGERPRINTER_CLASS
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: VERSION
+
+Default: :class:`scrapy.utils.request.RequestFingerprinter`
+
+A :ref:`request fingerprinter class ` or its
+import path.
+
+.. autoclass:: scrapy.utils.request.RequestFingerprinter
+
+
+.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
+
+REQUEST_FINGERPRINTER_IMPLEMENTATION
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: VERSION
+
+Default: ``'PREVIOUS_VERSION'``
+
+Determines which request fingerprinting algorithm is used by the default
+request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
+
+Possible values are:
+
+- ``'PREVIOUS_VERSION'`` (default)
+
+ This implementation uses the same request fingerprinting algorithm as
+ Scrapy PREVIOUS_VERSION and earlier versions.
+
+ Even though this is the default value for backward compatibility reasons,
+ it is a deprecated value.
+
+- ``'VERSION'``
+
+ This implementation was introduced in Scrapy VERSION to fix an issue of the
+ previous implementation.
+
+ New projects should use this value. The :command:`startproject` command
+ sets this value in the generated ``settings.py`` file.
+
+If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are
+using Scrapy components where changing the request fingerprinting algorithm
+would cause undesired results, you need to carefully decide when to change the
+value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
+setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request
+fingerprinting algorithm and does not log this warning (
+:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a
+class).
+
+Scenarios where changing the request fingerprinting algorithm may cause
+undesired results include, for example, using the HTTP cache middleware (see
+:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
+Changing the request fingerprinting algorithm would invalidade the current
+cache, requiring you to redownload all requests again.
+
+Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in
+your settings to switch already to the request fingerprinting implementation
+that will be the only request fingerprinting implementation available in a
+future version of Scrapy, and remove the deprecation warning triggered by using
+the default value (``'PREVIOUS_VERSION'``).
+
+
+.. _PREVIOUS_VERSION-request-fingerprinter:
+.. _custom-request-fingerprinter:
+
+Writing your own request fingerprinter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A request fingerprinter is a class that must implement the following method:
+
+.. method:: fingerprint(self, request)
+
+ Return a :class:`bytes` object that uniquely identifies *request*.
+
+ See also :ref:`request-fingerprint-restrictions`.
+
+ :param request: request to fingerprint
+ :type request: scrapy.http.Request
+
+Additionally, it may also implement the following methods:
+
+.. classmethod:: from_crawler(cls, crawler)
+
+ If present, this class method is called to create a request fingerprinter
+ instance from a :class:`~scrapy.crawler.Crawler` object. It must return a
+ new instance of the request fingerprinter.
+
+ *crawler* provides access to all Scrapy core components like settings and
+ signals; it is a way for the request fingerprinter to access them and hook
+ its functionality into Scrapy.
+
+ :param crawler: crawler that uses this request fingerprinter
+ :type crawler: :class:`~scrapy.crawler.Crawler` object
+
+.. classmethod:: from_settings(cls, settings)
+
+ If present, and ``from_crawler`` is not defined, this class method is called
+ to create a request fingerprinter instance from a
+ :class:`~scrapy.settings.Settings` object. It must return a new instance of
+ the request fingerprinter.
+
+The ``fingerprint`` method of the default request fingerprinter,
+:class:`scrapy.utils.request.RequestFingerprinter`, uses
+:func:`scrapy.utils.request.fingerprint` with its default parameters. For some
+common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well
+in your ``fingerprint`` method implementation:
+
+.. autofunction:: scrapy.utils.request.fingerprint
+
+For example, to take the value of a request header named ``X-ID`` into
+account::
+
+ # my_project/settings.py
+ REQUEST_FINGERPRINTER_CLASS = 'my_project.utils.RequestFingerprinter'
+
+ # my_project/utils.py
+ from scrapy.utils.request import fingerprint
+
+ class RequestFingerprinter:
+
+ def fingerprint(self, request):
+ return fingerprint(request, include_headers=['X-ID'])
+
+You can also write your own fingerprinting logic from scratch.
+
+However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure
+you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
+
+- Caching saves CPU by ensuring that fingerprints are calculated only once
+ per request, and not once per Scrapy component that needs the fingerprint
+ of a request.
+
+- Using :class:`~weakref.WeakKeyDictionary` saves memory by ensuring that
+ request objects do not stay in memory forever just because you have
+ references to them in your cache dictionary.
+
+For example, to take into account only the URL of a request, without any prior
+URL canonicalization or taking the request method or body into account::
+
+ from hashlib import sha1
+ from weakref import WeakKeyDictionary
+
+ from scrapy.utils.python import to_bytes
+
+ class RequestFingerprinter:
+
+ cache = WeakKeyDictionary()
+
+ def fingerprint(self, request):
+ if request not in self.cache:
+ fp = sha1()
+ fp.update(to_bytes(request.url))
+ self.cache[request] = fp.digest()
+ return self.cache[request]
+
+If you need to be able to override the request fingerprinting for arbitrary
+requests from your spider callbacks, you may implement a request fingerprinter
+that reads fingerprints from :attr:`request.meta `
+when available, and then falls back to
+:func:`~scrapy.utils.request.fingerprint`. For example::
+
+ from scrapy.utils.request import fingerprint
+
+ class RequestFingerprinter:
+
+ def fingerprint(self, request):
+ if 'fingerprint' in request.meta:
+ return request.meta['fingerprint']
+ return fingerprint(request)
+
+If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION
+without using the deprecated ``'PREVIOUS_VERSION'`` value of the
+:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
+request fingerprinter::
+
+ from hashlib import sha1
+ from weakref import WeakKeyDictionary
+
+ from scrapy.utils.python import to_bytes
+ from w3lib.url import canonicalize_url
+
+ class RequestFingerprinter:
+
+ cache = WeakKeyDictionary()
+
+ def fingerprint(self, request):
+ if request not in self.cache:
+ fp = sha1()
+ fp.update(to_bytes(request.method))
+ fp.update(to_bytes(canonicalize_url(request.url)))
+ fp.update(request.body or b'')
+ self.cache[request] = fp.digest()
+ return self.cache[request]
+
+
+.. _request-fingerprint-restrictions:
+
+Request fingerprint restrictions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Scrapy components that use request fingerprints may impose additional
+restrictions on the format of the fingerprints that your :ref:`request
+fingerprinter ` generates.
+
+The following built-in Scrapy components have such restrictions:
+
+- :class:`scrapy.extensions.httpcache.FilesystemCacheStorage` (default
+ value of :setting:`HTTPCACHE_STORAGE`)
+
+ Request fingerprints must be at least 1 byte long.
+
+ Path and filename length limits of the file system of
+ :setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`,
+ the following directory structure is created:
+
+ - :attr:`Spider.name `
+
+ - first byte of a request fingerprint as hexadecimal
+
+ - fingerprint as hexadecimal
+
+ - filenames up to 16 characters long
+
+ For example, if a request fingerprint is made of 20 bytes (default),
+ :setting:`HTTPCACHE_DIR` is ``'/home/user/project/.scrapy/httpcache'``,
+ and the name of your spider is ``'my_spider'`` your file system must
+ support a file path like::
+
+ /home/user/project/.scrapy/httpcache/my_spider/01/0123456789abcdef0123456789abcdef01234567/response_headers
+
+- :class:`scrapy.extensions.httpcache.DbmCacheStorage`
+
+ The underlying DBM implementation must support keys as long as twice
+ the number of bytes of a request fingerprint, plus 5. For example,
+ if a request fingerprint is made of 20 bytes (default),
+ 45-character-long keys must be supported.
+
+
.. _topics-request-meta:
Request.meta special keys
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 4e105642d..2046c6446 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -825,12 +825,8 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'``
The class used to detect and filter duplicate requests.
-The default (``RFPDupeFilter``) filters based on request fingerprint using
-the ``scrapy.utils.request.request_fingerprint`` function. In order to change
-the way duplicates are checked you could subclass ``RFPDupeFilter`` and
-override its ``request_fingerprint`` method. This method should accept
-scrapy :class:`~scrapy.Request` object and return its fingerprint
-(a string).
+The default (``RFPDupeFilter``) filters based on the
+:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
You can disable filtering of duplicate requests by setting
:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 2fbd0b51c..17bd16156 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -51,12 +51,12 @@ Deferred signal handlers
========================
Some signals support returning :class:`~twisted.internet.defer.Deferred`
-objects from their handlers, allowing you to run asynchronous code that
-does not block Scrapy. If a signal handler returns a
-:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that
-:class:`~twisted.internet.defer.Deferred` to fire.
+or :term:`awaitable objects ` from their handlers, allowing
+you to run asynchronous code that does not block Scrapy. If a signal
+handler returns one of these objects, Scrapy waits for that asynchronous
+operation to finish.
-Let's take an example::
+Let's take an example using :ref:`coroutines `::
class SignalSpider(scrapy.Spider):
name = 'signals'
@@ -68,17 +68,15 @@ Let's take an example::
crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
return spider
- def item_scraped(self, item):
+ async def item_scraped(self, item):
# Send the scraped item to the server
- d = treq.post(
+ response = await treq.post(
'http://example.com/post',
json.dumps(item).encode('ascii'),
headers={b'Content-Type': [b'application/json']}
)
- # The next item will be scraped only after
- # deferred (d) is fired
- return d
+ return response
def parse(self, response):
for quote in response.css('div.quote'):
@@ -89,7 +87,7 @@ Let's take an example::
}
See the :ref:`topics-signals-ref` below to know which signals support
-:class:`~twisted.internet.defer.Deferred`.
+:class:`~twisted.internet.defer.Deferred` and :term:`awaitable objects `.
.. _topics-signals-ref:
diff --git a/docs/topics/webservice.rst b/docs/topics/webservice.rst
deleted file mode 100644
index 2c4052c04..000000000
--- a/docs/topics/webservice.rst
+++ /dev/null
@@ -1,11 +0,0 @@
-.. _topics-webservice:
-
-===========
-Web Service
-===========
-
-webservice has been moved into a separate project.
-
-It is hosted at:
-
- https://github.com/scrapy-plugins/scrapy-jsonrpc
diff --git a/pytest.ini b/pytest.ini
index ae2ed2029..af0f2fb6e 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -21,3 +21,6 @@ addopts =
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
+filterwarnings =
+ ignore:scrapy.downloadermiddlewares.decompression is deprecated
+ ignore:Module scrapy.utils.reqser is deprecated
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 396f98219..86e584396 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Check minimum required Python version
-if sys.version_info < (3, 6):
- print(f"Scrapy {__version__} requires Python 3.6+")
+if sys.version_info < (3, 7):
+ print(f"Scrapy {__version__} requires Python 3.7+")
sys.exit(1)
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index a3f6b96f4..99fc8f955 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -146,7 +146,8 @@ class Command(BaseRunSpiderCommand):
def _start_requests(spider):
yield self.prepare_request(spider, Request(url), opts)
- self.spidercls.start_requests = _start_requests
+ if self.spidercls:
+ self.spidercls.start_requests = _start_requests
def start_parsing(self, url, opts):
self.crawler_process.crawl(self.spidercls, **opts.spargs)
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 3ef129587..a495874bd 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -102,11 +102,11 @@ class FTPDownloadHandler:
def _build_response(self, result, request, protocol):
self.result = result
- respcls = responsetypes.from_args(url=request.url)
protocol.close()
- body = protocol.filename or protocol.body.read()
headers = {"local filename": protocol.filename or '', "size": protocol.size}
- return respcls(url=request.url, status=200, body=to_bytes(body), headers=headers)
+ body = to_bytes(protocol.filename or protocol.body.read())
+ respcls = responsetypes.from_args(url=request.url, body=body)
+ return respcls(url=request.url, status=200, body=body, headers=headers)
def _failed(self, result, request):
message = result.getErrorMessage()
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 06cb96489..7d048c1e4 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -112,7 +112,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
request.meta['download_latency'] = self.headers_time - self.start_time
status = int(self.status)
headers = Headers(self.response_headers)
- respcls = responsetypes.from_args(headers=headers, url=self._url)
+ respcls = responsetypes.from_args(headers=headers, url=self._url, body=body)
return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))
def _set_connection_attributes(self, request):
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index a638254f1..fdca7b335 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -51,6 +51,7 @@ class Crawler:
self.spidercls.update_settings(self.settings)
self.signals = SignalManager(self)
+
self.stats = load_object(self.settings['STATS_CLASS'])(self)
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
@@ -71,6 +72,12 @@ class Crawler:
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
+ self.request_fingerprinter = create_instance(
+ load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
+ settings=self.settings,
+ crawler=self,
+ )
+
reactor_class = self.settings.get("TWISTED_REACTOR")
if init_reactor:
# this needs to be done after the spider settings are merged,
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 3afa06077..c592acb57 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -104,8 +104,8 @@ class CookiesMiddleware:
for key in ("name", "value", "path", "domain"):
if cookie.get(key) is None:
if key in ("name", "value"):
- msg = "Invalid cookie found in request {}: {} ('{}' is missing)"
- logger.warning(msg.format(request, cookie, key))
+ msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
+ logger.warning(msg)
return
continue
if isinstance(cookie[key], (bool, float, int, str)):
diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index 0fcf8fb8c..e01e9cc76 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -9,10 +9,19 @@ import tarfile
import zipfile
from io import BytesIO
from tempfile import mktemp
+from warnings import warn
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.responsetypes import responsetypes
+warn(
+ 'scrapy.downloadermiddlewares.decompression is deprecated',
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+)
+
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 292c68099..d1b0559ef 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -1,14 +1,16 @@
import logging
import os
from typing import Optional, Set, Type, TypeVar
+from warnings import warn
from twisted.internet.defer import Deferred
from scrapy.http.request import Request
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.job import job_dir
-from scrapy.utils.request import referer_str, request_fingerprint
+from scrapy.utils.request import referer_str, RequestFingerprinter
BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")
@@ -39,8 +41,15 @@ RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter")
class RFPDupeFilter(BaseDupeFilter):
"""Request Fingerprint duplicates filter"""
- def __init__(self, path: Optional[str] = None, debug: bool = False) -> None:
+ def __init__(
+ self,
+ path: Optional[str] = None,
+ debug: bool = False,
+ *,
+ fingerprinter=None,
+ ) -> None:
self.file = None
+ self.fingerprinter = fingerprinter or RequestFingerprinter()
self.fingerprints: Set[str] = set()
self.logdupes = True
self.debug = debug
@@ -51,9 +60,39 @@ class RFPDupeFilter(BaseDupeFilter):
self.fingerprints.update(x.rstrip() for x in self.file)
@classmethod
- def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings) -> RFPDupeFilterTV:
+ def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None) -> RFPDupeFilterTV:
debug = settings.getbool('DUPEFILTER_DEBUG')
- return cls(job_dir(settings), debug)
+ try:
+ return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
+ except TypeError:
+ warn(
+ "RFPDupeFilter subclasses must either modify their '__init__' "
+ "method to support a 'fingerprinter' parameter or reimplement "
+ "the 'from_settings' class method.",
+ ScrapyDeprecationWarning,
+ )
+ result = cls(job_dir(settings), debug)
+ result.fingerprinter = fingerprinter
+ return result
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ try:
+ return cls.from_settings(
+ crawler.settings,
+ fingerprinter=crawler.request_fingerprinter,
+ )
+ except TypeError:
+ warn(
+ "RFPDupeFilter subclasses must either modify their overridden "
+ "'__init__' method and 'from_settings' class method to "
+ "support a 'fingerprinter' parameter, or reimplement the "
+ "'from_crawler' class method.",
+ ScrapyDeprecationWarning,
+ )
+ result = cls.from_settings(crawler.settings)
+ result.fingerprinter = crawler.request_fingerprinter
+ return result
def request_seen(self, request: Request) -> bool:
fp = self.request_fingerprint(request)
@@ -65,7 +104,7 @@ class RFPDupeFilter(BaseDupeFilter):
return False
def request_fingerprint(self, request: Request) -> str:
- return request_fingerprint(request)
+ return self.fingerprinter.fingerprint(request).hex()
def close(self, reason: str) -> None:
if self.file:
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 1c26e81db..76cbe4d4b 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -8,6 +8,7 @@ import marshal
import pickle
import pprint
import warnings
+from collections.abc import Mapping
from xml.sax.saxutils import XMLGenerator
from itemadapter import is_item, ItemAdapter
@@ -68,6 +69,14 @@ class BaseItemExporter:
field_iter = item.field_names()
else:
field_iter = item.keys()
+ elif isinstance(self.fields_to_export, Mapping):
+ if include_empty:
+ field_iter = self.fields_to_export.items()
+ else:
+ field_iter = (
+ (x, y) for x, y in self.fields_to_export.items()
+ if x in item
+ )
else:
if include_empty:
field_iter = self.fields_to_export
@@ -75,13 +84,17 @@ class BaseItemExporter:
field_iter = (x for x in self.fields_to_export if x in item)
for field_name in field_iter:
- if field_name in item:
- field_meta = item.get_field_meta(field_name)
- value = self.serialize_field(field_meta, field_name, item[field_name])
+ if isinstance(field_name, str):
+ item_field, output_field = field_name, field_name
+ else:
+ item_field, output_field = field_name
+ if item_field in item:
+ field_meta = item.get_field_meta(item_field)
+ value = self.serialize_field(field_meta, output_field, item[item_field])
else:
value = default_value
- yield field_name, value
+ yield output_field, value
class JsonLinesItemExporter(BaseItemExporter):
@@ -246,7 +259,11 @@ class CsvItemExporter(BaseItemExporter):
if not self.fields_to_export:
# use declared field names, or keys if the item is a dict
self.fields_to_export = ItemAdapter(item).field_names()
- row = list(self._build_row(self.fields_to_export))
+ if isinstance(self.fields_to_export, Mapping):
+ fields = self.fields_to_export.values()
+ else:
+ fields = self.fields_to_export
+ row = list(self._build_row(fields))
self.csv_writer.writerow(row)
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index d0ae29b90..843e14812 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -14,7 +14,6 @@ from scrapy.responsetypes import responsetypes
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.project import data_path
from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.request import request_fingerprint
logger = logging.getLogger(__name__)
@@ -228,6 +227,8 @@ class DbmCacheStorage:
logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
+ self._fingerprinter = spider.crawler.request_fingerprinter
+
def close_spider(self, spider):
self.db.close()
@@ -239,12 +240,12 @@ class DbmCacheStorage:
status = data['status']
headers = Headers(data['headers'])
body = data['body']
- respcls = responsetypes.from_args(headers=headers, url=url)
+ respcls = responsetypes.from_args(headers=headers, url=url, body=body)
response = respcls(url=url, headers=headers, status=status, body=body)
return response
def store_response(self, spider, request, response):
- key = self._request_key(request)
+ key = self._fingerprinter.fingerprint(request).hex()
data = {
'status': response.status,
'url': response.url,
@@ -255,7 +256,7 @@ class DbmCacheStorage:
self.db[f'{key}_time'] = str(time())
def _read_data(self, spider, request):
- key = self._request_key(request)
+ key = self._fingerprinter.fingerprint(request).hex()
db = self.db
tkey = f'{key}_time'
if tkey not in db:
@@ -267,9 +268,6 @@ class DbmCacheStorage:
return pickle.loads(db[f'{key}_data'])
- def _request_key(self, request):
- return request_fingerprint(request)
-
class FilesystemCacheStorage:
@@ -283,6 +281,8 @@ class FilesystemCacheStorage:
logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
extra={'spider': spider})
+ self._fingerprinter = spider.crawler.request_fingerprinter
+
def close_spider(self, spider):
pass
@@ -299,7 +299,7 @@ class FilesystemCacheStorage:
url = metadata.get('response_url')
status = metadata['status']
headers = Headers(headers_raw_to_dict(rawheaders))
- respcls = responsetypes.from_args(headers=headers, url=url)
+ respcls = responsetypes.from_args(headers=headers, url=url, body=body)
response = respcls(url=url, headers=headers, status=status, body=body)
return response
@@ -329,7 +329,7 @@ class FilesystemCacheStorage:
f.write(request.body)
def _get_request_path(self, spider, request):
- key = request_fingerprint(request)
+ key = self._fingerprinter.fingerprint(request).hex()
return os.path.join(self.cachedir, spider.name, key[0:2], key)
def _read_meta(self, spider, request):
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 9de119a10..f5081a7d7 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -33,8 +33,8 @@ class MemoryUsage:
self.crawler = crawler
self.warned = False
self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')
- self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
- self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
+ self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024
+ self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024
self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
self.mail = MailSender.from_settings(crawler.settings)
crawler.signals.connect(self.engine_started, signal=signals.engine_started)
@@ -77,7 +77,7 @@ class MemoryUsage:
def _check_limit(self):
if self.get_virtual_size() > self.limit:
self.crawler.stats.set_value('memusage/limit_reached', 1)
- mem = self.limit/1024/1024
+ mem = self.limit / 1024 / 1024
logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
{'memusage': mem}, extra={'crawler': self.crawler})
if self.notify_mails:
@@ -94,11 +94,11 @@ class MemoryUsage:
self.crawler.stop()
def _check_warning(self):
- if self.warned: # warn only once
+ if self.warned: # warn only once
return
if self.get_virtual_size() > self.warning:
self.crawler.stats.set_value('memusage/warning_reached', 1)
- mem = self.warning/1024/1024
+ mem = self.warning / 1024 / 1024
logger.warning("Memory usage reached %(memusage)dM",
{'memusage': mem}, extra={'crawler': self.crawler})
if self.notify_mails:
diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py
index bcdbaff24..739e6b958 100644
--- a/scrapy/extensions/statsmailer.py
+++ b/scrapy/extensions/statsmailer.py
@@ -8,6 +8,7 @@ from scrapy import signals
from scrapy.mail import MailSender
from scrapy.exceptions import NotConfigured
+
class StatsMailer:
def __init__(self, stats, recipients, mail):
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 1a2b99b0a..9c03fe54f 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,3 +1,5 @@
+from collections.abc import Mapping
+
from w3lib.http import headers_dict_to_raw
from scrapy.utils.datatypes import CaselessDict
from scrapy.utils.python import to_unicode
@@ -10,6 +12,13 @@ class Headers(CaselessDict):
self.encoding = encoding
super().__init__(seq)
+ def update(self, seq):
+ seq = seq.items() if isinstance(seq, Mapping) else seq
+ iseq = {}
+ for k, v in seq:
+ iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
+ super().update(iseq)
+
def normkey(self, key):
"""Normalize key to bytes"""
return self._tobytes(key.title())
@@ -86,4 +95,5 @@ class Headers(CaselessDict):
def __copy__(self):
return self.__class__(self)
+
copy = __copy__
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 5c52c6c28..906e7eb24 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -222,8 +222,8 @@ class GCSFilesStore:
return {'checksum': checksum, 'last_modified': last_modified}
else:
return {}
-
- return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess)
+ blob_path = self._get_blob_path(path)
+ return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess)
def _get_content_type(self, headers):
if headers and 'Content-Type' in headers:
@@ -231,8 +231,12 @@ class GCSFilesStore:
else:
return 'application/octet-stream'
+ def _get_blob_path(self, path):
+ return self.prefix + path
+
def persist_file(self, path, buf, info, meta=None, headers=None):
- blob = self.bucket.blob(self.prefix + path)
+ blob_path = self._get_blob_path(path)
+ blob = self.bucket.blob(blob_path)
blob.cache_control = self.CACHE_CONTROL
blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
return threads.deferToThread(
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 9c99dc69e..6b97190ee 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -141,7 +141,7 @@ class ImagesPipeline(FilesPipeline):
yield path, image, buf
for thumb_id, size in self.thumbs.items():
- thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
+ thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item)
thumb_image, thumb_buf = self.convert_image(image, size)
yield thumb_path, thumb_image, thumb_buf
@@ -179,6 +179,6 @@ class ImagesPipeline(FilesPipeline):
image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
return f'full/{image_guid}.jpg'
- def thumb_path(self, request, thumb_id, response=None, info=None):
+ def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
return f'thumbs/{thumb_id}/{thumb_guid}.jpg'
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index d1bccf323..5308a9793 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -11,7 +11,6 @@ from scrapy.settings import Settings
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.defer import mustbe_deferred, defer_result
from scrapy.utils.deprecate import ScrapyDeprecationWarning
-from scrapy.utils.request import request_fingerprint
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.log import failure_to_exc_info
@@ -77,6 +76,7 @@ class MediaPipeline:
except AttributeError:
pipe = cls()
pipe.crawler = crawler
+ pipe._fingerprinter = crawler.request_fingerprinter
return pipe
def open_spider(self, spider):
@@ -90,7 +90,7 @@ class MediaPipeline:
return dfd.addCallback(self.item_completed, item, info)
def _process_request(self, request, info, item):
- fp = request_fingerprint(request)
+ fp = self._fingerprinter.fingerprint(request)
cb = request.callback or (lambda _: _)
eb = request.errback
request.callback = None
@@ -121,7 +121,7 @@ class MediaPipeline:
def _make_compatible(self):
"""Make overridable methods of MediaPipeline and subclasses backwards compatible"""
methods = [
- "file_path", "media_to_download", "media_downloaded",
+ "file_path", "thumb_path", "media_to_download", "media_downloaded",
"file_downloaded", "image_downloaded", "get_images"
]
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 6ed9f8b8f..3efd4d2fd 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -95,12 +95,14 @@ class ResponseTypes:
chunk = to_bytes(chunk)
if not binary_is_text(chunk):
return self.from_mimetype('application/octet-stream')
- elif b"" in chunk.lower():
+ lowercase_chunk = chunk.lower()
+ if b"" in lowercase_chunk:
return self.from_mimetype('text/html')
- elif b"' in lowercase_chunk:
+ return self.from_mimetype('text/html')
+ return self.from_mimetype('text')
def from_args(self, headers=None, url=None, filename=None, body=None):
"""Guess the most appropriate Response class based on
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 6b1ad0828..6cacc63e1 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -197,6 +197,38 @@ class BaseSettings(MutableMapping):
value = json.loads(value)
return dict(value)
+ def getdictorlist(self, name, default=None):
+ """Get a setting value as either a :class:`dict` or a :class:`list`.
+
+ If the setting is already a dict or a list, a copy of it will be
+ returned.
+
+ If it is a string it will be evaluated as JSON, or as a comma-separated
+ list of strings as a fallback.
+
+ For example, settings populated from the command line will return:
+
+ - ``{'key1': 'value1', 'key2': 'value2'}`` if set to
+ ``'{"key1": "value1", "key2": "value2"}'``
+
+ - ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
+
+ :param name: the setting name
+ :type name: string
+
+ :param default: the value to return if no setting is found
+ :type default: any
+ """
+ value = self.get(name, default)
+ if value is None:
+ return {}
+ if isinstance(value, str):
+ try:
+ return json.loads(value)
+ except ValueError:
+ return value.split(',')
+ return copy.deepcopy(value)
+
def getwithbase(self, name):
"""Get a composition of a dictionary-like setting and its `_BASE`
counterpart.
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 8389a70cb..ff86af125 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -154,6 +154,7 @@ FEED_EXPORTERS = {}
FEED_EXPORTERS_BASE = {
'json': 'scrapy.exporters.JsonItemExporter',
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
+ 'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
'jl': 'scrapy.exporters.JsonLinesItemExporter',
'csv': 'scrapy.exporters.CsvItemExporter',
'xml': 'scrapy.exporters.XmlItemExporter',
@@ -246,6 +247,9 @@ REDIRECT_PRIORITY_ADJUST = +2
REFERER_ENABLED = True
REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
+REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter'
+REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION'
+
RETRY_ENABLED = True
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index a414b5fde..5e541e2c0 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -86,3 +86,6 @@ ROBOTSTXT_OBEY = True
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+
+# Set settings whose default value is deprecated to a future-proof value
+REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION'
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 24a6187b9..00cc53725 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -118,7 +118,7 @@ def feed_complete_default_values_from_settings(feed, settings):
out = feed.copy()
out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))
out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
- out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
+ out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None)
out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
out.setdefault("item_export_kwargs", {})
diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py
deleted file mode 100644
index 653e2bbbb..000000000
--- a/scrapy/utils/py36.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import warnings
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.asyncgen import collect_asyncgen # noqa: F401
-
-
-warnings.warn(
- "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.asyncgen` instead.",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
-)
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 70ef3ba2b..cf33317ce 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -4,7 +4,9 @@ scrapy.http.Request objects
"""
import hashlib
-from typing import Dict, Iterable, Optional, Tuple, Union
+import json
+import warnings
+from typing import Dict, Iterable, List, Optional, Tuple, Union
from urllib.parse import urlunparse
from weakref import WeakKeyDictionary
@@ -12,13 +14,22 @@ from w3lib.http import basic_auth_header
from w3lib.url import canonicalize_url
from scrapy import Request, Spider
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
from scrapy.utils.python import to_bytes, to_unicode
-_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
-_fingerprint_cache = WeakKeyDictionary()
+_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
+_deprecated_fingerprint_cache = WeakKeyDictionary()
+
+
+def _serialize_headers(headers, request):
+ for header in headers:
+ if header in request.headers:
+ yield header
+ for value in request.headers.getlist(header):
+ yield value
def request_fingerprint(
@@ -26,6 +37,123 @@ def request_fingerprint(
include_headers: Optional[Iterable[Union[bytes, str]]] = None,
keep_fragments: bool = False,
) -> str:
+ """
+ Return the request fingerprint as an hexadecimal string.
+
+ The request fingerprint is a hash that uniquely identifies the resource the
+ request points to. For example, take the following two urls:
+
+ http://www.example.com/query?id=111&cat=222
+ http://www.example.com/query?cat=222&id=111
+
+ Even though those are two different URLs both point to the same resource
+ and are equivalent (i.e. they should return the same response).
+
+ Another example are cookies used to store session ids. Suppose the
+ following page is only accessible to authenticated users:
+
+ http://www.example.com/members/offers.html
+
+ Lots of sites use a cookie to store the session id, which adds a random
+ component to the HTTP Request and thus should be ignored when calculating
+ the fingerprint.
+
+ For this reason, request headers are ignored by default when calculating
+ the fingerprint. If you want to include specific headers use the
+ include_headers argument, which is a list of Request headers to include.
+
+ Also, servers usually ignore fragments in urls when handling requests,
+ so they are also ignored by default when calculating the fingerprint.
+ If you want to include them, set the keep_fragments argument to True
+ (for instance when handling requests with a headless browser).
+ """
+ if include_headers or keep_fragments:
+ message = (
+ 'Call to deprecated function '
+ 'scrapy.utils.request.request_fingerprint().\n'
+ '\n'
+ 'If you are using this function in a Scrapy component because you '
+ 'need a non-default fingerprinting algorithm, and you are OK '
+ 'with that non-default fingerprinting algorithm being used by '
+ 'all Scrapy components and not just the one calling this '
+ 'function, use crawler.request_fingerprinter.fingerprint() '
+ 'instead in your Scrapy component (you can get the crawler '
+ 'object from the \'from_crawler\' class method), and use the '
+ '\'REQUEST_FINGERPRINTER_CLASS\' setting to configure your '
+ 'non-default fingerprinting algorithm.\n'
+ '\n'
+ 'Otherwise, consider using the '
+ 'scrapy.utils.request.fingerprint() function instead.\n'
+ '\n'
+ 'If you switch to \'fingerprint()\', or assign the '
+ '\'REQUEST_FINGERPRINTER_CLASS\' setting a class that uses '
+ '\'fingerprint()\', the generated fingerprints will not only be '
+ 'bytes instead of a string, but they will also be different from '
+ 'those generated by \'request_fingerprint()\'. Before you switch, '
+ 'make sure that you understand the consequences of this (e.g. '
+ 'cache invalidation) and are OK with them; otherwise, consider '
+ 'implementing your own function which returns the same '
+ 'fingerprints as the deprecated \'request_fingerprint()\' function.'
+ )
+ else:
+ message = (
+ 'Call to deprecated function '
+ 'scrapy.utils.request.request_fingerprint().\n'
+ '\n'
+ 'If you are using this function in a Scrapy component, and you '
+ 'are OK with users of your component changing the fingerprinting '
+ 'algorithm through settings, use '
+ 'crawler.request_fingerprinter.fingerprint() instead in your '
+ 'Scrapy component (you can get the crawler object from the '
+ '\'from_crawler\' class method).\n'
+ '\n'
+ 'Otherwise, consider using the '
+ 'scrapy.utils.request.fingerprint() function instead.\n'
+ '\n'
+ 'Either way, the resulting fingerprints will be returned as '
+ 'bytes, not as a string, and they will also be different from '
+ 'those generated by \'request_fingerprint()\'. Before you switch, '
+ 'make sure that you understand the consequences of this (e.g. '
+ 'cache invalidation) and are OK with them; otherwise, consider '
+ 'implementing your own function which returns the same '
+ 'fingerprints as the deprecated \'request_fingerprint()\' function.'
+ )
+ warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
+ processed_include_headers: Optional[Tuple[bytes, ...]] = None
+ if include_headers:
+ processed_include_headers = tuple(
+ to_bytes(h.lower()) for h in sorted(include_headers)
+ )
+ cache = _deprecated_fingerprint_cache.setdefault(request, {})
+ cache_key = (processed_include_headers, keep_fragments)
+ if cache_key not in cache:
+ fp = hashlib.sha1()
+ fp.update(to_bytes(request.method))
+ fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
+ fp.update(request.body or b'')
+ if processed_include_headers:
+ for part in _serialize_headers(processed_include_headers, request):
+ fp.update(part)
+ cache[cache_key] = fp.hexdigest()
+ return cache[cache_key]
+
+
+def _request_fingerprint_as_bytes(*args, **kwargs):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ return bytes.fromhex(request_fingerprint(*args, **kwargs))
+
+
+_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
+_fingerprint_cache = WeakKeyDictionary()
+
+
+def fingerprint(
+ request: Request,
+ *,
+ include_headers: Optional[Iterable[Union[bytes, str]]] = None,
+ keep_fragments: bool = False,
+) -> bytes:
"""
Return the request fingerprint.
@@ -43,7 +171,7 @@ def request_fingerprint(
http://www.example.com/members/offers.html
- Lot of sites use a cookie to store the session id, which adds a random
+ Lots of sites use a cookie to store the session id, which adds a random
component to the HTTP Request and thus should be ignored when calculating
the fingerprint.
@@ -55,29 +183,96 @@ def request_fingerprint(
so they are also ignored by default when calculating the fingerprint.
If you want to include them, set the keep_fragments argument to True
(for instance when handling requests with a headless browser).
-
"""
- headers: Optional[Tuple[bytes, ...]] = None
+ processed_include_headers: Optional[Tuple[bytes, ...]] = None
if include_headers:
- headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
+ processed_include_headers = tuple(
+ to_bytes(h.lower()) for h in sorted(include_headers)
+ )
cache = _fingerprint_cache.setdefault(request, {})
- cache_key = (headers, keep_fragments)
+ cache_key = (processed_include_headers, keep_fragments)
if cache_key not in cache:
- fp = hashlib.sha1()
- fp.update(to_bytes(request.method))
- fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
- fp.update(request.body or b'')
- if headers:
- for hdr in headers:
- if hdr in request.headers:
- fp.update(hdr)
- for v in request.headers.getlist(hdr):
- fp.update(v)
- cache[cache_key] = fp.hexdigest()
+ # To decode bytes reliably (JSON does not support bytes), regardless of
+ # character encoding, we use bytes.hex()
+ headers: Dict[str, List[str]] = {}
+ if processed_include_headers:
+ for header in processed_include_headers:
+ if header in request.headers:
+ headers[header.hex()] = [
+ header_value.hex()
+ for header_value in request.headers.getlist(header)
+ ]
+ fingerprint_data = {
+ 'method': to_unicode(request.method),
+ 'url': canonicalize_url(request.url, keep_fragments=keep_fragments),
+ 'body': (request.body or b'').hex(),
+ 'headers': headers,
+ }
+ fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)
+ cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest()
return cache[cache_key]
-def request_authenticate(request: Request, username: str, password: str) -> None:
+class RequestFingerprinter:
+ """Default fingerprinter.
+
+ It takes into account a canonical version
+ (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url
+ ` and the values of :attr:`request.method
+ ` and :attr:`request.body
+ `. It then generates an `SHA1
+ `_ hash.
+
+ .. seealso:: :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION`.
+ """
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
+
+ def __init__(self, crawler=None):
+ if crawler:
+ implementation = crawler.settings.get(
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION'
+ )
+ else:
+ implementation = 'PREVIOUS_VERSION'
+ if implementation == 'PREVIOUS_VERSION':
+ message = (
+ '\'PREVIOUS_VERSION\' is a deprecated value for the '
+ '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n'
+ '\n'
+ 'It is also the default value. In other words, it is normal '
+ 'to get this warning if you have not defined a value for the '
+ '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting. This is so '
+ 'for backward compatibility reasons, but it will change in a '
+ 'future version of Scrapy.\n'
+ '\n'
+ 'See the documentation of the '
+ '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting for '
+ 'information on how to handle this deprecation.'
+ )
+ warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
+ self._fingerprint = _request_fingerprint_as_bytes
+ elif implementation == 'VERSION':
+ self._fingerprint = fingerprint
+ else:
+ raise ValueError(
+ f'Got an invalid value on setting '
+ f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': '
+ f'{implementation!r}. Valid values are \'PREVIOUS_VERSION\' (deprecated) '
+ f'and \'VERSION\'.'
+ )
+
+ def fingerprint(self, request):
+ return self._fingerprint(request)
+
+
+def request_authenticate(
+ request: Request,
+ username: str,
+ password: str,
+) -> None:
"""Authenticate the given request (in place) using the HTTP basic access
authentication mechanism (RFC 2617) and the given username and password
"""
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index d8fc25094..0b828f7c0 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -54,7 +54,7 @@ def get_ftp_content_and_delete(
return "".join(ftp_data)
-def get_crawler(spidercls=None, settings_dict=None):
+def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):
"""Return an unconfigured Crawler object. If settings_dict is given, it
will be used to populate the crawler settings with a project level
priority.
@@ -62,7 +62,12 @@ def get_crawler(spidercls=None, settings_dict=None):
from scrapy.crawler import CrawlerRunner
from scrapy.spiders import Spider
- runner = CrawlerRunner(settings_dict)
+ # Set by default settings that prevent deprecation warnings.
+ settings = {}
+ if prevent_warnings:
+ settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = 'VERSION'
+ settings.update(settings_dict or {})
+ runner = CrawlerRunner(settings)
return runner.create_crawler(spidercls or Spider)
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index a6a2a9e8b..4d5e9ae82 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -5,7 +5,6 @@ library.
Some of the functions that used to be imported from this module have been moved
to the w3lib.url module. Always import those from there instead.
"""
-import posixpath
import re
from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
@@ -31,8 +30,9 @@ def url_is_from_spider(url, spider):
def url_has_any_extension(url, extensions):
- return posixpath.splitext(parse_url(url).path)[1].lower() in extensions
-
+ """Return True if the url ends with one of the extensions provided"""
+ lowercase_path = parse_url(url).path.lower()
+ return any(lowercase_path.endswith(ext) for ext in extensions)
def parse_url(url, encoding=None):
"""Return urlparsed url from the given argument (which could be an already
diff --git a/sep/sep-014.rst b/sep/sep-014.rst
index 0859e3f7c..2521aa0e5 100644
--- a/sep/sep-014.rst
+++ b/sep/sep-014.rst
@@ -590,11 +590,11 @@ Request Generator
def generate_requests(self, response):
"""
- Extract and process new requets from response
+ Extract and process new requests from response
"""
requests = []
for ext in self._request_extractors:
- requets.extend(ext.extract_requests(response))
+ requests.extend(ext.extract_requests(response))
for proc in self._request_processors:
requests = proc(requests)
diff --git a/setup.py b/setup.py
index d86c0f285..ed197273f 100644
--- a/setup.py
+++ b/setup.py
@@ -19,35 +19,29 @@ def has_environment_marker_platform_impl_support():
install_requires = [
- 'Twisted>=17.9.0',
- 'cryptography>=2.0',
+ 'Twisted>=18.9.0',
+ 'cryptography>=2.8',
'cssselect>=0.9.1',
'itemloaders>=1.0.1',
'parsel>=1.5.0',
- 'pyOpenSSL>=16.2.0',
+ 'pyOpenSSL>=19.1.0',
'queuelib>=1.4.2',
'service_identity>=16.0.0',
'w3lib>=1.17.0',
- 'zope.interface>=4.1.3',
+ 'zope.interface>=5.1.0',
'protego>=0.1.15',
'itemadapter>=0.1.0',
'setuptools',
'tldextract',
+ 'lxml>=4.3.0',
]
extras_require = {}
cpython_dependencies = [
- 'lxml>=3.5.0',
'PyDispatcher>=2.0.5',
]
if has_environment_marker_platform_impl_support():
extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies
extras_require[':platform_python_implementation == "PyPy"'] = [
- # Earlier lxml versions are affected by
- # https://foss.heptapod.net/pypy/pypy/-/issues/2498,
- # which was fixed in Cython 0.26, released on 2017-06-19, and used to
- # generate the C headers of lxml release tarballs published since then, the
- # first of which was:
- 'lxml>=4.0.0',
'PyPyDispatcher>=2.1.0',
]
else:
@@ -84,7 +78,6 @@ setup(
'Operating System :: OS Independent',
'Programming Language :: Python',
'Programming Language :: Python :: 3',
- 'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
@@ -95,7 +88,7 @@ setup(
'Topic :: Software Development :: Libraries :: Application Frameworks',
'Topic :: Software Development :: Libraries :: Python Modules',
],
- python_requires='>=3.6',
+ python_requires='>=3.7',
install_requires=install_requires,
extras_require=extras_require,
)
diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
index 1f5a44010..72bb986bc 100644
--- a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
@@ -8,6 +8,7 @@ class AsyncioReactorSpider1(scrapy.Spider):
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
}
+
class AsyncioReactorSpider2(scrapy.Spider):
name = 'asyncio_reactor2'
custom_settings = {
diff --git a/tests/__init__.py b/tests/__init__.py
index 12ce79fa9..bb62851dc 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -5,6 +5,7 @@ see https://docs.scrapy.org/en/latest/contributing.html#running-tests
"""
import os
+import socket
# ignore system-wide proxies for tests
# which would send requests to a totally unsuspecting server
@@ -25,6 +26,15 @@ tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'sample_data')
+# In some environments accessing a non-existing host doesn't raise an
+# error. In such cases we're going to skip tests which rely on it.
+try:
+ socket.getaddrinfo('non-existing-host', 80)
+ NON_EXISTING_RESOLVABLE = True
+except socket.gaierror:
+ NON_EXISTING_RESOLVABLE = False
+
+
def get_testdata(*paths):
"""Return test data"""
path = os.path.join(tests_datadir, *paths)
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 398d1d16d..d9373dfa8 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,18 +1,16 @@
# Tests requirements
attrs
-dataclasses; python_version == '3.6'
pyftpdlib
pytest
pytest-cov==3.0.0
pytest-xdist
sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
testfixtures
-uvloop < 0.15.0; platform_system != "Windows" and python_version == '3.6'
-uvloop; platform_system != "Windows" and python_version > '3.6'
+uvloop; platform_system != "Windows"
# optional for shell wrapper tests
bpython
-brotlipy # optional for HTTP compress downloader middleware tests
+brotli # optional for HTTP compress downloader middleware tests
zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests
ipython
pywin32; sys_platform == "win32"
diff --git a/tests/sample_data/link_extractor/linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html
index 2307ea865..e3a2a4145 100644
--- a/tests/sample_data/link_extractor/linkextractor.html
+++ b/tests/sample_data/link_extractor/linkextractor.html
@@ -1,20 +1,22 @@
+
+
-
-
-Sample page with links for testing LinkExtractor
-
-
-
-
-
+
+
+ Sample page with links for testing LinkExtractor
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/sample_data/link_extractor/linkextractor_latin1.html b/tests/sample_data/link_extractor/linkextractor_latin1.html
index e7eee18de..1e05bf0f0 100644
--- a/tests/sample_data/link_extractor/linkextractor_latin1.html
+++ b/tests/sample_data/link_extractor/linkextractor_latin1.html
@@ -1,3 +1,5 @@
+
+
@@ -7,11 +9,11 @@
diff --git a/tests/sample_data/link_extractor/linkextractor_no_href.html b/tests/sample_data/link_extractor/linkextractor_no_href.html
index 0b01cede8..2d67ec6ff 100644
--- a/tests/sample_data/link_extractor/linkextractor_no_href.html
+++ b/tests/sample_data/link_extractor/linkextractor_no_href.html
@@ -1,3 +1,5 @@
+
+
@@ -21,5 +23,4 @@
-
\ No newline at end of file
diff --git a/tests/sample_data/link_extractor/linkextractor_noenc.html b/tests/sample_data/link_extractor/linkextractor_noenc.html
index f9166adbe..6fa137cd9 100644
--- a/tests/sample_data/link_extractor/linkextractor_noenc.html
+++ b/tests/sample_data/link_extractor/linkextractor_noenc.html
@@ -1,14 +1,17 @@
+
+
-
-
-Sample page without encoding for testing LinkExtractor
-
+
+
+ Sample page without encoding for testing LinkExtractor
+
+
-
+
diff --git a/tests/sample_data/test_site/index.html b/tests/sample_data/test_site/index.html
index d268c846a..afe17d8e2 100644
--- a/tests/sample_data/test_site/index.html
+++ b/tests/sample_data/test_site/index.html
@@ -1,18 +1,15 @@
+
+
-
-
-Scrapy test site
-
-
-
-
-Scrapy test site
-
-
-
-
-
+
+ Scrapy test site
+
+
+ Scrapy test site
+
+
+
\ No newline at end of file
diff --git a/tests/sample_data/test_site/item1.html b/tests/sample_data/test_site/item1.html
index ceeb6dc87..ee39f16f3 100644
--- a/tests/sample_data/test_site/item1.html
+++ b/tests/sample_data/test_site/item1.html
@@ -1,17 +1,14 @@
+
+
-
-
-Item 1 - Scrapy test site
-
-
-
-
-Item 1 name
-
-
-- Price: $100
-- Stock: 12
-
-
-
+
+ Item 1 - Scrapy test site
+
+
+ Item 1 name
+
+ - Price: $100
+ - Stock: 12
+
+
diff --git a/tests/sample_data/test_site/item2.html b/tests/sample_data/test_site/item2.html
index a64c92810..f40f70750 100644
--- a/tests/sample_data/test_site/item2.html
+++ b/tests/sample_data/test_site/item2.html
@@ -1,17 +1,14 @@
+
+
-
-
-Item 2 - Scrapy test site
-
-
-
-
-Item 2 name
-
-
-
-
-
+
+ Item 2 - Scrapy test site
+
+
+ Item 2 name
+
+ - Price: $200
+ - Stock: 5
+
+
+
\ No newline at end of file
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index f21ee971d..0d992be56 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -1,6 +1,7 @@
import os
import argparse
from os.path import join, abspath, isfile, exists
+
from twisted.internet import defer
from scrapy.commands import parse
from scrapy.settings import Settings
@@ -222,6 +223,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""")
self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))
+ @defer.inlineCallbacks
+ def test_crawlspider_not_exists_with_not_matched_url(self):
+ status, out, stderr = yield self.execute([self.url('/invalid_url')])
+ self.assertEqual(status, 0)
+
@defer.inlineCallbacks
def test_output_flag(self):
"""Checks if a file was created successfully having
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 16c9559b5..33189e9be 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -6,7 +6,7 @@ from twisted.internet import defer
from scrapy.utils.testsite import SiteTest
from scrapy.utils.testproc import ProcessTest
-from tests import tests_datadir
+from tests import tests_datadir, NON_EXISTING_RESOLVABLE
class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
@@ -109,6 +109,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
@defer.inlineCallbacks
def test_dns_failures(self):
+ if NON_EXISTING_RESOLVABLE:
+ raise unittest.SkipTest("Non-existing hosts are resolvable")
url = 'www.somedomainthatdoesntexi.st'
errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False)
self.assertEqual(errcode, 1, out or err)
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index d0f4a68c2..136056f50 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -5,11 +5,11 @@ from twisted.python import failure
from twisted.trial import unittest
from scrapy import FormRequest
-from scrapy.crawler import CrawlerRunner
from scrapy.spidermiddlewares.httperror import HttpError
from scrapy.spiders import Spider
from scrapy.http import Request
from scrapy.item import Item, Field
+from scrapy.utils.test import get_crawler
from scrapy.contracts import ContractsManager, Contract
from scrapy.contracts.default import (
UrlContract,
@@ -398,7 +398,7 @@ class ContractsManagerTest(unittest.TestCase):
TestSameUrlSpider.parse_first.__doc__ = contract_doc
TestSameUrlSpider.parse_second.__doc__ = contract_doc
- crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
+ crawler = get_crawler(TestSameUrlSpider)
yield crawler.crawl()
self.assertEqual(crawler.spider.visited, 2)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index cefa79156..c11871745 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -3,6 +3,7 @@ import logging
from ipaddress import IPv4Address
from socket import gethostbyname
from urllib.parse import urlparse
+import unittest
from pytest import mark
from testfixtures import LogCapture
@@ -17,6 +18,8 @@ from scrapy.exceptions import StopDownload
from scrapy.http import Request
from scrapy.http.response import Response
from scrapy.utils.python import to_unicode
+from scrapy.utils.test import get_crawler
+from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver import MockServer
from tests.spiders import (
AsyncDefAsyncioGenComplexSpider,
@@ -51,14 +54,13 @@ class CrawlTestCase(TestCase):
def setUp(self):
self.mockserver = MockServer()
self.mockserver.__enter__()
- self.runner = CrawlerRunner()
def tearDown(self):
self.mockserver.__exit__(None, None, None)
@defer.inlineCallbacks
def test_follow_all(self):
- crawler = self.runner.create_crawler(FollowAllSpider)
+ crawler = get_crawler(FollowAllSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url
@@ -81,7 +83,7 @@ class CrawlTestCase(TestCase):
settings = {"DOWNLOAD_DELAY": delay,
'RANDOMIZE_DOWNLOAD_DELAY': randomize}
- crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
+ crawler = get_crawler(FollowAllSpider, settings)
yield crawler.crawl(**crawl_kwargs)
times = crawler.spider.times
total_time = times[-1] - times[0]
@@ -94,7 +96,7 @@ class CrawlTestCase(TestCase):
# of ``total`` and ``delay`` values that are too small for the test
# code above to have any meaning.
settings["DOWNLOAD_DELAY"] = 0
- crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
+ crawler = get_crawler(FollowAllSpider, settings)
yield crawler.crawl(**crawl_kwargs)
times = crawler.spider.times
total_time = times[-1] - times[0]
@@ -104,7 +106,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_timeout_success(self):
- crawler = self.runner.create_crawler(DelaySpider)
+ crawler = get_crawler(DelaySpider)
yield crawler.crawl(n=0.5, mockserver=self.mockserver)
self.assertTrue(crawler.spider.t1 > 0)
self.assertTrue(crawler.spider.t2 > 0)
@@ -112,7 +114,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_timeout_failure(self):
- crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider)
+ crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})
yield crawler.crawl(n=0.5, mockserver=self.mockserver)
self.assertTrue(crawler.spider.t1 > 0)
self.assertTrue(crawler.spider.t2 == 0)
@@ -127,21 +129,23 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_retry_503(self):
- crawler = self.runner.create_crawler(SimpleSpider)
+ crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
self._assert_retried(log)
@defer.inlineCallbacks
def test_retry_conn_failed(self):
- crawler = self.runner.create_crawler(SimpleSpider)
+ crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver)
self._assert_retried(log)
@defer.inlineCallbacks
def test_retry_dns_error(self):
- crawler = self.runner.create_crawler(SimpleSpider)
+ if NON_EXISTING_RESOLVABLE:
+ raise unittest.SkipTest("Non-existing hosts are resolvable")
+ crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
# try to fetch the homepage of a non-existent domain
yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver)
@@ -150,7 +154,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_start_requests_bug_before_yield(self):
with LogCapture('scrapy', level=logging.ERROR) as log:
- crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
+ crawler = get_crawler(BrokenStartRequestsSpider)
yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)
self.assertEqual(len(log.records), 1)
@@ -161,7 +165,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_start_requests_bug_yielding(self):
with LogCapture('scrapy', level=logging.ERROR) as log:
- crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
+ crawler = get_crawler(BrokenStartRequestsSpider)
yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)
self.assertEqual(len(log.records), 1)
@@ -172,7 +176,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_start_requests_lazyness(self):
settings = {"CONCURRENT_REQUESTS": 1}
- crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider)
+ crawler = get_crawler(BrokenStartRequestsSpider, settings)
yield crawler.crawl(mockserver=self.mockserver)
self.assertTrue(
crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
@@ -181,7 +185,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_start_requests_dupes(self):
settings = {"CONCURRENT_REQUESTS": 1}
- crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider)
+ crawler = get_crawler(DuplicateStartRequestsSpider, settings)
yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver)
self.assertEqual(crawler.spider.visited, 6)
@@ -210,7 +214,7 @@ Connection: close
foo body
with multiples lines
'''})
- crawler = self.runner.create_crawler(SimpleSpider)
+ crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver)
self.assertEqual(str(log).count("Got response 200"), 1)
@@ -218,7 +222,7 @@ with multiples lines
@defer.inlineCallbacks
def test_retry_conn_lost(self):
# connection lost after receiving data
- crawler = self.runner.create_crawler(SimpleSpider)
+ crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver)
self._assert_retried(log)
@@ -226,7 +230,7 @@ with multiples lines
@defer.inlineCallbacks
def test_retry_conn_aborted(self):
# connection lost before receiving data
- crawler = self.runner.create_crawler(SimpleSpider)
+ crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver)
self._assert_retried(log)
@@ -245,7 +249,7 @@ with multiples lines
req0.meta['next'] = req1
req1.meta['next'] = req2
req2.meta['next'] = req3
- crawler = self.runner.create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
yield crawler.crawl(seed=req0, mockserver=self.mockserver)
# basic asserts in case of weird communication errors
self.assertIn('responses', crawler.spider.meta)
@@ -271,7 +275,7 @@ with multiples lines
def cb(response):
est.append(get_engine_status(crawler.engine))
- crawler = self.runner.create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver)
self.assertEqual(len(est), 1, est)
s = dict(est[0])
@@ -286,7 +290,7 @@ with multiples lines
def cb(response):
est.append(format_engine_status(crawler.engine))
- crawler = self.runner.create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver)
self.assertEqual(len(est), 1, est)
est = est[0].split("\n")[2:-2] # remove header & footer
@@ -317,7 +321,7 @@ with multiples lines
def start_requests(self):
raise TestError
- crawler = self.runner.create_crawler(FaultySpider)
+ crawler = get_crawler(FaultySpider)
yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError)
self.assertFalse(crawler.crawling)
@@ -328,26 +332,28 @@ with multiples lines
"tests.pipelines.ZeroDivisionErrorPipeline": 300,
}
}
- crawler = CrawlerRunner(settings).create_crawler(SimpleSpider)
+ crawler = get_crawler(SimpleSpider, settings)
yield self.assertFailure(
- self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver),
+ crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver),
ZeroDivisionError)
self.assertFalse(crawler.crawling)
@defer.inlineCallbacks
def test_crawlerrunner_accepts_crawler(self):
- crawler = self.runner.create_crawler(SimpleSpider)
+ crawler = get_crawler(SimpleSpider)
+ runner = CrawlerRunner()
with LogCapture() as log:
- yield self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+ yield runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
self.assertIn("Got response 200", str(log))
@defer.inlineCallbacks
def test_crawl_multiple(self):
- self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
- self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
+ runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'})
+ runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+ runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
with LogCapture() as log:
- yield self.runner.join()
+ yield runner.join()
self._assert_retried(log)
self.assertIn("Got response 200", str(log))
@@ -358,7 +364,6 @@ class CrawlSpiderTestCase(TestCase):
def setUp(self):
self.mockserver = MockServer()
self.mockserver.__enter__()
- self.runner = CrawlerRunner()
def tearDown(self):
self.mockserver.__exit__(None, None, None)
@@ -370,7 +375,7 @@ class CrawlSpiderTestCase(TestCase):
def _on_item_scraped(item):
items.append(item)
- crawler = self.runner.create_crawler(spider_cls)
+ crawler = get_crawler(spider_cls)
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
@@ -378,10 +383,9 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_crawlspider_with_parse(self):
- self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver)
-
+ crawler = get_crawler(CrawlSpiderWithParseMethod)
with LogCapture() as log:
- yield self.runner.join()
+ yield crawler.crawl(mockserver=self.mockserver)
self.assertIn("[parse] status 200 (foo: None)", str(log))
self.assertIn("[parse] status 201 (foo: None)", str(log))
@@ -389,10 +393,9 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_crawlspider_with_errback(self):
- self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver)
-
+ crawler = get_crawler(CrawlSpiderWithErrback)
with LogCapture() as log:
- yield self.runner.join()
+ yield crawler.crawl(mockserver=self.mockserver)
self.assertIn("[parse] status 200 (foo: None)", str(log))
self.assertIn("[parse] status 201 (foo: None)", str(log))
@@ -403,18 +406,19 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_async_def_parse(self):
- self.runner.crawl(AsyncDefSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+ crawler = get_crawler(AsyncDefSpider)
with LogCapture() as log:
- yield self.runner.join()
+ yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
self.assertIn("Got response 200", str(log))
@mark.only_asyncio()
@defer.inlineCallbacks
def test_async_def_asyncio_parse(self):
- runner = CrawlerRunner({"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"})
- runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+ crawler = get_crawler(AsyncDefAsyncioSpider, {
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+ })
with LogCapture() as log:
- yield runner.join()
+ yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
self.assertIn("Got response 200", str(log))
@mark.only_asyncio()
@@ -433,7 +437,7 @@ class CrawlSpiderTestCase(TestCase):
def _on_item_scraped(item):
items.append(item)
- crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSingleElementSpider)
+ crawler = get_crawler(AsyncDefAsyncioReturnSingleElementSpider)
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
@@ -508,14 +512,14 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_response_ssl_certificate_none(self):
- crawler = self.runner.create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url("/echo?body=test", is_secure=False)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta['responses'][0].certificate)
@defer.inlineCallbacks
def test_response_ssl_certificate(self):
- crawler = self.runner.create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url("/echo?body=test", is_secure=True)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
cert = crawler.spider.meta['responses'][0].certificate
@@ -526,7 +530,7 @@ class CrawlSpiderTestCase(TestCase):
@mark.xfail(reason="Responses with no body return early and contain no certificate")
@defer.inlineCallbacks
def test_response_ssl_certificate_empty_response(self):
- crawler = self.runner.create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url("/status?n=200", is_secure=True)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
cert = crawler.spider.meta['responses'][0].certificate
@@ -536,7 +540,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_dns_server_ip_address_none(self):
- crawler = self.runner.create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url('/status?n=200')
yield crawler.crawl(seed=url, mockserver=self.mockserver)
ip_address = crawler.spider.meta['responses'][0].ip_address
@@ -544,7 +548,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_dns_server_ip_address(self):
- crawler = self.runner.create_crawler(SingleRequestSpider)
+ crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url('/echo?body=test')
expected_netloc, _ = urlparse(url).netloc.split(':')
yield crawler.crawl(seed=url, mockserver=self.mockserver)
@@ -554,7 +558,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_bytes_received_stop_download_callback(self):
- crawler = self.runner.create_crawler(BytesReceivedCallbackSpider)
+ crawler = get_crawler(BytesReceivedCallbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta.get("failure"))
self.assertIsInstance(crawler.spider.meta["response"], Response)
@@ -563,7 +567,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_bytes_received_stop_download_errback(self):
- crawler = self.runner.create_crawler(BytesReceivedErrbackSpider)
+ crawler = get_crawler(BytesReceivedErrbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta.get("response"))
self.assertIsInstance(crawler.spider.meta["failure"], Failure)
@@ -578,7 +582,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_headers_received_stop_download_callback(self):
- crawler = self.runner.create_crawler(HeadersReceivedCallbackSpider)
+ crawler = get_crawler(HeadersReceivedCallbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta.get("failure"))
self.assertIsInstance(crawler.spider.meta["response"], Response)
@@ -586,7 +590,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_headers_received_stop_download_errback(self):
- crawler = self.runner.create_crawler(HeadersReceivedErrbackSpider)
+ crawler = get_crawler(HeadersReceivedErrbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta.get("response"))
self.assertIsInstance(crawler.spider.meta["failure"], Failure)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 8f6227109..d67abed7c 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -13,11 +13,13 @@ from twisted.trial import unittest
import scrapy
from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings, default_settings
from scrapy.spiderloader import SpiderLoader
from scrapy.utils.log import configure_logging, get_scrapy_root_handler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.misc import load_object
+from scrapy.utils.test import get_crawler
from scrapy.extensions.throttle import AutoThrottle
from scrapy.extensions import telnet
from scrapy.utils.test import get_testenv
@@ -34,9 +36,6 @@ class BaseCrawlerTest(unittest.TestCase):
class CrawlerTestCase(BaseCrawlerTest):
- def setUp(self):
- self.crawler = Crawler(DefaultSpider, Settings())
-
def test_populate_spidercls_settings(self):
spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}
project_settings = {'TEST1': 'project', 'TEST3': 'project'}
@@ -46,7 +45,9 @@ class CrawlerTestCase(BaseCrawlerTest):
settings = Settings()
settings.setdict(project_settings, priority='project')
- crawler = Crawler(CustomSettingsSpider, settings)
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+ crawler = Crawler(CustomSettingsSpider, settings)
self.assertEqual(crawler.settings.get('TEST1'), 'spider')
self.assertEqual(crawler.settings.get('TEST2'), 'spider')
@@ -56,12 +57,14 @@ class CrawlerTestCase(BaseCrawlerTest):
self.assertTrue(crawler.settings.frozen)
def test_crawler_accepts_dict(self):
- crawler = Crawler(DefaultSpider, {'foo': 'bar'})
+ crawler = get_crawler(DefaultSpider, {'foo': 'bar'})
self.assertEqual(crawler.settings['foo'], 'bar')
self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
def test_crawler_accepts_None(self):
- crawler = Crawler(DefaultSpider)
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+ crawler = Crawler(DefaultSpider)
self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
def test_crawler_rejects_spider_objects(self):
@@ -77,7 +80,7 @@ class SpiderSettingsTestCase(unittest.TestCase):
'AUTOTHROTTLE_ENABLED': True
}
- crawler = Crawler(MySpider, {})
+ crawler = get_crawler(MySpider)
enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
self.assertIn(AutoThrottle, enabled_exts)
@@ -91,7 +94,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
class MySpider(scrapy.Spider):
name = 'spider'
- Crawler(MySpider, {})
+ get_crawler(MySpider)
assert get_scrapy_root_handler() is None
def test_spider_custom_settings_log_level(self):
@@ -104,13 +107,14 @@ class CrawlerLoggingTestCase(unittest.TestCase):
custom_settings = {
'LOG_LEVEL': 'INFO',
'LOG_FILE': log_file,
- # disable telnet if not available to avoid an extra warning
+ # settings to avoid extra warnings
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
}
configure_logging()
self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)
- crawler = Crawler(MySpider, {})
+ crawler = get_crawler(MySpider)
self.assertEqual(get_scrapy_root_handler().level, logging.INFO)
info_count = crawler.stats.get_value('log_count/INFO')
logging.debug('debug message')
@@ -147,7 +151,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
}
configure_logging()
- Crawler(MySpider, {})
+ get_crawler(MySpider)
logging.debug('debug message')
with open(log_file, 'rb') as fo:
@@ -228,22 +232,25 @@ class NoRequestsSpider(scrapy.Spider):
@mark.usefixtures('reactor_pytest')
class CrawlerRunnerHasSpider(unittest.TestCase):
+ def _runner(self):
+ return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'})
+
@defer.inlineCallbacks
def test_crawler_runner_bootstrap_successful(self):
- runner = CrawlerRunner()
+ runner = self._runner()
yield runner.crawl(NoRequestsSpider)
self.assertEqual(runner.bootstrap_failed, False)
@defer.inlineCallbacks
def test_crawler_runner_bootstrap_successful_for_several(self):
- runner = CrawlerRunner()
+ runner = self._runner()
yield runner.crawl(NoRequestsSpider)
yield runner.crawl(NoRequestsSpider)
self.assertEqual(runner.bootstrap_failed, False)
@defer.inlineCallbacks
def test_crawler_runner_bootstrap_failed(self):
- runner = CrawlerRunner()
+ runner = self._runner()
try:
yield runner.crawl(ExceptionSpider)
@@ -256,7 +263,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
@defer.inlineCallbacks
def test_crawler_runner_bootstrap_failed_for_several(self):
- runner = CrawlerRunner()
+ runner = self._runner()
try:
yield runner.crawl(ExceptionSpider)
@@ -274,12 +281,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
if self.reactor_pytest == 'asyncio':
CrawlerRunner(settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION",
})
else:
msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
with self.assertRaisesRegex(Exception, msg):
runner = CrawlerRunner(settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION",
})
yield runner.crawl(NoRequestsSpider)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 2bb53950d..883960084 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -4,7 +4,7 @@ import shutil
import sys
import tempfile
from typing import Optional, Type
-from unittest import mock
+from unittest import mock, SkipTest
from testfixtures import LogCapture
from twisted.cred import checkers, credentials, portal
@@ -25,13 +25,14 @@ from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.http import Headers, Request
+from scrapy.http import Headers, HtmlResponse, Request
from scrapy.http.response.text import TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
from scrapy.utils.misc import create_instance
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler, skip_if_no_boto
+from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver import (
Echo,
ForeverTakingResource,
@@ -389,6 +390,23 @@ class HttpTestCase(unittest.TestCase):
d.addCallback(self.assertEqual, b'159')
return d
+ def _test_response_class(self, filename, body, response_class):
+ def _test(response):
+ self.assertEqual(type(response), response_class)
+
+ request = Request(self.getURL(filename), body=body)
+ return self.download_request(request, Spider('foo')).addCallback(_test)
+
+ def test_response_class_from_url(self):
+ return self._test_response_class('foo.html', b'', HtmlResponse)
+
+ def test_response_class_from_body(self):
+ return self._test_response_class(
+ 'foo',
+ b"\n.",
+ HtmlResponse,
+ )
+
class Http10TestCase(HttpTestCase):
"""HTTP 1.0 test case"""
@@ -774,6 +792,8 @@ class Http11ProxyTestCase(HttpProxyTestCase):
@defer.inlineCallbacks
def test_download_with_proxy_https_timeout(self):
""" Test TunnelingTCP4ClientEndpoint """
+ if NON_EXISTING_RESOLVABLE:
+ raise SkipTest("Non-existing hosts are resolvable")
http_proxy = self.getURL('')
domain = 'https://no-such-domain.nosuch'
request = Request(
@@ -971,6 +991,12 @@ class BaseFTPTestCase(unittest.TestCase):
password = "passwd"
req_meta = {"ftp_user": username, "ftp_password": password}
+ test_files = (
+ ('file.txt', b"I have the power!"),
+ ('file with spaces.txt', b"Moooooooooo power!"),
+ ('html-file-without-extension', b"\n."),
+ )
+
def setUp(self):
from twisted.protocols.ftp import FTPRealm, FTPFactory
from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
@@ -981,8 +1007,8 @@ class BaseFTPTestCase(unittest.TestCase):
userdir = os.path.join(self.directory, self.username)
os.mkdir(userdir)
fp = FilePath(userdir)
- fp.child('file.txt').setContent(b"I have the power!")
- fp.child('file with spaces.txt').setContent(b"Moooooooooo power!")
+ for filename, content in self.test_files:
+ fp.child(filename).setContent(content)
# setup server
realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory)
@@ -1069,6 +1095,27 @@ class BaseFTPTestCase(unittest.TestCase):
return self._add_test_callbacks(d, _test)
+ def _test_response_class(self, filename, response_class):
+ f, local_fname = tempfile.mkstemp()
+ local_fname = to_bytes(local_fname)
+ os.close(f)
+ meta = {}
+ meta.update(self.req_meta)
+ request = Request(url=f"ftp://127.0.0.1:{self.portNum}/{filename}",
+ meta=meta)
+ d = self.download_handler.download_request(request, None)
+
+ def _test(r):
+ self.assertEqual(type(r), response_class)
+ os.remove(local_fname)
+ return self._add_test_callbacks(d, _test)
+
+ def test_response_class_from_url(self):
+ return self._test_response_class('file.txt', TextResponse)
+
+ def test_response_class_from_body(self):
+ return self._test_response_class('html-file-without-extension', HtmlResponse)
+
class FTPTestCase(BaseFTPTestCase):
@@ -1104,8 +1151,8 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
os.mkdir(self.directory)
fp = FilePath(self.directory)
- fp.child('file.txt').setContent(b"I have the power!")
- fp.child('file with spaces.txt').setContent(b"Moooooooooo power!")
+ for filename, content in self.test_files:
+ fp.child(filename).setContent(content)
# setup server for anonymous access
realm = FTPRealm(anonymousRoot=self.directory)
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index b538a0ed3..38be915f2 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -1,13 +1,11 @@
import asyncio
-from unittest import mock, SkipTest
+from unittest import mock
from pytest import mark
-from twisted import version as twisted_version
from twisted.internet import defer
from twisted.internet.defer import Deferred
from twisted.trial.unittest import TestCase
from twisted.python.failure import Failure
-from twisted.python.versions import Version
from scrapy.http import Request, Response
from scrapy.spiders import Spider
@@ -218,16 +216,6 @@ class MiddlewareUsingCoro(ManagerTestCase):
"""Middlewares using asyncio coroutines should work"""
def test_asyncdef(self):
- if (
- self.reactor_pytest == 'asyncio'
- and twisted_version < Version('twisted', 18, 4, 0)
- ):
- raise SkipTest(
- 'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
- 'hangs when using AsyncIO and Twisted versions lower than '
- '18.4.0'
- )
-
resp = Response('http://example.com/index.html')
class CoroMiddleware:
@@ -248,12 +236,6 @@ class MiddlewareUsingCoro(ManagerTestCase):
@mark.only_asyncio()
def test_asyncdef_asyncio(self):
- if twisted_version < Version('twisted', 18, 4, 0):
- raise SkipTest(
- 'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
- 'hangs when using Twisted versions lower than 18.4.0'
- )
-
resp = Response('http://example.com/index.html')
class CoroMiddleware:
diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py
index 0362e2018..b9f3e24a4 100644
--- a/tests/test_downloadermiddleware_httpauth.py
+++ b/tests/test_downloadermiddleware_httpauth.py
@@ -1,7 +1,9 @@
import unittest
+import pytest
from w3lib.http import basic_auth_header
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
from scrapy.spiders import Spider
@@ -30,8 +32,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
self.spider = TestSpiderLegacy('foo')
def test_auth(self):
- mw = HttpAuthMiddleware()
- mw.spider_opened(self.spider)
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="Using HttpAuthMiddleware without http_auth_domain is deprecated"):
+ mw = HttpAuthMiddleware()
+ mw.spider_opened(self.spider)
# initial request, sets the domain and sends the header
req = Request('http://example.com/')
@@ -49,8 +53,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
self.assertNotIn('Authorization', req.headers)
def test_auth_already_set(self):
- mw = HttpAuthMiddleware()
- mw.spider_opened(self.spider)
+ with pytest.warns(ScrapyDeprecationWarning,
+ match="Using HttpAuthMiddleware without http_auth_domain is deprecated"):
+ mw = HttpAuthMiddleware()
+ mw.spider_opened(self.spider)
req = Request('http://example.com/',
headers=dict(Authorization='Digest 123'))
assert mw.process_request(req, self.spider) is None
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 0c6dcf2aa..928c007f5 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -122,6 +122,21 @@ class DefaultStorageTest(_BaseTest):
time.sleep(0.5) # give the chance to expire
assert storage.retrieve_response(self.spider, self.request)
+ def test_storage_no_content_type_header(self):
+ """Test that the response body is used to get the right response class
+ even if there is no Content-Type header"""
+ with self._storage() as storage:
+ assert storage.retrieve_response(self.spider, self.request) is None
+ response = Response(
+ 'http://www.example.com',
+ body=b'\n.',
+ status=202,
+ )
+ storage.store_response(self.spider, self.request, response)
+ cached_response = storage.retrieve_response(self.spider, self.request)
+ self.assertIsInstance(cached_response, HtmlResponse)
+ self.assertEqualResponse(response, cached_response)
+
class DbmStorageTest(DefaultStorageTest):
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 7c97bf32a..4ac85c1ec 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -1,13 +1,13 @@
import os
-from functools import partial
+
+import pytest
from twisted.trial.unittest import TestCase
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
from scrapy.exceptions import NotConfigured
from scrapy.http import Request
from scrapy.spiders import Spider
-from scrapy.crawler import Crawler
-from scrapy.settings import Settings
+from scrapy.utils.test import get_crawler
spider = Spider('foo')
@@ -23,9 +23,9 @@ class TestHttpProxyMiddleware(TestCase):
os.environ = self._oldenv
def test_not_enabled(self):
- settings = Settings({'HTTPPROXY_ENABLED': False})
- crawler = Crawler(Spider, settings)
- self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler))
+ crawler = get_crawler(Spider, {'HTTPPROXY_ENABLED': False})
+ with pytest.raises(NotConfigured):
+ HttpProxyMiddleware.from_crawler(crawler)
def test_no_environment_proxies(self):
os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index 9e75f0a50..7d88ba4d2 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -1,7 +1,9 @@
+import warnings
from itertools import product
from unittest import TestCase
from scrapy.downloadermiddlewares.stats import DownloaderStats
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.response import response_httprepr
@@ -54,7 +56,10 @@ class TestDownloaderStats(TestCase):
for test_response in test_responses:
self.crawler.stats.set_value('downloader/response_bytes', 0)
self.mw.process_response(self.req, test_response, self.spider)
- self.assertStatsEqual('downloader/response_bytes', len(response_httprepr(test_response)))
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+ resp_size = len(response_httprepr(test_response))
+ self.assertStatsEqual('downloader/response_bytes', resp_size)
def test_process_exception(self):
self.mw.process_exception(self.req, MyException(), self.spider)
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 680bb6dc8..8a37a8ebe 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -10,17 +10,25 @@ from scrapy.dupefilters import RFPDupeFilter
from scrapy.http import Request
from scrapy.core.scheduler import Scheduler
from scrapy.utils.python import to_bytes
-from scrapy.utils.job import job_dir
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
+def _get_dupefilter(*, crawler=None, settings=None, open=True):
+ if crawler is None:
+ crawler = get_crawler(settings_dict=settings)
+ scheduler = Scheduler.from_crawler(crawler)
+ dupefilter = scheduler.df
+ if open:
+ dupefilter.open()
+ return dupefilter
+
+
class FromCrawlerRFPDupeFilter(RFPDupeFilter):
@classmethod
def from_crawler(cls, crawler):
- debug = crawler.settings.getbool('DUPEFILTER_DEBUG')
- df = cls(job_dir(crawler.settings), debug)
+ df = super().from_crawler(crawler)
df.method = 'from_crawler'
return df
@@ -28,9 +36,8 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter):
class FromSettingsRFPDupeFilter(RFPDupeFilter):
@classmethod
- def from_settings(cls, settings):
- debug = settings.getbool('DUPEFILTER_DEBUG')
- df = cls(job_dir(settings), debug)
+ def from_settings(cls, settings, *, fingerprinter=None):
+ df = super().from_settings(settings, fingerprinter=fingerprinter)
df.method = 'from_settings'
return df
@@ -43,7 +50,8 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_df_from_crawler_scheduler(self):
settings = {'DUPEFILTER_DEBUG': True,
- 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
+ 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
self.assertTrue(scheduler.df.debug)
@@ -51,22 +59,22 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_df_from_settings_scheduler(self):
settings = {'DUPEFILTER_DEBUG': True,
- 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter}
+ 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter,
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
self.assertTrue(scheduler.df.debug)
self.assertEqual(scheduler.df.method, 'from_settings')
def test_df_direct_scheduler(self):
- settings = {'DUPEFILTER_CLASS': DirectDupeFilter}
+ settings = {'DUPEFILTER_CLASS': DirectDupeFilter,
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
self.assertEqual(scheduler.df.method, 'n/a')
def test_filter(self):
- dupefilter = RFPDupeFilter()
- dupefilter.open()
-
+ dupefilter = _get_dupefilter()
r1 = Request('http://scrapytest.org/1')
r2 = Request('http://scrapytest.org/2')
r3 = Request('http://scrapytest.org/2')
@@ -85,7 +93,7 @@ class RFPDupeFilterTest(unittest.TestCase):
path = tempfile.mkdtemp()
try:
- df = RFPDupeFilter(path)
+ df = _get_dupefilter(settings={'JOBDIR': path}, open=False)
try:
df.open()
assert not df.request_seen(r1)
@@ -93,7 +101,8 @@ class RFPDupeFilterTest(unittest.TestCase):
finally:
df.close('finished')
- df2 = RFPDupeFilter(path)
+ df2 = _get_dupefilter(settings={'JOBDIR': path}, open=False)
+ assert df != df2
try:
df2.open()
assert df2.request_seen(r1)
@@ -109,26 +118,24 @@ class RFPDupeFilterTest(unittest.TestCase):
output of request_seen.
"""
+ dupefilter = _get_dupefilter()
r1 = Request('http://scrapytest.org/index.html')
r2 = Request('http://scrapytest.org/INDEX.html')
- dupefilter = RFPDupeFilter()
- dupefilter.open()
-
assert not dupefilter.request_seen(r1)
assert not dupefilter.request_seen(r2)
dupefilter.close('finished')
- class CaseInsensitiveRFPDupeFilter(RFPDupeFilter):
+ class RequestFingerprinter:
- def request_fingerprint(self, request):
+ def fingerprint(self, request):
fp = hashlib.sha1()
fp.update(to_bytes(request.url.lower()))
- return fp.hexdigest()
+ return fp.digest()
- case_insensitive_dupefilter = CaseInsensitiveRFPDupeFilter()
- case_insensitive_dupefilter.open()
+ settings = {'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter}
+ case_insensitive_dupefilter = _get_dupefilter(settings=settings)
assert not case_insensitive_dupefilter.request_seen(r1)
assert case_insensitive_dupefilter.request_seen(r2)
@@ -142,8 +149,10 @@ class RFPDupeFilterTest(unittest.TestCase):
r1 = Request('http://scrapytest.org/1')
path = tempfile.mkdtemp()
+ crawler = get_crawler(settings_dict={'JOBDIR': path})
try:
- df = RFPDupeFilter(path)
+ scheduler = Scheduler.from_crawler(crawler)
+ df = scheduler.df
df.open()
df.request_seen(r1)
df.close('finished')
@@ -162,13 +171,11 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_log(self):
with LogCapture() as log:
settings = {'DUPEFILTER_DEBUG': False,
- 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
+ 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
- scheduler = Scheduler.from_crawler(crawler)
spider = SimpleSpider.from_crawler(crawler)
-
- dupefilter = scheduler.df
- dupefilter.open()
+ dupefilter = _get_dupefilter(crawler=crawler)
r1 = Request('http://scrapytest.org/index.html')
r2 = Request('http://scrapytest.org/index.html')
@@ -191,13 +198,45 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_log_debug(self):
with LogCapture() as log:
settings = {'DUPEFILTER_DEBUG': True,
- 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
+ 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
- scheduler = Scheduler.from_crawler(crawler)
spider = SimpleSpider.from_crawler(crawler)
-
- dupefilter = scheduler.df
- dupefilter.open()
+ dupefilter = _get_dupefilter(crawler=crawler)
+
+ r1 = Request('http://scrapytest.org/index.html')
+ r2 = Request('http://scrapytest.org/index.html',
+ headers={'Referer': 'http://scrapytest.org/INDEX.html'})
+
+ dupefilter.log(r1, spider)
+ dupefilter.log(r2, spider)
+
+ assert crawler.stats.get_value('dupefilter/filtered') == 2
+ log.check_present(
+ (
+ 'scrapy.dupefilters',
+ 'DEBUG',
+ 'Filtered duplicate request: (referer: None)'
+ )
+ )
+ log.check_present(
+ (
+ 'scrapy.dupefilters',
+ 'DEBUG',
+ 'Filtered duplicate request: '
+ ' (referer: http://scrapytest.org/INDEX.html)'
+ )
+ )
+
+ dupefilter.close('finished')
+
+ def test_log_debug_default_dupefilter(self):
+ with LogCapture() as log:
+ settings = {'DUPEFILTER_DEBUG': True,
+ 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
+ crawler = get_crawler(SimpleSpider, settings_dict=settings)
+ spider = SimpleSpider.from_crawler(crawler)
+ dupefilter = _get_dupefilter(crawler=crawler)
r1 = Request('http://scrapytest.org/index.html')
r2 = Request('http://scrapytest.org/index.html',
diff --git a/tests/test_engine.py b/tests/test_engine.py
index fa7d0c8d4..1bd802bcf 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -13,10 +13,11 @@ module with the ``runserver`` argument::
import os
import re
import sys
-import warnings
from collections import defaultdict
from urllib.parse import urlparse
+from dataclasses import dataclass
+import pytest
import attr
from itemadapter import ItemAdapter
from pydispatch import dispatcher
@@ -50,6 +51,13 @@ class AttrsItem:
price = attr.ib(default=0)
+@dataclass
+class DataClassItem:
+ name: str = ""
+ url: str = ""
+ price: int = 0
+
+
class TestSpider(Spider):
name = "scrapytest.org"
allowed_domains = ["scrapytest.org", "localhost"]
@@ -92,17 +100,8 @@ class AttrsItemsSpider(TestSpider):
item_cls = AttrsItem
-try:
- from dataclasses import make_dataclass
-except ImportError:
- DataClassItemsSpider = None
-else:
- TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)])
-
- class DataClassItemsSpider(DictItemsSpider): # type: ignore[no-redef]
- def parse_item(self, response):
- item = super().parse_item(response)
- return TestDataClass(**item)
+class DataClassItemsSpider(TestSpider):
+ item_cls = DataClassItem
class ItemZeroDivisionErrorSpider(TestSpider):
@@ -188,7 +187,7 @@ class CrawlerRun:
return self.deferred
def stop(self):
- self.port.stopListening()
+ self.port.stopListening() # FIXME: wait for this Deferred
for name, signal in vars(signals).items():
if not name.startswith('_'):
disconnect_all(signal)
@@ -239,79 +238,77 @@ class EngineTest(unittest.TestCase):
def test_crawler(self):
for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
- if spider is None:
- continue
- self.run = CrawlerRun(spider)
- yield self.run.run()
- self._assert_visited_urls()
- self._assert_scheduled_requests(count=9)
- self._assert_downloaded_responses(count=9)
- self._assert_scraped_items()
- self._assert_signals_caught()
- self._assert_bytes_received()
+ run = CrawlerRun(spider)
+ yield run.run()
+ self._assert_visited_urls(run)
+ self._assert_scheduled_requests(run, count=9)
+ self._assert_downloaded_responses(run, count=9)
+ self._assert_scraped_items(run)
+ self._assert_signals_caught(run)
+ self._assert_bytes_received(run)
@defer.inlineCallbacks
def test_crawler_dupefilter(self):
- self.run = CrawlerRun(TestDupeFilterSpider)
- yield self.run.run()
- self._assert_scheduled_requests(count=8)
- self._assert_dropped_requests()
+ run = CrawlerRun(TestDupeFilterSpider)
+ yield run.run()
+ self._assert_scheduled_requests(run, count=8)
+ self._assert_dropped_requests(run)
@defer.inlineCallbacks
def test_crawler_itemerror(self):
- self.run = CrawlerRun(ItemZeroDivisionErrorSpider)
- yield self.run.run()
- self._assert_items_error()
+ run = CrawlerRun(ItemZeroDivisionErrorSpider)
+ yield run.run()
+ self._assert_items_error(run)
@defer.inlineCallbacks
def test_crawler_change_close_reason_on_idle(self):
- self.run = CrawlerRun(ChangeCloseReasonSpider)
- yield self.run.run()
- self.assertEqual({'spider': self.run.spider, 'reason': 'custom_reason'},
- self.run.signals_caught[signals.spider_closed])
+ run = CrawlerRun(ChangeCloseReasonSpider)
+ yield run.run()
+ self.assertEqual({'spider': run.spider, 'reason': 'custom_reason'},
+ run.signals_caught[signals.spider_closed])
- def _assert_visited_urls(self):
+ def _assert_visited_urls(self, run: CrawlerRun):
must_be_visited = ["/", "/redirect", "/redirected",
"/item1.html", "/item2.html", "/item999.html"]
- urls_visited = {rp[0].url for rp in self.run.respplug}
- urls_expected = {self.run.geturl(p) for p in must_be_visited}
+ urls_visited = {rp[0].url for rp in run.respplug}
+ urls_expected = {run.geturl(p) for p in must_be_visited}
assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}"
- def _assert_scheduled_requests(self, count=None):
- self.assertEqual(count, len(self.run.reqplug))
+ def _assert_scheduled_requests(self, run: CrawlerRun, count=None):
+ self.assertEqual(count, len(run.reqplug))
paths_expected = ['/item999.html', '/item2.html', '/item1.html']
- urls_requested = {rq[0].url for rq in self.run.reqplug}
- urls_expected = {self.run.geturl(p) for p in paths_expected}
+ urls_requested = {rq[0].url for rq in run.reqplug}
+ urls_expected = {run.geturl(p) for p in paths_expected}
assert urls_expected <= urls_requested
- scheduled_requests_count = len(self.run.reqplug)
- dropped_requests_count = len(self.run.reqdropped)
- responses_count = len(self.run.respplug)
+ scheduled_requests_count = len(run.reqplug)
+ dropped_requests_count = len(run.reqdropped)
+ responses_count = len(run.respplug)
self.assertEqual(scheduled_requests_count,
dropped_requests_count + responses_count)
- self.assertEqual(len(self.run.reqreached),
+ self.assertEqual(len(run.reqreached),
responses_count)
- def _assert_dropped_requests(self):
- self.assertEqual(len(self.run.reqdropped), 1)
+ def _assert_dropped_requests(self, run: CrawlerRun):
+ self.assertEqual(len(run.reqdropped), 1)
- def _assert_downloaded_responses(self, count):
+ def _assert_downloaded_responses(self, run: CrawlerRun, count):
# response tests
- self.assertEqual(count, len(self.run.respplug))
- self.assertEqual(count, len(self.run.reqreached))
+ self.assertEqual(count, len(run.respplug))
+ self.assertEqual(count, len(run.reqreached))
- for response, _ in self.run.respplug:
- if self.run.getpath(response.url) == '/item999.html':
+ for response, _ in run.respplug:
+ if run.getpath(response.url) == '/item999.html':
self.assertEqual(404, response.status)
- if self.run.getpath(response.url) == '/redirect':
+ if run.getpath(response.url) == '/redirect':
self.assertEqual(302, response.status)
- def _assert_items_error(self):
- self.assertEqual(2, len(self.run.itemerror))
- for item, response, spider, failure in self.run.itemerror:
+ def _assert_items_error(self, run: CrawlerRun):
+ self.assertEqual(2, len(run.itemerror))
+ for item, response, spider, failure in run.itemerror:
self.assertEqual(failure.value.__class__, ZeroDivisionError)
- self.assertEqual(spider, self.run.spider)
+ self.assertEqual(spider, run.spider)
self.assertEqual(item['url'], response.url)
if 'item1.html' in item['url']:
@@ -321,9 +318,9 @@ class EngineTest(unittest.TestCase):
self.assertEqual('Item 2 name', item['name'])
self.assertEqual('200', item['price'])
- def _assert_scraped_items(self):
- self.assertEqual(2, len(self.run.itemresp))
- for item, response in self.run.itemresp:
+ def _assert_scraped_items(self, run: CrawlerRun):
+ self.assertEqual(2, len(run.itemresp))
+ for item, response in run.itemresp:
item = ItemAdapter(item)
self.assertEqual(item['url'], response.url)
if 'item1.html' in item['url']:
@@ -333,26 +330,26 @@ class EngineTest(unittest.TestCase):
self.assertEqual('Item 2 name', item['name'])
self.assertEqual('200', item['price'])
- def _assert_headers_received(self):
- for headers in self.run.headers.values():
+ def _assert_headers_received(self, run: CrawlerRun):
+ for headers in run.headers.values():
self.assertIn(b"Server", headers)
self.assertIn(b"TwistedWeb", headers[b"Server"])
self.assertIn(b"Date", headers)
self.assertIn(b"Content-Type", headers)
- def _assert_bytes_received(self):
- self.assertEqual(9, len(self.run.bytes))
- for request, data in self.run.bytes.items():
+ def _assert_bytes_received(self, run: CrawlerRun):
+ self.assertEqual(9, len(run.bytes))
+ for request, data in run.bytes.items():
joined_data = b"".join(data)
- if self.run.getpath(request.url) == "/":
+ if run.getpath(request.url) == "/":
self.assertEqual(joined_data, get_testdata("test_site", "index.html"))
- elif self.run.getpath(request.url) == "/item1.html":
+ elif run.getpath(request.url) == "/item1.html":
self.assertEqual(joined_data, get_testdata("test_site", "item1.html"))
- elif self.run.getpath(request.url) == "/item2.html":
+ elif run.getpath(request.url) == "/item2.html":
self.assertEqual(joined_data, get_testdata("test_site", "item2.html"))
- elif self.run.getpath(request.url) == "/redirected":
+ elif run.getpath(request.url) == "/redirected":
self.assertEqual(joined_data, b"Redirected here")
- elif self.run.getpath(request.url) == '/redirect':
+ elif run.getpath(request.url) == '/redirect':
self.assertEqual(
joined_data,
b"\n\n"
@@ -364,7 +361,7 @@ class EngineTest(unittest.TestCase):
b" \n"
b"\n"
)
- elif self.run.getpath(request.url) == "/tem999.html":
+ elif run.getpath(request.url) == "/tem999.html":
self.assertEqual(
joined_data,
b"\n\n"
@@ -375,27 +372,27 @@ class EngineTest(unittest.TestCase):
b" \n"
b"
- Name: My image 2 
- Name: My image 3 
- Name: My image 4 
- Name: My image 5 
- 
-