Merge remote-tracking branch 'upstream/master' into tox-pip20.2

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.3.0
+current_version = 2.4.1
 commit = True
 tag = True
 tag_name = {new_version}

.github/workflows/main.yml

@@ -0,0 +1,31 @@
+name: Run test suite
+on: [push, pull_request]
+
+jobs:
+  test-windows:
+    name: "Windows Tests"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest]
+        python-version: [3.7, 3.8]
+        env: [TOXENV: py]
+        include:
+        - os: windows-latest
+          python-version: 3.6
+          env:
+            TOXENV: windows-pinned
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Run test suite
+      env: ${{ matrix.env }}
+      run: |
+        pip install -U tox twine wheel codecov
+        tox

azure-pipelines.yml

@@ -1,22 +0,0 @@
-variables:
-  TOXENV: py
-pool:
-  vmImage: 'windows-latest'
-strategy:
-  matrix:
-    Python36:
-      python.version: '3.6'
-      TOXENV: windows-pinned
-    Python37:
-      python.version: '3.7'
-    Python38:
-      python.version: '3.8'
-steps:
-- task: UsePythonVersion@0
-  inputs:
-    versionSpec: '$(python.version)'
-  displayName: 'Use Python $(python.version)'
-- script: |
-    pip install -U tox twine wheel codecov
-    tox
-  displayName: 'Run test suite'

docs/conf.py

@@ -283,6 +283,7 @@ coverage_ignore_pyobjects = [
 intersphinx_mapping = {
     'attrs': ('https://www.attrs.org/en/stable/', None),
     'coverage': ('https://coverage.readthedocs.io/en/stable', None),
+    'cryptography' : ('https://cryptography.io/en/latest/', None),
     'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
     'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None),
     'pytest': ('https://docs.pytest.org/en/latest', None),

docs/contributing.rst

@@ -140,7 +140,7 @@ original pull request author hasn't had time to address them.
 In this case consider picking up this pull request: open
 a new pull request with all commits from the original pull request, as well as
 additional changes to address the raised issues. Doing so helps a lot; it is
-not considered rude as soon as the original author is acknowledged by keeping
+not considered rude as long as the original author is acknowledged by keeping
 his/her commits.
 
 You can pull an existing pull request to a local branch

docs/index.rst

@@ -78,7 +78,6 @@ Basic concepts
    topics/settings
    topics/exceptions
 
-
 :doc:`topics/commands`
     Learn about the command-line tool used to manage your Scrapy project.
 

docs/intro/install.rst

@@ -69,10 +69,9 @@ In case of any trouble related to these dependencies,
 please refer to their respective installation instructions:
 
 * `lxml installation`_
-* `cryptography installation`_
+* :doc:`cryptography installation <cryptography:installation>`
 
 .. _lxml installation: https://lxml.de/installation.html
-.. _cryptography installation: https://cryptography.io/en/latest/installation/
 
 
 .. _intro-using-virtualenv:
@@ -265,7 +264,6 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
 .. _cryptography: https://cryptography.io/en/latest/
 .. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/
 .. _setuptools: https://pypi.python.org/pypi/setuptools
-.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
 .. _homebrew: https://brew.sh/
 .. _zsh: https://www.zsh.org/
 .. _Scrapinghub: https://scrapinghub.com

docs/news.rst

@@ -3,6 +3,333 @@
 Release notes
 =============
 
+.. _release-2.4.1:
+
+Scrapy 2.4.1 (2020-11-17)
+-------------------------
+
+-   Fixed :ref:`feed exports <topics-feed-exports>` overwrite support (:issue:`4845`, :issue:`4857`, :issue:`4859`)
+
+-   Fixed the AsyncIO event loop handling, which could make code hang
+    (:issue:`4855`, :issue:`4872`)
+
+-   Fixed the IPv6-capable DNS resolver
+    :class:`~scrapy.resolver.CachingHostnameResolver` for download handlers
+    that call
+    :meth:`reactor.resolve <twisted.internet.interfaces.IReactorCore.resolve>`
+    (:issue:`4802`, :issue:`4803`)
+
+-   Fixed the output of the :command:`genspider` command showing placeholders
+    instead of the import path of the generated spider module (:issue:`4874`)
+
+-   Migrated Windows CI from Azure Pipelines to GitHub Actions (:issue:`4869`,
+    :issue:`4876`)
+
+
+.. _release-2.4.0:
+
+Scrapy 2.4.0 (2020-10-11)
+-------------------------
+
+Highlights:
+
+*   Python 3.5 support has been dropped.
+
+*   The ``file_path`` method of :ref:`media pipelines <topics-media-pipeline>`
+    can now access the source :ref:`item <topics-items>`.
+
+    This allows you to set a download file path based on item data.
+
+*   The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows
+    to define keyword parameters to pass to :ref:`item exporter classes
+    <topics-exporters>`
+
+*   You can now choose whether :ref:`feed exports <topics-feed-exports>`
+    overwrite or append to the output file.
+
+    For example, when using the :command:`crawl` or :command:`runspider`
+    commands, you can use the ``-O`` option instead of ``-o`` to overwrite the
+    output file.
+
+*   Zstd-compressed responses are now supported if zstandard_ is installed.
+
+*   In settings, where the import path of a class is required, it is now
+    possible to pass a class object instead.
+
+Modified requirements
+~~~~~~~~~~~~~~~~~~~~~
+
+*   Python 3.6 or greater is now required; support for Python 3.5 has been
+    dropped
+
+    As a result:
+
+    -   When using PyPy, PyPy 7.2.0 or greater :ref:`is now required
+        <faq-python-versions>`
+
+    -   For Amazon S3 storage support in :ref:`feed exports
+        <topics-feed-storage-s3>` or :ref:`media pipelines
+        <media-pipelines-s3>`, botocore_ 1.4.87 or greater is now required
+
+    -   To use the :ref:`images pipeline <images-pipeline>`, Pillow_ 4.0.0 or
+        greater is now required
+
+    (:issue:`4718`, :issue:`4732`, :issue:`4733`, :issue:`4742`, :issue:`4743`,
+    :issue:`4764`)
+
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+*   :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` once again
+    discards cookies defined in :attr:`Request.headers
+    <scrapy.http.Request.headers>`.
+
+    We decided to revert this bug fix, introduced in Scrapy 2.2.0, because it
+    was reported that the current implementation could break existing code.
+
+    If you need to set cookies for a request, use the :class:`Request.cookies
+    <scrapy.http.Request>` parameter.
+
+    A future version of Scrapy will include a new, better implementation of the
+    reverted bug fix.
+
+    (:issue:`4717`, :issue:`4823`)
+
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+*   :class:`scrapy.extensions.feedexport.S3FeedStorage` no longer reads the
+    values of ``access_key`` and ``secret_key`` from the running project
+    settings when they are not passed to its ``__init__`` method; you must
+    either pass those parameters to its ``__init__`` method or use
+    :class:`S3FeedStorage.from_crawler
+    <scrapy.extensions.feedexport.S3FeedStorage.from_crawler>`
+    (:issue:`4356`, :issue:`4411`, :issue:`4688`)
+
+*   :attr:`Rule.process_request <scrapy.spiders.crawl.Rule.process_request>`
+    no longer admits callables which expect a single ``request`` parameter,
+    rather than both ``request`` and ``response`` (:issue:`4818`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+*   In custom :ref:`media pipelines <topics-media-pipeline>`, signatures that
+    do not accept a keyword-only ``item`` parameter in any of the  methods that
+    :ref:`now support this parameter <media-pipeline-item-parameter>` are now
+    deprecated (:issue:`4628`, :issue:`4686`)
+
+*   In custom :ref:`feed storage backend classes <topics-feed-storage>`,
+    ``__init__`` method signatures that do not accept a keyword-only
+    ``feed_options`` parameter are now deprecated (:issue:`547`, :issue:`716`,
+    :issue:`4512`)
+
+*   The :class:`scrapy.utils.python.WeakKeyCache` class is now deprecated
+    (:issue:`4684`, :issue:`4701`)
+
+*   The :func:`scrapy.utils.boto.is_botocore` function is now deprecated, use
+    :func:`scrapy.utils.boto.is_botocore_available` instead (:issue:`4734`,
+    :issue:`4776`)
+
+
+New features
+~~~~~~~~~~~~
+
+.. _media-pipeline-item-parameter:
+
+*   The following methods of :ref:`media pipelines <topics-media-pipeline>` now
+    accept an ``item`` keyword-only parameter containing the source
+    :ref:`item <topics-items>`:
+
+    -   In :class:`scrapy.pipelines.files.FilesPipeline`:
+
+        -   :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded`
+
+        -   :meth:`~scrapy.pipelines.files.FilesPipeline.file_path`
+
+        -   :meth:`~scrapy.pipelines.files.FilesPipeline.media_downloaded`
+
+        -   :meth:`~scrapy.pipelines.files.FilesPipeline.media_to_download`
+
+    -   In :class:`scrapy.pipelines.images.ImagesPipeline`:
+
+        -   :meth:`~scrapy.pipelines.images.ImagesPipeline.file_downloaded`
+
+        -   :meth:`~scrapy.pipelines.images.ImagesPipeline.file_path`
+
+        -   :meth:`~scrapy.pipelines.images.ImagesPipeline.get_images`
+
+        -   :meth:`~scrapy.pipelines.images.ImagesPipeline.image_downloaded`
+
+        -   :meth:`~scrapy.pipelines.images.ImagesPipeline.media_downloaded`
+
+        -   :meth:`~scrapy.pipelines.images.ImagesPipeline.media_to_download`
+
+    (:issue:`4628`, :issue:`4686`)
+
+*   The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows
+    to define keyword parameters to pass to :ref:`item exporter classes
+    <topics-exporters>` (:issue:`4606`, :issue:`4768`)
+
+*   :ref:`Feed exports <topics-feed-exports>` gained overwrite support:
+
+    *   When using the :command:`crawl` or :command:`runspider` commands, you
+        can use the ``-O`` option instead of ``-o`` to overwrite the output
+        file
+
+    *   You can use the ``overwrite`` key in the :setting:`FEEDS` setting to
+        configure whether to overwrite the output file (``True``) or append to
+        its content (``False``)
+
+    *   The ``__init__`` and ``from_crawler`` methods of :ref:`feed storage
+        backend classes <topics-feed-storage>` now receive a new keyword-only
+        parameter, ``feed_options``, which is a dictionary of :ref:`feed
+        options <feed-options>`
+
+    (:issue:`547`, :issue:`716`, :issue:`4512`)
+
+*   Zstd-compressed responses are now supported if zstandard_ is installed
+    (:issue:`4831`)
+
+*   In settings, where the import path of a class is required, it is now
+    possible to pass a class object instead (:issue:`3870`, :issue:`3873`).
+
+    This includes also settings where only part of its value is made of an
+    import path, such as :setting:`DOWNLOADER_MIDDLEWARES` or
+    :setting:`DOWNLOAD_HANDLERS`.
+
+*   :ref:`Downloader middlewares <topics-downloader-middleware>` can now
+    override :class:`response.request <scrapy.http.Response.request>`.
+
+    If a :ref:`downloader middleware <topics-downloader-middleware>` returns
+    a :class:`~scrapy.http.Response` object from
+    :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`
+    or
+    :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`
+    with a custom :class:`~scrapy.http.Request` object assigned to
+    :class:`response.request <scrapy.http.Response.request>`:
+
+    -   The response is handled by the callback of that custom
+        :class:`~scrapy.http.Request` object, instead of being handled by the
+        callback of the original :class:`~scrapy.http.Request` object
+
+    -   That custom :class:`~scrapy.http.Request` object is now sent as the
+        ``request`` argument to the :signal:`response_received` signal, instead
+        of the original :class:`~scrapy.http.Request` object
+
+    (:issue:`4529`, :issue:`4632`)
+
+*   When using the :ref:`FTP feed storage backend <topics-feed-storage-ftp>`:
+
+    -   It is now possible to set the new ``overwrite`` :ref:`feed option
+        <feed-options>` to ``False`` to append to an existing file instead of
+        overwriting it
+
+    -   The FTP password can now be omitted if it is not necessary
+
+    (:issue:`547`, :issue:`716`, :issue:`4512`)
+
+*   The ``__init__`` method of :class:`~scrapy.exporters.CsvItemExporter` now
+    supports an ``errors`` parameter to indicate how to handle encoding errors
+    (:issue:`4755`)
+
+*   When :ref:`using asyncio <using-asyncio>`, it is now possible to
+    :ref:`set a custom asyncio loop <using-custom-loops>` (:issue:`4306`,
+    :issue:`4414`)
+
+*   Serialized requests (see :ref:`topics-jobs`) now support callbacks that are
+    spider methods that delegate on other callable (:issue:`4756`)
+
+*   When a response is larger than :setting:`DOWNLOAD_MAXSIZE`, the logged
+    message is now a warning, instead of an error (:issue:`3874`,
+    :issue:`3886`, :issue:`4752`)
+
+
+Bug fixes
+~~~~~~~~~
+
+*   The :command:`genspider` command no longer overwrites existing files
+    unless the ``--force`` option is used (:issue:`4561`, :issue:`4616`,
+    :issue:`4623`)
+
+*   Cookies with an empty value are no longer considered invalid cookies
+    (:issue:`4772`)
+
+*   The :command:`runspider` command now supports files with the ``.pyw`` file
+    extension (:issue:`4643`, :issue:`4646`)
+
+*   The :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
+    middleware now simply ignores unsupported proxy values (:issue:`3331`,
+    :issue:`4778`)
+
+*   Checks for generator callbacks with a ``return`` statement no longer warn
+    about ``return`` statements in nested functions (:issue:`4720`,
+    :issue:`4721`)
+
+*   The system file mode creation mask no longer affects the permissions of
+    files generated using the :command:`startproject` command (:issue:`4722`)
+
+*   :func:`scrapy.utils.iterators.xmliter` now supports namespaced node names
+    (:issue:`861`, :issue:`4746`)
+
+*   :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can
+    work when using a headless browser (:issue:`4835`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+*   The :setting:`FEED_URI_PARAMS` setting is now documented (:issue:`4671`,
+    :issue:`4724`)
+
+*   Improved the documentation of
+    :ref:`link extractors <topics-link-extractors>` with an usage example from
+    a spider callback and reference documentation for the
+    :class:`~scrapy.link.Link` class (:issue:`4751`, :issue:`4775`)
+
+*   Clarified the impact of :setting:`CONCURRENT_REQUESTS` when using the
+    :class:`~scrapy.extensions.closespider.CloseSpider` extension
+    (:issue:`4836`)
+
+*   Removed references to Python 2’s ``unicode`` type (:issue:`4547`,
+    :issue:`4703`)
+
+*   We now have an :ref:`official deprecation policy <deprecation-policy>`
+    (:issue:`4705`)
+
+*   Our :ref:`documentation policies <documentation-policies>` now cover usage
+    of Sphinx’s :rst:dir:`versionadded` and :rst:dir:`versionchanged`
+    directives, and we have removed usages referencing Scrapy 1.4.0 and earlier
+    versions (:issue:`3971`, :issue:`4310`)
+
+*   Other documentation cleanups (:issue:`4090`, :issue:`4782`, :issue:`4800`,
+    :issue:`4801`, :issue:`4809`, :issue:`4816`, :issue:`4825`)
+
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+*   Extended typing hints (:issue:`4243`, :issue:`4691`)
+
+*   Added tests for the :command:`check` command (:issue:`4663`)
+
+*   Fixed test failures on Debian (:issue:`4726`, :issue:`4727`, :issue:`4735`)
+
+*   Improved Windows test coverage (:issue:`4723`)
+
+*   Switched to :ref:`formatted string literals <f-strings>` where possible
+    (:issue:`4307`, :issue:`4324`, :issue:`4672`)
+
+*   Modernized :func:`super` usage (:issue:`4707`)
+
+*   Other code and test cleanups (:issue:`1790`, :issue:`3288`, :issue:`4165`,
+    :issue:`4564`, :issue:`4651`, :issue:`4714`, :issue:`4738`, :issue:`4745`,
+    :issue:`4747`, :issue:`4761`, :issue:`4765`, :issue:`4804`, :issue:`4817`,
+    :issue:`4820`, :issue:`4822`, :issue:`4839`)
+
+
 .. _release-2.3.0:
 
 Scrapy 2.3.0 (2020-08-04)
@@ -4008,9 +4335,9 @@ First release of Scrapy.
 .. _six: https://six.readthedocs.io/
 .. _tox: https://pypi.org/project/tox/
 .. _Twisted: https://twistedmatrix.com/trac/
-.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
 .. _w3lib: https://github.com/scrapy/w3lib
 .. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
 .. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
 .. _zope.interface: https://zopeinterface.readthedocs.io/en/latest/
 .. _Zsh: https://www.zsh.org/
+.. _zstandard: https://pypi.org/project/zstandard/

docs/topics/asyncio.rst

@@ -1,3 +1,5 @@
+.. _using-asyncio:
+
 =======
 asyncio
 =======

docs/topics/downloader-middleware.rst

@@ -207,6 +207,11 @@ CookiesMiddleware
       a warning. Refer to :ref:`topics-logging-advanced-customization`
       to customize the logging behaviour.
 
+   .. caution:: Cookies set via the ``Cookie`` header are not considered by the
+      :ref:`cookies-mw`. If you need to set cookies for a request, use the
+      :class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
+      current limitation that is being worked on.
+
 The following settings can be used to configure the cookie middleware:
 
 * :setting:`COOKIES_ENABLED`
@@ -684,11 +689,14 @@ HttpCompressionMiddleware
    This middleware allows compressed (gzip, deflate) traffic to be
    sent/received from web sites.
 
-   This middleware also supports decoding `brotli-compressed`_ responses,
+   This middleware also supports decoding `brotli-compressed`_ as well as
-   provided `brotlipy`_ is installed.
+   `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is
+   installed, respectively.
 
 .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
 .. _brotlipy: https://pypi.org/project/brotlipy/
+.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
+.. _zstandard: https://pypi.org/project/zstandard/
 
 HttpCompressionMiddleware Settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/topics/exporters.rst

@@ -123,7 +123,7 @@ Example::
           def serialize_field(self, field, name, value):
               if field == 'price':
                   return f'$ {str(value)}'
-              return super(Product, self).serialize_field(field, name, value)
+              return super().serialize_field(field, name, value)
 
 .. _topics-exporters-reference:
 

docs/topics/extensions.rst

@@ -257,6 +257,12 @@ settings:
 * :setting:`CLOSESPIDER_PAGECOUNT`
 * :setting:`CLOSESPIDER_ERRORCOUNT`
 
+.. note::
+
+   When a certain closing condition is met, requests which are 
+   currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS` 
+   requests) are still processed.
+
 .. setting:: CLOSESPIDER_TIMEOUT
 
 CLOSESPIDER_TIMEOUT
@@ -279,8 +285,6 @@ Default: ``0``
 An integer which specifies a number of items. If the spider scrapes more than
 that amount and those items are passed by the item pipeline, the
 spider will be closed with the reason ``closespider_itemcount``.
-Requests which  are currently in the downloader queue (up to
-:setting:`CONCURRENT_REQUESTS` requests) are still processed.
 If zero (or non set), spiders won't be closed by number of passed items.
 
 .. setting:: CLOSESPIDER_PAGECOUNT

docs/topics/feed-exports.rst

@@ -184,7 +184,7 @@ The feeds are stored on `Amazon S3`_.
    * ``s3://mybucket/path/to/export.csv``
    * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
 
- * Required external libraries: `botocore`_
+ * Required external libraries: `botocore`_ >= 1.4.87
 
 The AWS credentials can be passed as user/password in the URI, or they can be
 passed through the following settings:
@@ -303,6 +303,9 @@ For instance::
             'store_empty': False,
             'fields': None,
             'indent': 4,
+            'item_export_kwargs': {
+               'export_empty_fields': True,
+            },
         }, 
         '/home/user/documents/items.xml': {
             'format': 'xml',
@@ -316,6 +319,8 @@ For instance::
         },
     }
 
+.. _feed-options:
+
 The following is a list of the accepted keys and the setting that is used
 as a fallback value if that key is not provided for a specific feed definition:
 
@@ -326,12 +331,18 @@ as a fallback value if that key is not provided for a specific feed definition:
 -   ``batch_item_count``: falls back to
     :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
 
+    .. versionadded:: 2.3.0
+
 -   ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`.
 
 -   ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.
 
 -   ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
 
+-   ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.
+
+    .. versionadded:: 2.4.0
+
 -   ``overwrite``: whether to overwrite the file if it already exists
     (``True``) or append to its content (``False``).
 
@@ -350,6 +361,8 @@ as a fallback value if that key is not provided for a specific feed definition:
 
     -   :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
 
+    .. versionadded:: 2.4.0
+
 -   ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`.
 
 -   ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.
@@ -512,7 +525,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 .. setting:: FEED_EXPORT_BATCH_ITEM_COUNT
 
 FEED_EXPORT_BATCH_ITEM_COUNT
------------------------------
+----------------------------
+
+.. versionadded:: 2.3.0
 
 Default: ``0``
 
@@ -581,11 +596,15 @@ The function signature should be as follows:
             If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id``
             is always ``1``.
 
+            .. versionadded:: 2.3.0
+
         -   ``batch_time``: UTC date and time, in ISO format with ``:``
             replaced with ``-``.
 
             See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
 
+            .. versionadded:: 2.3.0
+
         -   ``time``: ``batch_time``, with microseconds set to ``0``.
    :type params: dict
 

docs/topics/link-extractors.rst

@@ -10,12 +10,19 @@ The ``__init__`` method of
 :class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` takes settings that
 determine which links may be extracted. :class:`LxmlLinkExtractor.extract_links
 <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` returns a
-list of matching :class:`scrapy.link.Link` objects from a
+list of matching :class:`~scrapy.link.Link` objects from a
 :class:`~scrapy.http.Response` object.
 
 Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders
-through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link
+through a set of :class:`~scrapy.spiders.Rule` objects.
-extractors in regular spiders.
+
+You can also use link extractors in regular spiders. For example, you can instantiate
+:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` into a class
+variable in your spider, and use it from your spider callbacks::
+
+    def parse(self, response):
+        for link in self.link_extractor.extract_links(response):
+            yield Request(link.url, callback=self.parse)
 
 .. _topics-link-extractors-ref:
 
@@ -145,4 +152,12 @@ LxmlLinkExtractor
 
     .. automethod:: extract_links
 
+Link
+----
+
+.. module:: scrapy.link
+   :synopsis: Link from link extractors
+
+.. autoclass:: Link
+
 .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py

docs/topics/media-pipeline.rst

@@ -56,6 +56,8 @@ this:
    error will be logged and the file won't be present in the ``files`` field.
 
 
+.. _images-pipeline:
+
 Using the Images Pipeline
 =========================
 
@@ -68,14 +70,10 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
 can configure some extra functions like generating thumbnails and filtering
 the images based on their size.
 
-The Images Pipeline uses `Pillow`_ for thumbnailing and normalizing images to
+The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for
-JPEG/RGB format, so you need to install this library in order to use it.
+thumbnailing and normalizing images to JPEG/RGB format.
-`Python Imaging Library`_ (PIL) should also work in most cases, but it is known
-to cause troubles in some setups, so we recommend to use `Pillow`_ instead of
-PIL.
 
 .. _Pillow: https://github.com/python-pillow/Pillow
-.. _Python Imaging Library: http://www.pythonware.com/products/pil/
 
 
 .. _topics-media-pipeline-enabling:
@@ -164,14 +162,17 @@ FTP supports two different connection modes: active or passive. Scrapy uses
 the passive connection mode by default. To use the active connection mode instead,
 set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
 
+.. _media-pipelines-s3:
+
 Amazon S3 storage
 -----------------
 
 .. setting:: FILES_STORE_S3_ACL
 .. setting:: IMAGES_STORE_S3_ACL
 
-:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3
+If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and
-bucket. Scrapy will automatically upload the files to the bucket.
+:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will
+automatically upload the files to the bucket.
 
 For example, this is a valid :setting:`IMAGES_STORE` value::
 
@@ -187,8 +188,9 @@ policy::
 
 For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
 
-Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like
+You can also use other S3-like storages. Storages like self-hosted `Minio`_ or
-self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
+`s3.scality`_. All you need to do is set endpoint option in you Scrapy
+settings::
 
     AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
 
@@ -197,9 +199,10 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
     AWS_USE_SSL = False # or True (None by default)
     AWS_VERIFY = False # or True (None by default)
 
+.. _botocore: https://github.com/boto/botocore
+.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 .. _Minio: https://github.com/minio/minio
 .. _s3.scality: https://s3.scality.com/
-.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
 
 .. _media-pipeline-gcs:
@@ -446,6 +449,9 @@ See here the methods that you can override in your custom Files Pipeline:
       By default the :meth:`file_path` method returns
       ``full/<request URL hash>.<extension>``.
 
+      .. versionadded:: 2.4
+         The *item* parameter.
+
    .. method:: FilesPipeline.get_media_requests(item, info)
 
       As seen on the workflow, the pipeline will get the URLs of the images to
@@ -582,6 +588,9 @@ See here the methods that you can override in your custom Images Pipeline:
       By default the :meth:`file_path` method returns
       ``full/<request URL hash>.<extension>``.
 
+      .. versionadded:: 2.4
+         The *item* parameter.
+
    .. method:: ImagesPipeline.get_media_requests(item, info)
 
       Works the same way as :meth:`FilesPipeline.get_media_requests` method,

docs/topics/request-response.rst

@@ -61,6 +61,12 @@ Request objects
     :param headers: the headers of this request. The dict values can be strings
        (for single valued headers) or lists (for multi-valued headers). If
        ``None`` is passed as value, the HTTP header will not be sent at all.
+
+        .. caution:: Cookies set via the ``Cookie`` header are not considered by the
+            :ref:`cookies-mw`. If you need to set cookies for a request, use the
+            :class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
+            current limitation that is being worked on.
+
     :type headers: dict
 
     :param cookies: the request cookies. These can be sent in two forms.
@@ -102,6 +108,12 @@ Request objects
             )
 
         For more info see :ref:`cookies-mw`.
+
+        .. caution:: Cookies set via the ``Cookie`` header are not considered by the
+            :ref:`cookies-mw`. If you need to set cookies for a request, use the
+            :class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
+            current limitation that is being worked on.
+
     :type cookies: dict or list
 
     :param encoding: the encoding of this request (defaults to ``'utf-8'``).
@@ -681,9 +693,19 @@ Response objects
     :param ip_address: The IP address of the server from which the Response originated.
     :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
 
+    :param protocol: The protocol that was used to download the response.
+        For instance: "HTTP/1.0", "HTTP/1.1"
+    :type protocol: :class:`str`
+
+    .. versionadded:: 2.0.0
+       The ``certificate`` parameter.
+
     .. versionadded:: 2.1.0
        The ``ip_address`` parameter.
 
+    .. versionadded:: VERSION
+       The ``protocol`` parameter.
+
     .. attribute:: Response.url
 
         A string containing the URL of the response.
@@ -768,6 +790,8 @@ Response objects
 
     .. attribute:: Response.certificate
 
+        .. versionadded:: 2.0.0
+
         A :class:`twisted.internet.ssl.Certificate` object representing
         the server's SSL certificate.
 
@@ -783,6 +807,17 @@ Response objects
         handler, i.e. for ``http(s)`` responses. For other handlers,
         :attr:`ip_address` is always ``None``.
 
+    .. attribute:: Response.protocol
+
+        .. versionadded:: VERSION
+
+        The protocol that was used to download the response.
+        For instance: "HTTP/1.0", "HTTP/1.1"
+
+        This attribute is currently only populated by the HTTP download
+        handlers, i.e. for ``http(s)`` responses. For other handlers,
+        :attr:`protocol` is always ``None``.
+
     .. method:: Response.copy()
 
        Returns a new Response which is a copy of this Response.

docs/topics/settings.rst

@@ -102,7 +102,7 @@ module and documented in the :ref:`topics-settings-ref` section.
 Import paths and classes
 ========================
 
-.. versionadded:: VERSION
+.. versionadded:: 2.4.0
 
 When a setting references a callable object to be imported by Scrapy, such as a
 class or a function, there are two different ways you can specify that object:
@@ -249,19 +249,25 @@ ASYNCIO_EVENT_LOOP
 
 Default: ``None``
 
-Import path of a given asyncio event loop class.
+Import path of a given ``asyncio`` event loop class.
 
-If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the 
+If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the
-asyncio event loop to be used with it. Set the setting to the import path of the 
+asyncio event loop to be used with it. Set the setting to the import path of the
 desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
 event loop will be used.
 
 If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor`
-function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop 
+function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
-class to be used.  
+class to be used.
 
 Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`.
 
+.. caution:: Please be aware that, when using a non-default event loop
+    (either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with
+    :func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call
+    :func:`asyncio.set_event_loop`, which will set the specified event loop
+    as the current loop for the current OS thread.
+
 .. setting:: BOT_NAME
 
 BOT_NAME
@@ -352,6 +358,11 @@ Default::
 The default headers used for Scrapy HTTP Requests. They're populated in the
 :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`.
 
+.. caution:: Cookies set via the ``Cookie`` header are not considered by the
+    :ref:`cookies-mw`. If you need to set cookies for a request, use the
+    :class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
+    current limitation that is being worked on.
+
 .. setting:: DEPTH_LIMIT
 
 DEPTH_LIMIT

pytest.ini

@@ -1,4 +1,5 @@
 [pytest]
+xfail_strict = true
 usefixtures = chdir
 python_files=test_*.py __init__.py
 python_classes=
@@ -35,9 +36,5 @@ flake8-ignore =
     scrapy/spiders/__init__.py E402 F401
 
     # Issues pending a review:
-    scrapy/utils/http.py F403
-    scrapy/utils/markup.py F403
-    scrapy/utils/multipart.py F403
     scrapy/utils/url.py F403 F405
     tests/test_loader.py E741
-    

scrapy/VERSION

@@ -1 +1 @@
-2.3.0
+2.4.1

scrapy/commands/genspider.py

@@ -98,7 +98,7 @@ class Command(ScrapyCommand):
         print(f"Created spider {name!r} using template {template_name!r} ",
               end=('' if spiders_module else '\n'))
         if spiders_module:
-            print("in module:\n  {spiders_module.__name__}.{module}")
+            print(f"in module:\n  {spiders_module.__name__}.{module}")
 
     def _find_template(self, template):
         template_file = join(self.templates_dir, f'{template}.tmpl')

scrapy/commands/runspider.py

@@ -11,7 +11,7 @@ def _import_file(filepath):
     abspath = os.path.abspath(filepath)
     dirname, file = os.path.split(abspath)
     fname, fext = os.path.splitext(file)
-    if fext != '.py':
+    if fext not in ('.py', '.pyw'):
         raise ValueError(f"Not a Python source file: {abspath}")
     if dirname:
         sys.path = [dirname] + sys.path

scrapy/core/downloader/handlers/http11.py

@@ -434,6 +434,11 @@ class ScrapyAgent:
     def _cb_bodydone(self, result, request, url):
         headers = Headers(result["txresponse"].headers.getAllRawHeaders())
         respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
+        try:
+            version = result["txresponse"].version
+            protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}"
+        except (AttributeError, TypeError, IndexError):
+            protocol = None
         response = respcls(
             url=url,
             status=int(result["txresponse"].code),
@@ -442,6 +447,7 @@ class ScrapyAgent:
             flags=result["flags"],
             certificate=result["certificate"],
             ip_address=result["ip_address"],
+            protocol=protocol,
         )
         if result.get("failure"):
             result["failure"].value.response = response

scrapy/core/downloader/tls.py

@@ -5,7 +5,6 @@ from service_identity.exceptions import CertificateError
 from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError
 from twisted.internet.ssl import AcceptableCiphers
 
-from scrapy import twisted_version
 from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
 
 
@@ -28,13 +27,6 @@ openssl_methods = {
 }
 
 
-if twisted_version < (17, 0, 0):
-    from twisted.internet._sslverify import _maybeSetHostNameIndication as set_tlsext_host_name
-else:
-    def set_tlsext_host_name(connection, hostNameBytes):
-        connection.set_tlsext_host_name(hostNameBytes)
-
-
 class ScrapyClientTLSOptions(ClientTLSOptions):
     """
     SSL Client connection creator ignoring certificate verification errors
@@ -52,21 +44,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
 
     def _identityVerifyingInfoCallback(self, connection, where, ret):
         if where & SSL.SSL_CB_HANDSHAKE_START:
-            set_tlsext_host_name(connection, self._hostnameBytes)
+            connection.set_tlsext_host_name(self._hostnameBytes)
         elif where & SSL.SSL_CB_HANDSHAKE_DONE:
             if self.verbose_logging:
-                if hasattr(connection, 'get_cipher_name'):  # requires pyOPenSSL 0.15
+                logger.debug('SSL connection to %s using protocol %s, cipher %s',
-                    if hasattr(connection, 'get_protocol_version_name'):  # requires pyOPenSSL 16.0.0
+                             self._hostnameASCII,
-                        logger.debug('SSL connection to %s using protocol %s, cipher %s',
+                             connection.get_protocol_version_name(),
-                                     self._hostnameASCII,
+                             connection.get_cipher_name(),
-                                     connection.get_protocol_version_name(),
+                             )
-                                     connection.get_cipher_name(),
-                                     )
-                    else:
-                        logger.debug('SSL connection to %s using cipher %s',
-                                     self._hostnameASCII,
-                                     connection.get_cipher_name(),
-                                     )
                 server_cert = connection.get_peer_certificate()
                 logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
                              x509name_to_string(server_cert.get_issuer()),

scrapy/core/downloader/webclient.py

@@ -7,7 +7,7 @@ from twisted.internet.protocol import ClientFactory
 
 from scrapy.http import Headers
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_bytes
+from scrapy.utils.python import to_bytes, to_unicode
 from scrapy.responsetypes import responsetypes
 
 
@@ -110,7 +110,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
         status = int(self.status)
         headers = Headers(self.response_headers)
         respcls = responsetypes.from_args(headers=headers, url=self._url)
-        return respcls(url=self._url, status=status, headers=headers, body=body)
+        return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))
 
     def _set_connection_attributes(self, request):
         parsed = urlparse_cached(request)

scrapy/core/scheduler.py

@@ -1,14 +1,10 @@
 import os
 import json
 import logging
-import warnings
 from os.path import join, exists
 
-from queuelib import PriorityQueue
-
 from scrapy.utils.misc import load_object, create_instance
 from scrapy.utils.job import job_dir
-from scrapy.utils.deprecate import ScrapyDeprecationWarning
 
 
 logger = logging.getLogger(__name__)
@@ -56,14 +52,6 @@ class Scheduler:
         dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
         dupefilter = create_instance(dupefilter_cls, settings, crawler)
         pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
-        if pqclass is PriorityQueue:
-            warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'"
-                          " is no longer supported because of API changes; "
-                          "please use 'scrapy.pqueues.ScrapyPriorityQueue'",
-                          ScrapyDeprecationWarning)
-            from scrapy.pqueues import ScrapyPriorityQueue
-            pqclass = ScrapyPriorityQueue
-
         dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
         mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
         logunser = settings.getbool('SCHEDULER_DEBUG')

scrapy/crawler.py

@@ -180,9 +180,9 @@ class CrawlerRunner:
         :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance,
             :class:`~scrapy.spiders.Spider` subclass or string
 
-        :param list args: arguments to initialize the spider
+        :param args: arguments to initialize the spider
 
-        :param dict kwargs: keyword arguments to initialize the spider
+        :param kwargs: keyword arguments to initialize the spider
         """
         if isinstance(crawler_or_spidercls, Spider):
             raise ValueError(

scrapy/downloadermiddlewares/cookies.py

@@ -97,35 +97,14 @@ class CookiesMiddleware:
 
     def _get_request_cookies(self, jar, request):
         """
-        Extract cookies from a Request. Values from the `Request.cookies` attribute
+        Extract cookies from the Request.cookies attribute
-        take precedence over values from the `Cookie` request header.
         """
-        def get_cookies_from_header(jar, request):
+        if not request.cookies:
-            cookie_header = request.headers.get("Cookie")
+            return []
-            if not cookie_header:
+        elif isinstance(request.cookies, dict):
-                return []
+            cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
-            cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";"))
+        else:
-            cookie_list_unicode = []
+            cookies = request.cookies
-            for cookie_bytes in cookie_gen_bytes:
+        formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
-                try:
+        response = Response(request.url, headers={"Set-Cookie": formatted})
-                    cookie_unicode = cookie_bytes.decode("utf8")
+        return jar.make_cookies(response, request)
-                except UnicodeDecodeError:
-                    logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
-                                   request, cookie_bytes)
-                    cookie_unicode = cookie_bytes.decode("latin1", errors="replace")
-                cookie_list_unicode.append(cookie_unicode)
-            response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode})
-            return jar.make_cookies(response, request)
-
-        def get_cookies_from_attribute(jar, request):
-            if not request.cookies:
-                return []
-            elif isinstance(request.cookies, dict):
-                cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
-            else:
-                cookies = request.cookies
-            formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
-            response = Response(request.url, headers={"Set-Cookie": formatted})
-            return jar.make_cookies(response, request)
-
-        return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request)

scrapy/downloadermiddlewares/httpcompression.py

@@ -1,3 +1,4 @@
+import io
 import zlib
 
 from scrapy.utils.gz import gunzip
@@ -14,6 +15,12 @@ try:
 except ImportError:
     pass
 
+try:
+    import zstandard
+    ACCEPTED_ENCODINGS.append(b'zstd')
+except ImportError:
+    pass
+
 
 class HttpCompressionMiddleware:
     """This middleware allows compressed (gzip, deflate) traffic to be
@@ -67,4 +74,9 @@ class HttpCompressionMiddleware:
                 body = zlib.decompress(body, -15)
         if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS:
             body = brotli.decompress(body)
+        if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS:
+            # Using its streaming API since its simple API could handle only cases
+            # where there is content size data embedded in the frame
+            reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body))
+            body = reader.read()
         return body

scrapy/downloadermiddlewares/httpproxy.py

@@ -13,7 +13,12 @@ class HttpProxyMiddleware:
         self.auth_encoding = auth_encoding
         self.proxies = {}
         for type_, url in getproxies().items():
-            self.proxies[type_] = self._get_proxy(url, type_)
+            try:
+                self.proxies[type_] = self._get_proxy(url, type_)
+            # some values such as '/var/run/docker.sock' can't be parsed
+            # by _parse_proxy and as such should be skipped
+            except ValueError:
+                continue
 
     @classmethod
     def from_crawler(cls, crawler):

scrapy/extensions/feedexport.py

@@ -319,18 +319,26 @@ class FeedExporter:
         # Use `largs=log_args` to copy log_args into function's scope
         # instead of using `log_args` from the outer scope
         d.addCallback(
-            lambda _, largs=log_args: logger.info(
+            self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__
-                logfmt % "Stored", largs, extra={'spider': spider}
-            )
         )
         d.addErrback(
-            lambda f, largs=log_args: logger.error(
+            self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__
-                logfmt % "Error storing", largs,
-                exc_info=failure_to_exc_info(f), extra={'spider': spider}
-            )
         )
         return d
 
+    def _handle_store_error(self, f, largs, logfmt, spider, slot_type):
+        logger.error(
+            logfmt % "Error storing", largs,
+            exc_info=failure_to_exc_info(f), extra={'spider': spider}
+        )
+        self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
+
+    def _handle_store_success(self, f, largs, logfmt, spider, slot_type):
+        logger.info(
+            logfmt % "Stored", largs, extra={'spider': spider}
+        )
+        self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}")
+
     def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template):
         """
         Redirect the output data stream to a new file.
@@ -349,6 +357,7 @@ class FeedExporter:
             fields_to_export=feed_options['fields'],
             encoding=feed_options['encoding'],
             indent=feed_options['indent'],
+            **feed_options['item_export_kwargs'],
         )
         slot = _FeedSlot(
             file=file,
@@ -451,7 +460,7 @@ class FeedExporter:
         crawler = getattr(self, 'crawler', None)
 
         def build_instance(builder, *preargs):
-            return build_storage(builder, uri, preargs=preargs)
+            return build_storage(builder, uri, feed_options=feed_options, preargs=preargs)
 
         if crawler and hasattr(feedcls, 'from_crawler'):
             instance = build_instance(feedcls.from_crawler, crawler)

scrapy/http/request/__init__.py

@@ -65,7 +65,11 @@ class Request(object_ref):
         s = safe_url_string(url, self.encoding)
         self._url = escape_ajax(s)
 
-        if ('://' not in self._url) and (not self._url.startswith('data:')):
+        if (
+            '://' not in self._url
+            and not self._url.startswith('about:')
+            and not self._url.startswith('data:')
+        ):
             raise ValueError(f'Missing scheme in request url: {self._url}')
 
     url = property(_get_url, obsolete_setter(_set_url, 'url'))

scrapy/http/response/__init__.py

@@ -17,8 +17,18 @@ from scrapy.utils.trackref import object_ref
 
 class Response(object_ref):
 
-    def __init__(self, url, status=200, headers=None, body=b'', flags=None,
+    def __init__(
-                 request=None, certificate=None, ip_address=None):
+        self,
+        url,
+        status=200,
+        headers=None,
+        body=b"",
+        flags=None,
+        request=None,
+        certificate=None,
+        ip_address=None,
+        protocol=None,
+    ):
         self.headers = Headers(headers or {})
         self.status = int(status)
         self._set_body(body)
@@ -27,6 +37,7 @@ class Response(object_ref):
         self.flags = [] if flags is None else list(flags)
         self.certificate = certificate
         self.ip_address = ip_address
+        self.protocol = protocol
 
     @property
     def cb_kwargs(self):
@@ -89,8 +100,9 @@ class Response(object_ref):
         """Create a new Response with the same attributes except for those
         given new values.
         """
-        for x in ['url', 'status', 'headers', 'body',
+        for x in [
-                  'request', 'flags', 'certificate', 'ip_address']:
+            "url", "status", "headers", "body", "request", "flags", "certificate", "ip_address", "protocol",
+        ]:
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop('cls', self.__class__)
         return cls(*args, **kwargs)

scrapy/link.py

@@ -7,7 +7,22 @@ its documentation in: docs/topics/link-extractors.rst
 
 
 class Link:
-    """Link objects represent an extracted link by the LinkExtractor."""
+    """Link objects represent an extracted link by the LinkExtractor.
+
+    Using the anchor tag sample below to illustrate the parameters::
+
+            <a href="https://example.com/nofollow.html#foo" rel="nofollow">Dont follow this one</a>
+
+    :param url: the absolute url being linked to in the anchor tag.
+                From the sample, this is ``https://example.com/nofollow.html``.
+
+    :param text: the text in the anchor tag. From the sample, this is ``Dont follow this one``.
+
+    :param fragment: the part of the url after the hash symbol. From the sample, this is ``foo``.
+
+    :param nofollow: an indication of the presence or absence of a nofollow value in the ``rel`` attribute
+                    of the anchor tag.
+    """
 
     __slots__ = ['url', 'text', 'fragment', 'nofollow']
 

scrapy/resolver.py

@@ -1,6 +1,6 @@
 from twisted.internet import defer
 from twisted.internet.base import ThreadedResolver
-from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple
+from twisted.internet.interfaces import IHostResolution, IHostnameResolver, IResolutionReceiver, IResolverSimple
 from zope.interface.declarations import implementer, provider
 
 from scrapy.utils.datatypes import LocalCache
@@ -50,6 +50,36 @@ class CachingThreadedResolver(ThreadedResolver):
         return result
 
 
+@implementer(IHostResolution)
+class HostResolution:
+    def __init__(self, name):
+        self.name = name
+
+    def cancel(self):
+        raise NotImplementedError()
+
+
+@provider(IResolutionReceiver)
+class _CachingResolutionReceiver:
+    def __init__(self, resolutionReceiver, hostName):
+        self.resolutionReceiver = resolutionReceiver
+        self.hostName = hostName
+        self.addresses = []
+
+    def resolutionBegan(self, resolution):
+        self.resolutionReceiver.resolutionBegan(resolution)
+        self.resolution = resolution
+
+    def addressResolved(self, address):
+        self.resolutionReceiver.addressResolved(address)
+        self.addresses.append(address)
+
+    def resolutionComplete(self):
+        self.resolutionReceiver.resolutionComplete()
+        if self.addresses:
+            dnscache[self.hostName] = self.addresses
+
+
 @implementer(IHostnameResolver)
 class CachingHostnameResolver:
     """
@@ -73,33 +103,22 @@ class CachingHostnameResolver:
     def install_on_reactor(self):
         self.reactor.installNameResolver(self)
 
-    def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
+    def resolveHostName(
-                        addressTypes=None, transportSemantics='TCP'):
+        self, resolutionReceiver, hostName, portNumber=0, addressTypes=None, transportSemantics="TCP"
-
+    ):
-        @provider(IResolutionReceiver)
-        class CachingResolutionReceiver(resolutionReceiver):
-
-            def resolutionBegan(self, resolution):
-                super().resolutionBegan(resolution)
-                self.resolution = resolution
-                self.resolved = False
-
-            def addressResolved(self, address):
-                super().addressResolved(address)
-                self.resolved = True
-
-            def resolutionComplete(self):
-                super().resolutionComplete()
-                if self.resolved:
-                    dnscache[hostName] = self.resolution
-
         try:
-            return dnscache[hostName]
+            addresses = dnscache[hostName]
         except KeyError:
             return self.original_resolver.resolveHostName(
-                CachingResolutionReceiver(),
+                _CachingResolutionReceiver(resolutionReceiver, hostName),
                 hostName,
                 portNumber,
                 addressTypes,
-                transportSemantics
+                transportSemantics,
             )
+        else:
+            resolutionReceiver.resolutionBegan(HostResolution(hostName))
+            for addr in addresses:
+                resolutionReceiver.addressResolved(addr)
+            resolutionReceiver.resolutionComplete()
+            return resolutionReceiver

scrapy/spiders/crawl.py

@@ -6,14 +6,11 @@ See documentation in docs/topics/spiders.rst
 """
 
 import copy
-import warnings
 from typing import Sequence
 
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, HtmlResponse
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
-from scrapy.utils.python import get_func_args
 from scrapy.utils.spider import iterate_spider_output
 
 
@@ -37,15 +34,22 @@ _default_link_extractor = LinkExtractor()
 
 class Rule:
 
-    def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None,
+    def __init__(
-                 process_links=None, process_request=None, errback=None):
+        self,
+        link_extractor=None,
+        callback=None,
+        cb_kwargs=None,
+        follow=None,
+        process_links=None,
+        process_request=None,
+        errback=None,
+    ):
         self.link_extractor = link_extractor or _default_link_extractor
         self.callback = callback
         self.errback = errback
         self.cb_kwargs = cb_kwargs or {}
         self.process_links = process_links or _identity
         self.process_request = process_request or _identity_process_request
-        self.process_request_argcount = None
         self.follow = follow if follow is not None else not callback
 
     def _compile(self, spider):
@@ -53,22 +57,6 @@ class Rule:
         self.errback = _get_method(self.errback, spider)
         self.process_links = _get_method(self.process_links, spider)
         self.process_request = _get_method(self.process_request, spider)
-        self.process_request_argcount = len(get_func_args(self.process_request))
-        if self.process_request_argcount == 1:
-            warnings.warn(
-                "Rule.process_request should accept two arguments "
-                "(request, response), accepting only one is deprecated",
-                category=ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
-
-    def _process_request(self, request, response):
-        """
-        Wrapper around the request processing function to maintain backward
-        compatibility with functions that do not take a Response object
-        """
-        args = [request] if self.process_request_argcount == 1 else [request, response]
-        return self.process_request(*args)
 
 
 class CrawlSpider(Spider):
@@ -111,7 +99,7 @@ class CrawlSpider(Spider):
             for link in rule.process_links(links):
                 seen.add(link)
                 request = self._build_request(rule_index, link)
-                yield rule._process_request(request, response)
+                yield rule.process_request(request, response)
 
     def _callback(self, response):
         rule = self._rules[response.meta['rule']]

scrapy/utils/asyncgen.py

@@ -0,0 +1,5 @@
+async def collect_asyncgen(result):
+    results = []
+    async for x in result:
+        results.append(x)
+    return results

scrapy/utils/conf.py

@@ -121,6 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings):
     out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
     out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
     out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
+    out.setdefault("item_export_kwargs", dict())
     if settings["FEED_EXPORT_INDENT"] is None:
         out.setdefault("indent", None)
     else:

scrapy/utils/gz.py

@@ -1,7 +1,6 @@
+import struct
 from gzip import GzipFile
 from io import BytesIO
-import re
-import struct
 
 from scrapy.utils.decorators import deprecated
 
@@ -42,17 +41,5 @@ def gunzip(data):
     return b''.join(output_list)
 
 
-_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search
-_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search
-
-
-@deprecated
-def is_gzipped(response):
-    """Return True if the response is gzipped, or False otherwise"""
-    ctype = response.headers.get('Content-Type', b'')
-    cenc = response.headers.get('Content-Encoding', b'').lower()
-    return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')
-
-
 def gzip_magic_number(response):
     return response.body[:3] == b'\x1f\x8b\x08'

scrapy/utils/http.py

@@ -1,36 +0,0 @@
-"""
-Transitional module for moving to the w3lib library.
-
-For new code, always import from w3lib.http instead of this module
-"""
-
-import warnings
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.decorators import deprecated
-from w3lib.http import *  # noqa: F401
-
-
-warnings.warn("Module `scrapy.utils.http` is deprecated, "
-              "Please import from `w3lib.http` instead.",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-
-@deprecated
-def decode_chunked_transfer(chunked_body):
-    """Parsed body received with chunked transfer encoding, and return the
-    decoded body.
-
-    For more info see:
-    https://en.wikipedia.org/wiki/Chunked_transfer_encoding
-
-    """
-    body, h, t = '', '', chunked_body
-    while t:
-        h, t = t.split('\r\n', 1)
-        if h == '0':
-            break
-        size = int(h, 16)
-        body += t[:size]
-        t = t[size + 2:]
-    return body

scrapy/utils/iterators.py

@@ -22,25 +22,41 @@ def xmliter(obj, nodename):
     """
     nodename_patt = re.escape(nodename)
 
-    HEADER_START_RE = re.compile(fr'^(.*?)<\s*{nodename_patt}(?:\s|>)', re.S)
+    DOCUMENT_HEADER_RE = re.compile(r'<\?xml[^>]+>\s*', re.S)
     HEADER_END_RE = re.compile(fr'<\s*/{nodename_patt}\s*>', re.S)
+    END_TAG_RE = re.compile(r'<\s*/([^\s>]+)\s*>', re.S)
+    NAMESPACE_RE = re.compile(r'((xmlns[:A-Za-z]*)=[^>\s]+)', re.S)
     text = _body_or_str(obj)
 
-    header_start = re.search(HEADER_START_RE, text)
+    document_header = re.search(DOCUMENT_HEADER_RE, text)
-    header_start = header_start.group(1).strip() if header_start else ''
+    document_header = document_header.group().strip() if document_header else ''
-    header_end = re_rsearch(HEADER_END_RE, text)
+    header_end_idx = re_rsearch(HEADER_END_RE, text)
-    header_end = text[header_end[1]:].strip() if header_end else ''
+    header_end = text[header_end_idx[1]:].strip() if header_end_idx else ''
+    namespaces = {}
+    if header_end:
+        for tagname in reversed(re.findall(END_TAG_RE, header_end)):
+            tag = re.search(fr'<\s*{tagname}.*?xmlns[:=][^>]*>', text[:header_end_idx[1]], re.S)
+            if tag:
+                namespaces.update(reversed(x) for x in re.findall(NAMESPACE_RE, tag.group()))
 
     r = re.compile(fr'<{nodename_patt}[\s>].*?</{nodename_patt}>', re.DOTALL)
     for match in r.finditer(text):
-        nodetext = header_start + match.group() + header_end
+        nodetext = (
-        yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0]
+            document_header
+            + match.group().replace(
+                nodename,
+                f'{nodename} {" ".join(namespaces.values())}',
+                1
+            )
+            + header_end
+        )
+        yield Selector(text=nodetext, type='xml')
 
 
 def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
     from lxml import etree
     reader = _StreamReader(obj)
-    tag = f'{{{namespace}}}{nodename}'if namespace else nodename
+    tag = f'{{{namespace}}}{nodename}' if namespace else nodename
     iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
     selxpath = '//' + (f'{prefix}:{nodename}' if namespace else nodename)
     for _, node in iterable:

scrapy/utils/markup.py

@@ -1,14 +0,0 @@
-"""
-Transitional module for moving to the w3lib library.
-
-For new code, always import from w3lib.html instead of this module
-"""
-import warnings
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-from w3lib.html import *  # noqa: F401
-
-
-warnings.warn("Module `scrapy.utils.markup` is deprecated. "
-              "Please import from `w3lib.html` instead.",
-              ScrapyDeprecationWarning, stacklevel=2)

scrapy/utils/multipart.py

@@ -1,15 +0,0 @@
-"""
-Transitional module for moving to the w3lib library.
-
-For new code, always import from w3lib.form instead of this module
-"""
-import warnings
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-from w3lib.form import *  # noqa: F401
-
-
-warnings.warn("Module `scrapy.utils.multipart` is deprecated. "
-              "If you're using `encode_multipart` function, please use "
-              "`urllib3.filepost.encode_multipart_formdata` instead",
-              ScrapyDeprecationWarning, stacklevel=2)

scrapy/utils/project.py

@@ -1,5 +1,4 @@
 import os
-import pickle
 import warnings
 
 from importlib import import_module
@@ -68,18 +67,10 @@ def get_project_settings():
     if settings_module_path:
         settings.setmodule(settings_module_path, priority='project')
 
-    pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
-    if pickled_settings:
-        warnings.warn("Use of environment variable "
-                      "'SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE' "
-                      "is deprecated.", ScrapyDeprecationWarning)
-        settings.setdict(pickle.loads(pickled_settings), priority='project')
-
     scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if
                       k.startswith('SCRAPY_')}
     valid_envvars = {
         'CHECK',
-        'PICKLED_SETTINGS_TO_OVERRIDE',
         'PROJECT',
         'PYTHON_SHELL',
         'SETTINGS_MODULE',

scrapy/utils/py36.py

@@ -1,10 +1,11 @@
-"""
+import warnings
-Helpers using Python 3.6+ syntax (ignore SyntaxError on import).
+
-"""
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.asyncgen import collect_asyncgen  # noqa: F401
 
 
-async def collect_asyncgen(result):
+warnings.warn(
-    results = []
+    "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.asyncgen` instead.",
-    async for x in result:
+    category=ScrapyDeprecationWarning,
-        results.append(x)
+    stacklevel=2,
-    return results
+)

scrapy/utils/reactor.py

@@ -60,8 +60,9 @@ def install_reactor(reactor_path, event_loop_path=None):
             if event_loop_path is not None:
                 event_loop_class = load_object(event_loop_path)
                 event_loop = event_loop_class()
+                asyncio.set_event_loop(event_loop)
             else:
-                event_loop = asyncio.new_event_loop()
+                event_loop = asyncio.get_event_loop()
             asyncioreactor.install(eventloop=event_loop)
     else:
         *module, _ = reactor_path.split(".")

scrapy/utils/spider.py

@@ -4,17 +4,14 @@ import logging
 from scrapy.spiders import Spider
 from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.misc import arg_to_iter
-try:
+from scrapy.utils.asyncgen import collect_asyncgen
-    from scrapy.utils.py36 import collect_asyncgen
-except SyntaxError:
-    collect_asyncgen = None
 
 
 logger = logging.getLogger(__name__)
 
 
 def iterate_spider_output(result):
-    if collect_asyncgen and hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result):
+    if inspect.isasyncgen(result):
         d = deferred_from_coro(collect_asyncgen(result))
         d.addCallback(iterate_spider_output)
         return d

scrapy/utils/test.py

@@ -13,15 +13,6 @@ from twisted.trial.unittest import SkipTest
 from scrapy.utils.boto import is_botocore_available
 
 
-def assert_aws_environ():
-    """Asserts the current environment is suitable for running AWS testsi.
-    Raises SkipTest with the reason if it's not.
-    """
-    skip_if_no_boto()
-    if 'AWS_ACCESS_KEY_ID' not in os.environ:
-        raise SkipTest("AWS keys not found")
-
-
 def assert_gcs_environ():
     if 'GCS_PROJECT_ID' not in os.environ:
         raise SkipTest("GCS_PROJECT_ID not found")
@@ -32,18 +23,6 @@ def skip_if_no_boto():
         raise SkipTest('missing botocore library')
 
 
-def get_s3_content_and_delete(bucket, path, with_key=False):
-    """ Get content from s3 key, and delete key afterwards.
-    """
-    import botocore.session
-    session = botocore.session.get_session()
-    client = session.create_client('s3')
-    key = client.get_object(Bucket=bucket, Key=path)
-    content = key['Body'].read()
-    client.delete_object(Bucket=bucket, Key=path)
-    return (content, key) if with_key else content
-
-
 def get_gcs_content_and_delete(bucket, path):
     from google.cloud import storage
     client = storage.Client(project=os.environ.get('GCS_PROJECT_ID'))

setup.cfg

@@ -55,9 +55,6 @@ ignore_errors = True
 [mypy-scrapy.utils.response]
 ignore_errors = True
 
-[mypy-scrapy.utils.spider]
-ignore_errors = True
-
 [mypy-scrapy.utils.trackref]
 ignore_errors = True
 

setup.py

@@ -56,7 +56,7 @@ setup(
     name='Scrapy',
     version=version,
     url='https://scrapy.org',
-    project_urls = {
+    project_urls={
         'Documentation': 'https://docs.scrapy.org/',
         'Source': 'https://github.com/scrapy/scrapy',
         'Tracker': 'https://github.com/scrapy/scrapy/issues',

tests/CrawlerProcess/alternative_name_resolver.py

@@ -1,15 +0,0 @@
-import scrapy
-from scrapy.crawler import CrawlerProcess
-
-
-class IPv6Spider(scrapy.Spider):
-    name = "ipv6_spider"
-    start_urls = ["http://[::1]"]
-
-
-process = CrawlerProcess(settings={
-    "RETRY_ENABLED": False,
-    "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
-})
-process.crawl(IPv6Spider)
-process.start()

tests/CrawlerProcess/asyncio_deferred_signal.py

@@ -0,0 +1,44 @@
+import asyncio
+import sys
+
+from scrapy import Spider
+from scrapy.crawler import CrawlerProcess
+from scrapy.utils.defer import deferred_from_coro
+from twisted.internet.defer import Deferred
+
+
+class UppercasePipeline:
+    async def _open_spider(self, spider):
+        spider.logger.info("async pipeline opened!")
+        await asyncio.sleep(0.1)
+
+    def open_spider(self, spider):
+        return deferred_from_coro(self._open_spider(spider))
+
+    def process_item(self, item, spider):
+        return {"url": item["url"].upper()}
+
+
+class UrlSpider(Spider):
+    name = "url_spider"
+    start_urls = ["data:,"]
+    custom_settings = {
+        "ITEM_PIPELINES": {UppercasePipeline: 100},
+    }
+
+    def parse(self, response):
+        yield {"url": response.url}
+
+
+if __name__ == "__main__":
+    try:
+        ASYNCIO_EVENT_LOOP = sys.argv[1]
+    except IndexError:
+        ASYNCIO_EVENT_LOOP = None
+
+    process = CrawlerProcess(settings={
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+        "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP,
+    })
+    process.crawl(UrlSpider)
+    process.start()

tests/CrawlerProcess/caching_hostname_resolver.py

@@ -0,0 +1,30 @@
+import sys
+
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class CachingHostnameResolverSpider(scrapy.Spider):
+    """
+    Finishes in a finite amount of time (does not hang indefinitely in the DNS resolution)
+    """
+    name = "caching_hostname_resolver_spider"
+
+    def start_requests(self):
+        yield scrapy.Request(self.url)
+
+    def parse(self, response):
+        for _ in range(10):
+            yield scrapy.Request(response.url, dont_filter=True, callback=self.ignore_response)
+
+    def ignore_response(self, response):
+        self.logger.info(repr(response.ip_address))
+
+
+if __name__ == "__main__":
+    process = CrawlerProcess(settings={
+        "RETRY_ENABLED": False,
+        "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+    })
+    process.crawl(CachingHostnameResolverSpider, url=sys.argv[1])
+    process.start()

tests/CrawlerProcess/caching_hostname_resolver_ipv6.py

@@ -0,0 +1,19 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class CachingHostnameResolverSpider(scrapy.Spider):
+    """
+    Finishes without a twisted.internet.error.DNSLookupError exception
+    """
+    name = "caching_hostname_resolver_spider"
+    start_urls = ["http://[::1]"]
+
+
+if __name__ == "__main__":
+    process = CrawlerProcess(settings={
+        "RETRY_ENABLED": False,
+        "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+    })
+    process.crawl(CachingHostnameResolverSpider)
+    process.start()

tests/CrawlerProcess/default_name_resolver.py

@@ -3,10 +3,15 @@ from scrapy.crawler import CrawlerProcess
 
 
 class IPv6Spider(scrapy.Spider):
+    """
+    Raises a twisted.internet.error.DNSLookupError:
+    the default name resolver does not handle IPv6 addresses.
+    """
     name = "ipv6_spider"
     start_urls = ["http://[::1]"]
 
 
-process = CrawlerProcess(settings={"RETRY_ENABLED": False})
+if __name__ == "__main__":
-process.crawl(IPv6Spider)
+    process = CrawlerProcess(settings={"RETRY_ENABLED": False})
-process.start()
+    process.crawl(IPv6Spider)
+    process.start()

tests/requirements-py3.txt

@@ -4,7 +4,6 @@ dataclasses; python_version == '3.6'
 pyftpdlib
 # https://github.com/pytest-dev/pytest-twisted/issues/93
 pytest != 5.4, != 5.4.1
-pytest-azurepipelines
 pytest-cov
 pytest-twisted >= 1.11
 pytest-xdist
@@ -14,6 +13,7 @@ uvloop; platform_system != "Windows"
 
 # optional for shell wrapper tests
 bpython
-brotlipy
+brotlipy  # optional for HTTP compress downloader middleware tests
+zstandard  # optional for HTTP compress downloader middleware tests
 ipython
 pywin32; sys_platform == "win32"

tests/test_commands.py

@@ -389,8 +389,9 @@ class GenspiderCommandTest(CommandTest):
     def test_template(self, tplname='crawl'):
         args = [f'--template={tplname}'] if tplname else []
         spname = 'test_spider'
+        spmodule = f"{self.project_name}.spiders.{spname}"
         p, out, err = self.proc('genspider', spname, 'test.com', *args)
-        self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module", out)
+        self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep}  {spmodule}", out)
         self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
         modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py'))
         p, out, err = self.proc('genspider', spname, 'test.com', *args)
@@ -496,6 +497,8 @@ class MiscCommandsTest(CommandTest):
 
 class RunSpiderCommandTest(CommandTest):
 
+    spider_filename = 'myspider.py'
+
     debug_log_spider = """
 import scrapy
 
@@ -507,11 +510,23 @@ class MySpider(scrapy.Spider):
         return []
 """
 
+    badspider = """
+import scrapy
+
+class BadSpider(scrapy.Spider):
+    name = "bad"
+    def start_requests(self):
+        raise Exception("oops!")
+        """
+
     @contextmanager
-    def _create_file(self, content, name):
+    def _create_file(self, content, name=None):
         tmpdir = self.mktemp()
         os.mkdir(tmpdir)
-        fname = abspath(join(tmpdir, name))
+        if name:
+            fname = abspath(join(tmpdir, name))
+        else:
+            fname = abspath(join(tmpdir, self.spider_filename))
         with open(fname, 'w') as f:
             f.write(content)
         try:
@@ -519,12 +534,12 @@ class MySpider(scrapy.Spider):
         finally:
             rmtree(tmpdir)
 
-    def runspider(self, code, name='myspider.py', args=()):
+    def runspider(self, code, name=None, args=()):
         with self._create_file(code, name) as fname:
             return self.proc('runspider', fname, *args)
 
-    def get_log(self, code, name='myspider.py', args=()):
+    def get_log(self, code, name=None, args=()):
-        p, stdout, stderr = self.runspider(code, name=name, args=args)
+        p, stdout, stderr = self.runspider(code, name, args=args)
         return stderr
 
     def test_runspider(self):
@@ -556,7 +571,7 @@ class MySpider(scrapy.Spider):
         # which is intended,
         # but this should not be because of DNS lookup error
         # assumption: localhost will resolve in all cases (true?)
-        log = self.get_log("""
+        dnscache_spider = """
 import scrapy
 
 class MySpider(scrapy.Spider):
@@ -565,23 +580,20 @@ class MySpider(scrapy.Spider):
 
     def parse(self, response):
         return {'test': 'value'}
-""",
+"""
-                           args=('-s', 'DNSCACHE_ENABLED=False'))
+        log = self.get_log(dnscache_spider, args=('-s', 'DNSCACHE_ENABLED=False'))
-        print(log)
         self.assertNotIn("DNSLookupError", log)
         self.assertIn("INFO: Spider opened", log)
 
     def test_runspider_log_short_names(self):
         log1 = self.get_log(self.debug_log_spider,
                             args=('-s', 'LOG_SHORT_NAMES=1'))
-        print(log1)
         self.assertIn("[myspider] DEBUG: It Works!", log1)
         self.assertIn("[scrapy]", log1)
         self.assertNotIn("[scrapy.core.engine]", log1)
 
         log2 = self.get_log(self.debug_log_spider,
                             args=('-s', 'LOG_SHORT_NAMES=0'))
-        print(log2)
         self.assertIn("[myspider] DEBUG: It Works!", log2)
         self.assertNotIn("[scrapy]", log2)
         self.assertIn("[scrapy.core.engine]", log2)
@@ -599,15 +611,7 @@ class MySpider(scrapy.Spider):
         self.assertIn('Unable to load', log)
 
     def test_start_requests_errors(self):
-        log = self.get_log("""
+        log = self.get_log(self.badspider, name='badspider.py')
-import scrapy
-
-class BadSpider(scrapy.Spider):
-    name = "bad"
-    def start_requests(self):
-        raise Exception("oops!")
-        """, name="badspider.py")
-        print(log)
         self.assertIn("start_requests", log)
         self.assertIn("badspider.py", log)
 
@@ -677,9 +681,14 @@ class MySpider(scrapy.Spider):
         )
         return []
 """
+        with open(os.path.join(self.cwd, "example.json"), "w") as f1:
+            f1.write("not empty")
         args = ['-O', 'example.json']
         log = self.get_log(spider_code, args=args)
         self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
+        with open(os.path.join(self.cwd, "example.json")) as f2:
+            first_line = f2.readline()
+        self.assertNotEqual(first_line, "not empty")
 
     def test_output_and_overwrite_output(self):
         spider_code = """
@@ -696,6 +705,54 @@ class MySpider(scrapy.Spider):
         self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
 
 
+class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
+
+    spider_filename = 'myspider.pyw'
+
+    def setUp(self):
+        super(WindowsRunSpiderCommandTest, self).setUp()
+
+    def test_start_requests_errors(self):
+        log = self.get_log(self.badspider, name='badspider.pyw')
+        self.assertIn("start_requests", log)
+        self.assertIn("badspider.pyw", log)
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_run_good_spider(self):
+        super().test_run_good_spider()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_runspider(self):
+        super().test_runspider()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_runspider_dnscache_disabled(self):
+        super().test_runspider_dnscache_disabled()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_runspider_log_level(self):
+        super().test_runspider_log_level()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_runspider_log_short_names(self):
+        super().test_runspider_log_short_names()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_runspider_no_spider_found(self):
+        super().test_runspider_no_spider_found()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_output(self):
+        super().test_output()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_overwrite_output(self):
+        super().test_overwrite_output()
+
+    def test_runspider_unable_to_load(self):
+        raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ")
+
+
 class BenchCommandTest(CommandTest):
 
     def test_run(self):
@@ -762,9 +819,14 @@ class MySpider(scrapy.Spider):
         )
         return []
 """
+        with open(os.path.join(self.cwd, "example.json"), "w") as f1:
+            f1.write("not empty")
         args = ['-O', 'example.json']
         log = self.get_log(spider_code, args=args)
         self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
+        with open(os.path.join(self.cwd, "example.json")) as f2:
+            first_line = f2.readline()
+        self.assertNotEqual(first_line, "not empty")
 
     def test_output_and_overwrite_output(self):
         spider_code = """

tests/test_crawler.py

@@ -22,6 +22,8 @@ from scrapy.extensions.throttle import AutoThrottle
 from scrapy.extensions import telnet
 from scrapy.utils.test import get_testenv
 
+from tests.mockserver import MockServer
+
 
 class BaseCrawlerTest(unittest.TestCase):
 
@@ -280,9 +282,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
 
 class ScriptRunnerMixin:
-    def run_script(self, script_name):
+    def run_script(self, script_name, *script_args):
         script_path = os.path.join(self.script_dir, script_name)
-        args = (sys.executable, script_path)
+        args = [sys.executable, script_path] + list(script_args)
         p = subprocess.Popen(args, env=get_testenv(),
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         stdout, stderr = p.communicate()
@@ -321,11 +323,20 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
             "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.",
             log)
 
-    def test_ipv6_alternative_name_resolver(self):
+    def test_caching_hostname_resolver_ipv6(self):
-        log = self.run_script('alternative_name_resolver.py')
+        log = self.run_script("caching_hostname_resolver_ipv6.py")
-        self.assertIn('Spider closed (finished)', log)
+        self.assertIn("Spider closed (finished)", log)
         self.assertNotIn("twisted.internet.error.DNSLookupError", log)
 
+    def test_caching_hostname_resolver_finite_execution(self):
+        with MockServer() as mock_server:
+            http_address = mock_server.http_address.replace("0.0.0.0", "127.0.0.1")
+            log = self.run_script("caching_hostname_resolver.py", http_address)
+            self.assertIn("Spider closed (finished)", log)
+            self.assertNotIn("ERROR: Error downloading", log)
+            self.assertNotIn("TimeoutError", log)
+            self.assertNotIn("twisted.internet.error.DNSLookupError", log)
+
     def test_reactor_select(self):
         log = self.run_script("twisted_reactor_select.py")
         self.assertIn("Spider closed (finished)", log)
@@ -353,6 +364,25 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
 
+    @mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly")
+    @mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows")
+    def test_custom_loop_asyncio_deferred_signal(self):
+        log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertIn("Using asyncio event loop: uvloop.Loop", log)
+        self.assertIn("async pipeline opened!", log)
+
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
+    def test_default_loop_asyncio_deferred_signal(self):
+        log = self.run_script("asyncio_deferred_signal.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertNotIn("Using asyncio event loop: uvloop.Loop", log)
+        self.assertIn("async pipeline opened!", log)
+
 
 class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
     script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner')

tests/test_downloader_handlers.py

@@ -115,6 +115,7 @@ class FileTestCase(unittest.TestCase):
             self.assertEqual(response.url, request.url)
             self.assertEqual(response.status, 200)
             self.assertEqual(response.body, b'0123456789')
+            self.assertEqual(response.protocol, None)
 
         request = Request(path_to_file_uri(self.tmpname + '^'))
         assert request.url.upper().endswith('%5E')
@@ -360,6 +361,13 @@ class Http10TestCase(HttpTestCase):
     """HTTP 1.0 test case"""
     download_handler_cls = HTTP10DownloadHandler
 
+    def test_protocol(self):
+        request = Request(self.getURL("host"), method="GET")
+        d = self.download_request(request, Spider("foo"))
+        d.addCallback(lambda r: r.protocol)
+        d.addCallback(self.assertEqual, "HTTP/1.0")
+        return d
+
 
 class Https10TestCase(Http10TestCase):
     scheme = 'https'
@@ -489,6 +497,13 @@ class Http11TestCase(HttpTestCase):
     def test_download_broken_chunked_content_allow_data_loss_via_setting(self):
         return self.test_download_broken_content_allow_data_loss_via_setting('broken-chunked')
 
+    def test_protocol(self):
+        request = Request(self.getURL("host"), method="GET")
+        d = self.download_request(request, Spider("foo"))
+        d.addCallback(lambda r: r.protocol)
+        d.addCallback(self.assertEqual, "HTTP/1.1")
+        return d
+
 
 class Https11TestCase(Http11TestCase):
     scheme = 'https'
@@ -868,29 +883,6 @@ class S3TestCase(unittest.TestCase):
         self.assertEqual(httpreq.headers['Authorization'],
                          b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
 
-    def test_request_signing5(self):
-        try:
-            import botocore  # noqa: F401
-        except ImportError:
-            pass
-        else:
-            raise unittest.SkipTest(
-                'botocore does not support overriding date with x-amz-date')
-        # deletes an object from the 'johnsmith' bucket using the
-        # path-style and Date alternative.
-        date = 'Tue, 27 Mar 2007 21:20:27 +0000'
-        req = Request(
-            's3://johnsmith/photos/puppy.jpg', method='DELETE', headers={
-                'Date': date,
-                'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
-            })
-        with self._mocked_date(date):
-            httpreq = self.download_request(req, self.spider)
-        # botocore does not override Date with x-amz-date
-        self.assertEqual(
-            httpreq.headers['Authorization'],
-            b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
-
     def test_request_signing6(self):
         # uploads an object to a CNAME style virtual hosted bucket with metadata.
         date = 'Tue, 27 Mar 2007 21:06:08 +0000'
@@ -985,6 +977,7 @@ class BaseFTPTestCase(unittest.TestCase):
             self.assertEqual(r.status, 200)
             self.assertEqual(r.body, b'I have the power!')
             self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']})
+            self.assertIsNone(r.protocol)
         return self._add_test_callbacks(d, _test)
 
     def test_ftp_download_path_with_spaces(self):
@@ -1143,3 +1136,10 @@ class DataURITestCase(unittest.TestCase):
 
         request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D')
         return self.download_request(request, self.spider).addCallback(_test)
+
+    def test_protocol(self):
+        def _test(response):
+            self.assertIsNone(response.protocol)
+
+        request = Request("data:,")
+        return self.download_request(request, self.spider).addCallback(_test)

tests/test_downloadermiddleware_cookies.py

@@ -2,6 +2,8 @@ import logging
 from testfixtures import LogCapture
 from unittest import TestCase
 
+import pytest
+
 from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
 from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
 from scrapy.exceptions import NotConfigured
@@ -243,6 +245,7 @@ class CookiesMiddlewareTest(TestCase):
         self.assertIn('Cookie', request.headers)
         self.assertEqual(b'currencyCookie=USD', request.headers['Cookie'])
 
+    @pytest.mark.xfail(reason="Cookie header is not currently being processed")
     def test_keep_cookie_from_default_request_headers_middleware(self):
         DEFAULT_REQUEST_HEADERS = dict(Cookie='default=value; asdf=qwerty')
         mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items())
@@ -257,6 +260,7 @@ class CookiesMiddlewareTest(TestCase):
         assert self.mw.process_request(req2, self.spider) is None
         self.assertCookieValEqual(req2.headers['Cookie'], b'default=value; a=b; asdf=qwerty')
 
+    @pytest.mark.xfail(reason="Cookie header is not currently being processed")
     def test_keep_cookie_header(self):
         # keep only cookies from 'Cookie' request header
         req1 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'})
@@ -291,6 +295,7 @@ class CookiesMiddlewareTest(TestCase):
         assert self.mw.process_request(req3, self.spider) is None
         self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
 
+    @pytest.mark.xfail(reason="Cookie header is not currently being processed")
     def test_request_headers_cookie_encoding(self):
         # 1) UTF8-encoded bytes
         req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')})

tests/test_downloadermiddleware_httpcompression.py

@@ -20,6 +20,12 @@ FORMAT = {
     'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
     'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
     'br': ('html-br.bin', 'br'),
+    # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin
+    'zstd-static-content-size': ('html-zstd-static-content-size.bin', 'zstd'),
+    # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin
+    'zstd-static-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'),
+    # $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin
+    'zstd-streaming-no-content-size': ('html-zstd-streaming-no-content-size.bin', 'zstd'),
 }
 
 
@@ -80,6 +86,27 @@ class HttpCompressionTest(TestCase):
         assert newresponse.body.startswith(b"<!DOCTYPE")
         assert 'Content-Encoding' not in newresponse.headers
 
+    def test_process_response_zstd(self):
+        try:
+            import zstandard  # noqa: F401
+        except ImportError:
+            raise SkipTest("no zstd support (zstandard)")
+        raw_content = None
+        for check_key in FORMAT:
+            if not check_key.startswith('zstd-'):
+                continue
+            response = self._getresponse(check_key)
+            request = response.request
+            self.assertEqual(response.headers['Content-Encoding'], b'zstd')
+            newresponse = self.mw.process_response(request, response, self.spider)
+            if raw_content is None:
+                raw_content = newresponse.body
+            else:
+                assert raw_content == newresponse.body
+            assert newresponse is not response
+            assert newresponse.body.startswith(b"<!DOCTYPE")
+            assert 'Content-Encoding' not in newresponse.headers
+
     def test_process_response_rawdeflate(self):
         response = self._getresponse('rawdeflate')
         request = response.request

tests/test_downloadermiddleware_httpproxy.py

@@ -145,3 +145,10 @@ class TestHttpProxyMiddleware(TestCase):
         req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'})
         assert mw.process_request(req, spider) is None
         self.assertEqual(req.meta, {'proxy': 'http://proxy.com'})
+
+    def test_no_proxy_invalid_values(self):
+        os.environ['no_proxy'] = '/var/run/docker.sock'
+        mw = HttpProxyMiddleware()
+        # '/var/run/docker.sock' may be used by the user for
+        # no_proxy value but is not parseable and should be skipped
+        assert 'no' not in mw.proxies

tests/test_dupefilters.py

@@ -43,7 +43,7 @@ class RFPDupeFilterTest(unittest.TestCase):
 
     def test_df_from_crawler_scheduler(self):
         settings = {'DUPEFILTER_DEBUG': True,
-                    'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
+                    'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertTrue(scheduler.df.debug)
@@ -51,14 +51,14 @@ class RFPDupeFilterTest(unittest.TestCase):
 
     def test_df_from_settings_scheduler(self):
         settings = {'DUPEFILTER_DEBUG': True,
-                    'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'}
+                    'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertTrue(scheduler.df.debug)
         self.assertEqual(scheduler.df.method, 'from_settings')
 
     def test_df_direct_scheduler(self):
-        settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'}
+        settings = {'DUPEFILTER_CLASS': DirectDupeFilter}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertEqual(scheduler.df.method, 'n/a')
@@ -162,7 +162,7 @@ class RFPDupeFilterTest(unittest.TestCase):
     def test_log(self):
         with LogCapture() as log:
             settings = {'DUPEFILTER_DEBUG': False,
-                        'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
+                        'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
             scheduler = Scheduler.from_crawler(crawler)
             spider = SimpleSpider.from_crawler(crawler)
@@ -191,7 +191,7 @@ class RFPDupeFilterTest(unittest.TestCase):
     def test_log_debug(self):
         with LogCapture() as log:
             settings = {'DUPEFILTER_DEBUG': True,
-                        'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
+                        'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
             scheduler = Scheduler.from_crawler(crawler)
             spider = SimpleSpider.from_crawler(crawler)

tests/test_feedexport.py

@@ -8,12 +8,13 @@ import tempfile
 import warnings
 from abc import ABC, abstractmethod
 from collections import defaultdict
+from contextlib import ExitStack
 from io import BytesIO
 from logging import getLogger
 from pathlib import Path
 from string import ascii_letters, digits
 from unittest import mock
-from urllib.parse import urljoin, urlparse, quote
+from urllib.parse import urljoin, quote
 from urllib.request import pathname2url
 
 import lxml.etree
@@ -41,14 +42,27 @@ from scrapy.extensions.feedexport import (
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import (
-    assert_aws_environ,
-    get_s3_content_and_delete,
     get_crawler,
     mock_google_cloud_storage,
     skip_if_no_boto,
 )
 
 from tests.mockserver import MockFTPServer, MockServer
+from tests.spiders import ItemSpider
+
+
+def path_to_url(path):
+    return urljoin('file:', pathname2url(str(path)))
+
+
+def printf_escape(string):
+    return string.replace('%', '%%')
+
+
+def build_url(path):
+    if path[0] != '/':
+        path = '/' + path
+    return urljoin('file:', path)
 
 
 class FileFeedStorageTest(unittest.TestCase):
@@ -254,21 +268,42 @@ class S3FeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_store(self):
-        assert_aws_environ()
+        skip_if_no_boto()
-        uri = os.environ.get('S3_TEST_FILE_URI')
+
-        if not uri:
+        settings = {
-            raise unittest.SkipTest("No S3 URI available for testing")
+            'AWS_ACCESS_KEY_ID': 'access_key',
-        access_key = os.environ.get('AWS_ACCESS_KEY_ID')
+            'AWS_SECRET_ACCESS_KEY': 'secret_key',
-        secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
+        }
-        storage = S3FeedStorage(uri, access_key, secret_key)
+        crawler = get_crawler(settings_dict=settings)
+        bucket = 'mybucket'
+        key = 'export.csv'
+        storage = S3FeedStorage.from_crawler(crawler, f's3://{bucket}/{key}')
         verifyObject(IFeedStorage, storage)
-        file = storage.open(scrapy.Spider("default"))
+
-        expected_content = b"content: \xe2\x98\x83"
+        file = mock.MagicMock()
-        file.write(expected_content)
+        from botocore.stub import Stubber
-        yield storage.store(file)
+        with Stubber(storage.s3_client) as stub:
-        u = urlparse(uri)
+            stub.add_response(
-        content = get_s3_content_and_delete(u.hostname, u.path[1:])
+                'put_object',
-        self.assertEqual(content, expected_content)
+                expected_params={
+                    'Body': file,
+                    'Bucket': bucket,
+                    'Key': key,
+                },
+                service_response={},
+            )
+
+            yield storage.store(file)
+
+            stub.assert_no_pending_responses()
+            self.assertEqual(
+                file.method_calls,
+                [
+                    mock.call.seek(0),
+                    # The call to read does not happen with Stubber
+                    mock.call.close(),
+                ]
+            )
 
     def test_init_without_acl(self):
         storage = S3FeedStorage(
@@ -601,12 +636,6 @@ class FeedExportTest(FeedExportTestBase):
     def run_and_export(self, spider_cls, settings):
         """ Run spider with specified settings; return exported data. """
 
-        def path_to_url(path):
-            return urljoin('file:', pathname2url(str(path)))
-
-        def printf_escape(string):
-            return string.replace('%', '%%')
-
         FEEDS = settings.get('FEEDS') or {}
         settings['FEEDS'] = {
             printf_escape(path_to_url(file_path)): feed_options
@@ -729,6 +758,69 @@ class FeedExportTest(FeedExportTestBase):
         result = self._load_until_eof(data['marshal'], load_func=marshal.load)
         self.assertEqual(expected, result)
 
+    @defer.inlineCallbacks
+    def test_stats_file_success(self):
+        settings = {
+            "FEEDS": {
+                printf_escape(path_to_url(self._random_temp_filename())): {
+                    "format": "json",
+                }
+            },
+        }
+        crawler = get_crawler(ItemSpider, settings)
+        with MockServer() as mockserver:
+            yield crawler.crawl(mockserver=mockserver)
+        self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats())
+        self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1)
+
+    @defer.inlineCallbacks
+    def test_stats_file_failed(self):
+        settings = {
+            "FEEDS": {
+                printf_escape(path_to_url(self._random_temp_filename())): {
+                    "format": "json",
+                }
+            },
+        }
+        crawler = get_crawler(ItemSpider, settings)
+        with ExitStack() as stack:
+            mockserver = stack.enter_context(MockServer())
+            stack.enter_context(
+                mock.patch(
+                    "scrapy.extensions.feedexport.FileFeedStorage.store",
+                    side_effect=KeyError("foo"))
+            )
+            yield crawler.crawl(mockserver=mockserver)
+        self.assertIn("feedexport/failed_count/FileFeedStorage", crawler.stats.get_stats())
+        self.assertEqual(crawler.stats.get_value("feedexport/failed_count/FileFeedStorage"), 1)
+
+    @defer.inlineCallbacks
+    def test_stats_multiple_file(self):
+        settings = {
+            'AWS_ACCESS_KEY_ID': 'access_key',
+            'AWS_SECRET_ACCESS_KEY': 'secret_key',
+            "FEEDS": {
+                printf_escape(path_to_url(self._random_temp_filename())): {
+                    "format": "json",
+                },
+                "s3://bucket/key/foo.csv": {
+                    "format": "csv",
+                },
+                "stdout:": {
+                    "format": "xml",
+                }
+            },
+        }
+        crawler = get_crawler(ItemSpider, settings)
+        with MockServer() as mockserver, mock.patch.object(S3FeedStorage, "store"):
+            yield crawler.crawl(mockserver=mockserver)
+        self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats())
+        self.assertIn("feedexport/success_count/S3FeedStorage", crawler.stats.get_stats())
+        self.assertIn("feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats())
+        self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1)
+        self.assertEqual(crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1)
+        self.assertEqual(crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1)
+
     @defer.inlineCallbacks
     def test_export_items(self):
         # feed exporters use field names from Item
@@ -1191,6 +1283,43 @@ class FeedExportTest(FeedExportTestBase):
         for fmt in ['json', 'xml', 'csv']:
             self.assertIn(f'Error storing {fmt} feed (2 items)', str(log))
 
+    @defer.inlineCallbacks
+    def test_extend_kwargs(self):
+        items = [{'foo': 'FOO', 'bar': 'BAR'}]
+
+        expected_with_title_csv = 'foo,bar\r\nFOO,BAR\r\n'.encode('utf-8')
+        expected_without_title_csv = 'FOO,BAR\r\n'.encode('utf-8')
+        test_cases = [
+            # with title
+            {
+                'options': {
+                    'format': 'csv',
+                    'item_export_kwargs': {'include_headers_line': True},
+                },
+                'expected': expected_with_title_csv,
+            },
+            # without title
+            {
+                'options': {
+                    'format': 'csv',
+                    'item_export_kwargs': {'include_headers_line': False},
+                },
+                'expected': expected_without_title_csv,
+            },
+        ]
+
+        for row in test_cases:
+            feed_options = row['options']
+            settings = {
+                'FEEDS': {
+                    self._random_temp_filename(): feed_options,
+                },
+                'FEED_EXPORT_INDENT': None,
+            }
+
+            data = yield self.exported_data(items, settings)
+            self.assertEqual(row['expected'], data[feed_options['format']])
+
 
 class BatchDeliveriesTest(FeedExportTestBase):
     __test__ = True
@@ -1200,11 +1329,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
     def run_and_export(self, spider_cls, settings):
         """ Run spider with specified settings; return exported data. """
 
-        def build_url(path):
-            if path[0] != '/':
-                path = '/' + path
-            return urljoin('file:', path)
-
         FEEDS = settings.get('FEEDS') or {}
         settings['FEEDS'] = {
             build_url(file_path): feed
@@ -1495,47 +1619,70 @@ class BatchDeliveriesTest(FeedExportTestBase):
         self.assertEqual(len(items) + 1, len(data['json']))
 
     @defer.inlineCallbacks
-    def test_s3_export(self):
+    def test_stats_batch_file_success(self):
-        """
+        settings = {
-        Test export of items into s3 bucket.
+            "FEEDS": {
-        S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini
+                build_url(os.path.join(self._random_temp_filename(), "json", self._file_mark)): {
-        to perform this test:
+                    "format": "json",
-        [testenv]
+                }
-        setenv =
-            AWS_SECRET_ACCESS_KEY = ABCD
-            AWS_ACCESS_KEY_ID = EFGH
-            S3_TEST_BUCKET_NAME = IJKL
-        """
-        try:
-            import boto3
-        except ImportError:
-            raise unittest.SkipTest("S3FeedStorage requires boto3")
-
-        assert_aws_environ()
-        s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME')
-        access_key = os.environ.get('AWS_ACCESS_KEY_ID')
-        secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
-        if not s3_test_bucket_name:
-            raise unittest.SkipTest("No S3 BUCKET available for testing")
-
-        chars = [random.choice(ascii_letters + digits) for _ in range(15)]
-        filename = ''.join(chars)
-        prefix = f'tmp/{filename}'
-        s3_test_file_uri = f's3://{s3_test_bucket_name}/{prefix}/%(batch_time)s.json'
-        storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)
-        settings = Settings({
-            'FEEDS': {
-                s3_test_file_uri: {
-                    'format': 'json',
-                },
             },
-            'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
+            "FEED_EXPORT_BATCH_ITEM_COUNT": 1,
-        })
+        }
+        crawler = get_crawler(ItemSpider, settings)
+        with MockServer() as mockserver:
+            yield crawler.crawl(total=2, mockserver=mockserver)
+        self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats())
+        self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12)
+
+    @defer.inlineCallbacks
+    def test_s3_export(self):
+        skip_if_no_boto()
+
+        bucket = 'mybucket'
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
             self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
             self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
         ]
+
+        class CustomS3FeedStorage(S3FeedStorage):
+
+            stubs = []
+
+            def open(self, *args, **kwargs):
+                from botocore.stub import ANY, Stubber
+                stub = Stubber(self.s3_client)
+                stub.activate()
+                CustomS3FeedStorage.stubs.append(stub)
+                stub.add_response(
+                    'put_object',
+                    expected_params={
+                        'Body': ANY,
+                        'Bucket': bucket,
+                        'Key': ANY,
+                    },
+                    service_response={},
+                )
+                return super().open(*args, **kwargs)
+
+        key = 'export.csv'
+        uri = f's3://{bucket}/{key}/%(batch_time)s.json'
+        batch_item_count = 1
+        settings = {
+            'AWS_ACCESS_KEY_ID': 'access_key',
+            'AWS_SECRET_ACCESS_KEY': 'secret_key',
+            'FEED_EXPORT_BATCH_ITEM_COUNT': batch_item_count,
+            'FEED_STORAGES': {
+                's3': CustomS3FeedStorage,
+            },
+            'FEEDS': {
+                uri: {
+                    'format': 'json',
+                },
+            },
+        }
+        crawler = get_crawler(settings_dict=settings)
+        storage = S3FeedStorage.from_crawler(crawler, uri)
         verifyObject(IFeedStorage, storage)
 
         class TestSpider(scrapy.Spider):
@@ -1545,22 +1692,14 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 for item in items:
                     yield item
 
-        s3 = boto3.resource('s3')
+        with MockServer() as server:
-        my_bucket = s3.Bucket(s3_test_bucket_name)
-        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
-
-        with MockServer() as s:
             runner = CrawlerRunner(Settings(settings))
-            TestSpider.start_urls = [s.url('/')]
+            TestSpider.start_urls = [server.url('/')]
             yield runner.crawl(TestSpider)
 
-        for file_uri in my_bucket.objects.filter(Prefix=prefix):
+        self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1)
-            content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key)
+        for stub in CustomS3FeedStorage.stubs[:-1]:
-            if not content and not items:
+            stub.assert_no_pending_responses()
-                break
-            content = json.loads(content.decode('utf-8'))
-            expected_batch, items = items[:batch_size], items[batch_size:]
-            self.assertEqual(expected_batch, content)
 
 
 class FeedExportInitTest(unittest.TestCase):

tests/test_http_request.py

@@ -43,6 +43,15 @@ class RequestTest(unittest.TestCase):
         assert r.headers is not headers
         self.assertEqual(r.headers[b"caca"], b"coco")
 
+    def test_url_scheme(self):
+        # This test passes by not raising any (ValueError) exception
+        self.request_class('http://example.org')
+        self.request_class('https://example.org')
+        self.request_class('s3://example.org')
+        self.request_class('ftp://example.org')
+        self.request_class('about:config')
+        self.request_class('data:,Hello%2C%20World!')
+
     def test_url_no_scheme(self):
         self.assertRaises(ValueError, self.request_class, 'foo')
         self.assertRaises(ValueError, self.request_class, '/foo/')

tests/test_logformatter.py

@@ -193,7 +193,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
         self.base_settings = {
             'LOG_LEVEL': 'DEBUG',
             'ITEM_PIPELINES': {
-                __name__ + '.DropSomeItemsPipeline': 300,
+                DropSomeItemsPipeline: 300,
             },
         }
 
@@ -212,7 +212,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
     @defer.inlineCallbacks
     def test_skip_messages(self):
         settings = self.base_settings.copy()
-        settings['LOG_FORMATTER'] = __name__ + '.SkipMessagesLogFormatter'
+        settings['LOG_FORMATTER'] = SkipMessagesLogFormatter
         crawler = CrawlerRunner(settings).create_crawler(ItemSpider)
         with LogCapture() as lc:
             yield crawler.crawl(mockserver=self.mockserver)

tests/test_pipeline_files.py

@@ -1,6 +1,7 @@
 import os
 import random
 import time
+from datetime import datetime
 from io import BytesIO
 from shutil import rmtree
 from tempfile import mkdtemp
@@ -23,11 +24,10 @@ from scrapy.pipelines.files import (
 )
 from scrapy.settings import Settings
 from scrapy.utils.test import (
-    assert_aws_environ,
     assert_gcs_environ,
     get_ftp_content_and_delete,
     get_gcs_content_and_delete,
-    get_s3_content_and_delete,
+    skip_if_no_boto,
 )
 
 
@@ -414,32 +414,88 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
 
 
 class TestS3FilesStore(unittest.TestCase):
+
     @defer.inlineCallbacks
     def test_persist(self):
-        assert_aws_environ()
+        skip_if_no_boto()
-        uri = os.environ.get('S3_TEST_FILE_URI')
+
-        if not uri:
+        bucket = 'mybucket'
-            raise unittest.SkipTest("No S3 URI available for testing")
+        key = 'export.csv'
-        data = b"TestS3FilesStore: \xe2\x98\x83"
+        uri = f's3://{bucket}/{key}'
-        buf = BytesIO(data)
+        buffer = mock.MagicMock()
         meta = {'foo': 'bar'}
         path = ''
+        content_type = 'image/png'
+
         store = S3FilesStore(uri)
-        yield store.persist_file(
+        from botocore.stub import Stubber
-            path, buf, info=None, meta=meta,
+        with Stubber(store.s3_client) as stub:
-            headers={'Content-Type': 'image/png'})
+            stub.add_response(
-        s = yield store.stat_file(path, info=None)
+                'put_object',
-        self.assertIn('last_modified', s)
+                expected_params={
-        self.assertIn('checksum', s)
+                    'ACL': S3FilesStore.POLICY,
-        self.assertEqual(s['checksum'], '3187896a9657a28163abb31667df64c8')
+                    'Body': buffer,
-        u = urlparse(uri)
+                    'Bucket': bucket,
-        content, key = get_s3_content_and_delete(
+                    'CacheControl': S3FilesStore.HEADERS['Cache-Control'],
-            u.hostname, u.path[1:], with_key=True)
+                    'ContentType': content_type,
-        self.assertEqual(content, data)
+                    'Key': key,
-        self.assertEqual(key['Metadata'], {'foo': 'bar'})
+                    'Metadata': meta,
-        self.assertEqual(
+                },
-            key['CacheControl'], S3FilesStore.HEADERS['Cache-Control'])
+                service_response={},
-        self.assertEqual(key['ContentType'], 'image/png')
+            )
+
+            yield store.persist_file(
+                path,
+                buffer,
+                info=None,
+                meta=meta,
+                headers={'Content-Type': content_type},
+            )
+
+            stub.assert_no_pending_responses()
+            self.assertEqual(
+                buffer.method_calls,
+                [
+                    mock.call.seek(0),
+                    # The call to read does not happen with Stubber
+                ]
+            )
+
+    @defer.inlineCallbacks
+    def test_stat(self):
+        skip_if_no_boto()
+
+        bucket = 'mybucket'
+        key = 'export.csv'
+        uri = f's3://{bucket}/{key}'
+        checksum = '3187896a9657a28163abb31667df64c8'
+        last_modified = datetime(2019, 12, 1)
+
+        store = S3FilesStore(uri)
+        from botocore.stub import Stubber
+        with Stubber(store.s3_client) as stub:
+            stub.add_response(
+                'head_object',
+                expected_params={
+                    'Bucket': bucket,
+                    'Key': key,
+                },
+                service_response={
+                    'ETag': f'"{checksum}"',
+                    'LastModified': last_modified,
+                },
+            )
+
+            file_stats = yield store.stat_file('', info=None)
+            self.assertEqual(
+                file_stats,
+                {
+                    'checksum': checksum,
+                    'last_modified': last_modified.timestamp(),
+                },
+            )
+
+            stub.assert_no_pending_responses()
 
 
 class TestGCSFilesStore(unittest.TestCase):

tests/test_pipelines.py

@@ -68,7 +68,7 @@ class PipelineTestCase(unittest.TestCase):
 
     def _create_crawler(self, pipeline_class):
         settings = {
-            'ITEM_PIPELINES': {__name__ + '.' + pipeline_class.__name__: 1},
+            'ITEM_PIPELINES': {pipeline_class: 1},
         }
         crawler = get_crawler(ItemSpider, settings)
         crawler.signals.connect(self._on_item_scraped, signals.item_scraped)

tests/test_request_attribute_binding.py

@@ -92,7 +92,7 @@ class CrawlTestCase(TestCase):
         url = self.mockserver.url("/status?n=200")
         runner = CrawlerRunner(settings={
             "DOWNLOADER_MIDDLEWARES": {
-                __name__ + ".RaiseExceptionRequestMiddleware": 590,
+                RaiseExceptionRequestMiddleware: 590,
             },
         })
         crawler = runner.create_crawler(SingleRequestSpider)
@@ -119,7 +119,7 @@ class CrawlTestCase(TestCase):
         url = self.mockserver.url("/status?n=200")
         runner = CrawlerRunner(settings={
             "DOWNLOADER_MIDDLEWARES": {
-                __name__ + ".ProcessResponseMiddleware": 595,
+                ProcessResponseMiddleware: 595,
             }
         })
         crawler = runner.create_crawler(SingleRequestSpider)
@@ -149,8 +149,8 @@ class CrawlTestCase(TestCase):
         url = self.mockserver.url("/status?n=200")
         runner = CrawlerRunner(settings={
             "DOWNLOADER_MIDDLEWARES": {
-                __name__ + ".RaiseExceptionRequestMiddleware": 590,
+                RaiseExceptionRequestMiddleware: 590,
-                __name__ + ".CatchExceptionOverrideRequestMiddleware": 595,
+                CatchExceptionOverrideRequestMiddleware: 595,
             },
         })
         crawler = runner.create_crawler(SingleRequestSpider)
@@ -170,8 +170,8 @@ class CrawlTestCase(TestCase):
         url = self.mockserver.url("/status?n=200")
         runner = CrawlerRunner(settings={
             "DOWNLOADER_MIDDLEWARES": {
-                __name__ + ".RaiseExceptionRequestMiddleware": 590,
+                RaiseExceptionRequestMiddleware: 590,
-                __name__ + ".CatchExceptionDoNotOverrideRequestMiddleware": 595,
+                CatchExceptionDoNotOverrideRequestMiddleware: 595,
             },
         })
         crawler = runner.create_crawler(SingleRequestSpider)
@@ -188,7 +188,7 @@ class CrawlTestCase(TestCase):
         """
         runner = CrawlerRunner(settings={
             "DOWNLOADER_MIDDLEWARES": {
-                __name__ + ".AlternativeCallbacksMiddleware": 595,
+                AlternativeCallbacksMiddleware: 595,
             }
         })
         crawler = runner.create_crawler(AlternativeCallbacksSpider)

tests/test_request_cb_kwargs.py

@@ -50,10 +50,10 @@ class KeywordArgumentsSpider(MockServerSpider):
     name = 'kwargs'
     custom_settings = {
         'DOWNLOADER_MIDDLEWARES': {
-            __name__ + '.InjectArgumentsDownloaderMiddleware': 750,
+            InjectArgumentsDownloaderMiddleware: 750,
         },
         'SPIDER_MIDDLEWARES': {
-            __name__ + '.InjectArgumentsSpiderMiddleware': 750,
+            InjectArgumentsSpiderMiddleware: 750,
         },
     }
 

tests/test_spider.py

@@ -1,8 +1,8 @@
 import gzip
 import inspect
-from unittest import mock
 import warnings
 from io import BytesIO
+from unittest import mock
 
 from testfixtures import LogCapture
 from twisted.trial import unittest
@@ -20,7 +20,6 @@ from scrapy.spiders import (
     XMLFeedSpider,
 )
 from scrapy.linkextractors import LinkExtractor
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.test import get_crawler
 
 
@@ -280,7 +279,7 @@ class CrawlSpiderTest(SpiderTest):
 
         response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
 
-        def process_request_change_domain(request):
+        def process_request_change_domain(request, response):
             return request.replace(url=request.url.replace('.org', '.com'))
 
         class _CrawlSpider(self.spider_class):
@@ -290,17 +289,14 @@ class CrawlSpiderTest(SpiderTest):
                 Rule(LinkExtractor(), process_request=process_request_change_domain),
             )
 
-        with warnings.catch_warnings(record=True) as cw:
+        spider = _CrawlSpider()
-            spider = _CrawlSpider()
+        output = list(spider._requests_to_follow(response))
-            output = list(spider._requests_to_follow(response))
+        self.assertEqual(len(output), 3)
-            self.assertEqual(len(output), 3)
+        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-            self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertEqual([r.url for r in output],
-            self.assertEqual([r.url for r in output],
+                         ['http://example.com/somepage/item/12.html',
-                             ['http://example.com/somepage/item/12.html',
+                          'http://example.com/about.html',
-                              'http://example.com/about.html',
+                          'http://example.com/nofollow.html'])
-                              'http://example.com/nofollow.html'])
-            self.assertEqual(len(cw), 1)
-            self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
 
     def test_process_request_with_response(self):
 
@@ -339,20 +335,17 @@ class CrawlSpiderTest(SpiderTest):
                 Rule(LinkExtractor(), process_request='process_request_upper'),
             )
 
-            def process_request_upper(self, request):
+            def process_request_upper(self, request, response):
                 return request.replace(url=request.url.upper())
 
-        with warnings.catch_warnings(record=True) as cw:
+        spider = _CrawlSpider()
-            spider = _CrawlSpider()
+        output = list(spider._requests_to_follow(response))
-            output = list(spider._requests_to_follow(response))
+        self.assertEqual(len(output), 3)
-            self.assertEqual(len(output), 3)
+        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-            self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertEqual([r.url for r in output],
-            self.assertEqual([r.url for r in output],
+                         ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
-                             ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
+                          'http://EXAMPLE.ORG/ABOUT.HTML',
-                              'http://EXAMPLE.ORG/ABOUT.HTML',
+                          'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
-                              'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
-            self.assertEqual(len(cw), 1)
-            self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
 
     def test_process_request_instance_method_with_response(self):
 

tests/test_spidermiddleware_output_chain.py

@@ -16,11 +16,20 @@ class LogExceptionMiddleware:
 
 # ================================================================================
 # (0) recover from an exception on a spider callback
+class RecoveryMiddleware:
+    def process_spider_exception(self, response, exception, spider):
+        spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
+        return [
+            {'from': 'process_spider_exception'},
+            Request(response.url, meta={'dont_fail': True}, dont_filter=True),
+        ]
+
+
 class RecoverySpider(Spider):
     name = 'RecoverySpider'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
-            __name__ + '.RecoveryMiddleware': 10,
+            RecoveryMiddleware: 10,
         },
     }
 
@@ -34,15 +43,6 @@ class RecoverySpider(Spider):
             raise TabError()
 
 
-class RecoveryMiddleware:
-    def process_spider_exception(self, response, exception, spider):
-        spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
-        return [
-            {'from': 'process_spider_exception'},
-            Request(response.url, meta={'dont_fail': True}, dont_filter=True),
-        ]
-
-
 # ================================================================================
 # (1) exceptions from a spider middleware's process_spider_input method
 class FailProcessSpiderInputMiddleware:
@@ -56,9 +56,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider):
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
             # spider
-            __name__ + '.LogExceptionMiddleware': 10,
+            FailProcessSpiderInputMiddleware: 8,
-            __name__ + '.FailProcessSpiderInputMiddleware': 8,
+            LogExceptionMiddleware: 6,
-            __name__ + '.LogExceptionMiddleware': 6,
             # engine
         }
     }
@@ -87,7 +86,7 @@ class GeneratorCallbackSpider(Spider):
     name = 'GeneratorCallbackSpider'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
-            __name__ + '.LogExceptionMiddleware': 10,
+            LogExceptionMiddleware: 10,
         },
     }
 
@@ -106,7 +105,7 @@ class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider)
     name = 'GeneratorCallbackSpiderMiddlewareRightAfterSpider'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
-            __name__ + '.LogExceptionMiddleware': 100000,
+            LogExceptionMiddleware: 100000,
         },
     }
 
@@ -117,7 +116,7 @@ class NotGeneratorCallbackSpider(Spider):
     name = 'NotGeneratorCallbackSpider'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
-            __name__ + '.LogExceptionMiddleware': 10,
+            LogExceptionMiddleware: 10,
         },
     }
 
@@ -134,32 +133,13 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS
     name = 'NotGeneratorCallbackSpiderMiddlewareRightAfterSpider'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
-            __name__ + '.LogExceptionMiddleware': 100000,
+            LogExceptionMiddleware: 100000,
         },
     }
 
 
 # ================================================================================
 # (4) exceptions from a middleware process_spider_output method (generator)
-class GeneratorOutputChainSpider(Spider):
-    name = 'GeneratorOutputChainSpider'
-    custom_settings = {
-        'SPIDER_MIDDLEWARES': {
-            __name__ + '.GeneratorFailMiddleware': 10,
-            __name__ + '.GeneratorDoNothingAfterFailureMiddleware': 8,
-            __name__ + '.GeneratorRecoverMiddleware': 5,
-            __name__ + '.GeneratorDoNothingAfterRecoveryMiddleware': 3,
-        },
-    }
-
-    def start_requests(self):
-        yield Request(self.mockserver.url('/status?n=200'))
-
-    def parse(self, response):
-        yield {'processed': ['parse-first-item']}
-        yield {'processed': ['parse-second-item']}
-
-
 class _GeneratorDoNothingMiddleware:
     def process_spider_output(self, response, result, spider):
         for r in result:
@@ -205,26 +185,28 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
     pass
 
 
-# ================================================================================
+class GeneratorOutputChainSpider(Spider):
-# (5) exceptions from a middleware process_spider_output method (not generator)
+    name = 'GeneratorOutputChainSpider'
-class NotGeneratorOutputChainSpider(Spider):
-    name = 'NotGeneratorOutputChainSpider'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
-            __name__ + '.NotGeneratorFailMiddleware': 10,
+            GeneratorFailMiddleware: 10,
-            __name__ + '.NotGeneratorDoNothingAfterFailureMiddleware': 8,
+            GeneratorDoNothingAfterFailureMiddleware: 8,
-            __name__ + '.NotGeneratorRecoverMiddleware': 5,
+            GeneratorRecoverMiddleware: 5,
-            __name__ + '.NotGeneratorDoNothingAfterRecoveryMiddleware': 3,
+            GeneratorDoNothingAfterRecoveryMiddleware: 3,
         },
     }
 
     def start_requests(self):
-        return [Request(self.mockserver.url('/status?n=200'))]
+        yield Request(self.mockserver.url('/status?n=200'))
 
     def parse(self, response):
-        return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}]
+        yield {'processed': ['parse-first-item']}
+        yield {'processed': ['parse-second-item']}
 
 
+# ================================================================================
+# (5) exceptions from a middleware process_spider_output method (not generator)
+
 class _NotGeneratorDoNothingMiddleware:
     def process_spider_output(self, response, result, spider):
         out = []
@@ -276,6 +258,24 @@ class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddlew
     pass
 
 
+class NotGeneratorOutputChainSpider(Spider):
+    name = 'NotGeneratorOutputChainSpider'
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            NotGeneratorFailMiddleware: 10,
+            NotGeneratorDoNothingAfterFailureMiddleware: 8,
+            NotGeneratorRecoverMiddleware: 5,
+            NotGeneratorDoNothingAfterRecoveryMiddleware: 3,
+        },
+    }
+
+    def start_requests(self):
+        return [Request(self.mockserver.url('/status?n=200'))]
+
+    def parse(self, response):
+        return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}]
+
+
 # ================================================================================
 class TestSpiderMiddleware(TestCase):
     @classmethod

tests/test_utils_conf.py

@@ -176,6 +176,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "store_empty": True,
             "uri_params": (1, 2, 3, 4),
             "batch_item_count": 2,
+            "item_export_kwargs": dict(),
         })
 
     def test_feed_complete_default_values_from_settings_non_empty(self):
@@ -198,6 +199,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "store_empty": True,
             "uri_params": None,
             "batch_item_count": 2,
+            "item_export_kwargs": dict(),
         })
 
 

tests/test_utils_gz.py

@@ -3,10 +3,11 @@ from os.path import join
 
 from w3lib.encoding import html_to_unicode
 
-from scrapy.utils.gz import gunzip, is_gzipped
+from scrapy.utils.gz import gunzip, gzip_magic_number
-from scrapy.http import Response, Headers
+from scrapy.http import Response
 from tests import tests_datadir
 
+
 SAMPLEDIR = join(tests_datadir, 'compressed')
 
 
@@ -14,8 +15,12 @@ class GunzipTest(unittest.TestCase):
 
     def test_gunzip_basic(self):
         with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f:
-            text = gunzip(f.read())
+            r1 = Response("http://www.example.com", body=f.read())
-            self.assertEqual(len(text), 9950)
+            self.assertTrue(gzip_magic_number(r1))
+
+            r2 = Response("http://www.example.com", body=gunzip(r1.body))
+            self.assertFalse(gzip_magic_number(r2))
+            self.assertEqual(len(r2.body), 9950)
 
     def test_gunzip_truncated(self):
         with open(join(SAMPLEDIR, 'truncated-crc-error.gz'), 'rb') as f:
@@ -28,46 +33,16 @@ class GunzipTest(unittest.TestCase):
 
     def test_gunzip_truncated_short(self):
         with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f:
-            text = gunzip(f.read())
+            r1 = Response("http://www.example.com", body=f.read())
-            assert text.endswith(b'</html>')
+            self.assertTrue(gzip_magic_number(r1))
 
-    def test_is_x_gzipped_right(self):
+            r2 = Response("http://www.example.com", body=gunzip(r1.body))
-        hdrs = Headers({"Content-Type": "application/x-gzip"})
+            assert r2.body.endswith(b'</html>')
-        r1 = Response("http://www.example.com", headers=hdrs)
+            self.assertFalse(gzip_magic_number(r2))
-        self.assertTrue(is_gzipped(r1))
-
-    def test_is_gzipped_right(self):
-        hdrs = Headers({"Content-Type": "application/gzip"})
-        r1 = Response("http://www.example.com", headers=hdrs)
-        self.assertTrue(is_gzipped(r1))
-
-    def test_is_gzipped_not_quite(self):
-        hdrs = Headers({"Content-Type": "application/gzippppp"})
-        r1 = Response("http://www.example.com", headers=hdrs)
-        self.assertFalse(is_gzipped(r1))
-
-    def test_is_gzipped_case_insensitive(self):
-        hdrs = Headers({"Content-Type": "Application/X-Gzip"})
-        r1 = Response("http://www.example.com", headers=hdrs)
-        self.assertTrue(is_gzipped(r1))
-
-        hdrs = Headers({"Content-Type": "application/X-GZIP ; charset=utf-8"})
-        r1 = Response("http://www.example.com", headers=hdrs)
-        self.assertTrue(is_gzipped(r1))
 
     def test_is_gzipped_empty(self):
         r1 = Response("http://www.example.com")
-        self.assertFalse(is_gzipped(r1))
+        self.assertFalse(gzip_magic_number(r1))
-
-    def test_is_gzipped_wrong(self):
-        hdrs = Headers({"Content-Type": "application/javascript"})
-        r1 = Response("http://www.example.com", headers=hdrs)
-        self.assertFalse(is_gzipped(r1))
-
-    def test_is_gzipped_with_charset(self):
-        hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"})
-        r1 = Response("http://www.example.com", headers=hdrs)
-        self.assertTrue(is_gzipped(r1))
 
     def test_gunzip_illegal_eof(self):
         with open(join(SAMPLEDIR, 'unexpected-eof.gz'), 'rb') as f:

tests/test_utils_http.py

@@ -1,19 +0,0 @@
-import unittest
-
-from scrapy.utils.http import decode_chunked_transfer
-
-
-class ChunkedTest(unittest.TestCase):
-
-    def test_decode_chunked_transfer(self):
-        """Example taken from: http://en.wikipedia.org/wiki/Chunked_transfer_encoding"""
-        chunked_body = "25\r\n" + "This is the data in the first chunk\r\n\r\n"
-        chunked_body += "1C\r\n" + "and this is the second one\r\n\r\n"
-        chunked_body += "3\r\n" + "con\r\n"
-        chunked_body += "8\r\n" + "sequence\r\n"
-        chunked_body += "0\r\n\r\n"
-        body = decode_chunked_transfer(chunked_body)
-        self.assertEqual(
-            body,
-            "This is the data in the first chunk\r\nand this is the second one\r\nconsequence"
-        )

tests/test_utils_iterators.py

@@ -1,5 +1,6 @@
 import os
 
+from pytest import mark
 from twisted.trial import unittest
 
 from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml
@@ -134,7 +135,6 @@ class XmliterTestCase(unittest.TestCase):
         """
         response = XmlResponse(url='http://mydummycompany.com', body=body)
         my_iter = self.xmliter(response, 'item')
-
         node = next(my_iter)
         node.register_namespace('g', 'http://base.google.com/ns/1.0')
         self.assertEqual(node.xpath('title/text()').getall(), ['Item 1'])
@@ -150,6 +150,55 @@ class XmliterTestCase(unittest.TestCase):
         self.assertEqual(node.xpath('id/text()').getall(), [])
         self.assertEqual(node.xpath('price/text()').getall(), [])
 
+    def test_xmliter_namespaced_nodename(self):
+        body = b"""
+            <?xml version="1.0" encoding="UTF-8"?>
+            <rss version="2.0" xmlns:g="http://base.google.com/ns/1.0">
+                <channel>
+                <title>My Dummy Company</title>
+                <link>http://www.mydummycompany.com</link>
+                <description>This is a dummy company. We do nothing.</description>
+                <item>
+                    <title>Item 1</title>
+                    <description>This is item 1</description>
+                    <link>http://www.mydummycompany.com/items/1</link>
+                    <g:image_link>http://www.mydummycompany.com/images/item1.jpg</g:image_link>
+                    <g:id>ITEM_1</g:id>
+                    <g:price>400</g:price>
+                </item>
+                </channel>
+            </rss>
+        """
+        response = XmlResponse(url='http://mydummycompany.com', body=body)
+        my_iter = self.xmliter(response, 'g:image_link')
+        node = next(my_iter)
+        node.register_namespace('g', 'http://base.google.com/ns/1.0')
+        self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
+
+    def test_xmliter_namespaced_nodename_missing(self):
+        body = b"""
+            <?xml version="1.0" encoding="UTF-8"?>
+            <rss version="2.0" xmlns:g="http://base.google.com/ns/1.0">
+                <channel>
+                <title>My Dummy Company</title>
+                <link>http://www.mydummycompany.com</link>
+                <description>This is a dummy company. We do nothing.</description>
+                <item>
+                    <title>Item 1</title>
+                    <description>This is item 1</description>
+                    <link>http://www.mydummycompany.com/items/1</link>
+                    <g:image_link>http://www.mydummycompany.com/images/item1.jpg</g:image_link>
+                    <g:id>ITEM_1</g:id>
+                    <g:price>400</g:price>
+                </item>
+                </channel>
+            </rss>
+        """
+        response = XmlResponse(url='http://mydummycompany.com', body=body)
+        my_iter = self.xmliter(response, 'g:link_image')
+        with self.assertRaises(StopIteration):
+            next(my_iter)
+
     def test_xmliter_exception(self):
         body = (
             '<?xml version="1.0" encoding="UTF-8"?>'
@@ -183,6 +232,10 @@ class XmliterTestCase(unittest.TestCase):
 class LxmlXmliterTestCase(XmliterTestCase):
     xmliter = staticmethod(xmliter_lxml)
 
+    @mark.xfail(reason='known bug of the current implementation')
+    def test_xmliter_namespaced_nodename(self):
+        super().test_xmliter_namespaced_nodename()
+
     def test_xmliter_iterate_namespace(self):
         body = b"""
             <?xml version="1.0" encoding="UTF-8"?>

tox.ini

@@ -16,7 +16,6 @@ deps =
     mitmproxy; python_version >= '3.7' and implementation_name != 'pypy'
     mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
     # Extras
-    boto3>=1.13.0
     botocore>=1.4.87
     Pillow>=4.0.0
 passenv =