Merge remote-tracking branch 'origin/master' into asyncio-eagerness

.travis.yml

@@ -18,19 +18,25 @@ matrix:
     - env: TOXENV=typing
       python: 3.8
 
-    - env: TOXENV=pypy3
     - env: TOXENV=pinned
       python: 3.5.2
-    - env: TOXENV=asyncio
+    - env: TOXENV=asyncio-pinned
       python: 3.5.2  # We use additional code to support 3.5.3 and earlier
+    - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0
+
     - env: TOXENV=py
       python: 3.5
     - env: TOXENV=asyncio
       python: 3.5  # We use specific code to support >= 3.5.4, < 3.6
+    - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0
+
     - env: TOXENV=py
       python: 3.6
+    - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1
+
     - env: TOXENV=py
       python: 3.7
+
     - env: TOXENV=py PYPI_RELEASE_JOB=true
       python: 3.8
       dist: bionic
@@ -42,9 +48,9 @@ matrix:
       dist: bionic
 install:
   - |
-      if [ "$TOXENV" = "pypy3" ]; then
-        export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
-        wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
+      if [[ ! -z "$PYPY_VERSION" ]]; then
+        export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
+        wget "https://bitbucket.org/pypy/pypy/downloads/${PYPY_VERSION}.tar.bz2"
         tar -jxf ${PYPY_VERSION}.tar.bz2
         virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"

azure-pipelines.yml

@@ -0,0 +1,24 @@
+variables:
+  TOXENV: py
+pool:
+  vmImage: 'windows-latest'
+strategy:
+  matrix:
+    Python35:
+      python.version: '3.5'
+      TOXENV: windows-pinned
+    Python36:
+      python.version: '3.6'
+    Python37:
+      python.version: '3.7'
+    Python38:
+      python.version: '3.8'
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: '$(python.version)'
+  displayName: 'Use Python $(python.version)'
+- script: |
+    pip install -U tox twine wheel codecov
+    tox
+  displayName: 'Run test suite'

docs/conf.py

@@ -284,6 +284,7 @@ intersphinx_mapping = {
     'attrs': ('https://www.attrs.org/en/stable/', None),
     'coverage': ('https://coverage.readthedocs.io/en/stable', None),
     'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
+    'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None),
     'pytest': ('https://docs.pytest.org/en/latest', None),
     'python': ('https://docs.python.org/3', None),
     'sphinx': ('https://www.sphinx-doc.org/en/master', None),
@@ -305,3 +306,15 @@ hoverxref_role_types = {
     "ref": "tooltip",
 }
 hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']
+
+
+def setup(app):
+    app.connect('autodoc-skip-member', maybe_skip_member)
+
+
+def maybe_skip_member(app, what, name, obj, skip, options):
+    if not skip:
+        # autodocs was generating a text "alias of" for the following members
+        # https://github.com/sphinx-doc/sphinx/issues/4422
+        return name in {'default_item_class', 'default_selector_class'}
+    return skip

docs/faq.rst

@@ -64,20 +64,6 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
 
 .. _BeautifulSoup's official documentation: https://www.crummy.com/software/BeautifulSoup/bs4/doc/#specifying-the-parser-to-use
 
-.. _faq-python-versions:
-
-What Python versions does Scrapy support?
------------------------------------------
-
-Scrapy is supported under Python 3.5.2+
-under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
-Python 3 support was added in Scrapy 1.1.
-PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
-Python 2 support was dropped in Scrapy 2.0.
-
-.. note::
-    For Python 3 support on Windows, it is recommended to use
-    Anaconda/Miniconda as :ref:`outlined in the installation guide <intro-install-windows>`.
 
 Did Scrapy "steal" X from Django?
 ---------------------------------

docs/intro/install.rst

@@ -4,12 +4,18 @@
 Installation guide
 ==================
 
+.. _faq-python-versions:
+
+Supported Python versions
+=========================
+
+Scrapy requires Python 3.5.2+, either the CPython implementation (default) or
+the PyPy 5.9+ implementation (see :ref:`python:implementations`).
+
+
 Installing Scrapy
 =================
 
-Scrapy runs on Python 3.5.2 or above under CPython (default Python
-implementation) and PyPy (starting with PyPy 5.9).
-
 If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
 the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
 and macOS.

docs/topics/commands.rst

@@ -468,7 +468,7 @@ Supported options:
 * ``--callback`` or ``-c``: spider method to use as callback for parsing the
   response
 
-* ``--meta`` or ``-m``: additional request meta that will be passed to the callback 
+* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
   request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
 
 * ``--cbkwargs``: additional keyword arguments that will be passed to the callback.
@@ -491,6 +491,8 @@ Supported options:
 
 * ``--verbose`` or ``-v``: display information for each depth level
 
+* ``--output`` or ``-o``: dump scraped items to a file
+
 .. skip: start
 
 Usage example::

docs/topics/feed-exports.rst

@@ -194,6 +194,27 @@ You can also define a custom ACL for exported feeds using this setting:
 
  * :setting:`FEED_STORAGE_S3_ACL`
 
+.. _topics-feed-storage-gcs:
+
+Google Cloud Storage (GCS)
+--------------------------
+
+The feeds are stored on `Google Cloud Storage`_.
+
+ * URI scheme: ``gs``
+ * Example URIs:
+
+   * ``gs://mybucket/path/to/export.csv``
+
+ * Required external libraries: `google-cloud-storage <https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python>`_.
+
+For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
+
+You can set a *Project ID* and *Access Control List (ACL)* through the following settings:
+
+ * :setting:`FEED_STORAGE_GCS_ACL`
+ * :setting:`GCS_PROJECT_ID`
+
 .. _topics-feed-storage-stdout:
 
 Standard output
@@ -429,3 +450,4 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 .. _Amazon S3: https://aws.amazon.com/s3/
 .. _botocore: https://github.com/boto/botocore
 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+.. _Google Cloud Storage: https://cloud.google.com/storage/

docs/topics/loaders.rst

@@ -20,6 +20,10 @@ Item Loaders are designed to provide a flexible, efficient and easy mechanism
 for extending and overriding different field parsing rules, either by spider,
 or by source format (HTML, XML, etc) without becoming a nightmare to maintain.
 
+.. note:: Item Loaders are an extension of the itemloaders_ library that make it
+    easier to work with Scrapy by adding support for
+    :ref:`responses <topics-request-response>`.
+
 Using Item Loaders to populate items
 ====================================
 
@@ -88,29 +92,17 @@ item loaders: unless a pre-populated item is passed to the loader, fields
 will be populated incrementally using the loader's :meth:`~ItemLoader.add_xpath`,
 :meth:`~ItemLoader.add_css` and :meth:`~ItemLoader.add_value` methods.
 
-Given the way that item loaders store data internally, one approach
-to overcome this is to define items using the :func:`~dataclasses.field`
-function, with ``list`` as the ``default_factory`` argument::
+One approach to overcome this is to define items using the
+:func:`~dataclasses.field` function, with a ``default`` argument::
 
     from dataclasses import dataclass, field
+    from typing import Optional
 
     @dataclass
     class InventoryItem:
-        name: str = field(default_factory=list)
-        price: float = field(default_factory=list)
-        stock: int = field(default_factory=list)
-
-Note that in order to keep the example simple, the types do not match
-completely. A more accurate but verbose definition would be::
-
-    from dataclasses import dataclass, field
-    from typing import List, Union
-
-    @dataclass
-    class InventoryItem:
-        name: Union[str, List[str]] = field(default_factory=list)
-        price: Union[float, List[float]] = field(default_factory=list)
-        stock: Union[int, List[int]] = field(default_factory=list)
+        name: Optional[str] = field(default=None)
+        price: Optional[float] = field(default=None)
+        stock: Optional[int] = field(default=None)
 
 
 .. _topics-loaders-processors:
@@ -185,8 +177,8 @@ The other thing you need to keep in mind is that the values returned by input
 processors are collected internally (in lists) and then passed to output
 processors to populate the fields.
 
-Last, but not least, Scrapy comes with some :ref:`commonly used processors
-<topics-loaders-available-processors>` built-in for convenience.
+Last, but not least, itemloaders_ comes with some :ref:`commonly used
+processors <itemloaders:built-in-processors>` built-in for convenience.
 
 
 Declaring Item Loaders
@@ -194,8 +186,8 @@ Declaring Item Loaders
 
 Item Loaders are declared using a class definition syntax. Here is an example::
 
+    from itemloaders.processors import TakeFirst, MapCompose, Join
     from scrapy.loader import ItemLoader
-    from scrapy.loader.processors import TakeFirst, MapCompose, Join
 
     class ProductLoader(ItemLoader):
 
@@ -226,7 +218,7 @@ output processors to use: in the :ref:`Item Field <topics-items-fields>`
 metadata. Here is an example::
 
     import scrapy
-    from scrapy.loader.processors import Join, MapCompose, TakeFirst
+    from itemloaders.processors import Join, MapCompose, TakeFirst
     from w3lib.html import remove_tags
 
     def filter_price(value):
@@ -307,250 +299,9 @@ There are several ways to modify Item Loader context values:
 ItemLoader objects
 ==================
 
-.. class:: ItemLoader([item, selector, response], **kwargs)
-
-    Return a new Item Loader for populating the given :ref:`item object
-    <topics-items>`. If no item object is given, one is instantiated
-    automatically using the class in :attr:`default_item_class`.
-
-    When instantiated with a ``selector`` or a ``response`` parameters
-    the :class:`ItemLoader` class provides convenient mechanisms for extracting
-    data from web pages using :ref:`selectors <topics-selectors>`.
-
-    :param item: The item instance to populate using subsequent calls to
-        :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
-        or :meth:`~ItemLoader.add_value`.
-    :type item: :ref:`item object <topics-items>`
-
-    :param selector: The selector to extract data from, when using the
-        :meth:`add_xpath` (resp. :meth:`add_css`) or :meth:`replace_xpath`
-        (resp. :meth:`replace_css`) method.
-    :type selector: :class:`~scrapy.selector.Selector` object
-
-    :param response: The response used to construct the selector using the
-        :attr:`default_selector_class`, unless the selector argument is given,
-        in which case this argument is ignored.
-    :type response: :class:`~scrapy.http.Response` object
-
-    The item, selector, response and the remaining keyword arguments are
-    assigned to the Loader context (accessible through the :attr:`context` attribute).
-
-    :class:`ItemLoader` instances have the following methods:
-
-    .. method:: get_value(value, *processors, **kwargs)
-
-        Process the given ``value`` by the given ``processors`` and keyword
-        arguments.
-
-        Available keyword arguments:
-
-        :param re: a regular expression to use for extracting data from the
-            given value using :meth:`~scrapy.utils.misc.extract_regex` method,
-            applied before processors
-        :type re: str or compiled regex
-
-        Examples:
-
-        >>> from scrapy.loader.processors import TakeFirst
-        >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
-        'FOO`
-
-    .. method:: add_value(field_name, value, *processors, **kwargs)
-
-        Process and then add the given ``value`` for the given field.
-
-        The value is first passed through :meth:`get_value` by giving the
-        ``processors`` and ``kwargs``, and then passed through the
-        :ref:`field input processor <topics-loaders-processors>` and its result
-        appended to the data collected for that field. If the field already
-        contains collected data, the new data is added.
-
-        The given ``field_name`` can be ``None``, in which case values for
-        multiple fields may be added. And the processed value should be a dict
-        with field_name mapped to values.
-
-        Examples::
-
-            loader.add_value('name', u'Color TV')
-            loader.add_value('colours', [u'white', u'blue'])
-            loader.add_value('length', u'100')
-            loader.add_value('name', u'name: foo', TakeFirst(), re='name: (.+)')
-            loader.add_value(None, {'name': u'foo', 'sex': u'male'})
-
-    .. method:: replace_value(field_name, value, *processors, **kwargs)
-
-        Similar to :meth:`add_value` but replaces the collected data with the
-        new value instead of adding it.
-    .. method:: get_xpath(xpath, *processors, **kwargs)
-
-        Similar to :meth:`ItemLoader.get_value` but receives an XPath instead of a
-        value, which is used to extract a list of unicode strings from the
-        selector associated with this :class:`ItemLoader`.
-
-        :param xpath: the XPath to extract data from
-        :type xpath: str
-
-        :param re: a regular expression to use for extracting data from the
-            selected XPath region
-        :type re: str or compiled regex
-
-        Examples::
-
-            # HTML snippet: <p class="product-name">Color TV</p>
-            loader.get_xpath('//p[@class="product-name"]')
-            # HTML snippet: <p id="price">the price is $1200</p>
-            loader.get_xpath('//p[@id="price"]', TakeFirst(), re='the price is (.*)')
-
-    .. method:: add_xpath(field_name, xpath, *processors, **kwargs)
-
-        Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a
-        value, which is used to extract a list of unicode strings from the
-        selector associated with this :class:`ItemLoader`.
-
-        See :meth:`get_xpath` for ``kwargs``.
-
-        :param xpath: the XPath to extract data from
-        :type xpath: str
-
-        Examples::
-
-            # HTML snippet: <p class="product-name">Color TV</p>
-            loader.add_xpath('name', '//p[@class="product-name"]')
-            # HTML snippet: <p id="price">the price is $1200</p>
-            loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)')
-
-    .. method:: replace_xpath(field_name, xpath, *processors, **kwargs)
-
-        Similar to :meth:`add_xpath` but replaces collected data instead of
-        adding it.
-
-    .. method:: get_css(css, *processors, **kwargs)
-
-        Similar to :meth:`ItemLoader.get_value` but receives a CSS selector
-        instead of a value, which is used to extract a list of unicode strings
-        from the selector associated with this :class:`ItemLoader`.
-
-        :param css: the CSS selector to extract data from
-        :type css: str
-
-        :param re: a regular expression to use for extracting data from the
-            selected CSS region
-        :type re: str or compiled regex
-
-        Examples::
-
-            # HTML snippet: <p class="product-name">Color TV</p>
-            loader.get_css('p.product-name')
-            # HTML snippet: <p id="price">the price is $1200</p>
-            loader.get_css('p#price', TakeFirst(), re='the price is (.*)')
-
-    .. method:: add_css(field_name, css, *processors, **kwargs)
-
-        Similar to :meth:`ItemLoader.add_value` but receives a CSS selector
-        instead of a value, which is used to extract a list of unicode strings
-        from the selector associated with this :class:`ItemLoader`.
-
-        See :meth:`get_css` for ``kwargs``.
-
-        :param css: the CSS selector to extract data from
-        :type css: str
-
-        Examples::
-
-            # HTML snippet: <p class="product-name">Color TV</p>
-            loader.add_css('name', 'p.product-name')
-            # HTML snippet: <p id="price">the price is $1200</p>
-            loader.add_css('price', 'p#price', re='the price is (.*)')
-
-    .. method:: replace_css(field_name, css, *processors, **kwargs)
-
-        Similar to :meth:`add_css` but replaces collected data instead of
-        adding it.
-
-    .. method:: load_item()
-
-        Populate the item with the data collected so far, and return it. The
-        data collected is first passed through the :ref:`output processors
-        <topics-loaders-processors>` to get the final value to assign to each
-        item field.
-
-    .. method:: nested_xpath(xpath)
-
-        Create a nested loader with an xpath selector.
-        The supplied selector is applied relative to selector associated
-        with this :class:`ItemLoader`. The nested loader shares the :ref:`item
-        object <topics-items>` with the parent :class:`ItemLoader` so calls to
-        :meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
-        behave as expected.
-
-    .. method:: nested_css(css)
-
-        Create a nested loader with a css selector.
-        The supplied selector is applied relative to selector associated
-        with this :class:`ItemLoader`. The nested loader shares the :ref:`item
-        object <topics-items>` with the parent :class:`ItemLoader` so calls to
-        :meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
-        behave as expected.
-
-    .. method:: get_collected_values(field_name)
-
-        Return the collected values for the given field.
-
-    .. method:: get_output_value(field_name)
-
-        Return the collected values parsed using the output processor, for the
-        given field. This method doesn't populate or modify the item at all.
-
-    .. method:: get_input_processor(field_name)
-
-        Return the input processor for the given field.
-
-    .. method:: get_output_processor(field_name)
-
-        Return the output processor for the given field.
-
-    :class:`ItemLoader` instances have the following attributes:
-
-    .. attribute:: item
-
-        The :ref:`item object <topics-items>` being parsed by this Item Loader.
-        This is mostly used as a property so when attempting to override this
-        value, you may want to check out :attr:`default_item_class` first.
-
-    .. attribute:: context
-
-        The currently active :ref:`Context <topics-loaders-context>` of this
-        Item Loader.
-
-    .. attribute:: default_item_class
-
-        An :ref:`item object <topics-items>` class or factory, used to
-        instantiate items when not given in the ``__init__`` method.
-
-    .. attribute:: default_input_processor
-
-        The default input processor to use for those fields which don't specify
-        one.
-
-    .. attribute:: default_output_processor
-
-        The default output processor to use for those fields which don't specify
-        one.
-
-    .. attribute:: default_selector_class
-
-        The class used to construct the :attr:`selector` of this
-        :class:`ItemLoader`, if only a response is given in the ``__init__`` method.
-        If a selector is given in the ``__init__`` method this attribute is ignored.
-        This attribute is sometimes overridden in subclasses.
-
-    .. attribute:: selector
-
-        The :class:`~scrapy.selector.Selector` object to extract data from.
-        It's either the selector given in the ``__init__`` method or one created from
-        the response given in the ``__init__`` method using the
-        :attr:`default_selector_class`. This attribute is meant to be
-        read-only.
+.. autoclass:: scrapy.loader.ItemLoader
+    :members:
+    :inherited-members:
 
 .. _topics-loaders-nested:
 
@@ -621,7 +372,7 @@ those dashes in the final product names.
 Here's how you can remove those dashes by reusing and extending the default
 Product Item Loader (``ProductLoader``)::
 
-    from scrapy.loader.processors import MapCompose
+    from itemloaders.processors import MapCompose
     from myproject.ItemLoaders import ProductLoader
 
     def strip_dashes(x):
@@ -634,7 +385,7 @@ Another case where extending Item Loaders can be very helpful is when you have
 multiple source formats, for example XML and HTML. In the XML version you may
 want to remove ``CDATA`` occurrences. Here's an example of how to do it::
 
-    from scrapy.loader.processors import MapCompose
+    from itemloaders.processors import MapCompose
     from myproject.ItemLoaders import ProductLoader
     from myproject.utils.xml import remove_cdata
 
@@ -654,156 +405,5 @@ projects. Scrapy only provides the mechanism; it doesn't impose any specific
 organization of your Loaders collection - that's up to you and your project's
 needs.
 
-.. _topics-loaders-available-processors:
-
-Available built-in processors
-=============================
-
-.. module:: scrapy.loader.processors
-   :synopsis: A collection of processors to use with Item Loaders
-
-Even though you can use any callable function as input and output processors,
-Scrapy provides some commonly used processors, which are described below. Some
-of them, like the :class:`MapCompose` (which is typically used as input
-processor) compose the output of several functions executed in order, to
-produce the final parsed value.
-
-Here is a list of all built-in processors:
-
-.. class:: Identity
-
-    The simplest processor, which doesn't do anything. It returns the original
-    values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
-    accept Loader contexts.
-
-    Example:
-
-    >>> from scrapy.loader.processors import Identity
-    >>> proc = Identity()
-    >>> proc(['one', 'two', 'three'])
-    ['one', 'two', 'three']
-
-.. class:: TakeFirst
-
-    Returns the first non-null/non-empty value from the values received,
-    so it's typically used as an output processor to single-valued fields.
-    It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
-
-    Example:
-
-    >>> from scrapy.loader.processors import TakeFirst
-    >>> proc = TakeFirst()
-    >>> proc(['', 'one', 'two', 'three'])
-    'one'
-
-.. class:: Join(separator=u' ')
-
-    Returns the values joined with the separator given in the ``__init__`` method, which
-    defaults to ``u' '``. It doesn't accept Loader contexts.
-
-    When using the default separator, this processor is equivalent to the
-    function: ``u' '.join``
-
-    Examples:
-
-    >>> from scrapy.loader.processors import Join
-    >>> proc = Join()
-    >>> proc(['one', 'two', 'three'])
-    'one two three'
-    >>> proc = Join('<br>')
-    >>> proc(['one', 'two', 'three'])
-    'one<br>two<br>three'
-
-.. class:: Compose(*functions, **default_loader_context)
-
-    A processor which is constructed from the composition of the given
-    functions. This means that each input value of this processor is passed to
-    the first function, and the result of that function is passed to the second
-    function, and so on, until the last function returns the output value of
-    this processor.
-
-    By default, stop process on ``None`` value. This behaviour can be changed by
-    passing keyword argument ``stop_on_none=False``.
-
-    Example:
-
-    >>> from scrapy.loader.processors import Compose
-    >>> proc = Compose(lambda v: v[0], str.upper)
-    >>> proc(['hello', 'world'])
-    'HELLO'
-
-    Each function can optionally receive a ``loader_context`` parameter. For
-    those which do, this processor will pass the currently active :ref:`Loader
-    context <topics-loaders-context>` through that parameter.
-
-    The keyword arguments passed in the ``__init__`` method are used as the default
-    Loader context values passed to each function call. However, the final
-    Loader context values passed to functions are overridden with the currently
-    active Loader context accessible through the :meth:`ItemLoader.context`
-    attribute.
-
-.. class:: MapCompose(*functions, **default_loader_context)
-
-    A processor which is constructed from the composition of the given
-    functions, similar to the :class:`Compose` processor. The difference with
-    this processor is the way internal results are passed among functions,
-    which is as follows:
-
-    The input value of this processor is *iterated* and the first function is
-    applied to each element. The results of these function calls (one for each element)
-    are concatenated to construct a new iterable, which is then used to apply the
-    second function, and so on, until the last function is applied to each
-    value of the list of values collected so far. The output values of the last
-    function are concatenated together to produce the output of this processor.
-
-    Each particular function can return a value or a list of values, which is
-    flattened with the list of values returned by the same function applied to
-    the other input values. The functions can also return ``None`` in which
-    case the output of that function is ignored for further processing over the
-    chain.
-
-    This processor provides a convenient way to compose functions that only
-    work with single values (instead of iterables). For this reason the
-    :class:`MapCompose` processor is typically used as input processor, since
-    data is often extracted using the
-    :meth:`~scrapy.selector.Selector.extract` method of :ref:`selectors
-    <topics-selectors>`, which returns a list of unicode strings.
-
-    The example below should clarify how it works:
-
-    >>> def filter_world(x):
-    ...     return None if x == 'world' else x
-    ...
-    >>> from scrapy.loader.processors import MapCompose
-    >>> proc = MapCompose(filter_world, str.upper)
-    >>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
-    ['HELLO, 'THIS', 'IS', 'SCRAPY']
-
-    As with the Compose processor, functions can receive Loader contexts, and
-    ``__init__`` method keyword arguments are used as default context values. See
-    :class:`Compose` processor for more info.
-
-.. class:: SelectJmes(json_path)
-
-    Queries the value using the json path provided to the ``__init__`` method and returns the output.
-    Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
-    This processor takes only one input at a time.
-
-    Example:
-
-    >>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
-    >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
-    >>> proc({'foo': 'bar'})
-    'bar'
-    >>> proc({'foo': {'bar': 'baz'}})
-    {'bar': 'baz'}
-
-    Working with Json:
-
-    >>> import json
-    >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
-    >>> proc_single_json_str('{"foo": "bar"}')
-    'bar'
-    >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
-    >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
-    ['bar']
+.. _itemloaders: https://itemloaders.readthedocs.io/en/latest/
+.. _processors: https://itemloaders.readthedocs.io/en/latest/built-in-processors.html

docs/topics/media-pipeline.rst

@@ -207,7 +207,6 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
 Google Cloud Storage
 ---------------------
 
-.. setting:: GCS_PROJECT_ID
 .. setting:: FILES_STORE_GCS_ACL
 .. setting:: IMAGES_STORE_GCS_ACL
 

docs/topics/settings.rst

@@ -469,7 +469,7 @@ necessary to access certain HTTPS websites: for example, you may need to use
 ``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a
 specific cipher that is not included in ``DEFAULT`` if a website requires it.
 
-.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/ciphers.html#CIPHER-LIST-FORMAT
+.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT
 
 .. setting:: DOWNLOADER_CLIENT_TLS_METHOD
 
@@ -786,6 +786,14 @@ The Feed Temp dir allows you to set a custom folder to save crawler
 temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
 :ref:`Amazon S3 <topics-feed-storage-s3>`.
 
+.. setting:: FEED_STORAGE_GCS_ACL
+
+FEED_STORAGE_GCS_ACL
+--------------------
+
+The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage <topics-feed-storage-gcs>`.
+For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation <https://cloud.google.com/storage/docs/access-control/lists>`_.
+
 .. setting:: FTP_PASSIVE_MODE
 
 FTP_PASSIVE_MODE
@@ -825,6 +833,15 @@ Default: ``"anonymous"``
 The username to use for FTP connections when there is no ``"ftp_user"``
 in ``Request`` meta.
 
+.. setting:: GCS_PROJECT_ID
+
+GCS_PROJECT_ID
+-----------------
+
+Default: ``None``
+
+The Project ID that will be used when storing data on `Google Cloud Storage`_.
+
 .. setting:: ITEM_PIPELINES
 
 ITEM_PIPELINES
@@ -1544,3 +1561,4 @@ case to see how to enable and use them.
 .. _Amazon web services: https://aws.amazon.com/
 .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search
 .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search
+.. _Google Cloud Storage: https://cloud.google.com/storage/

docs/topics/spiders.rst

@@ -360,9 +360,10 @@ CrawlSpider
 
    This spider also exposes an overrideable method:
 
-   .. method:: parse_start_url(response)
+   .. method:: parse_start_url(response, **kwargs)
 
-      This method is called for the start_urls responses. It allows to parse
+      This method is called for each response produced for the URLs in
+      the spider's ``start_urls`` attribute. It allows to parse
       the initial responses and must return either an
       :ref:`item object <topics-items>`, a :class:`~scrapy.http.Request`
       object, or an iterable containing any of them.
@@ -388,11 +389,6 @@ Crawling rules
    object will contain the text of the link that produced the :class:`~scrapy.http.Request`
    in its ``meta`` dictionary (under the ``link_text`` key)
 
-   .. warning:: When writing crawl spider rules, avoid using ``parse`` as
-       callback, since the :class:`CrawlSpider` uses the ``parse`` method
-       itself to implement its logic. So if you override the ``parse`` method,
-       the crawl spider will no longer work.
-
    ``cb_kwargs`` is a dict containing the keyword arguments to be passed to the
    callback function.
 
@@ -418,6 +414,11 @@ Crawling rules
    It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
    instance as first parameter.
 
+
+.. warning:: Because of its internal implementation, you must explicitly set
+   callbacks for new requests when writing :class:`CrawlSpider`-based spiders;
+   unexpected behaviour can occur otherwise.
+
    .. versionadded:: 2.0
       The *errback* parameter.
 
@@ -451,6 +452,11 @@ Let's now take a look at an example CrawlSpider with rules::
             item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
             item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
             item['link_text'] = response.meta['link_text']
+            url = response.xpath('//td[@id="additional_data"]/@href').get()
+            return response.follow(url, self.parse_additional_page, cb_kwargs=dict(item=item))
+
+        def parse_additional_page(self, response, item):
+            item['additional_data'] = response.xpath('//p[@id="additional_data"]/text()').get()
             return item
 
 
@@ -544,6 +550,11 @@ XMLFeedSpider
         those results. It must return a list of results (items or requests).
 
 
+.. warning:: Because of its internal implementation, you must explicitly set
+   callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders;
+   unexpected behaviour can occur otherwise.
+
+
 XMLFeedSpider example
 ~~~~~~~~~~~~~~~~~~~~~
 

scrapy/cmdline.py

@@ -19,10 +19,12 @@ def _iter_command_classes(module_name):
     # scrapy.utils.spider.iter_spider_classes
     for module in walk_modules(module_name):
         for obj in vars(module).values():
-            if inspect.isclass(obj) and \
-                    issubclass(obj, ScrapyCommand) and \
-                    obj.__module__ == module.__name__ and \
-                    not obj == ScrapyCommand:
+            if (
+                inspect.isclass(obj)
+                and issubclass(obj, ScrapyCommand)
+                and obj.__module__ == module.__name__
+                and not obj == ScrapyCommand
+            ):
                 yield obj
 
 

scrapy/commands/__init__.py

@@ -108,7 +108,7 @@ class ScrapyCommand:
 
 class BaseRunSpiderCommand(ScrapyCommand):
     """
-    Common class used to share functionality between the crawl and runspider commands
+    Common class used to share functionality between the crawl, parse and runspider commands
     """
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)

scrapy/commands/crawl.py

@@ -26,6 +26,8 @@ class Command(BaseRunSpiderCommand):
         else:
             self.crawler_process.start()
 
-            if self.crawler_process.bootstrap_failed or \
-                    (hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception):
+            if (
+                self.crawler_process.bootstrap_failed
+                or hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception
+            ):
                 self.exitcode = 1

scrapy/commands/fetch.py

@@ -19,8 +19,10 @@ class Command(ScrapyCommand):
         return "Fetch a URL using the Scrapy downloader"
 
     def long_desc(self):
-        return "Fetch a URL using the Scrapy downloader and print its content " \
-            "to stdout. You may want to use --nolog to disable logging"
+        return (
+            "Fetch a URL using the Scrapy downloader and print its content"
+            " to stdout. You may want to use --nolog to disable logging"
+        )
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)

scrapy/commands/genspider.py

@@ -121,6 +121,7 @@ class Command(ScrapyCommand):
 
     @property
     def templates_dir(self):
-        _templates_base_dir = self.settings['TEMPLATES_DIR'] or \
-            join(scrapy.__path__[0], 'templates')
-        return join(_templates_base_dir, 'spiders')
+        return join(
+            self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
+            'spiders'
+        )

scrapy/commands/parse.py

@@ -4,18 +4,16 @@ import logging
 from itemadapter import is_item, ItemAdapter
 from w3lib.url import is_url
 
-from scrapy.commands import ScrapyCommand
+from scrapy.commands import BaseRunSpiderCommand
 from scrapy.http import Request
 from scrapy.utils import display
-from scrapy.utils.conf import arglist_to_dict
 from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
 from scrapy.exceptions import UsageError
 
 logger = logging.getLogger(__name__)
 
 
-class Command(ScrapyCommand):
-
+class Command(BaseRunSpiderCommand):
     requires_project = True
 
     spider = None
@@ -31,11 +29,9 @@ class Command(ScrapyCommand):
         return "Parse URL (using its spider) and print the results"
 
     def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+        BaseRunSpiderCommand.add_options(self, parser)
         parser.add_option("--spider", dest="spider", default=None,
                           help="use this spider without looking for one")
-        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
-                          help="set spider argument (may be repeated)")
         parser.add_option("--pipelines", action="store_true",
                           help="process items through pipelines")
         parser.add_option("--nolinks", dest="nolinks", action="store_true",
@@ -200,12 +196,15 @@ class Command(ScrapyCommand):
             self.add_items(depth, items)
             self.add_requests(depth, requests)
 
+            scraped_data = items if opts.output else []
             if depth < opts.depth:
                 for req in requests:
                     req.meta['_depth'] = depth + 1
                     req.meta['_callback'] = req.callback
                     req.callback = callback
-                return requests
+                scraped_data += requests
+
+            return scraped_data
 
         # update request meta if any extra meta was passed through the --meta/-m opts.
         if opts.meta:
@@ -221,18 +220,11 @@ class Command(ScrapyCommand):
         return request
 
     def process_options(self, args, opts):
-        ScrapyCommand.process_options(self, args, opts)
+        BaseRunSpiderCommand.process_options(self, args, opts)
 
-        self.process_spider_arguments(opts)
         self.process_request_meta(opts)
         self.process_request_cb_kwargs(opts)
 
-    def process_spider_arguments(self, opts):
-        try:
-            opts.spargs = arglist_to_dict(opts.spargs)
-        except ValueError:
-            raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
-
     def process_request_meta(self, opts):
         if opts.meta:
             try:

scrapy/commands/startproject.py

@@ -1,10 +1,10 @@
 import re
 import os
-import stat
 import string
 from importlib import import_module
 from os.path import join, exists, abspath
 from shutil import ignore_patterns, move, copy2, copystat
+from stat import S_IWUSR as OWNER_WRITE_PERMISSION
 
 import scrapy
 from scrapy.commands import ScrapyCommand
@@ -20,7 +20,12 @@ TEMPLATES_TO_RENDER = (
     ('${project_name}', 'middlewares.py.tmpl'),
 )
 
-IGNORE = ignore_patterns('*.pyc', '.svn')
+IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn')
+
+
+def _make_writable(path):
+    current_permissions = os.stat(path).st_mode
+    os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION)
 
 
 class Command(ScrapyCommand):
@@ -78,30 +83,10 @@ class Command(ScrapyCommand):
                 self._copytree(srcname, dstname)
             else:
                 copy2(srcname, dstname)
+                _make_writable(dstname)
+
         copystat(src, dst)
-        self._set_rw_permissions(dst)
-
-    def _set_rw_permissions(self, path):
-        """
-        Sets permissions of a directory tree to +rw and +rwx for folders.
-        This is necessary if the start template files come without write
-        permissions.
-        """
-        mode_rw = (stat.S_IRUSR
-                   | stat.S_IWUSR
-                   | stat.S_IRGRP
-                   | stat.S_IROTH)
-
-        mode_x = (stat.S_IXUSR
-                  | stat.S_IXGRP
-                  | stat.S_IXOTH)
-
-        os.chmod(path, mode_rw | mode_x)
-        for root, dirs, files in os.walk(path):
-            for dir in dirs:
-                os.chmod(join(root, dir), mode_rw | mode_x)
-            for file in files:
-                os.chmod(join(root, file), mode_rw)
+        _make_writable(dst)
 
     def run(self, args, opts):
         if len(args) not in (1, 2):
@@ -137,6 +122,7 @@ class Command(ScrapyCommand):
 
     @property
     def templates_dir(self):
-        _templates_base_dir = self.settings['TEMPLATES_DIR'] or \
-            join(scrapy.__path__[0], 'templates')
-        return join(_templates_base_dir, 'project')
+        return join(
+            self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
+            'project'
+        )

scrapy/commands/view.py

@@ -8,8 +8,7 @@ class Command(fetch.Command):
         return "Open URL in browser, as seen by Scrapy"
 
     def long_desc(self):
-        return "Fetch a URL using the Scrapy downloader and show its " \
-            "contents in a browser"
+        return "Fetch a URL using the Scrapy downloader and show its contents in a browser"
 
     def add_options(self, parser):
         super(Command, self).add_options(parser)

scrapy/core/engine.py

@@ -159,10 +159,12 @@ class ExecutionEngine:
 
     def _needs_backout(self, spider):
         slot = self.slot
-        return not self.running \
-            or slot.closing \
-            or self.downloader.needs_backout() \
+        return (
+            not self.running
+            or slot.closing
+            or self.downloader.needs_backout()
             or self.scraper.slot.needs_backout()
+        )
 
     def _next_request_from_scheduler(self, spider):
         slot = self.slot

scrapy/core/scraper.py

@@ -148,7 +148,7 @@ class Scraper:
     def call_spider(self, result, request, spider):
         result.request = request
         dfd = defer_result(result)
-        callback = request.callback or spider.parse
+        callback = request.callback or spider._parse
         warn_on_generator_with_return_value(spider, callback)
         warn_on_generator_with_return_value(spider, request.errback)
         dfd.addCallbacks(callback=callback,

scrapy/downloadermiddlewares/redirect.py

@@ -33,10 +33,8 @@ class BaseRedirectMiddleware:
         if ttl and redirects <= self.max_redirect_times:
             redirected.meta['redirect_times'] = redirects
             redirected.meta['redirect_ttl'] = ttl - 1
-            redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \
-                [request.url]
-            redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \
-                [reason]
+            redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + [request.url]
+            redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + [reason]
             redirected.dont_filter = request.dont_filter
             redirected.priority = request.priority + self.priority_adjust
             logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s",
@@ -99,8 +97,11 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
         self._maxdelay = settings.getint('METAREFRESH_MAXDELAY')
 
     def process_response(self, request, response, spider):
-        if request.meta.get('dont_redirect', False) or request.method == 'HEAD' or \
-                not isinstance(response, HtmlResponse):
+        if (
+            request.meta.get('dont_redirect', False)
+            or request.method == 'HEAD'
+            or not isinstance(response, HtmlResponse)
+        ):
             return response
 
         interval, url = get_meta_refresh(response,

scrapy/downloadermiddlewares/retry.py

@@ -60,8 +60,10 @@ class RetryMiddleware:
         return response
 
     def process_exception(self, request, exception, spider):
-        if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \
-                and not request.meta.get('dont_retry', False):
+        if (
+            isinstance(exception, self.EXCEPTIONS_TO_RETRY)
+            and not request.meta.get('dont_retry', False)
+        ):
             return self._retry(request, exception, spider)
 
     def _retry(self, request, reason, spider):

scrapy/extensions/feedexport.py

@@ -153,6 +153,32 @@ class S3FeedStorage(BlockingFeedStorage):
             key.close()
 
 
+class GCSFeedStorage(BlockingFeedStorage):
+
+    def __init__(self, uri, project_id, acl):
+        self.project_id = project_id
+        self.acl = acl
+        u = urlparse(uri)
+        self.bucket_name = u.hostname
+        self.blob_name = u.path[1:]  # remove first "/"
+
+    @classmethod
+    def from_crawler(cls, crawler, uri):
+        return cls(
+            uri,
+            crawler.settings['GCS_PROJECT_ID'],
+            crawler.settings['FEED_STORAGE_GCS_ACL'] or None
+        )
+
+    def _store_in_thread(self, file):
+        file.seek(0)
+        from google.cloud.storage import Client
+        client = Client(project=self.project_id)
+        bucket = client.get_bucket(self.bucket_name)
+        blob = bucket.blob(self.blob_name)
+        blob.upload_from_file(file, predefined_acl=self.acl)
+
+
 class FTPFeedStorage(BlockingFeedStorage):
 
     def __init__(self, uri, use_active_mode=False):

scrapy/extensions/memusage.py

@@ -81,8 +81,10 @@ class MemoryUsage:
             logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
                          {'memusage': mem}, extra={'crawler': self.crawler})
             if self.notify_mails:
-                subj = "%s terminated: memory usage exceeded %dM at %s" % \
-                        (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
+                subj = (
+                    "%s terminated: memory usage exceeded %dM at %s"
+                    % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
+                )
                 self._send_report(self.notify_mails, subj)
                 self.crawler.stats.set_value('memusage/limit_notified', 1)
 
@@ -102,8 +104,10 @@ class MemoryUsage:
             logger.warning("Memory usage reached %(memusage)dM",
                            {'memusage': mem}, extra={'crawler': self.crawler})
             if self.notify_mails:
-                subj = "%s warning: memory usage reached %dM at %s" % \
-                        (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
+                subj = (
+                    "%s warning: memory usage reached %dM at %s"
+                    % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
+                )
                 self._send_report(self.notify_mails, subj)
                 self.crawler.stats.set_value('memusage/warning_notified', 1)
             self.warned = True

scrapy/http/cookies.py

@@ -186,9 +186,6 @@ class WrappedResponse:
     def info(self):
         return self
 
-    # python3 cookiejars calls get_all
     def get_all(self, name, default=None):
         return [to_unicode(v, errors='replace')
                 for v in self.response.headers.getlist(name)]
-    # python2 cookiejars calls getheaders
-    getheaders = get_all

scrapy/http/request/form.py

@@ -205,8 +205,7 @@ def _get_clickable(clickdata, form):
 
     # We didn't find it, so now we build an XPath expression out of the other
     # arguments, because they can be used as such
-    xpath = u'.//*' + \
-            u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
+    xpath = u'.//*' + u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
     el = form.xpath(xpath)
     if len(el) == 1:
         return (el[0].get('name'), el[0].get('value') or '')

scrapy/http/response/text.py

@@ -62,8 +62,11 @@ class TextResponse(Response):
         return self._declared_encoding() or self._body_inferred_encoding()
 
     def _declared_encoding(self):
-        return self._encoding or self._headers_encoding() \
+        return (
+            self._encoding
+            or self._headers_encoding()
             or self._body_declared_encoding()
+        )
 
     def body_as_unicode(self):
         """Return body as unicode"""

scrapy/link.py

@@ -21,12 +21,18 @@ class Link:
         self.nofollow = nofollow
 
     def __eq__(self, other):
-        return self.url == other.url and self.text == other.text and \
-            self.fragment == other.fragment and self.nofollow == other.nofollow
+        return (
+            self.url == other.url
+            and self.text == other.text
+            and self.fragment == other.fragment
+            and self.nofollow == other.nofollow
+        )
 
     def __hash__(self):
         return hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow)
 
     def __repr__(self):
-        return 'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' % \
-            (self.url, self.text, self.fragment, self.nofollow)
+        return (
+            'Link(url=%r, text=%r, fragment=%r, nofollow=%r)'
+            % (self.url, self.text, self.fragment, self.nofollow)
+        )

scrapy/linkextractors/htmlparser.py

@@ -1,91 +0,0 @@
-"""
-HTMLParser-based link extractor
-"""
-import warnings
-from html.parser import HTMLParser
-from urllib.parse import urljoin
-
-from w3lib.url import safe_url_string
-from w3lib.html import strip_html5_whitespace
-
-from scrapy.link import Link
-from scrapy.utils.python import unique as unique_list
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-
-class HtmlParserLinkExtractor(HTMLParser):
-
-    def __init__(self, tag="a", attr="href", process=None, unique=False,
-                 strip=True):
-        HTMLParser.__init__(self)
-
-        warnings.warn(
-            "HtmlParserLinkExtractor is deprecated and will be removed in "
-            "future releases. Please use scrapy.linkextractors.LinkExtractor",
-            ScrapyDeprecationWarning, stacklevel=2,
-        )
-
-        self.scan_tag = tag if callable(tag) else lambda t: t == tag
-        self.scan_attr = attr if callable(attr) else lambda a: a == attr
-        self.process_attr = process if callable(process) else lambda v: v
-        self.unique = unique
-        self.strip = strip
-
-    def _extract_links(self, response_text, response_url, response_encoding):
-        self.reset()
-        self.feed(response_text)
-        self.close()
-
-        links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links
-
-        ret = []
-        base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
-        for link in links:
-            if isinstance(link.url, str):
-                link.url = link.url.encode(response_encoding)
-            try:
-                link.url = urljoin(base_url, link.url)
-            except ValueError:
-                continue
-            link.url = safe_url_string(link.url, response_encoding)
-            link.text = link.text.decode(response_encoding)
-            ret.append(link)
-
-        return ret
-
-    def extract_links(self, response):
-        # wrapper needed to allow to work directly with text
-        return self._extract_links(response.body, response.url, response.encoding)
-
-    def reset(self):
-        HTMLParser.reset(self)
-
-        self.base_url = None
-        self.current_link = None
-        self.links = []
-
-    def handle_starttag(self, tag, attrs):
-        if tag == 'base':
-            self.base_url = dict(attrs).get('href')
-        if self.scan_tag(tag):
-            for attr, value in attrs:
-                if self.scan_attr(attr):
-                    if self.strip:
-                        value = strip_html5_whitespace(value)
-                    url = self.process_attr(value)
-                    link = Link(url=url)
-                    self.links.append(link)
-                    self.current_link = link
-
-    def handle_endtag(self, tag):
-        if self.scan_tag(tag):
-            self.current_link = None
-
-    def handle_data(self, data):
-        if self.current_link:
-            self.current_link.text = self.current_link.text + data
-
-    def matches(self, url):
-        """This extractor matches with any url, since
-        it doesn't contain any patterns"""
-        return True

scrapy/linkextractors/regex.py

@@ -1,41 +0,0 @@
-import re
-from urllib.parse import urljoin
-
-from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
-
-from scrapy.link import Link
-from scrapy.linkextractors.sgml import SgmlLinkExtractor
-
-
-linkre = re.compile(
-        "<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
-        re.DOTALL | re.IGNORECASE)
-
-
-def clean_link(link_text):
-    """Remove leading and trailing whitespace and punctuation"""
-    return link_text.strip("\t\r\n '\"\x0c")
-
-
-class RegexLinkExtractor(SgmlLinkExtractor):
-    """High performant link extractor"""
-
-    def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
-        def clean_text(text):
-            return replace_escape_chars(remove_tags(text.decode(response_encoding))).strip()
-
-        def clean_url(url):
-            clean_url = ''
-            try:
-                clean_url = urljoin(base_url, replace_entities(clean_link(url.decode(response_encoding))))
-            except ValueError:
-                pass
-            return clean_url
-
-        if base_url is None:
-            base_url = get_base_url(response_text, response_url, response_encoding)
-
-        links_text = linkre.findall(response_text)
-        return [Link(clean_url(url).encode(response_encoding),
-                     clean_text(text))
-                for url, _, text in links_text]

scrapy/linkextractors/sgml.py

@@ -1,151 +0,0 @@
-"""
-SGMLParser-based Link extractors
-"""
-import warnings
-from urllib.parse import urljoin
-from sgmllib import SGMLParser
-
-from w3lib.url import safe_url_string, canonicalize_url
-from w3lib.html import strip_html5_whitespace
-
-from scrapy.link import Link
-from scrapy.linkextractors import FilteringLinkExtractor
-from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
-from scrapy.utils.python import unique as unique_list, to_unicode
-from scrapy.utils.response import get_base_url
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-
-class BaseSgmlLinkExtractor(SGMLParser):
-
-    def __init__(self, tag="a", attr="href", unique=False, process_value=None,
-                 strip=True, canonicalized=False):
-        warnings.warn(
-            "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
-            "Please use scrapy.linkextractors.LinkExtractor",
-            ScrapyDeprecationWarning, stacklevel=2,
-        )
-        SGMLParser.__init__(self)
-        self.scan_tag = tag if callable(tag) else lambda t: t == tag
-        self.scan_attr = attr if callable(attr) else lambda a: a == attr
-        self.process_value = (lambda v: v) if process_value is None else process_value
-        self.current_link = None
-        self.unique = unique
-        self.strip = strip
-        if canonicalized:
-            self.link_key = lambda link: link.url
-        else:
-            self.link_key = lambda link: canonicalize_url(link.url,
-                                                          keep_fragments=True)
-
-    def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
-        """ Do the real extraction work """
-        self.reset()
-        self.feed(response_text)
-        self.close()
-
-        ret = []
-        if base_url is None:
-            base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
-        for link in self.links:
-            if isinstance(link.url, str):
-                link.url = link.url.encode(response_encoding)
-            try:
-                link.url = urljoin(base_url, link.url)
-            except ValueError:
-                continue
-            link.url = safe_url_string(link.url, response_encoding)
-            link.text = to_unicode(link.text, response_encoding, errors='replace').strip()
-            ret.append(link)
-
-        return ret
-
-    def _process_links(self, links):
-        """ Normalize and filter extracted links
-
-        The subclass should override it if necessary
-        """
-        return unique_list(links, key=self.link_key) if self.unique else links
-
-    def extract_links(self, response):
-        # wrapper needed to allow to work directly with text
-        links = self._extract_links(response.body, response.url, response.encoding)
-        links = self._process_links(links)
-        return links
-
-    def reset(self):
-        SGMLParser.reset(self)
-        self.links = []
-        self.base_url = None
-        self.current_link = None
-
-    def unknown_starttag(self, tag, attrs):
-        if tag == 'base':
-            self.base_url = dict(attrs).get('href')
-        if self.scan_tag(tag):
-            for attr, value in attrs:
-                if self.scan_attr(attr):
-                    if self.strip and value is not None:
-                        value = strip_html5_whitespace(value)
-                    url = self.process_value(value)
-                    if url is not None:
-                        link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel')))
-                        self.links.append(link)
-                        self.current_link = link
-
-    def unknown_endtag(self, tag):
-        if self.scan_tag(tag):
-            self.current_link = None
-
-    def handle_data(self, data):
-        if self.current_link:
-            self.current_link.text = self.current_link.text + data
-
-    def matches(self, url):
-        """This extractor matches with any url, since
-        it doesn't contain any patterns"""
-        return True
-
-
-class SgmlLinkExtractor(FilteringLinkExtractor):
-
-    def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
-                 tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True,
-                 process_value=None, deny_extensions=None, restrict_css=(),
-                 strip=True, restrict_text=()):
-        warnings.warn(
-            "SgmlLinkExtractor is deprecated and will be removed in future releases. "
-            "Please use scrapy.linkextractors.LinkExtractor",
-            ScrapyDeprecationWarning, stacklevel=2,
-        )
-
-        tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
-        tag_func = lambda x: x in tags
-        attr_func = lambda x: x in attrs
-
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
-            lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
-                                       unique=unique, process_value=process_value, strip=strip,
-                                       canonicalized=canonicalize)
-
-        super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
-                                                allow_domains=allow_domains, deny_domains=deny_domains,
-                                                restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
-                                                canonicalize=canonicalize, deny_extensions=deny_extensions,
-                                                restrict_text=restrict_text)
-
-    def extract_links(self, response):
-        base_url = None
-        if self.restrict_xpaths:
-            base_url = get_base_url(response)
-            body = u''.join(f
-                            for x in self.restrict_xpaths
-                            for f in response.xpath(x).getall()
-                            ).encode(response.encoding, errors='xmlcharrefreplace')
-        else:
-            body = response.body
-
-        links = self._extract_links(body, response.url, response.encoding, base_url)
-        links = self._process_links(links)
-        return links

scrapy/loader/__init__.py

@@ -3,217 +3,86 @@ Item Loader
 
 See documentation in docs/topics/loaders.rst
 """
-from collections import defaultdict
-from contextlib import suppress
-
-from itemadapter import ItemAdapter
+import itemloaders
 
 from scrapy.item import Item
-from scrapy.loader.common import wrap_loader_context
-from scrapy.loader.processors import Identity
 from scrapy.selector import Selector
-from scrapy.utils.misc import arg_to_iter, extract_regex
-from scrapy.utils.python import flatten
 
 
-def unbound_method(method):
+class ItemLoader(itemloaders.ItemLoader):
     """
-    Allow to use single-argument functions as input or output processors
-    (no need to define an unused first 'self' argument)
+    A user-friendly abstraction to populate an :ref:`item <topics-items>` with data
+    by applying :ref:`field processors <topics-loaders-processors>` to scraped data.
+    When instantiated with a ``selector`` or a ``response`` it supports
+    data extraction from web pages using :ref:`selectors <topics-selectors>`.
+
+    :param item: The item instance to populate using subsequent calls to
+        :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
+        or :meth:`~ItemLoader.add_value`.
+    :type item: scrapy.item.Item
+
+    :param selector: The selector to extract data from, when using the
+        :meth:`add_xpath`, :meth:`add_css`, :meth:`replace_xpath`, or
+        :meth:`replace_css` method.
+    :type selector: :class:`~scrapy.selector.Selector` object
+
+    :param response: The response used to construct the selector using the
+        :attr:`default_selector_class`, unless the selector argument is given,
+        in which case this argument is ignored.
+    :type response: :class:`~scrapy.http.Response` object
+
+    If no item is given, one is instantiated automatically using the class in
+    :attr:`default_item_class`.
+
+    The item, selector, response and remaining keyword arguments are
+    assigned to the Loader context (accessible through the :attr:`context` attribute).
+
+    .. attribute:: item
+
+        The item object being parsed by this Item Loader.
+        This is mostly used as a property so, when attempting to override this
+        value, you may want to check out :attr:`default_item_class` first.
+
+    .. attribute:: context
+
+        The currently active :ref:`Context <loaders-context>` of this Item Loader.
+
+    .. attribute:: default_item_class
+
+        An :ref:`item <topics-items>` class (or factory), used to instantiate
+        items when not given in the ``__init__`` method.
+
+    .. attribute:: default_input_processor
+
+        The default input processor to use for those fields which don't specify
+        one.
+
+    .. attribute:: default_output_processor
+
+        The default output processor to use for those fields which don't specify
+        one.
+
+    .. attribute:: default_selector_class
+
+        The class used to construct the :attr:`selector` of this
+        :class:`ItemLoader`, if only a response is given in the ``__init__`` method.
+        If a selector is given in the ``__init__`` method this attribute is ignored.
+        This attribute is sometimes overridden in subclasses.
+
+    .. attribute:: selector
+
+        The :class:`~scrapy.selector.Selector` object to extract data from.
+        It's either the selector given in the ``__init__`` method or one created from
+        the response given in the ``__init__`` method using the
+        :attr:`default_selector_class`. This attribute is meant to be
+        read-only.
     """
-    with suppress(AttributeError):
-        if '.' not in method.__qualname__:
-            return method.__func__
-    return method
-
-
-class ItemLoader:
 
     default_item_class = Item
-    default_input_processor = Identity()
-    default_output_processor = Identity()
     default_selector_class = Selector
 
     def __init__(self, item=None, selector=None, response=None, parent=None, **context):
         if selector is None and response is not None:
             selector = self.default_selector_class(response)
-        self.selector = selector
-        context.update(selector=selector, response=response)
-        if item is None:
-            item = self.default_item_class()
-        self.context = context
-        self.parent = parent
-        self._local_item = context['item'] = item
-        self._local_values = defaultdict(list)
-        # values from initial item
-        for field_name, value in ItemAdapter(item).items():
-            self._values[field_name] += arg_to_iter(value)
-
-    @property
-    def _values(self):
-        if self.parent is not None:
-            return self.parent._values
-        else:
-            return self._local_values
-
-    @property
-    def item(self):
-        if self.parent is not None:
-            return self.parent.item
-        else:
-            return self._local_item
-
-    def nested_xpath(self, xpath, **context):
-        selector = self.selector.xpath(xpath)
-        context.update(selector=selector)
-        subloader = self.__class__(
-            item=self.item, parent=self, **context
-        )
-        return subloader
-
-    def nested_css(self, css, **context):
-        selector = self.selector.css(css)
-        context.update(selector=selector)
-        subloader = self.__class__(
-            item=self.item, parent=self, **context
-        )
-        return subloader
-
-    def add_value(self, field_name, value, *processors, **kw):
-        value = self.get_value(value, *processors, **kw)
-        if value is None:
-            return
-        if not field_name:
-            for k, v in value.items():
-                self._add_value(k, v)
-        else:
-            self._add_value(field_name, value)
-
-    def replace_value(self, field_name, value, *processors, **kw):
-        value = self.get_value(value, *processors, **kw)
-        if value is None:
-            return
-        if not field_name:
-            for k, v in value.items():
-                self._replace_value(k, v)
-        else:
-            self._replace_value(field_name, value)
-
-    def _add_value(self, field_name, value):
-        value = arg_to_iter(value)
-        processed_value = self._process_input_value(field_name, value)
-        if processed_value:
-            self._values[field_name] += arg_to_iter(processed_value)
-
-    def _replace_value(self, field_name, value):
-        self._values.pop(field_name, None)
-        self._add_value(field_name, value)
-
-    def get_value(self, value, *processors, **kw):
-        regex = kw.get('re', None)
-        if regex:
-            value = arg_to_iter(value)
-            value = flatten(extract_regex(regex, x) for x in value)
-
-        for proc in processors:
-            if value is None:
-                break
-            _proc = proc
-            proc = wrap_loader_context(proc, self.context)
-            try:
-                value = proc(value)
-            except Exception as e:
-                raise ValueError("Error with processor %s value=%r error='%s: %s'" %
-                                 (_proc.__class__.__name__, value,
-                                  type(e).__name__, str(e)))
-        return value
-
-    def load_item(self):
-        adapter = ItemAdapter(self.item)
-        for field_name in tuple(self._values):
-            value = self.get_output_value(field_name)
-            if value is not None:
-                adapter[field_name] = value
-        return adapter.item
-
-    def get_output_value(self, field_name):
-        proc = self.get_output_processor(field_name)
-        proc = wrap_loader_context(proc, self.context)
-        try:
-            return proc(self._values[field_name])
-        except Exception as e:
-            raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" %
-                             (field_name, self._values[field_name], type(e).__name__, str(e)))
-
-    def get_collected_values(self, field_name):
-        return self._values[field_name]
-
-    def get_input_processor(self, field_name):
-        proc = getattr(self, '%s_in' % field_name, None)
-        if not proc:
-            proc = self._get_item_field_attr(field_name, 'input_processor',
-                                             self.default_input_processor)
-        return unbound_method(proc)
-
-    def get_output_processor(self, field_name):
-        proc = getattr(self, '%s_out' % field_name, None)
-        if not proc:
-            proc = self._get_item_field_attr(field_name, 'output_processor',
-                                             self.default_output_processor)
-        return unbound_method(proc)
-
-    def _process_input_value(self, field_name, value):
-        proc = self.get_input_processor(field_name)
-        _proc = proc
-        proc = wrap_loader_context(proc, self.context)
-        try:
-            return proc(value)
-        except Exception as e:
-            raise ValueError(
-                "Error with input processor %s: field=%r value=%r "
-                "error='%s: %s'" % (_proc.__class__.__name__, field_name,
-                                    value, type(e).__name__, str(e)))
-
-    def _get_item_field_attr(self, field_name, key, default=None):
-        field_meta = ItemAdapter(self.item).get_field_meta(field_name)
-        return field_meta.get(key, default)
-
-    def _check_selector_method(self):
-        if self.selector is None:
-            raise RuntimeError("To use XPath or CSS selectors, "
-                               "%s must be instantiated with a selector "
-                               "or a response" % self.__class__.__name__)
-
-    def add_xpath(self, field_name, xpath, *processors, **kw):
-        values = self._get_xpathvalues(xpath, **kw)
-        self.add_value(field_name, values, *processors, **kw)
-
-    def replace_xpath(self, field_name, xpath, *processors, **kw):
-        values = self._get_xpathvalues(xpath, **kw)
-        self.replace_value(field_name, values, *processors, **kw)
-
-    def get_xpath(self, xpath, *processors, **kw):
-        values = self._get_xpathvalues(xpath, **kw)
-        return self.get_value(values, *processors, **kw)
-
-    def _get_xpathvalues(self, xpaths, **kw):
-        self._check_selector_method()
-        xpaths = arg_to_iter(xpaths)
-        return flatten(self.selector.xpath(xpath).getall() for xpath in xpaths)
-
-    def add_css(self, field_name, css, *processors, **kw):
-        values = self._get_cssvalues(css, **kw)
-        self.add_value(field_name, values, *processors, **kw)
-
-    def replace_css(self, field_name, css, *processors, **kw):
-        values = self._get_cssvalues(css, **kw)
-        self.replace_value(field_name, values, *processors, **kw)
-
-    def get_css(self, css, *processors, **kw):
-        values = self._get_cssvalues(css, **kw)
-        return self.get_value(values, *processors, **kw)
-
-    def _get_cssvalues(self, csss, **kw):
-        self._check_selector_method()
-        csss = arg_to_iter(csss)
-        return flatten(self.selector.css(css).getall() for css in csss)
+        context.update(response=response)
+        super().__init__(item=item, selector=selector, parent=parent, **context)

scrapy/loader/common.py

@@ -1,14 +1,21 @@
 """Common functions used in Item Loaders code"""
 
-from functools import partial
-from scrapy.utils.python import get_func_args
+import warnings
+
+from itemloaders import common
+
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 
 
 def wrap_loader_context(function, context):
     """Wrap functions that receive loader_context to contain the context
     "pre-loaded" and expose a interface that receives only one argument
     """
-    if 'loader_context' in get_func_args(function):
-        return partial(function, loader_context=context)
-    else:
-        return function
+    warnings.warn(
+        "scrapy.loader.common.wrap_loader_context has moved to a new library."
+        "Please update your reference to itemloaders.common.wrap_loader_context",
+        ScrapyDeprecationWarning,
+        stacklevel=2
+    )
+
+    return common.wrap_loader_context(function, context)

scrapy/loader/processors.py

@@ -3,102 +3,19 @@ This module provides some commonly used processors for Item Loaders.
 
 See documentation in docs/topics/loaders.rst
 """
-from collections import ChainMap
+from itemloaders import processors
 
-from scrapy.utils.misc import arg_to_iter
-from scrapy.loader.common import wrap_loader_context
+from scrapy.utils.deprecate import create_deprecated_class
 
 
-class MapCompose:
+MapCompose = create_deprecated_class('MapCompose', processors.MapCompose)
 
-    def __init__(self, *functions, **default_loader_context):
-        self.functions = functions
-        self.default_loader_context = default_loader_context
+Compose = create_deprecated_class('Compose', processors.Compose)
 
-    def __call__(self, value, loader_context=None):
-        values = arg_to_iter(value)
-        if loader_context:
-            context = ChainMap(loader_context, self.default_loader_context)
-        else:
-            context = self.default_loader_context
-        wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
-        for func in wrapped_funcs:
-            next_values = []
-            for v in values:
-                try:
-                    next_values += arg_to_iter(func(v))
-                except Exception as e:
-                    raise ValueError("Error in MapCompose with "
-                                     "%s value=%r error='%s: %s'" %
-                                     (str(func), value, type(e).__name__,
-                                      str(e)))
-            values = next_values
-        return values
+TakeFirst = create_deprecated_class('TakeFirst', processors.TakeFirst)
 
+Identity = create_deprecated_class('Identity', processors.Identity)
 
-class Compose:
+SelectJmes = create_deprecated_class('SelectJmes', processors.SelectJmes)
 
-    def __init__(self, *functions, **default_loader_context):
-        self.functions = functions
-        self.stop_on_none = default_loader_context.get('stop_on_none', True)
-        self.default_loader_context = default_loader_context
-
-    def __call__(self, value, loader_context=None):
-        if loader_context:
-            context = ChainMap(loader_context, self.default_loader_context)
-        else:
-            context = self.default_loader_context
-        wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
-        for func in wrapped_funcs:
-            if value is None and self.stop_on_none:
-                break
-            try:
-                value = func(value)
-            except Exception as e:
-                raise ValueError("Error in Compose with "
-                                 "%s value=%r error='%s: %s'" %
-                                 (str(func), value, type(e).__name__, str(e)))
-        return value
-
-
-class TakeFirst:
-
-    def __call__(self, values):
-        for value in values:
-            if value is not None and value != '':
-                return value
-
-
-class Identity:
-
-    def __call__(self, values):
-        return values
-
-
-class SelectJmes:
-    """
-        Query the input string for the jmespath (given at instantiation),
-        and return the answer
-        Requires : jmespath(https://github.com/jmespath/jmespath)
-        Note: SelectJmes accepts only one input element at a time.
-    """
-    def __init__(self, json_path):
-        self.json_path = json_path
-        import jmespath
-        self.compiled_path = jmespath.compile(self.json_path)
-
-    def __call__(self, value):
-        """Query value for the jmespath query and return answer
-        :param value: a data structure (dict, list) to extract from
-        :return: Element extracted according to jmespath query
-        """
-        return self.compiled_path.search(value)
-
-
-class Join:
-
-    def __init__(self, separator=u' '):
-        self.separator = separator
-
-    def __call__(self, values):
-        return self.separator.join(values)
+Join = create_deprecated_class('Join', processors.Join)

scrapy/settings/__init__.py

@@ -52,8 +52,7 @@ class SettingsAttribute:
             self.priority = priority
 
     def __str__(self):
-        return "<SettingsAttribute value={self.value!r} " \
-               "priority={self.priority}>".format(self=self)
+        return "<SettingsAttribute value={self.value!r} priority={self.priority}>".format(self=self)
 
     __repr__ = __str__
 

scrapy/settings/default_settings.py

@@ -142,9 +142,10 @@ FEED_STORAGES = {}
 FEED_STORAGES_BASE = {
     '': 'scrapy.extensions.feedexport.FileFeedStorage',
     'file': 'scrapy.extensions.feedexport.FileFeedStorage',
-    'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
-    's3': 'scrapy.extensions.feedexport.S3FeedStorage',
     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
+    'gs': 'scrapy.extensions.feedexport.GCSFeedStorage',
+    's3': 'scrapy.extensions.feedexport.S3FeedStorage',
+    'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
 }
 FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
@@ -159,6 +160,7 @@ FEED_EXPORTERS_BASE = {
 FEED_EXPORT_INDENT = 0
 
 FEED_STORAGE_FTP_ACTIVE = False
+FEED_STORAGE_GCS_ACL = ''
 FEED_STORAGE_S3_ACL = ''
 
 FILES_STORE_S3_ACL = 'private'
@@ -168,6 +170,8 @@ FTP_USER = 'anonymous'
 FTP_PASSWORD = 'guest'
 FTP_PASSIVE_MODE = True
 
+GCS_PROJECT_ID = None
+
 HTTPCACHE_ENABLED = False
 HTTPCACHE_DIR = 'httpcache'
 HTTPCACHE_IGNORE_MISSING = False

scrapy/spiders/__init__.py

@@ -96,7 +96,10 @@ class Spider(object_ref):
         )
         return Request(url, dont_filter=True)
 
-    def parse(self, response):
+    def _parse(self, response, **kwargs):
+        return self.parse(response, **kwargs)
+
+    def parse(self, response, **kwargs):
         raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__))
 
     @classmethod

scrapy/spiders/crawl.py

@@ -78,10 +78,15 @@ class CrawlSpider(Spider):
         super(CrawlSpider, self).__init__(*a, **kw)
         self._compile_rules()
 
-    def parse(self, response):
-        return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True)
+    def _parse(self, response, **kwargs):
+        return self._parse_response(
+            response=response,
+            callback=self.parse_start_url,
+            cb_kwargs=kwargs,
+            follow=True,
+        )
 
-    def parse_start_url(self, response):
+    def parse_start_url(self, response, **kwargs):
         return []
 
     def process_results(self, response, results):

scrapy/spiders/feed.py

@@ -61,7 +61,7 @@ class XMLFeedSpider(Spider):
             for result_item in self.process_results(response, ret):
                 yield result_item
 
-    def parse(self, response):
+    def _parse(self, response, **kwargs):
         if not hasattr(self, 'parse_node'):
             raise NotConfigured('You must define parse_node method in order to scrape this XML feed')
 
@@ -128,7 +128,7 @@ class CSVFeedSpider(Spider):
             for result_item in self.process_results(response, ret):
                 yield result_item
 
-    def parse(self, response):
+    def _parse(self, response, **kwargs):
         if not hasattr(self, 'parse_row'):
             raise NotConfigured('You must define parse_row method in order to scrape this CSV feed')
         response = self.adapt_response(response)

scrapy/utils/benchserver.py

@@ -28,8 +28,7 @@ class Root(Resource):
 
 
 def _getarg(request, name, default=None, type=str):
-    return type(request.args[name][0]) \
-        if name in request.args else default
+    return type(request.args[name][0]) if name in request.args else default
 
 
 if __name__ == '__main__':

scrapy/utils/conf.py

@@ -101,11 +101,13 @@ def get_config(use_closest=True):
 
 
 def get_sources(use_closest=True):
-    xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \
-        os.path.expanduser('~/.config')
-    sources = ['/etc/scrapy.cfg', r'c:\scrapy\scrapy.cfg',
-               xdg_config_home + '/scrapy.cfg',
-               os.path.expanduser('~/.scrapy.cfg')]
+    xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.expanduser('~/.config')
+    sources = [
+        '/etc/scrapy.cfg',
+        r'c:\scrapy\scrapy.cfg',
+        xdg_config_home + '/scrapy.cfg',
+        os.path.expanduser('~/.scrapy.cfg'),
+    ]
     if use_closest:
         sources.append(closest_scrapy_cfg())
     return sources

scrapy/utils/curl.py

@@ -9,8 +9,7 @@ from w3lib.http import basic_auth_header
 
 class CurlParser(argparse.ArgumentParser):
     def error(self, message):
-        error_msg = \
-            'There was an error parsing the curl command: {}'.format(message)
+        error_msg = 'There was an error parsing the curl command: {}'.format(message)
         raise ValueError(error_msg)
 
 

scrapy/utils/display.py

@@ -2,20 +2,42 @@
 pprint and pformat wrappers with colorization support
 """
 
+import ctypes
+import platform
 import sys
+from distutils.version import LooseVersion as parse_version
 from pprint import pformat as pformat_
 
 
+def _enable_windows_terminal_processing():
+    # https://stackoverflow.com/a/36760881
+    kernel32 = ctypes.windll.kernel32
+    return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7))
+
+
+def _tty_supports_color():
+    if sys.platform != "win32":
+        return True
+
+    if parse_version(platform.version()) < parse_version("10.0.14393"):
+        return True
+
+    # Windows >= 10.0.14393 interprets ANSI escape sequences providing terminal
+    # processing is enabled.
+    return _enable_windows_terminal_processing()
+
+
 def _colorize(text, colorize=True):
-    if not colorize or not sys.stdout.isatty():
+    if not colorize or not sys.stdout.isatty() or not _tty_supports_color():
         return text
     try:
         from pygments import highlight
+    except ImportError:
+        return text
+    else:
         from pygments.formatters import TerminalFormatter
         from pygments.lexers import PythonLexer
         return highlight(text, PythonLexer(), TerminalFormatter())
-    except ImportError:
-        return text
 
 
 def pformat(obj, *args, **kwargs):

scrapy/utils/misc.py

@@ -15,6 +15,7 @@ from w3lib.html import replace_entities
 from scrapy.utils.datatypes import LocalWeakReferencedCache
 from scrapy.utils.python import flatten, to_unicode
 from scrapy.item import _BaseItem
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 
 
 _ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes
@@ -86,6 +87,11 @@ def extract_regex(regex, text, encoding='utf-8'):
     * if the regex contains multiple numbered groups, all those will be returned (flattened)
     * if the regex doesn't contain any group the entire regex matching is returned
     """
+    warnings.warn(
+        "scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.",
+        ScrapyDeprecationWarning,
+        stacklevel=2
+    )
 
     if isinstance(regex, str):
         regex = re.compile(regex, re.UNICODE)

scrapy/utils/ossignal.py

@@ -18,8 +18,7 @@ def install_shutdown_handlers(function, override_sigint=True):
     from twisted.internet import reactor
     reactor._handleSignals()
     signal.signal(signal.SIGTERM, function)
-    if signal.getsignal(signal.SIGINT) == signal.default_int_handler or \
-            override_sigint:
+    if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint:
         signal.signal(signal.SIGINT, function)
     # Catch Ctrl-Break in windows
     if hasattr(signal, 'SIGBREAK'):

scrapy/utils/python.py

@@ -285,6 +285,7 @@ class WeakKeyCache:
         return self._weakdict[key]
 
 
+@deprecated
 def retry_on_eintr(function, *args, **kw):
     """Run a function and retry it while getting EINTR errors"""
     while True:

scrapy/utils/response.py

@@ -47,13 +47,17 @@ def response_httprepr(response):
     is provided only for reference, since it's not the exact stream of bytes
     that was received (that's not exposed by Twisted).
     """
-    s = b"HTTP/1.1 " + to_bytes(str(response.status)) + b" " + \
-        to_bytes(http.RESPONSES.get(response.status, b'')) + b"\r\n"
+    values = [
+        b"HTTP/1.1 ",
+        to_bytes(str(response.status)),
+        b" ",
+        to_bytes(http.RESPONSES.get(response.status, b'')),
+        b"\r\n",
+    ]
     if response.headers:
-        s += response.headers.to_string() + b"\r\n"
-    s += b"\r\n"
-    s += response.body
-    return s
+        values.extend([response.headers.to_string(), b"\r\n"])
+    values.extend([b"\r\n", response.body])
+    return b"".join(values)
 
 
 def open_in_browser(response, _openfunc=webbrowser.open):

scrapy/utils/spider.py

@@ -34,10 +34,12 @@ def iter_spider_classes(module):
     from scrapy.spiders import Spider
 
     for obj in vars(module).values():
-        if inspect.isclass(obj) and \
-           issubclass(obj, Spider) and \
-           obj.__module__ == module.__name__ and \
-           getattr(obj, 'name', None):
+        if (
+            inspect.isclass(obj)
+            and issubclass(obj, Spider)
+            and obj.__module__ == module.__name__
+            and getattr(obj, 'name', None)
+        ):
             yield obj
 
 

scrapy/utils/test.py

@@ -2,10 +2,10 @@
 This module contains some assorted functions used in tests
 """
 
-from __future__ import absolute_import
-from posixpath import split
 import asyncio
 import os
+from posixpath import split
+from unittest import mock
 
 from importlib import import_module
 from twisted.trial.unittest import SkipTest
@@ -126,3 +126,19 @@ def get_from_asyncio_queue(value):
     getter = q.get()
     q.put_nowait(value)
     return getter
+
+
+def mock_google_cloud_storage():
+    """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
+    classes and set their proper return values.
+    """
+    from google.cloud.storage import Client, Bucket, Blob
+    client_mock = mock.create_autospec(Client)
+
+    bucket_mock = mock.create_autospec(Bucket)
+    client_mock.get_bucket.return_value = bucket_mock
+
+    blob_mock = mock.create_autospec(Blob)
+    bucket_mock.blob.return_value = blob_mock
+
+    return (client_mock, bucket_mock, blob_mock)

scrapy/utils/url.py

@@ -83,25 +83,54 @@ def add_http_if_no_scheme(url):
     return url
 
 
+def _is_posix_path(string):
+    return bool(
+        re.match(
+            r'''
+            ^                   # start with...
+            (
+                \.              # ...a single dot,
+                (
+                    \. | [^/\.]+  # optionally followed by
+                )?                # either a second dot or some characters
+                |
+                ~   # $HOME
+            )?      # optional match of ".", ".." or ".blabla"
+            /       # at least one "/" for a file path,
+            .       # and something after the "/"
+            ''',
+            string,
+            flags=re.VERBOSE,
+        )
+    )
+
+
+def _is_windows_path(string):
+    return bool(
+        re.match(
+            r'''
+            ^
+            (
+                [a-z]:\\
+                | \\\\
+            )
+            ''',
+            string,
+            flags=re.IGNORECASE | re.VERBOSE,
+        )
+    )
+
+
+def _is_filesystem_path(string):
+    return _is_posix_path(string) or _is_windows_path(string)
+
+
 def guess_scheme(url):
-    """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise."""
-    parts = urlparse(url)
-    if parts.scheme:
-        return url
-    # Note: this does not match Windows filepath
-    if re.match(r'''^                   # start with...
-                    (
-                        \.              # ...a single dot,
-                        (
-                            \. | [^/\.]+  # optionally followed by
-                        )?                # either a second dot or some characters
-                    )?      # optional match of ".", ".." or ".blabla"
-                    /       # at least one "/" for a file path,
-                    .       # and something after the "/"
-                    ''', parts.path, flags=re.VERBOSE):
+    """Add an URL scheme if missing: file:// for filepath-like input or
+    http:// otherwise."""
+    if _is_filesystem_path(url):
         return any_to_uri(url)
-    else:
-        return add_http_if_no_scheme(url)
+    return add_http_if_no_scheme(url)
 
 
 def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True):

setup.cfg

@@ -13,9 +13,6 @@ follow_imports = skip
 [mypy-scrapy]
 ignore_errors = True
 
-[mypy-scrapy._monkeypatches]
-ignore_errors = True
-
 [mypy-scrapy.commands]
 ignore_errors = True
 
@@ -118,6 +115,9 @@ ignore_errors = True
 [mypy-tests.test_loader]
 ignore_errors = True
 
+[mypy-tests.test_loader_deprecated]
+ignore_errors = True
+
 [mypy-tests.test_pipeline_crawl]
 ignore_errors = True
 

setup.py

@@ -18,12 +18,39 @@ def has_environment_marker_platform_impl_support():
     return parse_version(setuptools_version) >= parse_version('18.5')
 
 
+install_requires = [
+    'Twisted>=17.9.0',
+    'cryptography>=2.0',
+    'cssselect>=0.9.1',
+    'itemloaders>=1.0.1',
+    'lxml>=3.5.0',
+    'parsel>=1.5.0',
+    'PyDispatcher>=2.0.5',
+    'pyOpenSSL>=16.2.0',
+    'queuelib>=1.4.2',
+    'service_identity>=16.0.0',
+    'w3lib>=1.17.0',
+    'zope.interface>=4.1.3',
+    'protego>=0.1.15',
+    'itemadapter>=0.1.0',
+]
 extras_require = {}
 
 if has_environment_marker_platform_impl_support():
+    extras_require[':platform_python_implementation == "CPython"'] = [
+        'lxml>=3.5.0',
+    ]
     extras_require[':platform_python_implementation == "PyPy"'] = [
+        # Earlier lxml versions are affected by
+        # https://bitbucket.org/pypy/pypy/issues/2498/cython-on-pypy-3-dict-object-has-no,
+        # which was fixed in Cython 0.26, released on 2017-06-19, and used to
+        # generate the C headers of lxml release tarballs published since then, the
+        # first of which was:
+        'lxml>=4.0.0',
         'PyPyDispatcher>=2.1.0',
     ]
+else:
+    install_requires.append('lxml>=3.5.0')
 
 
 setup(
@@ -67,20 +94,6 @@ setup(
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
     python_requires='>=3.5.2',
-    install_requires=[
-        'Twisted>=17.9.0',
-        'cryptography>=2.0',
-        'cssselect>=0.9.1',
-        'lxml>=3.5.0',
-        'parsel>=1.5.0',
-        'PyDispatcher>=2.0.5',
-        'pyOpenSSL>=16.2.0',
-        'queuelib>=1.4.2',
-        'service_identity>=16.0.0',
-        'w3lib>=1.17.0',
-        'zope.interface>=4.1.3',
-        'protego>=0.1.15',
-        'itemadapter>=0.1.0',
-    ],
+    install_requires=install_requires,
     extras_require=extras_require,
 )

tests/CrawlerRunner/ip_address.py

@@ -1,7 +1,9 @@
 from urllib.parse import urlparse
 
 from twisted.internet import reactor
-from twisted.names.client import createResolver
+from twisted.names import cache, hosts as hostsModule, resolve
+from twisted.names.client import Resolver
+from twisted.python.runtime import platform
 
 from scrapy import Spider, Request
 from scrapy.crawler import CrawlerRunner
@@ -10,6 +12,16 @@ from scrapy.utils.log import configure_logging
 from tests.mockserver import MockServer, MockDNSServer
 
 
+# https://stackoverflow.com/a/32784190
+def createResolver(servers=None, resolvconf=None, hosts=None):
+    if hosts is None:
+        hosts = b'/etc/hosts' if platform.getType() == 'posix' else r'c:\windows\hosts'
+    theResolver = Resolver(resolvconf, servers)
+    hostResolver = hostsModule.Resolver(hosts)
+    chain = [hostResolver, cache.CacheResolver(), theResolver]
+    return resolve.ResolverChain(chain)
+
+
 class LocalhostSpider(Spider):
     name = "localhost_spider"
 

tests/ignores.txt

@@ -1,6 +1,3 @@
-scrapy/linkextractors/sgml.py
-scrapy/linkextractors/regex.py
-scrapy/linkextractors/htmlparser.py
 scrapy/downloadermiddlewares/cookies.py
 scrapy/extensions/statsmailer.py
 scrapy/extensions/memusage.py

tests/mockserver.py

@@ -218,9 +218,8 @@ class MockServer:
         self.proc.communicate()
 
     def url(self, path, is_secure=False):
-        host = self.http_address.replace('0.0.0.0', '127.0.0.1')
-        if is_secure:
-            host = self.https_address
+        host = self.https_address if is_secure else self.http_address
+        host = host.replace('0.0.0.0', '127.0.0.1')
         return host + path
 
 
@@ -248,9 +247,8 @@ class MockDNSServer:
     def __enter__(self):
         self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'],
                           stdout=PIPE, env=get_testenv())
-        host, port = self.proc.stdout.readline().strip().decode('ascii').split(":")
-        self.host = host
-        self.port = int(port)
+        self.host = '127.0.0.1'
+        self.port = int(self.proc.stdout.readline().strip().decode('ascii').split(":")[1])
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):

tests/requirements-py3.txt

@@ -1,15 +1,15 @@
 # Tests requirements
 attrs
 dataclasses; python_version == '3.6'
-jmespath
 mitmproxy; python_version >= '3.6'
 mitmproxy<4.0.0; python_version < '3.6'
 # https://github.com/pytest-dev/pytest-twisted/issues/93
 pytest != 5.4, != 5.4.1
+pytest-azurepipelines
 pytest-cov
 pytest-twisted >= 1.11
 pytest-xdist
-sybil
+sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
 testfixtures
 
 # optional for shell wrapper tests

tests/sample_data/link_extractor/linkextractor.html

@@ -1,7 +1,7 @@
 <html>
 <head>
 <base href='http://example.com' />
-<title>Sample page with links for testing RegexLinkExtractor</title>
+<title>Sample page with links for testing LinkExtractor</title>
 </head>
 <body>
 <div id='wrapper'>

tests/sample_data/link_extractor/linkextractor_latin1.html

@@ -2,7 +2,7 @@
   <head>
     <meta http-equiv="Content-Type" content="text/html; charset=latin-1">
     <base href='http://example.com' />
-    <title>Sample page with links for testing RegexLinkExtractor</title>
+    <title>Sample page with links for testing LinkExtractor</title>
   </head>
 <body>
   <div id='wrapper'>

tests/spiders.py

@@ -246,13 +246,40 @@ class DuplicateStartRequestsSpider(MockServerSpider):
         self.visited += 1
 
 
-class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider):
-    name = 'crawl_spider_with_errback'
+class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
+    """
+    A CrawlSpider which overrides the 'parse' method
+    """
+    name = 'crawl_spider_with_parse_method'
     custom_settings = {
         'RETRY_HTTP_CODES': [],  # no need to retry
     }
     rules = (
-        Rule(LinkExtractor(), callback='callback', errback='errback', follow=True),
+        Rule(LinkExtractor(), callback='parse', follow=True),
+    )
+
+    def start_requests(self):
+        test_body = b"""
+        <html>
+            <head><title>Page title<title></head>
+            <body>
+                <p><a href="/status?n=200">Item 200</a></p>  <!-- callback -->
+                <p><a href="/status?n=201">Item 201</a></p>  <!-- callback -->
+            </body>
+        </html>
+        """
+        url = self.mockserver.url("/alpayload")
+        yield Request(url, method="POST", body=test_body)
+
+    def parse(self, response, foo=None):
+        self.logger.info('[parse] status %i (foo: %s)', response.status, foo)
+        yield Request(self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"})
+
+
+class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
+    name = 'crawl_spider_with_errback'
+    rules = (
+        Rule(LinkExtractor(), callback='parse', errback='errback', follow=True),
     )
 
     def start_requests(self):
@@ -271,9 +298,6 @@ class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider):
         url = self.mockserver.url("/alpayload")
         yield Request(url, method="POST", body=test_body)
 
-    def callback(self, response):
-        self.logger.info('[callback] status %i', response.status)
-
     def errback(self, failure):
         self.logger.info('[errback] status %i', failure.value.response.status)
 

tests/test_command_parse.py

@@ -1,5 +1,5 @@
 import os
-from os.path import join, abspath
+from os.path import join, abspath, isfile, exists
 from twisted.internet import defer
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
@@ -218,3 +218,24 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         )
         self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))
+
+    @defer.inlineCallbacks
+    def test_output_flag(self):
+        """Checks if a file was created successfully having
+        correct format containing correct data in it.
+        """
+        file_name = 'data.json'
+        file_path = join(self.proj_path, file_name)
+        yield self.execute([
+            '--spider', self.spider_name,
+            '-c', 'parse',
+            '-o', file_name,
+            self.url('/html')
+        ])
+
+        self.assertTrue(exists(file_path))
+        self.assertTrue(isfile(file_path))
+
+        content = '[\n{},\n{"foo": "bar"}\n]'
+        with open(file_path, 'r') as f:
+            self.assertEqual(f.read(), content)

tests/test_command_shell.py

@@ -96,7 +96,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_local_file(self):
-        filepath = join(tests_datadir, 'test_site/index.html')
+        filepath = join(tests_datadir, 'test_site', 'index.html')
         _, out, _ = yield self.execute([filepath, '-c', 'item'])
         assert b'{}' in out
 

tests/test_commands.py

@@ -2,19 +2,25 @@ import inspect
 import json
 import optparse
 import os
+import platform
 import subprocess
 import sys
 import tempfile
 from contextlib import contextmanager
+from itertools import chain
 from os.path import exists, join, abspath
+from pathlib import Path
 from shutil import rmtree, copytree
+from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
 from tempfile import mkdtemp
 from threading import Timer
+from unittest import skipIf
 
 from twisted.trial import unittest
 
 import scrapy
 from scrapy.commands import ScrapyCommand
+from scrapy.commands.startproject import IGNORE
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_testenv
@@ -119,6 +125,29 @@ class StartprojectTest(ProjectTest):
         self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params'))
 
 
+def get_permissions_dict(path, renamings=None, ignore=None):
+    renamings = renamings or tuple()
+    permissions_dict = {
+        '.': os.stat(path).st_mode,
+    }
+    for root, dirs, files in os.walk(path):
+        nodes = list(chain(dirs, files))
+        if ignore:
+            ignored_names = ignore(root, nodes)
+            nodes = [node for node in nodes if node not in ignored_names]
+        for node in nodes:
+            absolute_path = os.path.join(root, node)
+            relative_path = os.path.relpath(absolute_path, path)
+            for search_string, replacement in renamings:
+                relative_path = relative_path.replace(
+                    search_string,
+                    replacement
+                )
+            permissions = os.stat(absolute_path).st_mode
+            permissions_dict[relative_path] = permissions
+    return permissions_dict
+
+
 class StartprojectTemplatesTest(ProjectTest):
 
     def setUp(self):
@@ -139,6 +168,149 @@ class StartprojectTemplatesTest(ProjectTest):
         self.assertIn(self.tmpl_proj, out)
         assert exists(join(self.proj_path, 'root_template'))
 
+    def test_startproject_permissions_from_writable(self):
+        """Check that generated files have the right permissions when the
+        template folder has the same permissions as in the project, i.e.
+        everything is writable."""
+        scrapy_path = scrapy.__path__[0]
+        project_template = os.path.join(scrapy_path, 'templates', 'project')
+        project_name = 'startproject1'
+        renamings = (
+            ('module', project_name),
+            ('.tmpl', ''),
+        )
+        expected_permissions = get_permissions_dict(
+            project_template,
+            renamings,
+            IGNORE,
+        )
+
+        destination = mkdtemp()
+        process = subprocess.Popen(
+            (
+                sys.executable,
+                '-m',
+                'scrapy.cmdline',
+                'startproject',
+                project_name,
+            ),
+            cwd=destination,
+            env=self.env,
+        )
+        process.wait()
+
+        project_dir = os.path.join(destination, project_name)
+        actual_permissions = get_permissions_dict(project_dir)
+
+        self.assertEqual(actual_permissions, expected_permissions)
+
+    def test_startproject_permissions_from_read_only(self):
+        """Check that generated files have the right permissions when the
+        template folder has been made read-only, which is something that some
+        systems do.
+
+        See https://github.com/scrapy/scrapy/pull/4604
+        """
+        scrapy_path = scrapy.__path__[0]
+        templates_dir = os.path.join(scrapy_path, 'templates')
+        project_template = os.path.join(templates_dir, 'project')
+        project_name = 'startproject2'
+        renamings = (
+            ('module', project_name),
+            ('.tmpl', ''),
+        )
+        expected_permissions = get_permissions_dict(
+            project_template,
+            renamings,
+            IGNORE,
+        )
+
+        def _make_read_only(path):
+            current_permissions = os.stat(path).st_mode
+            os.chmod(path, current_permissions & ~ANYONE_WRITE_PERMISSION)
+
+        read_only_templates_dir = str(Path(mkdtemp()) / 'templates')
+        copytree(templates_dir, read_only_templates_dir)
+
+        for root, dirs, files in os.walk(read_only_templates_dir):
+            for node in chain(dirs, files):
+                _make_read_only(os.path.join(root, node))
+
+        destination = mkdtemp()
+        process = subprocess.Popen(
+            (
+                sys.executable,
+                '-m',
+                'scrapy.cmdline',
+                'startproject',
+                project_name,
+                '--set',
+                'TEMPLATES_DIR={}'.format(read_only_templates_dir),
+            ),
+            cwd=destination,
+            env=self.env,
+        )
+        process.wait()
+
+        project_dir = os.path.join(destination, project_name)
+        actual_permissions = get_permissions_dict(project_dir)
+
+        self.assertEqual(actual_permissions, expected_permissions)
+
+    def test_startproject_permissions_unchanged_in_destination(self):
+        """Check that pre-existing folders and files in the destination folder
+        do not see their permissions modified."""
+        scrapy_path = scrapy.__path__[0]
+        project_template = os.path.join(scrapy_path, 'templates', 'project')
+        project_name = 'startproject3'
+        renamings = (
+            ('module', project_name),
+            ('.tmpl', ''),
+        )
+        expected_permissions = get_permissions_dict(
+            project_template,
+            renamings,
+            IGNORE,
+        )
+
+        destination = mkdtemp()
+        project_dir = os.path.join(destination, project_name)
+
+        existing_nodes = {
+            oct(permissions)[2:] + extension: permissions
+            for extension in ('', '.d')
+            for permissions in (
+                0o444, 0o555, 0o644, 0o666, 0o755, 0o777,
+            )
+        }
+        os.mkdir(project_dir)
+        project_dir_path = Path(project_dir)
+        for node, permissions in existing_nodes.items():
+            path = project_dir_path / node
+            if node.endswith('.d'):
+                path.mkdir(mode=permissions)
+            else:
+                path.touch(mode=permissions)
+            expected_permissions[node] = path.stat().st_mode
+
+        process = subprocess.Popen(
+            (
+                sys.executable,
+                '-m',
+                'scrapy.cmdline',
+                'startproject',
+                project_name,
+                '.',
+            ),
+            cwd=project_dir,
+            env=self.env,
+        )
+        process.wait()
+
+        actual_permissions = get_permissions_dict(project_dir)
+
+        self.assertEqual(actual_permissions, expected_permissions)
+
 
 class CommandTest(ProjectTest):
 
@@ -319,6 +491,9 @@ class BadSpider(scrapy.Spider):
         self.assertIn("start_requests", log)
         self.assertIn("badspider.py", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_true(self):
         log = self.get_log(self.debug_log_spider, args=[
             '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'

tests/test_crawl.py

@@ -29,6 +29,7 @@ from tests.spiders import (
     BytesReceivedCallbackSpider,
     BytesReceivedErrbackSpider,
     CrawlSpiderWithErrback,
+    CrawlSpiderWithParseMethod,
     DelaySpider,
     DuplicateStartRequestsSpider,
     FollowAllSpider,
@@ -377,6 +378,28 @@ with multiples lines
         self._assert_retried(log)
         self.assertIn("Got response 200", str(log))
 
+
+class CrawlSpiderTestCase(TestCase):
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+        self.runner = CrawlerRunner()
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def test_crawlspider_with_parse(self):
+        self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver)
+
+        with LogCapture() as log:
+            yield self.runner.join()
+
+        self.assertIn("[parse] status 200 (foo: None)", str(log))
+        self.assertIn("[parse] status 201 (foo: None)", str(log))
+        self.assertIn("[parse] status 202 (foo: bar)", str(log))
+
     @defer.inlineCallbacks
     def test_crawlspider_with_errback(self):
         self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver)
@@ -384,10 +407,12 @@ with multiples lines
         with LogCapture() as log:
             yield self.runner.join()
 
-        self.assertIn("[callback] status 200", str(log))
-        self.assertIn("[callback] status 201", str(log))
+        self.assertIn("[parse] status 200 (foo: None)", str(log))
+        self.assertIn("[parse] status 201 (foo: None)", str(log))
+        self.assertIn("[parse] status 202 (foo: bar)", str(log))
         self.assertIn("[errback] status 404", str(log))
         self.assertIn("[errback] status 500", str(log))
+        self.assertIn("[errback] status 501", str(log))
 
     @defer.inlineCallbacks
     def test_async_def_parse(self):

tests/test_crawler.py

@@ -1,8 +1,10 @@
 import logging
 import os
+import platform
 import subprocess
 import sys
 import warnings
+from unittest import skipIf
 
 from pytest import raises, mark
 from testfixtures import LogCapture
@@ -251,6 +253,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
                 })
 
     @defer.inlineCallbacks
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_crawler_process_asyncio_enabled_true(self):
         with LogCapture(level=logging.DEBUG) as log:
             if self.reactor_pytest == 'asyncio':
@@ -292,11 +297,17 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn('Spider closed (finished)', log)
         self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_no_reactor(self):
         log = self.run_script('asyncio_enabled_no_reactor.py')
         self.assertIn('Spider closed (finished)', log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_reactor(self):
         log = self.run_script('asyncio_enabled_reactor.py')
         self.assertIn('Spider closed (finished)', log)
@@ -320,11 +331,15 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
 
+    @mark.skipif(platform.system() == 'Windows', reason="PollReactor is not supported on Windows")
     def test_reactor_poll(self):
         log = self.run_script("twisted_reactor_poll.py")
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_reactor_asyncio(self):
         log = self.run_script("twisted_reactor_asyncio.py")
         self.assertIn("Spider closed (finished)", log)

tests/test_downloadermiddleware_httpcompression.py

@@ -5,8 +5,7 @@ from gzip import GzipFile
 
 from scrapy.spiders import Spider
 from scrapy.http import Response, Request, HtmlResponse
-from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \
-    ACCEPTED_ENCODINGS
+from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.gz import gunzip
 from tests import tests_datadir

tests/test_downloadermiddleware_redirect.py

@@ -77,12 +77,9 @@ class RedirectMiddlewareTest(unittest.TestCase):
         assert isinstance(req2, Request)
         self.assertEqual(req2.url, url2)
         self.assertEqual(req2.method, 'GET')
-        assert 'Content-Type' not in req2.headers, \
-            "Content-Type header must not be present in redirected request"
-        assert 'Content-Length' not in req2.headers, \
-            "Content-Length header must not be present in redirected request"
-        assert not req2.body, \
-            "Redirected body must be empty, not '%s'" % req2.body
+        assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request"
+        assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request"
+        assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body
 
         # response without Location header but with status code is 3XX should be ignored
         del rsp.headers['Location']
@@ -244,12 +241,9 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
         assert isinstance(req2, Request)
         self.assertEqual(req2.url, 'http://example.org/newpage')
         self.assertEqual(req2.method, 'GET')
-        assert 'Content-Type' not in req2.headers, \
-            "Content-Type header must not be present in redirected request"
-        assert 'Content-Length' not in req2.headers, \
-            "Content-Length header must not be present in redirected request"
-        assert not req2.body, \
-            "Redirected body must be empty, not '%s'" % req2.body
+        assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request"
+        assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request"
+        assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body
 
     def test_max_redirect_times(self):
         self.mw.max_redirect_times = 1

tests/test_feedexport.py

@@ -25,11 +25,23 @@ from zope.interface.verify import verifyObject
 import scrapy
 from scrapy.crawler import CrawlerRunner
 from scrapy.exporters import CsvItemExporter
-from scrapy.extensions.feedexport import (BlockingFeedStorage, FileFeedStorage, FTPFeedStorage,
-                                          IFeedStorage, S3FeedStorage, StdoutFeedStorage)
+from scrapy.extensions.feedexport import (
+    BlockingFeedStorage,
+    FileFeedStorage,
+    FTPFeedStorage,
+    GCSFeedStorage,
+    IFeedStorage,
+    S3FeedStorage,
+    StdoutFeedStorage,
+)
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
-from scrapy.utils.test import assert_aws_environ, get_crawler, get_s3_content_and_delete
+from scrapy.utils.test import (
+    assert_aws_environ,
+    get_s3_content_and_delete,
+    get_crawler,
+    mock_google_cloud_storage,
+)
 
 from tests.mockserver import MockServer
 
@@ -364,6 +376,63 @@ class S3FeedStorageTest(unittest.TestCase):
         )
 
 
+class GCSFeedStorageTest(unittest.TestCase):
+
+    def test_parse_settings(self):
+        try:
+            from google.cloud.storage import Client  # noqa
+        except ImportError:
+            raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead'}
+        crawler = get_crawler(settings_dict=settings)
+        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
+        assert storage.project_id == '123'
+        assert storage.acl == 'publicRead'
+        assert storage.bucket_name == 'mybucket'
+        assert storage.blob_name == 'export.csv'
+
+    def test_parse_empty_acl(self):
+        try:
+            from google.cloud.storage import Client  # noqa
+        except ImportError:
+            raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': ''}
+        crawler = get_crawler(settings_dict=settings)
+        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
+        assert storage.acl is None
+
+        settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None}
+        crawler = get_crawler(settings_dict=settings)
+        storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
+        assert storage.acl is None
+
+    @defer.inlineCallbacks
+    def test_store(self):
+        try:
+            from google.cloud.storage import Client  # noqa
+        except ImportError:
+            raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+        uri = 'gs://mybucket/export.csv'
+        project_id = 'myproject-123'
+        acl = 'publicRead'
+        (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
+        with mock.patch('google.cloud.storage.Client') as m:
+            m.return_value = client_mock
+
+            f = mock.Mock()
+            storage = GCSFeedStorage(uri, project_id, acl)
+            yield storage.store(f)
+
+            f.seek.assert_called_once_with(0)
+            m.assert_called_once_with(project=project_id)
+            client_mock.get_bucket.assert_called_once_with('mybucket')
+            bucket_mock.blob.assert_called_once_with('export.csv')
+            blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
+
+
 class StdoutFeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks
@@ -455,9 +524,15 @@ class FeedExportTest(unittest.TestCase):
     def run_and_export(self, spider_cls, settings):
         """ Run spider with specified settings; return exported data. """
 
+        def path_to_url(path):
+            return urljoin('file:', pathname2url(str(path)))
+
+        def printf_escape(string):
+            return string.replace('%', '%%')
+
         FEEDS = settings.get('FEEDS') or {}
         settings['FEEDS'] = {
-            urljoin('file:', pathname2url(str(file_path))): feed
+            printf_escape(path_to_url(file_path)): feed
             for file_path, feed in FEEDS.items()
         }
 

tests/test_http_cookies.py

@@ -64,9 +64,6 @@ class WrappedResponseTest(TestCase):
     def test_info(self):
         self.assertIs(self.wrapped.info(), self.wrapped)
 
-    def test_getheaders(self):
-        self.assertEqual(self.wrapped.getheaders('content-type'), ['text/html'])
-
     def test_get_all(self):
         # get_all result must be native string
         self.assertEqual(self.wrapped.get_all('content-type'), ['text/html'])

tests/test_loader.py

@@ -1,14 +1,12 @@
-from functools import partial
 import unittest
 
 import attr
 from itemadapter import ItemAdapter
+from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst
 
 from scrapy.http import HtmlResponse
 from scrapy.item import Item, Field
 from scrapy.loader import ItemLoader
-from scrapy.loader.processors import (Compose, Identity, Join,
-                                      MapCompose, SelectJmes, TakeFirst)
 from scrapy.selector import Selector
 
 
@@ -69,6 +67,10 @@ def processor_with_args(value, other=None, loader_context=None):
 
 class BasicItemLoaderTest(unittest.TestCase):
 
+    def test_add_value_on_unknown_field(self):
+        il = TestItemLoader()
+        self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo'])
+
     def test_load_item_using_default_loader(self):
         i = TestItem()
         i['summary'] = u'lala'
@@ -85,391 +87,6 @@ class BasicItemLoaderTest(unittest.TestCase):
         item = il.load_item()
         self.assertEqual(item['name'], [u'Marta'])
 
-    def test_load_item_ignore_none_field_values(self):
-        def validate_sku(value):
-            # Let's assume a SKU is only digits.
-            if value.isdigit():
-                return value
-
-        class MyLoader(ItemLoader):
-            name_out = Compose(lambda vs: vs[0])  # take first which allows empty values
-            price_out = Compose(TakeFirst(), float)
-            sku_out = Compose(TakeFirst(), validate_sku)
-
-        valid_fragment = u'SKU: 1234'
-        invalid_fragment = u'SKU: not available'
-        sku_re = 'SKU: (.+)'
-
-        il = MyLoader(item={})
-        # Should not return "sku: None".
-        il.add_value('sku', [invalid_fragment], re=sku_re)
-        # Should not ignore empty values.
-        il.add_value('name', u'')
-        il.add_value('price', [u'0'])
-        self.assertEqual(il.load_item(), {
-            'name': u'',
-            'price': 0.0,
-        })
-
-        il.replace_value('sku', [valid_fragment], re=sku_re)
-        self.assertEqual(il.load_item()['sku'], u'1234')
-
-    def test_self_referencing_loader(self):
-        class MyLoader(ItemLoader):
-            url_out = TakeFirst()
-
-            def img_url_out(self, values):
-                return (self.get_output_value('url') or '') + values[0]
-
-        il = MyLoader(item={})
-        il.add_value('url', 'http://example.com/')
-        il.add_value('img_url', '1234.png')
-        self.assertEqual(il.load_item(), {
-            'url': 'http://example.com/',
-            'img_url': 'http://example.com/1234.png',
-        })
-
-        il = MyLoader(item={})
-        il.add_value('img_url', '1234.png')
-        self.assertEqual(il.load_item(), {
-            'img_url': '1234.png',
-        })
-
-    def test_add_value(self):
-        il = TestItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-        il.add_value('name', u'pepe')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe'])
-
-        # test add object value
-        il.add_value('summary', {'key': 1})
-        self.assertEqual(il.get_collected_values('summary'), [{'key': 1}])
-
-        il.add_value(None, u'Jim', lambda x: {'name': x})
-        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim'])
-
-    def test_add_zero(self):
-        il = NameItemLoader()
-        il.add_value('name', 0)
-        self.assertEqual(il.get_collected_values('name'), [0])
-
-    def test_replace_value(self):
-        il = TestItemLoader()
-        il.replace_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-        il.replace_value('name', u'pepe')
-        self.assertEqual(il.get_collected_values('name'), [u'Pepe'])
-        self.assertEqual(il.get_output_value('name'), [u'Pepe'])
-
-        il.replace_value(None, u'Jim', lambda x: {'name': x})
-        self.assertEqual(il.get_collected_values('name'), [u'Jim'])
-
-    def test_get_value(self):
-        il = NameItemLoader()
-        self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))
-        self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
-        self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
-
-        il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')
-        self.assertEqual([u'foo'], il.get_collected_values('name'))
-        il.replace_value('name', u'name:bar', re=u'name:(.*)$')
-        self.assertEqual([u'bar'], il.get_collected_values('name'))
-
-    def test_iter_on_input_processor_input(self):
-        class NameFirstItemLoader(NameItemLoader):
-            name_in = TakeFirst()
-
-        il = NameFirstItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
-        il = NameFirstItemLoader()
-        il.add_value('name', [u'marta', u'jose'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
-
-        il = NameFirstItemLoader()
-        il.replace_value('name', u'marta')
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
-        il = NameFirstItemLoader()
-        il.replace_value('name', [u'marta', u'jose'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta'])
-
-        il = NameFirstItemLoader()
-        il.add_value('name', u'marta')
-        il.add_value('name', [u'jose', u'pedro'])
-        self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose'])
-
-    def test_map_compose_filter(self):
-        def filter_world(x):
-            return None if x == 'world' else x
-
-        proc = MapCompose(filter_world, str.upper)
-        self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),
-                         ['HELLO', 'THIS', 'IS', 'SCRAPY'])
-
-    def test_map_compose_filter_multil(self):
-        class TestItemLoader(NameItemLoader):
-            name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1])
-
-        il = TestItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Mart'])
-        item = il.load_item()
-        self.assertEqual(item['name'], [u'Mart'])
-
-    def test_default_input_processor(self):
-        il = DefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mart'])
-
-    def test_inherited_default_input_processor(self):
-        class InheritDefaultedItemLoader(DefaultedItemLoader):
-            pass
-
-        il = InheritDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mart'])
-
-    def test_input_processor_inheritance(self):
-        class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(lambda v: v.lower())
-
-        il = ChildItemLoader()
-        il.add_value('url', u'HTTP://scrapy.ORG')
-        self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org'])
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-
-        class ChildChildItemLoader(ChildItemLoader):
-            url_in = MapCompose(lambda v: v.upper())
-            summary_in = MapCompose(lambda v: v)
-
-        il = ChildChildItemLoader()
-        il.add_value('url', u'http://scrapy.org')
-        self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG'])
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'Marta'])
-
-    def test_empty_map_compose(self):
-        class IdentityDefaultedItemLoader(DefaultedItemLoader):
-            name_in = MapCompose()
-
-        il = IdentityDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'marta'])
-
-    def test_identity_input_processor(self):
-        class IdentityDefaultedItemLoader(DefaultedItemLoader):
-            name_in = Identity()
-
-        il = IdentityDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'marta'])
-
-    def test_extend_custom_input_processors(self):
-        class ChildItemLoader(TestItemLoader):
-            name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
-
-        il = ChildItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'mARTA'])
-
-    def test_extend_default_input_processors(self):
-        class ChildDefaultedItemLoader(DefaultedItemLoader):
-            name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
-
-        il = ChildDefaultedItemLoader()
-        il.add_value('name', u'marta')
-        self.assertEqual(il.get_output_value('name'), [u'MART'])
-
-    def test_output_processor_using_function(self):
-        il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
-
-        class TakeFirstItemLoader(TestItemLoader):
-            name_out = u" ".join
-
-        il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
-
-    def test_output_processor_error(self):
-        class TestItemLoader(ItemLoader):
-            default_item_class = TestItem
-            name_out = MapCompose(float)
-
-        il = TestItemLoader()
-        il.add_value('name', [u'$10'])
-        try:
-            float(u'$10')
-        except Exception as e:
-            expected_exc_str = str(e)
-
-        exc = None
-        try:
-            il.load_item()
-        except Exception as e:
-            exc = e
-        assert isinstance(exc, ValueError)
-        s = str(exc)
-        assert 'name' in s, s
-        assert '$10' in s, s
-        assert 'ValueError' in s, s
-        assert expected_exc_str in s, s
-
-    def test_output_processor_using_classes(self):
-        il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
-
-        class TakeFirstItemLoader(TestItemLoader):
-            name_out = Join()
-
-        il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
-
-        class TakeFirstItemLoader(TestItemLoader):
-            name_out = Join("<br>")
-
-        il = TakeFirstItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), u'Mar<br>Ta')
-
-    def test_default_output_processor(self):
-        il = TestItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
-
-        class LalaItemLoader(TestItemLoader):
-            default_output_processor = Identity()
-
-        il = LalaItemLoader()
-        il.add_value('name', [u'mar', u'ta'])
-        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
-
-    def test_loader_context_on_declaration(self):
-        class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(processor_with_args, key=u'val')
-
-        il = ChildItemLoader()
-        il.add_value('url', u'text')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-
-    def test_loader_context_on_instantiation(self):
-        class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(processor_with_args)
-
-        il = ChildItemLoader(key=u'val')
-        il.add_value('url', u'text')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-
-    def test_loader_context_on_assign(self):
-        class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(processor_with_args)
-
-        il = ChildItemLoader()
-        il.context['key'] = u'val'
-        il.add_value('url', u'text')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-        il.replace_value('url', u'text2')
-        self.assertEqual(il.get_output_value('url'), ['val'])
-
-    def test_item_passed_to_input_processor_functions(self):
-        def processor(value, loader_context):
-            return loader_context['item']['name']
-
-        class ChildItemLoader(TestItemLoader):
-            url_in = MapCompose(processor)
-
-        it = TestItem(name='marta')
-        il = ChildItemLoader(item=it)
-        il.add_value('url', u'text')
-        self.assertEqual(il.get_output_value('url'), ['marta'])
-        il.replace_value('url', u'text2')
-        self.assertEqual(il.get_output_value('url'), ['marta'])
-
-    def test_add_value_on_unknown_field(self):
-        il = TestItemLoader()
-        self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo'])
-
-    def test_compose_processor(self):
-        class TestItemLoader(NameItemLoader):
-            name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1])
-
-        il = TestItemLoader()
-        il.add_value('name', [u'marta', u'other'])
-        self.assertEqual(il.get_output_value('name'), u'Mart')
-        item = il.load_item()
-        self.assertEqual(item['name'], u'Mart')
-
-    def test_partial_processor(self):
-        def join(values, sep=None, loader_context=None, ignored=None):
-            if sep is not None:
-                return sep.join(values)
-            elif loader_context and 'sep' in loader_context:
-                return loader_context['sep'].join(values)
-            else:
-                return ''.join(values)
-
-        class TestItemLoader(NameItemLoader):
-            name_out = Compose(partial(join, sep='+'))
-            url_out = Compose(partial(join, loader_context={'sep': '.'}))
-            summary_out = Compose(partial(join, ignored='foo'))
-
-        il = TestItemLoader()
-        il.add_value('name', [u'rabbit', u'hole'])
-        il.add_value('url', [u'rabbit', u'hole'])
-        il.add_value('summary', [u'rabbit', u'hole'])
-        item = il.load_item()
-        self.assertEqual(item['name'], u'rabbit+hole')
-        self.assertEqual(item['url'], u'rabbit.hole')
-        self.assertEqual(item['summary'], u'rabbithole')
-
-    def test_error_input_processor(self):
-        class TestItem(Item):
-            name = Field()
-
-        class TestItemLoader(ItemLoader):
-            default_item_class = TestItem
-            name_in = MapCompose(float)
-
-        il = TestItemLoader()
-        self.assertRaises(ValueError, il.add_value, 'name',
-                          [u'marta', u'other'])
-
-    def test_error_output_processor(self):
-        class TestItem(Item):
-            name = Field()
-
-        class TestItemLoader(ItemLoader):
-            default_item_class = TestItem
-            name_out = Compose(Join(), float)
-
-        il = TestItemLoader()
-        il.add_value('name', u'marta')
-        with self.assertRaises(ValueError):
-            il.load_item()
-
-    def test_error_processor_as_argument(self):
-        class TestItem(Item):
-            name = Field()
-
-        class TestItemLoader(ItemLoader):
-            default_item_class = TestItem
-
-        il = TestItemLoader()
-        self.assertRaises(ValueError, il.add_value, 'name',
-                          [u'marta', u'other'], Compose(float))
-
 
 class InitializationTestMixin:
 
@@ -587,41 +204,6 @@ class BaseNoInputReprocessingLoader(ItemLoader):
     title_out = TakeFirst()
 
 
-class NoInputReprocessingDictLoader(BaseNoInputReprocessingLoader):
-    default_item_class = dict
-
-
-class NoInputReprocessingFromDictTest(unittest.TestCase):
-    """
-    Loaders initialized from loaded items must not reprocess fields (dict instances)
-    """
-    def test_avoid_reprocessing_with_initial_values_single(self):
-        il = NoInputReprocessingDictLoader(item=dict(title='foo'))
-        il_loaded = il.load_item()
-        self.assertEqual(il_loaded, dict(title='foo'))
-        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
-
-    def test_avoid_reprocessing_with_initial_values_list(self):
-        il = NoInputReprocessingDictLoader(item=dict(title=['foo', 'bar']))
-        il_loaded = il.load_item()
-        self.assertEqual(il_loaded, dict(title='foo'))
-        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
-
-    def test_avoid_reprocessing_without_initial_values_single(self):
-        il = NoInputReprocessingDictLoader()
-        il.add_value('title', 'foo')
-        il_loaded = il.load_item()
-        self.assertEqual(il_loaded, dict(title='FOO'))
-        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
-
-    def test_avoid_reprocessing_without_initial_values_list(self):
-        il = NoInputReprocessingDictLoader()
-        il.add_value('title', ['foo', 'bar'])
-        il_loaded = il.load_item()
-        self.assertEqual(il_loaded, dict(title='FOO'))
-        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
-
-
 class NoInputReprocessingItem(Item):
     title = Field()
 
@@ -661,25 +243,6 @@ class NoInputReprocessingFromItemTest(unittest.TestCase):
         self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'FOO'})
 
 
-class TestOutputProcessorDict(unittest.TestCase):
-    def test_output_processor(self):
-
-        class TempDict(dict):
-            def __init__(self, *args, **kwargs):
-                super(TempDict, self).__init__(self, *args, **kwargs)
-                self.setdefault('temp', 0.3)
-
-        class TempLoader(ItemLoader):
-            default_item_class = TempDict
-            default_input_processor = Identity()
-            default_output_processor = Compose(TakeFirst())
-
-        loader = TempLoader()
-        item = loader.load_item()
-        self.assertIsInstance(item, TempDict)
-        self.assertEqual(dict(item), {'temp': 0.3})
-
-
 class TestOutputProcessorItem(unittest.TestCase):
     def test_output_processor(self):
 
@@ -701,49 +264,6 @@ class TestOutputProcessorItem(unittest.TestCase):
         self.assertEqual(dict(item), {'temp': 0.3})
 
 
-class ProcessorsTest(unittest.TestCase):
-
-    def test_take_first(self):
-        proc = TakeFirst()
-        self.assertEqual(proc([None, '', 'hello', 'world']), 'hello')
-        self.assertEqual(proc([None, '', 0, 'hello', 'world']), 0)
-
-    def test_identity(self):
-        proc = Identity()
-        self.assertEqual(proc([None, '', 'hello', 'world']),
-                         [None, '', 'hello', 'world'])
-
-    def test_join(self):
-        proc = Join()
-        self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
-        self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
-        self.assertEqual(proc(['hello', 'world']), u'hello world')
-        self.assertIsInstance(proc(['hello', 'world']), str)
-
-    def test_compose(self):
-        proc = Compose(lambda v: v[0], str.upper)
-        self.assertEqual(proc(['hello', 'world']), 'HELLO')
-        proc = Compose(str.upper)
-        self.assertEqual(proc(None), None)
-        proc = Compose(str.upper, stop_on_none=False)
-        self.assertRaises(ValueError, proc, None)
-        proc = Compose(str.upper, lambda x: x + 1)
-        self.assertRaises(ValueError, proc, 'hello')
-
-    def test_mapcompose(self):
-        def filter_world(x):
-            return None if x == 'world' else x
-        proc = MapCompose(filter_world, str.upper)
-        self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
-                         [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
-        proc = MapCompose(filter_world, str.upper)
-        self.assertEqual(proc(None), [])
-        proc = MapCompose(filter_world, str.upper)
-        self.assertRaises(ValueError, proc, [1])
-        proc = MapCompose(filter_world, lambda x: x + 1)
-        self.assertRaises(ValueError, proc, 'hello')
-
-
 class SelectortemLoaderTest(unittest.TestCase):
     response = HtmlResponse(url="", encoding='utf-8', body=b"""
     <html>
@@ -921,6 +441,7 @@ class SubselectorLoaderTest(unittest.TestCase):
 
     def test_nested_xpath(self):
         l = NestedItemLoader(response=self.response)
+
         nl = l.nested_xpath("//header")
         nl.add_xpath('name', 'div/text()')
         nl.add_css('name_div', '#id')
@@ -998,31 +519,6 @@ class SubselectorLoaderTest(unittest.TestCase):
         self.assertEqual(item['image'], [u'/images/logo.png'])
 
 
-class SelectJmesTestCase(unittest.TestCase):
-    test_list_equals = {
-        'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
-        'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
-        'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
-        'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
-        'dict': (
-            'foo.bar[*].name',
-            {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
-            ['one', 'two']
-        ),
-        'list': ('[1]', [1, 2], 2)
-    }
-
-    def test_output(self):
-        for l in self.test_list_equals:
-            expr, test_list, expected = self.test_list_equals[l]
-            test = SelectJmes(expr)(test_list)
-            self.assertEqual(
-                test,
-                expected,
-                msg='test "{}" got {} expected {}'.format(l, test, expected)
-            )
-
-
 # Functions as processors
 
 def function_processor_strip(iterable):
@@ -1044,12 +540,6 @@ class FunctionProcessorItemLoader(ItemLoader):
     default_item_class = FunctionProcessorItem
 
 
-class FunctionProcessorDictLoader(ItemLoader):
-    default_item_class = dict
-    foo_in = function_processor_strip
-    foo_out = function_processor_upper
-
-
 class FunctionProcessorTestCase(unittest.TestCase):
 
     def test_processor_defined_in_item(self):
@@ -1061,15 +551,6 @@ class FunctionProcessorTestCase(unittest.TestCase):
             {'foo': ['BAR', 'ASDF', 'QWERTY']}
         )
 
-    def test_processor_defined_in_item_loader(self):
-        lo = FunctionProcessorDictLoader()
-        lo.add_value('foo', '  bar  ')
-        lo.add_value('foo', ['  asdf  ', '  qwerty  '])
-        self.assertEqual(
-            dict(lo.load_item()),
-            {'foo': ['BAR', 'ASDF', 'QWERTY']}
-        )
-
 
 if __name__ == "__main__":
     unittest.main()

tests/test_loader_deprecated.py

@@ -0,0 +1,720 @@
+"""
+These tests are kept as references from the ones that were ported to a itemloaders library.
+Once we remove the references from scrapy, we can remove these tests.
+"""
+
+import unittest
+import warnings
+from functools import partial
+
+from itemloaders.processors import (Compose, Identity, Join,
+                                    MapCompose, SelectJmes, TakeFirst)
+
+from scrapy.item import Item, Field
+from scrapy.loader import ItemLoader
+from scrapy.loader.common import wrap_loader_context
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
+from scrapy.utils.misc import extract_regex
+
+
+# test items
+class NameItem(Item):
+    name = Field()
+
+
+class TestItem(NameItem):
+    url = Field()
+    summary = Field()
+
+
+# test item loaders
+class NameItemLoader(ItemLoader):
+    default_item_class = TestItem
+
+
+class TestItemLoader(NameItemLoader):
+    name_in = MapCompose(lambda v: v.title())
+
+
+class DefaultedItemLoader(NameItemLoader):
+    default_input_processor = MapCompose(lambda v: v[:-1])
+
+
+# test processors
+def processor_with_args(value, other=None, loader_context=None):
+    if 'key' in loader_context:
+        return loader_context['key']
+    return value
+
+
+class BasicItemLoaderTest(unittest.TestCase):
+
+    def test_load_item_using_default_loader(self):
+        i = TestItem()
+        i['summary'] = u'lala'
+        il = ItemLoader(item=i)
+        il.add_value('name', u'marta')
+        item = il.load_item()
+        assert item is i
+        self.assertEqual(item['summary'], [u'lala'])
+        self.assertEqual(item['name'], [u'marta'])
+
+    def test_load_item_using_custom_loader(self):
+        il = TestItemLoader()
+        il.add_value('name', u'marta')
+        item = il.load_item()
+        self.assertEqual(item['name'], [u'Marta'])
+
+    def test_load_item_ignore_none_field_values(self):
+        def validate_sku(value):
+            # Let's assume a SKU is only digits.
+            if value.isdigit():
+                return value
+
+        class MyLoader(ItemLoader):
+            name_out = Compose(lambda vs: vs[0])  # take first which allows empty values
+            price_out = Compose(TakeFirst(), float)
+            sku_out = Compose(TakeFirst(), validate_sku)
+
+        valid_fragment = u'SKU: 1234'
+        invalid_fragment = u'SKU: not available'
+        sku_re = 'SKU: (.+)'
+
+        il = MyLoader(item={})
+        # Should not return "sku: None".
+        il.add_value('sku', [invalid_fragment], re=sku_re)
+        # Should not ignore empty values.
+        il.add_value('name', u'')
+        il.add_value('price', [u'0'])
+        self.assertEqual(il.load_item(), {
+            'name': u'',
+            'price': 0.0,
+        })
+
+        il.replace_value('sku', [valid_fragment], re=sku_re)
+        self.assertEqual(il.load_item()['sku'], u'1234')
+
+    def test_self_referencing_loader(self):
+        class MyLoader(ItemLoader):
+            url_out = TakeFirst()
+
+            def img_url_out(self, values):
+                return (self.get_output_value('url') or '') + values[0]
+
+        il = MyLoader(item={})
+        il.add_value('url', 'http://example.com/')
+        il.add_value('img_url', '1234.png')
+        self.assertEqual(il.load_item(), {
+            'url': 'http://example.com/',
+            'img_url': 'http://example.com/1234.png',
+        })
+
+        il = MyLoader(item={})
+        il.add_value('img_url', '1234.png')
+        self.assertEqual(il.load_item(), {
+            'img_url': '1234.png',
+        })
+
+    def test_add_value(self):
+        il = TestItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
+        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+        il.add_value('name', u'pepe')
+        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe'])
+        self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe'])
+
+        # test add object value
+        il.add_value('summary', {'key': 1})
+        self.assertEqual(il.get_collected_values('summary'), [{'key': 1}])
+
+        il.add_value(None, u'Jim', lambda x: {'name': x})
+        self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim'])
+
+    def test_add_zero(self):
+        il = NameItemLoader()
+        il.add_value('name', 0)
+        self.assertEqual(il.get_collected_values('name'), [0])
+
+    def test_replace_value(self):
+        il = TestItemLoader()
+        il.replace_value('name', u'marta')
+        self.assertEqual(il.get_collected_values('name'), [u'Marta'])
+        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+        il.replace_value('name', u'pepe')
+        self.assertEqual(il.get_collected_values('name'), [u'Pepe'])
+        self.assertEqual(il.get_output_value('name'), [u'Pepe'])
+
+        il.replace_value(None, u'Jim', lambda x: {'name': x})
+        self.assertEqual(il.get_collected_values('name'), [u'Jim'])
+
+    def test_get_value(self):
+        il = NameItemLoader()
+        self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))
+        self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
+        self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
+
+        il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')
+        self.assertEqual([u'foo'], il.get_collected_values('name'))
+        il.replace_value('name', u'name:bar', re=u'name:(.*)$')
+        self.assertEqual([u'bar'], il.get_collected_values('name'))
+
+    def test_iter_on_input_processor_input(self):
+        class NameFirstItemLoader(NameItemLoader):
+            name_in = TakeFirst()
+
+        il = NameFirstItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il = NameFirstItemLoader()
+        il.add_value('name', [u'marta', u'jose'])
+        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+
+        il = NameFirstItemLoader()
+        il.replace_value('name', u'marta')
+        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+        il = NameFirstItemLoader()
+        il.replace_value('name', [u'marta', u'jose'])
+        self.assertEqual(il.get_collected_values('name'), [u'marta'])
+
+        il = NameFirstItemLoader()
+        il.add_value('name', u'marta')
+        il.add_value('name', [u'jose', u'pedro'])
+        self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose'])
+
+    def test_map_compose_filter(self):
+        def filter_world(x):
+            return None if x == 'world' else x
+
+        proc = MapCompose(filter_world, str.upper)
+        self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),
+                         ['HELLO', 'THIS', 'IS', 'SCRAPY'])
+
+    def test_map_compose_filter_multil(self):
+        class TestItemLoader(NameItemLoader):
+            name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1])
+
+        il = TestItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'Mart'])
+        item = il.load_item()
+        self.assertEqual(item['name'], [u'Mart'])
+
+    def test_default_input_processor(self):
+        il = DefaultedItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'mart'])
+
+    def test_inherited_default_input_processor(self):
+        class InheritDefaultedItemLoader(DefaultedItemLoader):
+            pass
+
+        il = InheritDefaultedItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'mart'])
+
+    def test_input_processor_inheritance(self):
+        class ChildItemLoader(TestItemLoader):
+            url_in = MapCompose(lambda v: v.lower())
+
+        il = ChildItemLoader()
+        il.add_value('url', u'HTTP://scrapy.ORG')
+        self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org'])
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+
+        class ChildChildItemLoader(ChildItemLoader):
+            url_in = MapCompose(lambda v: v.upper())
+            summary_in = MapCompose(lambda v: v)
+
+        il = ChildChildItemLoader()
+        il.add_value('url', u'http://scrapy.org')
+        self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG'])
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'Marta'])
+
+    def test_empty_map_compose(self):
+        class IdentityDefaultedItemLoader(DefaultedItemLoader):
+            name_in = MapCompose()
+
+        il = IdentityDefaultedItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'marta'])
+
+    def test_identity_input_processor(self):
+        class IdentityDefaultedItemLoader(DefaultedItemLoader):
+            name_in = Identity()
+
+        il = IdentityDefaultedItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'marta'])
+
+    def test_extend_custom_input_processors(self):
+        class ChildItemLoader(TestItemLoader):
+            name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
+
+        il = ChildItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'mARTA'])
+
+    def test_extend_default_input_processors(self):
+        class ChildDefaultedItemLoader(DefaultedItemLoader):
+            name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
+
+        il = ChildDefaultedItemLoader()
+        il.add_value('name', u'marta')
+        self.assertEqual(il.get_output_value('name'), [u'MART'])
+
+    def test_output_processor_using_function(self):
+        il = TestItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+
+        class TakeFirstItemLoader(TestItemLoader):
+            name_out = u" ".join
+
+        il = TakeFirstItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
+
+    def test_output_processor_error(self):
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+            name_out = MapCompose(float)
+
+        il = TestItemLoader()
+        il.add_value('name', [u'$10'])
+        try:
+            float(u'$10')
+        except Exception as e:
+            expected_exc_str = str(e)
+
+        exc = None
+        try:
+            il.load_item()
+        except Exception as e:
+            exc = e
+        assert isinstance(exc, ValueError)
+        s = str(exc)
+        assert 'name' in s, s
+        assert '$10' in s, s
+        assert 'ValueError' in s, s
+        assert expected_exc_str in s, s
+
+    def test_output_processor_using_classes(self):
+        il = TestItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+
+        class TakeFirstItemLoader(TestItemLoader):
+            name_out = Join()
+
+        il = TakeFirstItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), u'Mar Ta')
+
+        class TakeFirstItemLoader(TestItemLoader):
+            name_out = Join("<br>")
+
+        il = TakeFirstItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), u'Mar<br>Ta')
+
+    def test_default_output_processor(self):
+        il = TestItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+
+        class LalaItemLoader(TestItemLoader):
+            default_output_processor = Identity()
+
+        il = LalaItemLoader()
+        il.add_value('name', [u'mar', u'ta'])
+        self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+
+    def test_loader_context_on_declaration(self):
+        class ChildItemLoader(TestItemLoader):
+            url_in = MapCompose(processor_with_args, key=u'val')
+
+        il = ChildItemLoader()
+        il.add_value('url', u'text')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+        il.replace_value('url', u'text2')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+
+    def test_loader_context_on_instantiation(self):
+        class ChildItemLoader(TestItemLoader):
+            url_in = MapCompose(processor_with_args)
+
+        il = ChildItemLoader(key=u'val')
+        il.add_value('url', u'text')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+        il.replace_value('url', u'text2')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+
+    def test_loader_context_on_assign(self):
+        class ChildItemLoader(TestItemLoader):
+            url_in = MapCompose(processor_with_args)
+
+        il = ChildItemLoader()
+        il.context['key'] = u'val'
+        il.add_value('url', u'text')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+        il.replace_value('url', u'text2')
+        self.assertEqual(il.get_output_value('url'), ['val'])
+
+    def test_item_passed_to_input_processor_functions(self):
+        def processor(value, loader_context):
+            return loader_context['item']['name']
+
+        class ChildItemLoader(TestItemLoader):
+            url_in = MapCompose(processor)
+
+        it = TestItem(name='marta')
+        il = ChildItemLoader(item=it)
+        il.add_value('url', u'text')
+        self.assertEqual(il.get_output_value('url'), ['marta'])
+        il.replace_value('url', u'text2')
+        self.assertEqual(il.get_output_value('url'), ['marta'])
+
+    def test_compose_processor(self):
+        class TestItemLoader(NameItemLoader):
+            name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1])
+
+        il = TestItemLoader()
+        il.add_value('name', [u'marta', u'other'])
+        self.assertEqual(il.get_output_value('name'), u'Mart')
+        item = il.load_item()
+        self.assertEqual(item['name'], u'Mart')
+
+    def test_partial_processor(self):
+        def join(values, sep=None, loader_context=None, ignored=None):
+            if sep is not None:
+                return sep.join(values)
+            elif loader_context and 'sep' in loader_context:
+                return loader_context['sep'].join(values)
+            else:
+                return ''.join(values)
+
+        class TestItemLoader(NameItemLoader):
+            name_out = Compose(partial(join, sep='+'))
+            url_out = Compose(partial(join, loader_context={'sep': '.'}))
+            summary_out = Compose(partial(join, ignored='foo'))
+
+        il = TestItemLoader()
+        il.add_value('name', [u'rabbit', u'hole'])
+        il.add_value('url', [u'rabbit', u'hole'])
+        il.add_value('summary', [u'rabbit', u'hole'])
+        item = il.load_item()
+        self.assertEqual(item['name'], u'rabbit+hole')
+        self.assertEqual(item['url'], u'rabbit.hole')
+        self.assertEqual(item['summary'], u'rabbithole')
+
+    def test_error_input_processor(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+            name_in = MapCompose(float)
+
+        il = TestItemLoader()
+        self.assertRaises(ValueError, il.add_value, 'name',
+                          [u'marta', u'other'])
+
+    def test_error_output_processor(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+            name_out = Compose(Join(), float)
+
+        il = TestItemLoader()
+        il.add_value('name', u'marta')
+        with self.assertRaises(ValueError):
+            il.load_item()
+
+    def test_error_processor_as_argument(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+
+        il = TestItemLoader()
+        self.assertRaises(ValueError, il.add_value, 'name',
+                          [u'marta', u'other'], Compose(float))
+
+
+class InitializationFromDictTest(unittest.TestCase):
+
+    item_class = dict
+
+    def test_keep_single_value(self):
+        """Loaded item should contain values from the initial item"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo']})
+
+    def test_keep_list(self):
+        """Loaded item should contain values from the initial item"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
+
+    def test_add_value_singlevalue_singlevalue(self):
+        """Values added after initialization should be appended"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        il.add_value('name', 'bar')
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
+
+    def test_add_value_singlevalue_list(self):
+        """Values added after initialization should be appended"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        il.add_value('name', ['item', 'loader'])
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'item', 'loader']})
+
+    def test_add_value_list_singlevalue(self):
+        """Values added after initialization should be appended"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        il.add_value('name', 'qwerty')
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'qwerty']})
+
+    def test_add_value_list_list(self):
+        """Values added after initialization should be appended"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        il.add_value('name', ['item', 'loader'])
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'item', 'loader']})
+
+    def test_get_output_value_singlevalue(self):
+        """Getting output value must not remove value from item"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        self.assertEqual(il.get_output_value('name'), ['foo'])
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(loaded_item, dict({'name': ['foo']}))
+
+    def test_get_output_value_list(self):
+        """Getting output value must not remove value from item"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        self.assertEqual(il.get_output_value('name'), ['foo', 'bar'])
+        loaded_item = il.load_item()
+        self.assertIsInstance(loaded_item, self.item_class)
+        self.assertEqual(loaded_item, dict({'name': ['foo', 'bar']}))
+
+    def test_values_single(self):
+        """Values from initial item must be added to loader._values"""
+        input_item = self.item_class(name='foo')
+        il = ItemLoader(item=input_item)
+        self.assertEqual(il._values.get('name'), ['foo'])
+
+    def test_values_list(self):
+        """Values from initial item must be added to loader._values"""
+        input_item = self.item_class(name=['foo', 'bar'])
+        il = ItemLoader(item=input_item)
+        self.assertEqual(il._values.get('name'), ['foo', 'bar'])
+
+
+class BaseNoInputReprocessingLoader(ItemLoader):
+    title_in = MapCompose(str.upper)
+    title_out = TakeFirst()
+
+
+class NoInputReprocessingDictLoader(BaseNoInputReprocessingLoader):
+    default_item_class = dict
+
+
+class NoInputReprocessingFromDictTest(unittest.TestCase):
+    """
+    Loaders initialized from loaded items must not reprocess fields (dict instances)
+    """
+    def test_avoid_reprocessing_with_initial_values_single(self):
+        il = NoInputReprocessingDictLoader(item=dict(title='foo'))
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, dict(title='foo'))
+        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
+
+    def test_avoid_reprocessing_with_initial_values_list(self):
+        il = NoInputReprocessingDictLoader(item=dict(title=['foo', 'bar']))
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, dict(title='foo'))
+        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
+
+    def test_avoid_reprocessing_without_initial_values_single(self):
+        il = NoInputReprocessingDictLoader()
+        il.add_value('title', 'foo')
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, dict(title='FOO'))
+        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
+
+    def test_avoid_reprocessing_without_initial_values_list(self):
+        il = NoInputReprocessingDictLoader()
+        il.add_value('title', ['foo', 'bar'])
+        il_loaded = il.load_item()
+        self.assertEqual(il_loaded, dict(title='FOO'))
+        self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
+
+
+class TestOutputProcessorDict(unittest.TestCase):
+    def test_output_processor(self):
+
+        class TempDict(dict):
+            def __init__(self, *args, **kwargs):
+                super(TempDict, self).__init__(self, *args, **kwargs)
+                self.setdefault('temp', 0.3)
+
+        class TempLoader(ItemLoader):
+            default_item_class = TempDict
+            default_input_processor = Identity()
+            default_output_processor = Compose(TakeFirst())
+
+        loader = TempLoader()
+        item = loader.load_item()
+        self.assertIsInstance(item, TempDict)
+        self.assertEqual(dict(item), {'temp': 0.3})
+
+
+class ProcessorsTest(unittest.TestCase):
+
+    def test_take_first(self):
+        proc = TakeFirst()
+        self.assertEqual(proc([None, '', 'hello', 'world']), 'hello')
+        self.assertEqual(proc([None, '', 0, 'hello', 'world']), 0)
+
+    def test_identity(self):
+        proc = Identity()
+        self.assertEqual(proc([None, '', 'hello', 'world']),
+                         [None, '', 'hello', 'world'])
+
+    def test_join(self):
+        proc = Join()
+        self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
+        self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
+        self.assertEqual(proc(['hello', 'world']), u'hello world')
+        self.assertIsInstance(proc(['hello', 'world']), str)
+
+    def test_compose(self):
+        proc = Compose(lambda v: v[0], str.upper)
+        self.assertEqual(proc(['hello', 'world']), 'HELLO')
+        proc = Compose(str.upper)
+        self.assertEqual(proc(None), None)
+        proc = Compose(str.upper, stop_on_none=False)
+        self.assertRaises(ValueError, proc, None)
+        proc = Compose(str.upper, lambda x: x + 1)
+        self.assertRaises(ValueError, proc, 'hello')
+
+    def test_mapcompose(self):
+        def filter_world(x):
+            return None if x == 'world' else x
+        proc = MapCompose(filter_world, str.upper)
+        self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
+                         [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
+        proc = MapCompose(filter_world, str.upper)
+        self.assertEqual(proc(None), [])
+        proc = MapCompose(filter_world, str.upper)
+        self.assertRaises(ValueError, proc, [1])
+        proc = MapCompose(filter_world, lambda x: x + 1)
+        self.assertRaises(ValueError, proc, 'hello')
+
+
+class SelectJmesTestCase(unittest.TestCase):
+    test_list_equals = {
+        'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
+        'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
+        'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
+        'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
+        'dict': (
+            'foo.bar[*].name',
+            {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
+            ['one', 'two']
+        ),
+        'list': ('[1]', [1, 2], 2)
+    }
+
+    def test_output(self):
+        for tl in self.test_list_equals:
+            expr, test_list, expected = self.test_list_equals[tl]
+            test = SelectJmes(expr)(test_list)
+            self.assertEqual(
+                test,
+                expected,
+                msg='test "{}" got {} expected {}'.format(tl, test, expected)
+            )
+
+
+# Functions as processors
+
+def function_processor_strip(iterable):
+    return [x.strip() for x in iterable]
+
+
+def function_processor_upper(iterable):
+    return [x.upper() for x in iterable]
+
+
+class FunctionProcessorItem(Item):
+    foo = Field(
+        input_processor=function_processor_strip,
+        output_processor=function_processor_upper,
+    )
+
+
+class FunctionProcessorDictLoader(ItemLoader):
+    default_item_class = dict
+    foo_in = function_processor_strip
+    foo_out = function_processor_upper
+
+
+class FunctionProcessorTestCase(unittest.TestCase):
+
+    def test_processor_defined_in_item_loader(self):
+        lo = FunctionProcessorDictLoader()
+        lo.add_value('foo', '  bar  ')
+        lo.add_value('foo', ['  asdf  ', '  qwerty  '])
+        self.assertEqual(
+            dict(lo.load_item()),
+            {'foo': ['BAR', 'ASDF', 'QWERTY']}
+        )
+
+
+class DeprecatedUtilityFunctionsTestCase(unittest.TestCase):
+
+    def test_deprecated_wrap_loader_context(self):
+        def function(*args):
+            return None
+
+        with warnings.catch_warnings(record=True) as w:
+            wrap_loader_context(function, context=dict())
+
+            assert len(w) == 1
+            assert issubclass(w[0].category, ScrapyDeprecationWarning)
+
+    def test_deprecated_extract_regex(self):
+        with warnings.catch_warnings(record=True) as w:
+            extract_regex(r'\w+', 'this is a test')
+
+            assert len(w) == 1
+            assert issubclass(w[0].category, ScrapyDeprecationWarning)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_proxy_connect.py

@@ -1,5 +1,6 @@
 import json
 import os
+import platform
 import re
 import sys
 from subprocess import Popen, PIPE
@@ -59,6 +60,10 @@ def _wrong_credentials(proxy_url):
 
 @skipIf(sys.version_info < (3, 5, 4),
         "requires mitmproxy < 3.0.0, which these tests do not support")
+@skipIf("pypy" in sys.executable,
+        "mitmproxy does not support PyPy")
+@skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7),
+        "mitmproxy does not support Windows when running Python < 3.7")
 class ProxyConnectTestCase(TestCase):
 
     def setUp(self):

tests/test_selector.py

@@ -88,8 +88,7 @@ class SelectorTestCase(unittest.TestCase):
         """Check that classes are using slots and are weak-referenceable"""
         x = Selector(text='')
         weakref.ref(x)
-        assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
-            x.__class__.__name__
+        assert not hasattr(x, '__dict__'), "%s does not use __slots__" % x.__class__.__name__
 
     def test_selector_bad_args(self):
         with self.assertRaisesRegex(ValueError, 'received both response and text'):

tests/test_settings/__init__.py

@@ -86,8 +86,7 @@ class BaseSettingsTest(unittest.TestCase):
 
     def test_set_calls_settings_attributes_methods_on_update(self):
         attr = SettingsAttribute('value', 10)
-        with mock.patch.object(attr, '__setattr__') as mock_setattr, \
-                mock.patch.object(attr, 'set') as mock_set:
+        with mock.patch.object(attr, '__setattr__') as mock_setattr, mock.patch.object(attr, 'set') as mock_set:
 
             self.settings.attributes = {'TEST_OPTION': attr}
 

tests/test_spider.py

@@ -11,8 +11,14 @@ from scrapy import signals
 from scrapy.settings import Settings
 from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse
 from scrapy.spiders.init import InitSpider
-from scrapy.spiders import Spider, CrawlSpider, Rule, XMLFeedSpider, \
-    CSVFeedSpider, SitemapSpider
+from scrapy.spiders import (
+    CSVFeedSpider,
+    CrawlSpider,
+    Rule,
+    SitemapSpider,
+    Spider,
+    XMLFeedSpider,
+)
 from scrapy.linkextractors import LinkExtractor
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.test import get_crawler
@@ -144,7 +150,7 @@ class XMLFeedSpiderTest(SpiderTest):
 
         for iterator in ('iternodes', 'xml'):
             spider = _XMLSpider('example', iterator=iterator)
-            output = list(spider.parse(response))
+            output = list(spider._parse(response))
             self.assertEqual(len(output), 2, iterator)
             self.assertEqual(output, [
                 {'loc': [u'http://www.example.com/Special-Offers.html'],

tests/test_spiderloader/__init__.py

@@ -20,13 +20,20 @@ from scrapy.crawler import CrawlerRunner
 module_dir = os.path.dirname(os.path.abspath(__file__))
 
 
+def _copytree(source, target):
+    try:
+        shutil.copytree(source, target)
+    except shutil.Error:
+        pass
+
+
 class SpiderLoaderTest(unittest.TestCase):
 
     def setUp(self):
         orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
         self.tmpdir = tempfile.mkdtemp()
         self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
-        shutil.copytree(orig_spiders_dir, self.spiders_dir)
+        _copytree(orig_spiders_dir, self.spiders_dir)
         sys.path.append(self.tmpdir)
         settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
         self.spider_loader = SpiderLoader.from_settings(settings)
@@ -124,7 +131,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
         self.tmpdir = self.mktemp()
         os.mkdir(self.tmpdir)
         self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
-        shutil.copytree(orig_spiders_dir, self.spiders_dir)
+        _copytree(orig_spiders_dir, self.spiders_dir)
         sys.path.append(self.tmpdir)
         self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
 
@@ -134,8 +141,8 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
 
     def test_dupename_warning(self):
         # copy 1 spider module so as to have duplicate spider name
-        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider3.py'),
-                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider3dupe.py'))
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3dupe.py'))
 
         with warnings.catch_warnings(record=True) as w:
             spider_loader = SpiderLoader.from_settings(self.settings)
@@ -156,10 +163,10 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
     def test_multiple_dupename_warning(self):
         # copy 2 spider modules so as to have duplicate spider name
         # This should issue 2 warning, 1 for each duplicate spider name
-        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider1.py'),
-                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider1dupe.py'))
-        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider2.py'),
-                        os.path.join(self.tmpdir, 'test_spiders_xxx/spider2dupe.py'))
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1dupe.py'))
+        shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2.py'),
+                        os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2dupe.py'))
 
         with warnings.catch_warnings(record=True) as w:
             spider_loader = SpiderLoader.from_settings(self.settings)

tests/test_spidermiddleware_referer.py

@@ -6,16 +6,28 @@ from scrapy.http import Response, Request
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
 from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
-from scrapy.spidermiddlewares.referer import RefererMiddleware, \
-    POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \
-    POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \
-    POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \
-    POLICY_STRICT_ORIGIN, POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, \
-    DefaultReferrerPolicy, \
-    NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \
-    OriginWhenCrossOriginPolicy, OriginPolicy, \
-    StrictOriginWhenCrossOriginPolicy, StrictOriginPolicy, \
-    SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy
+from scrapy.spidermiddlewares.referer import (
+    DefaultReferrerPolicy,
+    NoReferrerPolicy,
+    NoReferrerWhenDowngradePolicy,
+    OriginPolicy,
+    OriginWhenCrossOriginPolicy,
+    POLICY_NO_REFERRER,
+    POLICY_NO_REFERRER_WHEN_DOWNGRADE,
+    POLICY_ORIGIN,
+    POLICY_ORIGIN_WHEN_CROSS_ORIGIN,
+    POLICY_SAME_ORIGIN,
+    POLICY_SCRAPY_DEFAULT,
+    POLICY_STRICT_ORIGIN,
+    POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN,
+    POLICY_UNSAFE_URL,
+    RefererMiddleware,
+    ReferrerPolicy,
+    SameOriginPolicy,
+    StrictOriginPolicy,
+    StrictOriginWhenCrossOriginPolicy,
+    UnsafeUrlPolicy,
+)
 
 
 class TestRefererMiddleware(TestCase):

tests/test_utils_asyncio.py

@@ -1,4 +1,6 @@
-from unittest import TestCase
+import platform
+import sys
+from unittest import skipIf, TestCase
 
 from pytest import mark
 
@@ -12,6 +14,9 @@ class AsyncioTest(TestCase):
         # the result should depend only on the pytest --reactor argument
         self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_install_asyncio_reactor(self):
         # this should do nothing
         install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")

tests/test_utils_curl.py

@@ -29,8 +29,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
         self._test_command(curl_command, expected_result)
 
     def test_get_basic_auth(self):
-        curl_command = 'curl "https://api.test.com/" -u ' \
-                       '"some_username:some_password"'
+        curl_command = 'curl "https://api.test.com/" -u "some_username:some_password"'
         expected_result = {
             "method": "GET",
             "url": "https://api.test.com/",
@@ -212,8 +211,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
         with warnings.catch_warnings():  # avoid warning when executing tests
             warnings.simplefilter('ignore')
             curl_command = 'curl --bar --baz http://www.example.com'
-            expected_result = \
-                {"method": "GET", "url": "http://www.example.com"}
+            expected_result = {"method": "GET", "url": "http://www.example.com"}
             self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
 
         # case 2: ignore_unknown_options=False (raise exception):

tests/test_utils_defer.py

@@ -2,8 +2,13 @@ from twisted.trial import unittest
 from twisted.internet import reactor, defer
 from twisted.python.failure import Failure
 
-from scrapy.utils.defer import mustbe_deferred, process_chain, \
-    process_chain_both, process_parallel, iter_errback
+from scrapy.utils.defer import (
+    iter_errback,
+    mustbe_deferred,
+    process_chain,
+    process_chain_both,
+    process_parallel,
+)
 
 
 class MustbeDeferredTest(unittest.TestCase):

tests/test_utils_display.py

@@ -0,0 +1,78 @@
+from io import StringIO
+
+from unittest import mock, TestCase
+
+from scrapy.utils.display import pformat, pprint
+
+
+class TestDisplay(TestCase):
+    object = {'a': 1}
+    colorized_string = (
+        "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"
+        "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}\n"
+    )
+    plain_string = "{'a': 1}"
+
+    @mock.patch('sys.platform', 'linux')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat(self, isatty):
+        isatty.return_value = True
+        self.assertEqual(pformat(self.object), self.colorized_string)
+
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_dont_colorize(self, isatty):
+        isatty.return_value = True
+        self.assertEqual(pformat(self.object, colorize=False), self.plain_string)
+
+    def test_pformat_not_tty(self):
+        self.assertEqual(pformat(self.object), self.plain_string)
+
+    @mock.patch('sys.platform', 'win32')
+    @mock.patch('platform.version')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_old_windows(self, isatty, version):
+        isatty.return_value = True
+        version.return_value = '10.0.14392'
+        self.assertEqual(pformat(self.object), self.colorized_string)
+
+    @mock.patch('sys.platform', 'win32')
+    @mock.patch('scrapy.utils.display._enable_windows_terminal_processing')
+    @mock.patch('platform.version')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_windows_no_terminal_processing(self, isatty, version, terminal_processing):
+        isatty.return_value = True
+        version.return_value = '10.0.14393'
+        terminal_processing.return_value = False
+        self.assertEqual(pformat(self.object), self.plain_string)
+
+    @mock.patch('sys.platform', 'win32')
+    @mock.patch('scrapy.utils.display._enable_windows_terminal_processing')
+    @mock.patch('platform.version')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_windows(self, isatty, version, terminal_processing):
+        isatty.return_value = True
+        version.return_value = '10.0.14393'
+        terminal_processing.return_value = True
+        self.assertEqual(pformat(self.object), self.colorized_string)
+
+    @mock.patch('sys.platform', 'linux')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_no_pygments(self, isatty):
+        isatty.return_value = True
+
+        import builtins
+        real_import = builtins.__import__
+
+        def mock_import(name, globals, locals, fromlist, level):
+            if 'pygments' in name:
+                raise ImportError
+            return real_import(name, globals, locals, fromlist, level)
+
+        builtins.__import__ = mock_import
+        self.assertEqual(pformat(self.object), self.plain_string)
+        builtins.__import__ = real_import
+
+    def test_pprint(self):
+        with mock.patch('sys.stdout', new=StringIO()) as mock_out:
+            pprint(self.object)
+            self.assertEqual(mock_out.getvalue(), "{'a': 1}\n")

tests/test_utils_iterators.py

@@ -7,7 +7,7 @@ from scrapy.http import XmlResponse, TextResponse, Response
 from tests import get_testdata
 
 
-FOOBAR_NL = u"foo\nbar"
+FOOBAR_NL = "foo{}bar".format(os.linesep)
 
 
 class XmliterTestCase(unittest.TestCase):
@@ -15,18 +15,20 @@ class XmliterTestCase(unittest.TestCase):
     xmliter = staticmethod(xmliter)
 
     def test_xmliter(self):
-        body = b"""<?xml version="1.0" encoding="UTF-8"?>\
+        body = b"""
+            <?xml version="1.0" encoding="UTF-8"?>
             <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-                      xsi:noNamespaceSchemaLocation="someschmea.xsd">\
-              <product id="001">\
-                <type>Type 1</type>\
-                <name>Name 1</name>\
-              </product>\
-              <product id="002">\
-                <type>Type 2</type>\
-                <name>Name 2</name>\
-              </product>\
-            </products>"""
+                      xsi:noNamespaceSchemaLocation="someschmea.xsd">
+              <product id="001">
+                <type>Type 1</type>
+                <name>Name 1</name>
+              </product>
+              <product id="002">
+                <type>Type 2</type>
+                <name>Name 2</name>
+              </product>
+            </products>
+        """
 
         response = XmlResponse(url="http://example.com", body=body)
         attrs = []
@@ -115,7 +117,7 @@ class XmliterTestCase(unittest.TestCase):
                          [[u'one'], [u'two']])
 
     def test_xmliter_namespaces(self):
-        body = b"""\
+        body = b"""
             <?xml version="1.0" encoding="UTF-8"?>
             <rss version="2.0" xmlns:g="http://base.google.com/ns/1.0">
                 <channel>
@@ -185,7 +187,7 @@ class LxmlXmliterTestCase(XmliterTestCase):
     xmliter = staticmethod(xmliter_lxml)
 
     def test_xmliter_iterate_namespace(self):
-        body = b"""\
+        body = b"""
             <?xml version="1.0" encoding="UTF-8"?>
             <rss version="2.0" xmlns="http://base.google.com/ns/1.0">
                 <channel>
@@ -214,7 +216,7 @@ class LxmlXmliterTestCase(XmliterTestCase):
         self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg'])
 
     def test_xmliter_namespaces_prefix(self):
-        body = b"""\
+        body = b"""
         <?xml version="1.0" encoding="UTF-8"?>
         <root>
             <h:table xmlns:h="http://www.w3.org/TR/html4/">

tests/test_utils_python.py

@@ -4,6 +4,7 @@ import operator
 import platform
 import unittest
 from itertools import count
+from sys import version_info
 from warnings import catch_warnings
 
 from scrapy.utils.python import (
@@ -214,9 +215,12 @@ class UtilsPythonTestCase(unittest.TestCase):
         else:
             self.assertEqual(
                 get_func_args(str.split, stripself=True), ['sep', 'maxsplit'])
-            self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
             self.assertEqual(
                 get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
+            if version_info < (3, 6):
+                self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
+            else:
+                self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])
 
     def test_without_none_values(self):
         self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])

tests/test_utils_request.py

@@ -1,7 +1,11 @@
 import unittest
 from scrapy.http import Request
-from scrapy.utils.request import request_fingerprint, _fingerprint_cache, \
-    request_authenticate, request_httprepr
+from scrapy.utils.request import (
+    _fingerprint_cache,
+    request_authenticate,
+    request_fingerprint,
+    request_httprepr,
+)
 
 
 class UtilsRequestTest(unittest.TestCase):

tests/test_utils_response.py

@@ -37,8 +37,7 @@ class ResponseUtilsTest(unittest.TestCase):
             self.assertIn(b'<base href="' + to_bytes(url) + b'">', bbody)
             return True
         response = HtmlResponse(url, body=body)
-        assert open_in_browser(response, _openfunc=browser_open), \
-            "Browser not called"
+        assert open_in_browser(response, _openfunc=browser_open), "Browser not called"
 
         resp = Response(url, body=body)
         self.assertRaises(TypeError, open_in_browser, resp, debug=True)

tests/test_utils_sitemap.py

@@ -156,8 +156,7 @@ Disallow: /forum/active/
 
     def test_sitemap_blanklines(self):
         """Assert we can deal with starting blank lines before <xml> tag"""
-        s = Sitemap(b"""\
-
+        s = Sitemap(b"""
 <?xml version="1.0" encoding="UTF-8"?>
 <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 

tests/test_utils_trackref.py

@@ -1,7 +1,10 @@
 import unittest
 from io import StringIO
+from time import sleep, time
 from unittest import mock
 
+from twisted.trial.unittest import SkipTest
+
 from scrapy.utils import trackref
 
 
@@ -55,7 +58,18 @@ Foo                                 1   oldest: 0s ago\n\n''')
 
     def test_get_oldest(self):
         o1 = Foo()  # NOQA
+
+        o1_time = time()
+
         o2 = Bar()  # NOQA
+
+        o3_time = time()
+        if o3_time <= o1_time:
+            sleep(0.01)
+            o3_time = time()
+        if o3_time <= o1_time:
+            raise SkipTest('time.time is not precise enough')
+
         o3 = Foo()  # NOQA
         self.assertIs(trackref.get_oldest('Foo'), o1)
         self.assertIs(trackref.get_oldest('Bar'), o2)

tests/test_utils_url.py

@@ -1,8 +1,14 @@
 import unittest
 
 from scrapy.spiders import Spider
-from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
-                              add_http_if_no_scheme, guess_scheme, strip_url)
+from scrapy.utils.url import (
+    add_http_if_no_scheme,
+    guess_scheme,
+    _is_filesystem_path,
+    strip_url,
+    url_is_from_any_domain,
+    url_is_from_spider,
+)
 
 
 __doctests__ = ['scrapy.utils.url']
@@ -207,8 +213,7 @@ def create_guess_scheme_t(args):
     def do_expected(self):
         url = guess_scheme(args[0])
         assert url.startswith(args[1]), \
-            'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (
-                args[0], url, args[1])
+            'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (args[0], url, args[1])
     return do_expected
 
 
@@ -434,5 +439,28 @@ class StripUrl(unittest.TestCase):
             self.assertEqual(strip_url(i, origin_only=True), o)
 
 
+class IsPathTestCase(unittest.TestCase):
+
+    def test_path(self):
+        for input_value, output_value in (
+            # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell
+            # Unix-like OS, Microsoft Windows / cmd.exe
+            ("/home/user/docs/Letter.txt", True),
+            ("./inthisdir", True),
+            ("../../greatgrandparent", True),
+            ("~/.rcinfo", True),
+            (r"C:\user\docs\Letter.txt", True),
+            ("/user/docs/Letter.txt", True),
+            (r"C:\Letter.txt", True),
+            (r"\\Server01\user\docs\Letter.txt", True),
+            (r"\\?\UNC\Server01\user\docs\Letter.txt", True),
+            (r"\\?\C:\user\docs\Letter.txt", True),
+            (r"C:\user\docs\somefile.ext:alternate_stream_name", True),
+
+            (r"https://example.com", False),
+        ):
+            self.assertEqual(_is_filesystem_path(input_value), output_value, input_value)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_webclient.py

@@ -356,9 +356,8 @@ class WebClientTestCase(unittest.TestCase):
         """ Test that non-standart body encoding matches
         Content-Encoding header """
         body = b'\xd0\x81\xd1\x8e\xd0\xaf'
-        return getPage(
-            self.getURL('encoding'), body=body, response_transform=lambda r: r)\
-            .addCallback(self._check_Encoding, body)
+        dfd = getPage(self.getURL('encoding'), body=body, response_transform=lambda r: r)
+        return dfd.addCallback(self._check_Encoding, body)
 
     def _check_Encoding(self, response, original_body):
         content_encoding = to_unicode(response.headers[b'Content-Encoding'])
@@ -414,7 +413,9 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
             self.getURL("payload"), body=s, contextFactory=client_context_factory
         ).addCallback(self.assertEqual, to_bytes(s))
 
-    def testPayloadDefaultCiphers(self):
+    def testPayloadDisabledCipher(self):
         s = "0123456789" * 10
-        d = getPage(self.getURL("payload"), body=s, contextFactory=ScrapyClientContextFactory())
+        settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'})
+        client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)
+        d = getPage(self.getURL("payload"), body=s, contextFactory=client_context_factory)
         return self.assertFailure(d, OpenSSL.SSL.Error)

tox.ini

@@ -58,19 +58,12 @@ deps =
 commands =
     pylint conftest.py docs extras scrapy setup.py tests
 
-[testenv:pypy3]
-basepython = pypy3
-commands =
-    py.test {posargs:--durations=10 docs scrapy tests}
-
-[testenv:pinned]
-basepython = python3
+[pinned]
 deps =
     -ctests/constraints.txt
     cryptography==2.0
     cssselect==0.9.1
     itemadapter==0.1.0
-    lxml==3.5.0
     parsel==1.5.0
     Protego==0.1.15
     PyDispatcher==2.0.5
@@ -83,14 +76,49 @@ deps =
     -rtests/requirements-py3.txt
     # Extras
     botocore==1.3.23
+    google-cloud-storage==1.29.0
     Pillow==3.4.2
 
+[testenv:pinned]
+deps =
+    {[pinned]deps}
+    lxml==3.5.0
+
+[testenv:windows-pinned]
+basepython = python3
+deps =
+    {[pinned]deps}
+    # First lxml version that includes a Windows wheel for Python 3.5, so we do
+    # not need to build lxml from sources in a CI Windows job:
+    lxml==3.8.0
+
 [testenv:extra-deps]
 deps =
     {[testenv]deps}
     reppy
     robotexclusionrulesparser
 
+[testenv:asyncio]
+commands =
+    {[testenv]commands} --reactor=asyncio
+
+[testenv:asyncio-pinned]
+commands = {[testenv:asyncio]commands}
+deps = {[testenv:pinned]deps}
+
+[testenv:pypy3]
+basepython = pypy3
+commands =
+    py.test {posargs:--durations=10 docs scrapy tests}
+
+[testenv:pypy3-pinned]
+basepython = {[testenv:pypy3]basepython}
+commands = {[testenv:pypy3]commands}
+deps =
+    {[pinned]deps}
+    lxml==4.0.0
+    PyPyDispatcher==2.1.0
+
 [docs]
 changedir = docs
 deps =
@@ -122,7 +150,3 @@ deps = {[docs]deps}
 setenv = {[docs]setenv}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
-
-[testenv:asyncio]
-commands =
-    {[testenv]commands} --reactor=asyncio