Merge remote-tracking branch 'origin/master' into asyncio-parse-asyncgen-proper-rebased

.github/workflows/tests-ubuntu.yml

@@ -20,7 +20,7 @@ jobs:
         - python-version: pypy3
           env:
             TOXENV: pypy3
-            PYPY_VERSION: 3.6-v7.3.1
+            PYPY_VERSION: 3.6-v7.3.3
 
         # pinned deps
         - python-version: 3.6.12

README.rst

@@ -1,3 +1,6 @@
+.. image:: /artwork/scrapy-logo.jpg
+   :width: 400px
+   
 ======
 Scrapy
 ======

docs/faq.rst

@@ -350,6 +350,7 @@ I'm scraping a XML document and my XPath selector doesn't return any items
 
 You may need to remove namespaces. See :ref:`removing-namespaces`.
 
+
 .. _faq-split-item:
 
 How to split an item into multiple items in an item pipeline?
@@ -406,8 +407,19 @@ or :class:`~scrapy.signals.headers_received` signals and raising a
 :ref:`topics-stop-response-download` topic for additional information and examples.
 
 
+Running ``runspider`` I get ``error: No spider found in file: <filename>``
+--------------------------------------------------------------------------
+
+This may happen if your Scrapy project has a spider module with a name that
+conflicts with the name of one of the `Python standard library modules`_, such
+as ``csv.py`` or ``os.py``, or any `Python package`_ that you have installed.
+See :issue:`2680`.
+
+
 .. _has been reported: https://github.com/scrapy/scrapy/issues/2905
+.. _Python standard library modules: https://docs.python.org/py-modindex.html
+.. _Python package: https://pypi.org/
 .. _user agents: https://en.wikipedia.org/wiki/User_agent
 .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
 .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search
-.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search
+.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search

docs/news.rst

@@ -3,6 +3,44 @@
 Release notes
 =============
 
+.. _release-2.5.1:
+
+Scrapy 2.5.1 (2021-10-05)
+-------------------------
+
+*   **Security bug fix:**
+
+    If you use
+    :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`
+    (i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP
+    authentication, any request exposes your credentials to the request target.
+
+    To prevent unintended exposure of authentication credentials to unintended
+    domains, you must now additionally set a new, additional spider attribute,
+    ``http_auth_domain``, and point it to the specific domain to which the
+    authentication credentials must be sent.
+
+    If the ``http_auth_domain`` spider attribute is not set, the domain of the
+    first request will be considered the HTTP authentication target, and
+    authentication credentials will only be sent in requests targeting that
+    domain.
+
+    If you need to send the same HTTP authentication credentials to multiple
+    domains, you can use :func:`w3lib.http.basic_auth_header` instead to
+    set the value of the ``Authorization`` header of your requests.
+
+    If you *really* want your spider to send the same HTTP authentication
+    credentials to any domain, set the ``http_auth_domain`` spider attribute
+    to ``None``.
+
+    Finally, if you are a user of `scrapy-splash`_, know that this version of
+    Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will
+    need to upgrade scrapy-splash to a greater version for it to continue to
+    work.
+
+.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
+
+
 .. _release-2.5.0:
 
 Scrapy 2.5.0 (2021-04-06)
@@ -1454,6 +1492,44 @@ affect subclasses:
 (:issue:`3884`)
 
 
+.. _release-1.8.1:
+
+Scrapy 1.8.1 (2021-10-05)
+-------------------------
+
+*   **Security bug fix:**
+
+    If you use
+    :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`
+    (i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP
+    authentication, any request exposes your credentials to the request target.
+
+    To prevent unintended exposure of authentication credentials to unintended
+    domains, you must now additionally set a new, additional spider attribute,
+    ``http_auth_domain``, and point it to the specific domain to which the
+    authentication credentials must be sent.
+
+    If the ``http_auth_domain`` spider attribute is not set, the domain of the
+    first request will be considered the HTTP authentication target, and
+    authentication credentials will only be sent in requests targeting that
+    domain.
+
+    If you need to send the same HTTP authentication credentials to multiple
+    domains, you can use :func:`w3lib.http.basic_auth_header` instead to
+    set the value of the ``Authorization`` header of your requests.
+
+    If you *really* want your spider to send the same HTTP authentication
+    credentials to any domain, set the ``http_auth_domain`` spider attribute
+    to ``None``.
+
+    Finally, if you are a user of `scrapy-splash`_, know that this version of
+    Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will
+    need to upgrade scrapy-splash to a greater version for it to continue to
+    work.
+
+.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
+
+
 .. _release-1.8.0:
 
 Scrapy 1.8.0 (2019-10-28)

docs/topics/downloader-middleware.rst

@@ -323,8 +323,21 @@ HttpAuthMiddleware
     This middleware authenticates all requests generated from certain spiders
     using `Basic access authentication`_ (aka. HTTP auth).
 
-    To enable HTTP authentication from certain spiders, set the ``http_user``
-    and ``http_pass`` attributes of those spiders.
+    To enable HTTP authentication for a spider, set the ``http_user`` and
+    ``http_pass`` spider attributes to the authentication data and the
+    ``http_auth_domain`` spider attribute to the domain which requires this
+    authentication (its subdomains will be also handled in the same way).
+    You can set ``http_auth_domain`` to ``None`` to enable the
+    authentication for all requests but you risk leaking your authentication
+    credentials to unrelated domains.
+
+    .. warning::
+        In previous Scrapy versions HttpAuthMiddleware sent the authentication
+        data with all requests, which is a security problem if the spider
+        makes requests to several different domains. Currently if the
+        ``http_auth_domain`` attribute is not set, the middleware will use the
+        domain of the first request, which will work for some spiders but not
+        for others. In the future the middleware will produce an error instead.
 
     Example::
 
@@ -334,6 +347,7 @@ HttpAuthMiddleware
 
             http_user = 'someuser'
             http_pass = 'somepass'
+            http_auth_domain = 'intranet.example.com'
             name = 'intranet.example.com'
 
             # .. rest of the spider code omitted ...
@@ -1070,9 +1084,13 @@ In order to use this parser:
 
 * Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
 
+    .. warning:: `Upstream issue #122
+        <https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in Python 3.9+.
+
 * Set :setting:`ROBOTSTXT_PARSER` setting to
   ``scrapy.robotstxt.ReppyRobotParser``
 
+
 .. _rerp-parser:
 
 Robotexclusionrulesparser

docs/topics/feed-exports.rst

@@ -272,6 +272,7 @@ in multiple files, with the specified maximum item count per file. That way, as
 soon as a file reaches the maximum item count, that file is delivered to the
 feed URI, allowing item delivery to start way before the end of the crawl.
 
+
 .. _item-filter:
 
 Item filtering
@@ -312,6 +313,63 @@ ItemFilter
    :members:
 
 
+.. _post-processing:
+
+Post-Processing
+===============
+
+.. versionadded:: VERSION
+
+Scrapy provides an option to activate plugins to post-process feeds before they are exported
+to feed storages. In addition to using :ref:`builtin plugins <builtin-plugins>`, you
+can create your own :ref:`plugins <custom-plugins>`. 
+
+These plugins can be activated through the ``postprocessing`` option of a feed.
+The option must be passed a list of post-processing plugins in the order you want
+the feed to be processed. These plugins can be declared either as an import string
+or with the imported class of the plugin. Parameters to plugins can be passed
+through the feed options. See :ref:`feed options <feed-options>` for examples.
+
+.. _builtin-plugins:
+
+Built-in Plugins
+----------------
+
+.. autoclass:: scrapy.extensions.postprocessing.GzipPlugin
+
+.. autoclass:: scrapy.extensions.postprocessing.LZMAPlugin
+
+.. autoclass:: scrapy.extensions.postprocessing.Bz2Plugin
+
+.. _custom-plugins:
+
+Custom Plugins
+--------------
+
+Each plugin is a class that must implement the following methods:
+
+.. method:: __init__(self, file, feed_options)
+
+    Initialize the plugin.
+
+    :param file: file-like object having at least the `write`, `tell` and `close` methods implemented
+
+    :param feed_options: feed-specific :ref:`options <feed-options>`
+    :type feed_options: :class:`dict`
+
+.. method:: write(self, data)
+
+   Process and write `data` (:class:`bytes` or :class:`memoryview`) into the plugin's target file.
+   It must return number of bytes written.
+
+.. method:: close(self)
+
+    Close the target file object.
+
+To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You 
+can then access those parameters from the ``__init__`` method of your plugin.
+
+
 Settings
 ========
 
@@ -368,10 +426,12 @@ For instance::
             'encoding': 'latin1',
             'indent': 8,
         },
-        pathlib.Path('items.csv'): {
+        pathlib.Path('items.csv.gz'): {
             'format': 'csv',
             'fields': ['price', 'name'],
             'item_filter': 'myproject.filters.MyCustomFilter2',
+            'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],
+            'gzip_compresslevel': 5,
         },
     }
 
@@ -435,6 +495,11 @@ as a fallback value if that key is not provided for a specific feed definition:
 
 -   ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.
 
+-   ``postprocessing``: list of :ref:`plugins <post-processing>` to use for post-processing.
+
+    The plugins will be used in the order of the list passed.
+
+    .. versionadded:: VERSION
 
 .. setting:: FEED_EXPORT_ENCODING
 

docs/topics/jobs.rst

@@ -39,6 +39,8 @@ a signal), and resume it later by issuing the same command::
 
     scrapy crawl somespider -s JOBDIR=crawls/somespider-1
 
+.. _topics-keeping-persistent-state-between-batches:
+
 Keeping persistent state between batches
 ========================================
 

docs/topics/request-response.rst

@@ -300,7 +300,7 @@ errors if needed::
             "http://www.httpbin.org/status/404",    # Not found error
             "http://www.httpbin.org/status/500",    # server issue
             "http://www.httpbin.org:12345/",        # non-responding host, timeout expected
-            "http://www.httphttpbinbin.org/",       # DNS error expected
+            "https://example.invalid/",             # DNS error expected
         ]
 
         def start_requests(self):

docs/topics/signals.rst

@@ -413,7 +413,7 @@ headers_received
 .. versionadded:: 2.5
 
 .. signal:: headers_received
-.. function:: headers_received(headers, request, spider)
+.. function:: headers_received(headers, body_length, request, spider)
 
     Sent by the HTTP 1.1 and S3 download handlers when the response headers are
     available for a given request, before downloading any additional content.

docs/topics/spider-middleware.rst

@@ -129,8 +129,8 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         method (from a previous spider middleware) raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an
-        iterable of :class:`~scrapy.Request` objects and :ref:`item objects
-        <topics-items>`.
+        iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>`
+        objects.
 
         If it returns ``None``, Scrapy will continue processing this exception,
         executing any other :meth:`process_spider_exception` in the following

docs/topics/spiders.rst

@@ -122,6 +122,11 @@ scrapy.Spider
       send log messages through it as described on
       :ref:`topics-logging-from-spiders`.
 
+   .. attribute:: state
+
+      A dict you can use to persist some spider state between batches.
+      See :ref:`topics-keeping-persistent-state-between-batches` to know more about it.
+
    .. method:: from_crawler(crawler, *args, **kwargs)
 
        This is the class method used by Scrapy to create your spiders.

pylintrc

@@ -105,6 +105,7 @@ disable=abstract-method,
         unnecessary-lambda,
         unnecessary-pass,
         unreachable,
+        unspecified-encoding,
         unsubscriptable-object,
         unused-argument,
         unused-import,

scrapy/__init__.py

@@ -29,7 +29,7 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
 
 # Check minimum required Python version
 if sys.version_info < (3, 6):
-    print("Scrapy %s requires Python 3.6+" % __version__)
+    print(f"Scrapy {__version__} requires Python 3.6+")
     sys.exit(1)
 
 

scrapy/commands/genspider.py

@@ -4,6 +4,7 @@ import string
 
 from importlib import import_module
 from os.path import join, dirname, abspath, exists, splitext
+from urllib.parse import urlparse
 
 import scrapy
 from scrapy.commands import ScrapyCommand
@@ -22,6 +23,14 @@ def sanitize_module_name(module_name):
     return module_name
 
 
+def extract_domain(url):
+    """Extract domain name from URL string"""
+    o = urlparse(url)
+    if o.scheme == '' and o.netloc == '':
+        o = urlparse("//" + url.lstrip("/"))
+    return o.netloc
+
+
 class Command(ScrapyCommand):
 
     requires_project = False
@@ -59,7 +68,8 @@ class Command(ScrapyCommand):
         if len(args) != 2:
             raise UsageError()
 
-        name, domain = args[0:2]
+        name, url = args[0:2]
+        domain = extract_domain(url)
         module = sanitize_module_name(name)
 
         if self.settings.get('BOT_NAME') == module:

scrapy/commands/startproject.py

@@ -1,7 +1,7 @@
 import re
 import os
 import string
-from importlib import import_module
+from importlib.util import find_spec
 from os.path import join, exists, abspath
 from shutil import ignore_patterns, move, copy2, copystat
 from stat import S_IWUSR as OWNER_WRITE_PERMISSION
@@ -42,11 +42,8 @@ class Command(ScrapyCommand):
 
     def _is_valid_name(self, project_name):
         def _module_exists(module_name):
-            try:
-                import_module(module_name)
-                return True
-            except ImportError:
-                return False
+            spec = find_spec(module_name)
+            return spec is not None and spec.loader is not None
 
         if not re.search(r'^[_a-zA-Z]\w*$', project_name):
             print('Error: Project names must begin with a letter and contain'

scrapy/core/downloader/contextfactory.py

@@ -135,11 +135,13 @@ def load_context_factory_from_settings(settings, crawler):
             settings=settings,
             crawler=crawler,
         )
-        msg = """
-            '%s' does not accept `method` argument (type OpenSSL.SSL method,\
-            e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
-            Please upgrade your context factory class to handle them or ignore them.""" % (
-            settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
+        msg = (
+            f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept "
+            "a `method` argument (type OpenSSL.SSL method, e.g. "
+            "OpenSSL.SSL.SSLv23_METHOD) and/or a `tls_verbose_logging` "
+            "argument and/or a `tls_ciphers` argument. Please, upgrade your "
+            "context factory class to handle them or ignore them."
+        )
         warnings.warn(msg)
 
     return context_factory

scrapy/core/downloader/tls.py

@@ -65,14 +65,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
                 verifyHostname(connection, self._hostnameASCII)
             except (CertificateError, VerificationError) as e:
                 logger.warning(
-                    'Remote certificate is not valid for hostname "{}"; {}'.format(
-                        self._hostnameASCII, e))
+                    'Remote certificate is not valid for hostname "%s"; %s',
+                    self._hostnameASCII, e)
 
             except ValueError as e:
                 logger.warning(
                     'Ignoring error while verifying certificate '
-                    'from host "{}" (exception: {})'.format(
-                        self._hostnameASCII, repr(e)))
+                    'from host "%s" (exception: %r)',
+                    self._hostnameASCII, e)
 
 
 DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')

scrapy/core/scraper.py

@@ -163,7 +163,7 @@ class Scraper:
             callback = result.request.callback or spider._parse
             warn_on_generator_with_return_value(spider, callback)
             dfd = defer_succeed(result)
-            dfd.addCallback(callback, **result.request.cb_kwargs)
+            dfd.addCallbacks(callback=callback, callbackKeywords=result.request.cb_kwargs)
         else:  # result is a Failure
             result.request = request
             warn_on_generator_with_return_value(spider, request.errback)

scrapy/downloadermiddlewares/cookies.py

@@ -80,8 +80,8 @@ class CookiesMiddleware:
                     logger.warning(msg.format(request, cookie, key))
                     return
                 continue
-            if isinstance(cookie[key], str):
-                decoded[key] = cookie[key]
+            if isinstance(cookie[key], (bool, float, int, str)):
+                decoded[key] = str(cookie[key])
             else:
                 try:
                     decoded[key] = cookie[key].decode("utf8")

scrapy/downloadermiddlewares/httpauth.py

@@ -3,10 +3,14 @@ HTTP basic auth downloader middleware
 
 See documentation in docs/topics/downloader-middleware.rst
 """
+import warnings
 
 from w3lib.http import basic_auth_header
 
 from scrapy import signals
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.url import url_is_from_any_domain
 
 
 class HttpAuthMiddleware:
@@ -24,8 +28,23 @@ class HttpAuthMiddleware:
         pwd = getattr(spider, 'http_pass', '')
         if usr or pwd:
             self.auth = basic_auth_header(usr, pwd)
+            if not hasattr(spider, 'http_auth_domain'):
+                warnings.warn('Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security '
+                              'problems if the spider makes requests to several different domains. http_auth_domain '
+                              'will be set to the domain of the first request, please set it to the correct value '
+                              'explicitly.',
+                              category=ScrapyDeprecationWarning)
+                self.domain_unset = True
+            else:
+                self.domain = spider.http_auth_domain
+                self.domain_unset = False
 
     def process_request(self, request, spider):
         auth = getattr(self, 'auth', None)
         if auth and b'Authorization' not in request.headers:
-            request.headers[b'Authorization'] = auth
+            domain = urlparse_cached(request).hostname
+            if self.domain_unset:
+                self.domain = domain
+                self.domain_unset = False
+            if not self.domain or url_is_from_any_domain(request.url, [self.domain]):
+                request.headers[b'Authorization'] = auth

scrapy/downloadermiddlewares/httpcache.py

@@ -70,7 +70,7 @@ class HttpCacheMiddleware:
             self.stats.inc_value('httpcache/miss', spider=spider)
             if self.ignore_missing:
                 self.stats.inc_value('httpcache/ignore', spider=spider)
-                raise IgnoreRequest("Ignored request not in cache: %s" % request)
+                raise IgnoreRequest(f"Ignored request not in cache: {request}")
             return None  # first time request
 
         # Return cached response only if not expired

scrapy/extensions/feedexport.py

@@ -20,6 +20,7 @@ from zope.interface import implementer, Interface
 
 from scrapy import signals
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
+from scrapy.extensions.postprocessing import PostProcessingManager
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.conf import feed_complete_default_values_from_settings
 from scrapy.utils.ftp import ftp_store_file
@@ -37,11 +38,10 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
         kwargs['feed_options'] = feed_options
     else:
         warnings.warn(
-            "{} does not support the 'feed_options' keyword argument. Add a "
+            f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a "
             "'feed_options' parameter to its signature to remove this "
             "warning. This parameter will become mandatory in a future "
-            "version of Scrapy."
-            .format(builder.__qualname__),
+            "version of Scrapy.",
             category=ScrapyDeprecationWarning
         )
     return builder(*preargs, uri, *args, **kwargs)
@@ -355,32 +355,28 @@ class FeedExporter:
             # properly closed.
             return defer.maybeDeferred(slot.storage.store, slot.file)
         slot.finish_exporting()
-        logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
-        log_args = {'format': slot.format,
-                    'itemcount': slot.itemcount,
-                    'uri': slot.uri}
+        logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
         d = defer.maybeDeferred(slot.storage.store, slot.file)
 
-        # Use `largs=log_args` to copy log_args into function's scope
-        # instead of using `log_args` from the outer scope
         d.addCallback(
-            self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__
+            self._handle_store_success, logmsg, spider, type(slot.storage).__name__
         )
         d.addErrback(
-            self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__
+            self._handle_store_error, logmsg, spider, type(slot.storage).__name__
         )
         return d
 
-    def _handle_store_error(self, f, largs, logfmt, spider, slot_type):
+    def _handle_store_error(self, f, logmsg, spider, slot_type):
         logger.error(
-            logfmt % "Error storing", largs,
+            "Error storing %s", logmsg,
             exc_info=failure_to_exc_info(f), extra={'spider': spider}
         )
         self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
 
-    def _handle_store_success(self, f, largs, logfmt, spider, slot_type):
+    def _handle_store_success(self, f, logmsg, spider, slot_type):
         logger.info(
-            logfmt % "Stored", largs, extra={'spider': spider}
+            "Stored %s", logmsg,
+            extra={'spider': spider}
         )
         self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}")
 
@@ -396,6 +392,9 @@ class FeedExporter:
         """
         storage = self._get_storage(uri, feed_options)
         file = storage.open(spider)
+        if "postprocessing" in feed_options:
+            file = PostProcessingManager(feed_options["postprocessing"], file, feed_options)
+
         exporter = self._get_exporter(
             file=file,
             format=feed_options['format'],
@@ -470,10 +469,10 @@ class FeedExporter:
         for uri_template, values in self.feeds.items():
             if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
                 logger.error(
-                    '%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT '
+                    '%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT '
                     'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: '
-                    'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
-                    ''.format(uri_template)
+                    'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count',
+                    uri_template
                 )
                 return False
         return True
@@ -522,7 +521,7 @@ class FeedExporter:
             instance = build_instance(feedcls)
             method_name = '__new__'
         if instance is None:
-            raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name))
+            raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None")
         return instance
 
     def _get_uri_params(self, spider, uri_params, slot=None):

scrapy/extensions/httpcache.py

@@ -226,7 +226,7 @@ class DbmCacheStorage:
         dbpath = os.path.join(self.cachedir, f'{spider.name}.db')
         self.db = self.dbmodule.open(dbpath, 'c')
 
-        logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
+        logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
 
     def close_spider(self, spider):
         self.db.close()
@@ -280,7 +280,7 @@ class FilesystemCacheStorage:
         self._open = gzip.open if self.use_gzip else open
 
     def open_spider(self, spider):
-        logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir},
+        logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
                      extra={'spider': spider})
 
     def close_spider(self, spider):

scrapy/extensions/postprocessing.py

@@ -0,0 +1,154 @@
+"""
+Extension for processing data before they are exported to feeds.
+"""
+from bz2 import BZ2File
+from gzip import GzipFile
+from io import IOBase
+from lzma import LZMAFile
+from typing import Any, BinaryIO, Dict, List
+
+from scrapy.utils.misc import load_object
+
+
+class GzipPlugin:
+    """
+    Compresses received data using `gzip <https://en.wikipedia.org/wiki/Gzip>`_.
+
+    Accepted ``feed_options`` parameters:
+
+    - `gzip_compresslevel`
+    - `gzip_mtime`
+    - `gzip_filename`
+
+    See :py:class:`gzip.GzipFile` for more info about parameters.
+    """
+
+    def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None:
+        self.file = file
+        self.feed_options = feed_options
+        compress_level = self.feed_options.get("gzip_compresslevel", 9)
+        mtime = self.feed_options.get("gzip_mtime")
+        filename = self.feed_options.get("gzip_filename")
+        self.gzipfile = GzipFile(fileobj=self.file, mode="wb", compresslevel=compress_level,
+                                 mtime=mtime, filename=filename)
+
+    def write(self, data: bytes) -> int:
+        return self.gzipfile.write(data)
+
+    def close(self) -> None:
+        self.gzipfile.close()
+        self.file.close()
+
+
+class Bz2Plugin:
+    """
+    Compresses received data using `bz2 <https://en.wikipedia.org/wiki/Bzip2>`_.
+
+    Accepted ``feed_options`` parameters:
+
+    - `bz2_compresslevel`
+
+    See :py:class:`bz2.BZ2File` for more info about parameters.
+    """
+
+    def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None:
+        self.file = file
+        self.feed_options = feed_options
+        compress_level = self.feed_options.get("bz2_compresslevel", 9)
+        self.bz2file = BZ2File(filename=self.file, mode="wb", compresslevel=compress_level)
+
+    def write(self, data: bytes) -> int:
+        return self.bz2file.write(data)
+
+    def close(self) -> None:
+        self.bz2file.close()
+        self.file.close()
+
+
+class LZMAPlugin:
+    """
+    Compresses received data using `lzma <https://en.wikipedia.org/wiki/Lempel–Ziv–Markov_chain_algorithm>`_.
+
+    Accepted ``feed_options`` parameters:
+
+    - `lzma_format`
+    - `lzma_check`
+    - `lzma_preset`
+    - `lzma_filters`
+
+    .. note::
+        ``lzma_filters`` cannot be used in pypy version 7.3.1 and older.
+
+    See :py:class:`lzma.LZMAFile` for more info about parameters.
+    """
+
+    def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None:
+        self.file = file
+        self.feed_options = feed_options
+
+        format = self.feed_options.get("lzma_format")
+        check = self.feed_options.get("lzma_check", -1)
+        preset = self.feed_options.get("lzma_preset")
+        filters = self.feed_options.get("lzma_filters")
+        self.lzmafile = LZMAFile(filename=self.file, mode="wb", format=format,
+                                 check=check, preset=preset, filters=filters)
+
+    def write(self, data: bytes) -> int:
+        return self.lzmafile.write(data)
+
+    def close(self) -> None:
+        self.lzmafile.close()
+        self.file.close()
+
+
+# io.IOBase is subclassed here, so that exporters can use the PostProcessingManager
+# instance as a file like writable object. This could be needed by some exporters
+# such as CsvItemExporter which wraps the feed storage with io.TextIOWrapper.
+class PostProcessingManager(IOBase):
+    """
+    This will manage and use declared plugins to process data in a
+    pipeline-ish way.
+    :param plugins: all the declared plugins for the feed
+    :type plugins: list
+    :param file: final target file where the processed data will be written
+    :type file: file like object
+    """
+
+    def __init__(self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any]) -> None:
+        self.plugins = self._load_plugins(plugins)
+        self.file = file
+        self.feed_options = feed_options
+        self.head_plugin = self._get_head_plugin()
+
+    def write(self, data: bytes) -> int:
+        """
+        Uses all the declared plugins to process data first, then writes
+        the processed data to target file.
+        :param data: data passed to be written to target file
+        :type data: bytes
+        :return: returns number of bytes written
+        :rtype: int
+        """
+        return self.head_plugin.write(data)
+
+    def tell(self) -> int:
+        return self.file.tell()
+
+    def close(self) -> None:
+        """
+        Close the target file along with all the plugins.
+        """
+        self.head_plugin.close()
+
+    def writable(self) -> bool:
+        return True
+
+    def _load_plugins(self, plugins: List[Any]) -> List[Any]:
+        plugins = [load_object(plugin) for plugin in plugins]
+        return plugins
+
+    def _get_head_plugin(self) -> Any:
+        prev = self.file
+        for plugin in self.plugins[::-1]:
+            prev = plugin(prev, self.feed_options)
+        return prev

scrapy/spiders/__init__.py

@@ -4,14 +4,12 @@ Base class for Scrapy spiders
 See documentation in docs/topics/spiders.rst
 """
 import logging
-import warnings
 from typing import Optional
 
 from scrapy import signals
 from scrapy.http import Request
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import url_is_from_spider
-from scrapy.utils.deprecate import method_is_overridden
 
 
 class Spider(object_ref):
@@ -57,34 +55,13 @@ class Spider(object_ref):
         crawler.signals.connect(self.close, signals.spider_closed)
 
     def start_requests(self):
-        cls = self.__class__
         if not self.start_urls and hasattr(self, 'start_url'):
             raise AttributeError(
                 "Crawling could not start: 'start_urls' not found "
                 "or empty (but found 'start_url' attribute instead, "
                 "did you miss an 's'?)")
-        if method_is_overridden(cls, Spider, 'make_requests_from_url'):
-            warnings.warn(
-                "Spider.make_requests_from_url method is deprecated; it "
-                "won't be called in future Scrapy releases. Please "
-                "override Spider.start_requests method instead "
-                f"(see {cls.__module__}.{cls.__name__}).",
-            )
-            for url in self.start_urls:
-                yield self.make_requests_from_url(url)
-        else:
-            for url in self.start_urls:
-                yield Request(url, dont_filter=True)
-
-    def make_requests_from_url(self, url):
-        """ This method is deprecated. """
-        warnings.warn(
-            "Spider.make_requests_from_url method is deprecated: "
-            "it will be removed and not be called by the default "
-            "Spider.start_requests method in future Scrapy releases. "
-            "Please override Spider.start_requests method instead."
-        )
-        return Request(url, dont_filter=True)
+        for url in self.start_urls:
+            yield Request(url, dont_filter=True)
 
     def _parse(self, response, **kwargs):
         return self.parse(response, **kwargs)

scrapy/utils/conf.py

@@ -121,7 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings):
     out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
     out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
     out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
-    out.setdefault("item_export_kwargs", dict())
+    out.setdefault("item_export_kwargs", {})
     if settings["FEED_EXPORT_INDENT"] is None:
         out.setdefault("indent", None)
     else:

scrapy/utils/deprecate.py

@@ -79,7 +79,7 @@ def create_deprecated_class(
         # for implementation details
         def __instancecheck__(cls, inst):
             return any(cls.__subclasscheck__(c)
-                       for c in {type(inst), inst.__class__})
+                       for c in (type(inst), inst.__class__))
 
         def __subclasscheck__(cls, sub):
             if cls is not DeprecatedClass.deprecated_class:

scrapy/utils/misc.py

@@ -50,7 +50,7 @@ def load_object(path):
             return path
         else:
             raise TypeError("Unexpected argument type, expected string "
-                            "or object, got: %s" % type(path))
+                            f"or object, got: {type(path)}")
 
     try:
         dot = path.rindex('.')

tests/keys/__init__.py

@@ -40,9 +40,9 @@ def generate_keys():
 
     subject = issuer = Name(
         [
-            NameAttribute(NameOID.COUNTRY_NAME, u"IE"),
-            NameAttribute(NameOID.ORGANIZATION_NAME, u"Scrapy"),
-            NameAttribute(NameOID.COMMON_NAME, u"localhost"),
+            NameAttribute(NameOID.COUNTRY_NAME, "IE"),
+            NameAttribute(NameOID.ORGANIZATION_NAME, "Scrapy"),
+            NameAttribute(NameOID.COMMON_NAME, "localhost"),
         ]
     )
     cert = (
@@ -54,7 +54,7 @@ def generate_keys():
         .not_valid_before(datetime.utcnow())
         .not_valid_after(datetime.utcnow() + timedelta(days=10))
         .add_extension(
-            SubjectAlternativeName([DNSName(u"localhost")]),
+            SubjectAlternativeName([DNSName("localhost")]),
             critical=False,
         )
         .sign(key, SHA256(), default_backend())

tests/spiders.py

@@ -87,7 +87,7 @@ class SimpleSpider(MetaSpider):
         self.start_urls = [url]
 
     def parse(self, response):
-        self.logger.info("Got response %d" % response.status)
+        self.logger.info(f"Got response {response.status}")
 
 
 class AsyncDefSpider(SimpleSpider):
@@ -96,7 +96,7 @@ class AsyncDefSpider(SimpleSpider):
 
     async def parse(self, response):
         await defer.succeed(42)
-        self.logger.info("Got response %d" % response.status)
+        self.logger.info(f"Got response {response.status}")
 
 
 class AsyncDefAsyncioSpider(SimpleSpider):
@@ -106,7 +106,7 @@ class AsyncDefAsyncioSpider(SimpleSpider):
     async def parse(self, response):
         await asyncio.sleep(0.2)
         status = await get_from_asyncio_queue(response.status)
-        self.logger.info("Got response %d" % status)
+        self.logger.info(f"Got response {status}")
 
 
 class AsyncDefAsyncioReturnSpider(SimpleSpider):
@@ -116,7 +116,7 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):
     async def parse(self, response):
         await asyncio.sleep(0.2)
         status = await get_from_asyncio_queue(response.status)
-        self.logger.info("Got response %d" % status)
+        self.logger.info(f"Got response {status}")
         return [{'id': 1}, {'id': 2}]
 
 
@@ -127,7 +127,7 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
     async def parse(self, response):
         await asyncio.sleep(0.1)
         status = await get_from_asyncio_queue(response.status)
-        self.logger.info("Got response %d" % status)
+        self.logger.info(f"Got response {status}")
         return {"foo": 42}
 
 
@@ -139,7 +139,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
         await asyncio.sleep(0.2)
         req_id = response.meta.get('req_id', 0)
         status = await get_from_asyncio_queue(response.status)
-        self.logger.info("Got response %d, req_id %d" % (status, req_id))
+        self.logger.info(f"Got response {status}, req_id {req_id}")
         if req_id > 0:
             return
         reqs = []
@@ -191,7 +191,7 @@ class AsyncDefAsyncioGenSpider(SimpleSpider):
     async def parse(self, response):
         await asyncio.sleep(0.2)
         yield {'foo': 42}
-        self.logger.info("Got response %d" % response.status)
+        self.logger.info(f"Got response {response.status}")
 
 
 class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
@@ -202,7 +202,7 @@ class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
         for i in range(10):
             await asyncio.sleep(0.1)
             yield {'foo': i}
-        self.logger.info("Got response %d" % response.status)
+        self.logger.info(f"Got response {response.status}")
 
 
 class AsyncDefAsyncioGenComplexSpider(SimpleSpider):

tests/test_closespider.py

@@ -41,7 +41,7 @@ class TestCloseSpider(TestCase):
         yield crawler.crawl(total=1000000, mockserver=self.mockserver)
         reason = crawler.spider.meta['close_reason']
         self.assertEqual(reason, 'closespider_errorcount')
-        key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__)
+        key = f'spider_exceptions/{crawler.spider.exception_cls.__name__}'
         errorcount = crawler.stats.get_value(key)
         self.assertTrue(errorcount >= close_on)
 

tests/test_commands.py

@@ -3,6 +3,7 @@ import json
 import optparse
 import os
 import platform
+import re
 import subprocess
 import sys
 import tempfile
@@ -71,9 +72,14 @@ class ProjectTest(unittest.TestCase):
 
     def proc(self, *new_args, **popen_kwargs):
         args = (sys.executable, '-m', 'scrapy.cmdline') + new_args
-        p = subprocess.Popen(args, cwd=self.cwd, env=self.env,
-                             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                             **popen_kwargs)
+        p = subprocess.Popen(
+            args,
+            cwd=popen_kwargs.pop('cwd', self.cwd),
+            env=self.env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            **popen_kwargs,
+        )
 
         def kill_proc():
             p.kill()
@@ -89,11 +95,23 @@ class ProjectTest(unittest.TestCase):
 
         return p, to_unicode(stdout), to_unicode(stderr)
 
+    def find_in_file(self, filename, regex):
+        """Find first pattern occurrence in file"""
+        pattern = re.compile(regex)
+        with open(filename, "r") as f:
+            for line in f:
+                match = pattern.search(line)
+                if match is not None:
+                    return match
+
 
 class StartprojectTest(ProjectTest):
 
     def test_startproject(self):
-        self.assertEqual(0, self.call('startproject', self.project_name))
+        p, out, err = self.proc('startproject', self.project_name)
+        print(out)
+        print(err, file=sys.stderr)
+        self.assertEqual(p.returncode, 0)
 
         assert exists(join(self.proj_path, 'scrapy.cfg'))
         assert exists(join(self.proj_path, 'testproject'))
@@ -128,6 +146,25 @@ class StartprojectTest(ProjectTest):
         self.assertEqual(2, self.call('startproject'))
         self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params'))
 
+    def test_existing_project_dir(self):
+        project_dir = mkdtemp()
+        project_name = self.project_name + '_existing'
+        project_path = os.path.join(project_dir, project_name)
+        os.mkdir(project_path)
+
+        p, out, err = self.proc('startproject', project_name, cwd=project_dir)
+        print(out)
+        print(err, file=sys.stderr)
+        self.assertEqual(p.returncode, 0)
+
+        assert exists(join(abspath(project_path), 'scrapy.cfg'))
+        assert exists(join(abspath(project_path), project_name))
+        assert exists(join(join(abspath(project_path), project_name), '__init__.py'))
+        assert exists(join(join(abspath(project_path), project_name), 'items.py'))
+        assert exists(join(join(abspath(project_path), project_name), 'pipelines.py'))
+        assert exists(join(join(abspath(project_path), project_name), 'settings.py'))
+        assert exists(join(join(abspath(project_path), project_name), 'spiders', '__init__.py'))
+
 
 def get_permissions_dict(path, renamings=None, ignore=None):
 
@@ -455,6 +492,26 @@ class GenspiderCommandTest(CommandTest):
     def test_same_filename_as_existing_spider_force(self):
         self.test_same_filename_as_existing_spider(force=True)
 
+    def test_url(self, url='test.com', domain="test.com"):
+        self.assertEqual(0, self.call('genspider', '--force', 'test_name', url))
+        self.assertEqual(domain,
+                         self.find_in_file(join(self.proj_mod_path,
+                                                'spiders', 'test_name.py'),
+                                           r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1))
+        self.assertEqual(f'http://{domain}/',
+                         self.find_in_file(join(self.proj_mod_path,
+                                                'spiders', 'test_name.py'),
+                                           r'start_urls\s*=\s*\[\'(.+)\'\]').group(1))
+
+    def test_url_schema(self):
+        self.test_url('http://test.com', 'test.com')
+
+    def test_url_path(self):
+        self.test_url('test.com/some/other/page', 'test.com')
+
+    def test_url_schema_path(self):
+        self.test_url('https://test.com/some/other/page', 'test.com')
+
 
 class GenspiderStandaloneCommandTest(ProjectTest):
 
@@ -651,7 +708,7 @@ class MySpider(scrapy.Spider):
         ])
         import asyncio
         loop = asyncio.new_event_loop()
-        self.assertIn("Using asyncio event loop: %s.%s" % (loop.__module__, loop.__class__.__name__), log)
+        self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log)
 
     def test_output(self):
         spider_code = """

tests/test_downloader_handlers_http2.py

@@ -219,7 +219,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
     certfile = 'keys/localhost.crt'
 
     scheme = 'https'
-    host = u'127.0.0.1'
+    host = '127.0.0.1'
 
     expected_http_proxy_request_body = b'/'
 
@@ -248,7 +248,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
             self.assertEqual(response.url, request.url)
             self.assertEqual(response.body, b'/')
 
-        http_proxy = '%s?noconnect' % self.getURL('')
+        http_proxy = f"{self.getURL('')}?noconnect"
         request = Request('https://example.com', meta={'proxy': http_proxy})
         with self.assertWarnsRegex(
             Warning,

tests/test_downloadermiddleware_cookies.py

@@ -347,3 +347,24 @@ class CookiesMiddlewareTest(TestCase):
         self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1')
         self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2')
         self.assertCookieValEqual(req3.headers['Cookie'], 'key=')
+
+    def test_primitive_type_cookies(self):
+        # Boolean
+        req1 = Request('http://example.org', cookies={'a': True})
+        assert self.mw.process_request(req1, self.spider) is None
+        self.assertCookieValEqual(req1.headers['Cookie'], b'a=True')
+
+        # Float
+        req2 = Request('http://example.org', cookies={'a': 9.5})
+        assert self.mw.process_request(req2, self.spider) is None
+        self.assertCookieValEqual(req2.headers['Cookie'], b'a=9.5')
+
+        # Integer
+        req3 = Request('http://example.org', cookies={'a': 10})
+        assert self.mw.process_request(req3, self.spider) is None
+        self.assertCookieValEqual(req3.headers['Cookie'], b'a=10')
+
+        # String
+        req4 = Request('http://example.org', cookies={'a': 'b'})
+        assert self.mw.process_request(req4, self.spider) is None
+        self.assertCookieValEqual(req4.headers['Cookie'], b'a=b')

tests/test_downloadermiddleware_httpauth.py

@@ -1,13 +1,60 @@
 import unittest
 
+from w3lib.http import basic_auth_header
+
 from scrapy.http import Request
 from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
 from scrapy.spiders import Spider
 
 
+class TestSpiderLegacy(Spider):
+    http_user = 'foo'
+    http_pass = 'bar'
+
+
 class TestSpider(Spider):
     http_user = 'foo'
     http_pass = 'bar'
+    http_auth_domain = 'example.com'
+
+
+class TestSpiderAny(Spider):
+    http_user = 'foo'
+    http_pass = 'bar'
+    http_auth_domain = None
+
+
+class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
+
+    def setUp(self):
+        self.spider = TestSpiderLegacy('foo')
+
+    def test_auth(self):
+        mw = HttpAuthMiddleware()
+        mw.spider_opened(self.spider)
+
+        # initial request, sets the domain and sends the header
+        req = Request('http://example.com/')
+        assert mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
+
+        # subsequent request to the same domain, should send the header
+        req = Request('http://example.com/')
+        assert mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
+
+        # subsequent request to a different domain, shouldn't send the header
+        req = Request('http://example-noauth.com/')
+        assert mw.process_request(req, self.spider) is None
+        self.assertNotIn('Authorization', req.headers)
+
+    def test_auth_already_set(self):
+        mw = HttpAuthMiddleware()
+        mw.spider_opened(self.spider)
+        req = Request('http://example.com/',
+                      headers=dict(Authorization='Digest 123'))
+        assert mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], b'Digest 123')
 
 
 class HttpAuthMiddlewareTest(unittest.TestCase):
@@ -20,13 +67,45 @@ class HttpAuthMiddlewareTest(unittest.TestCase):
     def tearDown(self):
         del self.mw
 
-    def test_auth(self):
-        req = Request('http://scrapytest.org/')
+    def test_no_auth(self):
+        req = Request('http://example-noauth.com/')
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEqual(req.headers['Authorization'], b'Basic Zm9vOmJhcg==')
+        self.assertNotIn('Authorization', req.headers)
+
+    def test_auth_domain(self):
+        req = Request('http://example.com/')
+        assert self.mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
+
+    def test_auth_subdomain(self):
+        req = Request('http://foo.example.com/')
+        assert self.mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
 
     def test_auth_already_set(self):
-        req = Request('http://scrapytest.org/',
+        req = Request('http://example.com/',
+                      headers=dict(Authorization='Digest 123'))
+        assert self.mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], b'Digest 123')
+
+
+class HttpAuthAnyMiddlewareTest(unittest.TestCase):
+
+    def setUp(self):
+        self.mw = HttpAuthMiddleware()
+        self.spider = TestSpiderAny('foo')
+        self.mw.spider_opened(self.spider)
+
+    def tearDown(self):
+        del self.mw
+
+    def test_auth(self):
+        req = Request('http://example.com/')
+        assert self.mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
+
+    def test_auth_already_set(self):
+        req = Request('http://example.com/',
                       headers=dict(Authorization='Digest 123'))
         assert self.mw.process_request(req, self.spider) is None
         self.assertEqual(req.headers['Authorization'], b'Digest 123')

tests/test_engine.py

@@ -152,7 +152,7 @@ class CrawlerRun:
         self.itemerror = []
         self.itemresp = []
         self.headers = {}
-        self.bytes = defaultdict(lambda: list())
+        self.bytes = defaultdict(list)
         self.signals_caught = {}
         self.spider_class = spider_class
 

tests/test_exporters.py

@@ -362,14 +362,14 @@ class CsvItemExporterTest(BaseItemExporterTest):
     def test_errors_default(self):
         with self.assertRaises(UnicodeEncodeError):
             self.assertExportResult(
-                item=dict(text=u'W\u0275\u200Brd'),
+                item=dict(text='W\u0275\u200Brd'),
                 expected=None,
                 encoding='windows-1251',
             )
 
     def test_errors_xmlcharrefreplace(self):
         self.assertExportResult(
-            item=dict(text=u'W\u0275\u200Brd'),
+            item=dict(text='W\u0275\u200Brd'),
             include_headers_line=False,
             expected='Wɵ​rd\r\n',
             encoding='windows-1251',

tests/test_feedexport.py

@@ -1,5 +1,8 @@
+import bz2
 import csv
+import gzip
 import json
+import lzma
 import os
 import random
 import shutil
@@ -1473,6 +1476,499 @@ class FeedExportTest(FeedExportTestBase):
             self.assertEqual(row['expected'], data[feed_options['format']])
 
 
+class FeedPostProcessedExportsTest(FeedExportTestBase):
+    __test__ = True
+
+    items = [{'foo': 'bar'}]
+    expected = b'foo\r\nbar\r\n'
+
+    class MyPlugin1:
+        def __init__(self, file, feed_options):
+            self.file = file
+            self.feed_options = feed_options
+            self.char = self.feed_options.get('plugin1_char', b'')
+
+        def write(self, data):
+            written_count = self.file.write(data)
+            written_count += self.file.write(self.char)
+            return written_count
+
+        def close(self):
+            self.file.close()
+
+    def _named_tempfile(self, name):
+        return os.path.join(self.temp_dir, name)
+
+    @defer.inlineCallbacks
+    def run_and_export(self, spider_cls, settings):
+        """ Run spider with specified settings; return exported data with filename. """
+
+        FEEDS = settings.get('FEEDS') or {}
+        settings['FEEDS'] = {
+            printf_escape(path_to_url(file_path)): feed_options
+            for file_path, feed_options in FEEDS.items()
+        }
+
+        content = {}
+        try:
+            with MockServer() as s:
+                runner = CrawlerRunner(Settings(settings))
+                spider_cls.start_urls = [s.url('/')]
+                yield runner.crawl(spider_cls)
+
+            for file_path, feed_options in FEEDS.items():
+                if not os.path.exists(str(file_path)):
+                    continue
+
+                with open(str(file_path), 'rb') as f:
+                    content[str(file_path)] = f.read()
+
+        finally:
+            for file_path in FEEDS.keys():
+                if not os.path.exists(str(file_path)):
+                    continue
+
+                os.remove(str(file_path))
+
+        return content
+
+    def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=''):
+        data_stream = BytesIO()
+        gzipf = gzip.GzipFile(fileobj=data_stream, filename=filename, mtime=mtime,
+                              compresslevel=compresslevel, mode="wb")
+        gzipf.write(data)
+        gzipf.close()
+        data_stream.seek(0)
+        return data_stream.read()
+
+    @defer.inlineCallbacks
+    def test_gzip_plugin(self):
+
+        filename = self._named_tempfile('gzip_file')
+
+        settings = {
+            'FEEDS': {
+                filename: {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+        try:
+            gzip.decompress(data[filename])
+        except OSError:
+            self.fail("Received invalid gzip data.")
+
+    @defer.inlineCallbacks
+    def test_gzip_plugin_compresslevel(self):
+
+        filename_to_compressed = {
+            self._named_tempfile('compresslevel_0'): self.get_gzip_compressed(self.expected, compresslevel=0),
+            self._named_tempfile('compresslevel_9'): self.get_gzip_compressed(self.expected, compresslevel=9),
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('compresslevel_0'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
+                    'gzip_compresslevel': 0,
+                    'gzip_mtime': 0,
+                    'gzip_filename': "",
+                },
+                self._named_tempfile('compresslevel_9'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
+                    'gzip_compresslevel': 9,
+                    'gzip_mtime': 0,
+                    'gzip_filename': "",
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, compressed in filename_to_compressed.items():
+            result = gzip.decompress(data[filename])
+            self.assertEqual(compressed, data[filename])
+            self.assertEqual(self.expected, result)
+
+    @defer.inlineCallbacks
+    def test_gzip_plugin_mtime(self):
+        filename_to_compressed = {
+            self._named_tempfile('mtime_123'): self.get_gzip_compressed(self.expected, mtime=123),
+            self._named_tempfile('mtime_123456789'): self.get_gzip_compressed(self.expected, mtime=123456789),
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('mtime_123'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
+                    'gzip_mtime': 123,
+                    'gzip_filename': "",
+                },
+                self._named_tempfile('mtime_123456789'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
+                    'gzip_mtime': 123456789,
+                    'gzip_filename': "",
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, compressed in filename_to_compressed.items():
+            result = gzip.decompress(data[filename])
+            self.assertEqual(compressed, data[filename])
+            self.assertEqual(self.expected, result)
+
+    @defer.inlineCallbacks
+    def test_gzip_plugin_filename(self):
+        filename_to_compressed = {
+            self._named_tempfile('filename_FILE1'): self.get_gzip_compressed(self.expected, filename="FILE1"),
+            self._named_tempfile('filename_FILE2'): self.get_gzip_compressed(self.expected, filename="FILE2"),
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('filename_FILE1'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
+                    'gzip_mtime': 0,
+                    'gzip_filename': "FILE1",
+                },
+                self._named_tempfile('filename_FILE2'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
+                    'gzip_mtime': 0,
+                    'gzip_filename': "FILE2",
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, compressed in filename_to_compressed.items():
+            result = gzip.decompress(data[filename])
+            self.assertEqual(compressed, data[filename])
+            self.assertEqual(self.expected, result)
+
+    @defer.inlineCallbacks
+    def test_lzma_plugin(self):
+
+        filename = self._named_tempfile('lzma_file')
+
+        settings = {
+            'FEEDS': {
+                filename: {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+        try:
+            lzma.decompress(data[filename])
+        except lzma.LZMAError:
+            self.fail("Received invalid lzma data.")
+
+    @defer.inlineCallbacks
+    def test_lzma_plugin_format(self):
+
+        filename_to_compressed = {
+            self._named_tempfile('format_FORMAT_XZ'): lzma.compress(self.expected, format=lzma.FORMAT_XZ),
+            self._named_tempfile('format_FORMAT_ALONE'): lzma.compress(self.expected, format=lzma.FORMAT_ALONE),
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('format_FORMAT_XZ'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
+                    'lzma_format': lzma.FORMAT_XZ,
+                },
+                self._named_tempfile('format_FORMAT_ALONE'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
+                    'lzma_format': lzma.FORMAT_ALONE,
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, compressed in filename_to_compressed.items():
+            result = lzma.decompress(data[filename])
+            self.assertEqual(compressed, data[filename])
+            self.assertEqual(self.expected, result)
+
+    @defer.inlineCallbacks
+    def test_lzma_plugin_check(self):
+
+        filename_to_compressed = {
+            self._named_tempfile('check_CHECK_NONE'): lzma.compress(self.expected, check=lzma.CHECK_NONE),
+            self._named_tempfile('check_CHECK_CRC256'): lzma.compress(self.expected, check=lzma.CHECK_SHA256),
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('check_CHECK_NONE'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
+                    'lzma_check': lzma.CHECK_NONE,
+                },
+                self._named_tempfile('check_CHECK_CRC256'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
+                    'lzma_check': lzma.CHECK_SHA256,
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, compressed in filename_to_compressed.items():
+            result = lzma.decompress(data[filename])
+            self.assertEqual(compressed, data[filename])
+            self.assertEqual(self.expected, result)
+
+    @defer.inlineCallbacks
+    def test_lzma_plugin_preset(self):
+
+        filename_to_compressed = {
+            self._named_tempfile('preset_PRESET_0'): lzma.compress(self.expected, preset=0),
+            self._named_tempfile('preset_PRESET_9'): lzma.compress(self.expected, preset=9),
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('preset_PRESET_0'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
+                    'lzma_preset': 0,
+                },
+                self._named_tempfile('preset_PRESET_9'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
+                    'lzma_preset': 9,
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, compressed in filename_to_compressed.items():
+            result = lzma.decompress(data[filename])
+            self.assertEqual(compressed, data[filename])
+            self.assertEqual(self.expected, result)
+
+    @defer.inlineCallbacks
+    def test_lzma_plugin_filters(self):
+        import sys
+        if "PyPy" in sys.version:
+            # https://foss.heptapod.net/pypy/pypy/-/issues/3527
+            raise unittest.SkipTest("lzma filters doesn't work in PyPy")
+
+        filters = [{'id': lzma.FILTER_LZMA2}]
+        compressed = lzma.compress(self.expected, filters=filters)
+        filename = self._named_tempfile('filters')
+
+        settings = {
+            'FEEDS': {
+                filename: {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
+                    'lzma_filters': filters,
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+        self.assertEqual(compressed, data[filename])
+        result = lzma.decompress(data[filename])
+        self.assertEqual(self.expected, result)
+
+    @defer.inlineCallbacks
+    def test_bz2_plugin(self):
+
+        filename = self._named_tempfile('bz2_file')
+
+        settings = {
+            'FEEDS': {
+                filename: {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'],
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+        try:
+            bz2.decompress(data[filename])
+        except OSError:
+            self.fail("Received invalid bz2 data.")
+
+    @defer.inlineCallbacks
+    def test_bz2_plugin_compresslevel(self):
+
+        filename_to_compressed = {
+            self._named_tempfile('compresslevel_1'): bz2.compress(self.expected, compresslevel=1),
+            self._named_tempfile('compresslevel_9'): bz2.compress(self.expected, compresslevel=9),
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('compresslevel_1'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'],
+                    'bz2_compresslevel': 1,
+                },
+                self._named_tempfile('compresslevel_9'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'],
+                    'bz2_compresslevel': 9,
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, compressed in filename_to_compressed.items():
+            result = bz2.decompress(data[filename])
+            self.assertEqual(compressed, data[filename])
+            self.assertEqual(self.expected, result)
+
+    @defer.inlineCallbacks
+    def test_custom_plugin(self):
+        filename = self._named_tempfile('csv_file')
+
+        settings = {
+            'FEEDS': {
+                filename: {
+                    'format': 'csv',
+                    'postprocessing': [self.MyPlugin1],
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+        self.assertEqual(self.expected, data[filename])
+
+    @defer.inlineCallbacks
+    def test_custom_plugin_with_parameter(self):
+
+        expected = b'foo\r\n\nbar\r\n\n'
+        filename = self._named_tempfile('newline')
+
+        settings = {
+            'FEEDS': {
+                filename: {
+                    'format': 'csv',
+                    'postprocessing': [self.MyPlugin1],
+                    'plugin1_char': b'\n'
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+        self.assertEqual(expected, data[filename])
+
+    @defer.inlineCallbacks
+    def test_custom_plugin_with_compression(self):
+
+        expected = b'foo\r\n\nbar\r\n\n'
+
+        filename_to_decompressor = {
+            self._named_tempfile('bz2'): bz2.decompress,
+            self._named_tempfile('lzma'): lzma.decompress,
+            self._named_tempfile('gzip'): gzip.decompress,
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('bz2'): {
+                    'format': 'csv',
+                    'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.Bz2Plugin'],
+                    'plugin1_char': b'\n',
+                },
+                self._named_tempfile('lzma'): {
+                    'format': 'csv',
+                    'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.LZMAPlugin'],
+                    'plugin1_char': b'\n',
+                },
+                self._named_tempfile('gzip'): {
+                    'format': 'csv',
+                    'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],
+                    'plugin1_char': b'\n',
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, decompressor in filename_to_decompressor.items():
+            result = decompressor(data[filename])
+            self.assertEqual(expected, result)
+
+    @defer.inlineCallbacks
+    def test_exports_compatibility_with_postproc(self):
+        import marshal
+        import pickle
+        filename_to_expected = {
+            self._named_tempfile('csv'): b'foo\r\nbar\r\n',
+            self._named_tempfile('json'): b'[\n{"foo": "bar"}\n]',
+            self._named_tempfile('jsonlines'): b'{"foo": "bar"}\n',
+            self._named_tempfile('xml'): b'<?xml version="1.0" encoding="utf-8"?>\n'
+                                         b'<items>\n<item><foo>bar</foo></item>\n</items>',
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('csv'): {
+                    'format': 'csv',
+                    'postprocessing': [self.MyPlugin1],
+                    # empty plugin to activate postprocessing.PostProcessingManager
+                },
+                self._named_tempfile('json'): {
+                    'format': 'json',
+                    'postprocessing': [self.MyPlugin1],
+                },
+                self._named_tempfile('jsonlines'): {
+                    'format': 'jsonlines',
+                    'postprocessing': [self.MyPlugin1],
+                },
+                self._named_tempfile('xml'): {
+                    'format': 'xml',
+                    'postprocessing': [self.MyPlugin1],
+                },
+                self._named_tempfile('marshal'): {
+                    'format': 'marshal',
+                    'postprocessing': [self.MyPlugin1],
+                },
+                self._named_tempfile('pickle'): {
+                    'format': 'pickle',
+                    'postprocessing': [self.MyPlugin1],
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, result in data.items():
+            if 'pickle' in filename:
+                expected, result = self.items[0], pickle.loads(result)
+            elif 'marshal' in filename:
+                expected, result = self.items[0], marshal.loads(result)
+            else:
+                expected = filename_to_expected[filename]
+            self.assertEqual(expected, result)
+
+
 class BatchDeliveriesTest(FeedExportTestBase):
     __test__ = True
     _file_mark = '_%(batch_time)s_#%(batch_id)02d_'

tests/test_http2_client_protocol.py

@@ -201,7 +201,7 @@ class Https2ClientProtocolTestCase(TestCase):
         self.site = Site(root, timeout=None)
 
         # Start server for testing
-        self.hostname = u'localhost'
+        self.hostname = 'localhost'
         context_factory = ssl_context_factory(self.key_file, self.certificate_file)
 
         server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname)

tests/test_http_request.py

@@ -1217,18 +1217,17 @@ class FormRequestTest(RequestTest):
                           response, formcss="input[name='abc']")
 
     def test_from_response_valid_form_methods(self):
-        body = """<form action="post.php" method="%s">
-            <input type="hidden" name="one" value="1">
-            </form>"""
+        form_methods = [[method, method] for method in self.request_class.valid_form_methods]
+        form_methods.append(['UNKNOWN', 'GET'])
 
-        for method in self.request_class.valid_form_methods:
-            response = _buildresponse(body % method)
+        for method, expected in form_methods:
+            response = _buildresponse(
+                f'<form action="post.php" method="{method}">'
+                '<input type="hidden" name="one" value="1">'
+                '</form>'
+            )
             r = self.request_class.from_response(response)
-            self.assertEqual(r.method, method)
-
-        response = _buildresponse(body % 'UNKNOWN')
-        r = self.request_class.from_response(response)
-        self.assertEqual(r.method, 'GET')
+            self.assertEqual(r.method, expected)
 
 
 def _buildresponse(body, **kwargs):

tests/test_loader_deprecated.py

@@ -703,7 +703,7 @@ class DeprecatedUtilityFunctionsTestCase(unittest.TestCase):
             return None
 
         with warnings.catch_warnings(record=True) as w:
-            wrap_loader_context(function, context=dict())
+            wrap_loader_context(function, context={})
 
             assert len(w) == 1
             assert issubclass(w[0].category, ScrapyDeprecationWarning)

tests/test_request_cb_kwargs.py

@@ -57,10 +57,10 @@ class KeywordArgumentsSpider(MockServerSpider):
         },
     }
 
-    checks = list()
+    checks = []
 
     def start_requests(self):
-        data = {'key': 'value', 'number': 123}
+        data = {'key': 'value', 'number': 123, 'callback': 'some_callback'}
         yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data)
         yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data)
         yield Request(self.mockserver.url('/general_without'), self.parse_general)
@@ -88,7 +88,8 @@ class KeywordArgumentsSpider(MockServerSpider):
         if response.url.endswith('/general_with'):
             self.checks.append(kwargs['key'] == 'value')
             self.checks.append(kwargs['number'] == 123)
-            self.crawler.stats.inc_value('boolean_checks', 2)
+            self.checks.append(kwargs['callback'] == 'some_callback')
+            self.crawler.stats.inc_value('boolean_checks', 3)
         elif response.url.endswith('/general_without'):
             self.checks.append(kwargs == {})
             self.crawler.stats.inc_value('boolean_checks')
@@ -104,13 +105,13 @@ class KeywordArgumentsSpider(MockServerSpider):
         self.checks.append(default == 99)
         self.crawler.stats.inc_value('boolean_checks', 4)
 
-    def parse_takes_less(self, response, key):
+    def parse_takes_less(self, response, key, callback):
         """
         Should raise
         TypeError: parse_takes_less() got an unexpected keyword argument 'number'
         """
 
-    def parse_takes_more(self, response, key, number, other):
+    def parse_takes_more(self, response, key, number, callback, other):
         """
         Should raise
         TypeError: parse_takes_more() missing 1 required positional argument: 'other'
@@ -161,11 +162,13 @@ class CallbackKeywordArgumentsTestCase(TestCase):
         self.assertTrue(
             str(exceptions['takes_less'].exc_info[1]).endswith(
                 "parse_takes_less() got an unexpected keyword argument 'number'"
-            )
+            ),
+            msg="Exception message: " + str(exceptions['takes_less'].exc_info[1]),
         )
         self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
         self.assertTrue(
             str(exceptions['takes_more'].exc_info[1]).endswith(
                 "parse_takes_more() missing 1 required positional argument: 'other'"
-            )
+            ),
+            msg="Exception message: " + str(exceptions['takes_more'].exc_info[1]),
         )

tests/test_scheduler.py

@@ -22,7 +22,7 @@ MockSlot = collections.namedtuple('MockSlot', ['active'])
 
 class MockDownloader:
     def __init__(self):
-        self.slots = dict()
+        self.slots = {}
 
     def _get_slot_key(self, request, spider):
         if Downloader.DOWNLOAD_SLOT in request.meta:
@@ -31,7 +31,7 @@ class MockDownloader:
         return urlparse_cached(request).hostname or ''
 
     def increment(self, slot_key):
-        slot = self.slots.setdefault(slot_key, MockSlot(active=list()))
+        slot = self.slots.setdefault(slot_key, MockSlot(active=[]))
         slot.active.append(1)
 
     def decrement(self, slot_key):
@@ -114,7 +114,7 @@ class BaseSchedulerInMemoryTester(SchedulerHandler):
         for url, priority in _PRIORITIES:
             self.scheduler.enqueue_request(Request(url, priority=priority))
 
-        priorities = list()
+        priorities = []
         while self.scheduler.has_pending_requests():
             priorities.append(self.scheduler.next_request().priority)
 
@@ -167,7 +167,7 @@ class BaseSchedulerOnDiskTester(SchedulerHandler):
         self.close_scheduler()
         self.create_scheduler()
 
-        priorities = list()
+        priorities = []
         while self.scheduler.has_pending_requests():
             priorities.append(self.scheduler.next_request().priority)
 
@@ -259,7 +259,7 @@ class DownloaderAwareSchedulerTestMixin:
             self.close_scheduler()
             self.create_scheduler()
 
-        dequeued_slots = list()
+        dequeued_slots = []
         requests = []
         downloader = self.mock_crawler.engine.downloader
         while self.scheduler.has_pending_requests():

tests/test_spider.py

@@ -584,39 +584,6 @@ class DeprecationTest(unittest.TestCase):
         assert issubclass(CrawlSpider, Spider)
         assert isinstance(CrawlSpider(name='foo'), Spider)
 
-    def test_make_requests_from_url_deprecated(self):
-        class MySpider4(Spider):
-            name = 'spider1'
-            start_urls = ['http://example.com']
-
-        class MySpider5(Spider):
-            name = 'spider2'
-            start_urls = ['http://example.com']
-
-            def make_requests_from_url(self, url):
-                return Request(url + "/foo", dont_filter=True)
-
-        with warnings.catch_warnings(record=True) as w:
-            # spider without overridden make_requests_from_url method
-            # doesn't issue a warning
-            spider1 = MySpider4()
-            self.assertEqual(len(list(spider1.start_requests())), 1)
-            self.assertEqual(len(w), 0)
-
-            # spider without overridden make_requests_from_url method
-            # should issue a warning when called directly
-            request = spider1.make_requests_from_url("http://www.example.com")
-            self.assertTrue(isinstance(request, Request))
-            self.assertEqual(len(w), 1)
-
-            # spider with overridden make_requests_from_url issues a warning,
-            # but the method still works
-            spider2 = MySpider5()
-            requests = list(spider2.start_requests())
-            self.assertEqual(len(requests), 1)
-            self.assertEqual(requests[0].url, 'http://example.com/foo')
-            self.assertEqual(len(w), 2)
-
 
 class NoParseMethodSpiderTest(unittest.TestCase):
 

tests/test_utils_conf.py

@@ -176,7 +176,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "store_empty": True,
             "uri_params": (1, 2, 3, 4),
             "batch_item_count": 2,
-            "item_export_kwargs": dict(),
+            "item_export_kwargs": {},
         })
 
     def test_feed_complete_default_values_from_settings_non_empty(self):
@@ -199,7 +199,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "store_empty": True,
             "uri_params": None,
             "batch_item_count": 2,
-            "item_export_kwargs": dict(),
+            "item_export_kwargs": {},
         })
 
 

tests/test_utils_misc/__init__.py

@@ -27,7 +27,7 @@ class UtilsMiscTestCase(unittest.TestCase):
     def test_load_object_exceptions(self):
         self.assertRaises(ImportError, load_object, 'nomodule999.mod.function')
         self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999')
-        self.assertRaises(TypeError, load_object, dict())
+        self.assertRaises(TypeError, load_object, {})
 
     def test_walk_modules(self):
         mods = walk_modules('tests.test_utils_misc.test_walk_modules')