Merge remote-tracking branch 'upstream/master' into ISSUE-4250-add_batch_deliveries

.travis.yml

@@ -11,23 +11,31 @@ matrix:
       python: 3.8
     - env: TOXENV=flake8
       python: 3.8
+    - env: TOXENV=pylint
+      python: 3.8
     - env: TOXENV=docs
       python: 3.7  # Keep in sync with .readthedocs.yml
 
     - env: TOXENV=pypy3
-    - python: 3.5
+    - env: TOXENV=py
+      python: 3.5
     - env: TOXENV=pinned
       python: 3.5
     - env: TOXENV=asyncio
       python: 3.5.2
-    - python: 3.6
-    - python: 3.7
-    - env: PYPI_RELEASE_JOB=true
+    - env: TOXENV=py
+      python: 3.6
+    - env: TOXENV=py
+      python: 3.7
+    - env: TOXENV=py PYPI_RELEASE_JOB=true
       python: 3.8
+      dist: bionic
     - env: TOXENV=extra-deps
       python: 3.8
+      dist: bionic
     - env: TOXENV=asyncio
       python: 3.8
+      dist: bionic
 install:
   - |
       if [ "$TOXENV" = "pypy3" ]; then

docs/conf.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-#
 # Scrapy documentation build configuration file, created by
 # sphinx-quickstart on Mon Nov 24 12:02:52 2008.
 #

docs/topics/dynamic-content.rst

@@ -184,6 +184,18 @@ data from it:
     >>> json.loads(json_data)
     {'field': 'value'}
 
+-   chompjs_ provides an API to parse JavaScript objects into a :class:`dict`.
+
+    For example, if the JavaScript code contains
+    ``var data = {field: "value", secondField: "second value"};``
+    you can extract that data as follows:
+
+    >>> import chompjs
+    >>> javascript = response.css('script::text').get()
+    >>> data = chompjs.parse_js_object(javascript)
+    >>> data
+    {'field': 'value', 'secondField': 'second value'}
+
 -   Otherwise, use js2xml_ to convert the JavaScript code into an XML document
     that you can parse using :ref:`selectors <topics-selectors>`.
 
@@ -241,6 +253,7 @@ along with `scrapy-selenium`_ for seamless integration.
 
 
 .. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
+.. _chompjs: https://github.com/Nykakin/chompjs
 .. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
 .. _curl: https://curl.haxx.se/
 .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser

docs/topics/feed-exports.rst

@@ -434,7 +434,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 .. setting:: FEED_STORAGE_BATCH_ITEM_COUNT
 
 FEED_STORAGE_BATCH_ITEM_COUNT
-----------------------
+-----------------------------
 Default: ``None``
 
 An integer number that represents the number of scraped items stored in each output

docs/topics/request-response.rst

@@ -834,11 +834,6 @@ TextResponse objects
 
     .. automethod:: TextResponse.follow_all
 
-    .. method:: TextResponse.body_as_unicode()
-
-        The same as :attr:`text`, but available as a method. This method is
-        kept for backward compatibility; please prefer ``response.text``.
-
 
 HtmlResponse objects
 --------------------

docs/topics/settings.rst

@@ -420,10 +420,9 @@ connections (for ``HTTP10DownloadHandler``).
 .. note::
 
     HTTP/1.0 is rarely used nowadays so you can safely ignore this setting,
-    unless you use Twisted<11.1, or if you really want to use HTTP/1.0
-    and override :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme
-    accordingly, i.e. to
-    ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
+    unless you really want to use HTTP/1.0 and override
+    :setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly,
+    i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
 
 .. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY
 
@@ -447,7 +446,6 @@ or even enable client-side authentication (and various other things).
     Scrapy also has another context factory class that you can set,
     ``'scrapy.core.downloader.contextfactory.BrowserLikeContextFactory'``,
     which uses the platform's certificates to validate remote endpoints.
-    **This is only available if you use Twisted>=14.0.**
 
 If you do use a custom ContextFactory, make sure its ``__init__`` method
 accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
@@ -494,10 +492,6 @@ This setting must be one of these string values:
 - ``'TLSv1.2'``: forces TLS version 1.2
 - ``'SSLv3'``: forces SSL version 3 (**not recommended**)
 
-.. note::
-
-    We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13
-    or above (Twisted>=14.0 if you can).
 
 .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
 
@@ -660,8 +654,6 @@ If you want to disable it set to 0.
     spider attribute and per-request using :reqmeta:`download_maxsize`
     Request.meta key.
 
-    This feature needs Twisted >= 11.1.
-
 .. setting:: DOWNLOAD_WARNSIZE
 
 DOWNLOAD_WARNSIZE
@@ -679,8 +671,6 @@ If you want to disable it set to 0.
     spider attribute and per-request using :reqmeta:`download_warnsize`
     Request.meta key.
 
-    This feature needs Twisted >= 11.1.
-
 .. setting:: DOWNLOAD_FAIL_ON_DATALOSS
 
 DOWNLOAD_FAIL_ON_DATALOSS

docs/topics/signals.rst

@@ -112,7 +112,7 @@ engine_started
 
     Sent when the Scrapy engine has started crawling.
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
 .. note:: This signal may be fired *after* the :signal:`spider_opened` signal,
     depending on how the spider was started. So **don't** rely on this signal
@@ -127,7 +127,7 @@ engine_stopped
     Sent when the Scrapy engine is stopped (for example, when a crawling
     process has finished).
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
 Item signals
 ------------
@@ -149,7 +149,7 @@ item_scraped
     Sent when an item has been scraped, after it has passed all the
     :ref:`topics-item-pipeline` stages (without being dropped).
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
     :param item: the item scraped
     :type item: dict or :class:`~scrapy.item.Item` object
@@ -169,7 +169,7 @@ item_dropped
     Sent after an item has been dropped from the :ref:`topics-item-pipeline`
     when some stage raised a :exc:`~scrapy.exceptions.DropItem` exception.
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
     :param item: the item dropped from the :ref:`topics-item-pipeline`
     :type item: dict or :class:`~scrapy.item.Item` object
@@ -194,7 +194,7 @@ item_error
     Sent when a :ref:`topics-item-pipeline` generates an error (i.e. raises
     an exception), except :exc:`~scrapy.exceptions.DropItem` exception.
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
     :param item: the item dropped from the :ref:`topics-item-pipeline`
     :type item: dict or :class:`~scrapy.item.Item` object
@@ -220,7 +220,7 @@ spider_closed
     Sent after a spider has been closed. This can be used to release per-spider
     resources reserved on :signal:`spider_opened`.
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
     :param spider: the spider which has been closed
     :type spider: :class:`~scrapy.spiders.Spider` object
@@ -244,7 +244,7 @@ spider_opened
     reserve per-spider resources, but can be used for any task that needs to be
     performed when a spider is opened.
 
-    This signal supports returning deferreds from their handlers.
+    This signal supports returning deferreds from its handlers.
 
     :param spider: the spider which has been opened
     :type spider: :class:`~scrapy.spiders.Spider` object
@@ -268,7 +268,7 @@ spider_idle
     You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to
     prevent the spider from being closed.
 
-    This signal does not support returning deferreds from their handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param spider: the spider which has gone idle
     :type spider: :class:`~scrapy.spiders.Spider` object
@@ -287,7 +287,7 @@ spider_error
 
     Sent when a spider callback generates an error (i.e. raises an exception).
 
-    This signal does not support returning deferreds from their handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param failure: the exception raised
     :type failure: twisted.python.failure.Failure
@@ -310,7 +310,7 @@ request_scheduled
     Sent when the engine schedules a :class:`~scrapy.http.Request`, to be
     downloaded later.
 
-    The signal does not support returning deferreds from their handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached the scheduler
     :type request: :class:`~scrapy.http.Request` object
@@ -327,7 +327,7 @@ request_dropped
     Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be
     downloaded later, is rejected by the scheduler.
 
-    The signal does not support returning deferreds from their handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached the scheduler
     :type request: :class:`~scrapy.http.Request` object
@@ -343,7 +343,7 @@ request_reached_downloader
 
     Sent when a :class:`~scrapy.http.Request` reached downloader.
 
-    The signal does not support returning deferreds from their handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached downloader
     :type request: :class:`~scrapy.http.Request` object
@@ -370,6 +370,29 @@ request_left_downloader
     :param spider: the spider that yielded the request
     :type spider: :class:`~scrapy.spiders.Spider` object
 
+bytes_received
+~~~~~~~~~~~~~~
+
+.. signal:: bytes_received
+.. function:: bytes_received(data, request, spider)
+
+    Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is
+    received for a specific request. This signal might be fired multiple
+    times for the same request, with partial data each time. For instance,
+    a possible scenario for a 25 kb response would be two signals fired
+    with 10 kb of data, and a final one with 5 kb of data.
+
+    This signal does not support returning deferreds from its handlers.
+
+    :param data: the data received by the download handler
+    :type spider: :class:`bytes` object
+
+    :param request: the request that generated the response
+    :type request: :class:`~scrapy.http.Request` object
+
+    :param spider: the spider associated with the response
+    :type spider: :class:`~scrapy.spiders.Spider` object
+
 Response signals
 ----------------
 
@@ -382,7 +405,7 @@ response_received
     Sent when the engine receives a new :class:`~scrapy.http.Response` from the
     downloader.
 
-    This signal does not support returning deferreds from their handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param response: the response received
     :type response: :class:`~scrapy.http.Response` object
@@ -401,7 +424,7 @@ response_downloaded
 
     Sent by the downloader right after a ``HTTPResponse`` is downloaded.
 
-    This signal does not support returning deferreds from their handlers.
+    This signal does not support returning deferreds from its handlers.
 
     :param response: the response downloaded
     :type response: :class:`~scrapy.http.Response` object

docs/utils/linkfix.py

@@ -14,50 +14,57 @@ Author: dufferzafar
 
 import re
 
-# Used for remembering the file (and its contents)
-# so we don't have to open the same file again.
-_filename = None
-_contents = None
 
-# A regex that matches standard linkcheck output lines
-line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
+def main():
 
-# Read lines from the linkcheck output file
-try:
-    with open("build/linkcheck/output.txt") as out:
-        output_lines = out.readlines()
-except IOError:
-    print("linkcheck output not found; please run linkcheck first.")
-    exit(1)
+    # Used for remembering the file (and its contents)
+    # so we don't have to open the same file again.
+    _filename = None
+    _contents = None
 
-# For every line, fix the respective file
-for line in output_lines:
-    match = re.match(line_re, line)
+    # A regex that matches standard linkcheck output lines
+    line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
 
-    if match:
-        newfilename = match.group(1)
-        errortype = match.group(2)
+    # Read lines from the linkcheck output file
+    try:
+        with open("build/linkcheck/output.txt") as out:
+            output_lines = out.readlines()
+    except IOError:
+        print("linkcheck output not found; please run linkcheck first.")
+        exit(1)
 
-        # Broken links can't be fixed and
-        # I am not sure what do with the local ones.
-        if errortype.lower() in ["broken", "local"]:
-            print("Not Fixed: " + line)
+    # For every line, fix the respective file
+    for line in output_lines:
+        match = re.match(line_re, line)
+
+        if match:
+            newfilename = match.group(1)
+            errortype = match.group(2)
+
+            # Broken links can't be fixed and
+            # I am not sure what do with the local ones.
+            if errortype.lower() in ["broken", "local"]:
+                print("Not Fixed: " + line)
+            else:
+                # If this is a new file
+                if newfilename != _filename:
+
+                    # Update the previous file
+                    if _filename:
+                        with open(_filename, "w") as _file:
+                            _file.write(_contents)
+
+                    _filename = newfilename
+
+                    # Read the new file to memory
+                    with open(_filename) as _file:
+                        _contents = _file.read()
+
+                _contents = _contents.replace(match.group(3), match.group(4))
         else:
-            # If this is a new file
-            if newfilename != _filename:
+            # We don't understand what the current line means!
+            print("Not Understood: " + line)
 
-                # Update the previous file
-                if _filename:
-                    with open(_filename, "w") as _file:
-                        _file.write(_contents)
 
-                _filename = newfilename
-
-                # Read the new file to memory
-                with open(_filename) as _file:
-                    _contents = _file.read()
-
-            _contents = _contents.replace(match.group(3), match.group(4))
-    else:
-        # We don't understand what the current line means!
-        print("Not Understood: " + line)
+if __name__ == '__main__':
+    main()

pylintrc

@@ -0,0 +1,113 @@
+[MASTER]
+persistent=no
+jobs=1  # >1 hides results
+
+[MESSAGES CONTROL]
+disable=abstract-method,
+        anomalous-backslash-in-string,
+        arguments-differ,
+        attribute-defined-outside-init,
+        bad-classmethod-argument,
+        bad-continuation,
+        bad-indentation,
+        bad-mcs-classmethod-argument,
+        bad-super-call,
+        bad-whitespace,
+        bare-except,
+        blacklisted-name,
+        broad-except,
+        c-extension-no-member,
+        catching-non-exception,
+        cell-var-from-loop,
+        comparison-with-callable,
+        consider-iterating-dictionary,
+        consider-using-in,
+        consider-using-set-comprehension,
+        consider-using-sys-exit,
+        cyclic-import,
+        dangerous-default-value,
+        deprecated-method,
+        deprecated-module,
+        duplicate-code,  # https://github.com/PyCQA/pylint/issues/214
+        eval-used,
+        expression-not-assigned,
+        fixme,
+        function-redefined,
+        global-statement,
+        import-error,
+        import-outside-toplevel,
+        import-self,
+        inconsistent-return-statements,
+        inherit-non-class,
+        invalid-name,
+        invalid-overridden-method,
+        isinstance-second-argument-not-valid-type,
+        keyword-arg-before-vararg,
+        line-too-long,
+        logging-format-interpolation,
+        logging-not-lazy,
+        lost-exception,
+        method-hidden,
+        misplaced-comparison-constant,
+        missing-docstring,
+        missing-final-newline,
+        multiple-imports,
+        multiple-statements,
+        no-else-continue,
+        no-else-raise,
+        no-else-return,
+        no-init,
+        no-member,
+        no-method-argument,
+        no-name-in-module,
+        no-self-argument,
+        no-self-use,
+        no-value-for-parameter,
+        not-an-iterable,
+        not-callable,
+        pointless-statement,
+        pointless-string-statement,
+        protected-access,
+        redefined-argument-from-local,
+        redefined-builtin,
+        redefined-outer-name,
+        reimported,
+        signature-differs,
+        singleton-comparison,
+        super-init-not-called,
+        superfluous-parens,
+        too-few-public-methods,
+        too-many-ancestors,
+        too-many-arguments,
+        too-many-branches,
+        too-many-format-args,
+        too-many-function-args,
+        too-many-instance-attributes,
+        too-many-lines,
+        too-many-locals,
+        too-many-public-methods,
+        too-many-return-statements,
+        trailing-newlines,
+        trailing-whitespace,
+        unbalanced-tuple-unpacking,
+        undefined-variable,
+        undefined-loop-variable,
+        unexpected-special-method-signature,
+        ungrouped-imports,
+        unidiomatic-typecheck,
+        unnecessary-comprehension,
+        unnecessary-lambda,
+        unnecessary-pass,
+        unreachable,
+        unsubscriptable-object,
+        unused-argument,
+        unused-import,
+        unused-variable,
+        unused-wildcard-import,
+        used-before-assignment,
+        useless-object-inheritance,  # Required for Python 2 support
+        useless-return,
+        useless-super-delegation,
+        wildcard-import,
+        wrong-import-order,
+        wrong-import-position

pytest.ini

@@ -35,82 +35,82 @@ flake8-ignore =
     scrapy/commands/check.py E501
     scrapy/commands/crawl.py E501
     scrapy/commands/edit.py E501
-    scrapy/commands/fetch.py E401 E501 E128
+    scrapy/commands/fetch.py E501 E128
     scrapy/commands/genspider.py E128 E501
     scrapy/commands/parse.py E128 E501
     scrapy/commands/runspider.py E501
     scrapy/commands/settings.py E128
     scrapy/commands/shell.py E128 E501
-    scrapy/commands/startproject.py E127 E501 E128
+    scrapy/commands/startproject.py E501 E128
     scrapy/commands/version.py E501 E128
     # scrapy/contracts
-    scrapy/contracts/__init__.py E501 W504
+    scrapy/contracts/__init__.py E501
     scrapy/contracts/default.py E128
     # scrapy/core
-    scrapy/core/engine.py E501 E128 E127
+    scrapy/core/engine.py E501 E128
     scrapy/core/scheduler.py E501
-    scrapy/core/scraper.py E501 E128 W504
-    scrapy/core/spidermw.py E501 E126
+    scrapy/core/scraper.py E501 E128
+    scrapy/core/spidermw.py E501
     scrapy/core/downloader/__init__.py E501
-    scrapy/core/downloader/contextfactory.py E501 E128 E126
+    scrapy/core/downloader/contextfactory.py E501 E128
     scrapy/core/downloader/middleware.py E501
     scrapy/core/downloader/tls.py E501
-    scrapy/core/downloader/webclient.py E501 E128 E126
+    scrapy/core/downloader/webclient.py E501 E128
     scrapy/core/downloader/handlers/__init__.py E501
-    scrapy/core/downloader/handlers/ftp.py E501 E128 E127
+    scrapy/core/downloader/handlers/ftp.py E501 E128
     scrapy/core/downloader/handlers/http10.py E501
     scrapy/core/downloader/handlers/http11.py E501
-    scrapy/core/downloader/handlers/s3.py E501 E128 E126
+    scrapy/core/downloader/handlers/s3.py E501 E128
     # scrapy/downloadermiddlewares
     scrapy/downloadermiddlewares/ajaxcrawl.py E501
     scrapy/downloadermiddlewares/decompression.py E501
     scrapy/downloadermiddlewares/defaultheaders.py E501
-    scrapy/downloadermiddlewares/httpcache.py E501 E126
+    scrapy/downloadermiddlewares/httpcache.py E501
     scrapy/downloadermiddlewares/httpcompression.py E501 E128
     scrapy/downloadermiddlewares/httpproxy.py E501
-    scrapy/downloadermiddlewares/redirect.py E501 W504
-    scrapy/downloadermiddlewares/retry.py E501 E126
+    scrapy/downloadermiddlewares/redirect.py E501
+    scrapy/downloadermiddlewares/retry.py E501
     scrapy/downloadermiddlewares/robotstxt.py E501
     scrapy/downloadermiddlewares/stats.py E501
     # scrapy/extensions
-    scrapy/extensions/closespider.py E501 E128 E123
+    scrapy/extensions/closespider.py E501 E128
     scrapy/extensions/corestats.py E501
     scrapy/extensions/feedexport.py E128 E501
     scrapy/extensions/httpcache.py E128 E501
     scrapy/extensions/memdebug.py E501
     scrapy/extensions/spiderstate.py E501
-    scrapy/extensions/telnet.py E501 W504
+    scrapy/extensions/telnet.py E501
     scrapy/extensions/throttle.py E501
     # scrapy/http
     scrapy/http/common.py E501
     scrapy/http/cookies.py E501
     scrapy/http/request/__init__.py E501
-    scrapy/http/request/form.py E501 E123
+    scrapy/http/request/form.py E501
     scrapy/http/request/json_request.py E501
     scrapy/http/response/__init__.py E501 E128
-    scrapy/http/response/text.py E501 E128 E124
+    scrapy/http/response/text.py E501 E128
     # scrapy/linkextractors
-    scrapy/linkextractors/__init__.py E501 E402 W504
+    scrapy/linkextractors/__init__.py E501 E402
     scrapy/linkextractors/lxmlhtml.py E501
     # scrapy/loader
     scrapy/loader/__init__.py E501 E128
     scrapy/loader/processors.py E501
     # scrapy/pipelines
     scrapy/pipelines/__init__.py E501
-    scrapy/pipelines/files.py E116 E501
+    scrapy/pipelines/files.py E501
     scrapy/pipelines/images.py E501
-    scrapy/pipelines/media.py E125 E501
+    scrapy/pipelines/media.py E501
     # scrapy/selector
     scrapy/selector/__init__.py F403
-    scrapy/selector/unified.py E501 E111
+    scrapy/selector/unified.py E501
     # scrapy/settings
     scrapy/settings/__init__.py E501
-    scrapy/settings/default_settings.py E501 E114 E116
+    scrapy/settings/default_settings.py E501
     scrapy/settings/deprecated.py E501
     # scrapy/spidermiddlewares
     scrapy/spidermiddlewares/httperror.py E501
     scrapy/spidermiddlewares/offsite.py E501
-    scrapy/spidermiddlewares/referer.py E501 E129 W504
+    scrapy/spidermiddlewares/referer.py E501
     scrapy/spidermiddlewares/urllength.py E501
     # scrapy/spiders
     scrapy/spiders/__init__.py E501 E402
@@ -124,8 +124,8 @@ flake8-ignore =
     scrapy/utils/datatypes.py E501
     scrapy/utils/decorators.py E501
     scrapy/utils/defer.py E501 E128
-    scrapy/utils/deprecate.py E128 E501 E127
-    scrapy/utils/gz.py E501 W504
+    scrapy/utils/deprecate.py E501
+    scrapy/utils/gz.py E501
     scrapy/utils/http.py F403
     scrapy/utils/httpobj.py E501
     scrapy/utils/iterators.py E501
@@ -137,7 +137,7 @@ flake8-ignore =
     scrapy/utils/python.py E501
     scrapy/utils/reactor.py E501
     scrapy/utils/reqser.py E501
-    scrapy/utils/request.py E127 E501
+    scrapy/utils/request.py E501
     scrapy/utils/response.py E501 E128
     scrapy/utils/signal.py E501 E128
     scrapy/utils/sitemap.py E501
@@ -164,87 +164,86 @@ flake8-ignore =
     scrapy/robotstxt.py E501
     scrapy/shell.py E501
     scrapy/signalmanager.py E501
-    scrapy/spiderloader.py F841 E501 E126
+    scrapy/spiderloader.py E501
     scrapy/squeues.py E128
+    scrapy/squeues.py E501
     scrapy/statscollectors.py E501
     # tests
     tests/__init__.py E402 E501
-    tests/mockserver.py E401 E501 E126 E123
-    tests/pipelines.py F841
-    tests/spiders.py E501 E127
-    tests/test_closespider.py E501 E127
+    tests/mockserver.py E501
+    tests/spiders.py E501
+    tests/test_closespider.py E501
     tests/test_command_fetch.py E501
     tests/test_command_parse.py E501 E128
     tests/test_command_shell.py E501 E128
     tests/test_commands.py E128 E501
     tests/test_contracts.py E501 E128
     tests/test_crawl.py E501 E741
-    tests/test_crawler.py F841 E501
-    tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E124 E127 E128 E501 E126 E123
+    tests/test_crawler.py E501
+    tests/test_dependencies.py E501
+    tests/test_downloader_handlers.py E128 E501
     tests/test_downloadermiddleware.py E501
     tests/test_downloadermiddleware_ajaxcrawlable.py E501
-    tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
-    tests/test_downloadermiddleware_decompression.py E127
+    tests/test_downloadermiddleware_cookies.py E741 E501 E128
     tests/test_downloadermiddleware_defaultheaders.py E501
     tests/test_downloadermiddleware_downloadtimeout.py E501
     tests/test_downloadermiddleware_httpcache.py E501
-    tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
+    tests/test_downloadermiddleware_httpcompression.py E501
+    tests/test_downloadermiddleware_decompression.py E501
     tests/test_downloadermiddleware_httpproxy.py E501 E128
-    tests/test_downloadermiddleware_redirect.py E501 E128 E127
-    tests/test_downloadermiddleware_retry.py E501 E128 E126
+    tests/test_downloadermiddleware_redirect.py E501 E128
+    tests/test_downloadermiddleware_retry.py E501 E128
     tests/test_downloadermiddleware_robotstxt.py E501
     tests/test_downloadermiddleware_stats.py E501
-    tests/test_dupefilters.py E501 E741 E128 E124
-    tests/test_engine.py E401 E501 E128
-    tests/test_exporters.py E501 E128 E124
-    tests/test_extension_telnet.py F841
-    tests/test_feedexport.py E501 F841
+    tests/test_dupefilters.py E501 E741 E128
+    tests/test_engine.py E501 E128
+    tests/test_exporters.py E501 E128
+    tests/test_feedexport.py E501
     tests/test_http_cookies.py E501
     tests/test_http_headers.py E501
-    tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
+    tests/test_http_request.py E402 E501 E128 E128
     tests/test_http_response.py E501 E128
-    tests/test_item.py E128 F841
+    tests/test_item.py E128
     tests/test_link.py E501
-    tests/test_linkextractors.py E501 E128 E124
-    tests/test_loader.py E501 E741 E128 E117
-    tests/test_logformatter.py E128 E501 E122
+    tests/test_linkextractors.py E501 E128
+    tests/test_loader.py E501 E741 E128
+    tests/test_logformatter.py E128 E501
     tests/test_mail.py E128 E501
     tests/test_middleware.py E501 E128
-    tests/test_pipeline_crawl.py E501 E128 E126
+    tests/test_pipeline_crawl.py E501 E128
     tests/test_pipeline_files.py E501
-    tests/test_pipeline_images.py F841 E501
+    tests/test_pipeline_images.py E501
     tests/test_pipeline_media.py E501 E741 E128
     tests/test_proxy_connect.py E501 E741
     tests/test_request_cb_kwargs.py E501
     tests/test_responsetypes.py E501
     tests/test_robotstxt_interface.py E501 E501
-    tests/test_scheduler.py E501 E126 E123
-    tests/test_selector.py E501 E127
+    tests/test_scheduler.py E501
+    tests/test_selector.py E501
     tests/test_spider.py E501
     tests/test_spidermiddleware.py E501
-    tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
-    tests/test_spidermiddleware_offsite.py E501 E128 E111
+    tests/test_spidermiddleware_httperror.py E128 E501
+    tests/test_spidermiddleware_offsite.py E501 E128
     tests/test_spidermiddleware_output_chain.py E501
-    tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121
+    tests/test_spidermiddleware_referer.py E501 E501
     tests/test_squeues.py E501 E741
     tests/test_utils_asyncio.py E501
     tests/test_utils_conf.py E501 E128
     tests/test_utils_curl.py E501
     tests/test_utils_datatypes.py E402 E501
-    tests/test_utils_defer.py E501 F841
-    tests/test_utils_deprecate.py F841 E501
-    tests/test_utils_http.py E501 E128 W504
-    tests/test_utils_iterators.py E501 E128 E129
+    tests/test_utils_defer.py E501
+    tests/test_utils_deprecate.py E501
+    tests/test_utils_http.py E501 E128
+    tests/test_utils_iterators.py E501 E128
     tests/test_utils_log.py E741
     tests/test_utils_python.py E501
     tests/test_utils_reqser.py E501 E128
     tests/test_utils_request.py E501 E128
     tests/test_utils_response.py E501
-    tests/test_utils_signal.py E741 F841
-    tests/test_utils_sitemap.py E128 E501 E124
-    tests/test_utils_url.py E501 E127 E125 E501 E126 E123
-    tests/test_webclient.py E501 E128 E122 E402 E123 E126
+    tests/test_utils_signal.py E741
+    tests/test_utils_sitemap.py E128 E501
+    tests/test_utils_url.py E501 E501
+    tests/test_webclient.py E501 E128 E402
     tests/test_cmdline/__init__.py E501
     tests/test_settings/__init__.py E501 E128
     tests/test_spiderloader/__init__.py E128 E501

scrapy/contracts/__init__.py

@@ -17,10 +17,10 @@ class ContractsManager:
             self.contracts[contract.name] = contract
 
     def tested_methods_from_spidercls(self, spidercls):
+        is_method = re.compile(r"^\s*@", re.MULTILINE).search
         methods = []
         for key, value in getmembers(spidercls):
-            if (callable(value) and value.__doc__ and
-                    re.search(r'^\s*@', value.__doc__, re.MULTILINE)):
+            if callable(value) and value.__doc__ and is_method(value.__doc__):
                 methods.append(key)
 
         return methods

scrapy/core/downloader/contextfactory.py

@@ -86,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
         #
         # This means that a website like https://www.cacert.org will be rejected
         # by default, since CAcert.org CA certificate is seldom shipped.
-        return optionsForClientTLS(hostname.decode("ascii"),
-                                   trustRoot=platformTrust(),
-                                   extraCertificateOptions={
-                                        'method': self._ssl_method,
-                                   })
+        return optionsForClientTLS(
+            hostname=hostname.decode("ascii"),
+            trustRoot=platformTrust(),
+            extraCertificateOptions={'method': self._ssl_method},
+        )

scrapy/core/downloader/handlers/ftp.py

@@ -94,11 +94,12 @@ class FTPDownloadHandler:
     def gotClient(self, client, request, filepath):
         self.client = client
         protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename"))
-        return client.retrieveFile(filepath, protocol)\
-                .addCallbacks(callback=self._build_response,
-                        callbackArgs=(request, protocol),
-                        errback=self._failed,
-                        errbackArgs=(request,))
+        return client.retrieveFile(filepath, protocol).addCallbacks(
+            callback=self._build_response,
+            callbackArgs=(request, protocol),
+            errback=self._failed,
+            errbackArgs=(request,),
+        )
 
     def _build_response(self, result, request, protocol):
         self.result = result

scrapy/core/downloader/handlers/http11.py

@@ -18,6 +18,7 @@ from twisted.web.http_headers import Headers as TxHeaders
 from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
 from zope.interface import implementer
 
+from scrapy import signals
 from scrapy.core.downloader.tls import openssl_methods
 from scrapy.core.downloader.webclient import _parse
 from scrapy.exceptions import ScrapyDeprecationWarning
@@ -34,6 +35,8 @@ class HTTP11DownloadHandler:
     lazy = False
 
     def __init__(self, settings, crawler=None):
+        self._crawler = crawler
+
         from twisted.internet import reactor
         self._pool = HTTPConnectionPool(reactor, persistent=True)
         self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
@@ -79,6 +82,7 @@ class HTTP11DownloadHandler:
             maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
             warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
             fail_on_dataloss=self._fail_on_dataloss,
+            crawler=self._crawler,
         )
         return agent.download_request(request)
 
@@ -276,7 +280,7 @@ class ScrapyAgent:
     _TunnelingAgent = TunnelingAgent
 
     def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
-                 maxsize=0, warnsize=0, fail_on_dataloss=True):
+                 maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None):
         self._contextFactory = contextFactory
         self._connectTimeout = connectTimeout
         self._bindAddress = bindAddress
@@ -285,6 +289,7 @@ class ScrapyAgent:
         self._warnsize = warnsize
         self._fail_on_dataloss = fail_on_dataloss
         self._txresponse = None
+        self._crawler = crawler
 
     def _get_agent(self, request, timeout):
         from twisted.internet import reactor
@@ -407,7 +412,15 @@ class ScrapyAgent:
 
         d = defer.Deferred(_cancel)
         txresponse.deliverBody(
-            _ResponseReader(d, txresponse, request, maxsize, warnsize, fail_on_dataloss)
+            _ResponseReader(
+                finished=d,
+                txresponse=txresponse,
+                request=request,
+                maxsize=maxsize,
+                warnsize=warnsize,
+                fail_on_dataloss=fail_on_dataloss,
+                crawler=self._crawler,
+            )
         )
 
         # save response for timeouts
@@ -449,7 +462,7 @@ class _RequestBodyProducer:
 
 class _ResponseReader(protocol.Protocol):
 
-    def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss):
+    def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler):
         self._finished = finished
         self._txresponse = txresponse
         self._request = request
@@ -462,6 +475,7 @@ class _ResponseReader(protocol.Protocol):
         self._bytes_received = 0
         self._certificate = None
         self._ip_address = None
+        self._crawler = crawler
 
     def connectionMade(self):
         if self._certificate is None:
@@ -479,6 +493,13 @@ class _ResponseReader(protocol.Protocol):
         self._bodybuf.write(bodyBytes)
         self._bytes_received += len(bodyBytes)
 
+        self._crawler.signals.send_catch_log(
+            signal=signals.bytes_received,
+            data=bodyBytes,
+            request=self._request,
+            spider=self._crawler.spider,
+        )
+
         if self._maxsize and self._bytes_received > self._maxsize:
             logger.error("Received (%(bytes)s) bytes larger than download "
                          "max size (%(maxsize)s) in request %(request)s.",

scrapy/core/downloader/handlers/s3.py

@@ -100,11 +100,12 @@ class S3DownloadHandler:
                 url=url, headers=awsrequest.headers.items())
         else:
             signed_headers = self.conn.make_request(
-                    method=request.method,
-                    bucket=bucket,
-                    key=unquote(p.path),
-                    query_args=unquote(p.query),
-                    headers=request.headers,
-                    data=request.body)
+                method=request.method,
+                bucket=bucket,
+                key=unquote(p.path),
+                query_args=unquote(p.query),
+                headers=request.headers,
+                data=request.body,
+            )
             request = request.replace(url=url, headers=signed_headers)
         return self._download_http(request, spider)

scrapy/core/downloader/webclient.py

@@ -88,8 +88,8 @@ class ScrapyHTTPPageGetter(HTTPClient):
             self.transport.stopProducing()
 
         self.factory.noPage(
-                defer.TimeoutError("Getting %s took longer than %s seconds." %
-                                   (self.factory.url, self.factory.timeout)))
+            defer.TimeoutError("Getting %s took longer than %s seconds."
+                               % (self.factory.url, self.factory.timeout)))
 
 
 class ScrapyHTTPClientFactory(HTTPClientFactory):

scrapy/core/engine.py

@@ -230,8 +230,7 @@ class ExecutionEngine:
 
     def _downloaded(self, response, slot, request, spider):
         slot.remove_request(request)
-        return self.download(response, spider) \
-                if isinstance(response, Request) else response
+        return self.download(response, spider) if isinstance(response, Request) else response
 
     def _download(self, request, spider):
         slot = self.slot

scrapy/downloadermiddlewares/ajaxcrawl.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import re
 import logging
 

scrapy/downloadermiddlewares/redirect.py

@@ -60,11 +60,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):
     Handle redirection of requests based on response status
     and meta-refresh html tag.
     """
+
     def process_response(self, request, response, spider):
-        if (request.meta.get('dont_redirect', False) or
-                response.status in getattr(spider, 'handle_httpstatus_list', []) or
-                response.status in request.meta.get('handle_httpstatus_list', []) or
-                request.meta.get('handle_httpstatus_all', False)):
+        if (
+            request.meta.get('dont_redirect', False)
+            or response.status in getattr(spider, 'handle_httpstatus_list', [])
+            or response.status in request.meta.get('handle_httpstatus_list', [])
+            or request.meta.get('handle_httpstatus_all', False)
+        ):
             return response
 
         allowed_status = (301, 302, 303, 307, 308)

scrapy/downloadermiddlewares/retry.py

@@ -12,9 +12,15 @@ once the spider has finished crawling all regular (non failed) pages.
 import logging
 
 from twisted.internet import defer
-from twisted.internet.error import TimeoutError, DNSLookupError, \
-        ConnectionRefusedError, ConnectionDone, ConnectError, \
-        ConnectionLost, TCPTimedOutError
+from twisted.internet.error import (
+    ConnectError,
+    ConnectionDone,
+    ConnectionLost,
+    ConnectionRefusedError,
+    DNSLookupError,
+    TCPTimedOutError,
+    TimeoutError,
+)
 from twisted.web.client import ResponseFailed
 
 from scrapy.exceptions import NotConfigured

scrapy/exporters.py

@@ -250,7 +250,7 @@ class CsvItemExporter(BaseItemExporter):
 
 class PickleItemExporter(BaseItemExporter):
 
-    def __init__(self, file, protocol=2, **kwargs):
+    def __init__(self, file, protocol=4, **kwargs):
         super().__init__(**kwargs)
         self.file = file
         self.protocol = protocol

scrapy/extensions/closespider.py

@@ -20,7 +20,7 @@ class CloseSpider:
             'itemcount': crawler.settings.getint('CLOSESPIDER_ITEMCOUNT'),
             'pagecount': crawler.settings.getint('CLOSESPIDER_PAGECOUNT'),
             'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'),
-            }
+        }
 
         if not any(self.close_on.values()):
             raise NotConfigured

scrapy/extensions/httpcache.py

@@ -250,7 +250,7 @@ class DbmCacheStorage:
             'headers': dict(response.headers),
             'body': response.body,
         }
-        self.db['%s_data' % key] = pickle.dumps(data, protocol=2)
+        self.db['%s_data' % key] = pickle.dumps(data, protocol=4)
         self.db['%s_time' % key] = str(time())
 
     def _read_data(self, spider, request):
@@ -317,7 +317,7 @@ class FilesystemCacheStorage:
         with self._open(os.path.join(rpath, 'meta'), 'wb') as f:
             f.write(to_bytes(repr(metadata)))
         with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f:
-            pickle.dump(metadata, f, protocol=2)
+            pickle.dump(metadata, f, protocol=4)
         with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f:
             f.write(headers_dict_to_raw(response.headers))
         with self._open(os.path.join(rpath, 'response_body'), 'wb') as f:

scrapy/extensions/spiderstate.py

@@ -26,7 +26,7 @@ class SpiderState:
     def spider_closed(self, spider):
         if self.jobdir:
             with open(self.statefn, 'wb') as f:
-                pickle.dump(spider.state, f, protocol=2)
+                pickle.dump(spider.state, f, protocol=4)
 
     def spider_opened(self, spider):
         if self.jobdir and os.path.exists(self.statefn):

scrapy/extensions/telnet.py

@@ -76,8 +76,10 @@ class TelnetConsole(protocol.ServerFactory):
             """An implementation of IPortal"""
             @defers
             def login(self_, credentials, mind, *interfaces):
-                if not (credentials.username == self.username.encode('utf8') and
-                        credentials.checkPassword(self.password.encode('utf8'))):
+                if not (
+                    credentials.username == self.username.encode('utf8')
+                    and credentials.checkPassword(self.password.encode('utf8'))
+                ):
                     raise ValueError("Invalid credentials")
 
                 protocol = telnet.TelnetBootstrapProtocol(

scrapy/http/request/form.py

@@ -178,12 +178,11 @@ def _get_clickable(clickdata, form):
     if the latter is given. If not, it returns the first
     clickable element found
     """
-    clickables = [
-        el for el in form.xpath(
-            'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
-            '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
-            namespaces={"re": "http://exslt.org/regular-expressions"})
-        ]
+    clickables = list(form.xpath(
+        'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
+        '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
+        namespaces={"re": "http://exslt.org/regular-expressions"}
+    ))
     if not clickables:
         return
 

scrapy/http/response/text.py

@@ -5,6 +5,7 @@ discovering (through HTTP headers) to base Response class.
 See documentation in docs/topics/request-response.rst
 """
 
+import warnings
 from contextlib import suppress
 from typing import Generator
 from urllib.parse import urljoin
@@ -14,6 +15,7 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
                             http_content_type_encoding, resolve_encoding)
 from w3lib.html import strip_html5_whitespace
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.http.response import Response
 from scrapy.utils.python import memoizemethod_noargs, to_unicode
@@ -61,6 +63,9 @@ class TextResponse(Response):
 
     def body_as_unicode(self):
         """Return body as unicode"""
+        warnings.warn('Response.body_as_unicode() is deprecated, '
+                      'please use Response.text instead.',
+                      ScrapyDeprecationWarning)
         return self.text
 
     @property

scrapy/linkextractors/__init__.py

@@ -61,8 +61,7 @@ class FilteringLinkExtractor:
 
     def __new__(cls, *args, **kwargs):
         from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
-        if (issubclass(cls, FilteringLinkExtractor) and
-                not issubclass(cls, LxmlLinkExtractor)):
+        if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor):
             warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
                  'please use scrapy.linkextractors.LinkExtractor instead',
                  ScrapyDeprecationWarning, stacklevel=2)

scrapy/linkextractors/lxmlhtml.py

@@ -1,6 +1,8 @@
 """
 Link extractor based on lxml.html
 """
+import operator
+from functools import partial
 from urllib.parse import urljoin
 
 import lxml.etree as etree
@@ -8,10 +10,10 @@ from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url, safe_url_string
 
 from scrapy.link import Link
+from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list
 from scrapy.utils.response import get_base_url
-from scrapy.linkextractors import FilteringLinkExtractor
 
 
 # from lxml/src/lxml/html/__init__.py
@@ -27,19 +29,24 @@ def _nons(tag):
     return tag
 
 
+def _identity(x):
+    return x
+
+
+def _canonicalize_link_url(link):
+    return canonicalize_url(link.url, keep_fragments=True)
+
+
 class LxmlParserLinkExtractor:
-    def __init__(self, tag="a", attr="href", process=None, unique=False,
-                 strip=True, canonicalized=False):
-        self.scan_tag = tag if callable(tag) else lambda t: t == tag
-        self.scan_attr = attr if callable(attr) else lambda a: a == attr
-        self.process_attr = process if callable(process) else lambda v: v
+    def __init__(
+        self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False
+    ):
+        self.scan_tag = tag if callable(tag) else partial(operator.eq, tag)
+        self.scan_attr = attr if callable(attr) else partial(operator.eq, attr)
+        self.process_attr = process if callable(process) else _identity
         self.unique = unique
         self.strip = strip
-        if canonicalized:
-            self.link_key = lambda link: link.url
-        else:
-            self.link_key = lambda link: canonicalize_url(link.url,
-                                                          keep_fragments=True)
+        self.link_key = operator.attrgetter("url") if canonicalized else _canonicalize_link_url
 
     def _iter_links(self, document):
         for el in document.iter(etree.Element):
@@ -93,25 +100,44 @@ class LxmlParserLinkExtractor:
 
 class LxmlLinkExtractor(FilteringLinkExtractor):
 
-    def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
-                 tags=('a', 'area'), attrs=('href',), canonicalize=False,
-                 unique=True, process_value=None, deny_extensions=None, restrict_css=(),
-                 strip=True, restrict_text=None):
+    def __init__(
+        self,
+        allow=(),
+        deny=(),
+        allow_domains=(),
+        deny_domains=(),
+        restrict_xpaths=(),
+        tags=('a', 'area'),
+        attrs=('href',),
+        canonicalize=False,
+        unique=True,
+        process_value=None,
+        deny_extensions=None,
+        restrict_css=(),
+        strip=True,
+        restrict_text=None,
+    ):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
         lx = LxmlParserLinkExtractor(
-            tag=lambda x: x in tags,
-            attr=lambda x: x in attrs,
+            tag=partial(operator.contains, tags),
+            attr=partial(operator.contains, attrs),
             unique=unique,
             process=process_value,
             strip=strip,
             canonicalized=canonicalize
         )
-
-        super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
-                                                allow_domains=allow_domains, deny_domains=deny_domains,
-                                                restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
-                                                canonicalize=canonicalize, deny_extensions=deny_extensions,
-                                                restrict_text=restrict_text)
+        super(LxmlLinkExtractor, self).__init__(
+            link_extractor=lx,
+            allow=allow,
+            deny=deny,
+            allow_domains=allow_domains,
+            deny_domains=deny_domains,
+            restrict_xpaths=restrict_xpaths,
+            restrict_css=restrict_css,
+            canonicalize=canonicalize,
+            deny_extensions=deny_extensions,
+            restrict_text=restrict_text,
+        )
 
     def extract_links(self, response):
         """Returns a list of :class:`~scrapy.link.Link` objects from the
@@ -124,9 +150,11 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
         """
         base_url = get_base_url(response)
         if self.restrict_xpaths:
-            docs = [subdoc
-                    for x in self.restrict_xpaths
-                    for subdoc in response.xpath(x)]
+            docs = [
+                subdoc
+                for x in self.restrict_xpaths
+                for subdoc in response.xpath(x)
+            ]
         else:
             docs = [response.selector]
         all_links = []

scrapy/pipelines/files.py

@@ -83,8 +83,7 @@ class S3FilesStore:
     AWS_USE_SSL = None
     AWS_VERIFY = None
 
-    POLICY = 'private'  # Overriden from settings.FILES_STORE_S3_ACL in
-                        # FilesPipeline.from_settings.
+    POLICY = 'private'  # Overriden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
     HEADERS = {
         'Cache-Control': 'max-age=172800',
     }
@@ -230,6 +229,20 @@ class GCSFilesStore:
         bucket, prefix = uri[5:].split('/', 1)
         self.bucket = client.bucket(bucket)
         self.prefix = prefix
+        permissions = self.bucket.test_iam_permissions(
+            ['storage.objects.get', 'storage.objects.create']
+        )
+        if 'storage.objects.get' not in permissions:
+            logger.warning(
+                "No 'storage.objects.get' permission for GSC bucket %(bucket)s. "
+                "Checking if files are up to date will be impossible. Files will be downloaded every time.",
+                {'bucket': bucket}
+            )
+        if 'storage.objects.create' not in permissions:
+            logger.error(
+                "No 'storage.objects.create' permission for GSC bucket %(bucket)s. Saving files will be impossible!",
+                {'bucket': bucket}
+            )
 
     def stat_file(self, path, info):
         def _onsuccess(blob):

scrapy/pipelines/media.py

@@ -43,8 +43,7 @@ class MediaPipeline:
         if allow_redirects:
             self.handle_httpstatus_list = SequenceExclude(range(300, 400))
 
-    def _key_for_pipe(self, key, base_class_name=None,
-                      settings=None):
+    def _key_for_pipe(self, key, base_class_name=None, settings=None):
         """
         >>> MediaPipeline()._key_for_pipe("IMAGES")
         'IMAGES'
@@ -55,8 +54,11 @@ class MediaPipeline:
         """
         class_name = self.__class__.__name__
         formatted_key = "{}_{}".format(class_name.upper(), key)
-        if class_name == base_class_name or not base_class_name \
-            or (settings and not settings.get(formatted_key)):
+        if (
+            not base_class_name
+            or class_name == base_class_name
+            or settings and not settings.get(formatted_key)
+        ):
             return key
         return formatted_key
 

scrapy/selector/unified.py

@@ -65,9 +65,9 @@ class Selector(_ParselSelector, object_ref):
     selectorlist_cls = SelectorList
 
     def __init__(self, response=None, text=None, type=None, root=None, **kwargs):
-        if not(response is None or text is None):
-           raise ValueError('%s.__init__() received both response and text'
-                            % self.__class__.__name__)
+        if response is not None and text is not None:
+            raise ValueError('%s.__init__() received both response and text'
+                             % self.__class__.__name__)
 
         st = _st(response, type or self._default_type)
 

scrapy/signals.py

@@ -17,6 +17,7 @@ request_reached_downloader = object()
 request_left_downloader = object()
 response_received = object()
 response_downloaded = object()
+bytes_received = object()
 item_scraped = object()
 item_dropped = object()
 item_error = object()

scrapy/spiderloader.py

@@ -1,7 +1,6 @@
-# -*- coding: utf-8 -*-
-from collections import defaultdict
 import traceback
 import warnings
+from collections import defaultdict
 
 from zope.interface import implementer
 
@@ -16,6 +15,7 @@ class SpiderLoader:
     SpiderLoader is a class which locates and loads spiders
     in a Scrapy project.
     """
+
     def __init__(self, settings):
         self.spider_modules = settings.getlist('SPIDER_MODULES')
         self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')
@@ -24,16 +24,21 @@ class SpiderLoader:
         self._load_all_spiders()
 
     def _check_name_duplicates(self):
-        dupes = ["\n".join("  {cls} named {name!r} (in {module})".format(
-                                module=mod, cls=cls, name=name)
-                           for (mod, cls) in locations)
-                 for name, locations in self._found.items()
-                 if len(locations) > 1]
+        dupes = []
+        for name, locations in self._found.items():
+            dupes.extend([
+                "  {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name)
+                for mod, cls in locations
+                if len(locations) > 1
+            ])
+
         if dupes:
-            msg = ("There are several spiders with the same name:\n\n"
-                   "{}\n\n  This can cause unexpected behavior.".format(
-                        "\n\n".join(dupes)))
-            warnings.warn(msg, UserWarning)
+            dupes_string = "\n\n".join(dupes)
+            warnings.warn(
+                "There are several spiders with the same name:\n\n"
+                "{}\n\n  This can cause unexpected behavior.".format(dupes_string),
+                category=UserWarning,
+            )
 
     def _load_spiders(self, module):
         for spcls in iter_spider_classes(module):
@@ -45,12 +50,15 @@ class SpiderLoader:
             try:
                 for module in walk_modules(name):
                     self._load_spiders(module)
-            except ImportError as e:
+            except ImportError:
                 if self.warn_only:
-                    msg = ("\n{tb}Could not load spiders from module '{modname}'. "
-                           "See above traceback for details.".format(
-                                modname=name, tb=traceback.format_exc()))
-                    warnings.warn(msg, RuntimeWarning)
+                    warnings.warn(
+                        "\n{tb}Could not load spiders from module '{modname}'. "
+                        "See above traceback for details.".format(
+                            modname=name, tb=traceback.format_exc()
+                        ),
+                        category=RuntimeWarning,
+                    )
                 else:
                     raise
         self._check_name_duplicates()
@@ -73,8 +81,10 @@ class SpiderLoader:
         """
         Return the list of spider names that can handle the given request.
         """
-        return [name for name, cls in self._spiders.items()
-                if cls.handles_request(request)]
+        return [
+            name for name, cls in self._spiders.items()
+            if cls.handles_request(request)
+        ]
 
     def list(self):
         """

scrapy/spidermiddlewares/referer.py

@@ -163,9 +163,10 @@ class StrictOriginPolicy(ReferrerPolicy):
     name = POLICY_STRICT_ORIGIN
 
     def referrer(self, response_url, request_url):
-        if ((self.tls_protected(response_url) and
-             self.potentially_trustworthy(request_url))
-            or not self.tls_protected(response_url)):
+        if (
+            self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
+            or not self.tls_protected(response_url)
+        ):
             return self.origin_referrer(response_url)
 
 
@@ -213,9 +214,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
         origin = self.origin(response_url)
         if origin == self.origin(request_url):
             return self.stripped_referrer(response_url)
-        elif ((self.tls_protected(response_url) and
-               self.potentially_trustworthy(request_url))
-              or not self.tls_protected(response_url)):
+        elif (
+            self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
+            or not self.tls_protected(response_url)
+        ):
             return self.origin_referrer(response_url)
 
 

scrapy/squeues.py

@@ -81,12 +81,11 @@ def _scrapy_non_serialization_queue(queue_class):
 
 def _pickle_serialize(obj):
     try:
-        return pickle.dumps(obj, protocol=2)
-    # Python <= 3.4 raises pickle.PicklingError here while
-    # 3.5 <= Python < 3.6 raises AttributeError and
-    # Python >= 3.6 raises TypeError
+        return pickle.dumps(obj, protocol=4)
+    # Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s)
+    # TypeError is raised from parsel.Selector
     except (pickle.PicklingError, AttributeError, TypeError) as e:
-        raise ValueError(str(e))
+        raise ValueError(str(e)) from e
 
 
 PickleFifoDiskQueueNonRequest = _serializable_queue(

scrapy/templates/project/module/items.py.tmpl

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Define here the models for your scraped items
 #
 # See documentation in:

scrapy/templates/project/module/middlewares.py.tmpl

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Define here the models for your spider middleware
 #
 # See documentation in:

scrapy/templates/project/module/pipelines.py.tmpl

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Define your item pipelines here
 #
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting

scrapy/templates/project/module/settings.py.tmpl

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Scrapy settings for $project_name project
 #
 # For simplicity, this file contains only settings considered important or

scrapy/templates/spiders/basic.tmpl

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import scrapy
 
 

scrapy/templates/spiders/crawl.tmpl

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import scrapy
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule

scrapy/templates/spiders/csvfeed.tmpl

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from scrapy.spiders import CSVFeedSpider
 
 

scrapy/templates/spiders/xmlfeed.tmpl

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from scrapy.spiders import XMLFeedSpider
 
 

scrapy/utils/deprecate.py

@@ -15,16 +15,17 @@ def attribute(obj, oldattr, newattr, version='0.12'):
         stacklevel=3)
 
 
-def create_deprecated_class(name, new_class, clsdict=None,
-                            warn_category=ScrapyDeprecationWarning,
-                            warn_once=True,
-                            old_class_path=None,
-                            new_class_path=None,
-                            subclass_warn_message="{cls} inherits from "
-                                    "deprecated class {old}, please inherit "
-                                    "from {new}.",
-                            instance_warn_message="{cls} is deprecated, "
-                                    "instantiate {new} instead."):
+def create_deprecated_class(
+    name,
+    new_class,
+    clsdict=None,
+    warn_category=ScrapyDeprecationWarning,
+    warn_once=True,
+    old_class_path=None,
+    new_class_path=None,
+    subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.",
+    instance_warn_message="{cls} is deprecated, instantiate {new} instead."
+):
     """
     Return a "deprecated" class that causes its subclasses to issue a warning.
     Subclasses of ``new_class`` are considered subclasses of this class.

scrapy/utils/gz.py

@@ -52,8 +52,7 @@ def is_gzipped(response):
     """Return True if the response is gzipped, or False otherwise"""
     ctype = response.headers.get('Content-Type', b'')
     cenc = response.headers.get('Content-Encoding', b'').lower()
-    return (_is_gzipped(ctype) or
-            (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')))
+    return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')
 
 
 def gzip_magic_number(response):

scrapy/utils/log.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 import logging
 import sys
 import warnings

scrapy/utils/misc.py

@@ -137,17 +137,26 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
     ``*args`` and ``**kwargs`` are forwarded to the constructors.
 
     Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
+
+    Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
+    extension has not been implemented correctly).
     """
     if settings is None:
         if crawler is None:
             raise ValueError("Specify at least one of settings and crawler.")
         settings = crawler.settings
     if crawler and hasattr(objcls, 'from_crawler'):
-        return objcls.from_crawler(crawler, *args, **kwargs)
+        instance = objcls.from_crawler(crawler, *args, **kwargs)
+        method_name = 'from_crawler'
     elif hasattr(objcls, 'from_settings'):
-        return objcls.from_settings(settings, *args, **kwargs)
+        instance = objcls.from_settings(settings, *args, **kwargs)
+        method_name = 'from_settings'
     else:
-        return objcls(*args, **kwargs)
+        instance = objcls(*args, **kwargs)
+        method_name = '__new__'
+    if instance is None:
+        raise TypeError("%s.%s returned None" % (objcls.__qualname__, method_name))
+    return instance
 
 
 @contextmanager

scrapy/utils/request.py

@@ -50,8 +50,7 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
 
     """
     if include_headers:
-        include_headers = tuple(to_bytes(h.lower())
-                                 for h in sorted(include_headers))
+        include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
     cache = _fingerprint_cache.setdefault(request, {})
     cache_key = (include_headers, keep_fragments)
     if cache_key not in cache:

scrapy/utils/ssl.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 import OpenSSL
 import OpenSSL._util as pyOpenSSLutil
 

tests/pipelines.py

@@ -6,7 +6,7 @@ Some pipelines used for testing
 class ZeroDivisionErrorPipeline:
 
     def open_spider(self, spider):
-        a = 1 / 0
+        1 / 0
 
     def process_item(self, item, spider):
         return item

tests/spiders.py

@@ -184,8 +184,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
             if self.fail_yielding:
                 2 / 0
 
-        assert self.seedsseen, \
-                'All start requests consumed before any download happened'
+        assert self.seedsseen, 'All start requests consumed before any download happened'
 
     def parse(self, response):
         self.seedsseen.append(response.meta.get('seed'))

tests/test_closespider.py

@@ -41,8 +41,7 @@ class TestCloseSpider(TestCase):
         yield crawler.crawl(total=1000000, mockserver=self.mockserver)
         reason = crawler.spider.meta['close_reason']
         self.assertEqual(reason, 'closespider_errorcount')
-        key = 'spider_exceptions/{name}'\
-                .format(name=crawler.spider.exception_cls.__name__)
+        key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__)
         errorcount = crawler.stats.get_value(key)
         self.assertTrue(errorcount >= close_on)
 

tests/test_crawler.py

@@ -87,7 +87,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
         class MySpider(scrapy.Spider):
             name = 'spider'
 
-        crawler = Crawler(MySpider, {})
+        Crawler(MySpider, {})
         assert get_scrapy_root_handler() is None
 
     def test_spider_custom_settings_log_level(self):
@@ -240,13 +240,13 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
     def test_crawler_runner_asyncio_enabled_true(self):
         if self.reactor_pytest == 'asyncio':
-            runner = CrawlerRunner(settings={
+            CrawlerRunner(settings={
                 "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
             })
         else:
             msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
             with self.assertRaisesRegex(Exception, msg):
-                runner = CrawlerRunner(settings={
+                CrawlerRunner(settings={
                     "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
                 })
 
@@ -311,14 +311,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
     def test_ipv6_alternative_name_resolver(self):
         log = self.run_script('alternative_name_resolver.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertTrue(any([
-            "twisted.internet.error.ConnectionRefusedError" in log,
-            "twisted.internet.error.ConnectError" in log,
-        ]))
-        self.assertTrue(any([
-            "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
-            "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
-        ]))
+        self.assertNotIn("twisted.internet.error.DNSLookupError", log)
 
     def test_reactor_select(self):
         log = self.run_script("twisted_reactor_select.py")

tests/test_dependencies.py

@@ -6,7 +6,7 @@ class ScrapyUtilsTest(unittest.TestCase):
     def test_required_openssl_version(self):
         try:
             module = import_module('OpenSSL')
-        except ImportError as ex:
+        except ImportError:
             raise unittest.SkipTest("OpenSSL is not available")
 
         if hasattr(module, '__version__'):

tests/test_downloader_handlers.py

@@ -730,6 +730,9 @@ class Http11ProxyTestCase(HttpProxyTestCase):
 
 class HttpDownloadHandlerMock:
 
+    def __init__(self, *args, **kwargs):
+        pass
+
     def download_request(self, request, spider):
         return request
 
@@ -822,11 +825,15 @@ class S3TestCase(unittest.TestCase):
     def test_request_signing2(self):
         # puts an object into the johnsmith bucket.
         date = 'Tue, 27 Mar 2007 21:15:45 +0000'
-        req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={
-            'Content-Type': 'image/jpeg',
-            'Date': date,
-            'Content-Length': '94328',
-            })
+        req = Request(
+            's3://johnsmith/photos/puppy.jpg',
+            method='PUT',
+            headers={
+                'Content-Type': 'image/jpeg',
+                'Date': date,
+                'Content-Length': '94328',
+            },
+        )
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'],
@@ -906,11 +913,10 @@ class S3TestCase(unittest.TestCase):
         # ensure that spaces are quoted properly before signing
         date = 'Tue, 27 Mar 2007 19:42:41 +0000'
         req = Request(
-            ("s3://johnsmith/photos/my puppy.jpg"
-             "?response-content-disposition=my puppy.jpg"),
+            "s3://johnsmith/photos/my puppy.jpg?response-content-disposition=my puppy.jpg",
             method='GET',
             headers={'Date': date},
-            )
+        )
         with self._mocked_date(date):
             httpreq = self.download_request(req, self.spider)
         self.assertEqual(
@@ -1090,8 +1096,7 @@ class DataURITestCase(unittest.TestCase):
     def test_default_mediatype_encoding(self):
         def _test(response):
             self.assertEqual(response.text, 'A brief note')
-            self.assertEqual(type(response),
-                              responsetypes.from_mimetype("text/plain"))
+            self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "US-ASCII")
 
         request = Request("data:,A%20brief%20note")
@@ -1100,8 +1105,7 @@ class DataURITestCase(unittest.TestCase):
     def test_default_mediatype(self):
         def _test(response):
             self.assertEqual(response.text, u'\u038e\u03a3\u038e')
-            self.assertEqual(type(response),
-                              responsetypes.from_mimetype("text/plain"))
+            self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "iso-8859-7")
 
         request = Request("data:;charset=iso-8859-7,%be%d3%be")
@@ -1119,8 +1123,7 @@ class DataURITestCase(unittest.TestCase):
     def test_mediatype_parameters(self):
         def _test(response):
             self.assertEqual(response.text, u'\u038e\u03a3\u038e')
-            self.assertEqual(type(response),
-                              responsetypes.from_mimetype("text/plain"))
+            self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
             self.assertEqual(response.encoding, "utf-8")
 
         request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;'

tests/test_downloadermiddleware_cookies.py

@@ -139,10 +139,12 @@ class CookiesMiddlewareTest(TestCase):
 
     def test_complex_cookies(self):
         # merge some cookies into jar
-        cookies = [{'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'},
-                {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'},
-                {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},
-                {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}]
+        cookies = [
+            {'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'},
+            {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'},
+            {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},
+            {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'},
+        ]
 
         req = Request('http://scrapytest.org/', cookies=cookies)
         self.mw.process_request(req, self.spider)

tests/test_downloadermiddleware_decompression.py

@@ -28,8 +28,8 @@ class DecompressionMiddlewareTest(TestCase):
         for fmt in self.test_formats:
             rsp = self.test_responses[fmt]
             new = self.mw.process_response(None, rsp, self.spider)
-            assert isinstance(new, XmlResponse), \
-                    'Failed %s, response type %s' % (fmt, type(new).__name__)
+            error_msg = 'Failed %s, response type %s' % (fmt, type(new).__name__)
+            assert isinstance(new, XmlResponse), error_msg
             assert_samelines(self, new.body, self.uncompressed_body, fmt)
 
     def test_plain_response(self):

tests/test_downloadermiddleware_httpcompression.py

@@ -16,12 +16,12 @@ from w3lib.encoding import resolve_encoding
 SAMPLEDIR = join(tests_datadir, 'compressed')
 
 FORMAT = {
-        'gzip': ('html-gzip.bin', 'gzip'),
-        'x-gzip': ('html-gzip.bin', 'gzip'),
-        'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
-        'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
-        'br': ('html-br.bin', 'br')
-        }
+    'gzip': ('html-gzip.bin', 'gzip'),
+    'x-gzip': ('html-gzip.bin', 'gzip'),
+    'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
+    'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
+    'br': ('html-br.bin', 'br'),
+}
 
 
 class HttpCompressionTest(TestCase):
@@ -40,12 +40,12 @@ class HttpCompressionTest(TestCase):
             body = sample.read()
 
         headers = {
-                'Server': 'Yaws/1.49 Yet Another Web Server',
-                'Date': 'Sun, 08 Mar 2009 00:41:03 GMT',
-                'Content-Length': len(body),
-                'Content-Type': 'text/html',
-                'Content-Encoding': contentencoding,
-                }
+            'Server': 'Yaws/1.49 Yet Another Web Server',
+            'Date': 'Sun, 08 Mar 2009 00:41:03 GMT',
+            'Content-Length': len(body),
+            'Content-Type': 'text/html',
+            'Content-Encoding': contentencoding,
+        }
 
         response = Response('http://scrapytest.org/', body=body, headers=headers)
         response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'})

tests/test_downloadermiddleware_redirect.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 import unittest
 
 from scrapy.downloadermiddlewares.redirect import RedirectMiddleware, MetaRefreshMiddleware
@@ -181,8 +179,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
             rsp = Response(url, headers={'Location': url2}, status=301, request=req)
             r = self.mw.process_response(req, rsp, self.spider)
             self.assertIs(r, rsp)
-        _test_passthrough(Request(url, meta={'handle_httpstatus_list':
-                                                           [404, 301, 302]}))
+        _test_passthrough(Request(url, meta={'handle_httpstatus_list': [404, 301, 302]}))
         _test_passthrough(Request(url, meta={'handle_httpstatus_all': True}))
 
     def test_latin1_location(self):

tests/test_downloadermiddleware_retry.py

@@ -1,8 +1,14 @@
 import unittest
 from twisted.internet import defer
-from twisted.internet.error import TimeoutError, DNSLookupError, \
-        ConnectionRefusedError, ConnectionDone, ConnectError, \
-        ConnectionLost, TCPTimedOutError
+from twisted.internet.error import (
+    ConnectError,
+    ConnectionDone,
+    ConnectionLost,
+    ConnectionRefusedError,
+    DNSLookupError,
+    TCPTimedOutError,
+    TimeoutError,
+)
 from twisted.web.client import ResponseFailed
 
 from scrapy.downloadermiddlewares.retry import RetryMiddleware

tests/test_downloadermiddleware_robotstxt.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from unittest import mock
 
 from twisted.internet import reactor, error

tests/test_dupefilters.py

@@ -197,8 +197,7 @@ class RFPDupeFilterTest(unittest.TestCase):
 
             r1 = Request('http://scrapytest.org/index.html')
             r2 = Request('http://scrapytest.org/index.html',
-                headers={'Referer': 'http://scrapytest.org/INDEX.html'}
-            )
+                         headers={'Referer': 'http://scrapytest.org/INDEX.html'})
 
             dupefilter.log(r1, spider)
             dupefilter.log(r2, spider)

tests/test_engine.py

@@ -13,22 +13,24 @@ module with the ``runserver`` argument::
 import os
 import re
 import sys
+from collections import defaultdict
 from urllib.parse import urlparse
 
 from twisted.internet import reactor, defer
-from twisted.web import server, static, util
 from twisted.trial import unittest
+from twisted.web import server, static, util
+from pydispatch import dispatcher
 
 from scrapy import signals
 from scrapy.core.engine import ExecutionEngine
-from scrapy.utils.test import get_crawler
-from pydispatch import dispatcher
-from tests import tests_datadir
-from scrapy.spiders import Spider
+from scrapy.http import Request
 from scrapy.item import Item, Field
 from scrapy.linkextractors import LinkExtractor
-from scrapy.http import Request
+from scrapy.spiders import Spider
 from scrapy.utils.signal import disconnect_all
+from scrapy.utils.test import get_crawler
+
+from tests import tests_datadir, get_testdata
 
 
 class TestItem(Item):
@@ -88,6 +90,8 @@ def start_test_site(debug=False):
     r = static.File(root_dir)
     r.putChild(b"redirect", util.Redirect(b"/redirected"))
     r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
+    numbers = [str(x).encode("utf8") for x in range(2**14)]
+    r.putChild(b"numbers", static.Data(b"".join(numbers), "text/plain"))
 
     port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1")
     if debug:
@@ -107,15 +111,20 @@ class CrawlerRun:
         self.reqreached = []
         self.itemerror = []
         self.itemresp = []
-        self.signals_catched = {}
+        self.bytes = defaultdict(lambda: list())
+        self.signals_caught = {}
         self.spider_class = spider_class
 
     def run(self):
         self.port = start_test_site()
         self.portno = self.port.getHost().port
 
-        start_urls = [self.geturl("/"), self.geturl("/redirect"),
-                      self.geturl("/redirect")]  # a duplicate
+        start_urls = [
+            self.geturl("/"),
+            self.geturl("/redirect"),
+            self.geturl("/redirect"),  # duplicate
+            self.geturl("/numbers"),
+        ]
 
         for name, signal in vars(signals).items():
             if not name.startswith('_'):
@@ -124,6 +133,7 @@ class CrawlerRun:
         self.crawler = get_crawler(self.spider_class)
         self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
         self.crawler.signals.connect(self.item_error, signals.item_error)
+        self.crawler.signals.connect(self.bytes_received, signals.bytes_received)
         self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
         self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
         self.crawler.signals.connect(self.request_reached, signals.request_reached_downloader)
@@ -155,6 +165,9 @@ class CrawlerRun:
     def item_scraped(self, item, spider, response):
         self.itemresp.append((item, response))
 
+    def bytes_received(self, data, request, spider):
+        self.bytes[request].append(data)
+
     def request_scheduled(self, request, spider):
         self.reqplug.append((request, spider))
 
@@ -172,7 +185,7 @@ class CrawlerRun:
         signalargs = kwargs.copy()
         sig = signalargs.pop('signal')
         signalargs.pop('sender', None)
-        self.signals_catched[sig] = signalargs
+        self.signals_caught[sig] = signalargs
 
 
 class EngineTest(unittest.TestCase):
@@ -183,16 +196,17 @@ class EngineTest(unittest.TestCase):
             self.run = CrawlerRun(spider)
             yield self.run.run()
             self._assert_visited_urls()
-            self._assert_scheduled_requests(urls_to_visit=8)
+            self._assert_scheduled_requests(urls_to_visit=9)
             self._assert_downloaded_responses()
             self._assert_scraped_items()
-            self._assert_signals_catched()
+            self._assert_signals_caught()
+            self._assert_bytes_received()
 
     @defer.inlineCallbacks
     def test_crawler_dupefilter(self):
         self.run = CrawlerRun(TestDupeFilterSpider)
         yield self.run.run()
-        self._assert_scheduled_requests(urls_to_visit=7)
+        self._assert_scheduled_requests(urls_to_visit=8)
         self._assert_dropped_requests()
 
     @defer.inlineCallbacks
@@ -229,8 +243,8 @@ class EngineTest(unittest.TestCase):
 
     def _assert_downloaded_responses(self):
         # response tests
-        self.assertEqual(8, len(self.run.respplug))
-        self.assertEqual(8, len(self.run.reqreached))
+        self.assertEqual(9, len(self.run.respplug))
+        self.assertEqual(9, len(self.run.reqreached))
 
         for response, _ in self.run.respplug:
             if self.run.getpath(response.url) == '/item999.html':
@@ -263,19 +277,61 @@ class EngineTest(unittest.TestCase):
                 self.assertEqual('Item 2 name', item['name'])
                 self.assertEqual('200', item['price'])
 
-    def _assert_signals_catched(self):
-        assert signals.engine_started in self.run.signals_catched
-        assert signals.engine_stopped in self.run.signals_catched
-        assert signals.spider_opened in self.run.signals_catched
-        assert signals.spider_idle in self.run.signals_catched
-        assert signals.spider_closed in self.run.signals_catched
+    def _assert_bytes_received(self):
+        self.assertEqual(9, len(self.run.bytes))
+        for request, data in self.run.bytes.items():
+            joined_data = b"".join(data)
+            if self.run.getpath(request.url) == "/":
+                self.assertEqual(joined_data, get_testdata("test_site", "index.html"))
+            elif self.run.getpath(request.url) == "/item1.html":
+                self.assertEqual(joined_data, get_testdata("test_site", "item1.html"))
+            elif self.run.getpath(request.url) == "/item2.html":
+                self.assertEqual(joined_data, get_testdata("test_site", "item2.html"))
+            elif self.run.getpath(request.url) == "/redirected":
+                self.assertEqual(joined_data, b"Redirected here")
+            elif self.run.getpath(request.url) == '/redirect':
+                self.assertEqual(
+                    joined_data,
+                    b"\n<html>\n"
+                    b"    <head>\n"
+                    b"        <meta http-equiv=\"refresh\" content=\"0;URL=/redirected\">\n"
+                    b"    </head>\n"
+                    b"    <body bgcolor=\"#FFFFFF\" text=\"#000000\">\n"
+                    b"    <a href=\"/redirected\">click here</a>\n"
+                    b"    </body>\n"
+                    b"</html>\n"
+                )
+            elif self.run.getpath(request.url) == "/tem999.html":
+                self.assertEqual(
+                    joined_data,
+                    b"\n<html>\n"
+                    b"  <head><title>404 - No Such Resource</title></head>\n"
+                    b"  <body>\n"
+                    b"    <h1>No Such Resource</h1>\n"
+                    b"    <p>File not found.</p>\n"
+                    b"  </body>\n"
+                    b"</html>\n"
+                )
+            elif self.run.getpath(request.url) == "/numbers":
+                # signal was fired multiple times
+                self.assertTrue(len(data) > 1)
+                # bytes were received in order
+                numbers = [str(x).encode("utf8") for x in range(2**14)]
+                self.assertEqual(joined_data, b"".join(numbers))
+
+    def _assert_signals_caught(self):
+        assert signals.engine_started in self.run.signals_caught
+        assert signals.engine_stopped in self.run.signals_caught
+        assert signals.spider_opened in self.run.signals_caught
+        assert signals.spider_idle in self.run.signals_caught
+        assert signals.spider_closed in self.run.signals_caught
 
         self.assertEqual({'spider': self.run.spider},
-                         self.run.signals_catched[signals.spider_opened])
+                         self.run.signals_caught[signals.spider_opened])
         self.assertEqual({'spider': self.run.spider},
-                         self.run.signals_catched[signals.spider_idle])
+                         self.run.signals_caught[signals.spider_idle])
         self.assertEqual({'spider': self.run.spider, 'reason': 'finished'},
-                         self.run.signals_catched[signals.spider_closed])
+                         self.run.signals_caught[signals.spider_closed])
 
     @defer.inlineCallbacks
     def test_close_downloader(self):

tests/test_exporters.py

@@ -342,20 +342,22 @@ class XmlItemExporterTest(BaseItemExporterTest):
         i2 = dict(name=u'bar', age=i1)
         i3 = TestItem(name=u'buz', age=i2)
 
-        self.assertExportResult(i3,
-            b'<?xml version="1.0" encoding="utf-8"?>\n'
-            b'<items>'
-                b'<item>'
-                    b'<age>'
-                        b'<age>'
-                            b'<age>22</age>'
-                            b'<name>foo\xc2\xa3hoo</name>'
-                        b'</age>'
-                        b'<name>bar</name>'
-                    b'</age>'
-                    b'<name>buz</name>'
-                b'</item>'
-            b'</items>'
+        self.assertExportResult(
+            i3,
+            b"""<?xml version="1.0" encoding="utf-8"?>\n
+                <items>
+                    <item>
+                        <age>
+                            <age>
+                                <age>22</age>
+                                <name>foo\xc2\xa3hoo</name>
+                            </age>
+                            <name>bar</name>
+                        </age>
+                        <name>buz</name>
+                    </item>
+                </items>
+            """
         )
 
     def test_nested_list_item(self):
@@ -363,31 +365,35 @@ class XmlItemExporterTest(BaseItemExporterTest):
         i2 = dict(name=u'bar', v2={"egg": ["spam"]})
         i3 = TestItem(name=u'buz', age=[i1, i2])
 
-        self.assertExportResult(i3,
-            b'<?xml version="1.0" encoding="utf-8"?>\n'
-            b'<items>'
-                b'<item>'
-                    b'<age>'
-                        b'<value><name>foo</name></value>'
-                        b'<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'
-                    b'</age>'
-                    b'<name>buz</name>'
-                b'</item>'
-            b'</items>'
+        self.assertExportResult(
+            i3,
+            b"""<?xml version="1.0" encoding="utf-8"?>\n
+                <items>
+                    <item>
+                        <age>
+                            <value><name>foo</name></value>
+                            <value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>
+                        </age>
+                        <name>buz</name>
+                    </item>
+                </items>
+            """
         )
 
     def test_nonstring_types_item(self):
         item = self._get_nonstring_types_item()
-        self.assertExportResult(item,
-            b'<?xml version="1.0" encoding="utf-8"?>\n'
-            b'<items>'
-               b'<item>'
-                   b'<float>3.14</float>'
-                   b'<boolean>False</boolean>'
-                   b'<number>22</number>'
-                   b'<time>2015-01-01 01:01:01</time>'
-               b'</item>'
-            b'</items>'
+        self.assertExportResult(
+            item,
+            b"""<?xml version="1.0" encoding="utf-8"?>\n
+                <items>
+                   <item>
+                       <float>3.14</float>
+                       <boolean>False</boolean>
+                       <number>22</number>
+                       <time>2015-01-01 01:01:01</time>
+                   </item>
+                </items>
+            """
         )
 
 

tests/test_extension_telnet.py

@@ -11,8 +11,6 @@ class TelnetExtensionTest(unittest.TestCase):
     def _get_console_and_portal(self, settings=None):
         crawler = get_crawler(settings_dict=settings)
         console = TelnetConsole(crawler)
-        username = console.username
-        password = console.password
 
         # This function has some side effects we don't need for this test
         console._get_telnet_vars = lambda: {}

tests/test_feedexport.py

@@ -730,7 +730,6 @@ class FeedExportTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_export_encoding(self):
         items = [dict({'foo': u'Test\xd6'})]
-        header = ['foo']
 
         formats = {
             'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'),

tests/test_http_request.py

@@ -399,8 +399,7 @@ class FormRequestTest(RequestTest):
 
     def test_custom_encoding_bytes(self):
         data = {b'\xb5 one': b'two', b'price': b'\xa3 100'}
-        r2 = self.request_class("http://www.example.com", formdata=data,
-                                    encoding='latin1')
+        r2 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
         self.assertEqual(r2.method, 'POST')
         self.assertEqual(r2.encoding, 'latin1')
         self.assertQueryEqual(r2.body, b'price=%A3+100&%B5+one=two')
@@ -408,8 +407,7 @@ class FormRequestTest(RequestTest):
 
     def test_custom_encoding_textual_data(self):
         data = {'price': u'£ 100'}
-        r3 = self.request_class("http://www.example.com", formdata=data,
-                                    encoding='latin1')
+        r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
         self.assertEqual(r3.encoding, 'latin1')
         self.assertEqual(r3.body, b'price=%A3+100')
 
@@ -469,7 +467,7 @@ class FormRequestTest(RequestTest):
             </form>""",
             url="http://www.example.com/this/list.html",
             encoding='latin1',
-            )
+        )
         req = self.request_class.from_response(response,
                 formdata={'one': ['two', 'three'], 'six': 'seven'})
 
@@ -504,11 +502,13 @@ class FormRequestTest(RequestTest):
 
     def test_from_response_duplicate_form_key(self):
         response = _buildresponse(
-                '<form></form>',
-                url='http://www.example.com')
-        req = self.request_class.from_response(response,
-                method='GET',
-                formdata=(('foo', 'bar'), ('foo', 'baz')))
+            '<form></form>',
+            url='http://www.example.com')
+        req = self.request_class.from_response(
+            response=response,
+            method='GET',
+            formdata=(('foo', 'bar'), ('foo', 'baz')),
+        )
         self.assertEqual(urlparse(req.url).hostname, 'www.example.com')
         self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz')
 
@@ -532,9 +532,11 @@ class FormRequestTest(RequestTest):
             <input type="hidden" name="test" value="val2">
             <input type="hidden" name="test2" value="xxx">
             </form>""")
-        req = self.request_class.from_response(response,
-                formdata={'one': ['two', 'three'], 'six': 'seven'},
-                headers={"Accept-Encoding": "gzip,deflate"})
+        req = self.request_class.from_response(
+            response=response,
+            formdata={'one': ['two', 'three'], 'six': 'seven'},
+            headers={"Accept-Encoding": "gzip,deflate"},
+        )
         self.assertEqual(req.method, 'POST')
         self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded')
         self.assertEqual(req.headers['Accept-Encoding'], b'gzip,deflate')
@@ -582,9 +584,9 @@ class FormRequestTest(RequestTest):
 
     def test_from_response_override_method(self):
         response = _buildresponse(
-                '''<html><body>
-                <form action="/app"></form>
-                </body></html>''')
+            '''<html><body>
+            <form action="/app"></form>
+            </body></html>''')
         request = FormRequest.from_response(response)
         self.assertEqual(request.method, 'GET')
         request = FormRequest.from_response(response, method='POST')
@@ -592,9 +594,9 @@ class FormRequestTest(RequestTest):
 
     def test_from_response_override_url(self):
         response = _buildresponse(
-                '''<html><body>
-                <form action="/app"></form>
-                </body></html>''')
+            '''<html><body>
+            <form action="/app"></form>
+            </body></html>''')
         request = FormRequest.from_response(response)
         self.assertEqual(request.url, 'http://example.com/app')
         request = FormRequest.from_response(response, url='http://foo.bar/absolute')

tests/test_http_response.py

@@ -1,8 +1,9 @@
-# -*- coding: utf-8 -*-
 import unittest
+from warnings import catch_warnings
 
 from w3lib.encoding import resolve_encoding
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
                          XmlResponse, Headers)
 from scrapy.selector import Selector
@@ -661,6 +662,13 @@ class TextResponseTest(BaseResponseTest):
         with self.assertRaises(ValueError):
             response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]')
 
+    def test_body_as_unicode_deprecation_warning(self):
+        with catch_warnings(record=True) as warnings:
+            r1 = self.response_class("http://www.example.com", body=u'Hello', encoding='utf-8')
+            self.assertEqual(r1.body_as_unicode(), u'Hello')
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+
 
 class HtmlResponseTest(TextResponseTest):
 

tests/test_item.py

@@ -264,12 +264,12 @@ class ItemTest(unittest.TestCase):
         """Make sure the DictItem deprecation warning is not issued for
         Item"""
         with catch_warnings(record=True) as warnings:
-            item = Item()
+            Item()
             self.assertEqual(len(warnings), 0)
 
             class SubclassedItem(Item):
                 pass
-            subclassed_item = SubclassedItem()
+            SubclassedItem()
             self.assertEqual(len(warnings), 0)
 
 
@@ -321,13 +321,13 @@ class DictItemTest(unittest.TestCase):
 
     def test_deprecation_warning(self):
         with catch_warnings(record=True) as warnings:
-            dict_item = DictItem()
+            DictItem()
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
         with catch_warnings(record=True) as warnings:
             class SubclassedDictItem(DictItem):
                 pass
-            subclassed_dict_item = SubclassedDictItem()
+            SubclassedDictItem()
             self.assertEqual(len(warnings), 1)
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
 

tests/test_linkextractors.py

@@ -1,3 +1,4 @@
+import pickle
 import re
 import unittest
 from warnings import catch_warnings
@@ -413,24 +414,30 @@ class Base:
             response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)
 
             lx = self.extractor_cls()
-            self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                              Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                              Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                              Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
-                              Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
-                            )
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
+                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
+                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                ]
+            )
 
             response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
 
             lx = self.extractor_cls()
-            self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                              Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                              Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                              Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
-                              Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
-                            )
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
+                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
+                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                ]
+            )
 
         def test_link_wrong_href(self):
             html = b"""
@@ -456,6 +463,10 @@ class Base:
                 Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False),
             ])
 
+        def test_pickle_extractor(self):
+            lx = self.extractor_cls()
+            self.assertIsInstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls)
+
 
 class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
     extractor_cls = LxmlLinkExtractor

tests/test_pipeline_crawl.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import os
 import shutil
 
@@ -44,9 +43,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):
     name = 'redirectedmedia'
 
     def _process_url(self, url):
-        return add_or_replace_parameter(
-                    self.mockserver.url('/redirect-to'),
-                    'goto', url)
+        return add_or_replace_parameter(self.mockserver.url('/redirect-to'), 'goto', url)
 
 
 class FileDownloadCrawlTestCase(TestCase):

tests/test_pipeline_images.py

@@ -15,7 +15,7 @@ from scrapy.utils.python import to_bytes
 skip = False
 try:
     from PIL import Image
-except ImportError as e:
+except ImportError:
     skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
 else:
     encoders = set(('jpeg_encoder', 'jpeg_decoder'))

tests/test_responsetypes.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 from scrapy.responsetypes import responsetypes
 

tests/test_scheduler.py

@@ -46,13 +46,13 @@ class MockCrawler(Crawler):
     def __init__(self, priority_queue_cls, jobdir):
 
         settings = dict(
-                SCHEDULER_DEBUG=False,
-                SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
-                SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
-                SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
-                JOBDIR=jobdir,
-                DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter'
-                )
+            SCHEDULER_DEBUG=False,
+            SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
+            SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
+            SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
+            JOBDIR=jobdir,
+            DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
+        )
         super(MockCrawler, self).__init__(Spider, settings)
         self.engine = MockEngine(downloader=MockDownloader())
 
@@ -305,10 +305,12 @@ class StartUrlsSpider(Spider):
 class TestIntegrationWithDownloaderAwareInMemory(TestCase):
     def setUp(self):
         self.crawler = get_crawler(
-                    StartUrlsSpider,
-                    {'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
-                     'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter'}
-                    )
+            spidercls=StartUrlsSpider,
+            settings_dict={
+                'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
+                'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter',
+            },
+        )
 
     @defer.inlineCallbacks
     def tearDown(self):
@@ -329,9 +331,9 @@ class TestIncompatibility(unittest.TestCase):
 
     def _incompatible(self):
         settings = dict(
-                SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
-                CONCURRENT_REQUESTS_PER_IP=1
-                )
+            SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
+            CONCURRENT_REQUESTS_PER_IP=1,
+        )
         crawler = Crawler(Spider, settings)
         scheduler = Scheduler.from_crawler(crawler)
         spider = Spider(name='spider')

tests/test_selector.py

@@ -67,8 +67,7 @@ class SelectorTestCase(unittest.TestCase):
         headers = {'Content-Type': ['text/html; charset=utf-8']}
         response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
         x = Selector(response)
-        self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(),
-                          [u'\xa3'])
+        self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3'])
 
     def test_badly_encoded_body(self):
         # \xe9 alone isn't valid utf8 sequence

tests/test_spiderloader/__init__.py

@@ -137,6 +137,11 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
             msg = str(w[0].message)
             self.assertIn("several spiders with the same name", msg)
             self.assertIn("'spider3'", msg)
+            self.assertTrue(msg.count("'spider3'") == 2)
+
+            self.assertNotIn("'spider1'", msg)
+            self.assertNotIn("'spider2'", msg)
+            self.assertNotIn("'spider4'", msg)
 
             spiders = set(spider_loader.list())
             self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))
@@ -156,7 +161,13 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
             msg = str(w[0].message)
             self.assertIn("several spiders with the same name", msg)
             self.assertIn("'spider1'", msg)
+            self.assertTrue(msg.count("'spider1'") == 2)
+
             self.assertIn("'spider2'", msg)
+            self.assertTrue(msg.count("'spider2'") == 2)
+
+            self.assertNotIn("'spider3'", msg)
+            self.assertNotIn("'spider4'", msg)
 
             spiders = set(spider_loader.list())
             self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))

tests/test_spidermiddleware_httperror.py

@@ -21,10 +21,10 @@ class _HttpErrorSpider(MockServerSpider):
     def __init__(self, *args, **kwargs):
         super(_HttpErrorSpider, self).__init__(*args, **kwargs)
         self.start_urls = [
-           self.mockserver.url("/status?n=200"),
-           self.mockserver.url("/status?n=404"),
-           self.mockserver.url("/status?n=402"),
-           self.mockserver.url("/status?n=500"),
+            self.mockserver.url("/status?n=200"),
+            self.mockserver.url("/status?n=404"),
+            self.mockserver.url("/status?n=402"),
+            self.mockserver.url("/status?n=500"),
         ]
         self.failed = set()
         self.skipped = set()
@@ -111,8 +111,7 @@ class TestHttpErrorMiddlewareSettings(TestCase):
                 self.mw.process_spider_input(self.res402, self.spider))
 
     def test_meta_overrides_settings(self):
-        request = Request('http://scrapytest.org',
-                              meta={'handle_httpstatus_list': [404]})
+        request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
         res404 = self.res404.copy()
         res404.request = request
         res402 = self.res402.copy()
@@ -146,8 +145,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
                 self.mw.process_spider_input(self.res404, self.spider))
 
     def test_meta_overrides_settings(self):
-        request = Request('http://scrapytest.org',
-                              meta={'handle_httpstatus_list': [404]})
+        request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
         res404 = self.res404.copy()
         res404.request = request
         res402 = self.res402.copy()

tests/test_spidermiddleware_referer.py

@@ -459,7 +459,6 @@ class TestRequestMetaSettingFallback(TestCase):
         target = 'http://www.example.com'
 
         for settings, response_headers, request_meta, policy_class, check_warning in self.params[3:]:
-            spider = Spider('foo')
             mw = RefererMiddleware(Settings(settings))
 
             response = Response(origin, headers=response_headers)
@@ -478,32 +477,32 @@ class TestSettingsPolicyByName(TestCase):
 
     def test_valid_name(self):
         for s, p in [
-                (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
-                (POLICY_NO_REFERRER, NoReferrerPolicy),
-                (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
-                (POLICY_SAME_ORIGIN, SameOriginPolicy),
-                (POLICY_ORIGIN, OriginPolicy),
-                (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
-                (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
-                (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
-                (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
-            ]:
+            (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
+            (POLICY_NO_REFERRER, NoReferrerPolicy),
+            (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
+            (POLICY_SAME_ORIGIN, SameOriginPolicy),
+            (POLICY_ORIGIN, OriginPolicy),
+            (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
+            (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
+            (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
+            (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
+        ]:
             settings = Settings({'REFERRER_POLICY': s})
             mw = RefererMiddleware(settings)
             self.assertEqual(mw.default_policy, p)
 
     def test_valid_name_casevariants(self):
         for s, p in [
-                (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
-                (POLICY_NO_REFERRER, NoReferrerPolicy),
-                (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
-                (POLICY_SAME_ORIGIN, SameOriginPolicy),
-                (POLICY_ORIGIN, OriginPolicy),
-                (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
-                (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
-                (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
-                (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
-            ]:
+            (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
+            (POLICY_NO_REFERRER, NoReferrerPolicy),
+            (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
+            (POLICY_SAME_ORIGIN, SameOriginPolicy),
+            (POLICY_ORIGIN, OriginPolicy),
+            (POLICY_STRICT_ORIGIN, StrictOriginPolicy),
+            (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
+            (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
+            (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
+        ]:
             settings = Settings({'REFERRER_POLICY': s.upper()})
             mw = RefererMiddleware(settings)
             self.assertEqual(mw.default_policy, p)
@@ -511,7 +510,7 @@ class TestSettingsPolicyByName(TestCase):
     def test_invalid_name(self):
         settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'})
         with self.assertRaises(RuntimeError):
-            mw = RefererMiddleware(settings)
+            RefererMiddleware(settings)
 
 
 class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):

tests/test_squeues.py

@@ -1,4 +1,5 @@
 import pickle
+import sys
 
 from queuelib.tests import test_queue as t
 from scrapy.squeues import (
@@ -28,31 +29,13 @@ class TestLoader(ItemLoader):
 
 def nonserializable_object_test(self):
     q = self.queue()
-    try:
-        pickle.dumps(lambda x: x)
-    except Exception:
-        # Trigger Twisted bug #7989
-        import twisted.persisted.styles  # NOQA
-        self.assertRaises(ValueError, q.push, lambda x: x)
-    else:
-        # Use a different unpickleable object
-        class A:
-            pass
-
-        a = A()
-        a.__reduce__ = a.__reduce_ex__ = None
-        self.assertRaises(ValueError, q.push, a)
+    self.assertRaises(ValueError, q.push, lambda x: x)
     # Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
     sel = Selector(text='<html><body><p>some text</p></body></html>')
     self.assertRaises(ValueError, q.push, sel)
 
 
-class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
-
-    chunksize = 100000
-
-    def queue(self):
-        return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize)
+class FifoDiskQueueTestMixin:
 
     def test_serialize(self):
         q = self.queue()
@@ -66,6 +49,13 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
     test_nonserializable_object = nonserializable_object_test
 
 
+class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
+    chunksize = 100000
+
+    def queue(self):
+        return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize)
+
+
 class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
     chunksize = 1
 
@@ -82,7 +72,7 @@ class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
     chunksize = 4
 
 
-class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest):
+class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
 
     chunksize = 100000
 
@@ -116,6 +106,21 @@ class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest):
         self.assertEqual(r.url, r2.url)
         assert r2.meta['request'] is r2
 
+    def test_non_pickable_object(self):
+        q = self.queue()
+        try:
+            q.push(lambda x: x)
+        except ValueError as exc:
+            if hasattr(sys, "pypy_version_info"):
+                self.assertIsInstance(exc.__context__, pickle.PicklingError)
+            else:
+                self.assertIsInstance(exc.__context__, AttributeError)
+        sel = Selector(text='<html><body><p>some text</p></body></html>')
+        try:
+            q.push(sel)
+        except ValueError as exc:
+            self.assertIsInstance(exc.__context__, TypeError)
+
 
 class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
     chunksize = 1
@@ -133,10 +138,7 @@ class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
     chunksize = 4
 
 
-class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest):
-
-    def queue(self):
-        return MarshalLifoDiskQueue(self.qpath)
+class LifoDiskQueueTestMixin:
 
     def test_serialize(self):
         q = self.queue()
@@ -150,7 +152,13 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest):
     test_nonserializable_object = nonserializable_object_test
 
 
-class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest):
+class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
+
+    def queue(self):
+        return MarshalLifoDiskQueue(self.qpath)
+
+
+class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
 
     def queue(self):
         return PickleLifoDiskQueue(self.qpath)

tests/test_utils_defer.py

@@ -64,7 +64,7 @@ class DeferUtilsTest(unittest.TestCase):
         gotexc = False
         try:
             yield process_chain([cb1, cb_fail, cb3], 'res', 'v1', 'v2')
-        except TypeError as e:
+        except TypeError:
             gotexc = True
         self.assertTrue(gotexc)
 
@@ -104,7 +104,7 @@ class IterErrbackTest(unittest.TestCase):
         def iterbad():
             for x in range(10):
                 if x == 5:
-                    a = 1 / 0
+                    1 / 0
                 yield x
 
         errors = []

tests/test_utils_deprecate.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import inspect
 import unittest
 from unittest import mock
@@ -26,7 +25,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
 
     def test_no_warning_on_definition(self):
         with warnings.catch_warnings(record=True) as w:
-            Deprecated = create_deprecated_class('Deprecated', NewName)
+            create_deprecated_class('Deprecated', NewName)
 
         w = self._mywarnings(w)
         self.assertEqual(w, [])
@@ -218,7 +217,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
     def test_deprecate_a_class_with_custom_metaclass(self):
         Meta1 = type('Meta1', (type,), {})
         New = Meta1('New', (), {})
-        Deprecated = create_deprecated_class('Deprecated', New)
+        create_deprecated_class('Deprecated', New)
 
     def test_deprecate_subclass_of_deprecated_class(self):
         with warnings.catch_warnings(record=True) as w:

tests/test_utils_http.py

@@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase):
         chunked_body += "8\r\n" + "sequence\r\n"
         chunked_body += "0\r\n\r\n"
         body = decode_chunked_transfer(chunked_body)
-        self.assertEqual(body,
-                         "This is the data in the first chunk\r\n" +
-                         "and this is the second one\r\n" +
-                         "consequence")
+        self.assertEqual(
+            body,
+            "This is the data in the first chunk\r\nand this is the second one\r\nconsequence"
+        )

tests/test_utils_iterators.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import os
 
 from twisted.trial import unittest
@@ -93,8 +92,8 @@ class XmliterTestCase(unittest.TestCase):
             # with bytes
             XmlResponse(url="http://example.com", body=body.encode('utf-8')),
             # Unicode body needs encoding information
-            XmlResponse(url="http://example.com", body=body, encoding='utf-8')):
-
+            XmlResponse(url="http://example.com", body=body, encoding='utf-8'),
+        ):
             attrs = []
             for x in self.xmliter(r, u'þingflokkur'):
                 attrs.append((x.attrib['id'],

tests/test_utils_log.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import sys
 import logging
 import unittest

tests/test_utils_misc/__init__.py

@@ -114,8 +114,12 @@ class UtilsMiscTestCase(unittest.TestCase):
         #   2. with from_settings() constructor
         #   3. with from_crawler() constructor
         #   4. with from_settings() and from_crawler() constructor
-        spec_sets = ([], ['from_settings'], ['from_crawler'],
-                     ['from_settings', 'from_crawler'])
+        spec_sets = (
+            ['__qualname__'],
+            ['__qualname__', 'from_settings'],
+            ['__qualname__', 'from_crawler'],
+            ['__qualname__', 'from_settings', 'from_crawler'],
+        )
         for specs in spec_sets:
             m = mock.MagicMock(spec_set=specs)
             _test_with_settings(m, settings)
@@ -123,7 +127,7 @@ class UtilsMiscTestCase(unittest.TestCase):
             _test_with_crawler(m, settings, crawler)
 
         # Check adoption of crawler settings
-        m = mock.MagicMock(spec_set=['from_settings'])
+        m = mock.MagicMock(spec_set=['__qualname__', 'from_settings'])
         create_instance(m, None, crawler, *args, **kwargs)
         m.from_settings.assert_called_once_with(crawler.settings, *args,
                                                 **kwargs)
@@ -131,6 +135,10 @@ class UtilsMiscTestCase(unittest.TestCase):
         with self.assertRaises(ValueError):
             create_instance(m, None, None)
 
+        m.from_settings.return_value = None
+        with self.assertRaises(TypeError):
+            create_instance(m, settings, None)
+
     def test_set_environ(self):
         assert os.environ.get('some_test_environ') is None
         with set_environ(some_test_environ='test_value'):

tests/test_utils_signal.py

@@ -44,7 +44,7 @@ class SendCatchLogTest(unittest.TestCase):
 
     def error_handler(self, arg, handlers_called):
         handlers_called.add(self.error_handler)
-        a = 1 / 0
+        1 / 0
 
     def ok_handler(self, arg, handlers_called):
         handlers_called.add(self.ok_handler)

tests/test_utils_sitemap.py

@@ -58,10 +58,13 @@ class SitemapTest(unittest.TestCase):
   </url>
 </urlset>
 """)
-        self.assertEqual(list(s),
-            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
-             {'loc': 'http://www.example.com/2', 'lastmod': ''},
-            ])
+        self.assertEqual(
+            list(s),
+            [
+                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+                {'loc': 'http://www.example.com/2', 'lastmod': ''},
+            ]
+        )
 
     def test_sitemap_wrong_ns(self):
         """We have seen sitemaps with wrongs ns. Presumably, Google still works
@@ -80,10 +83,13 @@ class SitemapTest(unittest.TestCase):
   </url>
 </urlset>
 """)
-        self.assertEqual(list(s),
-            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
-             {'loc': 'http://www.example.com/2', 'lastmod': ''},
-            ])
+        self.assertEqual(
+            list(s),
+            [
+                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+                {'loc': 'http://www.example.com/2', 'lastmod': ''},
+            ]
+        )
 
     def test_sitemap_wrong_ns2(self):
         """We have seen sitemaps with wrongs ns. Presumably, Google still works
@@ -103,10 +109,13 @@ class SitemapTest(unittest.TestCase):
 </urlset>
 """)
         assert s.type == 'urlset'
-        self.assertEqual(list(s),
-            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
-             {'loc': 'http://www.example.com/2', 'lastmod': ''},
-            ])
+        self.assertEqual(
+            list(s),
+            [
+                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+                {'loc': 'http://www.example.com/2', 'lastmod': ''},
+            ]
+        )
 
     def test_sitemap_urls_from_robots(self):
         robots = """User-agent: *
@@ -195,11 +204,19 @@ Disallow: /forum/active/
         </url>
     </urlset>""")
 
-        self.assertEqual(list(s), [
-            {'loc': 'http://www.example.com/english/',
-             'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/']
-            }
-        ])
+        self.assertEqual(
+            list(s),
+            [
+                {
+                    'loc': 'http://www.example.com/english/',
+                    'alternate': [
+                        'http://www.example.com/deutsch/',
+                        'http://www.example.com/schweiz-deutsch/',
+                        'http://www.example.com/english/',
+                    ],
+                }
+            ]
+        )
 
     def test_xml_entity_expansion(self):
         s = Sitemap(b"""<?xml version="1.0" encoding="utf-8"?>

tests/test_utils_url.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import unittest
 
 from scrapy.spiders import Spider
@@ -77,108 +76,124 @@ class UrlUtilsTest(unittest.TestCase):
 class AddHttpIfNoScheme(unittest.TestCase):
 
     def test_add_scheme(self):
-        self.assertEqual(add_http_if_no_scheme('www.example.com'),
-                                               'http://www.example.com')
+        self.assertEqual(add_http_if_no_scheme('www.example.com'), 'http://www.example.com')
 
     def test_without_subdomain(self):
-        self.assertEqual(add_http_if_no_scheme('example.com'),
-                                               'http://example.com')
+        self.assertEqual(add_http_if_no_scheme('example.com'), 'http://example.com')
 
     def test_path(self):
-        self.assertEqual(add_http_if_no_scheme('www.example.com/some/page.html'),
-                                               'http://www.example.com/some/page.html')
+        self.assertEqual(
+            add_http_if_no_scheme('www.example.com/some/page.html'),
+            'http://www.example.com/some/page.html')
 
     def test_port(self):
-        self.assertEqual(add_http_if_no_scheme('www.example.com:80'),
-                                               'http://www.example.com:80')
+        self.assertEqual(
+            add_http_if_no_scheme('www.example.com:80'),
+            'http://www.example.com:80')
 
     def test_fragment(self):
-        self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'),
-                                               'http://www.example.com/some/page#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('www.example.com/some/page#frag'),
+            'http://www.example.com/some/page#frag')
 
     def test_query(self):
-        self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
-                                               'http://www.example.com/do?a=1&b=2&c=3')
+        self.assertEqual(
+            add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
+            'http://www.example.com/do?a=1&b=2&c=3')
 
     def test_username_password(self):
-        self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'),
-                                               'http://username:password@www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('username:password@www.example.com'),
+            'http://username:password@www.example.com')
 
     def test_complete_url(self):
-        self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
 
     def test_preserve_http(self):
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com'),
-                                               'http://www.example.com')
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com'), 'http://www.example.com')
 
     def test_preserve_http_without_subdomain(self):
-        self.assertEqual(add_http_if_no_scheme('http://example.com'),
-                                               'http://example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('http://example.com'),
+            'http://example.com')
 
     def test_preserve_http_path(self):
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'),
-                                               'http://www.example.com/some/page.html')
+        self.assertEqual(
+            add_http_if_no_scheme('http://www.example.com/some/page.html'),
+            'http://www.example.com/some/page.html')
 
     def test_preserve_http_port(self):
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com:80'),
-                                               'http://www.example.com:80')
+        self.assertEqual(
+            add_http_if_no_scheme('http://www.example.com:80'),
+            'http://www.example.com:80')
 
     def test_preserve_http_fragment(self):
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'),
-                                               'http://www.example.com/some/page#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('http://www.example.com/some/page#frag'),
+            'http://www.example.com/some/page#frag')
 
     def test_preserve_http_query(self):
-        self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
-                                               'http://www.example.com/do?a=1&b=2&c=3')
+        self.assertEqual(
+            add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
+            'http://www.example.com/do?a=1&b=2&c=3')
 
     def test_preserve_http_username_password(self):
-        self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com'),
-                                               'http://username:password@www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('http://username:password@www.example.com'),
+            'http://username:password@www.example.com')
 
     def test_preserve_http_complete_url(self):
-        self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
 
     def test_protocol_relative(self):
-        self.assertEqual(add_http_if_no_scheme('//www.example.com'),
-                                               'http://www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('//www.example.com'), 'http://www.example.com')
 
     def test_protocol_relative_without_subdomain(self):
-        self.assertEqual(add_http_if_no_scheme('//example.com'),
-                                               'http://example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('//example.com'), 'http://example.com')
 
     def test_protocol_relative_path(self):
-        self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'),
-                                               'http://www.example.com/some/page.html')
+        self.assertEqual(
+            add_http_if_no_scheme('//www.example.com/some/page.html'),
+            'http://www.example.com/some/page.html')
 
     def test_protocol_relative_port(self):
-        self.assertEqual(add_http_if_no_scheme('//www.example.com:80'),
-                                               'http://www.example.com:80')
+        self.assertEqual(
+            add_http_if_no_scheme('//www.example.com:80'),
+            'http://www.example.com:80')
 
     def test_protocol_relative_fragment(self):
-        self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page#frag'),
-                                               'http://www.example.com/some/page#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('//www.example.com/some/page#frag'),
+            'http://www.example.com/some/page#frag')
 
     def test_protocol_relative_query(self):
-        self.assertEqual(add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
-                                               'http://www.example.com/do?a=1&b=2&c=3')
+        self.assertEqual(
+            add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
+            'http://www.example.com/do?a=1&b=2&c=3')
 
     def test_protocol_relative_username_password(self):
-        self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com'),
-                                               'http://username:password@www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('//username:password@www.example.com'),
+            'http://username:password@www.example.com')
 
     def test_protocol_relative_complete_url(self):
-        self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
-                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+        self.assertEqual(
+            add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+            'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
 
     def test_preserve_https(self):
-        self.assertEqual(add_http_if_no_scheme('https://www.example.com'),
-                                               'https://www.example.com')
+        self.assertEqual(
+            add_http_if_no_scheme('https://www.example.com'),
+            'https://www.example.com')
 
     def test_preserve_ftp(self):
-        self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'),
-                                               'ftp://www.example.com')
+        self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), 'ftp://www.example.com')
 
 
 class GuessSchemeTest(unittest.TestCase):
@@ -202,41 +217,49 @@ def create_skipped_scheme_t(args):
     return do_expected
 
 
-for k, args in enumerate([
-            ('/index', 'file://'),
-            ('/index.html', 'file://'),
-            ('./index.html', 'file://'),
-            ('../index.html', 'file://'),
-            ('../../index.html', 'file://'),
-            ('./data/index.html', 'file://'),
-            ('.hidden/data/index.html', 'file://'),
-            ('/home/user/www/index.html', 'file://'),
-            ('//home/user/www/index.html', 'file://'),
-            ('file:///home/user/www/index.html', 'file://'),
+for k, args in enumerate(
+    [
+        ('/index', 'file://'),
+        ('/index.html', 'file://'),
+        ('./index.html', 'file://'),
+        ('../index.html', 'file://'),
+        ('../../index.html', 'file://'),
+        ('./data/index.html', 'file://'),
+        ('.hidden/data/index.html', 'file://'),
+        ('/home/user/www/index.html', 'file://'),
+        ('//home/user/www/index.html', 'file://'),
+        ('file:///home/user/www/index.html', 'file://'),
 
-            ('index.html', 'http://'),
-            ('example.com', 'http://'),
-            ('www.example.com', 'http://'),
-            ('www.example.com/index.html', 'http://'),
-            ('http://example.com', 'http://'),
-            ('http://example.com/index.html', 'http://'),
-            ('localhost', 'http://'),
-            ('localhost/index.html', 'http://'),
+        ('index.html', 'http://'),
+        ('example.com', 'http://'),
+        ('www.example.com', 'http://'),
+        ('www.example.com/index.html', 'http://'),
+        ('http://example.com', 'http://'),
+        ('http://example.com/index.html', 'http://'),
+        ('localhost', 'http://'),
+        ('localhost/index.html', 'http://'),
 
-            # some corner cases (default to http://)
-            ('/', 'http://'),
-            ('.../test', 'http://'),
-
-        ], start=1):
+        # some corner cases (default to http://)
+        ('/', 'http://'),
+        ('.../test', 'http://'),
+    ],
+    start=1,
+):
     t_method = create_guess_scheme_t(args)
     t_method.__name__ = 'test_uri_%03d' % k
     setattr(GuessSchemeTest, t_method.__name__, t_method)
 
 # TODO: the following tests do not pass with current implementation
-for k, args in enumerate([
-            (r'C:\absolute\path\to\a\file.html', 'file://',
-             'Windows filepath are not supported for scrapy shell'),
-        ], start=1):
+for k, args in enumerate(
+    [
+        (
+            r'C:\absolute\path\to\a\file.html',
+            'file://',
+            'Windows filepath are not supported for scrapy shell',
+        ),
+    ],
+    start=1,
+):
     t_method = create_skipped_scheme_t(args)
     t_method.__name__ = 'test_uri_skipped_%03d' % k
     setattr(GuessSchemeTest, t_method.__name__, t_method)
@@ -272,7 +295,7 @@ class StripUrl(unittest.TestCase):
             ('http://www.example.com',
              True,
              'http://www.example.com/'),
-            ]:
+        ]:
             self.assertEqual(strip_url(input_url, origin_only=origin), output_url)
 
     def test_credentials(self):
@@ -285,7 +308,7 @@ class StripUrl(unittest.TestCase):
 
             ('ftp://username:password@www.example.com/index.html?somekey=somevalue#section',
              'ftp://www.example.com/index.html?somekey=somevalue'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i, strip_credentials=True), o)
 
     def test_credentials_encoded_delims(self):
@@ -304,7 +327,7 @@ class StripUrl(unittest.TestCase):
             # password: "user@domain.com"
             ('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section',
              'ftp://www.example.com/index.html?somekey=somevalue'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i, strip_credentials=True), o)
 
     def test_default_ports_creds_off(self):
@@ -332,7 +355,7 @@ class StripUrl(unittest.TestCase):
 
             ('ftp://username:password@www.example.com:221/file.txt',
              'ftp://www.example.com:221/file.txt'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i), o)
 
     def test_default_ports(self):
@@ -360,7 +383,7 @@ class StripUrl(unittest.TestCase):
 
             ('ftp://username:password@www.example.com:221/file.txt',
              'ftp://username:password@www.example.com:221/file.txt'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o)
 
     def test_default_ports_keep(self):
@@ -388,7 +411,7 @@ class StripUrl(unittest.TestCase):
 
             ('ftp://username:password@www.example.com:221/file.txt',
              'ftp://username:password@www.example.com:221/file.txt'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o)
 
     def test_origin_only(self):
@@ -404,7 +427,7 @@ class StripUrl(unittest.TestCase):
 
             ('https://username:password@www.example.com:443/index.html',
              'https://www.example.com/'),
-            ]:
+        ]:
             self.assertEqual(strip_url(i, origin_only=True), o)
 
 

tests/test_webclient.py

@@ -18,6 +18,14 @@ except ImportError:
 from twisted.python.filepath import FilePath
 from twisted.protocols.policies import WrappingFactory
 from twisted.internet.defer import inlineCallbacks
+from twisted.web.test.test_webclient import (
+    ForeverTakingResource,
+    ErrorResource,
+    NoLengthResource,
+    HostHeaderResource,
+    PayloadResource,
+    BrokenDownloadResource,
+)
 
 from scrapy.core.downloader import webclient as client
 from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
@@ -53,29 +61,29 @@ class ParseUrlTestCase(unittest.TestCase):
     def testParse(self):
         lip = '127.0.0.1'
         tests = (
-    ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
-    ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
-    ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
-    ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
-    ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
-    ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
+            ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
+            ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
+            ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
+            ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
+            ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
+            ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
 
-    ("http://127.0.0.1", ('http', lip, lip, 80, '/')),
-    ("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
-    ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
-    ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
-    ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
-    ("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
-    ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
-    ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
+            ("http://127.0.0.1", ('http', lip, lip, 80, '/')),
+            ("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
+            ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
+            ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
+            ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
+            ("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
+            ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
+            ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
 
-    ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
-    ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
-    ("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
+            ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
+            ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
+            ("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
 
-    ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
-    ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
-    )
+            ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
+            ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
+        )
 
         for url, test in tests:
             test = tuple(
@@ -149,7 +157,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             headers={
                 'X-Meta-Single': 'single',
                 'X-Meta-Multivalued': ['value1', 'value2'],
-                }))
+            },
+        ))
 
         self._test(factory,
             b"GET /bar HTTP/1.0\r\n"
@@ -165,7 +174,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             headers=Headers({
                 'X-Meta-Single': 'single',
                 'X-Meta-Multivalued': ['value1', 'value2'],
-                })))
+            }),
+        ))
 
         self._test(factory,
             b"GET /bar HTTP/1.0\r\n"
@@ -200,11 +210,6 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             Headers({'Hello': ['World'], 'Foo': ['Bar']}))
 
 
-from twisted.web.test.test_webclient import ForeverTakingResource, \
-        ErrorResource, NoLengthResource, HostHeaderResource, \
-        PayloadResource, BrokenDownloadResource
-
-
 class EncodingResource(resource.Resource):
     out_encoding = 'cp1251'
 

tox.ini

@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = security,flake8,py3
+envlist = security,flake8,py
 minversion = 1.7.0
 
 [testenv]
@@ -38,6 +38,19 @@ deps =
     pytest-flake8
 commands =
     py.test --flake8 {posargs:docs scrapy tests}
+    
+[testenv:pylint]
+basepython = python3
+deps =
+    {[testenv]deps}
+    # Optional dependencies
+    boto
+    reppy
+    robotexclusionrulesparser
+    # Test dependencies
+    pylint
+commands =
+    pylint conftest.py docs extras scrapy setup.py tests
 
 [testenv:pypy3]
 basepython = pypy3