Merge branch 'master' into nameless-spiders

2020-05-07 19:39:42 +02:00 · 2020-05-07 19:39:42 +02:00 · 9c58fbc3a9
parent 30643c1427 77492cd945
commit 9c58fbc3a9
19 changed files with 248 additions and 190 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -15,19 +15,25 @@ matrix:
      python: 3.7  # Keep in sync with .readthedocs.yml

    - env: TOXENV=pypy3
-    - python: 3.5
+    - env: TOXENV=py
+      python: 3.5
    - env: TOXENV=pinned
      python: 3.5
    - env: TOXENV=asyncio
      python: 3.5.2
-    - python: 3.6
-    - python: 3.7
-    - env: PYPI_RELEASE_JOB=true
+    - env: TOXENV=py
+      python: 3.6
+    - env: TOXENV=py
+      python: 3.7
+    - env: TOXENV=py PYPI_RELEASE_JOB=true
      python: 3.8
+      dist: bionic
    - env: TOXENV=extra-deps
      python: 3.8
+      dist: bionic
    - env: TOXENV=asyncio
      python: 3.8
+      dist: bionic
 install:
  - |
      if [ "$TOXENV" = "pypy3" ]; then
--- a/pytest.ini
+++ b/pytest.ini
@ -50,26 +50,26 @@ flake8-ignore =
    scrapy/core/engine.py E501 E128
    scrapy/core/scheduler.py E501
    scrapy/core/scraper.py E501 E128
-    scrapy/core/spidermw.py E501 E126
+    scrapy/core/spidermw.py E501
    scrapy/core/downloader/__init__.py E501
-    scrapy/core/downloader/contextfactory.py E501 E128 E126
+    scrapy/core/downloader/contextfactory.py E501 E128
    scrapy/core/downloader/middleware.py E501
    scrapy/core/downloader/tls.py E501
-    scrapy/core/downloader/webclient.py E501 E128 E126
+    scrapy/core/downloader/webclient.py E501 E128
    scrapy/core/downloader/handlers/__init__.py E501
    scrapy/core/downloader/handlers/ftp.py E501 E128
    scrapy/core/downloader/handlers/http10.py E501
    scrapy/core/downloader/handlers/http11.py E501
-    scrapy/core/downloader/handlers/s3.py E501 E128 E126
+    scrapy/core/downloader/handlers/s3.py E501 E128
    # scrapy/downloadermiddlewares
    scrapy/downloadermiddlewares/ajaxcrawl.py E501
    scrapy/downloadermiddlewares/decompression.py E501
    scrapy/downloadermiddlewares/defaultheaders.py E501
-    scrapy/downloadermiddlewares/httpcache.py E501 E126
+    scrapy/downloadermiddlewares/httpcache.py E501
    scrapy/downloadermiddlewares/httpcompression.py E501 E128
    scrapy/downloadermiddlewares/httpproxy.py E501
    scrapy/downloadermiddlewares/redirect.py E501
-    scrapy/downloadermiddlewares/retry.py E501 E126
+    scrapy/downloadermiddlewares/retry.py E501
    scrapy/downloadermiddlewares/robotstxt.py E501
    scrapy/downloadermiddlewares/stats.py E501
    # scrapy/extensions
@ -88,7 +88,7 @@ flake8-ignore =
    scrapy/http/request/form.py E501
    scrapy/http/request/json_request.py E501
    scrapy/http/response/__init__.py E501 E128
-    scrapy/http/response/text.py E501 E128 E124
+    scrapy/http/response/text.py E501 E128
    # scrapy/linkextractors
    scrapy/linkextractors/__init__.py E501 E402
    scrapy/linkextractors/lxmlhtml.py E501
@ -110,7 +110,7 @@ flake8-ignore =
    # scrapy/spidermiddlewares
    scrapy/spidermiddlewares/httperror.py E501
    scrapy/spidermiddlewares/offsite.py E501
-    scrapy/spidermiddlewares/referer.py E501 E129
+    scrapy/spidermiddlewares/referer.py E501
    scrapy/spidermiddlewares/urllength.py E501
    # scrapy/spiders
    scrapy/spiders/__init__.py E501 E402
@ -164,12 +164,12 @@ flake8-ignore =
    scrapy/robotstxt.py E501
    scrapy/shell.py E501
    scrapy/signalmanager.py E501
-    scrapy/spiderloader.py F841 E501 E126
+    scrapy/spiderloader.py F841 E501
    scrapy/squeues.py E128
    scrapy/statscollectors.py E501
    # tests
    tests/__init__.py E402 E501
-    tests/mockserver.py E501 E126
+    tests/mockserver.py E501
    tests/pipelines.py F841
    tests/spiders.py E501
    tests/test_closespider.py E501
@ -181,37 +181,37 @@ flake8-ignore =
    tests/test_crawl.py E501 E741
    tests/test_crawler.py F841 E501
    tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E124 E128 E501 E126
+    tests/test_downloader_handlers.py E128 E501
    tests/test_downloadermiddleware.py E501
    tests/test_downloadermiddleware_ajaxcrawlable.py E501
-    tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
+    tests/test_downloadermiddleware_cookies.py E741 E501 E128
    tests/test_downloadermiddleware_defaultheaders.py E501
    tests/test_downloadermiddleware_downloadtimeout.py E501
    tests/test_downloadermiddleware_httpcache.py E501
-    tests/test_downloadermiddleware_httpcompression.py E501 E126
+    tests/test_downloadermiddleware_httpcompression.py E501
    tests/test_downloadermiddleware_decompression.py E501
    tests/test_downloadermiddleware_httpproxy.py E501 E128
    tests/test_downloadermiddleware_redirect.py E501 E128
-    tests/test_downloadermiddleware_retry.py E501 E128 E126
+    tests/test_downloadermiddleware_retry.py E501 E128
    tests/test_downloadermiddleware_robotstxt.py E501
    tests/test_downloadermiddleware_stats.py E501
-    tests/test_dupefilters.py E501 E741 E128 E124
+    tests/test_dupefilters.py E501 E741 E128
    tests/test_engine.py E501 E128
-    tests/test_exporters.py E501 E128 E124
+    tests/test_exporters.py E501 E128
    tests/test_extension_telnet.py F841
    tests/test_feedexport.py E501 F841
    tests/test_http_cookies.py E501
    tests/test_http_headers.py E501
-    tests/test_http_request.py E402 E501 E128 E128 E126
+    tests/test_http_request.py E402 E501 E128 E128
    tests/test_http_response.py E501 E128
    tests/test_item.py E128 F841
    tests/test_link.py E501
-    tests/test_linkextractors.py E501 E128 E124
+    tests/test_linkextractors.py E501 E128
    tests/test_loader.py E501 E741 E128 E117
-    tests/test_logformatter.py E128 E501 E122
+    tests/test_logformatter.py E128 E501
    tests/test_mail.py E128 E501
    tests/test_middleware.py E501 E128
-    tests/test_pipeline_crawl.py E501 E128 E126
+    tests/test_pipeline_crawl.py E501 E128
    tests/test_pipeline_files.py E501
    tests/test_pipeline_images.py F841 E501
    tests/test_pipeline_media.py E501 E741 E128
@ -219,14 +219,14 @@ flake8-ignore =
    tests/test_request_cb_kwargs.py E501
    tests/test_responsetypes.py E501
    tests/test_robotstxt_interface.py E501 E501
-    tests/test_scheduler.py E501 E126
+    tests/test_scheduler.py E501
    tests/test_selector.py E501
    tests/test_spider.py E501
    tests/test_spidermiddleware.py E501
    tests/test_spidermiddleware_httperror.py E128 E501 E121
    tests/test_spidermiddleware_offsite.py E501 E128 E111
    tests/test_spidermiddleware_output_chain.py E501
-    tests/test_spidermiddleware_referer.py E501 F841 E124 E501 E121
+    tests/test_spidermiddleware_referer.py E501 F841 E501 E121
    tests/test_squeues.py E501 E741
    tests/test_utils_asyncio.py E501
    tests/test_utils_conf.py E501 E128
@ -235,16 +235,16 @@ flake8-ignore =
    tests/test_utils_defer.py E501 F841
    tests/test_utils_deprecate.py F841 E501
    tests/test_utils_http.py E501 E128
-    tests/test_utils_iterators.py E501 E128 E129
+    tests/test_utils_iterators.py E501 E128
    tests/test_utils_log.py E741
    tests/test_utils_python.py E501
    tests/test_utils_reqser.py E501 E128
    tests/test_utils_request.py E501 E128
    tests/test_utils_response.py E501
    tests/test_utils_signal.py E741 F841
-    tests/test_utils_sitemap.py E128 E501 E124
-    tests/test_utils_url.py E501 E501 E126
-    tests/test_webclient.py E501 E128 E122 E402 E126
+    tests/test_utils_sitemap.py E128 E501
+    tests/test_utils_url.py E501 E501
+    tests/test_webclient.py E501 E128 E402
    tests/test_cmdline/__init__.py E501
    tests/test_settings/__init__.py E501 E128
    tests/test_spiderloader/__init__.py E128 E501
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@ -86,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
        #
        # This means that a website like https://www.cacert.org will be rejected
        # by default, since CAcert.org CA certificate is seldom shipped.
-        return optionsForClientTLS(hostname.decode("ascii"),
-                                   trustRoot=platformTrust(),
-                                   extraCertificateOptions={
-                                        'method': self._ssl_method,
-                                   })
+        return optionsForClientTLS(
+            hostname=hostname.decode("ascii"),
+            trustRoot=platformTrust(),
+            extraCertificateOptions={'method': self._ssl_method},
+        )
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@ -100,11 +100,12 @@ class S3DownloadHandler:
                url=url, headers=awsrequest.headers.items())
        else:
            signed_headers = self.conn.make_request(
-                    method=request.method,
-                    bucket=bucket,
-                    key=unquote(p.path),
-                    query_args=unquote(p.query),
-                    headers=request.headers,
-                    data=request.body)
+                method=request.method,
+                bucket=bucket,
+                key=unquote(p.path),
+                query_args=unquote(p.query),
+                headers=request.headers,
+                data=request.body,
+            )
            request = request.replace(url=url, headers=signed_headers)
        return self._download_http(request, spider)
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@ -88,8 +88,8 @@ class ScrapyHTTPPageGetter(HTTPClient):
            self.transport.stopProducing()

        self.factory.noPage(
-                defer.TimeoutError("Getting %s took longer than %s seconds." %
-                                   (self.factory.url, self.factory.timeout)))
+            defer.TimeoutError("Getting %s took longer than %s seconds."
+                               % (self.factory.url, self.factory.timeout)))


 class ScrapyHTTPClientFactory(HTTPClientFactory):
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@ -12,9 +12,15 @@ once the spider has finished crawling all regular (non failed) pages.
 import logging

 from twisted.internet import defer
-from twisted.internet.error import TimeoutError, DNSLookupError, \
-        ConnectionRefusedError, ConnectionDone, ConnectError, \
-        ConnectionLost, TCPTimedOutError
+from twisted.internet.error import (
+    ConnectError,
+    ConnectionDone,
+    ConnectionLost,
+    ConnectionRefusedError,
+    DNSLookupError,
+    TCPTimedOutError,
+    TimeoutError,
+)
 from twisted.web.client import ResponseFailed

 from scrapy.exceptions import NotConfigured
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@ -16,6 +16,7 @@ class SpiderLoader:
    SpiderLoader is a class which locates and loads spiders
    in a Scrapy project.
    """
+
    def __init__(self, settings):
        self.require_name = settings.getbool('SPIDER_LOADER_REQUIRE_NAME')
        if self.require_name:
@ -34,16 +35,21 @@ class SpiderLoader:
        self._load_all_spiders()

    def _check_name_duplicates(self):
-        dupes = ["\n".join("  {cls} named {name!r} (in {module})".format(
-                                module=mod, cls=cls, name=name)
-                           for (mod, cls) in locations)
-                 for name, locations in self._found.items()
-                 if len(locations) > 1]
+        dupes = []
+        for name, locations in self._found.items():
+            dupes.extend([
+                "  {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name)
+                for mod, cls in locations
+                if len(locations) > 1
+            ])
+
        if dupes:
-            msg = ("There are several spiders with the same name:\n\n"
-                   "{}\n\n  This can cause unexpected behavior.".format(
-                        "\n\n".join(dupes)))
-            warnings.warn(msg, UserWarning)
+            dupes_string = "\n\n".join(dupes)
+            warnings.warn(
+                "There are several spiders with the same name:\n\n"
+                "{}\n\n  This can cause unexpected behavior.".format(dupes_string),
+                category=UserWarning,
+            )

    def _load_spiders(self, module):
        classes = iter_spider_classes(module, require_name=self.require_name)
@ -58,12 +64,15 @@ class SpiderLoader:
            try:
                for module in walk_modules(name):
                    self._load_spiders(module)
-            except ImportError as e:
+            except ImportError:
                if self.warn_only:
-                    msg = ("\n{tb}Could not load spiders from module '{modname}'. "
-                           "See above traceback for details.".format(
-                                modname=name, tb=traceback.format_exc()))
-                    warnings.warn(msg, RuntimeWarning)
+                    warnings.warn(
+                        "\n{tb}Could not load spiders from module '{modname}'. "
+                        "See above traceback for details.".format(
+                            modname=name, tb=traceback.format_exc()
+                        ),
+                        category=RuntimeWarning,
+                    )
                else:
                    raise
        self._check_name_duplicates()
@ -86,8 +95,10 @@ class SpiderLoader:
        """
        Return the list of spider names that can handle the given request.
        """
-        return [name for name, cls in self._spiders.items()
-                if cls.handles_request(request)]
+        return [
+            name for name, cls in self._spiders.items()
+            if cls.handles_request(request)
+        ]

    def list(self):
        """
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@ -313,14 +313,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
    def test_ipv6_alternative_name_resolver(self):
        log = self.run_script('alternative_name_resolver.py')
        self.assertIn('Spider closed (finished)', log)
-        self.assertTrue(any([
-            "twisted.internet.error.ConnectionRefusedError" in log,
-            "twisted.internet.error.ConnectError" in log,
-        ]))
-        self.assertTrue(any([
-            "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
-            "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
-        ]))
+        self.assertNotIn("twisted.internet.error.DNSLookupError", log)

    def test_reactor_select(self):
        log = self.run_script("twisted_reactor_select.py")
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@ -139,10 +139,12 @@ class CookiesMiddlewareTest(TestCase):

    def test_complex_cookies(self):
        # merge some cookies into jar
-        cookies = [{'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'},
-                {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'},
-                {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},
-                {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}]
+        cookies = [
+            {'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'},
+            {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'},
+            {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},
+            {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'},
+        ]

        req = Request('http://scrapytest.org/', cookies=cookies)
        self.mw.process_request(req, self.spider)
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@ -1,8 +1,14 @@
 import unittest
 from twisted.internet import defer
-from twisted.internet.error import TimeoutError, DNSLookupError, \
-        ConnectionRefusedError, ConnectionDone, ConnectError, \
-        ConnectionLost, TCPTimedOutError
+from twisted.internet.error import (
+    ConnectError,
+    ConnectionDone,
+    ConnectionLost,
+    ConnectionRefusedError,
+    DNSLookupError,
+    TCPTimedOutError,
+    TimeoutError,
+)
 from twisted.web.client import ResponseFailed

 from scrapy.downloadermiddlewares.retry import RetryMiddleware
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@ -197,8 +197,7 @@ class RFPDupeFilterTest(unittest.TestCase):

            r1 = Request('http://scrapytest.org/index.html')
            r2 = Request('http://scrapytest.org/index.html',
-                headers={'Referer': 'http://scrapytest.org/INDEX.html'}
-            )
+                         headers={'Referer': 'http://scrapytest.org/INDEX.html'})

            dupefilter.log(r1, spider)
            dupefilter.log(r2, spider)
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@ -342,20 +342,22 @@ class XmlItemExporterTest(BaseItemExporterTest):
        i2 = dict(name=u'bar', age=i1)
        i3 = TestItem(name=u'buz', age=i2)

-        self.assertExportResult(i3,
-            b'<?xml version="1.0" encoding="utf-8"?>\n'
-            b'<items>'
-                b'<item>'
-                    b'<age>'
-                        b'<age>'
-                            b'<age>22</age>'
-                            b'<name>foo\xc2\xa3hoo</name>'
-                        b'</age>'
-                        b'<name>bar</name>'
-                    b'</age>'
-                    b'<name>buz</name>'
-                b'</item>'
-            b'</items>'
+        self.assertExportResult(
+            i3,
+            b"""<?xml version="1.0" encoding="utf-8"?>\n
+                <items>
+                    <item>
+                        <age>
+                            <age>
+                                <age>22</age>
+                                <name>foo\xc2\xa3hoo</name>
+                            </age>
+                            <name>bar</name>
+                        </age>
+                        <name>buz</name>
+                    </item>
+                </items>
+            """
        )

    def test_nested_list_item(self):
@ -363,31 +365,35 @@ class XmlItemExporterTest(BaseItemExporterTest):
        i2 = dict(name=u'bar', v2={"egg": ["spam"]})
        i3 = TestItem(name=u'buz', age=[i1, i2])

-        self.assertExportResult(i3,
-            b'<?xml version="1.0" encoding="utf-8"?>\n'
-            b'<items>'
-                b'<item>'
-                    b'<age>'
-                        b'<value><name>foo</name></value>'
-                        b'<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'
-                    b'</age>'
-                    b'<name>buz</name>'
-                b'</item>'
-            b'</items>'
+        self.assertExportResult(
+            i3,
+            b"""<?xml version="1.0" encoding="utf-8"?>\n
+                <items>
+                    <item>
+                        <age>
+                            <value><name>foo</name></value>
+                            <value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>
+                        </age>
+                        <name>buz</name>
+                    </item>
+                </items>
+            """
        )

    def test_nonstring_types_item(self):
        item = self._get_nonstring_types_item()
-        self.assertExportResult(item,
-            b'<?xml version="1.0" encoding="utf-8"?>\n'
-            b'<items>'
-               b'<item>'
-                   b'<float>3.14</float>'
-                   b'<boolean>False</boolean>'
-                   b'<number>22</number>'
-                   b'<time>2015-01-01 01:01:01</time>'
-               b'</item>'
-            b'</items>'
+        self.assertExportResult(
+            item,
+            b"""<?xml version="1.0" encoding="utf-8"?>\n
+                <items>
+                   <item>
+                       <float>3.14</float>
+                       <boolean>False</boolean>
+                       <number>22</number>
+                       <time>2015-01-01 01:01:01</time>
+                   </item>
+                </items>
+            """
        )


--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@ -502,11 +502,13 @@ class FormRequestTest(RequestTest):

    def test_from_response_duplicate_form_key(self):
        response = _buildresponse(
-                '<form></form>',
-                url='http://www.example.com')
-        req = self.request_class.from_response(response,
-                method='GET',
-                formdata=(('foo', 'bar'), ('foo', 'baz')))
+            '<form></form>',
+            url='http://www.example.com')
+        req = self.request_class.from_response(
+            response=response,
+            method='GET',
+            formdata=(('foo', 'bar'), ('foo', 'baz')),
+        )
        self.assertEqual(urlparse(req.url).hostname, 'www.example.com')
        self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz')

@ -530,9 +532,11 @@ class FormRequestTest(RequestTest):
            <input type="hidden" name="test" value="val2">
            <input type="hidden" name="test2" value="xxx">
            </form>""")
-        req = self.request_class.from_response(response,
-                formdata={'one': ['two', 'three'], 'six': 'seven'},
-                headers={"Accept-Encoding": "gzip,deflate"})
+        req = self.request_class.from_response(
+            response=response,
+            formdata={'one': ['two', 'three'], 'six': 'seven'},
+            headers={"Accept-Encoding": "gzip,deflate"},
+        )
        self.assertEqual(req.method, 'POST')
        self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded')
        self.assertEqual(req.headers['Accept-Encoding'], b'gzip,deflate')
@ -580,9 +584,9 @@ class FormRequestTest(RequestTest):

    def test_from_response_override_method(self):
        response = _buildresponse(
-                '''<html><body>
-                <form action="/app"></form>
-                </body></html>''')
+            '''<html><body>
+            <form action="/app"></form>
+            </body></html>''')
        request = FormRequest.from_response(response)
        self.assertEqual(request.method, 'GET')
        request = FormRequest.from_response(response, method='POST')
@ -590,9 +594,9 @@ class FormRequestTest(RequestTest):

    def test_from_response_override_url(self):
        response = _buildresponse(
-                '''<html><body>
-                <form action="/app"></form>
-                </body></html>''')
+            '''<html><body>
+            <form action="/app"></form>
+            </body></html>''')
        request = FormRequest.from_response(response)
        self.assertEqual(request.url, 'http://example.com/app')
        request = FormRequest.from_response(response, url='http://foo.bar/absolute')
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@ -413,24 +413,30 @@ class Base:
            response = HtmlResponse("http://example.com/index.xhtml", body=xhtml)

            lx = self.extractor_cls()
-            self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                              Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                              Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                              Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
-                              Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
-                            )
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
+                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
+                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                ]
+            )

            response = XmlResponse("http://example.com/index.xhtml", body=xhtml)

            lx = self.extractor_cls()
-            self.assertEqual(lx.extract_links(response),
-                             [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
-                              Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
-                              Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
-                              Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
-                              Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)]
-                            )
+            self.assertEqual(
+                lx.extract_links(response),
+                [
+                    Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
+                    Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
+                    Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True),
+                    Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False),
+                    Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+                ]
+            )

        def test_link_wrong_href(self):
            html = b"""
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@ -44,9 +44,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):
    name = 'redirectedmedia'

    def _process_url(self, url):
-        return add_or_replace_parameter(
-                    self.mockserver.url('/redirect-to'),
-                    'goto', url)
+        return add_or_replace_parameter(self.mockserver.url('/redirect-to'), 'goto', url)


 class FileDownloadCrawlTestCase(TestCase):
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@ -93,8 +93,8 @@ class XmliterTestCase(unittest.TestCase):
            # with bytes
            XmlResponse(url="http://example.com", body=body.encode('utf-8')),
            # Unicode body needs encoding information
-            XmlResponse(url="http://example.com", body=body, encoding='utf-8')):
-
+            XmlResponse(url="http://example.com", body=body, encoding='utf-8'),
+        ):
            attrs = []
            for x in self.xmliter(r, u'þingflokkur'):
                attrs.append((x.attrib['id'],
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@ -58,10 +58,13 @@ class SitemapTest(unittest.TestCase):
  </url>
 </urlset>
 """)
-        self.assertEqual(list(s),
-            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
-             {'loc': 'http://www.example.com/2', 'lastmod': ''},
-            ])
+        self.assertEqual(
+            list(s),
+            [
+                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+                {'loc': 'http://www.example.com/2', 'lastmod': ''},
+            ]
+        )

    def test_sitemap_wrong_ns(self):
        """We have seen sitemaps with wrongs ns. Presumably, Google still works
@ -80,10 +83,13 @@ class SitemapTest(unittest.TestCase):
  </url>
 </urlset>
 """)
-        self.assertEqual(list(s),
-            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
-             {'loc': 'http://www.example.com/2', 'lastmod': ''},
-            ])
+        self.assertEqual(
+            list(s),
+            [
+                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+                {'loc': 'http://www.example.com/2', 'lastmod': ''},
+            ]
+        )

    def test_sitemap_wrong_ns2(self):
        """We have seen sitemaps with wrongs ns. Presumably, Google still works
@ -103,10 +109,13 @@ class SitemapTest(unittest.TestCase):
 </urlset>
 """)
        assert s.type == 'urlset'
-        self.assertEqual(list(s),
-            [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
-             {'loc': 'http://www.example.com/2', 'lastmod': ''},
-            ])
+        self.assertEqual(
+            list(s),
+            [
+                {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+                {'loc': 'http://www.example.com/2', 'lastmod': ''},
+            ]
+        )

    def test_sitemap_urls_from_robots(self):
        robots = """User-agent: *
@ -195,11 +204,19 @@ Disallow: /forum/active/
        </url>
    </urlset>""")

-        self.assertEqual(list(s), [
-            {'loc': 'http://www.example.com/english/',
-             'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/']
-            }
-        ])
+        self.assertEqual(
+            list(s),
+            [
+                {
+                    'loc': 'http://www.example.com/english/',
+                    'alternate': [
+                        'http://www.example.com/deutsch/',
+                        'http://www.example.com/schweiz-deutsch/',
+                        'http://www.example.com/english/',
+                    ],
+                }
+            ]
+        )

    def test_xml_entity_expansion(self):
        s = Sitemap(b"""<?xml version="1.0" encoding="utf-8"?>
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@ -18,6 +18,14 @@ except ImportError:
 from twisted.python.filepath import FilePath
 from twisted.protocols.policies import WrappingFactory
 from twisted.internet.defer import inlineCallbacks
+from twisted.web.test.test_webclient import (
+    ForeverTakingResource,
+    ErrorResource,
+    NoLengthResource,
+    HostHeaderResource,
+    PayloadResource,
+    BrokenDownloadResource,
+)

 from scrapy.core.downloader import webclient as client
 from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
@ -53,29 +61,29 @@ class ParseUrlTestCase(unittest.TestCase):
    def testParse(self):
        lip = '127.0.0.1'
        tests = (
-    ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
-    ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
-    ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
-    ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
-    ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
-    ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
+            ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
+            ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
+            ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
+            ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
+            ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
+            ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),

-    ("http://127.0.0.1", ('http', lip, lip, 80, '/')),
-    ("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
-    ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
-    ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
-    ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
-    ("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
-    ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
-    ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
+            ("http://127.0.0.1", ('http', lip, lip, 80, '/')),
+            ("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
+            ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
+            ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
+            ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
+            ("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
+            ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
+            ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),

-    ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
-    ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
-    ("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
+            ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
+            ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
+            ("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),

-    ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
-    ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
-    )
+            ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
+            ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
+        )

        for url, test in tests:
            test = tuple(
@ -202,11 +210,6 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
            Headers({'Hello': ['World'], 'Foo': ['Bar']}))


-from twisted.web.test.test_webclient import ForeverTakingResource, \
-        ErrorResource, NoLengthResource, HostHeaderResource, \
-        PayloadResource, BrokenDownloadResource
-
-
 class EncodingResource(resource.Resource):
    out_encoding = 'cp1251'

--- a/tox.ini
+++ b/tox.ini
@ -4,7 +4,7 @@
 # and then run "tox" from this directory.

 [tox]
-envlist = security,flake8,py3
+envlist = security,flake8,py
 minversion = 1.7.0

 [testenv]