E501 compliance

2020-03-22 21:34:20 -03:00 · 2020-03-22 21:34:20 -03:00 · 36a3913a41
parent 0ee04e1e91
commit 36a3913a41
5 changed files with 85 additions and 132 deletions
--- a/pytest.ini
+++ b/pytest.ini
@ -20,6 +20,7 @@ addopts =
 twisted = 1
 markers =
    only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
+flake8-max-line-length = 119
 flake8-ignore =
    W503
    # Files that are only meant to provide top-level imports are expected not
@ -27,146 +28,85 @@ flake8-ignore =
    scrapy/core/downloader/handlers/http.py F401
    scrapy/http/__init__.py F401
    # Issues pending a review:
-    # extras
-    extras/qps-bench-server.py E501
-    extras/qpsclient.py E501 E501
    # scrapy/commands
-    scrapy/commands/__init__.py E128 E501
-    scrapy/commands/check.py E501
-    scrapy/commands/crawl.py E501
-    scrapy/commands/edit.py E501
-    scrapy/commands/fetch.py E401 E501 E128 E731
-    scrapy/commands/genspider.py E128 E501 E502
-    scrapy/commands/parse.py E128 E501 E731
-    scrapy/commands/runspider.py E501
+    scrapy/commands/__init__.py E128
+    scrapy/commands/fetch.py E401 E128 E731
+    scrapy/commands/genspider.py E128 E502
+    scrapy/commands/parse.py E128 E731
+    scrapy/commands/runspider.py
    scrapy/commands/settings.py E128
-    scrapy/commands/shell.py E128 E501 E502
-    scrapy/commands/startproject.py E127 E501 E128
-    scrapy/commands/version.py E501 E128
+    scrapy/commands/shell.py E128 E502
+    scrapy/commands/startproject.py E127 E128
+    scrapy/commands/version.py E128
    # scrapy/contracts
-    scrapy/contracts/__init__.py E501 W504
+    scrapy/contracts/__init__.py W504
    scrapy/contracts/default.py E128
    # scrapy/core
-    scrapy/core/engine.py E501 E128 E127 E502
-    scrapy/core/scheduler.py E501
-    scrapy/core/scraper.py E501 E128 W504
-    scrapy/core/spidermw.py E501 E731 E126
-    scrapy/core/downloader/__init__.py E501
-    scrapy/core/downloader/contextfactory.py E501 E128 E126
-    scrapy/core/downloader/middleware.py E501 E502
-    scrapy/core/downloader/tls.py E501 E241
-    scrapy/core/downloader/webclient.py E731 E501 E128 E126
-    scrapy/core/downloader/handlers/__init__.py E501
-    scrapy/core/downloader/handlers/ftp.py E501 E128 E127
-    scrapy/core/downloader/handlers/http10.py E501
-    scrapy/core/downloader/handlers/http11.py E501
-    scrapy/core/downloader/handlers/s3.py E501 E128 E126
+    scrapy/core/engine.py E128 E127 E502
+    scrapy/core/scheduler.py
+    scrapy/core/scraper.py E128 W504
+    scrapy/core/spidermw.py E731 E126
+    scrapy/core/downloader/__init__.py
+    scrapy/core/downloader/contextfactory.py E128 E126
+    scrapy/core/downloader/middleware.py E502
+    scrapy/core/downloader/tls.py E241
+    scrapy/core/downloader/webclient.py E731 E128 E126
+    scrapy/core/downloader/handlers/ftp.py E128 E127
+    scrapy/core/downloader/handlers/s3.py E128 E126
    # scrapy/downloadermiddlewares
-    scrapy/downloadermiddlewares/ajaxcrawl.py E501
-    scrapy/downloadermiddlewares/decompression.py E501
-    scrapy/downloadermiddlewares/defaultheaders.py E501
-    scrapy/downloadermiddlewares/httpcache.py E501 E126
-    scrapy/downloadermiddlewares/httpcompression.py E501 E128
-    scrapy/downloadermiddlewares/httpproxy.py E501
-    scrapy/downloadermiddlewares/redirect.py E501 W504
-    scrapy/downloadermiddlewares/retry.py E501 E126
-    scrapy/downloadermiddlewares/robotstxt.py E501
-    scrapy/downloadermiddlewares/stats.py E501
+    scrapy/downloadermiddlewares/httpcache.py E126
+    scrapy/downloadermiddlewares/httpcompression.py E128
+    scrapy/downloadermiddlewares/redirect.py W504
+    scrapy/downloadermiddlewares/retry.py E126
    # scrapy/extensions
-    scrapy/extensions/closespider.py E501 E128 E123
-    scrapy/extensions/corestats.py E501
-    scrapy/extensions/feedexport.py E128 E501
-    scrapy/extensions/httpcache.py E128 E501
-    scrapy/extensions/memdebug.py E501
-    scrapy/extensions/spiderstate.py E501
-    scrapy/extensions/telnet.py E501 W504
-    scrapy/extensions/throttle.py E501
+    scrapy/extensions/closespider.py E128 E123
+    scrapy/extensions/feedexport.py E128
+    scrapy/extensions/httpcache.py E128
+    scrapy/extensions/telnet.py W504
    # scrapy/http
-    scrapy/http/common.py E501
-    scrapy/http/cookies.py E501
-    scrapy/http/request/__init__.py E501
-    scrapy/http/request/form.py E501 E123
-    scrapy/http/request/json_request.py E501
-    scrapy/http/response/__init__.py E501 E128
-    scrapy/http/response/text.py E501 E128 E124
+    scrapy/http/request/form.py E123
+    scrapy/http/response/__init__.py E128
+    scrapy/http/response/text.py E128 E124
    # scrapy/linkextractors
-    scrapy/linkextractors/__init__.py E731 E501 E402 W504
-    scrapy/linkextractors/lxmlhtml.py E501 E731
+    scrapy/linkextractors/__init__.py E731 E402 W504
+    scrapy/linkextractors/lxmlhtml.py E731
    # scrapy/loader
-    scrapy/loader/__init__.py E501 E128
-    scrapy/loader/processors.py E501
+    scrapy/loader/__init__.py E128
+    scrapy/loader/processors.py
    # scrapy/pipelines
-    scrapy/pipelines/__init__.py E501
-    scrapy/pipelines/files.py E116 E501 E266
-    scrapy/pipelines/images.py E265 E501
-    scrapy/pipelines/media.py E125 E501 E266
+    scrapy/pipelines/files.py E116 E266
+    scrapy/pipelines/images.py E265
+    scrapy/pipelines/media.py E125 E266
    # scrapy/selector
    scrapy/selector/__init__.py F403
-    scrapy/selector/unified.py E501 E111
+    scrapy/selector/unified.py E111
    # scrapy/settings
-    scrapy/settings/__init__.py E501
-    scrapy/settings/default_settings.py E501 E114 E116
-    scrapy/settings/deprecated.py E501
+    scrapy/settings/default_settings.py E114 E116
    # scrapy/spidermiddlewares
-    scrapy/spidermiddlewares/httperror.py E501
-    scrapy/spidermiddlewares/offsite.py E501
-    scrapy/spidermiddlewares/referer.py E501 E129 W504
-    scrapy/spidermiddlewares/urllength.py E501
+    scrapy/spidermiddlewares/referer.py E129 W504
    # scrapy/spiders
-    scrapy/spiders/__init__.py E501 E402
-    scrapy/spiders/crawl.py E501
-    scrapy/spiders/feed.py E501
-    scrapy/spiders/sitemap.py E501
+    scrapy/spiders/__init__.py E402
    # scrapy/utils
-    scrapy/utils/asyncio.py E501
-    scrapy/utils/benchserver.py E501
-    scrapy/utils/conf.py E402 E501
-    scrapy/utils/datatypes.py E501
-    scrapy/utils/decorators.py E501
-    scrapy/utils/defer.py E501 E128
-    scrapy/utils/deprecate.py E128 E501 E127 E502
-    scrapy/utils/gz.py E501 W504
+    scrapy/utils/defer.py E128
+    scrapy/utils/deprecate.py E128 E127 E502
+    scrapy/utils/gz.py W504
    scrapy/utils/http.py F403
-    scrapy/utils/httpobj.py E501
-    scrapy/utils/iterators.py E501
-    scrapy/utils/log.py E128 E501
+    scrapy/utils/log.py E128
    scrapy/utils/markup.py F403
-    scrapy/utils/misc.py E501
    scrapy/utils/multipart.py F403
-    scrapy/utils/project.py E501
-    scrapy/utils/python.py E501
-    scrapy/utils/reactor.py E501
-    scrapy/utils/reqser.py E501
-    scrapy/utils/request.py E127 E501
-    scrapy/utils/response.py E501 E128
-    scrapy/utils/signal.py E501 E128
-    scrapy/utils/sitemap.py E501
-    scrapy/utils/spider.py E501
-    scrapy/utils/ssl.py E501
-    scrapy/utils/test.py E501
-    scrapy/utils/url.py E501 F403 E128 F405
+    scrapy/utils/request.py E127
+    scrapy/utils/response.py E128
+    scrapy/utils/signal.py E128
+    scrapy/utils/url.py F403 E128 F405
    # scrapy
-    scrapy/__init__.py E402 E501
-    scrapy/cmdline.py E501
-    scrapy/crawler.py E501
-    scrapy/dupefilters.py E501 E202
-    scrapy/exceptions.py E501
-    scrapy/exporters.py E501
-    scrapy/interfaces.py E501
-    scrapy/item.py E501 E128
-    scrapy/link.py E501
-    scrapy/logformatter.py E501
-    scrapy/mail.py E402 E128 E501 E502
-    scrapy/middleware.py E128 E501
-    scrapy/pqueues.py E501
-    scrapy/resolver.py E501
-    scrapy/responsetypes.py E128 E501
-    scrapy/robotstxt.py E501
-    scrapy/shell.py E501
-    scrapy/signalmanager.py E501
-    scrapy/spiderloader.py F841 E501 E126
+    scrapy/__init__.py E402
+    scrapy/dupefilters.py E202
+    scrapy/item.py E128
+    scrapy/mail.py E402 E128 E502
+    scrapy/middleware.py E128
+    scrapy/responsetypes.py E128
+    scrapy/spiderloader.py F841 E126
    scrapy/squeues.py E128
-    scrapy/statscollectors.py E501
    # tests
    tests/__init__.py E402 E501
    tests/mockserver.py E401 E501 E126 E123
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@ -165,6 +165,7 @@ if __name__ == '__main__':
    try:
        execute()
    finally:
-        # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect()
-        # on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies
+        # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() on exit:
+        # http://doc.pypy.org/en/latest/cpython_differences.html
+        # ?highlight=gc.collect#differences-related-to-garbage-collection-strategies
        garbage_collect()
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@ -35,8 +35,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
            for method in self.methods['process_request']:
                response = yield deferred_from_coro(method(request=request, spider=spider))
                if response is not None and not isinstance(response, (Response, Request)):
-                    raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
-                                         (method.__self__.__class__.__name__, response.__class__.__name__))
+                    raise _InvalidOutput(
+                        "Middleware %s.process_request must return None, Response or Request, got %s"
+                        % (method.__self__.__class__.__name__, response.__class__.__name__)
+                    )
                if response:
                    defer.returnValue(response)
            defer.returnValue((yield download_func(request=request, spider=spider)))
@ -50,8 +52,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
            for method in self.methods['process_response']:
                response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
                if not isinstance(response, (Response, Request)):
-                    raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
-                                         (method.__self__.__class__.__name__, type(response)))
+                    raise _InvalidOutput(
+                        "Middleware %s.process_response must return Response or Request, got %s"
+                        % (method.__self__.__class__.__name__, type(response))
+                    )
                if isinstance(response, Request):
                    defer.returnValue(response)
            defer.returnValue(response)
@ -62,8 +66,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
            for method in self.methods['process_exception']:
                response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
                if response is not None and not isinstance(response, (Response, Request)):
-                    raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
-                                         (method.__self__.__class__.__name__, type(response)))
+                    raise _InvalidOutput(
+                        "Middleware %s.process_exception must return None, Response or Request, got %s"
+                        % (method.__self__.__class__.__name__, type(response))
+                    )
                if response:
                    defer.returnValue(response)
            defer.returnValue(_failure)
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@ -17,10 +17,12 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
    except UnicodeDecodeError:
        # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
        # Switch to 'allow all' state.
-        logger.warning("Failure while parsing robots.txt. "
-                       "File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
-                       exc_info=sys.exc_info(),
-                       extra={'spider': spider})
+        logger.warning(
+            "Failure while parsing robots.txt. File either contains garbage or "
+            "is in an encoding other than UTF-8, treating it as an empty file.",
+            exc_info=sys.exc_info(),
+            extra={'spider': spider},
+        )
        robotstxt_body = ''
    return robotstxt_body

--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@ -54,8 +54,12 @@ class Rule:
        self.process_request = _get_method(self.process_request, spider)
        self.process_request_argcount = len(get_func_args(self.process_request))
        if self.process_request_argcount == 1:
-            msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated'
-            warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
+            warnings.warn(
+                "Rule.process_request should accept two arguments"
+                " (request, response), accepting only one is deprecated",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )

    def _process_request(self, request, response):
        """