From b6bbb2819707a2202c87abd6b3dba6af13a7cc85 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 8 Nov 2019 22:13:03 -0300 Subject: [PATCH] PEP8 adjustments --- scrapy/crawler.py | 1 - scrapy/exporters.py | 1 - scrapy/http/cookies.py | 2 +- scrapy/link.py | 2 ++ tests/test_pipeline_media.py | 2 -- tests/test_proxy_connect.py | 53 ++++++++++++++++++++---------------- tests/test_utils_python.py | 2 +- 7 files changed, 34 insertions(+), 29 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 4d7d9bac4..ab62c678c 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,6 +1,5 @@ import logging import signal -import sys import warnings from twisted.internet import reactor, defer diff --git a/scrapy/exporters.py b/scrapy/exporters.py index f2999daea..e31ab1780 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -4,7 +4,6 @@ Item Exporters are used to export/serialize items into different formats. import csv import io -import sys import pprint import marshal import warnings diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index c39de0b52..0903fd4f8 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -165,7 +165,7 @@ class WrappedRequest(object): def get_header(self, name, default=None): return to_unicode(self.request.headers.get(name, default), - errors='replace') + errors='replace') def header_items(self): return [ diff --git a/scrapy/link.py b/scrapy/link.py index be1888ef0..a809c5ca4 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,6 +4,8 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ + + class Link(object): """Link objects represent an extracted link by the LinkExtractor.""" diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 70f11466b..0d23f51cc 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,5 +1,3 @@ -import sys - from testfixtures import LogCapture from twisted.trial import unittest from twisted.python.failure import Failure diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index bf56136b1..4147dc944 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -28,17 +28,24 @@ from mitmproxy.tools.main import mitmdump sys.argv[0] = "mitmdump" sys.exit(mitmdump()) """ - cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'keys', 'mitmproxy-ca.pem') - self.proc = Popen([sys.executable, - '-c', script, - '--listen-host', '127.0.0.1', - '--listen-port', '0', - '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass), - '--certs', cert_path, - '--ssl-insecure', - ], - stdout=PIPE, env=get_testenv()) + cert_path = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + 'keys', + 'mitmproxy-ca.pem' + ) + self.proc = Popen( + [ + sys.executable, + '-c', script, + '--listen-host', '127.0.0.1', + '--listen-port', '0', + '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass), + '--certs', cert_path, + '--ssl-insecure', + ], + stdout=PIPE, + env=get_testenv() + ) line = self.proc.stdout.readline().decode('utf-8') host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1) address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port) @@ -75,9 +82,9 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_connect_tunnel(self): crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as logs: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, l) + self._assert_got_response_code(200, logs) @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests') @defer.inlineCallbacks @@ -85,35 +92,35 @@ class ProxyConnectTestCase(TestCase): proxy = os.environ['https_proxy'] os.environ['https_proxy'] = proxy + '?noconnect' crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as logs: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, l) + self._assert_got_response_code(200, logs) @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6) @defer.inlineCallbacks def test_https_connect_tunnel_error(self): crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as logs: yield crawler.crawl("https://localhost:99999/status?n=200") - self._assert_got_tunnel_error(l) + self._assert_got_tunnel_error(logs) @defer.inlineCallbacks def test_https_tunnel_auth_error(self): os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as logs: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) # The proxy returns a 407 error code but it does not reach the client; # he just sees a TunnelError. - self._assert_got_tunnel_error(l) + self._assert_got_tunnel_error(logs) @defer.inlineCallbacks def test_https_tunnel_without_leak_proxy_authorization_header(self): request = Request(self.mockserver.url("/echo", is_secure=True)) crawler = get_crawler(SingleRequestSpider) - with LogCapture() as l: + with LogCapture() as logs: yield crawler.crawl(seed=request) - self._assert_got_response_code(200, l) + self._assert_got_response_code(200, logs) echo = json.loads(crawler.spider.meta['responses'][0].text) self.assertTrue('Proxy-Authorization' not in echo['headers']) @@ -122,9 +129,9 @@ class ProxyConnectTestCase(TestCase): def test_https_noconnect_auth_error(self): os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect' crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as logs: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(407, l) + self._assert_got_response_code(407, logs) def _assert_got_response_code(self, code, log): print(log) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index faf0d4b73..b36c2a5e3 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -7,7 +7,7 @@ from itertools import count from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, - WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode, + WeakKeyCache, get_func_args, to_bytes, to_unicode, without_none_values, MutableChain)