RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

PEP8 adjustments

This commit is contained in:
Eugenio Lacuesta 2019-11-08 22:13:03 -03:00
parent 342e3b5bd5
commit b6bbb28197
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
7 changed files with 34 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
scrapy/crawler.py
View File

@ -1,6 +1,5 @@
import logging
import signal
import sys
import warnings
from twisted.internet import reactor, defer

1
scrapy/exporters.py
View File

@ -4,7 +4,6 @@ Item Exporters are used to export/serialize items into different formats.
import csv
import io
import sys
import pprint
import marshal
import warnings

2
scrapy/http/cookies.py
View File

@ -165,7 +165,7 @@ class WrappedRequest(object):
def get_header(self, name, default=None):
return to_unicode(self.request.headers.get(name, default),
errors='replace')
errors='replace')
def header_items(self):
return [

2
scrapy/link.py
View File

@ -4,6 +4,8 @@ This module defines the Link object used in Link extractors.
For actual link extractors implementation see scrapy.linkextractors, or
its documentation in: docs/topics/link-extractors.rst
"""
class Link(object):
"""Link objects represent an extracted link by the LinkExtractor."""

2
tests/test_pipeline_media.py
View File

@ -1,5 +1,3 @@
import sys
from testfixtures import LogCapture
from twisted.trial import unittest
from twisted.python.failure import Failure

53
tests/test_proxy_connect.py
View File

@ -28,17 +28,24 @@ from mitmproxy.tools.main import mitmdump
sys.argv[0] = "mitmdump"
sys.exit(mitmdump())
"""
cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'keys', 'mitmproxy-ca.pem')
self.proc = Popen([sys.executable,
'-c', script,
'--listen-host', '127.0.0.1',
'--listen-port', '0',
'--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),
'--certs', cert_path,
'--ssl-insecure',
],
stdout=PIPE, env=get_testenv())
cert_path = os.path.join(
os.path.abspath(os.path.dirname(__file__)),
'keys',
'mitmproxy-ca.pem'
)
self.proc = Popen(
[
sys.executable,
'-c', script,
'--listen-host', '127.0.0.1',
'--listen-port', '0',
'--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),
'--certs', cert_path,
'--ssl-insecure',
],
stdout=PIPE,
env=get_testenv()
)
line = self.proc.stdout.readline().decode('utf-8')
host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1)
address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port)
@ -75,9 +82,9 @@ class ProxyConnectTestCase(TestCase):
@defer.inlineCallbacks
def test_https_connect_tunnel(self):
crawler = get_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as logs:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
self._assert_got_response_code(200, l)
self._assert_got_response_code(200, logs)
@pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
@defer.inlineCallbacks
@ -85,35 +92,35 @@ class ProxyConnectTestCase(TestCase):
proxy = os.environ['https_proxy']
os.environ['https_proxy'] = proxy + '?noconnect'
crawler = get_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as logs:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
self._assert_got_response_code(200, l)
self._assert_got_response_code(200, logs)
@pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
@defer.inlineCallbacks
def test_https_connect_tunnel_error(self):
crawler = get_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as logs:
yield crawler.crawl("https://localhost:99999/status?n=200")
self._assert_got_tunnel_error(l)
self._assert_got_tunnel_error(logs)
@defer.inlineCallbacks
def test_https_tunnel_auth_error(self):
os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
crawler = get_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as logs:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
# The proxy returns a 407 error code but it does not reach the client;
# he just sees a TunnelError.
self._assert_got_tunnel_error(l)
self._assert_got_tunnel_error(logs)
@defer.inlineCallbacks
def test_https_tunnel_without_leak_proxy_authorization_header(self):
request = Request(self.mockserver.url("/echo", is_secure=True))
crawler = get_crawler(SingleRequestSpider)
with LogCapture() as l:
with LogCapture() as logs:
yield crawler.crawl(seed=request)
self._assert_got_response_code(200, l)
self._assert_got_response_code(200, logs)
echo = json.loads(crawler.spider.meta['responses'][0].text)
self.assertTrue('Proxy-Authorization' not in echo['headers'])
@ -122,9 +129,9 @@ class ProxyConnectTestCase(TestCase):
def test_https_noconnect_auth_error(self):
os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
crawler = get_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as logs:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
self._assert_got_response_code(407, l)
self._assert_got_response_code(407, logs)
def _assert_got_response_code(self, code, log):
print(log)

2
tests/test_utils_python.py
View File

@ -7,7 +7,7 @@ from itertools import count
from scrapy.utils.python import (
memoizemethod_noargs, binary_is_text, equal_attributes,
WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode,
WeakKeyCache, get_func_args, to_bytes, to_unicode,
without_none_values, MutableChain)