RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/remove-py2-tests' into remove-six-code

This commit is contained in:
Eugenio Lacuesta 2019-11-19 11:04:46 -03:00
parent 05785c1c17 beb7d80d6a
commit 6f6426559d
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
2 changed files with 29 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pytest.ini
View File

@ -226,6 +226,7 @@ flake8-ignore =
tests/test_pipeline_files.py F401 E501 W293 E303 E272 E226
tests/test_pipeline_images.py F401 F841 E501 E303
tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
tests/test_proxy_connect.py E501 E741
tests/test_request_cb_kwargs.py E501
tests/test_responsetypes.py E501 E302 E305
tests/test_robotstxt_interface.py F401 E302 E501 W291 E501

56
tests/test_proxy_connect.py
View File

@ -28,24 +28,17 @@ from mitmproxy.tools.main import mitmdump
sys.argv[0] = "mitmdump"
sys.exit(mitmdump())
"""
cert_path = os.path.join(
os.path.abspath(os.path.dirname(__file__)),
'keys',
'mitmproxy-ca.pem'
)
self.proc = Popen(
[
sys.executable,
'-c', script,
'--listen-host', '127.0.0.1',
'--listen-port', '0',
'--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),
'--certs', cert_path,
'--ssl-insecure',
],
stdout=PIPE,
env=get_testenv()
)
cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'keys', 'mitmproxy-ca.pem')
self.proc = Popen([sys.executable,
'-c', script,
'--listen-host', '127.0.0.1',
'--listen-port', '0',
'--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),
'--certs', cert_path,
'--ssl-insecure',
],
stdout=PIPE, env=get_testenv())
line = self.proc.stdout.readline().decode('utf-8')
host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1)
address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port)
@ -86,16 +79,6 @@ class ProxyConnectTestCase(TestCase):
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
self._assert_got_response_code(200, logs)
@pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
@defer.inlineCallbacks
def test_https_noconnect(self):
proxy = os.environ['https_proxy']
os.environ['https_proxy'] = proxy + '?noconnect'
crawler = get_crawler(SimpleSpider)
with LogCapture() as logs:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
self._assert_got_response_code(200, logs)
@pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
@defer.inlineCallbacks
def test_https_connect_tunnel_error(self):
@ -124,6 +107,23 @@ class ProxyConnectTestCase(TestCase):
echo = json.loads(crawler.spider.meta['responses'][0].text)
self.assertTrue('Proxy-Authorization' not in echo['headers'])
# The noconnect mode isn't supported by the current mitmproxy, it returns
# "Invalid request scheme: https" as it doesn't seem to support full URLs in GET at all,
# and it's not clear what behavior is intended by Scrapy and by mitmproxy here.
# https://github.com/mitmproxy/mitmproxy/issues/848 may be related.
# The Scrapy noconnect mode was required, at least in the past, to work with Crawlera,
# and https://github.com/scrapy-plugins/scrapy-crawlera/pull/44 seems to be related.
@pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
@defer.inlineCallbacks
def test_https_noconnect(self):
proxy = os.environ['https_proxy']
os.environ['https_proxy'] = proxy + '?noconnect'
crawler = get_crawler(SimpleSpider)
with LogCapture() as l:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
self._assert_got_response_code(200, l)
@pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
@defer.inlineCallbacks
def test_https_noconnect_auth_error(self):