RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'py3-http-downloaders' into py3-downloader-middleware

This commit is contained in:
Konstantin Lopuhin 2016-01-18 15:51:26 +03:00
parent 7d44c5dcea f876654a0c
commit 6edd4dec33
4 changed files with 86 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
scrapy/core/downloader/handlers/http11.py
View File

@ -78,7 +78,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
for it.
"""
_responseMatcher = re.compile('HTTP/1\.. 200')
_responseMatcher = re.compile(b'HTTP/1\.. 200')
def __init__(self, reactor, host, port, proxyConf, contextFactory,
timeout=30, bindAddress=None):
@ -92,11 +92,15 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def requestTunnel(self, protocol):
"""Asks the proxy to open a tunnel."""
tunnelReq = 'CONNECT %s:%s HTTP/1.1\r\n' % (self._tunneledHost,
self._tunneledPort)
tunnelReq = (
b'CONNECT ' +
to_bytes(self._tunneledHost, encoding='ascii') + b':' +
to_bytes(str(self._tunneledPort)) +
b' HTTP/1.1\r\n')
if self._proxyAuthHeader:
tunnelReq += 'Proxy-Authorization: %s\r\n' % self._proxyAuthHeader
tunnelReq += '\r\n'
tunnelReq += \
b'Proxy-Authorization: ' + self._proxyAuthHeader + b'\r\n'
tunnelReq += b'\r\n'
protocol.transport.write(tunnelReq)
self._protocolDataReceived = protocol.dataReceived
protocol.dataReceived = self.processProxyResponse
@ -202,7 +206,7 @@ class ScrapyAgent(object):
agent = self._get_agent(request, timeout)
# request details
url = to_bytes(urldefrag(request.url)[0])
url = urldefrag(request.url)[0]
method = to_bytes(request.method)
headers = TxHeaders(request.headers)
if isinstance(agent, self._TunnelingAgent):
@ -210,7 +214,8 @@ class ScrapyAgent(object):
bodyproducer = _RequestBodyProducer(request.body) if request.body else None
start_time = time()
d = agent.request(method, url, headers, bodyproducer)
d = agent.request(
method, to_bytes(url, encoding='ascii'), headers, bodyproducer)
# set download latency
d.addCallback(self._cb_latency, request, start_time)
# response body is ready to be consumed
@ -263,10 +268,8 @@ class ScrapyAgent(object):
txresponse, body, flags = result
status = int(txresponse.code)
headers = Headers(txresponse.headers.getAllRawHeaders())
url = to_unicode(url)
respcls = responsetypes.from_args(headers=headers, url=url)
return respcls(
url=url, status=status, headers=headers, body=body, flags=flags)
return respcls(url=url, status=status, headers=headers, body=body, flags=flags)
@implementer(IBodyProducer)

21
scrapy/core/downloader/webclient.py
View File

@ -12,18 +12,26 @@ from scrapy.responsetypes import responsetypes
def _parsed_url_args(parsed):
# Assume parsed is urlparse-d from Request.url,
# which was passed via safe_url_string and is ascii-only.
b = lambda s: to_bytes(s, encoding='ascii')
path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
path = to_bytes(path)
host = to_bytes(parsed.hostname)
path = b(path)
host = b(parsed.hostname)
port = parsed.port
scheme = to_bytes(parsed.scheme, encoding='ascii')
netloc = to_bytes(parsed.netloc)
scheme = b(parsed.scheme)
netloc = b(parsed.netloc)
if port is None:
port = 443 if scheme == b'https' else 80
return scheme, netloc, host, port, path
def _parse(url):
""" Return tuple of (scheme, netloc, host, port, path),
all in bytes except for port which is int.
Assume url is from Request.url, which was passed via safe_url_string
and is ascii-only.
"""
url = url.strip()
parsed = urlparse(url)
return _parsed_url_args(parsed)
@ -66,7 +74,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
def handleResponse(self, response):
if self.factory.method.upper() == b'HEAD':
self.factory.page('')
self.factory.page(b'')
elif self.length is not None and self.length > 0:
self.factory.noPage(self._connection_lost_reason)
else:
@ -95,7 +103,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
def __init__(self, request, timeout=180):
self._url = urldefrag(request.url)[0]
# converting to bytes to comply to Twisted interface
self.url = to_bytes(self._url)
self.url = to_bytes(self._url, encoding='ascii')
self.method = to_bytes(request.method, encoding='ascii')
self.body = request.body or None
self.headers = Headers(request.headers)
@ -131,7 +139,6 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
status = int(self.status)
headers = Headers(self.response_headers)
respcls = responsetypes.from_args(headers=headers, url=self._url)
body = to_bytes(body)
return respcls(url=self._url, status=status, headers=headers, body=body)
def _set_connection_attributes(self, request):

25
tests/test_downloader_handlers.py
View File

@ -119,7 +119,7 @@ class HttpTestCase(unittest.TestCase):
r.putChild(b"broken", BrokenDownloadResource())
self.site = server.Site(r, timeout=None)
self.wrapper = WrappingFactory(self.site)
self.host = '127.0.0.1'
self.host = 'localhost'
if self.scheme == 'https':
self.port = reactor.listenSSL(
0, self.wrapper, ssl_context_factory(), interface=self.host)
@ -168,6 +168,9 @@ class HttpTestCase(unittest.TestCase):
@defer.inlineCallbacks
def test_timeout_download_from_spider(self):
if self.scheme == 'https':
raise unittest.SkipTest(
'test_timeout_download_from_spider skipped under https')
spider = Spider('foo')
meta = {'download_timeout': 0.2}
# client connects but no data is received
@ -181,7 +184,8 @@ class HttpTestCase(unittest.TestCase):
def test_host_header_not_in_request_headers(self):
def _test(response):
self.assertEquals(response.body, to_bytes('127.0.0.1:%d' % self.portno))
self.assertEquals(
response.body, to_bytes('%s:%d' % (self.host, self.portno)))
self.assertEquals(request.headers, {})
request = Request(self.getURL('host'))
@ -221,8 +225,6 @@ class Http10TestCase(HttpTestCase):
class Https10TestCase(Http10TestCase):
scheme = 'https'
def test_timeout_download_from_spider(self):
raise unittest.SkipTest("test_timeout_download_from_spider skipped under https")
class Http11TestCase(HttpTestCase):
@ -273,6 +275,10 @@ class Http11TestCase(HttpTestCase):
return d
class Https11TestCase(Http11TestCase):
scheme = 'https'
class Http11MockServerTestCase(unittest.TestCase):
"""HTTP 1.1 test case with MockServer"""
if twisted_version < (11, 1, 0):
@ -406,6 +412,17 @@ class Http11ProxyTestCase(HttpProxyTestCase):
if twisted_version < (11, 1, 0):
skip = 'HTTP1.1 not supported in twisted < 11.1.0'
@defer.inlineCallbacks
def test_download_with_proxy_https_timeout(self):
""" Test TunnelingTCP4ClientEndpoint """
http_proxy = self.getURL('')
domain = 'https://no-such-domain.nosuch'
request = Request(
domain, meta={'proxy': http_proxy, 'download_timeout': 0.2})
d = self.download_request(request, Spider('foo'))
timeout = yield self.assertFailure(d, error.TimeoutError)
self.assertIn(domain, timeout.osError)
class HttpDownloadHandlerMock(object):
def __init__(self, settings):

50
tests/test_webclient.py
View File

@ -7,7 +7,7 @@ import six
from six.moves.urllib.parse import urlparse
from twisted.trial import unittest
from twisted.web import server, static, error, util
from twisted.web import server, static, util, resource
from twisted.internet import reactor, defer
from twisted.test.proto_helpers import StringTransport
from twisted.python.filepath import FilePath
@ -18,14 +18,14 @@ from scrapy.http import Request, Headers
from scrapy.utils.python import to_bytes, to_unicode
def getPage(url, contextFactory=None, *args, **kwargs):
def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
"""Adapted version of twisted.web.client.getPage"""
def _clientfactory(url, *args, **kwargs):
url = to_unicode(url)
timeout = kwargs.pop('timeout', 0)
f = client.ScrapyHTTPClientFactory(
Request(url, *args, **kwargs), timeout=timeout)
f.deferred.addCallback(lambda r: r.body)
f.deferred.addCallback(response_transform or (lambda r: r.body))
return f
from twisted.web.client import _makeGetterFactory
@ -78,16 +78,17 @@ class ParseUrlTestCase(unittest.TestCase):
elements of its return tuple, even when passed an URL which has
previously been passed to L{urlparse} as a C{unicode} string.
"""
goodInput = u'http://example.com/path'
badInput = goodInput.encode('ascii')
if six.PY2:
goodInput, badInput = badInput, goodInput
urlparse(badInput)
if not six.PY2:
raise unittest.SkipTest(
"Applies only to Py2, as urls can be ONLY unicode on Py3")
badInput = u'http://example.com/path'
goodInput = badInput.encode('ascii')
self._parse(badInput) # cache badInput in urlparse_cached
scheme, netloc, host, port, path = self._parse(goodInput)
self.assertTrue(isinstance(scheme, bytes))
self.assertTrue(isinstance(netloc, bytes))
self.assertTrue(isinstance(host, bytes))
self.assertTrue(isinstance(path, bytes))
self.assertTrue(isinstance(scheme, str))
self.assertTrue(isinstance(netloc, str))
self.assertTrue(isinstance(host, str))
self.assertTrue(isinstance(path, str))
self.assertTrue(isinstance(port, int))
@ -213,6 +214,16 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \
ErrorResource, NoLengthResource, HostHeaderResource, \
PayloadResource, BrokenDownloadResource
class EncodingResource(resource.Resource):
out_encoding = 'cp1251'
def render(self, request):
body = to_unicode(request.content.read())
request.setHeader(b'content-encoding', self.out_encoding)
return body.encode(self.out_encoding)
class WebClientTestCase(unittest.TestCase):
def _listen(self, site):
return reactor.listenTCP(0, site, interface="127.0.0.1")
@ -229,6 +240,7 @@ class WebClientTestCase(unittest.TestCase):
r.putChild(b"host", HostHeaderResource())
r.putChild(b"payload", PayloadResource())
r.putChild(b"broken", BrokenDownloadResource())
r.putChild(b"encoding", EncodingResource())
self.site = server.Site(r, timeout=None)
self.wrapper = WrappingFactory(self.site)
self.port = self._listen(self.wrapper)
@ -338,3 +350,17 @@ class WebClientTestCase(unittest.TestCase):
b'\n<html>\n <head>\n <meta http-equiv="refresh" content="0;URL=/file">\n'
b' </head>\n <body bgcolor="#FFFFFF" text="#000000">\n '
b'<a href="/file">click here</a>\n </body>\n</html>\n')
def test_Encoding(self):
""" Test that non-standart body encoding matches
Content-Encoding header """
body = b'\xd0\x81\xd1\x8e\xd0\xaf'
return getPage(
self.getURL('encoding'), body=body, response_transform=lambda r: r)\
.addCallback(self._check_Encoding, body)
def _check_Encoding(self, response, original_body):
content_encoding = to_unicode(response.headers[b'Content-Encoding'])
self.assertEquals(content_encoding, EncodingResource.out_encoding)
self.assertEquals(
response.body.decode(content_encoding), to_unicode(original_body))