From 4018d25a9b968ce316bf48ab9322720163d14043 Mon Sep 17 00:00:00 2001 From: paul Date: Sat, 7 Apr 2012 23:00:21 +0200 Subject: [PATCH 01/18] Use twisted.web.client.Agent for download requests (use of HTTP/1.1) Adds http11.HttpDownloadHandler in scrapy.core.downloader.handlers --- scrapy/core/downloader/handlers/http11.py | 198 ++++++++++++++++++++++ scrapy/tests/test_downloader_handlers.py | 14 +- 2 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 scrapy/core/downloader/handlers/http11.py diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py new file mode 100644 index 000000000..daf1b700a --- /dev/null +++ b/scrapy/core/downloader/handlers/http11.py @@ -0,0 +1,198 @@ +"""Download handlers for http and https schemes""" + +from twisted.internet import reactor + +from scrapy.exceptions import NotSupported +from scrapy.utils.misc import load_object +from scrapy.conf import settings +from scrapy.utils.httpobj import urlparse_cached +from scrapy import optional_features + +from scrapy.utils.misc import load_object +from scrapy.conf import settings +from scrapy.http import Headers as ScrapyHeaders +from scrapy.utils.httpobj import urlparse_cached +from scrapy.responsetypes import responsetypes + + +from twisted.internet import defer, reactor, protocol +from twisted.web.client import Agent, ProxyAgent, ResponseDone, WebClientContextFactory + +from twisted.web.http_headers import Headers +from twisted.web._newclient import Response + +from twisted.web.http import PotentialDataLoss +from twisted.web._newclient import ResponseFailed +from twisted.web.iweb import IBodyProducer +from twisted.internet.error import TimeoutError + +from twisted.internet.endpoints import TCP4ClientEndpoint + +from time import time +from urlparse import urldefrag + +from zope.interface import implements + +from urlparse import urlparse, urlunparse, urldefrag + +def _parsed_url_args(parsed): + path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, '')) + host = parsed.hostname + port = parsed.port + scheme = parsed.scheme + netloc = parsed.netloc + if port is None: + port = 443 if scheme == 'https' else 80 + return scheme, netloc, host, port, path + +def _parse(url): + url = url.strip() + parsed = urlparse(url) + return _parsed_url_args(parsed) + + +class ScrapyAgent(object): + def __init__(self, reactor, contextFactory=WebClientContextFactory(), + connectTimeout=180, bindAddress=None): + self._reactor = reactor + self._agent = Agent(self._reactor, + contextFactory=contextFactory, + connectTimeout=connectTimeout, + bindAddress=bindAddress) + + def bindRequest(self, request): + self._scrapyrequest = request + + proxy = self._scrapyrequest.meta.get('proxy') + if proxy is not None and proxy != '': + scheme, _, host, port, _ = _parse(proxy) + endpoint = TCP4ClientEndpoint(self._reactor, host, port) + self._agent = ProxyAgent(endpoint) + self._agent._proxyEndpoint._timeout = request.meta.get('download_timeout') or self._agent._proxyEndpoint._timeout + else: + self._agent._connectTimeout = request.meta.get('download_timeout') or self._agent._connectTimeout + + def launch(self): + self._scrapyrequest._tw_start_time = time() + d = self._agent.request( + self._scrapyrequest.method, + urldefrag(self._scrapyrequest.url)[0], + Headers(self._scrapyrequest.headers), + ScrapyAgentRequestBodyProducer(self._scrapyrequest.body) if (self._scrapyrequest.body is not None) else None) + return d + + +class ScrapyAgentRequestBodyProducer(object): + implements(IBodyProducer) + + def __init__(self, body): + self.body = body + self.length = len(body) + + def startProducing(self, consumer): + consumer.write(self.body) + return defer.succeed(None) + + def pauseProducing(self): + pass + + def stopProducing(self): + pass + + +from cStringIO import StringIO +class ScrapyAgentResponseBodyReader(protocol.Protocol): + + def __init__(self, finished, response, scrapyRequest, debug=0): + + # finished is the deferred that will be fired + self._finished = finished + + self.debug = debug + + self.status = int(response.code) + self.resp_headers = list(response.headers.getAllRawHeaders()) + + self._scrapyrequest = scrapyRequest + self._scrapyrequest._tw_headers_time = time() + self._scrapyrequest.meta['download_latency'] = self._scrapyrequest._tw_headers_time - self._scrapyrequest._tw_start_time + + # body + self.bodyBuffer = StringIO() + + def dataReceived(self, bodyBytes): + if self.debug > 2: + print "dataReceived", len(bodyBytes), [bodyBytes[0:min(len(bodyBytes), 16)]] + self.bodyBuffer.write(bodyBytes) + + def connectionLost(self, reason): + + if reason.check(PotentialDataLoss): + if self.debug > 2: + print "PotentialDataLoss" + + elif reason.check(ResponseFailed): + if self.debug > 2: + print "ResponseFailed" + print reason.getErrorMessage() + reason.getBriefTraceback() + + elif reason.check(ResponseDone): + if self.debug > 2: + print "ReponseDone" + print "connection lost: %s" % reason + print 'Finished receiving body:', reason.getErrorMessage() + + # fire the deferred with Scrapy Response object + self._finished.callback(self._build_response()) + + + def _build_response(self): + headers = ScrapyHeaders(self.resp_headers) + respcls = responsetypes.from_args(headers=headers, url=urldefrag(self._scrapyrequest.url)[0]) + return respcls( + url=urldefrag(self._scrapyrequest.url)[0], + status=self.status, + headers=headers, + body=self.bodyBuffer.getvalue()) + + +class HttpDownloadHandler(object): + + def __init__(self, httpclientfactory=None): + self.debug = False + + + def download_request(self, request, spider): + """Return a deferred for the HTTP download""" + + agent = ScrapyAgent(reactor) + agent.bindRequest(request) + + d = agent.launch() + d.addCallback(self._agent_callback, request) + d.addErrback(self._agent_errback, request) + + return d + + + def _agent_callback(self, response, request): + finished = defer.Deferred() + reader = ScrapyAgentResponseBodyReader(finished, response, request, debug = 0) + response.deliverBody(reader) + + if request.method != 'HEAD': + return finished + else: + return reader._build_response() + + + def _agent_errback(self, failure, request): + if self.debug: + print "HttpDownloadHandler: errback called!" + print failure.getErrorMessage() + failure.getBriefTraceback() + failure.printTraceback() + + if failure.check(TimeoutError): + raise defer.TimeoutError diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index adf2d2433..9b7720ab1 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -12,6 +12,7 @@ from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HttpDownloadHandler +from scrapy.core.downloader.handlers.http11 import HttpDownloadHandler as Http11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.spider import BaseSpider from scrapy.http import Request @@ -132,6 +133,11 @@ class HttpTestCase(unittest.TestCase): return d +class Http11TestCase(HttpTestCase): + def setUp(self): + HttpTestCase.setUp(self) + self.download_request = Http11DownloadHandler().download_request + class UriResource(resource.Resource): """Return the full uri that was requested""" @@ -177,6 +183,12 @@ class HttpProxyTestCase(unittest.TestCase): return self.download_request(request, BaseSpider('foo')).addCallback(_test) +class Http11ProxyTestCase(HttpProxyTestCase): + def setUp(self): + HttpProxyTestCase.setUp(self) + self.download_request = Http11DownloadHandler().download_request + + class HttpDownloadHandlerMock(object): def __init__(self, settings): pass @@ -240,7 +252,7 @@ class S3TestCase(unittest.TestCase): 'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=') def test_request_signing5(self): - # deletes an object from the 'johnsmith' bucket using the + # deletes an object from the 'johnsmith' bucket using the # path-style and Date alternative. req = Request('s3://johnsmith/photos/puppy.jpg', \ method='DELETE', headers={ From 46341d5275930c63486529ef080a6e39641eb833 Mon Sep 17 00:00:00 2001 From: paul Date: Wed, 11 Apr 2012 23:47:07 +0200 Subject: [PATCH 02/18] Renamed downloader to Http11DownloadHandler and some refactoring Only for HTTP, not HTTPS Test on expected body length instead of request method (HEAD case) --- scrapy/core/downloader/handlers/http11.py | 71 +++++++++++++---------- scrapy/tests/test_downloader_handlers.py | 2 +- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index daf1b700a..2c4c4e188 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -1,4 +1,4 @@ -"""Download handlers for http and https schemes""" +"""Download handlers for http scheme""" from twisted.internet import reactor @@ -42,9 +42,10 @@ def _parsed_url_args(parsed): scheme = parsed.scheme netloc = parsed.netloc if port is None: - port = 443 if scheme == 'https' else 80 + port = 80 return scheme, netloc, host, port, path + def _parse(url): url = url.strip() parsed = urlparse(url) @@ -55,26 +56,31 @@ class ScrapyAgent(object): def __init__(self, reactor, contextFactory=WebClientContextFactory(), connectTimeout=180, bindAddress=None): self._reactor = reactor - self._agent = Agent(self._reactor, - contextFactory=contextFactory, - connectTimeout=connectTimeout, - bindAddress=bindAddress) + self._contextFactory = contextFactory + self._connectTimeout = connectTimeout + self._bindAddress = bindAddress - def bindRequest(self, request): + def launchRequest(self, request): self._scrapyrequest = request + request_timeout = request.meta.get('download_timeout') or self._connectTimeout proxy = self._scrapyrequest.meta.get('proxy') if proxy is not None and proxy != '': scheme, _, host, port, _ = _parse(proxy) - endpoint = TCP4ClientEndpoint(self._reactor, host, port) - self._agent = ProxyAgent(endpoint) - self._agent._proxyEndpoint._timeout = request.meta.get('download_timeout') or self._agent._proxyEndpoint._timeout - else: - self._agent._connectTimeout = request.meta.get('download_timeout') or self._agent._connectTimeout + endpoint = TCP4ClientEndpoint(self._reactor, + host, port, + timeout=request_timeout, + bindAddress=self._bindAddress) + agent = ProxyAgent(endpoint) + + else: + agent = Agent(self._reactor, + contextFactory=self._contextFactory, + connectTimeout=request_timeout, + bindAddress=self._bindAddress) - def launch(self): self._scrapyrequest._tw_start_time = time() - d = self._agent.request( + d = agent.request( self._scrapyrequest.method, urldefrag(self._scrapyrequest.url)[0], Headers(self._scrapyrequest.headers), @@ -101,23 +107,30 @@ class ScrapyAgentRequestBodyProducer(object): from cStringIO import StringIO -class ScrapyAgentResponseBodyReader(protocol.Protocol): +class ScrapyAgentResponseReader(protocol.Protocol): def __init__(self, finished, response, scrapyRequest, debug=0): + self.debug = debug # finished is the deferred that will be fired self._finished = finished - - self.debug = debug - self.status = int(response.code) - self.resp_headers = list(response.headers.getAllRawHeaders()) self._scrapyrequest = scrapyRequest self._scrapyrequest._tw_headers_time = time() self._scrapyrequest.meta['download_latency'] = self._scrapyrequest._tw_headers_time - self._scrapyrequest._tw_start_time - # body + # twisted.web._newclient.HTTPClientParser already decodes chunked response bodies, + # so prevent extra processing in scrapy.contrib.downloadermiddleware.chunked + # by removing the Transfer-Encoding header if found + txEncodings = response.headers.getRawHeaders('Transfer-Encoding') + if txEncodings is not None and 'chunked' in txEncodings: + # hopefully there's only one Transfer-Encoding header... + response.headers.removeHeader('Transfer-Encoding') + + self.resp_headers = list(response.headers.getAllRawHeaders()) + + # body, if any self.bodyBuffer = StringIO() def dataReceived(self, bodyBytes): @@ -146,7 +159,6 @@ class ScrapyAgentResponseBodyReader(protocol.Protocol): # fire the deferred with Scrapy Response object self._finished.callback(self._build_response()) - def _build_response(self): headers = ScrapyHeaders(self.resp_headers) respcls = responsetypes.from_args(headers=headers, url=urldefrag(self._scrapyrequest.url)[0]) @@ -157,31 +169,30 @@ class ScrapyAgentResponseBodyReader(protocol.Protocol): body=self.bodyBuffer.getvalue()) -class HttpDownloadHandler(object): +class Http11DownloadHandler(object): def __init__(self, httpclientfactory=None): self.debug = False + self._httpclientfactory = httpclientfactory def download_request(self, request, spider): """Return a deferred for the HTTP download""" - agent = ScrapyAgent(reactor) - agent.bindRequest(request) - - d = agent.launch() + agent = ScrapyAgent(reactor, self._httpclientfactory) + d = agent.launchRequest(request) d.addCallback(self._agent_callback, request) d.addErrback(self._agent_errback, request) - return d def _agent_callback(self, response, request): finished = defer.Deferred() - reader = ScrapyAgentResponseBodyReader(finished, response, request, debug = 0) - response.deliverBody(reader) + reader = ScrapyAgentResponseReader(finished, response, request, debug = 0) - if request.method != 'HEAD': + # is a response body expected? + if response.length > 0: + response.deliverBody(reader) return finished else: return reader._build_response() diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index 9b7720ab1..e79eec0f1 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -12,7 +12,7 @@ from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HttpDownloadHandler -from scrapy.core.downloader.handlers.http11 import HttpDownloadHandler as Http11DownloadHandler +from scrapy.core.downloader.handlers.http11 import Http11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.spider import BaseSpider from scrapy.http import Request From ef0360386921cf7b05f8b5e78554989072def5fd Mon Sep 17 00:00:00 2001 From: paul Date: Thu, 12 Apr 2012 00:16:03 +0200 Subject: [PATCH 03/18] Restore handling of HTTPS --- scrapy/core/downloader/handlers/http11.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 2c4c4e188..78ece7324 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -1,4 +1,4 @@ -"""Download handlers for http scheme""" +"""Download handlers for http and https schemes""" from twisted.internet import reactor @@ -42,7 +42,7 @@ def _parsed_url_args(parsed): scheme = parsed.scheme netloc = parsed.netloc if port is None: - port = 80 + port = 443 if scheme == 'https' else 80 return scheme, netloc, host, port, path From a7a354f9823872176e75c58329146d5ccb9578b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 25 Apr 2012 12:45:28 -0300 Subject: [PATCH 04/18] http11 cleanup --- scrapy/core/downloader/handlers/http11.py | 194 +++++++--------------- scrapy/settings/default_settings.py | 4 +- scrapy/tests/test_downloader_handlers.py | 4 +- 3 files changed, 60 insertions(+), 142 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 78ece7324..5ce482fbf 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -1,94 +1,56 @@ """Download handlers for http and https schemes""" -from twisted.internet import reactor - -from scrapy.exceptions import NotSupported -from scrapy.utils.misc import load_object -from scrapy.conf import settings -from scrapy.utils.httpobj import urlparse_cached -from scrapy import optional_features - -from scrapy.utils.misc import load_object -from scrapy.conf import settings -from scrapy.http import Headers as ScrapyHeaders -from scrapy.utils.httpobj import urlparse_cached -from scrapy.responsetypes import responsetypes - - -from twisted.internet import defer, reactor, protocol -from twisted.web.client import Agent, ProxyAgent, ResponseDone, WebClientContextFactory - -from twisted.web.http_headers import Headers -from twisted.web._newclient import Response - -from twisted.web.http import PotentialDataLoss -from twisted.web._newclient import ResponseFailed -from twisted.web.iweb import IBodyProducer -from twisted.internet.error import TimeoutError - -from twisted.internet.endpoints import TCP4ClientEndpoint - from time import time +from cStringIO import StringIO from urlparse import urldefrag from zope.interface import implements +from twisted.internet import defer, reactor, protocol +from twisted.web.client import Agent, ProxyAgent, ResponseDone, ResponseFailed +from twisted.web.http_headers import Headers +from twisted.web.http import PotentialDataLoss +from twisted.web.iweb import IBodyProducer +from twisted.internet.endpoints import TCP4ClientEndpoint -from urlparse import urlparse, urlunparse, urldefrag - -def _parsed_url_args(parsed): - path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, '')) - host = parsed.hostname - port = parsed.port - scheme = parsed.scheme - netloc = parsed.netloc - if port is None: - port = 443 if scheme == 'https' else 80 - return scheme, netloc, host, port, path - - -def _parse(url): - url = url.strip() - parsed = urlparse(url) - return _parsed_url_args(parsed) +from scrapy.http import Headers as ScrapyHeaders +from scrapy.responsetypes import responsetypes +from scrapy.core.downloader.webclient import _parse class ScrapyAgent(object): - def __init__(self, reactor, contextFactory=WebClientContextFactory(), - connectTimeout=180, bindAddress=None): - self._reactor = reactor + + def __init__(self, contextFactory=None, connectTimeout=180, bindAddress=None): self._contextFactory = contextFactory self._connectTimeout = connectTimeout self._bindAddress = bindAddress def launchRequest(self, request): - self._scrapyrequest = request request_timeout = request.meta.get('download_timeout') or self._connectTimeout - proxy = self._scrapyrequest.meta.get('proxy') + proxy = request.meta.get('proxy') if proxy is not None and proxy != '': scheme, _, host, port, _ = _parse(proxy) - endpoint = TCP4ClientEndpoint(self._reactor, + endpoint = TCP4ClientEndpoint(reactor, host, port, timeout=request_timeout, bindAddress=self._bindAddress) agent = ProxyAgent(endpoint) - else: - agent = Agent(self._reactor, + agent = Agent(reactor, contextFactory=self._contextFactory, connectTimeout=request_timeout, bindAddress=self._bindAddress) - self._scrapyrequest._tw_start_time = time() - d = agent.request( - self._scrapyrequest.method, - urldefrag(self._scrapyrequest.url)[0], - Headers(self._scrapyrequest.headers), - ScrapyAgentRequestBodyProducer(self._scrapyrequest.body) if (self._scrapyrequest.body is not None) else None) - return d + request._tw_start_time = time() + return agent.request( + request.method, + urldefrag(request.url)[0], + Headers(request.headers), + _RequestBodyProducer(request.body or ''), + ) -class ScrapyAgentRequestBodyProducer(object): +class _RequestBodyProducer(object): implements(IBodyProducer) def __init__(self, body): @@ -106,104 +68,60 @@ class ScrapyAgentRequestBodyProducer(object): pass -from cStringIO import StringIO -class ScrapyAgentResponseReader(protocol.Protocol): +class _ResponseReader(protocol.Protocol): - def __init__(self, finished, response, scrapyRequest, debug=0): - self.debug = debug - - # finished is the deferred that will be fired + def __init__(self, finished): self._finished = finished - self.status = int(response.code) - - self._scrapyrequest = scrapyRequest - self._scrapyrequest._tw_headers_time = time() - self._scrapyrequest.meta['download_latency'] = self._scrapyrequest._tw_headers_time - self._scrapyrequest._tw_start_time - - # twisted.web._newclient.HTTPClientParser already decodes chunked response bodies, - # so prevent extra processing in scrapy.contrib.downloadermiddleware.chunked - # by removing the Transfer-Encoding header if found - txEncodings = response.headers.getRawHeaders('Transfer-Encoding') - if txEncodings is not None and 'chunked' in txEncodings: - # hopefully there's only one Transfer-Encoding header... - response.headers.removeHeader('Transfer-Encoding') - - self.resp_headers = list(response.headers.getAllRawHeaders()) - - # body, if any - self.bodyBuffer = StringIO() + self._bodybuf = StringIO() def dataReceived(self, bodyBytes): - if self.debug > 2: - print "dataReceived", len(bodyBytes), [bodyBytes[0:min(len(bodyBytes), 16)]] - self.bodyBuffer.write(bodyBytes) + self._bodybuf.write(bodyBytes) def connectionLost(self, reason): - - if reason.check(PotentialDataLoss): - if self.debug > 2: - print "PotentialDataLoss" - - elif reason.check(ResponseFailed): - if self.debug > 2: - print "ResponseFailed" - print reason.getErrorMessage() - reason.getBriefTraceback() - - elif reason.check(ResponseDone): - if self.debug > 2: - print "ReponseDone" - print "connection lost: %s" % reason - print 'Finished receiving body:', reason.getErrorMessage() - - # fire the deferred with Scrapy Response object - self._finished.callback(self._build_response()) - - def _build_response(self): - headers = ScrapyHeaders(self.resp_headers) - respcls = responsetypes.from_args(headers=headers, url=urldefrag(self._scrapyrequest.url)[0]) - return respcls( - url=urldefrag(self._scrapyrequest.url)[0], - status=self.status, - headers=headers, - body=self.bodyBuffer.getvalue()) + body = self._bodybuf.getvalue() + if reason.check(ResponseDone): + self._finished.callback((body, None)) + elif reason.check(PotentialDataLoss, ResponseFailed): + self._finished.callback((body, 'partial_download')) + else: + self._finished.errback(reason) class Http11DownloadHandler(object): - def __init__(self, httpclientfactory=None): + def __init__(self): self.debug = False - self._httpclientfactory = httpclientfactory - def download_request(self, request, spider): """Return a deferred for the HTTP download""" - - agent = ScrapyAgent(reactor, self._httpclientfactory) + agent = ScrapyAgent(reactor) d = agent.launchRequest(request) + d.addBoth(self._download_latency, request, time()) d.addCallback(self._agent_callback, request) d.addErrback(self._agent_errback, request) return d + def _download_latency(self, any_, request, start_time): + request.meta['download_latency'] = time() - start_time + return any_ - def _agent_callback(self, response, request): + def _agent_callback(self, txresponse, request): + if txresponse.length == 0: + return self._build_response(('', None), txresponse, request) finished = defer.Deferred() - reader = ScrapyAgentResponseReader(finished, response, request, debug = 0) - - # is a response body expected? - if response.length > 0: - response.deliverBody(reader) - return finished - else: - return reader._build_response() + finished.addCallback(self._build_response, txresponse, request) + txresponse.deliverBody(_ResponseReader(finished)) + return finished + def _build_response(self, (body, flag), txresponse, request): + if flag is not None: + request.meta[flag] = True + url = urldefrag(request.url)[0] + status = int(txresponse.code) + headers = ScrapyHeaders(txresponse.headers.getAllRawHeaders()) + respcls = responsetypes.from_args(headers=headers, url=url) + return respcls(url=url, status=status, headers=headers, body=body) def _agent_errback(self, failure, request): - if self.debug: - print "HttpDownloadHandler: errback called!" - print failure.getErrorMessage() - failure.getBriefTraceback() - failure.printTraceback() - - if failure.check(TimeoutError): - raise defer.TimeoutError + #log.err(failure, 'HTTP11 failure: %s' % request) + return failure diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index b03aac626..d11e29bdd 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -53,8 +53,8 @@ DOWNLOAD_DELAY = 0 DOWNLOAD_HANDLERS = {} DOWNLOAD_HANDLERS_BASE = { 'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler', - 'http': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler', - 'https': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler', + 'http': 'scrapy.core.downloader.handlers.http11.Http11DownloadHandler', + 'https': 'scrapy.core.downloader.handlers.http11.Http11DownloadHandler', 's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler', } diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index e79eec0f1..e11da807f 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -3,7 +3,7 @@ import os from twisted.trial import unittest from twisted.protocols.policies import WrappingFactory from twisted.python.filepath import FilePath -from twisted.internet import reactor, defer +from twisted.internet import reactor, defer, error from twisted.web import server, static, util, resource from twisted.web.test.test_webclient import ForeverTakingResource, \ NoLengthResource, HostHeaderResource, \ @@ -101,7 +101,7 @@ class HttpTestCase(unittest.TestCase): def test_timeout_download_from_spider(self): request = Request(self.getURL('wait'), meta=dict(download_timeout=0.000001)) d = self.download_request(request, BaseSpider('foo')) - return self.assertFailure(d, defer.TimeoutError) + return self.assertFailure(d, defer.TimeoutError, error.TimeoutError) def test_host_header_not_in_request_headers(self): def _test(response): From e4fe7c63b0f84b893205269c2988c4e223d3c110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 14 May 2012 17:04:08 -0300 Subject: [PATCH 05/18] add http connection pool and custom ssl context factory --- scrapy/core/downloader/handlers/http11.py | 134 ++++++++++++---------- scrapy/core/downloader/webclient.py | 4 + 2 files changed, 76 insertions(+), 62 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 5ce482fbf..3793663a1 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -6,7 +6,8 @@ from urlparse import urldefrag from zope.interface import implements from twisted.internet import defer, reactor, protocol -from twisted.web.client import Agent, ProxyAgent, ResponseDone, ResponseFailed +from twisted.web.client import Agent, ProxyAgent, ResponseDone, \ + ResponseFailed, HTTPConnectionPool from twisted.web.http_headers import Headers from twisted.web.http import PotentialDataLoss from twisted.web.iweb import IBodyProducer @@ -15,39 +16,88 @@ from twisted.internet.endpoints import TCP4ClientEndpoint from scrapy.http import Headers as ScrapyHeaders from scrapy.responsetypes import responsetypes from scrapy.core.downloader.webclient import _parse +from scrapy.utils.misc import load_object +from scrapy.conf import settings +from scrapy import log + + +ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) + + +class Http11DownloadHandler(object): + + def __init__(self): + self._pool = HTTPConnectionPool(reactor, persistent=False) + self._contextFactory = ClientContextFactory() + + def download_request(self, request, spider): + """Return a deferred for the HTTP download""" + agent = ScrapyAgent(contextFactory=self._contextFactory, pool=self._pool) + return agent.download_request(request) class ScrapyAgent(object): - def __init__(self, contextFactory=None, connectTimeout=180, bindAddress=None): + _Agent = Agent + _ProxyAgent = ProxyAgent + + def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None): self._contextFactory = contextFactory self._connectTimeout = connectTimeout self._bindAddress = bindAddress + self._pool = pool - def launchRequest(self, request): - request_timeout = request.meta.get('download_timeout') or self._connectTimeout + def download_request(self, request): + url = urldefrag(request.url)[0] + method = request.method + headers = Headers(request.headers) + bodyproducer = _RequestBodyProducer(request.body or '') + agent = self._get_agent(request) + start_time = time() + d = agent.request(method, url, headers, bodyproducer) + d.addBoth(self._download_latency, request, start_time) + d.addCallback(self._agentrequest_downloaded, request) + d.addErrback(self._agentrequest_failed, request) + return d + def _get_agent(self, request): + timeout = request.meta.get('download_timeout') or self._connectTimeout + bindaddress = request.meta.get('bindaddress') or self._bindAddress proxy = request.meta.get('proxy') - if proxy is not None and proxy != '': + if proxy: scheme, _, host, port, _ = _parse(proxy) - endpoint = TCP4ClientEndpoint(reactor, - host, port, - timeout=request_timeout, - bindAddress=self._bindAddress) - agent = ProxyAgent(endpoint) - else: - agent = Agent(reactor, - contextFactory=self._contextFactory, - connectTimeout=request_timeout, - bindAddress=self._bindAddress) + endpoint = TCP4ClientEndpoint(reactor, host, port, timeout=timeout, + bindAddress=bindaddress) + return self._ProxyAgent(endpoint) - request._tw_start_time = time() - return agent.request( - request.method, - urldefrag(request.url)[0], - Headers(request.headers), - _RequestBodyProducer(request.body or ''), - ) + return self._Agent(reactor, contextFactory=self._contextFactory, + connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) + + def _download_latency(self, any_, request, start_time): + request.meta['download_latency'] = time() - start_time + return any_ + + def _agentrequest_downloaded(self, txresponse, request): + if txresponse.length == 0: + return self._build_response(('', None), txresponse, request) + finished = defer.Deferred() + finished.addCallback(self._build_response, txresponse, request) + txresponse.deliverBody(_ResponseReader(finished)) + return finished + + def _build_response(self, (body, flag), txresponse, request): + if flag is not None: + request.meta[flag] = True + url = urldefrag(request.url)[0] + status = int(txresponse.code) + headers = ScrapyHeaders(txresponse.headers.getAllRawHeaders()) + respcls = responsetypes.from_args(headers=headers, url=url) + return respcls(url=url, status=status, headers=headers, body=body) + + def _agentrequest_failed(self, failure, request): + # be clear it is an HTTP failure with new downloader + log.err(failure, 'HTTP11 failure: %s' % request) + return failure class _RequestBodyProducer(object): @@ -85,43 +135,3 @@ class _ResponseReader(protocol.Protocol): self._finished.callback((body, 'partial_download')) else: self._finished.errback(reason) - - -class Http11DownloadHandler(object): - - def __init__(self): - self.debug = False - - def download_request(self, request, spider): - """Return a deferred for the HTTP download""" - agent = ScrapyAgent(reactor) - d = agent.launchRequest(request) - d.addBoth(self._download_latency, request, time()) - d.addCallback(self._agent_callback, request) - d.addErrback(self._agent_errback, request) - return d - - def _download_latency(self, any_, request, start_time): - request.meta['download_latency'] = time() - start_time - return any_ - - def _agent_callback(self, txresponse, request): - if txresponse.length == 0: - return self._build_response(('', None), txresponse, request) - finished = defer.Deferred() - finished.addCallback(self._build_response, txresponse, request) - txresponse.deliverBody(_ResponseReader(finished)) - return finished - - def _build_response(self, (body, flag), txresponse, request): - if flag is not None: - request.meta[flag] = True - url = urldefrag(request.url)[0] - status = int(txresponse.code) - headers = ScrapyHeaders(txresponse.headers.getAllRawHeaders()) - respcls = responsetypes.from_args(headers=headers, url=url) - return respcls(url=url, status=status, headers=headers, body=body) - - def _agent_errback(self, failure, request): - #log.err(failure, 'HTTP11 failure: %s' % request) - return failure diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index a7a6140f9..ef17871de 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -144,12 +144,16 @@ class ScrapyClientContextFactory(ClientContextFactory): # see https://github.com/scrapy/scrapy/issues/82 # and https://github.com/scrapy/scrapy/issues/26 +<<<<<<< HEAD def __init__(self): # see this issue on why we use TLSv1_METHOD by default # https://github.com/scrapy/scrapy/issues/194 self.method = SSL.TLSv1_METHOD def getContext(self): +======= + def getContext(self, hostname, port): +>>>>>>> add http connection pool and custom ssl context factory ctx = ClientContextFactory.getContext(self) # Enable all workarounds to SSL bugs as documented by # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html From ab3407289cd493f45f10aba75f76561ebbbfdc64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 15 May 2012 08:36:32 -0300 Subject: [PATCH 06/18] enable persistent connections --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 3793663a1..dc7796093 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -27,7 +27,7 @@ ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) class Http11DownloadHandler(object): def __init__(self): - self._pool = HTTPConnectionPool(reactor, persistent=False) + self._pool = HTTPConnectionPool(reactor, persistent=True) self._contextFactory = ClientContextFactory() def download_request(self, request, spider): From 0a261700862c496dfabab8a5adc05d841bd95c05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 15 May 2012 11:27:04 -0300 Subject: [PATCH 07/18] move ssl context factory to its own module and implement a non-ssl version that warns about pyopenssl support --- scrapy/core/downloader/contextfactory.py | 20 ++++++++++++++++++++ scrapy/core/downloader/webclient.py | 24 +----------------------- scrapy/settings/default_settings.py | 2 +- 3 files changed, 22 insertions(+), 24 deletions(-) create mode 100644 scrapy/core/downloader/contextfactory.py diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py new file mode 100644 index 000000000..e20830c71 --- /dev/null +++ b/scrapy/core/downloader/contextfactory.py @@ -0,0 +1,20 @@ +from OpenSSL import SSL +from twisted.internet.ssl import ClientContextFactory + + +class ScrapyClientContextFactory(ClientContextFactory): + "A SSL context factory which is more permissive against SSL bugs." + # see https://github.com/scrapy/scrapy/issues/82 + # and https://github.com/scrapy/scrapy/issues/26 + + def __init__(self): + # see this issue on why we use TLSv1_METHOD by default + # https://github.com/scrapy/scrapy/issues/194 + self.method = SSL.TLSv1_METHOD + + def getContext(self, hostname=None, port=None): + ctx = ClientContextFactory.getContext(self) + # Enable all workarounds to SSL bugs as documented by + # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html + ctx.set_options(SSL.OP_ALL) + return ctx diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index ef17871de..ea69dc5ca 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -79,7 +79,7 @@ class ScrapyHTTPPageGetter(HTTPClient): class ScrapyHTTPClientFactory(HTTPClientFactory): """Scrapy implementation of the HTTPClientFactory overwriting the - serUrl method to make use of our Url object that cache the parse + serUrl method to make use of our Url object that cache the parse result. """ @@ -137,25 +137,3 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): self.headers_time = time() self.response_headers = headers - - -class ScrapyClientContextFactory(ClientContextFactory): - "A SSL context factory which is more permissive against SSL bugs." - # see https://github.com/scrapy/scrapy/issues/82 - # and https://github.com/scrapy/scrapy/issues/26 - -<<<<<<< HEAD - def __init__(self): - # see this issue on why we use TLSv1_METHOD by default - # https://github.com/scrapy/scrapy/issues/194 - self.method = SSL.TLSv1_METHOD - - def getContext(self): -======= - def getContext(self, hostname, port): ->>>>>>> add http connection pool and custom ssl context factory - ctx = ClientContextFactory.getContext(self) - # Enable all workarounds to SSL bugs as documented by - # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html - ctx.set_options(SSL.OP_ALL) - return ctx diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d11e29bdd..2eb15f96c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -63,7 +63,7 @@ DOWNLOAD_TIMEOUT = 180 # 3mins DOWNLOADER_DEBUG = False DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory' -DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.webclient.ScrapyClientContextFactory' +DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory' DOWNLOADER_MIDDLEWARES = {} From b34606030dff9c83bbd95532e7fc0b9f1604b9c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 15 May 2012 11:32:52 -0300 Subject: [PATCH 08/18] remove duplicate context factory handling for non-ssl support in http1.0 --- scrapy/core/downloader/handlers/http.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 7d91f1fb7..9f3799c59 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,9 +1,9 @@ -"""Download handlers for http and https schemes""" - +"""Download handlers for http and https schemes +""" from twisted.internet import reactor - from scrapy.utils.misc import load_object + class HttpDownloadHandler(object): def __init__(self, settings): @@ -19,7 +19,7 @@ class HttpDownloadHandler(object): def _connect(self, factory): host, port = factory.host, factory.port if factory.scheme == 'https': - return reactor.connectSSL(host, port, factory, \ - self.ClientContextFactory()) + return reactor.connectSSL(host, port, factory, + self.ClientContextFactory()) else: return reactor.connectTCP(host, port, factory) From db232da06839bc6f62daa0a89197cf7e57367bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 16 May 2012 10:31:03 -0300 Subject: [PATCH 09/18] close pool connections before finishing tests --- scrapy/core/downloader/handlers/http11.py | 3 +++ scrapy/tests/test_downloader_handlers.py | 30 +++++++++++++++-------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index dc7796093..1690899df 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -35,6 +35,9 @@ class Http11DownloadHandler(object): agent = ScrapyAgent(contextFactory=self._contextFactory, pool=self._pool) return agent.download_request(request) + def close(self): + return self._pool.closeCachedConnections() + class ScrapyAgent(object): diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index e11da807f..be7c797e3 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -47,6 +47,8 @@ class FileTestCase(unittest.TestCase): class HttpTestCase(unittest.TestCase): + download_handler_cls = HttpDownloadHandler + def setUp(self): name = self.mktemp() os.mkdir(name) @@ -62,10 +64,14 @@ class HttpTestCase(unittest.TestCase): self.wrapper = WrappingFactory(self.site) self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1') self.portno = self.port.getHost().port - self.download_request = HttpDownloadHandler(Settings()).download_request + self.download_handler = self.download_handler_cls(Settings()) + self.download_request = self.download_handler.download_request + @defer.inlineCallbacks def tearDown(self): - return self.port.stopListening() + yield self.port.stopListening() + if hasattr(self.download_handler, 'close'): + yield self.download_handler.close() def getURL(self, path): return "http://127.0.0.1:%d/%s" % (self.portno, path) @@ -134,9 +140,9 @@ class HttpTestCase(unittest.TestCase): class Http11TestCase(HttpTestCase): - def setUp(self): - HttpTestCase.setUp(self) - self.download_request = Http11DownloadHandler().download_request + """HTTP 1.1 test case""" + download_handler_cls = Http11DownloadHandler + class UriResource(resource.Resource): """Return the full uri that was requested""" @@ -150,15 +156,21 @@ class UriResource(resource.Resource): class HttpProxyTestCase(unittest.TestCase): + download_handler_cls = HttpDownloadHandler + def setUp(self): site = server.Site(UriResource(), timeout=None) wrapper = WrappingFactory(site) self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1') self.portno = self.port.getHost().port - self.download_request = HttpDownloadHandler(Settings()).download_request + self.download_handler = self.download_handler_cls(Settings()) + self.download_request = self.download_handler.download_request + @defer.inlineCallbacks def tearDown(self): - return self.port.stopListening() + yield self.port.stopListening() + if hasattr(self.download_handler, 'close'): + yield self.download_handler.close() def getURL(self, path): return "http://127.0.0.1:%d/%s" % (self.portno, path) @@ -184,9 +196,7 @@ class HttpProxyTestCase(unittest.TestCase): class Http11ProxyTestCase(HttpProxyTestCase): - def setUp(self): - HttpProxyTestCase.setUp(self) - self.download_request = Http11DownloadHandler().download_request + download_handler_cls = Http11DownloadHandler class HttpDownloadHandlerMock(object): From ba6545555f1d408fc88ac326cf148bdab02bbf3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 16 May 2012 12:24:24 -0300 Subject: [PATCH 10/18] empty bodies does not require a body producer --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1690899df..9195ba0d2 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -54,7 +54,7 @@ class ScrapyAgent(object): url = urldefrag(request.url)[0] method = request.method headers = Headers(request.headers) - bodyproducer = _RequestBodyProducer(request.body or '') + bodyproducer = _RequestBodyProducer(request.body) if request.body else None agent = self._get_agent(request) start_time = time() d = agent.request(method, url, headers, bodyproducer) From 334ad71a2e78ab0383049d5e1dddf187c3e85848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 4 Sep 2012 16:18:21 -0300 Subject: [PATCH 11/18] adapt for singletons removals --- scrapy/core/downloader/handlers/http11.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 9195ba0d2..2cf224dc2 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -17,18 +17,15 @@ from scrapy.http import Headers as ScrapyHeaders from scrapy.responsetypes import responsetypes from scrapy.core.downloader.webclient import _parse from scrapy.utils.misc import load_object -from scrapy.conf import settings from scrapy import log -ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) - class Http11DownloadHandler(object): - def __init__(self): + def __init__(self, settings): self._pool = HTTPConnectionPool(reactor, persistent=True) - self._contextFactory = ClientContextFactory() + self._contextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) def download_request(self, request, spider): """Return a deferred for the HTTP download""" From 3e62ce35a0234fa441dab0cd65a8482a515335b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 7 Sep 2012 11:12:40 -0300 Subject: [PATCH 12/18] cleanup http connection pool on engine stop --- scrapy/core/downloader/__init__.py | 2 +- scrapy/core/downloader/handlers/__init__.py | 22 +++++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 0aa017116..94fee1997 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -70,7 +70,7 @@ class Downloader(object): self.signals = crawler.signals self.slots = {} self.active = set() - self.handlers = DownloadHandlers(crawler.settings) + self.handlers = DownloadHandlers(crawler) self.total_concurrency = self.settings.getint('CONCURRENT_REQUESTS') self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP') diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 90c557abf..484a25280 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -1,32 +1,42 @@ """Download handlers for different schemes""" +from twisted.internet import defer from scrapy.exceptions import NotSupported, NotConfigured from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object +from scrapy import signals class DownloadHandlers(object): - def __init__(self, settings): + def __init__(self, crawler): self._handlers = {} self._notconfigured = {} - handlers = settings.get('DOWNLOAD_HANDLERS_BASE') - handlers.update(settings.get('DOWNLOAD_HANDLERS', {})) + handlers = crawler.settings.get('DOWNLOAD_HANDLERS_BASE') + handlers.update(crawler.settings.get('DOWNLOAD_HANDLERS', {})) for scheme, clspath in handlers.iteritems(): cls = load_object(clspath) try: - dh = cls(settings) + dh = cls(crawler.settings) except NotConfigured, ex: self._notconfigured[scheme] = str(ex) else: - self._handlers[scheme] = dh.download_request + self._handlers[scheme] = dh + + crawler.signals.connect(self._close, signals.engine_stopped) def download_request(self, request, spider): scheme = urlparse_cached(request).scheme try: - handler = self._handlers[scheme] + handler = self._handlers[scheme].download_request except KeyError: msg = self._notconfigured.get(scheme, \ 'no handler available for that scheme') raise NotSupported("Unsupported URL scheme '%s': %s" % (scheme, msg)) return handler(request, spider) + + @defer.inlineCallbacks + def _close(self, *_a, **_kw): + for dh in self._handlers.values(): + if hasattr(dh, 'close'): + yield dh.close() From 495acba223ec780f1260d3ec72e6b3a4b54f3c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 12 Sep 2012 13:04:42 -0300 Subject: [PATCH 13/18] agents requires an instance of contextFactory --- scrapy/core/downloader/handlers/http11.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 2cf224dc2..55044bf84 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,12 +8,12 @@ from zope.interface import implements from twisted.internet import defer, reactor, protocol from twisted.web.client import Agent, ProxyAgent, ResponseDone, \ ResponseFailed, HTTPConnectionPool -from twisted.web.http_headers import Headers +from twisted.web.http_headers import Headers as TxHeaders from twisted.web.http import PotentialDataLoss from twisted.web.iweb import IBodyProducer from twisted.internet.endpoints import TCP4ClientEndpoint -from scrapy.http import Headers as ScrapyHeaders +from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.core.downloader.webclient import _parse from scrapy.utils.misc import load_object @@ -25,7 +25,8 @@ class Http11DownloadHandler(object): def __init__(self, settings): self._pool = HTTPConnectionPool(reactor, persistent=True) - self._contextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) + self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) + self._contextFactory = self._contextFactoryClass() def download_request(self, request, spider): """Return a deferred for the HTTP download""" @@ -50,7 +51,7 @@ class ScrapyAgent(object): def download_request(self, request): url = urldefrag(request.url)[0] method = request.method - headers = Headers(request.headers) + headers = TxHeaders(request.headers) bodyproducer = _RequestBodyProducer(request.body) if request.body else None agent = self._get_agent(request) start_time = time() @@ -90,7 +91,7 @@ class ScrapyAgent(object): request.meta[flag] = True url = urldefrag(request.url)[0] status = int(txresponse.code) - headers = ScrapyHeaders(txresponse.headers.getAllRawHeaders()) + headers = Headers(txresponse.headers.getAllRawHeaders()) respcls = responsetypes.from_args(headers=headers, url=url) return respcls(url=url, status=status, headers=headers, body=body) From 7729f939e9fc3177d911b041b4c40c82fc4d4820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 3 Jun 2013 19:18:45 -0300 Subject: [PATCH 14/18] implement download timeouts based on deferred cancellation --- scrapy/core/downloader/handlers/http11.py | 31 +++++++++++------------ scrapy/tests/test_downloader_handlers.py | 2 +- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 55044bf84..f9f3b5424 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -12,6 +12,7 @@ from twisted.web.http_headers import Headers as TxHeaders from twisted.web.http import PotentialDataLoss from twisted.web.iweb import IBodyProducer from twisted.internet.endpoints import TCP4ClientEndpoint +from twisted.internet.error import TimeoutError from scrapy.http import Headers from scrapy.responsetypes import responsetypes @@ -49,20 +50,27 @@ class ScrapyAgent(object): self._pool = pool def download_request(self, request): + timeout = request.meta.get('download_timeout') or self._connectTimeout url = urldefrag(request.url)[0] method = request.method headers = TxHeaders(request.headers) bodyproducer = _RequestBodyProducer(request.body) if request.body else None - agent = self._get_agent(request) + agent = self._get_agent(request, timeout) start_time = time() d = agent.request(method, url, headers, bodyproducer) - d.addBoth(self._download_latency, request, start_time) - d.addCallback(self._agentrequest_downloaded, request) - d.addErrback(self._agentrequest_failed, request) + d.addBoth(self._both_cb, request, start_time, url, timeout) + d.addCallback(self._downloaded, request) + self._timeout_cl = reactor.callLater(timeout, d.cancel) return d - def _get_agent(self, request): - timeout = request.meta.get('download_timeout') or self._connectTimeout + def _both_cb(self, result, request, start_time, url, timeout): + request.meta['download_latency'] = time() - start_time + if self._timeout_cl.active(): + self._timeout_cl.cancel() + return result + raise TimeoutError("Getting %s took longer than %s seconds." % (url, timeout)) + + def _get_agent(self, request, timeout): bindaddress = request.meta.get('bindaddress') or self._bindAddress proxy = request.meta.get('proxy') if proxy: @@ -74,11 +82,7 @@ class ScrapyAgent(object): return self._Agent(reactor, contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) - def _download_latency(self, any_, request, start_time): - request.meta['download_latency'] = time() - start_time - return any_ - - def _agentrequest_downloaded(self, txresponse, request): + def _downloaded(self, txresponse, request): if txresponse.length == 0: return self._build_response(('', None), txresponse, request) finished = defer.Deferred() @@ -95,11 +99,6 @@ class ScrapyAgent(object): respcls = responsetypes.from_args(headers=headers, url=url) return respcls(url=url, status=status, headers=headers, body=body) - def _agentrequest_failed(self, failure, request): - # be clear it is an HTTP failure with new downloader - log.err(failure, 'HTTP11 failure: %s' % request) - return failure - class _RequestBodyProducer(object): implements(IBodyProducer) diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index be7c797e3..2e2497b08 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -105,7 +105,7 @@ class HttpTestCase(unittest.TestCase): return d def test_timeout_download_from_spider(self): - request = Request(self.getURL('wait'), meta=dict(download_timeout=0.000001)) + request = Request(self.getURL('wait'), meta=dict(download_timeout=0.1)) d = self.download_request(request, BaseSpider('foo')) return self.assertFailure(d, defer.TimeoutError, error.TimeoutError) From 8b0c9466d6733c6523aa8c630e70c607cd845128 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 4 Jun 2013 15:35:31 -0300 Subject: [PATCH 15/18] add required files to support twisted 11.1 (precise) --- scrapy/core/downloader/handlers/http11.py | 5 +- scrapy/xlib/tx/LICENSE | 57 + scrapy/xlib/tx/README | 2 + scrapy/xlib/tx/__init__.py | 14 + scrapy/xlib/tx/_newclient.py | 1516 +++++++++++++ scrapy/xlib/tx/client.py | 1168 ++++++++++ scrapy/xlib/tx/endpoints.py | 1269 +++++++++++ scrapy/xlib/tx/interfaces.py | 2442 +++++++++++++++++++++ scrapy/xlib/tx/iweb.py | 587 +++++ 9 files changed, 7057 insertions(+), 3 deletions(-) create mode 100644 scrapy/xlib/tx/LICENSE create mode 100644 scrapy/xlib/tx/README create mode 100644 scrapy/xlib/tx/__init__.py create mode 100644 scrapy/xlib/tx/_newclient.py create mode 100644 scrapy/xlib/tx/client.py create mode 100644 scrapy/xlib/tx/endpoints.py create mode 100644 scrapy/xlib/tx/interfaces.py create mode 100644 scrapy/xlib/tx/iweb.py diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index f9f3b5424..6f233a546 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -6,13 +6,12 @@ from urlparse import urldefrag from zope.interface import implements from twisted.internet import defer, reactor, protocol -from twisted.web.client import Agent, ProxyAgent, ResponseDone, \ - ResponseFailed, HTTPConnectionPool from twisted.web.http_headers import Headers as TxHeaders from twisted.web.http import PotentialDataLoss from twisted.web.iweb import IBodyProducer -from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError +from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \ + ResponseFailed, HTTPConnectionPool, TCP4ClientEndpoint from scrapy.http import Headers from scrapy.responsetypes import responsetypes diff --git a/scrapy/xlib/tx/LICENSE b/scrapy/xlib/tx/LICENSE new file mode 100644 index 000000000..8529f6edf --- /dev/null +++ b/scrapy/xlib/tx/LICENSE @@ -0,0 +1,57 @@ +Copyright (c) 2001-2013 +Allen Short +Andy Gayton +Andrew Bennetts +Antoine Pitrou +Apple Computer, Inc. +Benjamin Bruheim +Bob Ippolito +Canonical Limited +Christopher Armstrong +David Reid +Donovan Preston +Eric Mangold +Eyal Lotem +Itamar Turner-Trauring +James Knight +Jason A. Mobarak +Jean-Paul Calderone +Jessica McKellar +Jonathan Jacobs +Jonathan Lange +Jonathan D. Simms +Jürgen Hermann +Kevin Horn +Kevin Turner +Mary Gardiner +Matthew Lefkowitz +Massachusetts Institute of Technology +Moshe Zadka +Paul Swartz +Pavel Pergamenshchik +Ralph Meijer +Sean Riley +Software Freedom Conservancy +Travis B. Hartwell +Thijs Triemstra +Thomas Herve +Timothy Allen + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/scrapy/xlib/tx/README b/scrapy/xlib/tx/README new file mode 100644 index 000000000..75ef485ce --- /dev/null +++ b/scrapy/xlib/tx/README @@ -0,0 +1,2 @@ +This source files are adapted copies from Twisted trunk to support HTTP1.1 +handler under Twisted >= 11.1 and Twisted <= 13.0.0 diff --git a/scrapy/xlib/tx/__init__.py b/scrapy/xlib/tx/__init__.py new file mode 100644 index 000000000..9f731c5b6 --- /dev/null +++ b/scrapy/xlib/tx/__init__.py @@ -0,0 +1,14 @@ +import twisted +if twisted.__version__.split('.') < (13, 0, 1): + from . import client, endpoints +else: + from twisted.web import client + from twisted.internet import endpoints + + +Agent = client.Agent +ProxyAgent = client.ProxyAgent +ResponseDone = client.ResponseDone +ResponseFailed = client.ResponseFailed +HTTPConnectionPool = client.HTTPConnectionPool +TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py new file mode 100644 index 000000000..cce759e35 --- /dev/null +++ b/scrapy/xlib/tx/_newclient.py @@ -0,0 +1,1516 @@ +# -*- test-case-name: twisted.web.test.test_newclient -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +An U{HTTP 1.1} client. + +The way to use the functionality provided by this module is to: + + - Connect a L{HTTP11ClientProtocol} to an HTTP server + - Create a L{Request} with the appropriate data + - Pass the request to L{HTTP11ClientProtocol.request} + - The returned Deferred will fire with a L{Response} object + - Create a L{IProtocol} provider which can handle the response body + - Connect it to the response with L{Response.deliverBody} + - When the protocol's C{connectionLost} method is called, the response is + complete. See L{Response.deliverBody} for details. + +Various other classes in this module support this usage: + + - HTTPParser is the basic HTTP parser. It can handle the parts of HTTP which + are symmetric between requests and responses. + + - HTTPClientParser extends HTTPParser to handle response-specific parts of + HTTP. One instance is created for each request to parse the corresponding + response. +""" + +__metaclass__ = type + +from zope.interface import implements + +from twisted.python import log +from twisted.python.reflect import fullyQualifiedName +from twisted.python.failure import Failure +from twisted.internet.interfaces import IConsumer, IPushProducer +from twisted.internet.error import ConnectionDone +from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred +from twisted.internet.defer import CancelledError +from twisted.internet.protocol import Protocol +from twisted.protocols.basic import LineReceiver +from twisted.web.http_headers import Headers +from twisted.web.http import NO_CONTENT, NOT_MODIFIED +from twisted.web.http import _DataLoss, PotentialDataLoss +from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder + +from .iweb import IResponse, UNKNOWN_LENGTH + +# States HTTPParser can be in +STATUS = 'STATUS' +HEADER = 'HEADER' +BODY = 'BODY' +DONE = 'DONE' + + +class BadHeaders(Exception): + """ + Headers passed to L{Request} were in some way invalid. + """ + + + +class ExcessWrite(Exception): + """ + The body L{IBodyProducer} for a request tried to write data after + indicating it had finished writing data. + """ + + +class ParseError(Exception): + """ + Some received data could not be parsed. + + @ivar data: The string which could not be parsed. + """ + def __init__(self, reason, data): + Exception.__init__(self, reason, data) + self.data = data + + + +class BadResponseVersion(ParseError): + """ + The version string in a status line was unparsable. + """ + + + +class _WrapperException(Exception): + """ + L{_WrapperException} is the base exception type for exceptions which + include one or more other exceptions as the low-level causes. + + @ivar reasons: A list of exceptions. See subclass documentation for more + details. + """ + def __init__(self, reasons): + Exception.__init__(self, reasons) + self.reasons = reasons + + + +class RequestGenerationFailed(_WrapperException): + """ + There was an error while creating the bytes which make up a request. + + @ivar reasons: A C{list} of one or more L{Failure} instances giving the + reasons the request generation was considered to have failed. + """ + + + +class RequestTransmissionFailed(_WrapperException): + """ + There was an error while sending the bytes which make up a request. + + @ivar reasons: A C{list} of one or more L{Failure} instances giving the + reasons the request transmission was considered to have failed. + """ + + + +class ConnectionAborted(Exception): + """ + The connection was explicitly aborted by application code. + """ + + + +class WrongBodyLength(Exception): + """ + An L{IBodyProducer} declared the number of bytes it was going to + produce (via its C{length} attribute) and then produced a different number + of bytes. + """ + + + +class ResponseDone(Exception): + """ + L{ResponseDone} may be passed to L{IProtocol.connectionLost} on the + protocol passed to L{Response.deliverBody} and indicates that the entire + response has been delivered. + """ + + + +class ResponseFailed(_WrapperException): + """ + L{ResponseFailed} indicates that all of the response to a request was not + received for some reason. + + @ivar reasons: A C{list} of one or more L{Failure} instances giving the + reasons the response was considered to have failed. + + @ivar response: If specified, the L{Response} received from the server (and + in particular the status code and the headers). + """ + + def __init__(self, reasons, response=None): + _WrapperException.__init__(self, reasons) + self.response = response + + + +class ResponseNeverReceived(ResponseFailed): + """ + A L{ResponseFailed} that knows no response bytes at all have been received. + """ + + + +class RequestNotSent(Exception): + """ + L{RequestNotSent} indicates that an attempt was made to issue a request but + for reasons unrelated to the details of the request itself, the request + could not be sent. For example, this may indicate that an attempt was made + to send a request using a protocol which is no longer connected to a + server. + """ + + + +def _callAppFunction(function): + """ + Call C{function}. If it raises an exception, log it with a minimal + description of the source. + + @return: C{None} + """ + try: + function() + except: + log.err(None, "Unexpected exception from %s" % ( + fullyQualifiedName(function),)) + + + +class HTTPParser(LineReceiver): + """ + L{HTTPParser} handles the parsing side of HTTP processing. With a suitable + subclass, it can parse either the client side or the server side of the + connection. + + @ivar headers: All of the non-connection control message headers yet + received. + + @ivar state: State indicator for the response parsing state machine. One + of C{STATUS}, C{HEADER}, C{BODY}, C{DONE}. + + @ivar _partialHeader: C{None} or a C{list} of the lines of a multiline + header while that header is being received. + """ + + # NOTE: According to HTTP spec, we're supposed to eat the + # 'Proxy-Authenticate' and 'Proxy-Authorization' headers also, but that + # doesn't sound like a good idea to me, because it makes it impossible to + # have a non-authenticating transparent proxy in front of an authenticating + # proxy. An authenticating proxy can eat them itself. -jknight + # + # Further, quoting + # http://homepages.tesco.net/J.deBoynePollard/FGA/web-proxy-connection-header.html + # regarding the 'Proxy-Connection' header: + # + # The Proxy-Connection: header is a mistake in how some web browsers + # use HTTP. Its name is the result of a false analogy. It is not a + # standard part of the protocol. There is a different standard + # protocol mechanism for doing what it does. And its existence + # imposes a requirement upon HTTP servers such that no proxy HTTP + # server can be standards-conforming in practice. + # + # -exarkun + + # Some servers (like http://news.ycombinator.com/) return status lines and + # HTTP headers delimited by \n instead of \r\n. + delimiter = '\n' + + CONNECTION_CONTROL_HEADERS = set([ + 'content-length', 'connection', 'keep-alive', 'te', 'trailers', + 'transfer-encoding', 'upgrade', 'proxy-connection']) + + def connectionMade(self): + self.headers = Headers() + self.connHeaders = Headers() + self.state = STATUS + self._partialHeader = None + + + def switchToBodyMode(self, decoder): + """ + Switch to body parsing mode - interpret any more bytes delivered as + part of the message body and deliver them to the given decoder. + """ + if self.state == BODY: + raise RuntimeError("already in body mode") + + self.bodyDecoder = decoder + self.state = BODY + self.setRawMode() + + + def lineReceived(self, line): + """ + Handle one line from a response. + """ + # Handle the normal CR LF case. + if line[-1:] == '\r': + line = line[:-1] + + if self.state == STATUS: + self.statusReceived(line) + self.state = HEADER + elif self.state == HEADER: + if not line or line[0] not in ' \t': + if self._partialHeader is not None: + header = ''.join(self._partialHeader) + name, value = header.split(':', 1) + value = value.strip() + self.headerReceived(name, value) + if not line: + # Empty line means the header section is over. + self.allHeadersReceived() + else: + # Line not beginning with LWS is another header. + self._partialHeader = [line] + else: + # A line beginning with LWS is a continuation of a header + # begun on a previous line. + self._partialHeader.append(line) + + + def rawDataReceived(self, data): + """ + Pass data from the message body to the body decoder object. + """ + self.bodyDecoder.dataReceived(data) + + + def isConnectionControlHeader(self, name): + """ + Return C{True} if the given lower-cased name is the name of a + connection control header (rather than an entity header). + + According to RFC 2616, section 14.10, the tokens in the Connection + header are probably relevant here. However, I am not sure what the + practical consequences of either implementing or ignoring that are. + So I leave it unimplemented for the time being. + """ + return name in self.CONNECTION_CONTROL_HEADERS + + + def statusReceived(self, status): + """ + Callback invoked whenever the first line of a new message is received. + Override this. + + @param status: The first line of an HTTP request or response message + without trailing I{CR LF}. + @type status: C{str} + """ + + + def headerReceived(self, name, value): + """ + Store the given header in C{self.headers}. + """ + name = name.lower() + if self.isConnectionControlHeader(name): + headers = self.connHeaders + else: + headers = self.headers + headers.addRawHeader(name, value) + + + def allHeadersReceived(self): + """ + Callback invoked after the last header is passed to C{headerReceived}. + Override this to change to the C{BODY} or C{DONE} state. + """ + self.switchToBodyMode(None) + + + +class HTTPClientParser(HTTPParser): + """ + An HTTP parser which only handles HTTP responses. + + @ivar request: The request with which the expected response is associated. + @type request: L{Request} + + @ivar NO_BODY_CODES: A C{set} of response codes which B{MUST NOT} have a + body. + + @ivar finisher: A callable to invoke when this response is fully parsed. + + @ivar _responseDeferred: A L{Deferred} which will be called back with the + response when all headers in the response have been received. + Thereafter, C{None}. + + @ivar _everReceivedData: C{True} if any bytes have been received. + """ + NO_BODY_CODES = set([NO_CONTENT, NOT_MODIFIED]) + + _transferDecoders = { + 'chunked': _ChunkedTransferDecoder, + } + + bodyDecoder = None + + def __init__(self, request, finisher): + self.request = request + self.finisher = finisher + self._responseDeferred = Deferred() + self._everReceivedData = False + + + def dataReceived(self, data): + """ + Override so that we know if any response has been received. + """ + self._everReceivedData = True + HTTPParser.dataReceived(self, data) + + + def parseVersion(self, strversion): + """ + Parse version strings of the form Protocol '/' Major '.' Minor. E.g. + 'HTTP/1.1'. Returns (protocol, major, minor). Will raise ValueError + on bad syntax. + """ + try: + proto, strnumber = strversion.split('/') + major, minor = strnumber.split('.') + major, minor = int(major), int(minor) + except ValueError, e: + raise BadResponseVersion(str(e), strversion) + if major < 0 or minor < 0: + raise BadResponseVersion("version may not be negative", strversion) + return (proto, major, minor) + + + def statusReceived(self, status): + """ + Parse the status line into its components and create a response object + to keep track of this response's state. + """ + parts = status.split(' ', 2) + if len(parts) != 3: + raise ParseError("wrong number of parts", status) + + try: + statusCode = int(parts[1]) + except ValueError: + raise ParseError("non-integer status code", status) + + self.response = Response( + self.parseVersion(parts[0]), + statusCode, + parts[2], + self.headers, + self.transport) + + + def _finished(self, rest): + """ + Called to indicate that an entire response has been received. No more + bytes will be interpreted by this L{HTTPClientParser}. Extra bytes are + passed up and the state of this L{HTTPClientParser} is set to I{DONE}. + + @param rest: A C{str} giving any extra bytes delivered to this + L{HTTPClientParser} which are not part of the response being + parsed. + """ + self.state = DONE + self.finisher(rest) + + + def isConnectionControlHeader(self, name): + """ + Content-Length in the response to a HEAD request is an entity header, + not a connection control header. + """ + if self.request.method == 'HEAD' and name == 'content-length': + return False + return HTTPParser.isConnectionControlHeader(self, name) + + + def allHeadersReceived(self): + """ + Figure out how long the response body is going to be by examining + headers and stuff. + """ + if (self.response.code in self.NO_BODY_CODES + or self.request.method == 'HEAD'): + self.response.length = 0 + self._finished(self.clearLineBuffer()) + else: + transferEncodingHeaders = self.connHeaders.getRawHeaders( + 'transfer-encoding') + if transferEncodingHeaders: + + # This could be a KeyError. However, that would mean we do not + # know how to decode the response body, so failing the request + # is as good a behavior as any. Perhaps someday we will want + # to normalize/document/test this specifically, but failing + # seems fine to me for now. + transferDecoder = self._transferDecoders[transferEncodingHeaders[0].lower()] + + # If anyone ever invents a transfer encoding other than + # chunked (yea right), and that transfer encoding can predict + # the length of the response body, it might be sensible to + # allow the transfer decoder to set the response object's + # length attribute. + else: + contentLengthHeaders = self.connHeaders.getRawHeaders('content-length') + if contentLengthHeaders is None: + contentLength = None + elif len(contentLengthHeaders) == 1: + contentLength = int(contentLengthHeaders[0]) + self.response.length = contentLength + else: + # "HTTP Message Splitting" or "HTTP Response Smuggling" + # potentially happening. Or it's just a buggy server. + raise ValueError( + "Too many Content-Length headers; response is invalid") + + if contentLength == 0: + self._finished(self.clearLineBuffer()) + transferDecoder = None + else: + transferDecoder = lambda x, y: _IdentityTransferDecoder( + contentLength, x, y) + + if transferDecoder is None: + self.response._bodyDataFinished() + else: + # Make sure as little data as possible from the response body + # gets delivered to the response object until the response + # object actually indicates it is ready to handle bytes + # (probably because an application gave it a way to interpret + # them). + self.transport.pauseProducing() + self.switchToBodyMode(transferDecoder( + self.response._bodyDataReceived, + self._finished)) + + # This must be last. If it were first, then application code might + # change some state (for example, registering a protocol to receive the + # response body). Then the pauseProducing above would be wrong since + # the response is ready for bytes and nothing else would ever resume + # the transport. + self._responseDeferred.callback(self.response) + del self._responseDeferred + + + def connectionLost(self, reason): + if self.bodyDecoder is not None: + try: + try: + self.bodyDecoder.noMoreData() + except PotentialDataLoss: + self.response._bodyDataFinished(Failure()) + except _DataLoss: + self.response._bodyDataFinished( + Failure(ResponseFailed([reason, Failure()], + self.response))) + else: + self.response._bodyDataFinished() + except: + # Handle exceptions from both the except suites and the else + # suite. Those functions really shouldn't raise exceptions, + # but maybe there's some buggy application code somewhere + # making things difficult. + log.err() + elif self.state != DONE: + if self._everReceivedData: + exceptionClass = ResponseFailed + else: + exceptionClass = ResponseNeverReceived + self._responseDeferred.errback(Failure(exceptionClass([reason]))) + del self._responseDeferred + + + +class Request: + """ + A L{Request} instance describes an HTTP request to be sent to an HTTP + server. + + @ivar method: The HTTP method to for this request, ex: 'GET', 'HEAD', + 'POST', etc. + @type method: C{str} + + @ivar uri: The relative URI of the resource to request. For example, + C{'/foo/bar?baz=quux'}. + @type uri: C{str} + + @ivar headers: Headers to be sent to the server. It is important to + note that this object does not create any implicit headers. So it + is up to the HTTP Client to add required headers such as 'Host'. + @type headers: L{twisted.web.http_headers.Headers} + + @ivar bodyProducer: C{None} or an L{IBodyProducer} provider which + produces the content body to send to the remote HTTP server. + + @ivar persistent: Set to C{True} when you use HTTP persistent connection. + @type persistent: C{bool} + """ + def __init__(self, method, uri, headers, bodyProducer, persistent=False): + self.method = method + self.uri = uri + self.headers = headers + self.bodyProducer = bodyProducer + self.persistent = persistent + + + def _writeHeaders(self, transport, TEorCL): + hosts = self.headers.getRawHeaders('host', ()) + if len(hosts) != 1: + raise BadHeaders("Exactly one Host header required") + + # In the future, having the protocol version be a parameter to this + # method would probably be good. It would be nice if this method + # weren't limited to issueing HTTP/1.1 requests. + requestLines = [] + requestLines.append( + '%s %s HTTP/1.1\r\n' % (self.method, self.uri)) + if not self.persistent: + requestLines.append('Connection: close\r\n') + if TEorCL is not None: + requestLines.append(TEorCL) + for name, values in self.headers.getAllRawHeaders(): + requestLines.extend(['%s: %s\r\n' % (name, v) for v in values]) + requestLines.append('\r\n') + transport.writeSequence(requestLines) + + + def _writeToChunked(self, transport): + """ + Write this request to the given transport using chunked + transfer-encoding to frame the body. + """ + self._writeHeaders(transport, 'Transfer-Encoding: chunked\r\n') + encoder = ChunkedEncoder(transport) + encoder.registerProducer(self.bodyProducer, True) + d = self.bodyProducer.startProducing(encoder) + + def cbProduced(ignored): + encoder.unregisterProducer() + def ebProduced(err): + encoder._allowNoMoreWrites() + # Don't call the encoder's unregisterProducer because it will write + # a zero-length chunk. This would indicate to the server that the + # request body is complete. There was an error, though, so we + # don't want to do that. + transport.unregisterProducer() + return err + d.addCallbacks(cbProduced, ebProduced) + return d + + + def _writeToContentLength(self, transport): + """ + Write this request to the given transport using content-length to frame + the body. + """ + self._writeHeaders( + transport, + 'Content-Length: %d\r\n' % (self.bodyProducer.length,)) + + # This Deferred is used to signal an error in the data written to the + # encoder below. It can only errback and it will only do so before too + # many bytes have been written to the encoder and before the producer + # Deferred fires. + finishedConsuming = Deferred() + + # This makes sure the producer writes the correct number of bytes for + # the request body. + encoder = LengthEnforcingConsumer( + self.bodyProducer, transport, finishedConsuming) + + transport.registerProducer(self.bodyProducer, True) + + finishedProducing = self.bodyProducer.startProducing(encoder) + + def combine(consuming, producing): + # This Deferred is returned and will be fired when the first of + # consuming or producing fires. If it's cancelled, forward that + # cancellation to the producer. + def cancelConsuming(ign): + finishedProducing.cancel() + ultimate = Deferred(cancelConsuming) + + # Keep track of what has happened so far. This initially + # contains None, then an integer uniquely identifying what + # sequence of events happened. See the callbacks and errbacks + # defined below for the meaning of each value. + state = [None] + + def ebConsuming(err): + if state == [None]: + # The consuming Deferred failed first. This means the + # overall writeTo Deferred is going to errback now. The + # producing Deferred should not fire later (because the + # consumer should have called stopProducing on the + # producer), but if it does, a callback will be ignored + # and an errback will be logged. + state[0] = 1 + ultimate.errback(err) + else: + # The consuming Deferred errbacked after the producing + # Deferred fired. This really shouldn't ever happen. + # If it does, I goofed. Log the error anyway, just so + # there's a chance someone might notice and complain. + log.err( + err, + "Buggy state machine in %r/[%d]: " + "ebConsuming called" % (self, state[0])) + + def cbProducing(result): + if state == [None]: + # The producing Deferred succeeded first. Nothing will + # ever happen to the consuming Deferred. Tell the + # encoder we're done so it can check what the producer + # wrote and make sure it was right. + state[0] = 2 + try: + encoder._noMoreWritesExpected() + except: + # Fail the overall writeTo Deferred - something the + # producer did was wrong. + ultimate.errback() + else: + # Success - succeed the overall writeTo Deferred. + ultimate.callback(None) + # Otherwise, the consuming Deferred already errbacked. The + # producing Deferred wasn't supposed to fire, but it did + # anyway. It's buggy, but there's not really anything to be + # done about it. Just ignore this result. + + def ebProducing(err): + if state == [None]: + # The producing Deferred failed first. This means the + # overall writeTo Deferred is going to errback now. + # Tell the encoder that we're done so it knows to reject + # further writes from the producer (which should not + # happen, but the producer may be buggy). + state[0] = 3 + encoder._allowNoMoreWrites() + ultimate.errback(err) + else: + # The producing Deferred failed after the consuming + # Deferred failed. It shouldn't have, so it's buggy. + # Log the exception in case anyone who can fix the code + # is watching. + log.err(err, "Producer is buggy") + + consuming.addErrback(ebConsuming) + producing.addCallbacks(cbProducing, ebProducing) + + return ultimate + + d = combine(finishedConsuming, finishedProducing) + def f(passthrough): + # Regardless of what happens with the overall Deferred, once it + # fires, the producer registered way up above the definition of + # combine should be unregistered. + transport.unregisterProducer() + return passthrough + d.addBoth(f) + return d + + + def writeTo(self, transport): + """ + Format this L{Request} as an HTTP/1.1 request and write it to the given + transport. If bodyProducer is not None, it will be associated with an + L{IConsumer}. + + @return: A L{Deferred} which fires with C{None} when the request has + been completely written to the transport or with a L{Failure} if + there is any problem generating the request bytes. + """ + if self.bodyProducer is not None: + if self.bodyProducer.length is UNKNOWN_LENGTH: + return self._writeToChunked(transport) + else: + return self._writeToContentLength(transport) + else: + self._writeHeaders(transport, None) + return succeed(None) + + + def stopWriting(self): + """ + Stop writing this request to the transport. This can only be called + after C{writeTo} and before the L{Deferred} returned by C{writeTo} + fires. It should cancel any asynchronous task started by C{writeTo}. + The L{Deferred} returned by C{writeTo} need not be fired if this method + is called. + """ + # If bodyProducer is None, then the Deferred returned by writeTo has + # fired already and this method cannot be called. + _callAppFunction(self.bodyProducer.stopProducing) + + + +class LengthEnforcingConsumer: + """ + An L{IConsumer} proxy which enforces an exact length requirement on the + total data written to it. + + @ivar _length: The number of bytes remaining to be written. + + @ivar _producer: The L{IBodyProducer} which is writing to this + consumer. + + @ivar _consumer: The consumer to which at most C{_length} bytes will be + forwarded. + + @ivar _finished: A L{Deferred} which will be fired with a L{Failure} if too + many bytes are written to this consumer. + """ + def __init__(self, producer, consumer, finished): + self._length = producer.length + self._producer = producer + self._consumer = consumer + self._finished = finished + + + def _allowNoMoreWrites(self): + """ + Indicate that no additional writes are allowed. Attempts to write + after calling this method will be met with an exception. + """ + self._finished = None + + + def write(self, bytes): + """ + Write C{bytes} to the underlying consumer unless + C{_noMoreWritesExpected} has been called or there are/have been too + many bytes. + """ + if self._finished is None: + # No writes are supposed to happen any more. Try to convince the + # calling code to stop calling this method by calling its + # stopProducing method and then throwing an exception at it. This + # exception isn't documented as part of the API because you're + # never supposed to expect it: only buggy code will ever receive + # it. + self._producer.stopProducing() + raise ExcessWrite() + + if len(bytes) <= self._length: + self._length -= len(bytes) + self._consumer.write(bytes) + else: + # No synchronous exception is raised in *this* error path because + # we still have _finished which we can use to report the error to a + # better place than the direct caller of this method (some + # arbitrary application code). + _callAppFunction(self._producer.stopProducing) + self._finished.errback(WrongBodyLength("too many bytes written")) + self._allowNoMoreWrites() + + + def _noMoreWritesExpected(self): + """ + Called to indicate no more bytes will be written to this consumer. + Check to see that the correct number have been written. + + @raise WrongBodyLength: If not enough bytes have been written. + """ + if self._finished is not None: + self._allowNoMoreWrites() + if self._length: + raise WrongBodyLength("too few bytes written") + + + +def makeStatefulDispatcher(name, template): + """ + Given a I{dispatch} name and a function, return a function which can be + used as a method and which, when called, will call another method defined + on the instance and return the result. The other method which is called is + determined by the value of the C{_state} attribute of the instance. + + @param name: A string which is used to construct the name of the subsidiary + method to invoke. The subsidiary method is named like C{'_%s_%s' % + (name, _state)}. + + @param template: A function object which is used to give the returned + function a docstring. + + @return: The dispatcher function. + """ + def dispatcher(self, *args, **kwargs): + func = getattr(self, '_' + name + '_' + self._state, None) + if func is None: + raise RuntimeError( + "%r has no %s method in state %s" % (self, name, self._state)) + return func(*args, **kwargs) + dispatcher.__doc__ = template.__doc__ + return dispatcher + + + +class Response: + """ + A L{Response} instance describes an HTTP response received from an HTTP + server. + + L{Response} should not be subclassed or instantiated. + + @ivar _transport: The transport which is delivering this response. + + @ivar _bodyProtocol: The L{IProtocol} provider to which the body is + delivered. C{None} before one has been registered with + C{deliverBody}. + + @ivar _bodyBuffer: A C{list} of the strings passed to C{bodyDataReceived} + before C{deliverBody} is called. C{None} afterwards. + + @ivar _state: Indicates what state this L{Response} instance is in, + particularly with respect to delivering bytes from the response body + to an application-suppled protocol object. This may be one of + C{'INITIAL'}, C{'CONNECTED'}, C{'DEFERRED_CLOSE'}, or C{'FINISHED'}, + with the following meanings: + + - INITIAL: This is the state L{Response} objects start in. No + protocol has yet been provided and the underlying transport may + still have bytes to deliver to it. + + - DEFERRED_CLOSE: If the underlying transport indicates all bytes + have been delivered but no application-provided protocol is yet + available, the L{Response} moves to this state. Data is + buffered and waiting for a protocol to be delivered to. + + - CONNECTED: If a protocol is provided when the state is INITIAL, + the L{Response} moves to this state. Any buffered data is + delivered and any data which arrives from the transport + subsequently is given directly to the protocol. + + - FINISHED: If a protocol is provided in the DEFERRED_CLOSE state, + the L{Response} moves to this state after delivering all + buffered data to the protocol. Otherwise, if the L{Response} is + in the CONNECTED state, if the transport indicates there is no + more data, the L{Response} moves to this state. Nothing else + can happen once the L{Response} is in this state. + """ + implements(IResponse) + + length = UNKNOWN_LENGTH + + _bodyProtocol = None + _bodyFinished = False + + def __init__(self, version, code, phrase, headers, _transport): + self.version = version + self.code = code + self.phrase = phrase + self.headers = headers + self._transport = _transport + self._bodyBuffer = [] + self._state = 'INITIAL' + + + def deliverBody(self, protocol): + """ + Dispatch the given L{IProtocol} depending of the current state of the + response. + """ + deliverBody = makeStatefulDispatcher('deliverBody', deliverBody) + + + def _deliverBody_INITIAL(self, protocol): + """ + Deliver any buffered data to C{protocol} and prepare to deliver any + future data to it. Move to the C{'CONNECTED'} state. + """ + # Now that there's a protocol to consume the body, resume the + # transport. It was previously paused by HTTPClientParser to avoid + # reading too much data before it could be handled. + self._transport.resumeProducing() + + protocol.makeConnection(self._transport) + self._bodyProtocol = protocol + for data in self._bodyBuffer: + self._bodyProtocol.dataReceived(data) + self._bodyBuffer = None + self._state = 'CONNECTED' + + + def _deliverBody_CONNECTED(self, protocol): + """ + It is invalid to attempt to deliver data to a protocol when it is + already being delivered to another protocol. + """ + raise RuntimeError( + "Response already has protocol %r, cannot deliverBody " + "again" % (self._bodyProtocol,)) + + + def _deliverBody_DEFERRED_CLOSE(self, protocol): + """ + Deliver any buffered data to C{protocol} and then disconnect the + protocol. Move to the C{'FINISHED'} state. + """ + # Unlike _deliverBody_INITIAL, there is no need to resume the + # transport here because all of the response data has been received + # already. Some higher level code may want to resume the transport if + # that code expects further data to be received over it. + + protocol.makeConnection(self._transport) + + for data in self._bodyBuffer: + protocol.dataReceived(data) + self._bodyBuffer = None + protocol.connectionLost(self._reason) + self._state = 'FINISHED' + + + def _deliverBody_FINISHED(self, protocol): + """ + It is invalid to attempt to deliver data to a protocol after the + response body has been delivered to another protocol. + """ + raise RuntimeError( + "Response already finished, cannot deliverBody now.") + + + def _bodyDataReceived(self, data): + """ + Called by HTTPClientParser with chunks of data from the response body. + They will be buffered or delivered to the protocol passed to + deliverBody. + """ + _bodyDataReceived = makeStatefulDispatcher('bodyDataReceived', + _bodyDataReceived) + + + def _bodyDataReceived_INITIAL(self, data): + """ + Buffer any data received for later delivery to a protocol passed to + C{deliverBody}. + + Little or no data should be buffered by this method, since the + transport has been paused and will not be resumed until a protocol + is supplied. + """ + self._bodyBuffer.append(data) + + + def _bodyDataReceived_CONNECTED(self, data): + """ + Deliver any data received to the protocol to which this L{Response} + is connected. + """ + self._bodyProtocol.dataReceived(data) + + + def _bodyDataReceived_DEFERRED_CLOSE(self, data): + """ + It is invalid for data to be delivered after it has been indicated + that the response body has been completely delivered. + """ + raise RuntimeError("Cannot receive body data after _bodyDataFinished") + + + def _bodyDataReceived_FINISHED(self, data): + """ + It is invalid for data to be delivered after the response body has + been delivered to a protocol. + """ + raise RuntimeError("Cannot receive body data after protocol disconnected") + + + def _bodyDataFinished(self, reason=None): + """ + Called by HTTPClientParser when no more body data is available. If the + optional reason is supplied, this indicates a problem or potential + problem receiving all of the response body. + """ + _bodyDataFinished = makeStatefulDispatcher('bodyDataFinished', + _bodyDataFinished) + + + def _bodyDataFinished_INITIAL(self, reason=None): + """ + Move to the C{'DEFERRED_CLOSE'} state to wait for a protocol to + which to deliver the response body. + """ + self._state = 'DEFERRED_CLOSE' + if reason is None: + reason = Failure(ResponseDone("Response body fully received")) + self._reason = reason + + + def _bodyDataFinished_CONNECTED(self, reason=None): + """ + Disconnect the protocol and move to the C{'FINISHED'} state. + """ + if reason is None: + reason = Failure(ResponseDone("Response body fully received")) + self._bodyProtocol.connectionLost(reason) + self._bodyProtocol = None + self._state = 'FINISHED' + + + def _bodyDataFinished_DEFERRED_CLOSE(self): + """ + It is invalid to attempt to notify the L{Response} of the end of the + response body data more than once. + """ + raise RuntimeError("Cannot finish body data more than once") + + + def _bodyDataFinished_FINISHED(self): + """ + It is invalid to attempt to notify the L{Response} of the end of the + response body data more than once. + """ + raise RuntimeError("Cannot finish body data after protocol disconnected") + + + +class ChunkedEncoder: + """ + Helper object which exposes L{IConsumer} on top of L{HTTP11ClientProtocol} + for streaming request bodies to the server. + """ + implements(IConsumer) + + def __init__(self, transport): + self.transport = transport + + + def _allowNoMoreWrites(self): + """ + Indicate that no additional writes are allowed. Attempts to write + after calling this method will be met with an exception. + """ + self.transport = None + + + def registerProducer(self, producer, streaming): + """ + Register the given producer with C{self.transport}. + """ + self.transport.registerProducer(producer, streaming) + + + def write(self, data): + """ + Write the given request body bytes to the transport using chunked + encoding. + + @type data: C{str} + """ + if self.transport is None: + raise ExcessWrite() + self.transport.writeSequence(("%x\r\n" % len(data), data, "\r\n")) + + + def unregisterProducer(self): + """ + Indicate that the request body is complete and finish the request. + """ + self.write('') + self.transport.unregisterProducer() + self._allowNoMoreWrites() + + + +class TransportProxyProducer: + """ + An L{IPushProducer} implementation which wraps another such thing and + proxies calls to it until it is told to stop. + + @ivar _producer: The wrapped L{IPushProducer} provider or C{None} after + this proxy has been stopped. + """ + implements(IPushProducer) + + # LineReceiver uses this undocumented attribute of transports to decide + # when to stop calling lineReceived or rawDataReceived (if it finds it to + # be true, it doesn't bother to deliver any more data). Set disconnecting + # to False here and never change it to true so that all data is always + # delivered to us and so that LineReceiver doesn't fail with an + # AttributeError. + disconnecting = False + + def __init__(self, producer): + self._producer = producer + + + def _stopProxying(self): + """ + Stop forwarding calls of L{IPushProducer} methods to the underlying + L{IPushProvider} provider. + """ + self._producer = None + + + def stopProducing(self): + """ + Proxy the stoppage to the underlying producer, unless this proxy has + been stopped. + """ + if self._producer is not None: + self._producer.stopProducing() + + + def resumeProducing(self): + """ + Proxy the resumption to the underlying producer, unless this proxy has + been stopped. + """ + if self._producer is not None: + self._producer.resumeProducing() + + + def pauseProducing(self): + """ + Proxy the pause to the underlying producer, unless this proxy has been + stopped. + """ + if self._producer is not None: + self._producer.pauseProducing() + + + +class HTTP11ClientProtocol(Protocol): + """ + L{HTTP11ClientProtocol} is an implementation of the HTTP 1.1 client + protocol. It supports as few features as possible. + + @ivar _parser: After a request is issued, the L{HTTPClientParser} to + which received data making up the response to that request is + delivered. + + @ivar _finishedRequest: After a request is issued, the L{Deferred} which + will fire when a L{Response} object corresponding to that request is + available. This allows L{HTTP11ClientProtocol} to fail the request + if there is a connection or parsing problem. + + @ivar _currentRequest: After a request is issued, the L{Request} + instance used to make that request. This allows + L{HTTP11ClientProtocol} to stop request generation if necessary (for + example, if the connection is lost). + + @ivar _transportProxy: After a request is issued, the + L{TransportProxyProducer} to which C{_parser} is connected. This + allows C{_parser} to pause and resume the transport in a way which + L{HTTP11ClientProtocol} can exert some control over. + + @ivar _responseDeferred: After a request is issued, the L{Deferred} from + C{_parser} which will fire with a L{Response} when one has been + received. This is eventually chained with C{_finishedRequest}, but + only in certain cases to avoid double firing that Deferred. + + @ivar _state: Indicates what state this L{HTTP11ClientProtocol} instance + is in with respect to transmission of a request and reception of a + response. This may be one of the following strings: + + - QUIESCENT: This is the state L{HTTP11ClientProtocol} instances + start in. Nothing is happening: no request is being sent and no + response is being received or expected. + + - TRANSMITTING: When a request is made (via L{request}), the + instance moves to this state. L{Request.writeTo} has been used + to start to send a request but it has not yet finished. + + - TRANSMITTING_AFTER_RECEIVING_RESPONSE: The server has returned a + complete response but the request has not yet been fully sent + yet. The instance will remain in this state until the request + is fully sent. + + - GENERATION_FAILED: There was an error while the request. The + request was not fully sent to the network. + + - WAITING: The request was fully sent to the network. The + instance is now waiting for the response to be fully received. + + - ABORTING: Application code has requested that the HTTP connection + be aborted. + + - CONNECTION_LOST: The connection has been lost. + + @ivar _abortDeferreds: A list of C{Deferred} instances that will fire when + the connection is lost. + """ + _state = 'QUIESCENT' + _parser = None + _finishedRequest = None + _currentRequest = None + _transportProxy = None + _responseDeferred = None + + + def __init__(self, quiescentCallback=lambda c: None): + self._quiescentCallback = quiescentCallback + self._abortDeferreds = [] + + + @property + def state(self): + return self._state + + + def request(self, request): + """ + Issue C{request} over C{self.transport} and return a L{Deferred} which + will fire with a L{Response} instance or an error. + + @param request: The object defining the parameters of the request to + issue. + @type request: L{Request} + + @rtype: L{Deferred} + @return: The deferred may errback with L{RequestGenerationFailed} if + the request was not fully written to the transport due to a local + error. It may errback with L{RequestTransmissionFailed} if it was + not fully written to the transport due to a network error. It may + errback with L{ResponseFailed} if the request was sent (not + necessarily received) but some or all of the response was lost. It + may errback with L{RequestNotSent} if it is not possible to send + any more requests using this L{HTTP11ClientProtocol}. + """ + if self._state != 'QUIESCENT': + return fail(RequestNotSent()) + + self._state = 'TRANSMITTING' + _requestDeferred = maybeDeferred(request.writeTo, self.transport) + + def cancelRequest(ign): + # Explicitly cancel the request's deferred if it's still trying to + # write when this request is cancelled. + if self._state in ( + 'TRANSMITTING', 'TRANSMITTING_AFTER_RECEIVING_RESPONSE'): + _requestDeferred.cancel() + else: + self.transport.abortConnection() + self._disconnectParser(Failure(CancelledError())) + self._finishedRequest = Deferred(cancelRequest) + + # Keep track of the Request object in case we need to call stopWriting + # on it. + self._currentRequest = request + + self._transportProxy = TransportProxyProducer(self.transport) + self._parser = HTTPClientParser(request, self._finishResponse) + self._parser.makeConnection(self._transportProxy) + self._responseDeferred = self._parser._responseDeferred + + def cbRequestWrotten(ignored): + if self._state == 'TRANSMITTING': + self._state = 'WAITING' + self._responseDeferred.chainDeferred(self._finishedRequest) + + def ebRequestWriting(err): + if self._state == 'TRANSMITTING': + self._state = 'GENERATION_FAILED' + self.transport.abortConnection() + self._finishedRequest.errback( + Failure(RequestGenerationFailed([err]))) + else: + log.err(err, 'Error writing request, but not in valid state ' + 'to finalize request: %s' % self._state) + + _requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting) + + return self._finishedRequest + + + def _finishResponse(self, rest): + """ + Called by an L{HTTPClientParser} to indicate that it has parsed a + complete response. + + @param rest: A C{str} giving any trailing bytes which were given to + the L{HTTPClientParser} which were not part of the response it + was parsing. + """ + _finishResponse = makeStatefulDispatcher('finishResponse', _finishResponse) + + + def _finishResponse_WAITING(self, rest): + # Currently the rest parameter is ignored. Don't forget to use it if + # we ever add support for pipelining. And maybe check what trailers + # mean. + if self._state == 'WAITING': + self._state = 'QUIESCENT' + else: + # The server sent the entire response before we could send the + # whole request. That sucks. Oh well. Fire the request() + # Deferred with the response. But first, make sure that if the + # request does ever finish being written that it won't try to fire + # that Deferred. + self._state = 'TRANSMITTING_AFTER_RECEIVING_RESPONSE' + self._responseDeferred.chainDeferred(self._finishedRequest) + + # This will happen if we're being called due to connection being lost; + # if so, no need to disconnect parser again, or to call + # _quiescentCallback. + if self._parser is None: + return + + reason = ConnectionDone("synthetic!") + connHeaders = self._parser.connHeaders.getRawHeaders('connection', ()) + if (('close' in connHeaders) or self._state != "QUIESCENT" or + not self._currentRequest.persistent): + self._giveUp(Failure(reason)) + else: + # We call the quiescent callback first, to ensure connection gets + # added back to connection pool before we finish the request. + try: + self._quiescentCallback(self) + except: + # If callback throws exception, just log it and disconnect; + # keeping persistent connections around is an optimisation: + log.err() + self.transport.loseConnection() + self._disconnectParser(reason) + + + _finishResponse_TRANSMITTING = _finishResponse_WAITING + + + def _disconnectParser(self, reason): + """ + If there is still a parser, call its C{connectionLost} method with the + given reason. If there is not, do nothing. + + @type reason: L{Failure} + """ + if self._parser is not None: + parser = self._parser + self._parser = None + self._currentRequest = None + self._finishedRequest = None + self._responseDeferred = None + + # The parser is no longer allowed to do anything to the real + # transport. Stop proxying from the parser's transport to the real + # transport before telling the parser it's done so that it can't do + # anything. + self._transportProxy._stopProxying() + self._transportProxy = None + parser.connectionLost(reason) + + + def _giveUp(self, reason): + """ + Lose the underlying connection and disconnect the parser with the given + L{Failure}. + + Use this method instead of calling the transport's loseConnection + method directly otherwise random things will break. + """ + self.transport.loseConnection() + self._disconnectParser(reason) + + + def dataReceived(self, bytes): + """ + Handle some stuff from some place. + """ + try: + self._parser.dataReceived(bytes) + except: + self._giveUp(Failure()) + + + def connectionLost(self, reason): + """ + The underlying transport went away. If appropriate, notify the parser + object. + """ + connectionLost = makeStatefulDispatcher('connectionLost', connectionLost) + + + def _connectionLost_QUIESCENT(self, reason): + """ + Nothing is currently happening. Move to the C{'CONNECTION_LOST'} + state but otherwise do nothing. + """ + self._state = 'CONNECTION_LOST' + + + def _connectionLost_GENERATION_FAILED(self, reason): + """ + The connection was in an inconsistent state. Move to the + C{'CONNECTION_LOST'} state but otherwise do nothing. + """ + self._state = 'CONNECTION_LOST' + + + def _connectionLost_TRANSMITTING(self, reason): + """ + Fail the L{Deferred} for the current request, notify the request + object that it does not need to continue transmitting itself, and + move to the C{'CONNECTION_LOST'} state. + """ + self._state = 'CONNECTION_LOST' + self._finishedRequest.errback( + Failure(RequestTransmissionFailed([reason]))) + del self._finishedRequest + + # Tell the request that it should stop bothering now. + self._currentRequest.stopWriting() + + + def _connectionLost_TRANSMITTING_AFTER_RECEIVING_RESPONSE(self, reason): + """ + Move to the C{'CONNECTION_LOST'} state. + """ + self._state = 'CONNECTION_LOST' + + + def _connectionLost_WAITING(self, reason): + """ + Disconnect the response parser so that it can propagate the event as + necessary (for example, to call an application protocol's + C{connectionLost} method, or to fail a request L{Deferred}) and move + to the C{'CONNECTION_LOST'} state. + """ + self._disconnectParser(reason) + self._state = 'CONNECTION_LOST' + + + def _connectionLost_ABORTING(self, reason): + """ + Disconnect the response parser with a L{ConnectionAborted} failure, and + move to the C{'CONNECTION_LOST'} state. + """ + self._disconnectParser(Failure(ConnectionAborted())) + self._state = 'CONNECTION_LOST' + for d in self._abortDeferreds: + d.callback(None) + self._abortDeferreds = [] + + + def abort(self): + """ + Close the connection and cause all outstanding L{request} L{Deferred}s + to fire with an error. + """ + if self._state == "CONNECTION_LOST": + return succeed(None) + self.transport.loseConnection() + self._state = 'ABORTING' + d = Deferred() + self._abortDeferreds.append(d) + return d diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py new file mode 100644 index 000000000..c3830dc47 --- /dev/null +++ b/scrapy/xlib/tx/client.py @@ -0,0 +1,1168 @@ +# -*- test-case-name: twisted.web.test.test_webclient,twisted.web.test.test_agent -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +HTTP client. +""" + +from __future__ import division, absolute_import + +import os + +try: + from urlparse import urlunparse + from urllib import splithost, splittype +except ImportError: + from urllib.parse import splithost, splittype + from urllib.parse import urlunparse as _urlunparse + + def urlunparse(parts): + result = _urlunparse(tuple([p.decode("charmap") for p in parts])) + return result.encode("charmap") +import zlib + +from zope.interface import implementer + +from twisted.python import log +from twisted.python.failure import Failure +from twisted.web import http +from twisted.internet import defer, protocol, task, reactor +from twisted.internet.interfaces import IProtocol +from twisted.python import failure +from twisted.python.components import proxyForInterface +from twisted.web import error +from twisted.web.http_headers import Headers + +from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint +from .iweb import IResponse, UNKNOWN_LENGTH, IBodyProducer + + +class PartialDownloadError(error.Error): + """ + Page was only partially downloaded, we got disconnected in middle. + + @ivar response: All of the response body which was downloaded. + """ + + +class _URL(tuple): + """ + A parsed URL. + + At some point this should be replaced with a better URL implementation. + """ + def __new__(self, scheme, host, port, path): + return tuple.__new__(_URL, (scheme, host, port, path)) + + + def __init__(self, scheme, host, port, path): + self.scheme = scheme + self.host = host + self.port = port + self.path = path + + +def _parse(url, defaultPort=None): + """ + Split the given URL into the scheme, host, port, and path. + + @type url: C{bytes} + @param url: An URL to parse. + + @type defaultPort: C{int} or C{None} + @param defaultPort: An alternate value to use as the port if the URL does + not include one. + + @return: A four-tuple of the scheme, host, port, and path of the URL. All + of these are C{bytes} instances except for port, which is an C{int}. + """ + url = url.strip() + parsed = http.urlparse(url) + scheme = parsed[0] + path = urlunparse((b'', b'') + parsed[2:]) + + if defaultPort is None: + if scheme == b'https': + defaultPort = 443 + else: + defaultPort = 80 + + host, port = parsed[1], defaultPort + if b':' in host: + host, port = host.split(b':') + try: + port = int(port) + except ValueError: + port = defaultPort + + if path == b'': + path = b'/' + + return _URL(scheme, host, port, path) + + +def _makeGetterFactory(url, factoryFactory, contextFactory=None, + *args, **kwargs): + """ + Create and connect an HTTP page getting factory. + + Any additional positional or keyword arguments are used when calling + C{factoryFactory}. + + @param factoryFactory: Factory factory that is called with C{url}, C{args} + and C{kwargs} to produce the getter + + @param contextFactory: Context factory to use when creating a secure + connection, defaulting to C{None} + + @return: The factory created by C{factoryFactory} + """ + scheme, host, port, path = _parse(url) + factory = factoryFactory(url, *args, **kwargs) + if scheme == b'https': + from twisted.internet import ssl + if contextFactory is None: + contextFactory = ssl.ClientContextFactory() + reactor.connectSSL(host, port, factory, contextFactory) + else: + reactor.connectTCP(host, port, factory) + return factory + + +# The code which follows is based on the new HTTP client implementation. It +# should be significantly better than anything above, though it is not yet +# feature equivalent. + +from twisted.web.error import SchemeNotSupported +from ._newclient import Request, Response, HTTP11ClientProtocol +from ._newclient import ResponseDone, ResponseFailed +from ._newclient import RequestNotSent, RequestTransmissionFailed +from ._newclient import ( + ResponseNeverReceived, PotentialDataLoss, _WrapperException) + +try: + from twisted.internet.ssl import ClientContextFactory +except ImportError: + class WebClientContextFactory(object): + """ + A web context factory which doesn't work because the necessary SSL + support is missing. + """ + def getContext(self, hostname, port): + raise NotImplementedError("SSL support unavailable") +else: + class WebClientContextFactory(ClientContextFactory): + """ + A web context factory which ignores the hostname and port and does no + certificate verification. + """ + def getContext(self, hostname, port): + return ClientContextFactory.getContext(self) + + + +class _WebToNormalContextFactory(object): + """ + Adapt a web context factory to a normal context factory. + + @ivar _webContext: A web context factory which accepts a hostname and port + number to its C{getContext} method. + + @ivar _hostname: The hostname which will be passed to + C{_webContext.getContext}. + + @ivar _port: The port number which will be passed to + C{_webContext.getContext}. + """ + def __init__(self, webContext, hostname, port): + self._webContext = webContext + self._hostname = hostname + self._port = port + + + def getContext(self): + """ + Called the wrapped web context factory's C{getContext} method with a + hostname and port number and return the resulting context object. + """ + return self._webContext.getContext(self._hostname, self._port) + + + +@implementer(IBodyProducer) +class FileBodyProducer(object): + """ + L{FileBodyProducer} produces bytes from an input file object incrementally + and writes them to a consumer. + + Since file-like objects cannot be read from in an event-driven manner, + L{FileBodyProducer} uses a L{Cooperator} instance to schedule reads from + the file. This process is also paused and resumed based on notifications + from the L{IConsumer} provider being written to. + + The file is closed after it has been read, or if the producer is stopped + early. + + @ivar _inputFile: Any file-like object, bytes read from which will be + written to a consumer. + + @ivar _cooperate: A method like L{Cooperator.cooperate} which is used to + schedule all reads. + + @ivar _readSize: The number of bytes to read from C{_inputFile} at a time. + """ + + # Python 2.4 doesn't have these symbolic constants + _SEEK_SET = getattr(os, 'SEEK_SET', 0) + _SEEK_END = getattr(os, 'SEEK_END', 2) + + def __init__(self, inputFile, cooperator=task, readSize=2 ** 16): + self._inputFile = inputFile + self._cooperate = cooperator.cooperate + self._readSize = readSize + self.length = self._determineLength(inputFile) + + + def _determineLength(self, fObj): + """ + Determine how many bytes can be read out of C{fObj} (assuming it is not + modified from this point on). If the determination cannot be made, + return C{UNKNOWN_LENGTH}. + """ + try: + seek = fObj.seek + tell = fObj.tell + except AttributeError: + return UNKNOWN_LENGTH + originalPosition = tell() + seek(0, self._SEEK_END) + end = tell() + seek(originalPosition, self._SEEK_SET) + return end - originalPosition + + + def stopProducing(self): + """ + Permanently stop writing bytes from the file to the consumer by + stopping the underlying L{CooperativeTask}. + """ + self._inputFile.close() + self._task.stop() + + + def startProducing(self, consumer): + """ + Start a cooperative task which will read bytes from the input file and + write them to C{consumer}. Return a L{Deferred} which fires after all + bytes have been written. + + @param consumer: Any L{IConsumer} provider + """ + self._task = self._cooperate(self._writeloop(consumer)) + d = self._task.whenDone() + def maybeStopped(reason): + # IBodyProducer.startProducing's Deferred isn't support to fire if + # stopProducing is called. + reason.trap(task.TaskStopped) + return defer.Deferred() + d.addCallbacks(lambda ignored: None, maybeStopped) + return d + + + def _writeloop(self, consumer): + """ + Return an iterator which reads one chunk of bytes from the input file + and writes them to the consumer for each time it is iterated. + """ + while True: + bytes = self._inputFile.read(self._readSize) + if not bytes: + self._inputFile.close() + break + consumer.write(bytes) + yield None + + + def pauseProducing(self): + """ + Temporarily suspend copying bytes from the input file to the consumer + by pausing the L{CooperativeTask} which drives that activity. + """ + self._task.pause() + + + def resumeProducing(self): + """ + Undo the effects of a previous C{pauseProducing} and resume copying + bytes to the consumer by resuming the L{CooperativeTask} which drives + the write activity. + """ + self._task.resume() + + + +class _HTTP11ClientFactory(protocol.Factory): + """ + A factory for L{HTTP11ClientProtocol}, used by L{HTTPConnectionPool}. + + @ivar _quiescentCallback: The quiescent callback to be passed to protocol + instances, used to return them to the connection pool. + + @since: 11.1 + """ + def __init__(self, quiescentCallback): + self._quiescentCallback = quiescentCallback + + + def buildProtocol(self, addr): + return HTTP11ClientProtocol(self._quiescentCallback) + + + +class _RetryingHTTP11ClientProtocol(object): + """ + A wrapper for L{HTTP11ClientProtocol} that automatically retries requests. + + @ivar _clientProtocol: The underlying L{HTTP11ClientProtocol}. + + @ivar _newConnection: A callable that creates a new connection for a + retry. + """ + + def __init__(self, clientProtocol, newConnection): + self._clientProtocol = clientProtocol + self._newConnection = newConnection + + + def _shouldRetry(self, method, exception, bodyProducer): + """ + Indicate whether request should be retried. + + Only returns C{True} if method is idempotent, no response was + received, the reason for the failed request was not due to + user-requested cancellation, and no body was sent. The latter + requirement may be relaxed in the future, and PUT added to approved + method list. + """ + if method not in ("GET", "HEAD", "OPTIONS", "DELETE", "TRACE"): + return False + if not isinstance(exception, (RequestNotSent, RequestTransmissionFailed, + ResponseNeverReceived)): + return False + if isinstance(exception, _WrapperException): + for failure in exception.reasons: + if failure.check(defer.CancelledError): + return False + if bodyProducer is not None: + return False + return True + + + def request(self, request): + """ + Do a request, and retry once (with a new connection) it it fails in + a retryable manner. + + @param request: A L{Request} instance that will be requested using the + wrapped protocol. + """ + d = self._clientProtocol.request(request) + + def failed(reason): + if self._shouldRetry(request.method, reason.value, + request.bodyProducer): + return self._newConnection().addCallback( + lambda connection: connection.request(request)) + else: + return reason + d.addErrback(failed) + return d + + + +class HTTPConnectionPool(object): + """ + A pool of persistent HTTP connections. + + Features: + - Cached connections will eventually time out. + - Limits on maximum number of persistent connections. + + Connections are stored using keys, which should be chosen such that any + connections stored under a given key can be used interchangeably. + + Failed requests done using previously cached connections will be retried + once if they use an idempotent method (e.g. GET), in case the HTTP server + timed them out. + + @ivar persistent: Boolean indicating whether connections should be + persistent. Connections are persistent by default. + + @ivar maxPersistentPerHost: The maximum number of cached persistent + connections for a C{host:port} destination. + @type maxPersistentPerHost: C{int} + + @ivar cachedConnectionTimeout: Number of seconds a cached persistent + connection will stay open before disconnecting. + + @ivar retryAutomatically: C{boolean} indicating whether idempotent + requests should be retried once if no response was received. + + @ivar _factory: The factory used to connect to the proxy. + + @ivar _connections: Map (scheme, host, port) to lists of + L{HTTP11ClientProtocol} instances. + + @ivar _timeouts: Map L{HTTP11ClientProtocol} instances to a + C{IDelayedCall} instance of their timeout. + + @since: 12.1 + """ + + _factory = _HTTP11ClientFactory + maxPersistentPerHost = 2 + cachedConnectionTimeout = 240 + retryAutomatically = True + + def __init__(self, reactor, persistent=True): + self._reactor = reactor + self.persistent = persistent + self._connections = {} + self._timeouts = {} + + + def getConnection(self, key, endpoint): + """ + Supply a connection, newly created or retrieved from the pool, to be + used for one HTTP request. + + The connection will remain out of the pool (not available to be + returned from future calls to this method) until one HTTP request has + been completed over it. + + Afterwards, if the connection is still open, it will automatically be + added to the pool. + + @param key: A unique key identifying connections that can be used + interchangeably. + + @param endpoint: An endpoint that can be used to open a new connection + if no cached connection is available. + + @return: A C{Deferred} that will fire with a L{HTTP11ClientProtocol} + (or a wrapper) that can be used to send a single HTTP request. + """ + # Try to get cached version: + connections = self._connections.get(key) + while connections: + connection = connections.pop(0) + # Cancel timeout: + self._timeouts[connection].cancel() + del self._timeouts[connection] + if connection.state == "QUIESCENT": + if self.retryAutomatically: + newConnection = lambda: self._newConnection(key, endpoint) + connection = _RetryingHTTP11ClientProtocol( + connection, newConnection) + return defer.succeed(connection) + + return self._newConnection(key, endpoint) + + + def _newConnection(self, key, endpoint): + """ + Create a new connection. + + This implements the new connection code path for L{getConnection}. + """ + def quiescentCallback(protocol): + self._putConnection(key, protocol) + factory = self._factory(quiescentCallback) + return endpoint.connect(factory) + + + def _removeConnection(self, key, connection): + """ + Remove a connection from the cache and disconnect it. + """ + connection.transport.loseConnection() + self._connections[key].remove(connection) + del self._timeouts[connection] + + + def _putConnection(self, key, connection): + """ + Return a persistent connection to the pool. This will be called by + L{HTTP11ClientProtocol} when the connection becomes quiescent. + """ + if connection.state != "QUIESCENT": + # Log with traceback for debugging purposes: + try: + raise RuntimeError( + "BUG: Non-quiescent protocol added to connection pool.") + except: + log.err() + return + connections = self._connections.setdefault(key, []) + if len(connections) == self.maxPersistentPerHost: + dropped = connections.pop(0) + dropped.transport.loseConnection() + self._timeouts[dropped].cancel() + del self._timeouts[dropped] + connections.append(connection) + cid = self._reactor.callLater(self.cachedConnectionTimeout, + self._removeConnection, + key, connection) + self._timeouts[connection] = cid + + + def closeCachedConnections(self): + """ + Close all persistent connections and remove them from the pool. + + @return: L{defer.Deferred} that fires when all connections have been + closed. + """ + results = [] + for protocols in self._connections.itervalues(): + for p in protocols: + results.append(p.abort()) + self._connections = {} + for dc in self._timeouts.values(): + dc.cancel() + self._timeouts = {} + return defer.gatherResults(results).addCallback(lambda ign: None) + + + +class _AgentBase(object): + """ + Base class offering common facilities for L{Agent}-type classes. + + @ivar _reactor: The C{IReactorTime} implementation which will be used by + the pool, and perhaps by subclasses as well. + + @ivar _pool: The L{HTTPConnectionPool} used to manage HTTP connections. + """ + + def __init__(self, reactor, pool): + if pool is None: + pool = HTTPConnectionPool(reactor, False) + self._reactor = reactor + self._pool = pool + + + def _computeHostValue(self, scheme, host, port): + """ + Compute the string to use for the value of the I{Host} header, based on + the given scheme, host name, and port number. + """ + if (scheme, port) in (('http', 80), ('https', 443)): + return host + return '%s:%d' % (host, port) + + + def _requestWithEndpoint(self, key, endpoint, method, parsedURI, + headers, bodyProducer, requestPath): + """ + Issue a new request, given the endpoint and the path sent as part of + the request. + """ + # Create minimal headers, if necessary: + if headers is None: + headers = Headers() + if not headers.hasHeader('host'): + #headers = headers.copy() # not supported in twisted <= 11.1, and it doesn't affects us + headers.addRawHeader( + 'host', self._computeHostValue(parsedURI.scheme, parsedURI.host, + parsedURI.port)) + + d = self._pool.getConnection(key, endpoint) + def cbConnected(proto): + return proto.request( + Request(method, requestPath, headers, bodyProducer, + persistent=self._pool.persistent)) + d.addCallback(cbConnected) + return d + + + +class Agent(_AgentBase): + """ + L{Agent} is a very basic HTTP client. It supports I{HTTP} and I{HTTPS} + scheme URIs (but performs no certificate checking by default). + + @param pool: A L{HTTPConnectionPool} instance, or C{None}, in which case a + non-persistent L{HTTPConnectionPool} instance will be created. + + @ivar _contextFactory: A web context factory which will be used to create + SSL context objects for any SSL connections the agent needs to make. + + @ivar _connectTimeout: If not C{None}, the timeout passed to C{connectTCP} + or C{connectSSL} for specifying the connection timeout. + + @ivar _bindAddress: If not C{None}, the address passed to C{connectTCP} or + C{connectSSL} for specifying the local address to bind to. + + @since: 9.0 + """ + + def __init__(self, reactor, contextFactory=WebClientContextFactory(), + connectTimeout=None, bindAddress=None, + pool=None): + _AgentBase.__init__(self, reactor, pool) + self._contextFactory = contextFactory + self._connectTimeout = connectTimeout + self._bindAddress = bindAddress + + + def _wrapContextFactory(self, host, port): + """ + Create and return a normal context factory wrapped around + C{self._contextFactory} in such a way that C{self._contextFactory} will + have the host and port information passed to it. + + @param host: A C{str} giving the hostname which will be connected to in + order to issue a request. + + @param port: An C{int} giving the port number the connection will be + on. + + @return: A context factory suitable to be passed to + C{reactor.connectSSL}. + """ + return _WebToNormalContextFactory(self._contextFactory, host, port) + + + def _getEndpoint(self, scheme, host, port): + """ + Get an endpoint for the given host and port, using a transport + selected based on scheme. + + @param scheme: A string like C{'http'} or C{'https'} (the only two + supported values) to use to determine how to establish the + connection. + + @param host: A C{str} giving the hostname which will be connected to in + order to issue a request. + + @param port: An C{int} giving the port number the connection will be + on. + + @return: An endpoint which can be used to connect to given address. + """ + kwargs = {} + if self._connectTimeout is not None: + kwargs['timeout'] = self._connectTimeout + kwargs['bindAddress'] = self._bindAddress + if scheme == 'http': + return TCP4ClientEndpoint(self._reactor, host, port, **kwargs) + elif scheme == 'https': + return SSL4ClientEndpoint(self._reactor, host, port, + self._wrapContextFactory(host, port), + **kwargs) + else: + raise SchemeNotSupported("Unsupported scheme: %r" % (scheme,)) + + + def request(self, method, uri, headers=None, bodyProducer=None): + """ + Issue a new request. + + @param method: The request method to send. + @type method: C{str} + + @param uri: The request URI send. + @type uri: C{str} + + @param headers: The request headers to send. If no I{Host} header is + included, one will be added based on the request URI. + @type headers: L{Headers} + + @param bodyProducer: An object which will produce the request body or, + if the request body is to be empty, L{None}. + @type bodyProducer: L{IBodyProducer} provider + + @return: A L{Deferred} which fires with the result of the request (a + L{twisted.web.iweb.IResponse} provider), or fails if there is a + problem setting up a connection over which to issue the request. + It may also fail with L{SchemeNotSupported} if the scheme of the + given URI is not supported. + @rtype: L{Deferred} + """ + parsedURI = _parse(uri) + try: + endpoint = self._getEndpoint(parsedURI.scheme, parsedURI.host, + parsedURI.port) + except SchemeNotSupported: + return defer.fail(Failure()) + key = (parsedURI.scheme, parsedURI.host, parsedURI.port) + return self._requestWithEndpoint(key, endpoint, method, parsedURI, + headers, bodyProducer, parsedURI.path) + + + +class ProxyAgent(_AgentBase): + """ + An HTTP agent able to cross HTTP proxies. + + @ivar _proxyEndpoint: The endpoint used to connect to the proxy. + + @since: 11.1 + """ + + def __init__(self, endpoint, reactor=None, pool=None): + if reactor is None: + from twisted.internet import reactor + _AgentBase.__init__(self, reactor, pool) + self._proxyEndpoint = endpoint + + + def request(self, method, uri, headers=None, bodyProducer=None): + """ + Issue a new request via the configured proxy. + """ + # Cache *all* connections under the same key, since we are only + # connecting to a single destination, the proxy: + key = ("http-proxy", self._proxyEndpoint) + + # To support proxying HTTPS via CONNECT, we will use key + # ("http-proxy-CONNECT", scheme, host, port), and an endpoint that + # wraps _proxyEndpoint with an additional callback to do the CONNECT. + return self._requestWithEndpoint(key, self._proxyEndpoint, method, + _parse(uri), headers, bodyProducer, + uri) + + + +class _FakeUrllib2Request(object): + """ + A fake C{urllib2.Request} object for C{cookielib} to work with. + + @see: U{http://docs.python.org/library/urllib2.html#request-objects} + + @type uri: C{str} + @ivar uri: Request URI. + + @type headers: L{twisted.web.http_headers.Headers} + @ivar headers: Request headers. + + @type type: C{str} + @ivar type: The scheme of the URI. + + @type host: C{str} + @ivar host: The host[:port] of the URI. + + @since: 11.1 + """ + def __init__(self, uri): + self.uri = uri + self.headers = Headers() + self.type, rest = splittype(self.uri) + self.host, rest = splithost(rest) + + + def has_header(self, header): + return self.headers.hasHeader(header) + + + def add_unredirected_header(self, name, value): + self.headers.addRawHeader(name, value) + + + def get_full_url(self): + return self.uri + + + def get_header(self, name, default=None): + headers = self.headers.getRawHeaders(name, default) + if headers is not None: + return headers[0] + return None + + + def get_host(self): + return self.host + + + def get_type(self): + return self.type + + + def is_unverifiable(self): + # In theory this shouldn't be hardcoded. + return False + + + +class _FakeUrllib2Response(object): + """ + A fake C{urllib2.Response} object for C{cookielib} to work with. + + @type response: C{twisted.web.iweb.IResponse} + @ivar response: Underlying Twisted Web response. + + @since: 11.1 + """ + def __init__(self, response): + self.response = response + + + def info(self): + class _Meta(object): + def getheaders(zelf, name): + return self.response.headers.getRawHeaders(name, []) + return _Meta() + + + +class CookieAgent(object): + """ + L{CookieAgent} extends the basic L{Agent} to add RFC-compliant + handling of HTTP cookies. Cookies are written to and extracted + from a C{cookielib.CookieJar} instance. + + The same cookie jar instance will be used for any requests through this + agent, mutating it whenever a I{Set-Cookie} header appears in a response. + + @type _agent: L{twisted.web.client.Agent} + @ivar _agent: Underlying Twisted Web agent to issue requests through. + + @type cookieJar: C{cookielib.CookieJar} + @ivar cookieJar: Initialized cookie jar to read cookies from and store + cookies to. + + @since: 11.1 + """ + def __init__(self, agent, cookieJar): + self._agent = agent + self.cookieJar = cookieJar + + + def request(self, method, uri, headers=None, bodyProducer=None): + """ + Issue a new request to the wrapped L{Agent}. + + Send a I{Cookie} header if a cookie for C{uri} is stored in + L{CookieAgent.cookieJar}. Cookies are automatically extracted and + stored from requests. + + If a C{'cookie'} header appears in C{headers} it will override the + automatic cookie header obtained from the cookie jar. + + @see: L{Agent.request} + """ + if headers is None: + headers = Headers() + lastRequest = _FakeUrllib2Request(uri) + # Setting a cookie header explicitly will disable automatic request + # cookies. + if not headers.hasHeader('cookie'): + self.cookieJar.add_cookie_header(lastRequest) + cookieHeader = lastRequest.get_header('Cookie', None) + if cookieHeader is not None: + headers = headers.copy() + headers.addRawHeader('cookie', cookieHeader) + + d = self._agent.request(method, uri, headers, bodyProducer) + d.addCallback(self._extractCookies, lastRequest) + return d + + + def _extractCookies(self, response, request): + """ + Extract response cookies and store them in the cookie jar. + + @type response: L{twisted.web.iweb.IResponse} + @param response: Twisted Web response. + + @param request: A urllib2 compatible request object. + """ + resp = _FakeUrllib2Response(response) + self.cookieJar.extract_cookies(resp, request) + return response + + + +class GzipDecoder(proxyForInterface(IResponse)): + """ + A wrapper for a L{Response} instance which handles gzip'ed body. + + @ivar original: The original L{Response} object. + + @since: 11.1 + """ + + def __init__(self, response): + self.original = response + self.length = UNKNOWN_LENGTH + + + def deliverBody(self, protocol): + """ + Override C{deliverBody} to wrap the given C{protocol} with + L{_GzipProtocol}. + """ + self.original.deliverBody(_GzipProtocol(protocol, self.original)) + + + +class _GzipProtocol(proxyForInterface(IProtocol)): + """ + A L{Protocol} implementation which wraps another one, transparently + decompressing received data. + + @ivar _zlibDecompress: A zlib decompress object used to decompress the data + stream. + + @ivar _response: A reference to the original response, in case of errors. + + @since: 11.1 + """ + + def __init__(self, protocol, response): + self.original = protocol + self._response = response + self._zlibDecompress = zlib.decompressobj(16 + zlib.MAX_WBITS) + + + def dataReceived(self, data): + """ + Decompress C{data} with the zlib decompressor, forwarding the raw data + to the original protocol. + """ + try: + rawData = self._zlibDecompress.decompress(data) + except zlib.error: + raise ResponseFailed([failure.Failure()], self._response) + if rawData: + self.original.dataReceived(rawData) + + + def connectionLost(self, reason): + """ + Forward the connection lost event, flushing remaining data from the + decompressor if any. + """ + try: + rawData = self._zlibDecompress.flush() + except zlib.error: + raise ResponseFailed([reason, failure.Failure()], self._response) + if rawData: + self.original.dataReceived(rawData) + self.original.connectionLost(reason) + + + +class ContentDecoderAgent(object): + """ + An L{Agent} wrapper to handle encoded content. + + It takes care of declaring the support for content in the + I{Accept-Encoding} header, and automatically decompresses the received data + if it's effectively using compression. + + @param decoders: A list or tuple of (name, decoder) objects. The name + declares which decoding the decoder supports, and the decoder must + return a response object when called/instantiated. For example, + C{(('gzip', GzipDecoder))}. The order determines how the decoders are + going to be advertized to the server. + + @since: 11.1 + """ + + def __init__(self, agent, decoders): + self._agent = agent + self._decoders = dict(decoders) + self._supported = ','.join([decoder[0] for decoder in decoders]) + + + def request(self, method, uri, headers=None, bodyProducer=None): + """ + Send a client request which declares supporting compressed content. + + @see: L{Agent.request}. + """ + if headers is None: + headers = Headers() + else: + headers = headers.copy() + headers.addRawHeader('accept-encoding', self._supported) + deferred = self._agent.request(method, uri, headers, bodyProducer) + return deferred.addCallback(self._handleResponse) + + + def _handleResponse(self, response): + """ + Check if the response is encoded, and wrap it to handle decompression. + """ + contentEncodingHeaders = response.headers.getRawHeaders( + 'content-encoding', []) + contentEncodingHeaders = ','.join(contentEncodingHeaders).split(',') + while contentEncodingHeaders: + name = contentEncodingHeaders.pop().strip() + decoder = self._decoders.get(name) + if decoder is not None: + response = decoder(response) + else: + # Add it back + contentEncodingHeaders.append(name) + break + if contentEncodingHeaders: + response.headers.setRawHeaders( + 'content-encoding', [','.join(contentEncodingHeaders)]) + else: + response.headers.removeHeader('content-encoding') + return response + + + +class RedirectAgent(object): + """ + An L{Agent} wrapper which handles HTTP redirects. + + The implementation is rather strict: 301 and 302 behaves like 307, not + redirecting automatically on methods different from C{GET} and C{HEAD}. + + @param redirectLimit: The maximum number of times the agent is allowed to + follow redirects before failing with a L{error.InfiniteRedirection}. + + @since: 11.1 + """ + + def __init__(self, agent, redirectLimit=20): + self._agent = agent + self._redirectLimit = redirectLimit + + + def request(self, method, uri, headers=None, bodyProducer=None): + """ + Send a client request following HTTP redirects. + + @see: L{Agent.request}. + """ + deferred = self._agent.request(method, uri, headers, bodyProducer) + return deferred.addCallback( + self._handleResponse, method, uri, headers, 0) + + + def _handleRedirect(self, response, method, uri, headers, redirectCount): + """ + Handle a redirect response, checking the number of redirects already + followed, and extracting the location header fields. + """ + if redirectCount >= self._redirectLimit: + err = error.InfiniteRedirection( + response.code, + 'Infinite redirection detected', + location=uri) + raise ResponseFailed([failure.Failure(err)], response) + locationHeaders = response.headers.getRawHeaders('location', []) + if not locationHeaders: + err = error.RedirectWithNoLocation( + response.code, 'No location header field', uri) + raise ResponseFailed([failure.Failure(err)], response) + location = locationHeaders[0] + deferred = self._agent.request(method, location, headers) + return deferred.addCallback( + self._handleResponse, method, uri, headers, redirectCount + 1) + + + def _handleResponse(self, response, method, uri, headers, redirectCount): + """ + Handle the response, making another request if it indicates a redirect. + """ + if response.code in (http.MOVED_PERMANENTLY, http.FOUND, + http.TEMPORARY_REDIRECT): + if method not in ('GET', 'HEAD'): + err = error.PageRedirect(response.code, location=uri) + raise ResponseFailed([failure.Failure(err)], response) + return self._handleRedirect(response, method, uri, headers, + redirectCount) + elif response.code == http.SEE_OTHER: + return self._handleRedirect(response, 'GET', uri, headers, + redirectCount) + return response + + + +class _ReadBodyProtocol(protocol.Protocol): + """ + Protocol that collects data sent to it. + + This is a helper for L{IResponse.deliverBody}, which collects the body and + fires a deferred with it. + + @ivar deferred: See L{__init__}. + @ivar status: See L{__init__}. + @ivar message: See L{__init__}. + + @ivar dataBuffer: list of byte-strings received + @type dataBuffer: L{list} of L{bytes} + """ + + def __init__(self, status, message, deferred): + """ + @param status: Status of L{IResponse} + @ivar status: L{int} + + @param message: Message of L{IResponse} + @type message: L{bytes} + + @param deferred: deferred to fire when response is complete + @type deferred: L{Deferred} firing with L{bytes} + """ + self.deferred = deferred + self.status = status + self.message = message + self.dataBuffer = [] + + + def dataReceived(self, data): + """ + Accumulate some more bytes from the response. + """ + self.dataBuffer.append(data) + + + def connectionLost(self, reason): + """ + Deliver the accumulated response bytes to the waiting L{Deferred}, if + the response body has been completely received without error. + """ + if reason.check(ResponseDone): + self.deferred.callback(b''.join(self.dataBuffer)) + elif reason.check(PotentialDataLoss): + self.deferred.errback( + PartialDownloadError(self.status, self.message, + b''.join(self.dataBuffer))) + else: + self.deferred.errback(reason) + + + +def readBody(response): + """ + Get the body of an L{IResponse} and return it as a byte string. + + This is a helper function for clients that don't want to incrementally + receive the body of an HTTP response. + + @param response: The HTTP response for which the body will be read. + @type response: L{IResponse} provider + + @return: A L{Deferred} which will fire with the body of the response. + """ + d = defer.Deferred() + response.deliverBody(_ReadBodyProtocol(response.code, response.phrase, d)) + return d + + + +__all__ = [ + 'PartialDownloadError', 'HTTPPageGetter', 'HTTPPageDownloader', + 'HTTPClientFactory', 'HTTPDownloader', 'getPage', 'downloadPage', + 'ResponseDone', 'Response', 'ResponseFailed', 'Agent', 'CookieAgent', + 'ProxyAgent', 'ContentDecoderAgent', 'GzipDecoder', 'RedirectAgent', + 'HTTPConnectionPool', 'readBody'] diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py new file mode 100644 index 000000000..5548dbfbf --- /dev/null +++ b/scrapy/xlib/tx/endpoints.py @@ -0,0 +1,1269 @@ +# -*- test-case-name: twisted.internet.test.test_endpoints -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Implementations of L{IStreamServerEndpoint} and L{IStreamClientEndpoint} that +wrap the L{IReactorTCP}, L{IReactorSSL}, and L{IReactorUNIX} interfaces. + +This also implements an extensible mini-language for describing endpoints, +parsed by the L{clientFromString} and L{serverFromString} functions. + +@since: 10.1 +""" + +from __future__ import division, absolute_import + +import os +import socket + +from zope.interface import implementer, directlyProvides +import warnings + +from twisted.internet import interfaces, defer, error, fdesc, threads +from twisted.internet.protocol import ( + ClientFactory, Protocol, ProcessProtocol, Factory) +from twisted.internet.interfaces import IStreamServerEndpointStringParser +from twisted.internet.interfaces import IStreamClientEndpointStringParser +from twisted.python.filepath import FilePath +from twisted.python.failure import Failure +from twisted.python import log +from twisted.python.components import proxyForInterface + +from twisted.plugin import IPlugin, getPlugins +from twisted.internet import stdio + +from .interfaces import IFileDescriptorReceiver + + +__all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"] + + +class _WrappingProtocol(Protocol): + """ + Wrap another protocol in order to notify my user when a connection has + been made. + """ + + def __init__(self, connectedDeferred, wrappedProtocol): + """ + @param connectedDeferred: The L{Deferred} that will callback + with the C{wrappedProtocol} when it is connected. + + @param wrappedProtocol: An L{IProtocol} provider that will be + connected. + """ + self._connectedDeferred = connectedDeferred + self._wrappedProtocol = wrappedProtocol + + for iface in [interfaces.IHalfCloseableProtocol, + IFileDescriptorReceiver]: + if iface.providedBy(self._wrappedProtocol): + directlyProvides(self, iface) + + + def logPrefix(self): + """ + Transparently pass through the wrapped protocol's log prefix. + """ + if interfaces.ILoggingContext.providedBy(self._wrappedProtocol): + return self._wrappedProtocol.logPrefix() + return self._wrappedProtocol.__class__.__name__ + + + def connectionMade(self): + """ + Connect the C{self._wrappedProtocol} to our C{self.transport} and + callback C{self._connectedDeferred} with the C{self._wrappedProtocol} + """ + self._wrappedProtocol.makeConnection(self.transport) + self._connectedDeferred.callback(self._wrappedProtocol) + + + def dataReceived(self, data): + """ + Proxy C{dataReceived} calls to our C{self._wrappedProtocol} + """ + return self._wrappedProtocol.dataReceived(data) + + + def fileDescriptorReceived(self, descriptor): + """ + Proxy C{fileDescriptorReceived} calls to our C{self._wrappedProtocol} + """ + return self._wrappedProtocol.fileDescriptorReceived(descriptor) + + + def connectionLost(self, reason): + """ + Proxy C{connectionLost} calls to our C{self._wrappedProtocol} + """ + return self._wrappedProtocol.connectionLost(reason) + + + def readConnectionLost(self): + """ + Proxy L{IHalfCloseableProtocol.readConnectionLost} to our + C{self._wrappedProtocol} + """ + self._wrappedProtocol.readConnectionLost() + + + def writeConnectionLost(self): + """ + Proxy L{IHalfCloseableProtocol.writeConnectionLost} to our + C{self._wrappedProtocol} + """ + self._wrappedProtocol.writeConnectionLost() + + + +class _WrappingFactory(ClientFactory): + """ + Wrap a factory in order to wrap the protocols it builds. + + @ivar _wrappedFactory: A provider of I{IProtocolFactory} whose buildProtocol + method will be called and whose resulting protocol will be wrapped. + + @ivar _onConnection: A L{Deferred} that fires when the protocol is + connected + + @ivar _connector: A L{connector } + that is managing the current or previous connection attempt. + """ + protocol = _WrappingProtocol + + def __init__(self, wrappedFactory): + """ + @param wrappedFactory: A provider of I{IProtocolFactory} whose + buildProtocol method will be called and whose resulting protocol + will be wrapped. + """ + self._wrappedFactory = wrappedFactory + self._onConnection = defer.Deferred(canceller=self._canceller) + + + def startedConnecting(self, connector): + """ + A connection attempt was started. Remember the connector which started + said attempt, for use later. + """ + self._connector = connector + + + def _canceller(self, deferred): + """ + The outgoing connection attempt was cancelled. Fail that L{Deferred} + with an L{error.ConnectingCancelledError}. + + @param deferred: The L{Deferred } that was cancelled; + should be the same as C{self._onConnection}. + @type deferred: L{Deferred } + + @note: This relies on startedConnecting having been called, so it may + seem as though there's a race condition where C{_connector} may not + have been set. However, using public APIs, this condition is + impossible to catch, because a connection API + (C{connectTCP}/C{SSL}/C{UNIX}) is always invoked before a + L{_WrappingFactory}'s L{Deferred } is returned to + C{connect()}'s caller. + + @return: C{None} + """ + deferred.errback( + error.ConnectingCancelledError( + self._connector.getDestination())) + self._connector.stopConnecting() + + + def doStart(self): + """ + Start notifications are passed straight through to the wrapped factory. + """ + self._wrappedFactory.doStart() + + + def doStop(self): + """ + Stop notifications are passed straight through to the wrapped factory. + """ + self._wrappedFactory.doStop() + + + def buildProtocol(self, addr): + """ + Proxy C{buildProtocol} to our C{self._wrappedFactory} or errback + the C{self._onConnection} L{Deferred}. + + @return: An instance of L{_WrappingProtocol} or C{None} + """ + try: + proto = self._wrappedFactory.buildProtocol(addr) + except: + self._onConnection.errback() + else: + return self.protocol(self._onConnection, proto) + + + def clientConnectionFailed(self, connector, reason): + """ + Errback the C{self._onConnection} L{Deferred} when the + client connection fails. + """ + if not self._onConnection.called: + self._onConnection.errback(reason) + + + + + +@implementer(interfaces.ITransport) +class _ProcessEndpointTransport(proxyForInterface( + interfaces.IProcessTransport, '_process')): + """ + An L{ITransport} provider for the L{IProtocol} instance passed to the + process endpoint. + + @ivar _process: An active process transport which will be used by write + methods on this object to write data to a child process. + @type _process: L{interfaces.IProcessTransport} provider + """ + + def write(self, data): + """ + Write to the child process's standard input. + + @param data: The data to write on stdin. + """ + self._process.writeToChild(0, data) + + + def writeSequence(self, data): + """ + Write a list of strings to child process's stdin. + + @param data: The list of chunks to write on stdin. + """ + for chunk in data: + self._process.writeToChild(0, chunk) + + +@implementer(interfaces.IStreamServerEndpoint) +class _TCPServerEndpoint(object): + """ + A TCP server endpoint interface + """ + + def __init__(self, reactor, port, backlog, interface): + """ + @param reactor: An L{IReactorTCP} provider. + + @param port: The port number used for listening + @type port: int + + @param backlog: Size of the listen queue + @type backlog: int + + @param interface: The hostname to bind to + @type interface: str + """ + self._reactor = reactor + self._port = port + self._backlog = backlog + self._interface = interface + + + def listen(self, protocolFactory): + """ + Implement L{IStreamServerEndpoint.listen} to listen on a TCP + socket + """ + return defer.execute(self._reactor.listenTCP, + self._port, + protocolFactory, + backlog=self._backlog, + interface=self._interface) + + + +class TCP4ServerEndpoint(_TCPServerEndpoint): + """ + Implements TCP server endpoint with an IPv4 configuration + """ + def __init__(self, reactor, port, backlog=50, interface=''): + """ + @param reactor: An L{IReactorTCP} provider. + + @param port: The port number used for listening + @type port: int + + @param backlog: Size of the listen queue + @type backlog: int + + @param interface: The hostname to bind to, defaults to '' (all) + @type interface: str + """ + _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface) + + + +class TCP6ServerEndpoint(_TCPServerEndpoint): + """ + Implements TCP server endpoint with an IPv6 configuration + """ + def __init__(self, reactor, port, backlog=50, interface='::'): + """ + @param reactor: An L{IReactorTCP} provider. + + @param port: The port number used for listening + @type port: int + + @param backlog: Size of the listen queue + @type backlog: int + + @param interface: The hostname to bind to, defaults to '' (all) + @type interface: str + """ + _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface) + + + +@implementer(interfaces.IStreamClientEndpoint) +class TCP4ClientEndpoint(object): + """ + TCP client endpoint with an IPv4 configuration. + """ + + def __init__(self, reactor, host, port, timeout=30, bindAddress=None): + """ + @param reactor: An L{IReactorTCP} provider + + @param host: A hostname, used when connecting + @type host: str + + @param port: The port number, used when connecting + @type port: int + + @param timeout: The number of seconds to wait before assuming the + connection has failed. + @type timeout: int + + @param bindAddress: A (host, port) tuple of local address to bind to, + or None. + @type bindAddress: tuple + """ + self._reactor = reactor + self._host = host + self._port = port + self._timeout = timeout + self._bindAddress = bindAddress + + + def connect(self, protocolFactory): + """ + Implement L{IStreamClientEndpoint.connect} to connect via TCP. + """ + try: + wf = _WrappingFactory(protocolFactory) + self._reactor.connectTCP( + self._host, self._port, wf, + timeout=self._timeout, bindAddress=self._bindAddress) + return wf._onConnection + except: + return defer.fail() + + + + +@implementer(interfaces.IStreamServerEndpoint) +class SSL4ServerEndpoint(object): + """ + SSL secured TCP server endpoint with an IPv4 configuration. + """ + + def __init__(self, reactor, port, sslContextFactory, + backlog=50, interface=''): + """ + @param reactor: An L{IReactorSSL} provider. + + @param port: The port number used for listening + @type port: int + + @param sslContextFactory: An instance of + L{twisted.internet.ssl.ContextFactory}. + + @param backlog: Size of the listen queue + @type backlog: int + + @param interface: The hostname to bind to, defaults to '' (all) + @type interface: str + """ + self._reactor = reactor + self._port = port + self._sslContextFactory = sslContextFactory + self._backlog = backlog + self._interface = interface + + + def listen(self, protocolFactory): + """ + Implement L{IStreamServerEndpoint.listen} to listen for SSL on a + TCP socket. + """ + return defer.execute(self._reactor.listenSSL, self._port, + protocolFactory, + contextFactory=self._sslContextFactory, + backlog=self._backlog, + interface=self._interface) + + + +@implementer(interfaces.IStreamClientEndpoint) +class SSL4ClientEndpoint(object): + """ + SSL secured TCP client endpoint with an IPv4 configuration + """ + + def __init__(self, reactor, host, port, sslContextFactory, + timeout=30, bindAddress=None): + """ + @param reactor: An L{IReactorSSL} provider. + + @param host: A hostname, used when connecting + @type host: str + + @param port: The port number, used when connecting + @type port: int + + @param sslContextFactory: SSL Configuration information as an instance + of L{twisted.internet.ssl.ContextFactory}. + + @param timeout: Number of seconds to wait before assuming the + connection has failed. + @type timeout: int + + @param bindAddress: A (host, port) tuple of local address to bind to, + or None. + @type bindAddress: tuple + """ + self._reactor = reactor + self._host = host + self._port = port + self._sslContextFactory = sslContextFactory + self._timeout = timeout + self._bindAddress = bindAddress + + + def connect(self, protocolFactory): + """ + Implement L{IStreamClientEndpoint.connect} to connect with SSL over + TCP. + """ + try: + wf = _WrappingFactory(protocolFactory) + self._reactor.connectSSL( + self._host, self._port, wf, self._sslContextFactory, + timeout=self._timeout, bindAddress=self._bindAddress) + return wf._onConnection + except: + return defer.fail() + + + +@implementer(interfaces.IStreamServerEndpoint) +class UNIXServerEndpoint(object): + """ + UnixSocket server endpoint. + """ + def __init__(self, reactor, address, backlog=50, mode=0o666, wantPID=0): + """ + @param reactor: An L{IReactorUNIX} provider. + @param address: The path to the Unix socket file, used when listening + @param backlog: number of connections to allow in backlog. + @param mode: mode to set on the unix socket. This parameter is + deprecated. Permissions should be set on the directory which + contains the UNIX socket. + @param wantPID: If True, create a pidfile for the socket. + """ + self._reactor = reactor + self._address = address + self._backlog = backlog + self._mode = mode + self._wantPID = wantPID + + + def listen(self, protocolFactory): + """ + Implement L{IStreamServerEndpoint.listen} to listen on a UNIX socket. + """ + return defer.execute(self._reactor.listenUNIX, self._address, + protocolFactory, + backlog=self._backlog, + mode=self._mode, + wantPID=self._wantPID) + + + +@implementer(interfaces.IStreamClientEndpoint) +class UNIXClientEndpoint(object): + """ + UnixSocket client endpoint. + """ + def __init__(self, reactor, path, timeout=30, checkPID=0): + """ + @param reactor: An L{IReactorUNIX} provider. + + @param path: The path to the Unix socket file, used when connecting + @type path: str + + @param timeout: Number of seconds to wait before assuming the + connection has failed. + @type timeout: int + + @param checkPID: If True, check for a pid file to verify that a server + is listening. + @type checkPID: bool + """ + self._reactor = reactor + self._path = path + self._timeout = timeout + self._checkPID = checkPID + + + def connect(self, protocolFactory): + """ + Implement L{IStreamClientEndpoint.connect} to connect via a + UNIX Socket + """ + try: + wf = _WrappingFactory(protocolFactory) + self._reactor.connectUNIX( + self._path, wf, + timeout=self._timeout, + checkPID=self._checkPID) + return wf._onConnection + except: + return defer.fail() + + + +@implementer(interfaces.IStreamServerEndpoint) +class AdoptedStreamServerEndpoint(object): + """ + An endpoint for listening on a file descriptor initialized outside of + Twisted. + + @ivar _used: A C{bool} indicating whether this endpoint has been used to + listen with a factory yet. C{True} if so. + """ + _close = os.close + _setNonBlocking = staticmethod(fdesc.setNonBlocking) + + def __init__(self, reactor, fileno, addressFamily): + """ + @param reactor: An L{IReactorSocket} provider. + + @param fileno: An integer file descriptor corresponding to a listening + I{SOCK_STREAM} socket. + + @param addressFamily: The address family of the socket given by + C{fileno}. + """ + self.reactor = reactor + self.fileno = fileno + self.addressFamily = addressFamily + self._used = False + + + def listen(self, factory): + """ + Implement L{IStreamServerEndpoint.listen} to start listening on, and + then close, C{self._fileno}. + """ + if self._used: + return defer.fail(error.AlreadyListened()) + self._used = True + + try: + self._setNonBlocking(self.fileno) + port = self.reactor.adoptStreamPort( + self.fileno, self.addressFamily, factory) + self._close(self.fileno) + except: + return defer.fail() + return defer.succeed(port) + + + +def _parseTCP(factory, port, interface="", backlog=50): + """ + Internal parser function for L{_parseServer} to convert the string + arguments for a TCP(IPv4) stream endpoint into the structured arguments. + + @param factory: the protocol factory being parsed, or C{None}. (This was a + leftover argument from when this code was in C{strports}, and is now + mostly None and unused.) + + @type factory: L{IProtocolFactory} or C{NoneType} + + @param port: the integer port number to bind + @type port: C{str} + + @param interface: the interface IP to listen on + @param backlog: the length of the listen queue + @type backlog: C{str} + + @return: a 2-tuple of (args, kwargs), describing the parameters to + L{IReactorTCP.listenTCP} (or, modulo argument 2, the factory, arguments + to L{TCP4ServerEndpoint}. + """ + return (int(port), factory), {'interface': interface, + 'backlog': int(backlog)} + + + +def _parseUNIX(factory, address, mode='666', backlog=50, lockfile=True): + """ + Internal parser function for L{_parseServer} to convert the string + arguments for a UNIX (AF_UNIX/SOCK_STREAM) stream endpoint into the + structured arguments. + + @param factory: the protocol factory being parsed, or C{None}. (This was a + leftover argument from when this code was in C{strports}, and is now + mostly None and unused.) + + @type factory: L{IProtocolFactory} or C{NoneType} + + @param address: the pathname of the unix socket + @type address: C{str} + + @param backlog: the length of the listen queue + @type backlog: C{str} + + @param lockfile: A string '0' or '1', mapping to True and False + respectively. See the C{wantPID} argument to C{listenUNIX} + + @return: a 2-tuple of (args, kwargs), describing the parameters to + L{IReactorTCP.listenUNIX} (or, modulo argument 2, the factory, + arguments to L{UNIXServerEndpoint}. + """ + return ( + (address, factory), + {'mode': int(mode, 8), 'backlog': int(backlog), + 'wantPID': bool(int(lockfile))}) + + + +def _parseSSL(factory, port, privateKey="server.pem", certKey=None, + sslmethod=None, interface='', backlog=50): + """ + Internal parser function for L{_parseServer} to convert the string + arguments for an SSL (over TCP/IPv4) stream endpoint into the structured + arguments. + + @param factory: the protocol factory being parsed, or C{None}. (This was a + leftover argument from when this code was in C{strports}, and is now + mostly None and unused.) + @type factory: L{IProtocolFactory} or C{NoneType} + + @param port: the integer port number to bind + @type port: C{str} + + @param interface: the interface IP to listen on + @param backlog: the length of the listen queue + @type backlog: C{str} + + @param privateKey: The file name of a PEM format private key file. + @type privateKey: C{str} + + @param certKey: The file name of a PEM format certificate file. + @type certKey: C{str} + + @param sslmethod: The string name of an SSL method, based on the name of a + constant in C{OpenSSL.SSL}. Must be one of: "SSLv23_METHOD", + "SSLv2_METHOD", "SSLv3_METHOD", "TLSv1_METHOD". + @type sslmethod: C{str} + + @return: a 2-tuple of (args, kwargs), describing the parameters to + L{IReactorSSL.listenSSL} (or, modulo argument 2, the factory, arguments + to L{SSL4ServerEndpoint}. + """ + from twisted.internet import ssl + if certKey is None: + certKey = privateKey + kw = {} + if sslmethod is not None: + kw['method'] = getattr(ssl.SSL, sslmethod) + else: + kw['method'] = ssl.SSL.SSLv23_METHOD + certPEM = FilePath(certKey).getContent() + keyPEM = FilePath(privateKey).getContent() + privateCertificate = ssl.PrivateCertificate.loadPEM(certPEM + keyPEM) + cf = ssl.CertificateOptions( + privateKey=privateCertificate.privateKey.original, + certificate=privateCertificate.original, + **kw + ) + return ((int(port), factory, cf), + {'interface': interface, 'backlog': int(backlog)}) + + + +@implementer(IPlugin, IStreamServerEndpointStringParser) +class _StandardIOParser(object): + """ + Stream server endpoint string parser for the Standard I/O type. + + @ivar prefix: See L{IStreamClientEndpointStringParser.prefix}. + """ + prefix = "stdio" + + def _parseServer(self, reactor): + """ + Internal parser function for L{_parseServer} to convert the string + arguments into structured arguments for the L{StandardIOEndpoint} + + @param reactor: Reactor for the endpoint + """ + return StandardIOEndpoint(reactor) + + + def parseStreamServer(self, reactor, *args, **kwargs): + # Redirects to another function (self._parseServer), tricks zope.interface + # into believing the interface is correctly implemented. + return self._parseServer(reactor) + + + + +@implementer(IPlugin, IStreamServerEndpointStringParser) +class _TCP6ServerParser(object): + """ + Stream server endpoint string parser for the TCP6ServerEndpoint type. + + @ivar prefix: See L{IStreamClientEndpointStringParser.prefix}. + """ + prefix = "tcp6" # Used in _parseServer to identify the plugin with the endpoint type + + def _parseServer(self, reactor, port, backlog=50, interface='::'): + """ + Internal parser function for L{_parseServer} to convert the string + arguments into structured arguments for the L{TCP6ServerEndpoint} + + @param reactor: An L{IReactorTCP} provider. + + @param port: The port number used for listening + @type port: int + + @param backlog: Size of the listen queue + @type backlog: int + + @param interface: The hostname to bind to + @type interface: str + """ + port = int(port) + backlog = int(backlog) + return TCP6ServerEndpoint(reactor, port, backlog, interface) + + + def parseStreamServer(self, reactor, *args, **kwargs): + # Redirects to another function (self._parseServer), tricks zope.interface + # into believing the interface is correctly implemented. + return self._parseServer(reactor, *args, **kwargs) + + + +_serverParsers = {"tcp": _parseTCP, + "unix": _parseUNIX, + "ssl": _parseSSL, + } + +_OP, _STRING = range(2) + +def _tokenize(description): + """ + Tokenize a strports string and yield each token. + + @param description: a string as described by L{serverFromString} or + L{clientFromString}. + + @return: an iterable of 2-tuples of (L{_OP} or L{_STRING}, string). Tuples + starting with L{_OP} will contain a second element of either ':' (i.e. + 'next parameter') or '=' (i.e. 'assign parameter value'). For example, + the string 'hello:greet\=ing=world' would result in a generator + yielding these values:: + + _STRING, 'hello' + _OP, ':' + _STRING, 'greet=ing' + _OP, '=' + _STRING, 'world' + """ + current = '' + ops = ':=' + nextOps = {':': ':=', '=': ':'} + description = iter(description) + for n in description: + if n in ops: + yield _STRING, current + yield _OP, n + current = '' + ops = nextOps[n] + elif n == '\\': + current += description.next() + else: + current += n + yield _STRING, current + + + +def _parse(description): + """ + Convert a description string into a list of positional and keyword + parameters, using logic vaguely like what Python does. + + @param description: a string as described by L{serverFromString} or + L{clientFromString}. + + @return: a 2-tuple of C{(args, kwargs)}, where 'args' is a list of all + ':'-separated C{str}s not containing an '=' and 'kwargs' is a map of + all C{str}s which do contain an '='. For example, the result of + C{_parse('a:b:d=1:c')} would be C{(['a', 'b', 'c'], {'d': '1'})}. + """ + args, kw = [], {} + def add(sofar): + if len(sofar) == 1: + args.append(sofar[0]) + else: + kw[sofar[0]] = sofar[1] + sofar = () + for (type, value) in _tokenize(description): + if type is _STRING: + sofar += (value,) + elif value == ':': + add(sofar) + sofar = () + add(sofar) + return args, kw + + +# Mappings from description "names" to endpoint constructors. +_endpointServerFactories = { + 'TCP': TCP4ServerEndpoint, + 'SSL': SSL4ServerEndpoint, + 'UNIX': UNIXServerEndpoint, + } + +_endpointClientFactories = { + 'TCP': TCP4ClientEndpoint, + 'SSL': SSL4ClientEndpoint, + 'UNIX': UNIXClientEndpoint, + } + + +_NO_DEFAULT = object() + +def _parseServer(description, factory, default=None): + """ + Parse a stports description into a 2-tuple of arguments and keyword values. + + @param description: A description in the format explained by + L{serverFromString}. + @type description: C{str} + + @param factory: A 'factory' argument; this is left-over from + twisted.application.strports, it's not really used. + @type factory: L{IProtocolFactory} or L{None} + + @param default: Deprecated argument, specifying the default parser mode to + use for unqualified description strings (those which do not have a ':' + and prefix). + @type default: C{str} or C{NoneType} + + @return: a 3-tuple of (plugin or name, arguments, keyword arguments) + """ + args, kw = _parse(description) + if not args or (len(args) == 1 and not kw): + deprecationMessage = ( + "Unqualified strport description passed to 'service'." + "Use qualified endpoint descriptions; for example, 'tcp:%s'." + % (description,)) + if default is None: + default = 'tcp' + warnings.warn( + deprecationMessage, category=DeprecationWarning, stacklevel=4) + elif default is _NO_DEFAULT: + raise ValueError(deprecationMessage) + # If the default has been otherwise specified, the user has already + # been warned. + args[0:0] = [default] + endpointType = args[0] + parser = _serverParsers.get(endpointType) + if parser is None: + # If the required parser is not found in _server, check if + # a plugin exists for the endpointType + for plugin in getPlugins(IStreamServerEndpointStringParser): + if plugin.prefix == endpointType: + return (plugin, args[1:], kw) + raise ValueError("Unknown endpoint type: '%s'" % (endpointType,)) + return (endpointType.upper(),) + parser(factory, *args[1:], **kw) + + + +def _serverFromStringLegacy(reactor, description, default): + """ + Underlying implementation of L{serverFromString} which avoids exposing the + deprecated 'default' argument to anything but L{strports.service}. + """ + nameOrPlugin, args, kw = _parseServer(description, None, default) + if type(nameOrPlugin) is not str: + plugin = nameOrPlugin + return plugin.parseStreamServer(reactor, *args, **kw) + else: + name = nameOrPlugin + # Chop out the factory. + args = args[:1] + args[2:] + return _endpointServerFactories[name](reactor, *args, **kw) + + + +def serverFromString(reactor, description): + """ + Construct a stream server endpoint from an endpoint description string. + + The format for server endpoint descriptions is a simple string. It is a + prefix naming the type of endpoint, then a colon, then the arguments for + that endpoint. + + For example, you can call it like this to create an endpoint that will + listen on TCP port 80:: + + serverFromString(reactor, "tcp:80") + + Additional arguments may be specified as keywords, separated with colons. + For example, you can specify the interface for a TCP server endpoint to + bind to like this:: + + serverFromString(reactor, "tcp:80:interface=127.0.0.1") + + SSL server endpoints may be specified with the 'ssl' prefix, and the + private key and certificate files may be specified by the C{privateKey} and + C{certKey} arguments:: + + serverFromString(reactor, "ssl:443:privateKey=key.pem:certKey=crt.pem") + + If a private key file name (C{privateKey}) isn't provided, a "server.pem" + file is assumed to exist which contains the private key. If the certificate + file name (C{certKey}) isn't provided, the private key file is assumed to + contain the certificate as well. + + You may escape colons in arguments with a backslash, which you will need to + use if you want to specify a full pathname argument on Windows:: + + serverFromString(reactor, + "ssl:443:privateKey=C\\:/key.pem:certKey=C\\:/cert.pem") + + finally, the 'unix' prefix may be used to specify a filesystem UNIX socket, + optionally with a 'mode' argument to specify the mode of the socket file + created by C{listen}:: + + serverFromString(reactor, "unix:/var/run/finger") + serverFromString(reactor, "unix:/var/run/finger:mode=660") + + This function is also extensible; new endpoint types may be registered as + L{IStreamServerEndpointStringParser} plugins. See that interface for more + information. + + @param reactor: The server endpoint will be constructed with this reactor. + + @param description: The strports description to parse. + + @return: A new endpoint which can be used to listen with the parameters + given by by C{description}. + + @rtype: L{IStreamServerEndpoint} + + @raise ValueError: when the 'description' string cannot be parsed. + + @since: 10.2 + """ + return _serverFromStringLegacy(reactor, description, _NO_DEFAULT) + + + +def quoteStringArgument(argument): + """ + Quote an argument to L{serverFromString} and L{clientFromString}. Since + arguments are separated with colons and colons are escaped with + backslashes, some care is necessary if, for example, you have a pathname, + you may be tempted to interpolate into a string like this:: + + serverFromString("ssl:443:privateKey=%s" % (myPathName,)) + + This may appear to work, but will have portability issues (Windows + pathnames, for example). Usually you should just construct the appropriate + endpoint type rather than interpolating strings, which in this case would + be L{SSL4ServerEndpoint}. There are some use-cases where you may need to + generate such a string, though; for example, a tool to manipulate a + configuration file which has strports descriptions in it. To be correct in + those cases, do this instead:: + + serverFromString("ssl:443:privateKey=%s" % + (quoteStringArgument(myPathName),)) + + @param argument: The part of the endpoint description string you want to + pass through. + + @type argument: C{str} + + @return: The quoted argument. + + @rtype: C{str} + """ + return argument.replace('\\', '\\\\').replace(':', '\\:') + + + +def _parseClientTCP(*args, **kwargs): + """ + Perform any argument value coercion necessary for TCP client parameters. + + Valid positional arguments to this function are host and port. + + Valid keyword arguments to this function are all L{IReactorTCP.connectTCP} + arguments. + + @return: The coerced values as a C{dict}. + """ + + if len(args) == 2: + kwargs['port'] = int(args[1]) + kwargs['host'] = args[0] + elif len(args) == 1: + if 'host' in kwargs: + kwargs['port'] = int(args[0]) + else: + kwargs['host'] = args[0] + + try: + kwargs['port'] = int(kwargs['port']) + except KeyError: + pass + + try: + kwargs['timeout'] = int(kwargs['timeout']) + except KeyError: + pass + return kwargs + + + +def _loadCAsFromDir(directoryPath): + """ + Load certificate-authority certificate objects in a given directory. + + @param directoryPath: a L{FilePath} pointing at a directory to load .pem + files from. + + @return: a C{list} of L{OpenSSL.crypto.X509} objects. + """ + from twisted.internet import ssl + + caCerts = {} + for child in directoryPath.children(): + if not child.basename().split('.')[-1].lower() == 'pem': + continue + try: + data = child.getContent() + except IOError: + # Permission denied, corrupt disk, we don't care. + continue + try: + theCert = ssl.Certificate.loadPEM(data) + except ssl.SSL.Error: + # Duplicate certificate, invalid certificate, etc. We don't care. + pass + else: + caCerts[theCert.digest()] = theCert.original + return caCerts.values() + + + +def _parseClientSSL(*args, **kwargs): + """ + Perform any argument value coercion necessary for SSL client parameters. + + Valid keyword arguments to this function are all L{IReactorSSL.connectSSL} + arguments except for C{contextFactory}. Instead, C{certKey} (the path name + of the certificate file) C{privateKey} (the path name of the private key + associated with the certificate) are accepted and used to construct a + context factory. + + Valid positional arguments to this function are host and port. + + @param caCertsDir: The one parameter which is not part of + L{IReactorSSL.connectSSL}'s signature, this is a path name used to + construct a list of certificate authority certificates. The directory + will be scanned for files ending in C{.pem}, all of which will be + considered valid certificate authorities for this connection. + + @type caCertsDir: C{str} + + @return: The coerced values as a C{dict}. + """ + from twisted.internet import ssl + kwargs = _parseClientTCP(*args, **kwargs) + certKey = kwargs.pop('certKey', None) + privateKey = kwargs.pop('privateKey', None) + caCertsDir = kwargs.pop('caCertsDir', None) + if certKey is not None: + certx509 = ssl.Certificate.loadPEM( + FilePath(certKey).getContent()).original + else: + certx509 = None + if privateKey is not None: + privateKey = ssl.PrivateCertificate.loadPEM( + FilePath(privateKey).getContent()).privateKey.original + else: + privateKey = None + if caCertsDir is not None: + verify = True + caCerts = _loadCAsFromDir(FilePath(caCertsDir)) + else: + verify = False + caCerts = None + kwargs['sslContextFactory'] = ssl.CertificateOptions( + method=ssl.SSL.SSLv23_METHOD, + certificate=certx509, + privateKey=privateKey, + verify=verify, + caCerts=caCerts + ) + return kwargs + + + +def _parseClientUNIX(*args, **kwargs): + """ + Perform any argument value coercion necessary for UNIX client parameters. + + Valid keyword arguments to this function are all L{IReactorUNIX.connectUNIX} + keyword arguments except for C{checkPID}. Instead, C{lockfile} is accepted + and has the same meaning. Also C{path} is used instead of C{address}. + + Valid positional arguments to this function are C{path}. + + @return: The coerced values as a C{dict}. + """ + if len(args) == 1: + kwargs['path'] = args[0] + + try: + kwargs['checkPID'] = bool(int(kwargs.pop('lockfile'))) + except KeyError: + pass + try: + kwargs['timeout'] = int(kwargs['timeout']) + except KeyError: + pass + return kwargs + +_clientParsers = { + 'TCP': _parseClientTCP, + 'SSL': _parseClientSSL, + 'UNIX': _parseClientUNIX, + } + + + +def clientFromString(reactor, description): + """ + Construct a client endpoint from a description string. + + Client description strings are much like server description strings, + although they take all of their arguments as keywords, aside from host and + port. + + You can create a TCP client endpoint with the 'host' and 'port' arguments, + like so:: + + clientFromString(reactor, "tcp:host=www.example.com:port=80") + + or, without specifying host and port keywords:: + + clientFromString(reactor, "tcp:www.example.com:80") + + Or you can specify only one or the other, as in the following 2 examples:: + + clientFromString(reactor, "tcp:host=www.example.com:80") + clientFromString(reactor, "tcp:www.example.com:port=80") + + or an SSL client endpoint with those arguments, plus the arguments used by + the server SSL, for a client certificate:: + + clientFromString(reactor, "ssl:web.example.com:443:" + "privateKey=foo.pem:certKey=foo.pem") + + to specify your certificate trust roots, you can identify a directory with + PEM files in it with the C{caCertsDir} argument:: + + clientFromString(reactor, "ssl:host=web.example.com:port=443:" + "caCertsDir=/etc/ssl/certs") + + You can create a UNIX client endpoint with the 'path' argument and optional + 'lockfile' and 'timeout' arguments:: + + clientFromString(reactor, "unix:path=/var/foo/bar:lockfile=1:timeout=9") + + or, with the path as a positional argument with or without optional + arguments as in the following 2 examples:: + + clientFromString(reactor, "unix:/var/foo/bar") + clientFromString(reactor, "unix:/var/foo/bar:lockfile=1:timeout=9") + + This function is also extensible; new endpoint types may be registered as + L{IStreamClientEndpointStringParser} plugins. See that interface for more + information. + + @param reactor: The client endpoint will be constructed with this reactor. + + @param description: The strports description to parse. + + @return: A new endpoint which can be used to connect with the parameters + given by by C{description}. + @rtype: L{IStreamClientEndpoint} + + @since: 10.2 + """ + args, kwargs = _parse(description) + aname = args.pop(0) + name = aname.upper() + for plugin in getPlugins(IStreamClientEndpointStringParser): + if plugin.prefix.upper() == name: + return plugin.parseStreamClient(*args, **kwargs) + if name not in _clientParsers: + raise ValueError("Unknown endpoint type: %r" % (aname,)) + kwargs = _clientParsers[name](*args, **kwargs) + return _endpointClientFactories[name](reactor, **kwargs) + + + +def connectProtocol(endpoint, protocol): + """ + Connect a protocol instance to an endpoint. + + This allows using a client endpoint without having to create a factory. + + @param endpoint: A client endpoint to connect to. + + @param protocol: A protocol instance. + + @return: The result of calling C{connect} on the endpoint, i.e. a + L{Deferred} that will fire with the protocol when connected, or an + appropriate error. + """ + class OneShotFactory(Factory): + def buildProtocol(self, addr): + return protocol + return endpoint.connect(OneShotFactory()) + diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py new file mode 100644 index 000000000..7e294c450 --- /dev/null +++ b/scrapy/xlib/tx/interfaces.py @@ -0,0 +1,2442 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Interface documentation. + +Maintainer: Itamar Shtull-Trauring +""" + +from __future__ import division, absolute_import + +from zope.interface import Interface, Attribute + + +class IAddress(Interface): + """ + An address, e.g. a TCP C{(host, port)}. + + Default implementations are in L{twisted.internet.address}. + """ + +### Reactor Interfaces + +class IConnector(Interface): + """ + Object used to interface between connections and protocols. + + Each L{IConnector} manages one connection. + """ + + def stopConnecting(): + """ + Stop attempting to connect. + """ + + def disconnect(): + """ + Disconnect regardless of the connection state. + + If we are connected, disconnect, if we are trying to connect, + stop trying. + """ + + def connect(): + """ + Try to connect to remote address. + """ + + def getDestination(): + """ + Return destination this will try to connect to. + + @return: An object which provides L{IAddress}. + """ + + + +class IResolverSimple(Interface): + def getHostByName(name, timeout = (1, 3, 11, 45)): + """ + Resolve the domain name C{name} into an IP address. + + @type name: C{str} + @type timeout: C{tuple} + @rtype: L{twisted.internet.defer.Deferred} + @return: The callback of the Deferred that is returned will be + passed a string that represents the IP address of the specified + name, or the errback will be called if the lookup times out. If + multiple types of address records are associated with the name, + A6 records will be returned in preference to AAAA records, which + will be returned in preference to A records. If there are multiple + records of the type to be returned, one will be selected at random. + + @raise twisted.internet.defer.TimeoutError: Raised (asynchronously) + if the name cannot be resolved within the specified timeout period. + """ + + + +class IResolver(IResolverSimple): + def query(query, timeout=None): + """ + Dispatch C{query} to the method which can handle its type. + + @type query: L{twisted.names.dns.Query} + @param query: The DNS query being issued, to which a response is to be + generated. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupAddress(name, timeout=None): + """ + Perform an A record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupAddress6(name, timeout=None): + """ + Perform an A6 record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupIPV6Address(name, timeout=None): + """ + Perform an AAAA record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupMailExchange(name, timeout=None): + """ + Perform an MX record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupNameservers(name, timeout=None): + """ + Perform an NS record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupCanonicalName(name, timeout=None): + """ + Perform a CNAME record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupMailBox(name, timeout=None): + """ + Perform an MB record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupMailGroup(name, timeout=None): + """ + Perform an MG record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupMailRename(name, timeout=None): + """ + Perform an MR record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupPointer(name, timeout=None): + """ + Perform a PTR record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupAuthority(name, timeout=None): + """ + Perform an SOA record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupNull(name, timeout=None): + """ + Perform a NULL record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupWellKnownServices(name, timeout=None): + """ + Perform a WKS record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupHostInfo(name, timeout=None): + """ + Perform a HINFO record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupMailboxInfo(name, timeout=None): + """ + Perform an MINFO record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupText(name, timeout=None): + """ + Perform a TXT record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupResponsibility(name, timeout=None): + """ + Perform an RP record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupAFSDatabase(name, timeout=None): + """ + Perform an AFSDB record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupService(name, timeout=None): + """ + Perform an SRV record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupAllRecords(name, timeout=None): + """ + Perform an ALL_RECORD lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupSenderPolicy(name, timeout= 10): + """ + Perform a SPF record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupNamingAuthorityPointer(name, timeout=None): + """ + Perform a NAPTR record lookup. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: Sequence of C{int} + @param timeout: Number of seconds after which to reissue the query. + When the last timeout expires, the query is considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. The first element of the + tuple gives answers. The second element of the tuple gives + authorities. The third element of the tuple gives additional + information. The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + def lookupZone(name, timeout=None): + """ + Perform an AXFR record lookup. + + NB This is quite different from other DNS requests. See + U{http://cr.yp.to/djbdns/axfr-notes.html} for more + information. + + NB Unlike other C{lookup*} methods, the timeout here is not a + list of ints, it is a single int. + + @type name: C{str} + @param name: DNS name to resolve. + + @type timeout: C{int} + @param timeout: When this timeout expires, the query is + considered failed. + + @rtype: L{Deferred} + @return: A L{Deferred} which fires with a three-tuple of lists of + L{twisted.names.dns.RRHeader} instances. + The first element of the tuple gives answers. + The second and third elements are always empty. + The L{Deferred} may instead fail with one of the + exceptions defined in L{twisted.names.error} or with + C{NotImplementedError}. + """ + + + +class IReactorTCP(Interface): + + def listenTCP(port, factory, backlog=50, interface=''): + """ + Connects a given protocol factory to the given numeric TCP/IP port. + + @param port: a port number on which to listen + + @param factory: a L{twisted.internet.protocol.ServerFactory} instance + + @param backlog: size of the listen queue + + @param interface: The local IPv4 or IPv6 address to which to bind; + defaults to '', ie all IPv4 addresses. To bind to all IPv4 and IPv6 + addresses, you must call this method twice. + + @return: an object that provides L{IListeningPort}. + + @raise CannotListenError: as defined here + L{twisted.internet.error.CannotListenError}, + if it cannot listen on this port (e.g., it + cannot bind to the required port number) + """ + + def connectTCP(host, port, factory, timeout=30, bindAddress=None): + """ + Connect a TCP client. + + @param host: a host name + + @param port: a port number + + @param factory: a L{twisted.internet.protocol.ClientFactory} instance + + @param timeout: number of seconds to wait before assuming the + connection has failed. + + @param bindAddress: a (host, port) tuple of local address to bind + to, or None. + + @return: An object which provides L{IConnector}. This connector will + call various callbacks on the factory when a connection is + made, failed, or lost - see + L{ClientFactory} + docs for details. + """ + +class IReactorSSL(Interface): + + def connectSSL(host, port, factory, contextFactory, timeout=30, bindAddress=None): + """ + Connect a client Protocol to a remote SSL socket. + + @param host: a host name + + @param port: a port number + + @param factory: a L{twisted.internet.protocol.ClientFactory} instance + + @param contextFactory: a L{twisted.internet.ssl.ClientContextFactory} object. + + @param timeout: number of seconds to wait before assuming the + connection has failed. + + @param bindAddress: a (host, port) tuple of local address to bind to, + or C{None}. + + @return: An object which provides L{IConnector}. + """ + + def listenSSL(port, factory, contextFactory, backlog=50, interface=''): + """ + Connects a given protocol factory to the given numeric TCP/IP port. + The connection is a SSL one, using contexts created by the context + factory. + + @param port: a port number on which to listen + + @param factory: a L{twisted.internet.protocol.ServerFactory} instance + + @param contextFactory: a L{twisted.internet.ssl.ContextFactory} instance + + @param backlog: size of the listen queue + + @param interface: the hostname to bind to, defaults to '' (all) + """ + + + +class IReactorUNIX(Interface): + """ + UNIX socket methods. + """ + + def connectUNIX(address, factory, timeout=30, checkPID=0): + """ + Connect a client protocol to a UNIX socket. + + @param address: a path to a unix socket on the filesystem. + + @param factory: a L{twisted.internet.protocol.ClientFactory} instance + + @param timeout: number of seconds to wait before assuming the connection + has failed. + + @param checkPID: if True, check for a pid file to verify that a server + is listening. If C{address} is a Linux abstract namespace path, + this must be C{False}. + + @return: An object which provides L{IConnector}. + """ + + + def listenUNIX(address, factory, backlog=50, mode=0o666, wantPID=0): + """ + Listen on a UNIX socket. + + @param address: a path to a unix socket on the filesystem. + + @param factory: a L{twisted.internet.protocol.Factory} instance. + + @param backlog: number of connections to allow in backlog. + + @param mode: The mode (B{not} umask) to set on the unix socket. See + platform specific documentation for information about how this + might affect connection attempts. + @type mode: C{int} + + @param wantPID: if True, create a pidfile for the socket. If C{address} + is a Linux abstract namespace path, this must be C{False}. + + @return: An object which provides L{IListeningPort}. + """ + + + +class IReactorUNIXDatagram(Interface): + """ + Datagram UNIX socket methods. + """ + + def connectUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666, bindAddress=None): + """ + Connect a client protocol to a datagram UNIX socket. + + @param address: a path to a unix socket on the filesystem. + + @param protocol: a L{twisted.internet.protocol.ConnectedDatagramProtocol} instance + + @param maxPacketSize: maximum packet size to accept + + @param mode: The mode (B{not} umask) to set on the unix socket. See + platform specific documentation for information about how this + might affect connection attempts. + @type mode: C{int} + + @param bindAddress: address to bind to + + @return: An object which provides L{IConnector}. + """ + + + def listenUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666): + """ + Listen on a datagram UNIX socket. + + @param address: a path to a unix socket on the filesystem. + + @param protocol: a L{twisted.internet.protocol.DatagramProtocol} instance. + + @param maxPacketSize: maximum packet size to accept + + @param mode: The mode (B{not} umask) to set on the unix socket. See + platform specific documentation for information about how this + might affect connection attempts. + @type mode: C{int} + + @return: An object which provides L{IListeningPort}. + """ + + + +class IReactorWin32Events(Interface): + """ + Win32 Event API methods + + @since: 10.2 + """ + + def addEvent(event, fd, action): + """ + Add a new win32 event to the event loop. + + @param event: a Win32 event object created using win32event.CreateEvent() + + @param fd: an instance of L{twisted.internet.abstract.FileDescriptor} + + @param action: a string that is a method name of the fd instance. + This method is called in response to the event. + + @return: None + """ + + + def removeEvent(event): + """ + Remove an event. + + @param event: a Win32 event object added using L{IReactorWin32Events.addEvent} + + @return: None + """ + + + +class IReactorUDP(Interface): + """ + UDP socket methods. + """ + + def listenUDP(port, protocol, interface='', maxPacketSize=8192): + """ + Connects a given DatagramProtocol to the given numeric UDP port. + + @return: object which provides L{IListeningPort}. + """ + + + +class IReactorMulticast(Interface): + """ + UDP socket methods that support multicast. + + IMPORTANT: This is an experimental new interface. It may change + without backwards compatability. Suggestions are welcome. + """ + + def listenMulticast(port, protocol, interface='', maxPacketSize=8192, + listenMultiple=False): + """ + Connects a given + L{DatagramProtocol} to the + given numeric UDP port. + + @param listenMultiple: If set to True, allows multiple sockets to + bind to the same address and port number at the same time. + @type listenMultiple: C{bool} + + @returns: An object which provides L{IListeningPort}. + + @see: L{twisted.internet.interfaces.IMulticastTransport} + @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html} + """ + + + +class IReactorSocket(Interface): + """ + Methods which allow a reactor to use externally created sockets. + + For example, to use C{adoptStreamPort} to implement behavior equivalent + to that of L{IReactorTCP.listenTCP}, you might write code like this:: + + from socket import SOMAXCONN, AF_INET, SOCK_STREAM, socket + portSocket = socket(AF_INET, SOCK_STREAM) + # Set FD_CLOEXEC on port, left as an exercise. Then make it into a + # non-blocking listening port: + portSocket.setblocking(False) + portSocket.bind(('192.168.1.2', 12345)) + portSocket.listen(SOMAXCONN) + + # Now have the reactor use it as a TCP port + port = reactor.adoptStreamPort( + portSocket.fileno(), AF_INET, YourFactory()) + + # portSocket itself is no longer necessary, and needs to be cleaned + # up by us. + portSocket.close() + + # Whenever the server is no longer needed, stop it as usual. + stoppedDeferred = port.stopListening() + + Another potential use is to inherit a listening descriptor from a parent + process (for example, systemd or launchd), or to receive one over a UNIX + domain socket. + + Some plans for extending this interface exist. See: + + - U{http://twistedmatrix.com/trac/ticket/5570}: established connections + - U{http://twistedmatrix.com/trac/ticket/5573}: AF_UNIX ports + - U{http://twistedmatrix.com/trac/ticket/5574}: SOCK_DGRAM sockets + """ + + def adoptStreamPort(fileDescriptor, addressFamily, factory): + """ + Add an existing listening I{SOCK_STREAM} socket to the reactor to + monitor for new connections to accept and handle. + + @param fileDescriptor: A file descriptor associated with a socket which + is already bound to an address and marked as listening. The socket + must be set non-blocking. Any additional flags (for example, + close-on-exec) must also be set by application code. Application + code is responsible for closing the file descriptor, which may be + done as soon as C{adoptStreamPort} returns. + @type fileDescriptor: C{int} + + @param addressFamily: The address family (or I{domain}) of the socket. + For example, L{socket.AF_INET6}. + + @param factory: A L{ServerFactory} instance to use to create new + protocols to handle connections accepted via this socket. + + @return: An object providing L{IListeningPort}. + + @raise UnsupportedAddressFamily: If the given address family is not + supported by this reactor, or not supported with the given socket + type. + + @raise UnsupportedSocketType: If the given socket type is not supported + by this reactor, or not supported with the given socket type. + """ + + + def adoptStreamConnection(fileDescriptor, addressFamily, factory): + """ + Add an existing connected I{SOCK_STREAM} socket to the reactor to + monitor for data. + + Note that the given factory won't have its C{startFactory} and + C{stopFactory} methods called, as there is no sensible time to call + them in this situation. + + @param fileDescriptor: A file descriptor associated with a socket which + is already connected. The socket must be set non-blocking. Any + additional flags (for example, close-on-exec) must also be set by + application code. Application code is responsible for closing the + file descriptor, which may be done as soon as + C{adoptStreamConnection} returns. + @type fileDescriptor: C{int} + + @param addressFamily: The address family (or I{domain}) of the socket. + For example, L{socket.AF_INET6}. + + @param factory: A L{ServerFactory} instance to use to create a new + protocol to handle the connection via this socket. + + @raise UnsupportedAddressFamily: If the given address family is not + supported by this reactor, or not supported with the given socket + type. + + @raise UnsupportedSocketType: If the given socket type is not supported + by this reactor, or not supported with the given socket type. + """ + + + +class IReactorProcess(Interface): + + def spawnProcess(processProtocol, executable, args=(), env={}, path=None, + uid=None, gid=None, usePTY=0, childFDs=None): + """ + Spawn a process, with a process protocol. + + @type processProtocol: L{IProcessProtocol} provider + @param processProtocol: An object which will be notified of all + events related to the created process. + + @param executable: the file name to spawn - the full path should be + used. + + @param args: the command line arguments to pass to the process; a + sequence of strings. The first string should be the + executable's name. + + @type env: a C{dict} mapping C{str} to C{str}, or C{None}. + @param env: the environment variables to pass to the child process. The + resulting behavior varies between platforms. If + - C{env} is not set: + - On POSIX: pass an empty environment. + - On Windows: pass C{os.environ}. + - C{env} is C{None}: + - On POSIX: pass C{os.environ}. + - On Windows: pass C{os.environ}. + - C{env} is a C{dict}: + - On POSIX: pass the key/value pairs in C{env} as the + complete environment. + - On Windows: update C{os.environ} with the key/value + pairs in the C{dict} before passing it. As a + consequence of U{bug #1640 + }, passing + keys with empty values in an effort to unset + environment variables I{won't} unset them. + + @param path: the path to run the subprocess in - defaults to the + current directory. + + @param uid: user ID to run the subprocess as. (Only available on + POSIX systems.) + + @param gid: group ID to run the subprocess as. (Only available on + POSIX systems.) + + @param usePTY: if true, run this process in a pseudo-terminal. + optionally a tuple of C{(masterfd, slavefd, ttyname)}, + in which case use those file descriptors. + (Not available on all systems.) + + @param childFDs: A dictionary mapping file descriptors in the new child + process to an integer or to the string 'r' or 'w'. + + If the value is an integer, it specifies a file + descriptor in the parent process which will be mapped + to a file descriptor (specified by the key) in the + child process. This is useful for things like inetd + and shell-like file redirection. + + If it is the string 'r', a pipe will be created and + attached to the child at that file descriptor: the + child will be able to write to that file descriptor + and the parent will receive read notification via the + L{IProcessProtocol.childDataReceived} callback. This + is useful for the child's stdout and stderr. + + If it is the string 'w', similar setup to the previous + case will occur, with the pipe being readable by the + child instead of writeable. The parent process can + write to that file descriptor using + L{IProcessTransport.writeToChild}. This is useful for + the child's stdin. + + If childFDs is not passed, the default behaviour is to + use a mapping that opens the usual stdin/stdout/stderr + pipes. + + @see: L{twisted.internet.protocol.ProcessProtocol} + + @return: An object which provides L{IProcessTransport}. + + @raise OSError: Raised with errno C{EAGAIN} or C{ENOMEM} if there are + insufficient system resources to create a new process. + """ + +class IReactorTime(Interface): + """ + Time methods that a Reactor should implement. + """ + + def seconds(): + """ + Get the current time in seconds. + + @return: A number-like object of some sort. + """ + + + def callLater(delay, callable, *args, **kw): + """ + Call a function later. + + @type delay: C{float} + @param delay: the number of seconds to wait. + + @param callable: the callable object to call later. + + @param args: the arguments to call it with. + + @param kw: the keyword arguments to call it with. + + @return: An object which provides L{IDelayedCall} and can be used to + cancel the scheduled call, by calling its C{cancel()} method. + It also may be rescheduled by calling its C{delay()} or + C{reset()} methods. + """ + + + def getDelayedCalls(): + """ + Retrieve all currently scheduled delayed calls. + + @return: A tuple of all L{IDelayedCall} providers representing all + currently scheduled calls. This is everything that has been + returned by C{callLater} but not yet called or canceled. + """ + + +class IDelayedCall(Interface): + """ + A scheduled call. + + There are probably other useful methods we can add to this interface; + suggestions are welcome. + """ + + def getTime(): + """ + Get time when delayed call will happen. + + @return: time in seconds since epoch (a float). + """ + + def cancel(): + """ + Cancel the scheduled call. + + @raises twisted.internet.error.AlreadyCalled: if the call has already + happened. + @raises twisted.internet.error.AlreadyCancelled: if the call has already + been cancelled. + """ + + def delay(secondsLater): + """ + Delay the scheduled call. + + @param secondsLater: how many seconds from its current firing time to delay + + @raises twisted.internet.error.AlreadyCalled: if the call has already + happened. + @raises twisted.internet.error.AlreadyCancelled: if the call has already + been cancelled. + """ + + def reset(secondsFromNow): + """ + Reset the scheduled call's timer. + + @param secondsFromNow: how many seconds from now it should fire, + equivalent to C{.cancel()} and then doing another + C{reactor.callLater(secondsLater, ...)} + + @raises twisted.internet.error.AlreadyCalled: if the call has already + happened. + @raises twisted.internet.error.AlreadyCancelled: if the call has already + been cancelled. + """ + + def active(): + """ + @return: True if this call is still active, False if it has been + called or cancelled. + """ + +class IReactorThreads(Interface): + """ + Dispatch methods to be run in threads. + + Internally, this should use a thread pool and dispatch methods to them. + """ + + def getThreadPool(): + """ + Return the threadpool used by L{callInThread}. Create it first if + necessary. + + @rtype: L{twisted.python.threadpool.ThreadPool} + """ + + + def callInThread(callable, *args, **kwargs): + """ + Run the callable object in a separate thread. + """ + + + def callFromThread(callable, *args, **kw): + """ + Cause a function to be executed by the reactor thread. + + Use this method when you want to run a function in the reactor's thread + from another thread. Calling L{callFromThread} should wake up the main + thread (where L{reactor.run()} is executing) and run the + given callable in that thread. + + If you're writing a multi-threaded application the C{callable} may need + to be thread safe, but this method doesn't require it as such. If you + want to call a function in the next mainloop iteration, but you're in + the same thread, use L{callLater} with a delay of 0. + """ + + + def suggestThreadPoolSize(size): + """ + Suggest the size of the internal threadpool used to dispatch functions + passed to L{callInThread}. + """ + + +class IReactorCore(Interface): + """ + Core methods that a Reactor must implement. + """ + + running = Attribute( + "A C{bool} which is C{True} from I{during startup} to " + "I{during shutdown} and C{False} the rest of the time.") + + + def resolve(name, timeout=10): + """ + Return a L{twisted.internet.defer.Deferred} that will resolve a hostname. + """ + + def run(): + """ + Fire 'startup' System Events, move the reactor to the 'running' + state, then run the main loop until it is stopped with C{stop()} or + C{crash()}. + """ + + def stop(): + """ + Fire 'shutdown' System Events, which will move the reactor to the + 'stopped' state and cause C{reactor.run()} to exit. + """ + + def crash(): + """ + Stop the main loop *immediately*, without firing any system events. + + This is named as it is because this is an extremely "rude" thing to do; + it is possible to lose data and put your system in an inconsistent + state by calling this. However, it is necessary, as sometimes a system + can become wedged in a pre-shutdown call. + """ + + def iterate(delay=0): + """ + Run the main loop's I/O polling function for a period of time. + + This is most useful in applications where the UI is being drawn "as + fast as possible", such as games. All pending L{IDelayedCall}s will + be called. + + The reactor must have been started (via the C{run()} method) prior to + any invocations of this method. It must also be stopped manually + after the last call to this method (via the C{stop()} method). This + method is not re-entrant: you must not call it recursively; in + particular, you must not call it while the reactor is running. + """ + + def fireSystemEvent(eventType): + """ + Fire a system-wide event. + + System-wide events are things like 'startup', 'shutdown', and + 'persist'. + """ + + def addSystemEventTrigger(phase, eventType, callable, *args, **kw): + """ + Add a function to be called when a system event occurs. + + Each "system event" in Twisted, such as 'startup', 'shutdown', and + 'persist', has 3 phases: 'before', 'during', and 'after' (in that + order, of course). These events will be fired internally by the + Reactor. + + An implementor of this interface must only implement those events + described here. + + Callbacks registered for the "before" phase may return either None or a + Deferred. The "during" phase will not execute until all of the + Deferreds from the "before" phase have fired. + + Once the "during" phase is running, all of the remaining triggers must + execute; their return values must be ignored. + + @param phase: a time to call the event -- either the string 'before', + 'after', or 'during', describing when to call it + relative to the event's execution. + + @param eventType: this is a string describing the type of event. + + @param callable: the object to call before shutdown. + + @param args: the arguments to call it with. + + @param kw: the keyword arguments to call it with. + + @return: an ID that can be used to remove this call with + removeSystemEventTrigger. + """ + + def removeSystemEventTrigger(triggerID): + """ + Removes a trigger added with addSystemEventTrigger. + + @param triggerID: a value returned from addSystemEventTrigger. + + @raise KeyError: If there is no system event trigger for the given + C{triggerID}. + + @raise ValueError: If there is no system event trigger for the given + C{triggerID}. + + @raise TypeError: If there is no system event trigger for the given + C{triggerID}. + """ + + def callWhenRunning(callable, *args, **kw): + """ + Call a function when the reactor is running. + + If the reactor has not started, the callable will be scheduled + to run when it does start. Otherwise, the callable will be invoked + immediately. + + @param callable: the callable object to call later. + + @param args: the arguments to call it with. + + @param kw: the keyword arguments to call it with. + + @return: None if the callable was invoked, otherwise a system + event id for the scheduled call. + """ + + +class IReactorPluggableResolver(Interface): + """ + A reactor with a pluggable name resolver interface. + """ + + def installResolver(resolver): + """ + Set the internal resolver to use to for name lookups. + + @type resolver: An object implementing the L{IResolverSimple} interface + @param resolver: The new resolver to use. + + @return: The previously installed resolver. + """ + + +class IReactorDaemonize(Interface): + """ + A reactor which provides hooks that need to be called before and after + daemonization. + + Notes: + - This interface SHOULD NOT be called by applications. + - This interface should only be implemented by reactors as a workaround + (in particular, it's implemented currently only by kqueue()). + For details please see the comments on ticket #1918. + """ + + def beforeDaemonize(): + """ + Hook to be called immediately before daemonization. No reactor methods + may be called until L{afterDaemonize} is called. + + @return: C{None}. + """ + + + def afterDaemonize(): + """ + Hook to be called immediately after daemonization. This may only be + called after L{beforeDaemonize} had been called previously. + + @return: C{None}. + """ + + + +class IReactorFDSet(Interface): + """ + Implement me to be able to use L{IFileDescriptor} type resources. + + This assumes that your main-loop uses UNIX-style numeric file descriptors + (or at least similarly opaque IDs returned from a .fileno() method) + """ + + def addReader(reader): + """ + I add reader to the set of file descriptors to get read events for. + + @param reader: An L{IReadDescriptor} provider that will be checked for + read events until it is removed from the reactor with + L{removeReader}. + + @return: C{None}. + """ + + def addWriter(writer): + """ + I add writer to the set of file descriptors to get write events for. + + @param writer: An L{IWriteDescriptor} provider that will be checked for + write events until it is removed from the reactor with + L{removeWriter}. + + @return: C{None}. + """ + + def removeReader(reader): + """ + Removes an object previously added with L{addReader}. + + @return: C{None}. + """ + + def removeWriter(writer): + """ + Removes an object previously added with L{addWriter}. + + @return: C{None}. + """ + + def removeAll(): + """ + Remove all readers and writers. + + Should not remove reactor internal reactor connections (like a waker). + + @return: A list of L{IReadDescriptor} and L{IWriteDescriptor} providers + which were removed. + """ + + def getReaders(): + """ + Return the list of file descriptors currently monitored for input + events by the reactor. + + @return: the list of file descriptors monitored for input events. + @rtype: C{list} of C{IReadDescriptor} + """ + + def getWriters(): + """ + Return the list file descriptors currently monitored for output events + by the reactor. + + @return: the list of file descriptors monitored for output events. + @rtype: C{list} of C{IWriteDescriptor} + """ + + +class IListeningPort(Interface): + """ + A listening port. + """ + + def startListening(): + """ + Start listening on this port. + + @raise CannotListenError: If it cannot listen on this port (e.g., it is + a TCP port and it cannot bind to the required + port number). + """ + + def stopListening(): + """ + Stop listening on this port. + + If it does not complete immediately, will return Deferred that fires + upon completion. + """ + + def getHost(): + """ + Get the host that this port is listening for. + + @return: An L{IAddress} provider. + """ + + +class ILoggingContext(Interface): + """ + Give context information that will be used to log events generated by + this item. + """ + + def logPrefix(): + """ + @return: Prefix used during log formatting to indicate context. + @rtype: C{str} + """ + + + +class IFileDescriptor(ILoggingContext): + """ + An interface representing a UNIX-style numeric file descriptor. + """ + + def fileno(): + """ + @raise: If the descriptor no longer has a valid file descriptor + number associated with it. + + @return: The platform-specified representation of a file descriptor + number. Or C{-1} if the descriptor no longer has a valid file + descriptor number associated with it. As long as the descriptor + is valid, calls to this method on a particular instance must + return the same value. + """ + + + def connectionLost(reason): + """ + Called when the connection was lost. + + This is called when the connection on a selectable object has been + lost. It will be called whether the connection was closed explicitly, + an exception occurred in an event handler, or the other end of the + connection closed it first. + + See also L{IHalfCloseableDescriptor} if your descriptor wants to be + notified separately of the two halves of the connection being closed. + + @param reason: A failure instance indicating the reason why the + connection was lost. L{error.ConnectionLost} and + L{error.ConnectionDone} are of special note, but the + failure may be of other classes as well. + """ + + + +class IReadDescriptor(IFileDescriptor): + """ + An L{IFileDescriptor} that can read. + + This interface is generally used in conjunction with L{IReactorFDSet}. + """ + + def doRead(): + """ + Some data is available for reading on your descriptor. + + @return: If an error is encountered which causes the descriptor to + no longer be valid, a L{Failure} should be returned. Otherwise, + C{None}. + """ + + +class IWriteDescriptor(IFileDescriptor): + """ + An L{IFileDescriptor} that can write. + + This interface is generally used in conjunction with L{IReactorFDSet}. + """ + + def doWrite(): + """ + Some data can be written to your descriptor. + + @return: If an error is encountered which causes the descriptor to + no longer be valid, a L{Failure} should be returned. Otherwise, + C{None}. + """ + + +class IReadWriteDescriptor(IReadDescriptor, IWriteDescriptor): + """ + An L{IFileDescriptor} that can both read and write. + """ + + +class IHalfCloseableDescriptor(Interface): + """ + A descriptor that can be half-closed. + """ + + def writeConnectionLost(reason): + """ + Indicates write connection was lost. + """ + + def readConnectionLost(reason): + """ + Indicates read connection was lost. + """ + + +class ISystemHandle(Interface): + """ + An object that wraps a networking OS-specific handle. + """ + + def getHandle(): + """ + Return a system- and reactor-specific handle. + + This might be a socket.socket() object, or some other type of + object, depending on which reactor is being used. Use and + manipulate at your own risk. + + This might be used in cases where you want to set specific + options not exposed by the Twisted APIs. + """ + + +class IConsumer(Interface): + """ + A consumer consumes data from a producer. + """ + + def registerProducer(producer, streaming): + """ + Register to receive data from a producer. + + This sets self to be a consumer for a producer. When this object runs + out of data (as when a send(2) call on a socket succeeds in moving the + last data from a userspace buffer into a kernelspace buffer), it will + ask the producer to resumeProducing(). + + For L{IPullProducer} providers, C{resumeProducing} will be called once + each time data is required. + + For L{IPushProducer} providers, C{pauseProducing} will be called + whenever the write buffer fills up and C{resumeProducing} will only be + called when it empties. + + @type producer: L{IProducer} provider + + @type streaming: C{bool} + @param streaming: C{True} if C{producer} provides L{IPushProducer}, + C{False} if C{producer} provides L{IPullProducer}. + + @raise RuntimeError: If a producer is already registered. + + @return: C{None} + """ + + + def unregisterProducer(): + """ + Stop consuming data from a producer, without disconnecting. + """ + + + def write(data): + """ + The producer will write data by calling this method. + + The implementation must be non-blocking and perform whatever + buffering is necessary. If the producer has provided enough data + for now and it is a L{IPushProducer}, the consumer may call its + C{pauseProducing} method. + """ + + + +class IProducer(Interface): + """ + A producer produces data for a consumer. + + Typically producing is done by calling the write method of an class + implementing L{IConsumer}. + """ + + def stopProducing(): + """ + Stop producing data. + + This tells a producer that its consumer has died, so it must stop + producing data for good. + """ + + +class IPushProducer(IProducer): + """ + A push producer, also known as a streaming producer is expected to + produce (write to this consumer) data on a continuous basis, unless + it has been paused. A paused push producer will resume producing + after its resumeProducing() method is called. For a push producer + which is not pauseable, these functions may be noops. + """ + + def pauseProducing(): + """ + Pause producing data. + + Tells a producer that it has produced too much data to process for + the time being, and to stop until resumeProducing() is called. + """ + def resumeProducing(): + """ + Resume producing data. + + This tells a producer to re-add itself to the main loop and produce + more data for its consumer. + """ + +class IPullProducer(IProducer): + """ + A pull producer, also known as a non-streaming producer, is + expected to produce data each time resumeProducing() is called. + """ + + def resumeProducing(): + """ + Produce data for the consumer a single time. + + This tells a producer to produce data for the consumer once + (not repeatedly, once only). Typically this will be done + by calling the consumer's write() method a single time with + produced data. + """ + +class IProtocol(Interface): + + def dataReceived(data): + """ + Called whenever data is received. + + Use this method to translate to a higher-level message. Usually, some + callback will be made upon the receipt of each complete protocol + message. + + @param data: a string of indeterminate length. Please keep in mind + that you will probably need to buffer some data, as partial + (or multiple) protocol messages may be received! I recommend + that unit tests for protocols call through to this method with + differing chunk sizes, down to one byte at a time. + """ + + def connectionLost(reason): + """ + Called when the connection is shut down. + + Clear any circular references here, and any external references + to this Protocol. The connection has been closed. The C{reason} + Failure wraps a L{twisted.internet.error.ConnectionDone} or + L{twisted.internet.error.ConnectionLost} instance (or a subclass + of one of those). + + @type reason: L{twisted.python.failure.Failure} + """ + + def makeConnection(transport): + """ + Make a connection to a transport and a server. + """ + + def connectionMade(): + """ + Called when a connection is made. + + This may be considered the initializer of the protocol, because + it is called when the connection is completed. For clients, + this is called once the connection to the server has been + established; for servers, this is called after an accept() call + stops blocking and a socket has been received. If you need to + send any greeting or initial message, do it here. + """ + + +class IProcessProtocol(Interface): + """ + Interface for process-related event handlers. + """ + + def makeConnection(process): + """ + Called when the process has been created. + + @type process: L{IProcessTransport} provider + @param process: An object representing the process which has been + created and associated with this protocol. + """ + + + def childDataReceived(childFD, data): + """ + Called when data arrives from the child process. + + @type childFD: C{int} + @param childFD: The file descriptor from which the data was + received. + + @type data: C{str} + @param data: The data read from the child's file descriptor. + """ + + + def childConnectionLost(childFD): + """ + Called when a file descriptor associated with the child process is + closed. + + @type childFD: C{int} + @param childFD: The file descriptor which was closed. + """ + + + def processExited(reason): + """ + Called when the child process exits. + + @type reason: L{twisted.python.failure.Failure} + @param reason: A failure giving the reason the child process + terminated. The type of exception for this failure is either + L{twisted.internet.error.ProcessDone} or + L{twisted.internet.error.ProcessTerminated}. + + @since: 8.2 + """ + + + def processEnded(reason): + """ + Called when the child process exits and all file descriptors associated + with it have been closed. + + @type reason: L{twisted.python.failure.Failure} + @param reason: A failure giving the reason the child process + terminated. The type of exception for this failure is either + L{twisted.internet.error.ProcessDone} or + L{twisted.internet.error.ProcessTerminated}. + """ + + + +class IHalfCloseableProtocol(Interface): + """ + Implemented to indicate they want notification of half-closes. + + TCP supports the notion of half-closing the connection, e.g. + closing the write side but still not stopping reading. A protocol + that implements this interface will be notified of such events, + instead of having connectionLost called. + """ + + def readConnectionLost(): + """ + Notification of the read connection being closed. + + This indicates peer did half-close of write side. It is now + the responsibility of the this protocol to call + loseConnection(). In addition, the protocol MUST make sure a + reference to it still exists (i.e. by doing a callLater with + one of its methods, etc.) as the reactor will only have a + reference to it if it is writing. + + If the protocol does not do so, it might get garbage collected + without the connectionLost method ever being called. + """ + + def writeConnectionLost(): + """ + Notification of the write connection being closed. + + This will never be called for TCP connections as TCP does not + support notification of this type of half-close. + """ + + + +class IFileDescriptorReceiver(Interface): + """ + Protocols may implement L{IFileDescriptorReceiver} to receive file + descriptors sent to them. This is useful in conjunction with + L{IUNIXTransport}, which allows file descriptors to be sent between + processes on a single host. + """ + def fileDescriptorReceived(descriptor): + """ + Called when a file descriptor is received over the connection. + + @param descriptor: The descriptor which was received. + @type descriptor: C{int} + + @return: C{None} + """ + + + +class IProtocolFactory(Interface): + """ + Interface for protocol factories. + """ + + def buildProtocol(addr): + """ + Called when a connection has been established to addr. + + If None is returned, the connection is assumed to have been refused, + and the Port will close the connection. + + @type addr: (host, port) + @param addr: The address of the newly-established connection + + @return: None if the connection was refused, otherwise an object + providing L{IProtocol}. + """ + + def doStart(): + """ + Called every time this is connected to a Port or Connector. + """ + + def doStop(): + """ + Called every time this is unconnected from a Port or Connector. + """ + + +class ITransport(Interface): + """ + I am a transport for bytes. + + I represent (and wrap) the physical connection and synchronicity + of the framework which is talking to the network. I make no + representations about whether calls to me will happen immediately + or require returning to a control loop, or whether they will happen + in the same or another thread. Consider methods of this class + (aside from getPeer) to be 'thrown over the wall', to happen at some + indeterminate time. + """ + + def write(data): + """ + Write some data to the physical connection, in sequence, in a + non-blocking fashion. + + If possible, make sure that it is all written. No data will + ever be lost, although (obviously) the connection may be closed + before it all gets through. + """ + + def writeSequence(data): + """ + Write a list of strings to the physical connection. + + If possible, make sure that all of the data is written to + the socket at once, without first copying it all into a + single string. + """ + + def loseConnection(): + """ + Close my connection, after writing all pending data. + + Note that if there is a registered producer on a transport it + will not be closed until the producer has been unregistered. + """ + + def getPeer(): + """ + Get the remote address of this connection. + + Treat this method with caution. It is the unfortunate result of the + CGI and Jabber standards, but should not be considered reliable for + the usual host of reasons; port forwarding, proxying, firewalls, IP + masquerading, etc. + + @return: An L{IAddress} provider. + """ + + def getHost(): + """ + Similar to getPeer, but returns an address describing this side of the + connection. + + @return: An L{IAddress} provider. + """ + + +class ITCPTransport(ITransport): + """ + A TCP based transport. + """ + + def loseWriteConnection(): + """ + Half-close the write side of a TCP connection. + + If the protocol instance this is attached to provides + IHalfCloseableProtocol, it will get notified when the operation is + done. When closing write connection, as with loseConnection this will + only happen when buffer has emptied and there is no registered + producer. + """ + + + def abortConnection(): + """ + Close the connection abruptly. + + Discards any buffered data, stops any registered producer, + and, if possible, notifies the other end of the unclean + closure. + + @since: 11.1 + """ + + + def getTcpNoDelay(): + """ + Return if C{TCP_NODELAY} is enabled. + """ + + def setTcpNoDelay(enabled): + """ + Enable/disable C{TCP_NODELAY}. + + Enabling C{TCP_NODELAY} turns off Nagle's algorithm. Small packets are + sent sooner, possibly at the expense of overall throughput. + """ + + def getTcpKeepAlive(): + """ + Return if C{SO_KEEPALIVE} is enabled. + """ + + def setTcpKeepAlive(enabled): + """ + Enable/disable C{SO_KEEPALIVE}. + + Enabling C{SO_KEEPALIVE} sends packets periodically when the connection + is otherwise idle, usually once every two hours. They are intended + to allow detection of lost peers in a non-infinite amount of time. + """ + + def getHost(): + """ + Returns L{IPv4Address} or L{IPv6Address}. + """ + + def getPeer(): + """ + Returns L{IPv4Address} or L{IPv6Address}. + """ + + + +class IUNIXTransport(ITransport): + """ + Transport for stream-oriented unix domain connections. + """ + def sendFileDescriptor(descriptor): + """ + Send a duplicate of this (file, socket, pipe, etc) descriptor to the + other end of this connection. + + The send is non-blocking and will be queued if it cannot be performed + immediately. The send will be processed in order with respect to other + C{sendFileDescriptor} calls on this transport, but not necessarily with + respect to C{write} calls on this transport. The send can only be + processed if there are also bytes in the normal connection-oriented send + buffer (ie, you must call C{write} at least as many times as you call + C{sendFileDescriptor}). + + @param descriptor: An C{int} giving a valid file descriptor in this + process. Note that a I{file descriptor} may actually refer to a + socket, a pipe, or anything else POSIX tries to treat in the same + way as a file. + + @return: C{None} + """ + + + +class ITLSTransport(ITCPTransport): + """ + A TCP transport that supports switching to TLS midstream. + + Once TLS mode is started the transport will implement L{ISSLTransport}. + """ + + def startTLS(contextFactory): + """ + Initiate TLS negotiation. + + @param contextFactory: A context factory (see L{ssl.py}) + """ + +class ISSLTransport(ITCPTransport): + """ + A SSL/TLS based transport. + """ + + def getPeerCertificate(): + """ + Return an object with the peer's certificate info. + """ + + +class IProcessTransport(ITransport): + """ + A process transport. + """ + + pid = Attribute( + "From before L{IProcessProtocol.makeConnection} is called to before " + "L{IProcessProtocol.processEnded} is called, C{pid} is an L{int} " + "giving the platform process ID of this process. C{pid} is L{None} " + "at all other times.") + + def closeStdin(): + """ + Close stdin after all data has been written out. + """ + + def closeStdout(): + """ + Close stdout. + """ + + def closeStderr(): + """ + Close stderr. + """ + + def closeChildFD(descriptor): + """ + Close a file descriptor which is connected to the child process, identified + by its FD in the child process. + """ + + def writeToChild(childFD, data): + """ + Similar to L{ITransport.write} but also allows the file descriptor in + the child process which will receive the bytes to be specified. + + @type childFD: C{int} + @param childFD: The file descriptor to which to write. + + @type data: C{str} + @param data: The bytes to write. + + @return: C{None} + + @raise KeyError: If C{childFD} is not a file descriptor that was mapped + in the child when L{IReactorProcess.spawnProcess} was used to create + it. + """ + + def loseConnection(): + """ + Close stdin, stderr and stdout. + """ + + def signalProcess(signalID): + """ + Send a signal to the process. + + @param signalID: can be + - one of C{"KILL"}, C{"TERM"}, or C{"INT"}. + These will be implemented in a + cross-platform manner, and so should be used + if possible. + - an integer, where it represents a POSIX + signal ID. + + @raise twisted.internet.error.ProcessExitedAlready: If the process has + already exited. + @raise OSError: If the C{os.kill} call fails with an errno different + from C{ESRCH}. + """ + + +class IServiceCollection(Interface): + """ + An object which provides access to a collection of services. + """ + + def getServiceNamed(serviceName): + """ + Retrieve the named service from this application. + + Raise a C{KeyError} if there is no such service name. + """ + + def addService(service): + """ + Add a service to this collection. + """ + + def removeService(service): + """ + Remove a service from this collection. + """ + + +class IUDPTransport(Interface): + """ + Transport for UDP DatagramProtocols. + """ + + def write(packet, addr=None): + """ + Write packet to given address. + + @param addr: a tuple of (ip, port). For connected transports must + be the address the transport is connected to, or None. + In non-connected mode this is mandatory. + + @raise twisted.internet.error.MessageLengthError: C{packet} was too + long. + """ + + def connect(host, port): + """ + Connect the transport to an address. + + This changes it to connected mode. Datagrams can only be sent to + this address, and will only be received from this address. In addition + the protocol's connectionRefused method might get called if destination + is not receiving datagrams. + + @param host: an IP address, not a domain name ('127.0.0.1', not 'localhost') + @param port: port to connect to. + """ + + def getHost(): + """ + Returns L{IPv4Address}. + """ + + def stopListening(): + """ + Stop listening on this port. + + If it does not complete immediately, will return L{Deferred} that fires + upon completion. + """ + + + +class IUNIXDatagramTransport(Interface): + """ + Transport for UDP PacketProtocols. + """ + + def write(packet, address): + """ + Write packet to given address. + """ + + def getHost(): + """ + Returns L{UNIXAddress}. + """ + + +class IUNIXDatagramConnectedTransport(Interface): + """ + Transport for UDP ConnectedPacketProtocols. + """ + + def write(packet): + """ + Write packet to address we are connected to. + """ + + def getHost(): + """ + Returns L{UNIXAddress}. + """ + + def getPeer(): + """ + Returns L{UNIXAddress}. + """ + + +class IMulticastTransport(Interface): + """ + Additional functionality for multicast UDP. + """ + + def getOutgoingInterface(): + """ + Return interface of outgoing multicast packets. + """ + + def setOutgoingInterface(addr): + """ + Set interface for outgoing multicast packets. + + Returns Deferred of success. + """ + + def getLoopbackMode(): + """ + Return if loopback mode is enabled. + """ + + def setLoopbackMode(mode): + """ + Set if loopback mode is enabled. + """ + + def getTTL(): + """ + Get time to live for multicast packets. + """ + + def setTTL(ttl): + """ + Set time to live on multicast packets. + """ + + def joinGroup(addr, interface=""): + """ + Join a multicast group. Returns L{Deferred} of success or failure. + + If an error occurs, the returned L{Deferred} will fail with + L{error.MulticastJoinError}. + """ + + def leaveGroup(addr, interface=""): + """ + Leave multicast group, return L{Deferred} of success. + """ + + +class IStreamClientEndpoint(Interface): + """ + A stream client endpoint is a place that L{ClientFactory} can connect to. + For example, a remote TCP host/port pair would be a TCP client endpoint. + + @since: 10.1 + """ + + def connect(protocolFactory): + """ + Connect the C{protocolFactory} to the location specified by this + L{IStreamClientEndpoint} provider. + + @param protocolFactory: A provider of L{IProtocolFactory} + @return: A L{Deferred} that results in an L{IProtocol} upon successful + connection otherwise a L{ConnectError} + """ + + + +class IStreamServerEndpoint(Interface): + """ + A stream server endpoint is a place that a L{Factory} can listen for + incoming connections. + + @since: 10.1 + """ + + def listen(protocolFactory): + """ + Listen with C{protocolFactory} at the location specified by this + L{IStreamServerEndpoint} provider. + + @param protocolFactory: A provider of L{IProtocolFactory} + @return: A L{Deferred} that results in an L{IListeningPort} or an + L{CannotListenError} + """ + + + +class IStreamServerEndpointStringParser(Interface): + """ + An L{IStreamServerEndpointStringParser} is like an + L{IStreamClientEndpointStringParser}, except for L{IStreamServerEndpoint}s + instead of clients. It integrates with L{endpoints.serverFromString} in + much the same way. + """ + + prefix = Attribute( + """ + @see: L{IStreamClientEndpointStringParser.prefix} + """ + ) + + + def parseStreamServer(reactor, *args, **kwargs): + """ + Parse a stream server endpoint from a reactor and string-only arguments + and keyword arguments. + + @see: L{IStreamClientEndpointStringParser.parseStreamClient} + + @return: a stream server endpoint + @rtype: L{IStreamServerEndpoint} + """ + + + +class IStreamClientEndpointStringParser(Interface): + """ + An L{IStreamClientEndpointStringParser} is a parser which can convert + a set of string C{*args} and C{**kwargs} into an L{IStreamClientEndpoint} + provider. + + This interface is really only useful in the context of the plugin system + for L{endpoints.clientFromString}. See the document entitled "I{The + Twisted Plugin System}" for more details on how to write a plugin. + + If you place an L{IStreamClientEndpointStringParser} plugin in the + C{twisted.plugins} package, that plugin's C{parseStreamClient} method will + be used to produce endpoints for any description string that begins with + the result of that L{IStreamClientEndpointStringParser}'s prefix attribute. + """ + + prefix = Attribute( + """ + A C{str}, the description prefix to respond to. For example, an + L{IStreamClientEndpointStringParser} plugin which had C{"foo"} for its + C{prefix} attribute would be called for endpoint descriptions like + C{"foo:bar:baz"} or C{"foo:"}. + """ + ) + + + def parseStreamClient(*args, **kwargs): + """ + This method is invoked by L{endpoints.clientFromString}, if the type of + endpoint matches the return value from this + L{IStreamClientEndpointStringParser}'s C{prefix} method. + + @param args: The string arguments, minus the endpoint type, in the + endpoint description string, parsed according to the rules + described in L{endpoints.quoteStringArgument}. For example, if the + description were C{"my-type:foo:bar:baz=qux"}, C{args} would be + C{('foo','bar')} + + @param kwargs: The string arguments from the endpoint description + passed as keyword arguments. For example, if the description were + C{"my-type:foo:bar:baz=qux"}, C{kwargs} would be + C{dict(baz='qux')}. + + @return: a client endpoint + @rtype: L{IStreamClientEndpoint} + """ diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py new file mode 100644 index 000000000..ddcb6ed7a --- /dev/null +++ b/scrapy/xlib/tx/iweb.py @@ -0,0 +1,587 @@ +# -*- test-case-name: twisted.web.test -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Interface definitions for L{twisted.web}. + +@var UNKNOWN_LENGTH: An opaque object which may be used as the value of + L{IBodyProducer.length} to indicate that the length of the entity + body is not known in advance. +""" + +from zope.interface import Interface, Attribute + +from twisted.internet.interfaces import IPushProducer + + +class IRequest(Interface): + """ + An HTTP request. + + @since: 9.0 + """ + + method = Attribute("A C{str} giving the HTTP method that was used.") + uri = Attribute( + "A C{str} giving the full encoded URI which was requested (including " + "query arguments).") + path = Attribute( + "A C{str} giving the encoded query path of the request URI.") + args = Attribute( + "A mapping of decoded query argument names as C{str} to " + "corresponding query argument values as C{list}s of C{str}. " + "For example, for a URI with C{'foo=bar&foo=baz&quux=spam'} " + "for its query part, C{args} will be C{{'foo': ['bar', 'baz'], " + "'quux': ['spam']}}.") + + received_headers = Attribute( + "Backwards-compatibility access to C{requestHeaders}. Use " + "C{requestHeaders} instead. C{received_headers} behaves mostly " + "like a C{dict} and does not provide access to all header values.") + + requestHeaders = Attribute( + "A L{http_headers.Headers} instance giving all received HTTP request " + "headers.") + + content = Attribute( + "A file-like object giving the request body. This may be a file on " + "disk, a C{StringIO}, or some other type. The implementation is free " + "to decide on a per-request basis.") + + headers = Attribute( + "Backwards-compatibility access to C{responseHeaders}. Use" + "C{responseHeaders} instead. C{headers} behaves mostly like a " + "C{dict} and does not provide access to all header values nor " + "does it allow multiple values for one header to be set.") + + responseHeaders = Attribute( + "A L{http_headers.Headers} instance holding all HTTP response " + "headers to be sent.") + + def getHeader(key): + """ + Get an HTTP request header. + + @type key: C{str} + @param key: The name of the header to get the value of. + + @rtype: C{str} or C{NoneType} + @return: The value of the specified header, or C{None} if that header + was not present in the request. + """ + + + def getCookie(key): + """ + Get a cookie that was sent from the network. + """ + + + def getAllHeaders(): + """ + Return dictionary mapping the names of all received headers to the last + value received for each. + + Since this method does not return all header information, + C{requestHeaders.getAllRawHeaders()} may be preferred. + """ + + + def getRequestHostname(): + """ + Get the hostname that the user passed in to the request. + + This will either use the Host: header (if it is available) or the + host we are listening on if the header is unavailable. + + @returns: the requested hostname + @rtype: C{str} + """ + + + def getHost(): + """ + Get my originally requesting transport's host. + + @return: An L{IAddress}. + """ + + + def getClientIP(): + """ + Return the IP address of the client who submitted this request. + + @returns: the client IP address or C{None} if the request was submitted + over a transport where IP addresses do not make sense. + @rtype: L{str} or C{NoneType} + """ + + + def getClient(): + """ + Return the hostname of the IP address of the client who submitted this + request, if possible. + + This method is B{deprecated}. See L{getClientIP} instead. + + @rtype: C{NoneType} or L{str} + @return: The canonical hostname of the client, as determined by + performing a name lookup on the IP address of the client. + """ + + + def getUser(): + """ + Return the HTTP user sent with this request, if any. + + If no user was supplied, return the empty string. + + @returns: the HTTP user, if any + @rtype: C{str} + """ + + + def getPassword(): + """ + Return the HTTP password sent with this request, if any. + + If no password was supplied, return the empty string. + + @returns: the HTTP password, if any + @rtype: C{str} + """ + + + def isSecure(): + """ + Return True if this request is using a secure transport. + + Normally this method returns True if this request's HTTPChannel + instance is using a transport that implements ISSLTransport. + + This will also return True if setHost() has been called + with ssl=True. + + @returns: True if this request is secure + @rtype: C{bool} + """ + + + def getSession(sessionInterface=None): + """ + Look up the session associated with this request or create a new one if + there is not one. + + @return: The L{Session} instance identified by the session cookie in + the request, or the C{sessionInterface} component of that session + if C{sessionInterface} is specified. + """ + + + def URLPath(): + """ + @return: A L{URLPath} instance which identifies the URL for which this + request is. + """ + + + def prePathURL(): + """ + @return: At any time during resource traversal, a L{str} giving an + absolute URL to the most nested resource which has yet been + reached. + """ + + + def rememberRootURL(): + """ + Remember the currently-processed part of the URL for later + recalling. + """ + + + def getRootURL(): + """ + Get a previously-remembered URL. + """ + + + # Methods for outgoing response + def finish(): + """ + Indicate that the response to this request is complete. + """ + + + def write(data): + """ + Write some data to the body of the response to this request. Response + headers are written the first time this method is called, after which + new response headers may not be added. + """ + + + def addCookie(k, v, expires=None, domain=None, path=None, max_age=None, comment=None, secure=None): + """ + Set an outgoing HTTP cookie. + + In general, you should consider using sessions instead of cookies, see + L{twisted.web.server.Request.getSession} and the + L{twisted.web.server.Session} class for details. + """ + + + def setResponseCode(code, message=None): + """ + Set the HTTP response code. + """ + + + def setHeader(k, v): + """ + Set an HTTP response header. Overrides any previously set values for + this header. + + @type name: C{str} + @param name: The name of the header for which to set the value. + + @type value: C{str} + @param value: The value to set for the named header. + """ + + + def redirect(url): + """ + Utility function that does a redirect. + + The request should have finish() called after this. + """ + + + def setLastModified(when): + """ + Set the C{Last-Modified} time for the response to this request. + + If I am called more than once, I ignore attempts to set Last-Modified + earlier, only replacing the Last-Modified time if it is to a later + value. + + If I am a conditional request, I may modify my response code to + L{NOT_MODIFIED} if appropriate for the time given. + + @param when: The last time the resource being returned was modified, in + seconds since the epoch. + @type when: L{int}, L{long} or L{float} + + @return: If I am a C{If-Modified-Since} conditional request and the time + given is not newer than the condition, I return + L{CACHED} to indicate that you should write no body. + Otherwise, I return a false value. + """ + + + def setETag(etag): + """ + Set an C{entity tag} for the outgoing response. + + That's "entity tag" as in the HTTP/1.1 I{ETag} header, "used for + comparing two or more entities from the same requested resource." + + If I am a conditional request, I may modify my response code to + L{NOT_MODIFIED} or + L{PRECONDITION_FAILED}, if appropriate for the + tag given. + + @param etag: The entity tag for the resource being returned. + @type etag: C{str} + + @return: If I am a C{If-None-Match} conditional request and the tag + matches one in the request, I return L{CACHED} to + indicate that you should write no body. Otherwise, I return a + false value. + """ + + + def setHost(host, port, ssl=0): + """ + Change the host and port the request thinks it's using. + + This method is useful for working with reverse HTTP proxies (e.g. both + Squid and Apache's mod_proxy can do this), when the address the HTTP + client is using is different than the one we're listening on. + + For example, Apache may be listening on https://www.example.com, and + then forwarding requests to http://localhost:8080, but we don't want + HTML produced by Twisted to say 'http://localhost:8080', they should + say 'https://www.example.com', so we do:: + + request.setHost('www.example.com', 443, ssl=1) + """ + + + +class ICredentialFactory(Interface): + """ + A credential factory defines a way to generate a particular kind of + authentication challenge and a way to interpret the responses to these + challenges. It creates + L{ICredentials} providers from + responses. These objects will be used with L{twisted.cred} to authenticate + an authorize requests. + """ + scheme = Attribute( + "A C{str} giving the name of the authentication scheme with which " + "this factory is associated. For example, C{'basic'} or C{'digest'}.") + + + def getChallenge(request): + """ + Generate a new challenge to be sent to a client. + + @type peer: L{twisted.web.http.Request} + @param peer: The request the response to which this challenge will be + included. + + @rtype: C{dict} + @return: A mapping from C{str} challenge fields to associated C{str} + values. + """ + + + def decode(response, request): + """ + Create a credentials object from the given response. + + @type response: C{str} + @param response: scheme specific response string + + @type request: L{twisted.web.http.Request} + @param request: The request being processed (from which the response + was taken). + + @raise twisted.cred.error.LoginFailed: If the response is invalid. + + @rtype: L{twisted.cred.credentials.ICredentials} provider + @return: The credentials represented by the given response. + """ + + + +class IBodyProducer(IPushProducer): + """ + Objects which provide L{IBodyProducer} write bytes to an object which + provides L{IConsumer} by calling its + C{write} method repeatedly. + + L{IBodyProducer} providers may start producing as soon as they have an + L{IConsumer} provider. That is, they + should not wait for a C{resumeProducing} call to begin writing data. + + L{IConsumer.unregisterProducer} + must not be called. Instead, the + L{Deferred} returned from C{startProducing} + must be fired when all bytes have been written. + + L{IConsumer.write} may + synchronously invoke any of C{pauseProducing}, C{resumeProducing}, or + C{stopProducing}. These methods must be implemented with this in mind. + + @since: 9.0 + """ + + # Despite the restrictions above and the additional requirements of + # stopProducing documented below, this interface still needs to be an + # IPushProducer subclass. Providers of it will be passed to IConsumer + # providers which only know about IPushProducer and IPullProducer, not + # about this interface. This interface needs to remain close enough to one + # of those interfaces for consumers to work with it. + + length = Attribute( + """ + C{length} is a C{int} indicating how many bytes in total this + L{IBodyProducer} will write to the consumer or L{UNKNOWN_LENGTH} + if this is not known in advance. + """) + + def startProducing(consumer): + """ + Start producing to the given + L{IConsumer} provider. + + @return: A L{Deferred} which fires with + C{None} when all bytes have been produced or with a + L{Failure} if there is any problem + before all bytes have been produced. + """ + + + def stopProducing(): + """ + In addition to the standard behavior of + L{IProducer.stopProducing} + (stop producing data), make sure the + L{Deferred} returned by + C{startProducing} is never fired. + """ + + + +class IRenderable(Interface): + """ + An L{IRenderable} is an object that may be rendered by the + L{twisted.web.template} templating system. + """ + + def lookupRenderMethod(name): + """ + Look up and return the render method associated with the given name. + + @type name: C{str} + @param name: The value of a render directive encountered in the + document returned by a call to L{IRenderable.render}. + + @return: A two-argument callable which will be invoked with the request + being responded to and the tag object on which the render directive + was encountered. + """ + + + def render(request): + """ + Get the document for this L{IRenderable}. + + @type request: L{IRequest} provider or C{NoneType} + @param request: The request in response to which this method is being + invoked. + + @return: An object which can be flattened. + """ + + + +class ITemplateLoader(Interface): + """ + A loader for templates; something usable as a value for + L{twisted.web.template.Element}'s C{loader} attribute. + """ + + def load(): + """ + Load a template suitable for rendering. + + @return: a C{list} of C{list}s, C{unicode} objects, C{Element}s and + other L{IRenderable} providers. + """ + + + +class IResponse(Interface): + """ + An object representing an HTTP response received from an HTTP server. + + @since: 11.1 + """ + + version = Attribute( + "A three-tuple describing the protocol and protocol version " + "of the response. The first element is of type C{str}, the second " + "and third are of type C{int}. For example, C{('HTTP', 1, 1)}.") + + + code = Attribute("The HTTP status code of this response, as a C{int}.") + + + phrase = Attribute( + "The HTTP reason phrase of this response, as a C{str}.") + + + headers = Attribute("The HTTP response L{Headers} of this response.") + + + length = Attribute( + "The C{int} number of bytes expected to be in the body of this " + "response or L{UNKNOWN_LENGTH} if the server did not indicate how " + "many bytes to expect. For I{HEAD} responses, this will be 0; if " + "the response includes a I{Content-Length} header, it will be " + "available in C{headers}.") + + + def deliverBody(protocol): + """ + Register an L{IProtocol} provider + to receive the response body. + + The protocol will be connected to a transport which provides + L{IPushProducer}. The protocol's C{connectionLost} method will be + called with: + + - ResponseDone, which indicates that all bytes from the response + have been successfully delivered. + + - PotentialDataLoss, which indicates that it cannot be determined + if the entire response body has been delivered. This only occurs + when making requests to HTTP servers which do not set + I{Content-Length} or a I{Transfer-Encoding} in the response. + + - ResponseFailed, which indicates that some bytes from the response + were lost. The C{reasons} attribute of the exception may provide + more specific indications as to why. + """ + + + +class _IRequestEncoder(Interface): + """ + An object encoding data passed to L{IRequest.write}, for example for + compression purpose. + + @since: 12.3 + """ + + def encode(data): + """ + Encode the data given and return the result. + + @param data: The content to encode. + @type data: C{str} + + @return: The encoded data. + @rtype: C{str} + """ + + + def finish(): + """ + Callback called when the request is closing. + + @return: If necessary, the pending data accumulated from previous + C{encode} calls. + @rtype: C{str} + """ + + + +class _IRequestEncoderFactory(Interface): + """ + A factory for returing L{_IRequestEncoder} instances. + + @since: 12.3 + """ + + def encoderForRequest(request): + """ + If applicable, returns a L{_IRequestEncoder} instance which will encode + the request. + """ + + + +UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH" + +__all__ = [ + "ICredentialFactory", "IRequest", + "IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder", + "_IRequestEncoderFactory", + + "UNKNOWN_LENGTH"] From bb806fa0f8f77cba59506153e33cab96f91008ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 4 Jun 2013 16:36:21 -0300 Subject: [PATCH 16/18] restore http10 as default handler and skip tests if http1.1 is not supported --- scrapy/settings/default_settings.py | 4 ++-- scrapy/tests/test_downloader_handlers.py | 7 +++++++ scrapy/xlib/tx/__init__.py | 16 +++++++++++++--- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 2eb15f96c..e18362b13 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -53,8 +53,8 @@ DOWNLOAD_DELAY = 0 DOWNLOAD_HANDLERS = {} DOWNLOAD_HANDLERS_BASE = { 'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler', - 'http': 'scrapy.core.downloader.handlers.http11.Http11DownloadHandler', - 'https': 'scrapy.core.downloader.handlers.http11.Http11DownloadHandler', + 'http': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler', + 'https': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler', 's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler', } diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index 2e2497b08..1307f57ad 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -1,4 +1,5 @@ import os +import twisted from twisted.trial import unittest from twisted.protocols.policies import WrappingFactory @@ -143,6 +144,9 @@ class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" download_handler_cls = Http11DownloadHandler + if twisted.__version__.split('.') < (11, 1, 0): + skip = 'HTTP1.1 not supported in twisted < 11.1.0' + class UriResource(resource.Resource): """Return the full uri that was requested""" @@ -198,6 +202,9 @@ class HttpProxyTestCase(unittest.TestCase): class Http11ProxyTestCase(HttpProxyTestCase): download_handler_cls = Http11DownloadHandler + if twisted.__version__.split('.') < (11, 1, 0): + skip = 'HTTP1.1 not supported in twisted < 11.1.0' + class HttpDownloadHandlerMock(object): def __init__(self, settings): diff --git a/scrapy/xlib/tx/__init__.py b/scrapy/xlib/tx/__init__.py index 9f731c5b6..e184a6735 100644 --- a/scrapy/xlib/tx/__init__.py +++ b/scrapy/xlib/tx/__init__.py @@ -1,9 +1,19 @@ import twisted -if twisted.__version__.split('.') < (13, 0, 1): - from . import client, endpoints -else: +txver = twisted.__version__.split('.') +if txver > (13, 0, 0): from twisted.web import client from twisted.internet import endpoints +if txver >= (11, 1, 0): + from . import client, endpoints +else: + from scrapy.exceptions import NotSupported + class _Mocked(object): + def __init__(self, *args, **kw): + raise NotSupported('HTTP1.1 not supported') + class _Mock(object): + def __getattr__(self, name): + return _Mocked + client = endpoints = _Mock() Agent = client.Agent From cf10474b8b6da1028a7a6cea13b509b0c6502306 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 4 Jun 2013 16:59:35 -0300 Subject: [PATCH 17/18] improve twisted version tests and enable HTTP11 handler by default if supported --- scrapy/__init__.py | 5 +++ scrapy/core/downloader/handlers/http.py | 32 ++++++------------- scrapy/core/downloader/handlers/http10.py | 25 +++++++++++++++ scrapy/core/downloader/handlers/http11.py | 2 +- scrapy/settings/default_settings.py | 4 +-- scrapy/tests/test_downloader_handlers.py | 39 ++++++++++++++++------- scrapy/xlib/tx/__init__.py | 7 ++-- 7 files changed, 73 insertions(+), 41 deletions(-) create mode 100644 scrapy/core/downloader/handlers/http10.py diff --git a/scrapy/__init__.py b/scrapy/__init__.py index a02f1644c..17bf64202 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -43,3 +43,8 @@ except ImportError: pass else: optional_features.add('django') + +from twisted import version as _txv +twisted_version = (_txv.major, _txv.minor, _txv.micro) +if twisted_version >= (11, 1, 0): + optional_features.add('http11') diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 9f3799c59..8d05907f1 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,25 +1,11 @@ -"""Download handlers for http and https schemes -""" -from twisted.internet import reactor -from scrapy.utils.misc import load_object +from scrapy import optional_features +from .http10 import HTTP10DownloadHandler + +if 'http11' in optional_features: + from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler +else: + HTTPDownloadHandler = HTTP10DownloadHandler -class HttpDownloadHandler(object): - - def __init__(self, settings): - self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) - self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) - - def download_request(self, request, spider): - """Return a deferred for the HTTP download""" - factory = self.HTTPClientFactory(request) - self._connect(factory) - return factory.deferred - - def _connect(self, factory): - host, port = factory.host, factory.port - if factory.scheme == 'https': - return reactor.connectSSL(host, port, factory, - self.ClientContextFactory()) - else: - return reactor.connectTCP(host, port, factory) +# backwards compatibility +HttpDownloadHandler = HTTP10DownloadHandler diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py new file mode 100644 index 000000000..11b2acdae --- /dev/null +++ b/scrapy/core/downloader/handlers/http10.py @@ -0,0 +1,25 @@ +"""Download handlers for http and https schemes +""" +from twisted.internet import reactor +from scrapy.utils.misc import load_object + + +class HTTP10DownloadHandler(object): + + def __init__(self, settings): + self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) + self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) + + def download_request(self, request, spider): + """Return a deferred for the HTTP download""" + factory = self.HTTPClientFactory(request) + self._connect(factory) + return factory.deferred + + def _connect(self, factory): + host, port = factory.host, factory.port + if factory.scheme == 'https': + return reactor.connectSSL(host, port, factory, + self.ClientContextFactory()) + else: + return reactor.connectTCP(host, port, factory) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 6f233a546..456f48054 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -21,7 +21,7 @@ from scrapy import log -class Http11DownloadHandler(object): +class HTTP11DownloadHandler(object): def __init__(self, settings): self._pool = HTTPConnectionPool(reactor, persistent=True) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index e18362b13..94fac60d6 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -53,8 +53,8 @@ DOWNLOAD_DELAY = 0 DOWNLOAD_HANDLERS = {} DOWNLOAD_HANDLERS_BASE = { 'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler', - 'http': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler', - 'https': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler', + 'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', + 'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler', } diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index 1307f57ad..472817777 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -12,8 +12,9 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \ from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers.file import FileDownloadHandler -from scrapy.core.downloader.handlers.http import HttpDownloadHandler -from scrapy.core.downloader.handlers.http11 import Http11DownloadHandler +from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler +from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler +from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.spider import BaseSpider from scrapy.http import Request @@ -48,7 +49,7 @@ class FileTestCase(unittest.TestCase): class HttpTestCase(unittest.TestCase): - download_handler_cls = HttpDownloadHandler + download_handler_cls = HTTPDownloadHandler def setUp(self): name = self.mktemp() @@ -140,11 +141,20 @@ class HttpTestCase(unittest.TestCase): return d +class DeprecatedHttpTestCase(HttpTestCase): + """HTTP 1.0 test case""" + download_handler_cls = HttpDownloadHandler + + +class Http10TestCase(HttpTestCase): + """HTTP 1.0 test case""" + download_handler_cls = HTTP10DownloadHandler + + class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" - download_handler_cls = Http11DownloadHandler - - if twisted.__version__.split('.') < (11, 1, 0): + download_handler_cls = HTTP11DownloadHandler + if 'http11' not in optional_features: skip = 'HTTP1.1 not supported in twisted < 11.1.0' @@ -159,8 +169,7 @@ class UriResource(resource.Resource): class HttpProxyTestCase(unittest.TestCase): - - download_handler_cls = HttpDownloadHandler + download_handler_cls = HTTPDownloadHandler def setUp(self): site = server.Site(UriResource(), timeout=None) @@ -199,10 +208,18 @@ class HttpProxyTestCase(unittest.TestCase): return self.download_request(request, BaseSpider('foo')).addCallback(_test) -class Http11ProxyTestCase(HttpProxyTestCase): - download_handler_cls = Http11DownloadHandler +class DeprecatedHttpProxyTestCase(unittest.TestCase): + """Old deprecated reference to http10 downloader handler""" + download_handler_cls = HttpDownloadHandler - if twisted.__version__.split('.') < (11, 1, 0): + +class Http10ProxyTestCase(HttpProxyTestCase): + download_handler_cls = HTTP10DownloadHandler + + +class Http11ProxyTestCase(HttpProxyTestCase): + download_handler_cls = HTTP11DownloadHandler + if 'http11' not in optional_features: skip = 'HTTP1.1 not supported in twisted < 11.1.0' diff --git a/scrapy/xlib/tx/__init__.py b/scrapy/xlib/tx/__init__.py index e184a6735..1ac4e0108 100644 --- a/scrapy/xlib/tx/__init__.py +++ b/scrapy/xlib/tx/__init__.py @@ -1,9 +1,8 @@ -import twisted -txver = twisted.__version__.split('.') -if txver > (13, 0, 0): +from scrapy import twisted_version +if twisted_version > (13, 0, 0): from twisted.web import client from twisted.internet import endpoints -if txver >= (11, 1, 0): +if twisted_version >= (11, 1, 0): from . import client, endpoints else: from scrapy.exceptions import NotSupported From e7acc2171df5ea12b12066d159b8c5d1f7e30131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 4 Jun 2013 17:28:43 -0300 Subject: [PATCH 18/18] show deprecation warning for HttpDownloadHandler --- scrapy/core/downloader/handlers/http.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 8d05907f1..1efebb939 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -8,4 +8,12 @@ else: # backwards compatibility -HttpDownloadHandler = HTTP10DownloadHandler +class HttpDownloadHandler(HTTP10DownloadHandler): + + def __init__(self, *args, **kwargs): + import warnings + from scrapy.exceptions import ScrapyDeprecationWarning + warnings.warn('HttpDownloadHandler is deprecated, import scrapy.core.downloader' + '.handlers.http10.HTTP10DownloadHandler instead', + category=ScrapyDeprecationWarning, stacklevel=1) + super(HttpDownloadHandler, self).__init__(*args, **kwargs)