From 985755d1fe0aabf922bcdb0e8bc22d67948820cb Mon Sep 17 00:00:00 2001 From: nyov Date: Thu, 1 Dec 2016 23:44:15 +0000 Subject: [PATCH] Remove obsolete xlib code for Twisted 13.1.0 --- scrapy/xlib/tx/_newclient.py | 1466 -------------------- scrapy/xlib/tx/client.py | 1116 ---------------- scrapy/xlib/tx/endpoints.py | 1253 ----------------- scrapy/xlib/tx/interfaces.py | 2433 ---------------------------------- scrapy/xlib/tx/iweb.py | 569 -------- 5 files changed, 6837 deletions(-) diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py index d20eda34f..39cd20f95 100644 --- a/scrapy/xlib/tx/_newclient.py +++ b/scrapy/xlib/tx/_newclient.py @@ -38,9 +38,7 @@ from twisted.internet.error import ConnectionDone from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred from twisted.internet.defer import CancelledError from twisted.internet.protocol import Protocol -#from twisted.protocols.basic import LineReceiver from twisted.web.iweb import UNKNOWN_LENGTH, IResponse -#from twisted.web.http_headers import Headers from twisted.web.http import NO_CONTENT, NOT_MODIFIED from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder @@ -59,1467 +57,3 @@ STATUS = 'STATUS' HEADER = 'HEADER' BODY = 'BODY' DONE = 'DONE' - -''' {{{ -class BadHeaders(Exception): - """ - Headers passed to L{Request} were in some way invalid. - """ - - - -class ExcessWrite(Exception): - """ - The body L{IBodyProducer} for a request tried to write data after - indicating it had finished writing data. - """ - - -class ParseError(Exception): - """ - Some received data could not be parsed. - - @ivar data: The string which could not be parsed. - """ - def __init__(self, reason, data): - Exception.__init__(self, reason, data) - self.data = data - - - -class BadResponseVersion(ParseError): - """ - The version string in a status line was unparsable. - """ - - - -class _WrapperException(Exception): - """ - L{_WrapperException} is the base exception type for exceptions which - include one or more other exceptions as the low-level causes. - - @ivar reasons: A list of exceptions. See subclass documentation for more - details. - """ - def __init__(self, reasons): - Exception.__init__(self, reasons) - self.reasons = reasons - - - -class RequestGenerationFailed(_WrapperException): - """ - There was an error while creating the bytes which make up a request. - - @ivar reasons: A C{list} of one or more L{Failure} instances giving the - reasons the request generation was considered to have failed. - """ - - - -class RequestTransmissionFailed(_WrapperException): - """ - There was an error while sending the bytes which make up a request. - - @ivar reasons: A C{list} of one or more L{Failure} instances giving the - reasons the request transmission was considered to have failed. - """ - - - -class ConnectionAborted(Exception): - """ - The connection was explicitly aborted by application code. - """ - - - -class WrongBodyLength(Exception): - """ - An L{IBodyProducer} declared the number of bytes it was going to - produce (via its C{length} attribute) and then produced a different number - of bytes. - """ - - - -class ResponseDone(Exception): - """ - L{ResponseDone} may be passed to L{IProtocol.connectionLost} on the - protocol passed to L{Response.deliverBody} and indicates that the entire - response has been delivered. - """ - - - -class ResponseFailed(_WrapperException): - """ - L{ResponseFailed} indicates that all of the response to a request was not - received for some reason. - - @ivar reasons: A C{list} of one or more L{Failure} instances giving the - reasons the response was considered to have failed. - - @ivar response: If specified, the L{Response} received from the server (and - in particular the status code and the headers). - """ - - def __init__(self, reasons, response=None): - _WrapperException.__init__(self, reasons) - self.response = response - - - -class ResponseNeverReceived(ResponseFailed): - """ - A L{ResponseFailed} that knows no response bytes at all have been received. - """ - - - -class RequestNotSent(Exception): - """ - L{RequestNotSent} indicates that an attempt was made to issue a request but - for reasons unrelated to the details of the request itself, the request - could not be sent. For example, this may indicate that an attempt was made - to send a request using a protocol which is no longer connected to a - server. - """ - - - -def _callAppFunction(function): - """ - Call C{function}. If it raises an exception, log it with a minimal - description of the source. - - @return: C{None} - """ - try: - function() - except: - log.err(None, "Unexpected exception from %s" % ( - fullyQualifiedName(function),)) - - - -class HTTPParser(LineReceiver): - """ - L{HTTPParser} handles the parsing side of HTTP processing. With a suitable - subclass, it can parse either the client side or the server side of the - connection. - - @ivar headers: All of the non-connection control message headers yet - received. - - @ivar state: State indicator for the response parsing state machine. One - of C{STATUS}, C{HEADER}, C{BODY}, C{DONE}. - - @ivar _partialHeader: C{None} or a C{list} of the lines of a multiline - header while that header is being received. - """ - - # NOTE: According to HTTP spec, we're supposed to eat the - # 'Proxy-Authenticate' and 'Proxy-Authorization' headers also, but that - # doesn't sound like a good idea to me, because it makes it impossible to - # have a non-authenticating transparent proxy in front of an authenticating - # proxy. An authenticating proxy can eat them itself. -jknight - # - # Further, quoting - # http://homepages.tesco.net/J.deBoynePollard/FGA/web-proxy-connection-header.html - # regarding the 'Proxy-Connection' header: - # - # The Proxy-Connection: header is a mistake in how some web browsers - # use HTTP. Its name is the result of a false analogy. It is not a - # standard part of the protocol. There is a different standard - # protocol mechanism for doing what it does. And its existence - # imposes a requirement upon HTTP servers such that no proxy HTTP - # server can be standards-conforming in practice. - # - # -exarkun - - # Some servers (like http://news.ycombinator.com/) return status lines and - # HTTP headers delimited by \n instead of \r\n. - delimiter = '\n' - - CONNECTION_CONTROL_HEADERS = set([ - 'content-length', 'connection', 'keep-alive', 'te', 'trailers', - 'transfer-encoding', 'upgrade', 'proxy-connection']) - - def connectionMade(self): - self.headers = Headers() - self.connHeaders = Headers() - self.state = STATUS - self._partialHeader = None - - - def switchToBodyMode(self, decoder): - """ - Switch to body parsing mode - interpret any more bytes delivered as - part of the message body and deliver them to the given decoder. - """ - if self.state == BODY: - raise RuntimeError("already in body mode") - - self.bodyDecoder = decoder - self.state = BODY - self.setRawMode() - - - def lineReceived(self, line): - """ - Handle one line from a response. - """ - # Handle the normal CR LF case. - if line[-1:] == '\r': - line = line[:-1] - - if self.state == STATUS: - self.statusReceived(line) - self.state = HEADER - elif self.state == HEADER: - if not line or line[0] not in ' \t': - if self._partialHeader is not None: - header = ''.join(self._partialHeader) - name, value = header.split(':', 1) - value = value.strip() - self.headerReceived(name, value) - if not line: - # Empty line means the header section is over. - self.allHeadersReceived() - else: - # Line not beginning with LWS is another header. - self._partialHeader = [line] - else: - # A line beginning with LWS is a continuation of a header - # begun on a previous line. - self._partialHeader.append(line) - - - def rawDataReceived(self, data): - """ - Pass data from the message body to the body decoder object. - """ - self.bodyDecoder.dataReceived(data) - - - def isConnectionControlHeader(self, name): - """ - Return C{True} if the given lower-cased name is the name of a - connection control header (rather than an entity header). - - According to RFC 2616, section 14.10, the tokens in the Connection - header are probably relevant here. However, I am not sure what the - practical consequences of either implementing or ignoring that are. - So I leave it unimplemented for the time being. - """ - return name in self.CONNECTION_CONTROL_HEADERS - - - def statusReceived(self, status): - """ - Callback invoked whenever the first line of a new message is received. - Override this. - - @param status: The first line of an HTTP request or response message - without trailing I{CR LF}. - @type status: C{str} - """ - - - def headerReceived(self, name, value): - """ - Store the given header in C{self.headers}. - """ - name = name.lower() - if self.isConnectionControlHeader(name): - headers = self.connHeaders - else: - headers = self.headers - headers.addRawHeader(name, value) - - - def allHeadersReceived(self): - """ - Callback invoked after the last header is passed to C{headerReceived}. - Override this to change to the C{BODY} or C{DONE} state. - """ - self.switchToBodyMode(None) - - - -class HTTPClientParser(HTTPParser): - """ - An HTTP parser which only handles HTTP responses. - - @ivar request: The request with which the expected response is associated. - @type request: L{Request} - - @ivar NO_BODY_CODES: A C{set} of response codes which B{MUST NOT} have a - body. - - @ivar finisher: A callable to invoke when this response is fully parsed. - - @ivar _responseDeferred: A L{Deferred} which will be called back with the - response when all headers in the response have been received. - Thereafter, C{None}. - - @ivar _everReceivedData: C{True} if any bytes have been received. - """ - NO_BODY_CODES = set([NO_CONTENT, NOT_MODIFIED]) - - _transferDecoders = { - 'chunked': _ChunkedTransferDecoder, - } - - bodyDecoder = None - - def __init__(self, request, finisher): - self.request = request - self.finisher = finisher - self._responseDeferred = Deferred() - self._everReceivedData = False - - - def dataReceived(self, data): - """ - Override so that we know if any response has been received. - """ - self._everReceivedData = True - HTTPParser.dataReceived(self, data) - - - def parseVersion(self, strversion): - """ - Parse version strings of the form Protocol '/' Major '.' Minor. E.g. - 'HTTP/1.1'. Returns (protocol, major, minor). Will raise ValueError - on bad syntax. - """ - try: - proto, strnumber = strversion.split('/') - major, minor = strnumber.split('.') - major, minor = int(major), int(minor) - except ValueError as e: - raise BadResponseVersion(str(e), strversion) - if major < 0 or minor < 0: - raise BadResponseVersion("version may not be negative", strversion) - return (proto, major, minor) - - - def statusReceived(self, status): - """ - Parse the status line into its components and create a response object - to keep track of this response's state. - """ - parts = status.split(' ', 2) - if len(parts) != 3: - raise ParseError("wrong number of parts", status) - - try: - statusCode = int(parts[1]) - except ValueError: - raise ParseError("non-integer status code", status) - - self.response = Response( - self.parseVersion(parts[0]), - statusCode, - parts[2], - self.headers, - self.transport) - - - def _finished(self, rest): - """ - Called to indicate that an entire response has been received. No more - bytes will be interpreted by this L{HTTPClientParser}. Extra bytes are - passed up and the state of this L{HTTPClientParser} is set to I{DONE}. - - @param rest: A C{str} giving any extra bytes delivered to this - L{HTTPClientParser} which are not part of the response being - parsed. - """ - self.state = DONE - self.finisher(rest) - - - def isConnectionControlHeader(self, name): - """ - Content-Length in the response to a HEAD request is an entity header, - not a connection control header. - """ - if self.request.method == 'HEAD' and name == 'content-length': - return False - return HTTPParser.isConnectionControlHeader(self, name) - - - def allHeadersReceived(self): - """ - Figure out how long the response body is going to be by examining - headers and stuff. - """ - if (self.response.code in self.NO_BODY_CODES - or self.request.method == 'HEAD'): - self.response.length = 0 - self._finished(self.clearLineBuffer()) - else: - transferEncodingHeaders = self.connHeaders.getRawHeaders( - 'transfer-encoding') - if transferEncodingHeaders: - - # This could be a KeyError. However, that would mean we do not - # know how to decode the response body, so failing the request - # is as good a behavior as any. Perhaps someday we will want - # to normalize/document/test this specifically, but failing - # seems fine to me for now. - transferDecoder = self._transferDecoders[transferEncodingHeaders[0].lower()] - - # If anyone ever invents a transfer encoding other than - # chunked (yea right), and that transfer encoding can predict - # the length of the response body, it might be sensible to - # allow the transfer decoder to set the response object's - # length attribute. - else: - contentLengthHeaders = self.connHeaders.getRawHeaders('content-length') - if contentLengthHeaders is None: - contentLength = None - elif len(contentLengthHeaders) == 1: - contentLength = int(contentLengthHeaders[0]) - self.response.length = contentLength - else: - # "HTTP Message Splitting" or "HTTP Response Smuggling" - # potentially happening. Or it's just a buggy server. - raise ValueError( - "Too many Content-Length headers; response is invalid") - - if contentLength == 0: - self._finished(self.clearLineBuffer()) - transferDecoder = None - else: - transferDecoder = lambda x, y: _IdentityTransferDecoder( - contentLength, x, y) - - if transferDecoder is None: - self.response._bodyDataFinished() - else: - # Make sure as little data as possible from the response body - # gets delivered to the response object until the response - # object actually indicates it is ready to handle bytes - # (probably because an application gave it a way to interpret - # them). - self.transport.pauseProducing() - self.switchToBodyMode(transferDecoder( - self.response._bodyDataReceived, - self._finished)) - - # This must be last. If it were first, then application code might - # change some state (for example, registering a protocol to receive the - # response body). Then the pauseProducing above would be wrong since - # the response is ready for bytes and nothing else would ever resume - # the transport. - self._responseDeferred.callback(self.response) - del self._responseDeferred - - - def connectionLost(self, reason): - if self.bodyDecoder is not None: - try: - try: - self.bodyDecoder.noMoreData() - except PotentialDataLoss: - self.response._bodyDataFinished(Failure()) - except _DataLoss: - self.response._bodyDataFinished( - Failure(ResponseFailed([reason, Failure()], - self.response))) - else: - self.response._bodyDataFinished() - except: - # Handle exceptions from both the except suites and the else - # suite. Those functions really shouldn't raise exceptions, - # but maybe there's some buggy application code somewhere - # making things difficult. - log.err() - elif self.state != DONE: - if self._everReceivedData: - exceptionClass = ResponseFailed - else: - exceptionClass = ResponseNeverReceived - self._responseDeferred.errback(Failure(exceptionClass([reason]))) - del self._responseDeferred - - - -class Request: - """ - A L{Request} instance describes an HTTP request to be sent to an HTTP - server. - - @ivar method: The HTTP method to for this request, ex: 'GET', 'HEAD', - 'POST', etc. - @type method: C{str} - - @ivar uri: The relative URI of the resource to request. For example, - C{'/foo/bar?baz=quux'}. - @type uri: C{str} - - @ivar headers: Headers to be sent to the server. It is important to - note that this object does not create any implicit headers. So it - is up to the HTTP Client to add required headers such as 'Host'. - @type headers: L{twisted.web.http_headers.Headers} - - @ivar bodyProducer: C{None} or an L{IBodyProducer} provider which - produces the content body to send to the remote HTTP server. - - @ivar persistent: Set to C{True} when you use HTTP persistent connection. - @type persistent: C{bool} - """ - def __init__(self, method, uri, headers, bodyProducer, persistent=False): - self.method = method - self.uri = uri - self.headers = headers - self.bodyProducer = bodyProducer - self.persistent = persistent - - - def _writeHeaders(self, transport, TEorCL): - hosts = self.headers.getRawHeaders('host', ()) - if len(hosts) != 1: - raise BadHeaders("Exactly one Host header required") - - # In the future, having the protocol version be a parameter to this - # method would probably be good. It would be nice if this method - # weren't limited to issuing HTTP/1.1 requests. - requestLines = [] - requestLines.append( - '%s %s HTTP/1.1\r\n' % (self.method, self.uri)) - if not self.persistent: - requestLines.append('Connection: close\r\n') - if TEorCL is not None: - requestLines.append(TEorCL) - for name, values in self.headers.getAllRawHeaders(): - requestLines.extend(['%s: %s\r\n' % (name, v) for v in values]) - requestLines.append('\r\n') - transport.writeSequence(requestLines) - - - def _writeToChunked(self, transport): - """ - Write this request to the given transport using chunked - transfer-encoding to frame the body. - """ - self._writeHeaders(transport, 'Transfer-Encoding: chunked\r\n') - encoder = ChunkedEncoder(transport) - encoder.registerProducer(self.bodyProducer, True) - d = self.bodyProducer.startProducing(encoder) - - def cbProduced(ignored): - encoder.unregisterProducer() - def ebProduced(err): - encoder._allowNoMoreWrites() - # Don't call the encoder's unregisterProducer because it will write - # a zero-length chunk. This would indicate to the server that the - # request body is complete. There was an error, though, so we - # don't want to do that. - transport.unregisterProducer() - return err - d.addCallbacks(cbProduced, ebProduced) - return d - - - def _writeToContentLength(self, transport): - """ - Write this request to the given transport using content-length to frame - the body. - """ - self._writeHeaders( - transport, - 'Content-Length: %d\r\n' % (self.bodyProducer.length,)) - - # This Deferred is used to signal an error in the data written to the - # encoder below. It can only errback and it will only do so before too - # many bytes have been written to the encoder and before the producer - # Deferred fires. - finishedConsuming = Deferred() - - # This makes sure the producer writes the correct number of bytes for - # the request body. - encoder = LengthEnforcingConsumer( - self.bodyProducer, transport, finishedConsuming) - - transport.registerProducer(self.bodyProducer, True) - - finishedProducing = self.bodyProducer.startProducing(encoder) - - def combine(consuming, producing): - # This Deferred is returned and will be fired when the first of - # consuming or producing fires. If it's cancelled, forward that - # cancellation to the producer. - def cancelConsuming(ign): - finishedProducing.cancel() - ultimate = Deferred(cancelConsuming) - - # Keep track of what has happened so far. This initially - # contains None, then an integer uniquely identifying what - # sequence of events happened. See the callbacks and errbacks - # defined below for the meaning of each value. - state = [None] - - def ebConsuming(err): - if state == [None]: - # The consuming Deferred failed first. This means the - # overall writeTo Deferred is going to errback now. The - # producing Deferred should not fire later (because the - # consumer should have called stopProducing on the - # producer), but if it does, a callback will be ignored - # and an errback will be logged. - state[0] = 1 - ultimate.errback(err) - else: - # The consuming Deferred errbacked after the producing - # Deferred fired. This really shouldn't ever happen. - # If it does, I goofed. Log the error anyway, just so - # there's a chance someone might notice and complain. - log.err( - err, - "Buggy state machine in %r/[%d]: " - "ebConsuming called" % (self, state[0])) - - def cbProducing(result): - if state == [None]: - # The producing Deferred succeeded first. Nothing will - # ever happen to the consuming Deferred. Tell the - # encoder we're done so it can check what the producer - # wrote and make sure it was right. - state[0] = 2 - try: - encoder._noMoreWritesExpected() - except: - # Fail the overall writeTo Deferred - something the - # producer did was wrong. - ultimate.errback() - else: - # Success - succeed the overall writeTo Deferred. - ultimate.callback(None) - # Otherwise, the consuming Deferred already errbacked. The - # producing Deferred wasn't supposed to fire, but it did - # anyway. It's buggy, but there's not really anything to be - # done about it. Just ignore this result. - - def ebProducing(err): - if state == [None]: - # The producing Deferred failed first. This means the - # overall writeTo Deferred is going to errback now. - # Tell the encoder that we're done so it knows to reject - # further writes from the producer (which should not - # happen, but the producer may be buggy). - state[0] = 3 - encoder._allowNoMoreWrites() - ultimate.errback(err) - else: - # The producing Deferred failed after the consuming - # Deferred failed. It shouldn't have, so it's buggy. - # Log the exception in case anyone who can fix the code - # is watching. - log.err(err, "Producer is buggy") - - consuming.addErrback(ebConsuming) - producing.addCallbacks(cbProducing, ebProducing) - - return ultimate - - d = combine(finishedConsuming, finishedProducing) - def f(passthrough): - # Regardless of what happens with the overall Deferred, once it - # fires, the producer registered way up above the definition of - # combine should be unregistered. - transport.unregisterProducer() - return passthrough - d.addBoth(f) - return d - - - def writeTo(self, transport): - """ - Format this L{Request} as an HTTP/1.1 request and write it to the given - transport. If bodyProducer is not None, it will be associated with an - L{IConsumer}. - - @return: A L{Deferred} which fires with C{None} when the request has - been completely written to the transport or with a L{Failure} if - there is any problem generating the request bytes. - """ - if self.bodyProducer is not None: - if self.bodyProducer.length is UNKNOWN_LENGTH: - return self._writeToChunked(transport) - else: - return self._writeToContentLength(transport) - else: - self._writeHeaders(transport, None) - return succeed(None) - - - def stopWriting(self): - """ - Stop writing this request to the transport. This can only be called - after C{writeTo} and before the L{Deferred} returned by C{writeTo} - fires. It should cancel any asynchronous task started by C{writeTo}. - The L{Deferred} returned by C{writeTo} need not be fired if this method - is called. - """ - # If bodyProducer is None, then the Deferred returned by writeTo has - # fired already and this method cannot be called. - _callAppFunction(self.bodyProducer.stopProducing) - - - -class LengthEnforcingConsumer: - """ - An L{IConsumer} proxy which enforces an exact length requirement on the - total data written to it. - - @ivar _length: The number of bytes remaining to be written. - - @ivar _producer: The L{IBodyProducer} which is writing to this - consumer. - - @ivar _consumer: The consumer to which at most C{_length} bytes will be - forwarded. - - @ivar _finished: A L{Deferred} which will be fired with a L{Failure} if too - many bytes are written to this consumer. - """ - def __init__(self, producer, consumer, finished): - self._length = producer.length - self._producer = producer - self._consumer = consumer - self._finished = finished - - - def _allowNoMoreWrites(self): - """ - Indicate that no additional writes are allowed. Attempts to write - after calling this method will be met with an exception. - """ - self._finished = None - - - def write(self, bytes): - """ - Write C{bytes} to the underlying consumer unless - C{_noMoreWritesExpected} has been called or there are/have been too - many bytes. - """ - if self._finished is None: - # No writes are supposed to happen any more. Try to convince the - # calling code to stop calling this method by calling its - # stopProducing method and then throwing an exception at it. This - # exception isn't documented as part of the API because you're - # never supposed to expect it: only buggy code will ever receive - # it. - self._producer.stopProducing() - raise ExcessWrite() - - if len(bytes) <= self._length: - self._length -= len(bytes) - self._consumer.write(bytes) - else: - # No synchronous exception is raised in *this* error path because - # we still have _finished which we can use to report the error to a - # better place than the direct caller of this method (some - # arbitrary application code). - _callAppFunction(self._producer.stopProducing) - self._finished.errback(WrongBodyLength("too many bytes written")) - self._allowNoMoreWrites() - - - def _noMoreWritesExpected(self): - """ - Called to indicate no more bytes will be written to this consumer. - Check to see that the correct number have been written. - - @raise WrongBodyLength: If not enough bytes have been written. - """ - if self._finished is not None: - self._allowNoMoreWrites() - if self._length: - raise WrongBodyLength("too few bytes written") - - - -def makeStatefulDispatcher(name, template): - """ - Given a I{dispatch} name and a function, return a function which can be - used as a method and which, when called, will call another method defined - on the instance and return the result. The other method which is called is - determined by the value of the C{_state} attribute of the instance. - - @param name: A string which is used to construct the name of the subsidiary - method to invoke. The subsidiary method is named like C{'_%s_%s' % - (name, _state)}. - - @param template: A function object which is used to give the returned - function a docstring. - - @return: The dispatcher function. - """ - def dispatcher(self, *args, **kwargs): - func = getattr(self, '_' + name + '_' + self._state, None) - if func is None: - raise RuntimeError( - "%r has no %s method in state %s" % (self, name, self._state)) - return func(*args, **kwargs) - dispatcher.__doc__ = template.__doc__ - return dispatcher - - - -class Response: - """ - A L{Response} instance describes an HTTP response received from an HTTP - server. - - L{Response} should not be subclassed or instantiated. - - @ivar _transport: The transport which is delivering this response. - - @ivar _bodyProtocol: The L{IProtocol} provider to which the body is - delivered. C{None} before one has been registered with - C{deliverBody}. - - @ivar _bodyBuffer: A C{list} of the strings passed to C{bodyDataReceived} - before C{deliverBody} is called. C{None} afterwards. - - @ivar _state: Indicates what state this L{Response} instance is in, - particularly with respect to delivering bytes from the response body - to an application-suppled protocol object. This may be one of - C{'INITIAL'}, C{'CONNECTED'}, C{'DEFERRED_CLOSE'}, or C{'FINISHED'}, - with the following meanings: - - - INITIAL: This is the state L{Response} objects start in. No - protocol has yet been provided and the underlying transport may - still have bytes to deliver to it. - - - DEFERRED_CLOSE: If the underlying transport indicates all bytes - have been delivered but no application-provided protocol is yet - available, the L{Response} moves to this state. Data is - buffered and waiting for a protocol to be delivered to. - - - CONNECTED: If a protocol is provided when the state is INITIAL, - the L{Response} moves to this state. Any buffered data is - delivered and any data which arrives from the transport - subsequently is given directly to the protocol. - - - FINISHED: If a protocol is provided in the DEFERRED_CLOSE state, - the L{Response} moves to this state after delivering all - buffered data to the protocol. Otherwise, if the L{Response} is - in the CONNECTED state, if the transport indicates there is no - more data, the L{Response} moves to this state. Nothing else - can happen once the L{Response} is in this state. - """ - implements(IResponse) - - length = UNKNOWN_LENGTH - - _bodyProtocol = None - _bodyFinished = False - - def __init__(self, version, code, phrase, headers, _transport): - self.version = version - self.code = code - self.phrase = phrase - self.headers = headers - self._transport = _transport - self._bodyBuffer = [] - self._state = 'INITIAL' - - - def deliverBody(self, protocol): - """ - Dispatch the given L{IProtocol} depending of the current state of the - response. - """ - deliverBody = makeStatefulDispatcher('deliverBody', deliverBody) - - - def _deliverBody_INITIAL(self, protocol): - """ - Deliver any buffered data to C{protocol} and prepare to deliver any - future data to it. Move to the C{'CONNECTED'} state. - """ - # Now that there's a protocol to consume the body, resume the - # transport. It was previously paused by HTTPClientParser to avoid - # reading too much data before it could be handled. - self._transport.resumeProducing() - - protocol.makeConnection(self._transport) - self._bodyProtocol = protocol - for data in self._bodyBuffer: - self._bodyProtocol.dataReceived(data) - self._bodyBuffer = None - self._state = 'CONNECTED' - - - def _deliverBody_CONNECTED(self, protocol): - """ - It is invalid to attempt to deliver data to a protocol when it is - already being delivered to another protocol. - """ - raise RuntimeError( - "Response already has protocol %r, cannot deliverBody " - "again" % (self._bodyProtocol,)) - - - def _deliverBody_DEFERRED_CLOSE(self, protocol): - """ - Deliver any buffered data to C{protocol} and then disconnect the - protocol. Move to the C{'FINISHED'} state. - """ - # Unlike _deliverBody_INITIAL, there is no need to resume the - # transport here because all of the response data has been received - # already. Some higher level code may want to resume the transport if - # that code expects further data to be received over it. - - protocol.makeConnection(self._transport) - - for data in self._bodyBuffer: - protocol.dataReceived(data) - self._bodyBuffer = None - protocol.connectionLost(self._reason) - self._state = 'FINISHED' - - - def _deliverBody_FINISHED(self, protocol): - """ - It is invalid to attempt to deliver data to a protocol after the - response body has been delivered to another protocol. - """ - raise RuntimeError( - "Response already finished, cannot deliverBody now.") - - - def _bodyDataReceived(self, data): - """ - Called by HTTPClientParser with chunks of data from the response body. - They will be buffered or delivered to the protocol passed to - deliverBody. - """ - _bodyDataReceived = makeStatefulDispatcher('bodyDataReceived', - _bodyDataReceived) - - - def _bodyDataReceived_INITIAL(self, data): - """ - Buffer any data received for later delivery to a protocol passed to - C{deliverBody}. - - Little or no data should be buffered by this method, since the - transport has been paused and will not be resumed until a protocol - is supplied. - """ - self._bodyBuffer.append(data) - - - def _bodyDataReceived_CONNECTED(self, data): - """ - Deliver any data received to the protocol to which this L{Response} - is connected. - """ - self._bodyProtocol.dataReceived(data) - - - def _bodyDataReceived_DEFERRED_CLOSE(self, data): - """ - It is invalid for data to be delivered after it has been indicated - that the response body has been completely delivered. - """ - raise RuntimeError("Cannot receive body data after _bodyDataFinished") - - - def _bodyDataReceived_FINISHED(self, data): - """ - It is invalid for data to be delivered after the response body has - been delivered to a protocol. - """ - raise RuntimeError("Cannot receive body data after protocol disconnected") - - - def _bodyDataFinished(self, reason=None): - """ - Called by HTTPClientParser when no more body data is available. If the - optional reason is supplied, this indicates a problem or potential - problem receiving all of the response body. - """ - _bodyDataFinished = makeStatefulDispatcher('bodyDataFinished', - _bodyDataFinished) - - - def _bodyDataFinished_INITIAL(self, reason=None): - """ - Move to the C{'DEFERRED_CLOSE'} state to wait for a protocol to - which to deliver the response body. - """ - self._state = 'DEFERRED_CLOSE' - if reason is None: - reason = Failure(ResponseDone("Response body fully received")) - self._reason = reason - - - def _bodyDataFinished_CONNECTED(self, reason=None): - """ - Disconnect the protocol and move to the C{'FINISHED'} state. - """ - if reason is None: - reason = Failure(ResponseDone("Response body fully received")) - self._bodyProtocol.connectionLost(reason) - self._bodyProtocol = None - self._state = 'FINISHED' - - - def _bodyDataFinished_DEFERRED_CLOSE(self): - """ - It is invalid to attempt to notify the L{Response} of the end of the - response body data more than once. - """ - raise RuntimeError("Cannot finish body data more than once") - - - def _bodyDataFinished_FINISHED(self): - """ - It is invalid to attempt to notify the L{Response} of the end of the - response body data more than once. - """ - raise RuntimeError("Cannot finish body data after protocol disconnected") - - - -class ChunkedEncoder: - """ - Helper object which exposes L{IConsumer} on top of L{HTTP11ClientProtocol} - for streaming request bodies to the server. - """ - implements(IConsumer) - - def __init__(self, transport): - self.transport = transport - - - def _allowNoMoreWrites(self): - """ - Indicate that no additional writes are allowed. Attempts to write - after calling this method will be met with an exception. - """ - self.transport = None - - - def registerProducer(self, producer, streaming): - """ - Register the given producer with C{self.transport}. - """ - self.transport.registerProducer(producer, streaming) - - - def write(self, data): - """ - Write the given request body bytes to the transport using chunked - encoding. - - @type data: C{str} - """ - if self.transport is None: - raise ExcessWrite() - self.transport.writeSequence(("%x\r\n" % len(data), data, "\r\n")) - - - def unregisterProducer(self): - """ - Indicate that the request body is complete and finish the request. - """ - self.write('') - self.transport.unregisterProducer() - self._allowNoMoreWrites() - - - -class TransportProxyProducer: - """ - An L{IPushProducer} implementation which wraps another such thing and - proxies calls to it until it is told to stop. - - @ivar _producer: The wrapped L{IPushProducer} provider or C{None} after - this proxy has been stopped. - """ - implements(IPushProducer) - - # LineReceiver uses this undocumented attribute of transports to decide - # when to stop calling lineReceived or rawDataReceived (if it finds it to - # be true, it doesn't bother to deliver any more data). Set disconnecting - # to False here and never change it to true so that all data is always - # delivered to us and so that LineReceiver doesn't fail with an - # AttributeError. - disconnecting = False - - def __init__(self, producer): - self._producer = producer - - - def _stopProxying(self): - """ - Stop forwarding calls of L{IPushProducer} methods to the underlying - L{IPushProvider} provider. - """ - self._producer = None - - - def stopProducing(self): - """ - Proxy the stoppage to the underlying producer, unless this proxy has - been stopped. - """ - if self._producer is not None: - self._producer.stopProducing() - - - def resumeProducing(self): - """ - Proxy the resumption to the underlying producer, unless this proxy has - been stopped. - """ - if self._producer is not None: - self._producer.resumeProducing() - - - def pauseProducing(self): - """ - Proxy the pause to the underlying producer, unless this proxy has been - stopped. - """ - if self._producer is not None: - self._producer.pauseProducing() - - - -class HTTP11ClientProtocol(Protocol): - """ - L{HTTP11ClientProtocol} is an implementation of the HTTP 1.1 client - protocol. It supports as few features as possible. - - @ivar _parser: After a request is issued, the L{HTTPClientParser} to - which received data making up the response to that request is - delivered. - - @ivar _finishedRequest: After a request is issued, the L{Deferred} which - will fire when a L{Response} object corresponding to that request is - available. This allows L{HTTP11ClientProtocol} to fail the request - if there is a connection or parsing problem. - - @ivar _currentRequest: After a request is issued, the L{Request} - instance used to make that request. This allows - L{HTTP11ClientProtocol} to stop request generation if necessary (for - example, if the connection is lost). - - @ivar _transportProxy: After a request is issued, the - L{TransportProxyProducer} to which C{_parser} is connected. This - allows C{_parser} to pause and resume the transport in a way which - L{HTTP11ClientProtocol} can exert some control over. - - @ivar _responseDeferred: After a request is issued, the L{Deferred} from - C{_parser} which will fire with a L{Response} when one has been - received. This is eventually chained with C{_finishedRequest}, but - only in certain cases to avoid double firing that Deferred. - - @ivar _state: Indicates what state this L{HTTP11ClientProtocol} instance - is in with respect to transmission of a request and reception of a - response. This may be one of the following strings: - - - QUIESCENT: This is the state L{HTTP11ClientProtocol} instances - start in. Nothing is happening: no request is being sent and no - response is being received or expected. - - - TRANSMITTING: When a request is made (via L{request}), the - instance moves to this state. L{Request.writeTo} has been used - to start to send a request but it has not yet finished. - - - TRANSMITTING_AFTER_RECEIVING_RESPONSE: The server has returned a - complete response but the request has not yet been fully sent - yet. The instance will remain in this state until the request - is fully sent. - - - GENERATION_FAILED: There was an error while the request. The - request was not fully sent to the network. - - - WAITING: The request was fully sent to the network. The - instance is now waiting for the response to be fully received. - - - ABORTING: Application code has requested that the HTTP connection - be aborted. - - - CONNECTION_LOST: The connection has been lost. - - @ivar _abortDeferreds: A list of C{Deferred} instances that will fire when - the connection is lost. - """ - _state = 'QUIESCENT' - _parser = None - _finishedRequest = None - _currentRequest = None - _transportProxy = None - _responseDeferred = None - - - def __init__(self, quiescentCallback=lambda c: None): - self._quiescentCallback = quiescentCallback - self._abortDeferreds = [] - - - @property - def state(self): - return self._state - - - def request(self, request): - """ - Issue C{request} over C{self.transport} and return a L{Deferred} which - will fire with a L{Response} instance or an error. - - @param request: The object defining the parameters of the request to - issue. - @type request: L{Request} - - @rtype: L{Deferred} - @return: The deferred may errback with L{RequestGenerationFailed} if - the request was not fully written to the transport due to a local - error. It may errback with L{RequestTransmissionFailed} if it was - not fully written to the transport due to a network error. It may - errback with L{ResponseFailed} if the request was sent (not - necessarily received) but some or all of the response was lost. It - may errback with L{RequestNotSent} if it is not possible to send - any more requests using this L{HTTP11ClientProtocol}. - """ - if self._state != 'QUIESCENT': - return fail(RequestNotSent()) - - self._state = 'TRANSMITTING' - _requestDeferred = maybeDeferred(request.writeTo, self.transport) - - def cancelRequest(ign): - # Explicitly cancel the request's deferred if it's still trying to - # write when this request is cancelled. - if self._state in ( - 'TRANSMITTING', 'TRANSMITTING_AFTER_RECEIVING_RESPONSE'): - _requestDeferred.cancel() - else: - self.transport.abortConnection() - self._disconnectParser(Failure(CancelledError())) - self._finishedRequest = Deferred(cancelRequest) - - # Keep track of the Request object in case we need to call stopWriting - # on it. - self._currentRequest = request - - self._transportProxy = TransportProxyProducer(self.transport) - self._parser = HTTPClientParser(request, self._finishResponse) - self._parser.makeConnection(self._transportProxy) - self._responseDeferred = self._parser._responseDeferred - - def cbRequestWrotten(ignored): - if self._state == 'TRANSMITTING': - self._state = 'WAITING' - self._responseDeferred.chainDeferred(self._finishedRequest) - - def ebRequestWriting(err): - if self._state == 'TRANSMITTING': - self._state = 'GENERATION_FAILED' - self.transport.abortConnection() - self._finishedRequest.errback( - Failure(RequestGenerationFailed([err]))) - else: - log.err(err, 'Error writing request, but not in valid state ' - 'to finalize request: %s' % self._state) - - _requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting) - - return self._finishedRequest - - - def _finishResponse(self, rest): - """ - Called by an L{HTTPClientParser} to indicate that it has parsed a - complete response. - - @param rest: A C{str} giving any trailing bytes which were given to - the L{HTTPClientParser} which were not part of the response it - was parsing. - """ - _finishResponse = makeStatefulDispatcher('finishResponse', _finishResponse) - - - def _finishResponse_WAITING(self, rest): - # Currently the rest parameter is ignored. Don't forget to use it if - # we ever add support for pipelining. And maybe check what trailers - # mean. - if self._state == 'WAITING': - self._state = 'QUIESCENT' - else: - # The server sent the entire response before we could send the - # whole request. That sucks. Oh well. Fire the request() - # Deferred with the response. But first, make sure that if the - # request does ever finish being written that it won't try to fire - # that Deferred. - self._state = 'TRANSMITTING_AFTER_RECEIVING_RESPONSE' - self._responseDeferred.chainDeferred(self._finishedRequest) - - # This will happen if we're being called due to connection being lost; - # if so, no need to disconnect parser again, or to call - # _quiescentCallback. - if self._parser is None: - return - - reason = ConnectionDone("synthetic!") - connHeaders = self._parser.connHeaders.getRawHeaders('connection', ()) - if (('close' in connHeaders) or self._state != "QUIESCENT" or - not self._currentRequest.persistent): - self._giveUp(Failure(reason)) - else: - # We call the quiescent callback first, to ensure connection gets - # added back to connection pool before we finish the request. - try: - self._quiescentCallback(self) - except: - # If callback throws exception, just log it and disconnect; - # keeping persistent connections around is an optimisation: - log.err() - self.transport.loseConnection() - self._disconnectParser(reason) - - - _finishResponse_TRANSMITTING = _finishResponse_WAITING - - - def _disconnectParser(self, reason): - """ - If there is still a parser, call its C{connectionLost} method with the - given reason. If there is not, do nothing. - - @type reason: L{Failure} - """ - if self._parser is not None: - parser = self._parser - self._parser = None - self._currentRequest = None - self._finishedRequest = None - self._responseDeferred = None - - # The parser is no longer allowed to do anything to the real - # transport. Stop proxying from the parser's transport to the real - # transport before telling the parser it's done so that it can't do - # anything. - self._transportProxy._stopProxying() - self._transportProxy = None - parser.connectionLost(reason) - - - def _giveUp(self, reason): - """ - Lose the underlying connection and disconnect the parser with the given - L{Failure}. - - Use this method instead of calling the transport's loseConnection - method directly otherwise random things will break. - """ - self.transport.loseConnection() - self._disconnectParser(reason) - - - def dataReceived(self, bytes): - """ - Handle some stuff from some place. - """ - try: - self._parser.dataReceived(bytes) - except: - self._giveUp(Failure()) - - - def connectionLost(self, reason): - """ - The underlying transport went away. If appropriate, notify the parser - object. - """ - connectionLost = makeStatefulDispatcher('connectionLost', connectionLost) - - - def _connectionLost_QUIESCENT(self, reason): - """ - Nothing is currently happening. Move to the C{'CONNECTION_LOST'} - state but otherwise do nothing. - """ - self._state = 'CONNECTION_LOST' - - - def _connectionLost_GENERATION_FAILED(self, reason): - """ - The connection was in an inconsistent state. Move to the - C{'CONNECTION_LOST'} state but otherwise do nothing. - """ - self._state = 'CONNECTION_LOST' - - - def _connectionLost_TRANSMITTING(self, reason): - """ - Fail the L{Deferred} for the current request, notify the request - object that it does not need to continue transmitting itself, and - move to the C{'CONNECTION_LOST'} state. - """ - self._state = 'CONNECTION_LOST' - self._finishedRequest.errback( - Failure(RequestTransmissionFailed([reason]))) - del self._finishedRequest - - # Tell the request that it should stop bothering now. - self._currentRequest.stopWriting() - - - def _connectionLost_TRANSMITTING_AFTER_RECEIVING_RESPONSE(self, reason): - """ - Move to the C{'CONNECTION_LOST'} state. - """ - self._state = 'CONNECTION_LOST' - - - def _connectionLost_WAITING(self, reason): - """ - Disconnect the response parser so that it can propagate the event as - necessary (for example, to call an application protocol's - C{connectionLost} method, or to fail a request L{Deferred}) and move - to the C{'CONNECTION_LOST'} state. - """ - self._disconnectParser(reason) - self._state = 'CONNECTION_LOST' - - - def _connectionLost_ABORTING(self, reason): - """ - Disconnect the response parser with a L{ConnectionAborted} failure, and - move to the C{'CONNECTION_LOST'} state. - """ - self._disconnectParser(Failure(ConnectionAborted())) - self._state = 'CONNECTION_LOST' - for d in self._abortDeferreds: - d.callback(None) - self._abortDeferreds = [] - - - def abort(self): - """ - Close the connection and cause all outstanding L{request} L{Deferred}s - to fire with an error. - """ - if self._state == "CONNECTION_LOST": - return succeed(None) - self.transport.loseConnection() - self._state = 'ABORTING' - d = Deferred() - self._abortDeferreds.append(d) - return d -}}} ''' diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py index 8e0b1df8b..c2d50648a 100644 --- a/scrapy/xlib/tx/client.py +++ b/scrapy/xlib/tx/client.py @@ -42,1129 +42,13 @@ from twisted.web.client import ( Agent, ProxyAgent, HTTPConnectionPool, readBody, ) -''' {{{ -class PartialDownloadError(error.Error): - """ - Page was only partially downloaded, we got disconnected in middle. - - @ivar response: All of the response body which was downloaded. - """ - - -class _URL(tuple): - """ - A parsed URL. - - At some point this should be replaced with a better URL implementation. - """ - def __new__(self, scheme, host, port, path): - return tuple.__new__(_URL, (scheme, host, port, path)) - - - def __init__(self, scheme, host, port, path): - self.scheme = scheme - self.host = host - self.port = port - self.path = path - - -def _parse(url, defaultPort=None): - """ - Split the given URL into the scheme, host, port, and path. - - @type url: C{bytes} - @param url: An URL to parse. - - @type defaultPort: C{int} or C{None} - @param defaultPort: An alternate value to use as the port if the URL does - not include one. - - @return: A four-tuple of the scheme, host, port, and path of the URL. All - of these are C{bytes} instances except for port, which is an C{int}. - """ - url = url.strip() - parsed = http.urlparse(url) - scheme = parsed[0] - path = urlunparse((b'', b'') + parsed[2:]) - - if defaultPort is None: - if scheme == b'https': - defaultPort = 443 - else: - defaultPort = 80 - - host, port = parsed[1], defaultPort - if b':' in host: - host, port = host.split(b':') - try: - port = int(port) - except ValueError: - port = defaultPort - - if path == b'': - path = b'/' - - return _URL(scheme, host, port, path) - - -def _makeGetterFactory(url, factoryFactory, contextFactory=None, - *args, **kwargs): - """ - Create and connect an HTTP page getting factory. - - Any additional positional or keyword arguments are used when calling - C{factoryFactory}. - - @param factoryFactory: Factory factory that is called with C{url}, C{args} - and C{kwargs} to produce the getter - - @param contextFactory: Context factory to use when creating a secure - connection, defaulting to C{None} - - @return: The factory created by C{factoryFactory} - """ - scheme, host, port, path = _parse(url) - factory = factoryFactory(url, *args, **kwargs) - if scheme == b'https': - from twisted.internet import ssl - if contextFactory is None: - contextFactory = ssl.ClientContextFactory() - reactor.connectSSL(host, port, factory, contextFactory) - else: - reactor.connectTCP(host, port, factory) - return factory -}}} ''' # The code which follows is based on the new HTTP client implementation. It # should be significantly better than anything above, though it is not yet # feature equivalent. -#from twisted.web.error import SchemeNotSupported from twisted.web._newclient import Response -#from twisted.web._newclient import Request, HTTP11ClientProtocol from twisted.web._newclient import ResponseDone, ResponseFailed -#from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed -#from twisted.web._newclient import ( -# ResponseNeverReceived, PotentialDataLoss, _WrapperException) - -''' {{{ -try: - from twisted.internet.ssl import ClientContextFactory -except ImportError: - class WebClientContextFactory(object): - """ - A web context factory which doesn't work because the necessary SSL - support is missing. - """ - def getContext(self, hostname, port): - raise NotImplementedError("SSL support unavailable") -else: - class WebClientContextFactory(ClientContextFactory): - """ - A web context factory which ignores the hostname and port and does no - certificate verification. - """ - def getContext(self, hostname, port): - return ClientContextFactory.getContext(self) - - - -class _WebToNormalContextFactory(object): - """ - Adapt a web context factory to a normal context factory. - - @ivar _webContext: A web context factory which accepts a hostname and port - number to its C{getContext} method. - - @ivar _hostname: The hostname which will be passed to - C{_webContext.getContext}. - - @ivar _port: The port number which will be passed to - C{_webContext.getContext}. - """ - def __init__(self, webContext, hostname, port): - self._webContext = webContext - self._hostname = hostname - self._port = port - - - def getContext(self): - """ - Called the wrapped web context factory's C{getContext} method with a - hostname and port number and return the resulting context object. - """ - return self._webContext.getContext(self._hostname, self._port) - - - -@implementer(IBodyProducer) -class FileBodyProducer(object): - """ - L{FileBodyProducer} produces bytes from an input file object incrementally - and writes them to a consumer. - - Since file-like objects cannot be read from in an event-driven manner, - L{FileBodyProducer} uses a L{Cooperator} instance to schedule reads from - the file. This process is also paused and resumed based on notifications - from the L{IConsumer} provider being written to. - - The file is closed after it has been read, or if the producer is stopped - early. - - @ivar _inputFile: Any file-like object, bytes read from which will be - written to a consumer. - - @ivar _cooperate: A method like L{Cooperator.cooperate} which is used to - schedule all reads. - - @ivar _readSize: The number of bytes to read from C{_inputFile} at a time. - """ - - # Python 2.4 doesn't have these symbolic constants - _SEEK_SET = getattr(os, 'SEEK_SET', 0) - _SEEK_END = getattr(os, 'SEEK_END', 2) - - def __init__(self, inputFile, cooperator=task, readSize=2 ** 16): - self._inputFile = inputFile - self._cooperate = cooperator.cooperate - self._readSize = readSize - self.length = self._determineLength(inputFile) - - - def _determineLength(self, fObj): - """ - Determine how many bytes can be read out of C{fObj} (assuming it is not - modified from this point on). If the determination cannot be made, - return C{UNKNOWN_LENGTH}. - """ - try: - seek = fObj.seek - tell = fObj.tell - except AttributeError: - return UNKNOWN_LENGTH - originalPosition = tell() - seek(0, self._SEEK_END) - end = tell() - seek(originalPosition, self._SEEK_SET) - return end - originalPosition - - - def stopProducing(self): - """ - Permanently stop writing bytes from the file to the consumer by - stopping the underlying L{CooperativeTask}. - """ - self._inputFile.close() - self._task.stop() - - - def startProducing(self, consumer): - """ - Start a cooperative task which will read bytes from the input file and - write them to C{consumer}. Return a L{Deferred} which fires after all - bytes have been written. - - @param consumer: Any L{IConsumer} provider - """ - self._task = self._cooperate(self._writeloop(consumer)) - d = self._task.whenDone() - def maybeStopped(reason): - # IBodyProducer.startProducing's Deferred isn't support to fire if - # stopProducing is called. - reason.trap(task.TaskStopped) - return defer.Deferred() - d.addCallbacks(lambda ignored: None, maybeStopped) - return d - - - def _writeloop(self, consumer): - """ - Return an iterator which reads one chunk of bytes from the input file - and writes them to the consumer for each time it is iterated. - """ - while True: - bytes = self._inputFile.read(self._readSize) - if not bytes: - self._inputFile.close() - break - consumer.write(bytes) - yield None - - - def pauseProducing(self): - """ - Temporarily suspend copying bytes from the input file to the consumer - by pausing the L{CooperativeTask} which drives that activity. - """ - self._task.pause() - - - def resumeProducing(self): - """ - Undo the effects of a previous C{pauseProducing} and resume copying - bytes to the consumer by resuming the L{CooperativeTask} which drives - the write activity. - """ - self._task.resume() - - - -class _HTTP11ClientFactory(protocol.Factory): - """ - A factory for L{HTTP11ClientProtocol}, used by L{HTTPConnectionPool}. - - @ivar _quiescentCallback: The quiescent callback to be passed to protocol - instances, used to return them to the connection pool. - - @since: 11.1 - """ - def __init__(self, quiescentCallback): - self._quiescentCallback = quiescentCallback - - - def buildProtocol(self, addr): - return HTTP11ClientProtocol(self._quiescentCallback) - - - -class _RetryingHTTP11ClientProtocol(object): - """ - A wrapper for L{HTTP11ClientProtocol} that automatically retries requests. - - @ivar _clientProtocol: The underlying L{HTTP11ClientProtocol}. - - @ivar _newConnection: A callable that creates a new connection for a - retry. - """ - - def __init__(self, clientProtocol, newConnection): - self._clientProtocol = clientProtocol - self._newConnection = newConnection - - - def _shouldRetry(self, method, exception, bodyProducer): - """ - Indicate whether request should be retried. - - Only returns C{True} if method is idempotent, no response was - received, the reason for the failed request was not due to - user-requested cancellation, and no body was sent. The latter - requirement may be relaxed in the future, and PUT added to approved - method list. - """ - if method not in ("GET", "HEAD", "OPTIONS", "DELETE", "TRACE"): - return False - if not isinstance(exception, (RequestNotSent, RequestTransmissionFailed, - ResponseNeverReceived)): - return False - if isinstance(exception, _WrapperException): - for failure in exception.reasons: - if failure.check(defer.CancelledError): - return False - if bodyProducer is not None: - return False - return True - - - def request(self, request): - """ - Do a request, and retry once (with a new connection) it it fails in - a retryable manner. - - @param request: A L{Request} instance that will be requested using the - wrapped protocol. - """ - d = self._clientProtocol.request(request) - - def failed(reason): - if self._shouldRetry(request.method, reason.value, - request.bodyProducer): - return self._newConnection().addCallback( - lambda connection: connection.request(request)) - else: - return reason - d.addErrback(failed) - return d - - - -class HTTPConnectionPool(object): - """ - A pool of persistent HTTP connections. - - Features: - - Cached connections will eventually time out. - - Limits on maximum number of persistent connections. - - Connections are stored using keys, which should be chosen such that any - connections stored under a given key can be used interchangeably. - - Failed requests done using previously cached connections will be retried - once if they use an idempotent method (e.g. GET), in case the HTTP server - timed them out. - - @ivar persistent: Boolean indicating whether connections should be - persistent. Connections are persistent by default. - - @ivar maxPersistentPerHost: The maximum number of cached persistent - connections for a C{host:port} destination. - @type maxPersistentPerHost: C{int} - - @ivar cachedConnectionTimeout: Number of seconds a cached persistent - connection will stay open before disconnecting. - - @ivar retryAutomatically: C{boolean} indicating whether idempotent - requests should be retried once if no response was received. - - @ivar _factory: The factory used to connect to the proxy. - - @ivar _connections: Map (scheme, host, port) to lists of - L{HTTP11ClientProtocol} instances. - - @ivar _timeouts: Map L{HTTP11ClientProtocol} instances to a - C{IDelayedCall} instance of their timeout. - - @since: 12.1 - """ - - _factory = _HTTP11ClientFactory - maxPersistentPerHost = 2 - cachedConnectionTimeout = 240 - retryAutomatically = True - - def __init__(self, reactor, persistent=True): - self._reactor = reactor - self.persistent = persistent - self._connections = {} - self._timeouts = {} - - - def getConnection(self, key, endpoint): - """ - Supply a connection, newly created or retrieved from the pool, to be - used for one HTTP request. - - The connection will remain out of the pool (not available to be - returned from future calls to this method) until one HTTP request has - been completed over it. - - Afterwards, if the connection is still open, it will automatically be - added to the pool. - - @param key: A unique key identifying connections that can be used - interchangeably. - - @param endpoint: An endpoint that can be used to open a new connection - if no cached connection is available. - - @return: A C{Deferred} that will fire with a L{HTTP11ClientProtocol} - (or a wrapper) that can be used to send a single HTTP request. - """ - # Try to get cached version: - connections = self._connections.get(key) - while connections: - connection = connections.pop(0) - # Cancel timeout: - self._timeouts[connection].cancel() - del self._timeouts[connection] - if connection.state == "QUIESCENT": - if self.retryAutomatically: - newConnection = lambda: self._newConnection(key, endpoint) - connection = _RetryingHTTP11ClientProtocol( - connection, newConnection) - return defer.succeed(connection) - - return self._newConnection(key, endpoint) - - - def _newConnection(self, key, endpoint): - """ - Create a new connection. - - This implements the new connection code path for L{getConnection}. - """ - def quiescentCallback(protocol): - self._putConnection(key, protocol) - factory = self._factory(quiescentCallback) - return endpoint.connect(factory) - - - def _removeConnection(self, key, connection): - """ - Remove a connection from the cache and disconnect it. - """ - connection.transport.loseConnection() - self._connections[key].remove(connection) - del self._timeouts[connection] - - - def _putConnection(self, key, connection): - """ - Return a persistent connection to the pool. This will be called by - L{HTTP11ClientProtocol} when the connection becomes quiescent. - """ - if connection.state != "QUIESCENT": - # Log with traceback for debugging purposes: - try: - raise RuntimeError( - "BUG: Non-quiescent protocol added to connection pool.") - except: - log.err() - return - connections = self._connections.setdefault(key, []) - if len(connections) == self.maxPersistentPerHost: - dropped = connections.pop(0) - dropped.transport.loseConnection() - self._timeouts[dropped].cancel() - del self._timeouts[dropped] - connections.append(connection) - cid = self._reactor.callLater(self.cachedConnectionTimeout, - self._removeConnection, - key, connection) - self._timeouts[connection] = cid - - - def closeCachedConnections(self): - """ - Close all persistent connections and remove them from the pool. - - @return: L{defer.Deferred} that fires when all connections have been - closed. - """ - results = [] - for protocols in self._connections.itervalues(): - for p in protocols: - results.append(p.abort()) - self._connections = {} - for dc in self._timeouts.values(): - dc.cancel() - self._timeouts = {} - return defer.gatherResults(results).addCallback(lambda ign: None) - - - -class _AgentBase(object): - """ - Base class offering common facilities for L{Agent}-type classes. - - @ivar _reactor: The C{IReactorTime} implementation which will be used by - the pool, and perhaps by subclasses as well. - - @ivar _pool: The L{HTTPConnectionPool} used to manage HTTP connections. - """ - - def __init__(self, reactor, pool): - if pool is None: - pool = HTTPConnectionPool(reactor, False) - self._reactor = reactor - self._pool = pool - - - def _computeHostValue(self, scheme, host, port): - """ - Compute the string to use for the value of the I{Host} header, based on - the given scheme, host name, and port number. - """ - if (scheme, port) in (('http', 80), ('https', 443)): - return host - return '%s:%d' % (host, port) - - - def _requestWithEndpoint(self, key, endpoint, method, parsedURI, - headers, bodyProducer, requestPath): - """ - Issue a new request, given the endpoint and the path sent as part of - the request. - """ - # Create minimal headers, if necessary: - if headers is None: - headers = Headers() - if not headers.hasHeader('host'): - #headers = headers.copy() # not supported in twisted <= 11.1, and it doesn't affects us - headers.addRawHeader( - 'host', self._computeHostValue(parsedURI.scheme, parsedURI.host, - parsedURI.port)) - - d = self._pool.getConnection(key, endpoint) - def cbConnected(proto): - return proto.request( - Request(method, requestPath, headers, bodyProducer, - persistent=self._pool.persistent)) - d.addCallback(cbConnected) - return d - - - -class Agent(_AgentBase): - """ - L{Agent} is a very basic HTTP client. It supports I{HTTP} and I{HTTPS} - scheme URIs (but performs no certificate checking by default). - - @param pool: A L{HTTPConnectionPool} instance, or C{None}, in which case a - non-persistent L{HTTPConnectionPool} instance will be created. - - @ivar _contextFactory: A web context factory which will be used to create - SSL context objects for any SSL connections the agent needs to make. - - @ivar _connectTimeout: If not C{None}, the timeout passed to C{connectTCP} - or C{connectSSL} for specifying the connection timeout. - - @ivar _bindAddress: If not C{None}, the address passed to C{connectTCP} or - C{connectSSL} for specifying the local address to bind to. - - @since: 9.0 - """ - - def __init__(self, reactor, contextFactory=WebClientContextFactory(), - connectTimeout=None, bindAddress=None, - pool=None): - _AgentBase.__init__(self, reactor, pool) - self._contextFactory = contextFactory - self._connectTimeout = connectTimeout - self._bindAddress = bindAddress - - - def _wrapContextFactory(self, host, port): - """ - Create and return a normal context factory wrapped around - C{self._contextFactory} in such a way that C{self._contextFactory} will - have the host and port information passed to it. - - @param host: A C{str} giving the hostname which will be connected to in - order to issue a request. - - @param port: An C{int} giving the port number the connection will be - on. - - @return: A context factory suitable to be passed to - C{reactor.connectSSL}. - """ - return _WebToNormalContextFactory(self._contextFactory, host, port) - - - def _getEndpoint(self, scheme, host, port): - """ - Get an endpoint for the given host and port, using a transport - selected based on scheme. - - @param scheme: A string like C{'http'} or C{'https'} (the only two - supported values) to use to determine how to establish the - connection. - - @param host: A C{str} giving the hostname which will be connected to in - order to issue a request. - - @param port: An C{int} giving the port number the connection will be - on. - - @return: An endpoint which can be used to connect to given address. - """ - kwargs = {} - if self._connectTimeout is not None: - kwargs['timeout'] = self._connectTimeout - kwargs['bindAddress'] = self._bindAddress - if scheme == 'http': - return TCP4ClientEndpoint(self._reactor, host, port, **kwargs) - elif scheme == 'https': - return SSL4ClientEndpoint(self._reactor, host, port, - self._wrapContextFactory(host, port), - **kwargs) - else: - raise SchemeNotSupported("Unsupported scheme: %r" % (scheme,)) - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Issue a new request. - - @param method: The request method to send. - @type method: C{str} - - @param uri: The request URI send. - @type uri: C{str} - - @param headers: The request headers to send. If no I{Host} header is - included, one will be added based on the request URI. - @type headers: L{Headers} - - @param bodyProducer: An object which will produce the request body or, - if the request body is to be empty, L{None}. - @type bodyProducer: L{IBodyProducer} provider - - @return: A L{Deferred} which fires with the result of the request (a - L{twisted.web.iweb.IResponse} provider), or fails if there is a - problem setting up a connection over which to issue the request. - It may also fail with L{SchemeNotSupported} if the scheme of the - given URI is not supported. - @rtype: L{Deferred} - """ - parsedURI = _parse(uri) - try: - endpoint = self._getEndpoint(parsedURI.scheme, parsedURI.host, - parsedURI.port) - except SchemeNotSupported: - return defer.fail(Failure()) - key = (parsedURI.scheme, parsedURI.host, parsedURI.port) - return self._requestWithEndpoint(key, endpoint, method, parsedURI, - headers, bodyProducer, parsedURI.path) - - - -class ProxyAgent(_AgentBase): - """ - An HTTP agent able to cross HTTP proxies. - - @ivar _proxyEndpoint: The endpoint used to connect to the proxy. - - @since: 11.1 - """ - - def __init__(self, endpoint, reactor=None, pool=None): - if reactor is None: - from twisted.internet import reactor - _AgentBase.__init__(self, reactor, pool) - self._proxyEndpoint = endpoint - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Issue a new request via the configured proxy. - """ - # Cache *all* connections under the same key, since we are only - # connecting to a single destination, the proxy: - key = ("http-proxy", self._proxyEndpoint) - - # To support proxying HTTPS via CONNECT, we will use key - # ("http-proxy-CONNECT", scheme, host, port), and an endpoint that - # wraps _proxyEndpoint with an additional callback to do the CONNECT. - return self._requestWithEndpoint(key, self._proxyEndpoint, method, - _parse(uri), headers, bodyProducer, - uri) - - - -class _FakeUrllib2Request(object): - """ - A fake C{urllib2.Request} object for C{cookielib} to work with. - - @see: U{http://docs.python.org/library/urllib2.html#request-objects} - - @type uri: C{str} - @ivar uri: Request URI. - - @type headers: L{twisted.web.http_headers.Headers} - @ivar headers: Request headers. - - @type type: C{str} - @ivar type: The scheme of the URI. - - @type host: C{str} - @ivar host: The host[:port] of the URI. - - @since: 11.1 - """ - def __init__(self, uri): - self.uri = uri - self.headers = Headers() - self.type, rest = splittype(self.uri) - self.host, rest = splithost(rest) - - - def has_header(self, header): - return self.headers.hasHeader(header) - - - def add_unredirected_header(self, name, value): - self.headers.addRawHeader(name, value) - - - def get_full_url(self): - return self.uri - - - def get_header(self, name, default=None): - headers = self.headers.getRawHeaders(name, default) - if headers is not None: - return headers[0] - return None - - - def get_host(self): - return self.host - - - def get_type(self): - return self.type - - - def is_unverifiable(self): - # In theory this shouldn't be hardcoded. - return False - - - -class _FakeUrllib2Response(object): - """ - A fake C{urllib2.Response} object for C{cookielib} to work with. - - @type response: C{twisted.web.iweb.IResponse} - @ivar response: Underlying Twisted Web response. - - @since: 11.1 - """ - def __init__(self, response): - self.response = response - - - def info(self): - class _Meta(object): - def getheaders(zelf, name): - return self.response.headers.getRawHeaders(name, []) - return _Meta() - - - -class CookieAgent(object): - """ - L{CookieAgent} extends the basic L{Agent} to add RFC-compliant - handling of HTTP cookies. Cookies are written to and extracted - from a C{cookielib.CookieJar} instance. - - The same cookie jar instance will be used for any requests through this - agent, mutating it whenever a I{Set-Cookie} header appears in a response. - - @type _agent: L{twisted.web.client.Agent} - @ivar _agent: Underlying Twisted Web agent to issue requests through. - - @type cookieJar: C{cookielib.CookieJar} - @ivar cookieJar: Initialized cookie jar to read cookies from and store - cookies to. - - @since: 11.1 - """ - def __init__(self, agent, cookieJar): - self._agent = agent - self.cookieJar = cookieJar - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Issue a new request to the wrapped L{Agent}. - - Send a I{Cookie} header if a cookie for C{uri} is stored in - L{CookieAgent.cookieJar}. Cookies are automatically extracted and - stored from requests. - - If a C{'cookie'} header appears in C{headers} it will override the - automatic cookie header obtained from the cookie jar. - - @see: L{Agent.request} - """ - if headers is None: - headers = Headers() - lastRequest = _FakeUrllib2Request(uri) - # Setting a cookie header explicitly will disable automatic request - # cookies. - if not headers.hasHeader('cookie'): - self.cookieJar.add_cookie_header(lastRequest) - cookieHeader = lastRequest.get_header('Cookie', None) - if cookieHeader is not None: - headers = headers.copy() - headers.addRawHeader('cookie', cookieHeader) - - d = self._agent.request(method, uri, headers, bodyProducer) - d.addCallback(self._extractCookies, lastRequest) - return d - - - def _extractCookies(self, response, request): - """ - Extract response cookies and store them in the cookie jar. - - @type response: L{twisted.web.iweb.IResponse} - @param response: Twisted Web response. - - @param request: A urllib2 compatible request object. - """ - resp = _FakeUrllib2Response(response) - self.cookieJar.extract_cookies(resp, request) - return response - - - -class GzipDecoder(proxyForInterface(IResponse)): - """ - A wrapper for a L{Response} instance which handles gzip'ed body. - - @ivar original: The original L{Response} object. - - @since: 11.1 - """ - - def __init__(self, response): - self.original = response - self.length = UNKNOWN_LENGTH - - - def deliverBody(self, protocol): - """ - Override C{deliverBody} to wrap the given C{protocol} with - L{_GzipProtocol}. - """ - self.original.deliverBody(_GzipProtocol(protocol, self.original)) - - - -class _GzipProtocol(proxyForInterface(IProtocol)): - """ - A L{Protocol} implementation which wraps another one, transparently - decompressing received data. - - @ivar _zlibDecompress: A zlib decompress object used to decompress the data - stream. - - @ivar _response: A reference to the original response, in case of errors. - - @since: 11.1 - """ - - def __init__(self, protocol, response): - self.original = protocol - self._response = response - self._zlibDecompress = zlib.decompressobj(16 + zlib.MAX_WBITS) - - - def dataReceived(self, data): - """ - Decompress C{data} with the zlib decompressor, forwarding the raw data - to the original protocol. - """ - try: - rawData = self._zlibDecompress.decompress(data) - except zlib.error: - raise ResponseFailed([failure.Failure()], self._response) - if rawData: - self.original.dataReceived(rawData) - - - def connectionLost(self, reason): - """ - Forward the connection lost event, flushing remaining data from the - decompressor if any. - """ - try: - rawData = self._zlibDecompress.flush() - except zlib.error: - raise ResponseFailed([reason, failure.Failure()], self._response) - if rawData: - self.original.dataReceived(rawData) - self.original.connectionLost(reason) - - - -class ContentDecoderAgent(object): - """ - An L{Agent} wrapper to handle encoded content. - - It takes care of declaring the support for content in the - I{Accept-Encoding} header, and automatically decompresses the received data - if it's effectively using compression. - - @param decoders: A list or tuple of (name, decoder) objects. The name - declares which decoding the decoder supports, and the decoder must - return a response object when called/instantiated. For example, - C{(('gzip', GzipDecoder))}. The order determines how the decoders are - going to be advertized to the server. - - @since: 11.1 - """ - - def __init__(self, agent, decoders): - self._agent = agent - self._decoders = dict(decoders) - self._supported = ','.join([decoder[0] for decoder in decoders]) - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Send a client request which declares supporting compressed content. - - @see: L{Agent.request}. - """ - if headers is None: - headers = Headers() - else: - headers = headers.copy() - headers.addRawHeader('accept-encoding', self._supported) - deferred = self._agent.request(method, uri, headers, bodyProducer) - return deferred.addCallback(self._handleResponse) - - - def _handleResponse(self, response): - """ - Check if the response is encoded, and wrap it to handle decompression. - """ - contentEncodingHeaders = response.headers.getRawHeaders( - 'content-encoding', []) - contentEncodingHeaders = ','.join(contentEncodingHeaders).split(',') - while contentEncodingHeaders: - name = contentEncodingHeaders.pop().strip() - decoder = self._decoders.get(name) - if decoder is not None: - response = decoder(response) - else: - # Add it back - contentEncodingHeaders.append(name) - break - if contentEncodingHeaders: - response.headers.setRawHeaders( - 'content-encoding', [','.join(contentEncodingHeaders)]) - else: - response.headers.removeHeader('content-encoding') - return response - - - -class RedirectAgent(object): - """ - An L{Agent} wrapper which handles HTTP redirects. - - The implementation is rather strict: 301 and 302 behaves like 307, not - redirecting automatically on methods different from C{GET} and C{HEAD}. - - @param redirectLimit: The maximum number of times the agent is allowed to - follow redirects before failing with a L{error.InfiniteRedirection}. - - @since: 11.1 - """ - - def __init__(self, agent, redirectLimit=20): - self._agent = agent - self._redirectLimit = redirectLimit - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Send a client request following HTTP redirects. - - @see: L{Agent.request}. - """ - deferred = self._agent.request(method, uri, headers, bodyProducer) - return deferred.addCallback( - self._handleResponse, method, uri, headers, 0) - - - def _handleRedirect(self, response, method, uri, headers, redirectCount): - """ - Handle a redirect response, checking the number of redirects already - followed, and extracting the location header fields. - """ - if redirectCount >= self._redirectLimit: - err = error.InfiniteRedirection( - response.code, - 'Infinite redirection detected', - location=uri) - raise ResponseFailed([failure.Failure(err)], response) - locationHeaders = response.headers.getRawHeaders('location', []) - if not locationHeaders: - err = error.RedirectWithNoLocation( - response.code, 'No location header field', uri) - raise ResponseFailed([failure.Failure(err)], response) - location = locationHeaders[0] - deferred = self._agent.request(method, location, headers) - return deferred.addCallback( - self._handleResponse, method, uri, headers, redirectCount + 1) - - - def _handleResponse(self, response, method, uri, headers, redirectCount): - """ - Handle the response, making another request if it indicates a redirect. - """ - if response.code in (http.MOVED_PERMANENTLY, http.FOUND, - http.TEMPORARY_REDIRECT): - if method not in ('GET', 'HEAD'): - err = error.PageRedirect(response.code, location=uri) - raise ResponseFailed([failure.Failure(err)], response) - return self._handleRedirect(response, method, uri, headers, - redirectCount) - elif response.code == http.SEE_OTHER: - return self._handleRedirect(response, 'GET', uri, headers, - redirectCount) - return response - - - -class _ReadBodyProtocol(protocol.Protocol): - """ - Protocol that collects data sent to it. - - This is a helper for L{IResponse.deliverBody}, which collects the body and - fires a deferred with it. - - @ivar deferred: See L{__init__}. - @ivar status: See L{__init__}. - @ivar message: See L{__init__}. - - @ivar dataBuffer: list of byte-strings received - @type dataBuffer: L{list} of L{bytes} - """ - - def __init__(self, status, message, deferred): - """ - @param status: Status of L{IResponse} - @ivar status: L{int} - - @param message: Message of L{IResponse} - @type message: L{bytes} - - @param deferred: deferred to fire when response is complete - @type deferred: L{Deferred} firing with L{bytes} - """ - self.deferred = deferred - self.status = status - self.message = message - self.dataBuffer = [] - - - def dataReceived(self, data): - """ - Accumulate some more bytes from the response. - """ - self.dataBuffer.append(data) - - - def connectionLost(self, reason): - """ - Deliver the accumulated response bytes to the waiting L{Deferred}, if - the response body has been completely received without error. - """ - if reason.check(ResponseDone): - self.deferred.callback(b''.join(self.dataBuffer)) - elif reason.check(PotentialDataLoss): - self.deferred.errback( - PartialDownloadError(self.status, self.message, - b''.join(self.dataBuffer))) - else: - self.deferred.errback(reason) - - - -def readBody(response): - """ - Get the body of an L{IResponse} and return it as a byte string. - - This is a helper function for clients that don't want to incrementally - receive the body of an HTTP response. - - @param response: The HTTP response for which the body will be read. - @type response: L{IResponse} provider - - @return: A L{Deferred} which will fire with the body of the response. - """ - d = defer.Deferred() - response.deliverBody(_ReadBodyProtocol(response.code, response.phrase, d)) - return d -}}} ''' __all__ = [ diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py index 3f4704064..197e43ed3 100644 --- a/scrapy/xlib/tx/endpoints.py +++ b/scrapy/xlib/tx/endpoints.py @@ -14,27 +14,6 @@ parsed by the L{clientFromString} and L{serverFromString} functions. from __future__ import division, absolute_import -import os -#import socket - -from zope.interface import implementer, directlyProvides -import warnings - -from twisted.internet import interfaces, defer, error, fdesc -#from twisted.internet.protocol import ( -# ClientFactory, Protocol) -from twisted.internet.protocol import Factory -#from twisted.internet import threads, ProcessProtocol -from twisted.internet.interfaces import IStreamServerEndpointStringParser -#from twisted.internet.interfaces import IStreamClientEndpointStringParser -from twisted.python.filepath import FilePath -#from twisted.python.failure import Failure -#from twisted.python import log -#from twisted.python.components import proxyForInterface - -from twisted.plugin import IPlugin, getPlugins -#from twisted.internet import stdio - from twisted.internet.endpoints import ( clientFromString, serverFromString, quoteStringArgument, TCP4ServerEndpoint, TCP6ServerEndpoint, @@ -44,1236 +23,4 @@ from twisted.internet.endpoints import ( AdoptedStreamServerEndpoint, connectProtocol, ) - __all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"] - -''' {{{ -class _WrappingProtocol(Protocol): - """ - Wrap another protocol in order to notify my user when a connection has - been made. - """ - - def __init__(self, connectedDeferred, wrappedProtocol): - """ - @param connectedDeferred: The L{Deferred} that will callback - with the C{wrappedProtocol} when it is connected. - - @param wrappedProtocol: An L{IProtocol} provider that will be - connected. - """ - self._connectedDeferred = connectedDeferred - self._wrappedProtocol = wrappedProtocol - - for iface in [interfaces.IHalfCloseableProtocol, - interfaces.IFileDescriptorReceiver]: - if iface.providedBy(self._wrappedProtocol): - directlyProvides(self, iface) - - - def logPrefix(self): - """ - Transparently pass through the wrapped protocol's log prefix. - """ - if interfaces.ILoggingContext.providedBy(self._wrappedProtocol): - return self._wrappedProtocol.logPrefix() - return self._wrappedProtocol.__class__.__name__ - - - def connectionMade(self): - """ - Connect the C{self._wrappedProtocol} to our C{self.transport} and - callback C{self._connectedDeferred} with the C{self._wrappedProtocol} - """ - self._wrappedProtocol.makeConnection(self.transport) - self._connectedDeferred.callback(self._wrappedProtocol) - - - def dataReceived(self, data): - """ - Proxy C{dataReceived} calls to our C{self._wrappedProtocol} - """ - return self._wrappedProtocol.dataReceived(data) - - - def fileDescriptorReceived(self, descriptor): - """ - Proxy C{fileDescriptorReceived} calls to our C{self._wrappedProtocol} - """ - return self._wrappedProtocol.fileDescriptorReceived(descriptor) - - - def connectionLost(self, reason): - """ - Proxy C{connectionLost} calls to our C{self._wrappedProtocol} - """ - return self._wrappedProtocol.connectionLost(reason) - - - def readConnectionLost(self): - """ - Proxy L{IHalfCloseableProtocol.readConnectionLost} to our - C{self._wrappedProtocol} - """ - self._wrappedProtocol.readConnectionLost() - - - def writeConnectionLost(self): - """ - Proxy L{IHalfCloseableProtocol.writeConnectionLost} to our - C{self._wrappedProtocol} - """ - self._wrappedProtocol.writeConnectionLost() - - - -class _WrappingFactory(ClientFactory): - """ - Wrap a factory in order to wrap the protocols it builds. - - @ivar _wrappedFactory: A provider of I{IProtocolFactory} whose buildProtocol - method will be called and whose resulting protocol will be wrapped. - - @ivar _onConnection: A L{Deferred} that fires when the protocol is - connected - - @ivar _connector: A L{connector } - that is managing the current or previous connection attempt. - """ - protocol = _WrappingProtocol - - def __init__(self, wrappedFactory): - """ - @param wrappedFactory: A provider of I{IProtocolFactory} whose - buildProtocol method will be called and whose resulting protocol - will be wrapped. - """ - self._wrappedFactory = wrappedFactory - self._onConnection = defer.Deferred(canceller=self._canceller) - - - def startedConnecting(self, connector): - """ - A connection attempt was started. Remember the connector which started - said attempt, for use later. - """ - self._connector = connector - - - def _canceller(self, deferred): - """ - The outgoing connection attempt was cancelled. Fail that L{Deferred} - with an L{error.ConnectingCancelledError}. - - @param deferred: The L{Deferred } that was cancelled; - should be the same as C{self._onConnection}. - @type deferred: L{Deferred } - - @note: This relies on startedConnecting having been called, so it may - seem as though there's a race condition where C{_connector} may not - have been set. However, using public APIs, this condition is - impossible to catch, because a connection API - (C{connectTCP}/C{SSL}/C{UNIX}) is always invoked before a - L{_WrappingFactory}'s L{Deferred } is returned to - C{connect()}'s caller. - - @return: C{None} - """ - deferred.errback( - error.ConnectingCancelledError( - self._connector.getDestination())) - self._connector.stopConnecting() - - - def doStart(self): - """ - Start notifications are passed straight through to the wrapped factory. - """ - self._wrappedFactory.doStart() - - - def doStop(self): - """ - Stop notifications are passed straight through to the wrapped factory. - """ - self._wrappedFactory.doStop() - - - def buildProtocol(self, addr): - """ - Proxy C{buildProtocol} to our C{self._wrappedFactory} or errback - the C{self._onConnection} L{Deferred}. - - @return: An instance of L{_WrappingProtocol} or C{None} - """ - try: - proto = self._wrappedFactory.buildProtocol(addr) - except: - self._onConnection.errback() - else: - return self.protocol(self._onConnection, proto) - - - def clientConnectionFailed(self, connector, reason): - """ - Errback the C{self._onConnection} L{Deferred} when the - client connection fails. - """ - if not self._onConnection.called: - self._onConnection.errback(reason) - - - - - -@implementer(interfaces.ITransport) -class _ProcessEndpointTransport(proxyForInterface( - interfaces.IProcessTransport, '_process')): - """ - An L{ITransport} provider for the L{IProtocol} instance passed to the - process endpoint. - - @ivar _process: An active process transport which will be used by write - methods on this object to write data to a child process. - @type _process: L{interfaces.IProcessTransport} provider - """ - - def write(self, data): - """ - Write to the child process's standard input. - - @param data: The data to write on stdin. - """ - self._process.writeToChild(0, data) - - - def writeSequence(self, data): - """ - Write a list of strings to child process's stdin. - - @param data: The list of chunks to write on stdin. - """ - for chunk in data: - self._process.writeToChild(0, chunk) - - -@implementer(interfaces.IStreamServerEndpoint) -class _TCPServerEndpoint(object): - """ - A TCP server endpoint interface - """ - - def __init__(self, reactor, port, backlog, interface): - """ - @param reactor: An L{IReactorTCP} provider. - - @param port: The port number used for listening - @type port: int - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to - @type interface: str - """ - self._reactor = reactor - self._port = port - self._backlog = backlog - self._interface = interface - - - def listen(self, protocolFactory): - """ - Implement L{IStreamServerEndpoint.listen} to listen on a TCP - socket - """ - return defer.execute(self._reactor.listenTCP, - self._port, - protocolFactory, - backlog=self._backlog, - interface=self._interface) - - - -class TCP4ServerEndpoint(_TCPServerEndpoint): - """ - Implements TCP server endpoint with an IPv4 configuration - """ - def __init__(self, reactor, port, backlog=50, interface=''): - """ - @param reactor: An L{IReactorTCP} provider. - - @param port: The port number used for listening - @type port: int - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to, defaults to '' (all) - @type interface: str - """ - _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface) - - - -class TCP6ServerEndpoint(_TCPServerEndpoint): - """ - Implements TCP server endpoint with an IPv6 configuration - """ - def __init__(self, reactor, port, backlog=50, interface='::'): - """ - @param reactor: An L{IReactorTCP} provider. - - @param port: The port number used for listening - @type port: int - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to, defaults to '' (all) - @type interface: str - """ - _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface) - - - -@implementer(interfaces.IStreamClientEndpoint) -class TCP4ClientEndpoint(object): - """ - TCP client endpoint with an IPv4 configuration. - """ - - def __init__(self, reactor, host, port, timeout=30, bindAddress=None): - """ - @param reactor: An L{IReactorTCP} provider - - @param host: A hostname, used when connecting - @type host: str - - @param port: The port number, used when connecting - @type port: int - - @param timeout: The number of seconds to wait before assuming the - connection has failed. - @type timeout: int - - @param bindAddress: A (host, port) tuple of local address to bind to, - or None. - @type bindAddress: tuple - """ - self._reactor = reactor - self._host = host - self._port = port - self._timeout = timeout - self._bindAddress = bindAddress - - - def connect(self, protocolFactory): - """ - Implement L{IStreamClientEndpoint.connect} to connect via TCP. - """ - try: - wf = _WrappingFactory(protocolFactory) - self._reactor.connectTCP( - self._host, self._port, wf, - timeout=self._timeout, bindAddress=self._bindAddress) - return wf._onConnection - except: - return defer.fail() - - - - -@implementer(interfaces.IStreamServerEndpoint) -class SSL4ServerEndpoint(object): - """ - SSL secured TCP server endpoint with an IPv4 configuration. - """ - - def __init__(self, reactor, port, sslContextFactory, - backlog=50, interface=''): - """ - @param reactor: An L{IReactorSSL} provider. - - @param port: The port number used for listening - @type port: int - - @param sslContextFactory: An instance of - L{twisted.internet.ssl.ContextFactory}. - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to, defaults to '' (all) - @type interface: str - """ - self._reactor = reactor - self._port = port - self._sslContextFactory = sslContextFactory - self._backlog = backlog - self._interface = interface - - - def listen(self, protocolFactory): - """ - Implement L{IStreamServerEndpoint.listen} to listen for SSL on a - TCP socket. - """ - return defer.execute(self._reactor.listenSSL, self._port, - protocolFactory, - contextFactory=self._sslContextFactory, - backlog=self._backlog, - interface=self._interface) - - - -@implementer(interfaces.IStreamClientEndpoint) -class SSL4ClientEndpoint(object): - """ - SSL secured TCP client endpoint with an IPv4 configuration - """ - - def __init__(self, reactor, host, port, sslContextFactory, - timeout=30, bindAddress=None): - """ - @param reactor: An L{IReactorSSL} provider. - - @param host: A hostname, used when connecting - @type host: str - - @param port: The port number, used when connecting - @type port: int - - @param sslContextFactory: SSL Configuration information as an instance - of L{twisted.internet.ssl.ContextFactory}. - - @param timeout: Number of seconds to wait before assuming the - connection has failed. - @type timeout: int - - @param bindAddress: A (host, port) tuple of local address to bind to, - or None. - @type bindAddress: tuple - """ - self._reactor = reactor - self._host = host - self._port = port - self._sslContextFactory = sslContextFactory - self._timeout = timeout - self._bindAddress = bindAddress - - - def connect(self, protocolFactory): - """ - Implement L{IStreamClientEndpoint.connect} to connect with SSL over - TCP. - """ - try: - wf = _WrappingFactory(protocolFactory) - self._reactor.connectSSL( - self._host, self._port, wf, self._sslContextFactory, - timeout=self._timeout, bindAddress=self._bindAddress) - return wf._onConnection - except: - return defer.fail() - - - -@implementer(interfaces.IStreamServerEndpoint) -class UNIXServerEndpoint(object): - """ - UnixSocket server endpoint. - """ - def __init__(self, reactor, address, backlog=50, mode=0o666, wantPID=0): - """ - @param reactor: An L{IReactorUNIX} provider. - @param address: The path to the Unix socket file, used when listening - @param backlog: number of connections to allow in backlog. - @param mode: mode to set on the unix socket. This parameter is - deprecated. Permissions should be set on the directory which - contains the UNIX socket. - @param wantPID: If True, create a pidfile for the socket. - """ - self._reactor = reactor - self._address = address - self._backlog = backlog - self._mode = mode - self._wantPID = wantPID - - - def listen(self, protocolFactory): - """ - Implement L{IStreamServerEndpoint.listen} to listen on a UNIX socket. - """ - return defer.execute(self._reactor.listenUNIX, self._address, - protocolFactory, - backlog=self._backlog, - mode=self._mode, - wantPID=self._wantPID) - - - -@implementer(interfaces.IStreamClientEndpoint) -class UNIXClientEndpoint(object): - """ - UnixSocket client endpoint. - """ - def __init__(self, reactor, path, timeout=30, checkPID=0): - """ - @param reactor: An L{IReactorUNIX} provider. - - @param path: The path to the Unix socket file, used when connecting - @type path: str - - @param timeout: Number of seconds to wait before assuming the - connection has failed. - @type timeout: int - - @param checkPID: If True, check for a pid file to verify that a server - is listening. - @type checkPID: bool - """ - self._reactor = reactor - self._path = path - self._timeout = timeout - self._checkPID = checkPID - - - def connect(self, protocolFactory): - """ - Implement L{IStreamClientEndpoint.connect} to connect via a - UNIX Socket - """ - try: - wf = _WrappingFactory(protocolFactory) - self._reactor.connectUNIX( - self._path, wf, - timeout=self._timeout, - checkPID=self._checkPID) - return wf._onConnection - except: - return defer.fail() - - - -@implementer(interfaces.IStreamServerEndpoint) -class AdoptedStreamServerEndpoint(object): - """ - An endpoint for listening on a file descriptor initialized outside of - Twisted. - - @ivar _used: A C{bool} indicating whether this endpoint has been used to - listen with a factory yet. C{True} if so. - """ - _close = os.close - _setNonBlocking = staticmethod(fdesc.setNonBlocking) - - def __init__(self, reactor, fileno, addressFamily): - """ - @param reactor: An L{IReactorSocket} provider. - - @param fileno: An integer file descriptor corresponding to a listening - I{SOCK_STREAM} socket. - - @param addressFamily: The address family of the socket given by - C{fileno}. - """ - self.reactor = reactor - self.fileno = fileno - self.addressFamily = addressFamily - self._used = False - - - def listen(self, factory): - """ - Implement L{IStreamServerEndpoint.listen} to start listening on, and - then close, C{self._fileno}. - """ - if self._used: - return defer.fail(error.AlreadyListened()) - self._used = True - - try: - self._setNonBlocking(self.fileno) - port = self.reactor.adoptStreamPort( - self.fileno, self.addressFamily, factory) - self._close(self.fileno) - except: - return defer.fail() - return defer.succeed(port) - - - - -def _parseTCP(factory, port, interface="", backlog=50): - """ - Internal parser function for L{_parseServer} to convert the string - arguments for a TCP(IPv4) stream endpoint into the structured arguments. - - @param factory: the protocol factory being parsed, or C{None}. (This was a - leftover argument from when this code was in C{strports}, and is now - mostly None and unused.) - - @type factory: L{IProtocolFactory} or C{NoneType} - - @param port: the integer port number to bind - @type port: C{str} - - @param interface: the interface IP to listen on - @param backlog: the length of the listen queue - @type backlog: C{str} - - @return: a 2-tuple of (args, kwargs), describing the parameters to - L{IReactorTCP.listenTCP} (or, modulo argument 2, the factory, arguments - to L{TCP4ServerEndpoint}. - """ - return (int(port), factory), {'interface': interface, - 'backlog': int(backlog)} - - - -def _parseUNIX(factory, address, mode='666', backlog=50, lockfile=True): - """ - Internal parser function for L{_parseServer} to convert the string - arguments for a UNIX (AF_UNIX/SOCK_STREAM) stream endpoint into the - structured arguments. - - @param factory: the protocol factory being parsed, or C{None}. (This was a - leftover argument from when this code was in C{strports}, and is now - mostly None and unused.) - - @type factory: L{IProtocolFactory} or C{NoneType} - - @param address: the pathname of the unix socket - @type address: C{str} - - @param backlog: the length of the listen queue - @type backlog: C{str} - - @param lockfile: A string '0' or '1', mapping to True and False - respectively. See the C{wantPID} argument to C{listenUNIX} - - @return: a 2-tuple of (args, kwargs), describing the parameters to - L{IReactorTCP.listenUNIX} (or, modulo argument 2, the factory, - arguments to L{UNIXServerEndpoint}. - """ - return ( - (address, factory), - {'mode': int(mode, 8), 'backlog': int(backlog), - 'wantPID': bool(int(lockfile))}) - - - -def _parseSSL(factory, port, privateKey="server.pem", certKey=None, - sslmethod=None, interface='', backlog=50): - """ - Internal parser function for L{_parseServer} to convert the string - arguments for an SSL (over TCP/IPv4) stream endpoint into the structured - arguments. - - @param factory: the protocol factory being parsed, or C{None}. (This was a - leftover argument from when this code was in C{strports}, and is now - mostly None and unused.) - @type factory: L{IProtocolFactory} or C{NoneType} - - @param port: the integer port number to bind - @type port: C{str} - - @param interface: the interface IP to listen on - @param backlog: the length of the listen queue - @type backlog: C{str} - - @param privateKey: The file name of a PEM format private key file. - @type privateKey: C{str} - - @param certKey: The file name of a PEM format certificate file. - @type certKey: C{str} - - @param sslmethod: The string name of an SSL method, based on the name of a - constant in C{OpenSSL.SSL}. Must be one of: "SSLv23_METHOD", - "SSLv2_METHOD", "SSLv3_METHOD", "TLSv1_METHOD". - @type sslmethod: C{str} - - @return: a 2-tuple of (args, kwargs), describing the parameters to - L{IReactorSSL.listenSSL} (or, modulo argument 2, the factory, arguments - to L{SSL4ServerEndpoint}. - """ - from twisted.internet import ssl - if certKey is None: - certKey = privateKey - kw = {} - if sslmethod is not None: - kw['method'] = getattr(ssl.SSL, sslmethod) - else: - kw['method'] = ssl.SSL.SSLv23_METHOD - certPEM = FilePath(certKey).getContent() - keyPEM = FilePath(privateKey).getContent() - privateCertificate = ssl.PrivateCertificate.loadPEM(certPEM + keyPEM) - cf = ssl.CertificateOptions( - privateKey=privateCertificate.privateKey.original, - certificate=privateCertificate.original, - **kw - ) - return ((int(port), factory, cf), - {'interface': interface, 'backlog': int(backlog)}) - - - -@implementer(IPlugin, IStreamServerEndpointStringParser) -class _StandardIOParser(object): - """ - Stream server endpoint string parser for the Standard I/O type. - - @ivar prefix: See L{IStreamClientEndpointStringParser.prefix}. - """ - prefix = "stdio" - - def _parseServer(self, reactor): - """ - Internal parser function for L{_parseServer} to convert the string - arguments into structured arguments for the L{StandardIOEndpoint} - - @param reactor: Reactor for the endpoint - """ - return StandardIOEndpoint(reactor) - - - def parseStreamServer(self, reactor, *args, **kwargs): - # Redirects to another function (self._parseServer), tricks zope.interface - # into believing the interface is correctly implemented. - return self._parseServer(reactor) - - - - -@implementer(IPlugin, IStreamServerEndpointStringParser) -class _TCP6ServerParser(object): - """ - Stream server endpoint string parser for the TCP6ServerEndpoint type. - - @ivar prefix: See L{IStreamClientEndpointStringParser.prefix}. - """ - prefix = "tcp6" # Used in _parseServer to identify the plugin with the endpoint type - - def _parseServer(self, reactor, port, backlog=50, interface='::'): - """ - Internal parser function for L{_parseServer} to convert the string - arguments into structured arguments for the L{TCP6ServerEndpoint} - - @param reactor: An L{IReactorTCP} provider. - - @param port: The port number used for listening - @type port: int - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to - @type interface: str - """ - port = int(port) - backlog = int(backlog) - return TCP6ServerEndpoint(reactor, port, backlog, interface) - - - def parseStreamServer(self, reactor, *args, **kwargs): - # Redirects to another function (self._parseServer), tricks zope.interface - # into believing the interface is correctly implemented. - return self._parseServer(reactor, *args, **kwargs) - - - -_serverParsers = {"tcp": _parseTCP, - "unix": _parseUNIX, - "ssl": _parseSSL, - } - -_OP, _STRING = range(2) - -def _tokenize(description): - """ - Tokenize a strports string and yield each token. - - @param description: a string as described by L{serverFromString} or - L{clientFromString}. - - @return: an iterable of 2-tuples of (L{_OP} or L{_STRING}, string). Tuples - starting with L{_OP} will contain a second element of either ':' (i.e. - 'next parameter') or '=' (i.e. 'assign parameter value'). For example, - the string 'hello:greet\=ing=world' would result in a generator - yielding these values:: - - _STRING, 'hello' - _OP, ':' - _STRING, 'greet=ing' - _OP, '=' - _STRING, 'world' - """ - current = '' - ops = ':=' - nextOps = {':': ':=', '=': ':'} - description = iter(description) - for n in description: - if n in ops: - yield _STRING, current - yield _OP, n - current = '' - ops = nextOps[n] - elif n == '\\': - current += next(description) - else: - current += n - yield _STRING, current - - - -def _parse(description): - """ - Convert a description string into a list of positional and keyword - parameters, using logic vaguely like what Python does. - - @param description: a string as described by L{serverFromString} or - L{clientFromString}. - - @return: a 2-tuple of C{(args, kwargs)}, where 'args' is a list of all - ':'-separated C{str}s not containing an '=' and 'kwargs' is a map of - all C{str}s which do contain an '='. For example, the result of - C{_parse('a:b:d=1:c')} would be C{(['a', 'b', 'c'], {'d': '1'})}. - """ - args, kw = [], {} - def add(sofar): - if len(sofar) == 1: - args.append(sofar[0]) - else: - kw[sofar[0]] = sofar[1] - sofar = () - for (type, value) in _tokenize(description): - if type is _STRING: - sofar += (value,) - elif value == ':': - add(sofar) - sofar = () - add(sofar) - return args, kw - - -# Mappings from description "names" to endpoint constructors. -_endpointServerFactories = { - 'TCP': TCP4ServerEndpoint, - 'SSL': SSL4ServerEndpoint, - 'UNIX': UNIXServerEndpoint, - } - -_endpointClientFactories = { - 'TCP': TCP4ClientEndpoint, - 'SSL': SSL4ClientEndpoint, - 'UNIX': UNIXClientEndpoint, - } - - -_NO_DEFAULT = object() - -def _parseServer(description, factory, default=None): - """ - Parse a stports description into a 2-tuple of arguments and keyword values. - - @param description: A description in the format explained by - L{serverFromString}. - @type description: C{str} - - @param factory: A 'factory' argument; this is left-over from - twisted.application.strports, it's not really used. - @type factory: L{IProtocolFactory} or L{None} - - @param default: Deprecated argument, specifying the default parser mode to - use for unqualified description strings (those which do not have a ':' - and prefix). - @type default: C{str} or C{NoneType} - - @return: a 3-tuple of (plugin or name, arguments, keyword arguments) - """ - args, kw = _parse(description) - if not args or (len(args) == 1 and not kw): - deprecationMessage = ( - "Unqualified strport description passed to 'service'." - "Use qualified endpoint descriptions; for example, 'tcp:%s'." - % (description,)) - if default is None: - default = 'tcp' - warnings.warn( - deprecationMessage, category=DeprecationWarning, stacklevel=4) - elif default is _NO_DEFAULT: - raise ValueError(deprecationMessage) - # If the default has been otherwise specified, the user has already - # been warned. - args[0:0] = [default] - endpointType = args[0] - parser = _serverParsers.get(endpointType) - if parser is None: - # If the required parser is not found in _server, check if - # a plugin exists for the endpointType - for plugin in getPlugins(IStreamServerEndpointStringParser): - if plugin.prefix == endpointType: - return (plugin, args[1:], kw) - raise ValueError("Unknown endpoint type: '%s'" % (endpointType,)) - return (endpointType.upper(),) + parser(factory, *args[1:], **kw) - - - -def _serverFromStringLegacy(reactor, description, default): - """ - Underlying implementation of L{serverFromString} which avoids exposing the - deprecated 'default' argument to anything but L{strports.service}. - """ - nameOrPlugin, args, kw = _parseServer(description, None, default) - if type(nameOrPlugin) is not str: - plugin = nameOrPlugin - return plugin.parseStreamServer(reactor, *args, **kw) - else: - name = nameOrPlugin - # Chop out the factory. - args = args[:1] + args[2:] - return _endpointServerFactories[name](reactor, *args, **kw) - - - -def serverFromString(reactor, description): - """ - Construct a stream server endpoint from an endpoint description string. - - The format for server endpoint descriptions is a simple string. It is a - prefix naming the type of endpoint, then a colon, then the arguments for - that endpoint. - - For example, you can call it like this to create an endpoint that will - listen on TCP port 80:: - - serverFromString(reactor, "tcp:80") - - Additional arguments may be specified as keywords, separated with colons. - For example, you can specify the interface for a TCP server endpoint to - bind to like this:: - - serverFromString(reactor, "tcp:80:interface=127.0.0.1") - - SSL server endpoints may be specified with the 'ssl' prefix, and the - private key and certificate files may be specified by the C{privateKey} and - C{certKey} arguments:: - - serverFromString(reactor, "ssl:443:privateKey=key.pem:certKey=crt.pem") - - If a private key file name (C{privateKey}) isn't provided, a "server.pem" - file is assumed to exist which contains the private key. If the certificate - file name (C{certKey}) isn't provided, the private key file is assumed to - contain the certificate as well. - - You may escape colons in arguments with a backslash, which you will need to - use if you want to specify a full pathname argument on Windows:: - - serverFromString(reactor, - "ssl:443:privateKey=C\\:/key.pem:certKey=C\\:/cert.pem") - - finally, the 'unix' prefix may be used to specify a filesystem UNIX socket, - optionally with a 'mode' argument to specify the mode of the socket file - created by C{listen}:: - - serverFromString(reactor, "unix:/var/run/finger") - serverFromString(reactor, "unix:/var/run/finger:mode=660") - - This function is also extensible; new endpoint types may be registered as - L{IStreamServerEndpointStringParser} plugins. See that interface for more - information. - - @param reactor: The server endpoint will be constructed with this reactor. - - @param description: The strports description to parse. - - @return: A new endpoint which can be used to listen with the parameters - given by by C{description}. - - @rtype: L{IStreamServerEndpoint} - - @raise ValueError: when the 'description' string cannot be parsed. - - @since: 10.2 - """ - return _serverFromStringLegacy(reactor, description, _NO_DEFAULT) - - - -def quoteStringArgument(argument): - """ - Quote an argument to L{serverFromString} and L{clientFromString}. Since - arguments are separated with colons and colons are escaped with - backslashes, some care is necessary if, for example, you have a pathname, - you may be tempted to interpolate into a string like this:: - - serverFromString("ssl:443:privateKey=%s" % (myPathName,)) - - This may appear to work, but will have portability issues (Windows - pathnames, for example). Usually you should just construct the appropriate - endpoint type rather than interpolating strings, which in this case would - be L{SSL4ServerEndpoint}. There are some use-cases where you may need to - generate such a string, though; for example, a tool to manipulate a - configuration file which has strports descriptions in it. To be correct in - those cases, do this instead:: - - serverFromString("ssl:443:privateKey=%s" % - (quoteStringArgument(myPathName),)) - - @param argument: The part of the endpoint description string you want to - pass through. - - @type argument: C{str} - - @return: The quoted argument. - - @rtype: C{str} - """ - return argument.replace('\\', '\\\\').replace(':', '\\:') - - - -def _parseClientTCP(*args, **kwargs): - """ - Perform any argument value coercion necessary for TCP client parameters. - - Valid positional arguments to this function are host and port. - - Valid keyword arguments to this function are all L{IReactorTCP.connectTCP} - arguments. - - @return: The coerced values as a C{dict}. - """ - - if len(args) == 2: - kwargs['port'] = int(args[1]) - kwargs['host'] = args[0] - elif len(args) == 1: - if 'host' in kwargs: - kwargs['port'] = int(args[0]) - else: - kwargs['host'] = args[0] - - try: - kwargs['port'] = int(kwargs['port']) - except KeyError: - pass - - try: - kwargs['timeout'] = int(kwargs['timeout']) - except KeyError: - pass - return kwargs - - - -def _loadCAsFromDir(directoryPath): - """ - Load certificate-authority certificate objects in a given directory. - - @param directoryPath: a L{FilePath} pointing at a directory to load .pem - files from. - - @return: a C{list} of L{OpenSSL.crypto.X509} objects. - """ - from twisted.internet import ssl - - caCerts = {} - for child in directoryPath.children(): - if not child.basename().split('.')[-1].lower() == 'pem': - continue - try: - data = child.getContent() - except IOError: - # Permission denied, corrupt disk, we don't care. - continue - try: - theCert = ssl.Certificate.loadPEM(data) - except ssl.SSL.Error: - # Duplicate certificate, invalid certificate, etc. We don't care. - pass - else: - caCerts[theCert.digest()] = theCert.original - return caCerts.values() - - - -def _parseClientSSL(*args, **kwargs): - """ - Perform any argument value coercion necessary for SSL client parameters. - - Valid keyword arguments to this function are all L{IReactorSSL.connectSSL} - arguments except for C{contextFactory}. Instead, C{certKey} (the path name - of the certificate file) C{privateKey} (the path name of the private key - associated with the certificate) are accepted and used to construct a - context factory. - - Valid positional arguments to this function are host and port. - - @param caCertsDir: The one parameter which is not part of - L{IReactorSSL.connectSSL}'s signature, this is a path name used to - construct a list of certificate authority certificates. The directory - will be scanned for files ending in C{.pem}, all of which will be - considered valid certificate authorities for this connection. - - @type caCertsDir: C{str} - - @return: The coerced values as a C{dict}. - """ - from twisted.internet import ssl - kwargs = _parseClientTCP(*args, **kwargs) - certKey = kwargs.pop('certKey', None) - privateKey = kwargs.pop('privateKey', None) - caCertsDir = kwargs.pop('caCertsDir', None) - if certKey is not None: - certx509 = ssl.Certificate.loadPEM( - FilePath(certKey).getContent()).original - else: - certx509 = None - if privateKey is not None: - privateKey = ssl.PrivateCertificate.loadPEM( - FilePath(privateKey).getContent()).privateKey.original - else: - privateKey = None - if caCertsDir is not None: - verify = True - caCerts = _loadCAsFromDir(FilePath(caCertsDir)) - else: - verify = False - caCerts = None - kwargs['sslContextFactory'] = ssl.CertificateOptions( - method=ssl.SSL.SSLv23_METHOD, - certificate=certx509, - privateKey=privateKey, - verify=verify, - caCerts=caCerts - ) - return kwargs - - - -def _parseClientUNIX(*args, **kwargs): - """ - Perform any argument value coercion necessary for UNIX client parameters. - - Valid keyword arguments to this function are all L{IReactorUNIX.connectUNIX} - keyword arguments except for C{checkPID}. Instead, C{lockfile} is accepted - and has the same meaning. Also C{path} is used instead of C{address}. - - Valid positional arguments to this function are C{path}. - - @return: The coerced values as a C{dict}. - """ - if len(args) == 1: - kwargs['path'] = args[0] - - try: - kwargs['checkPID'] = bool(int(kwargs.pop('lockfile'))) - except KeyError: - pass - try: - kwargs['timeout'] = int(kwargs['timeout']) - except KeyError: - pass - return kwargs - -_clientParsers = { - 'TCP': _parseClientTCP, - 'SSL': _parseClientSSL, - 'UNIX': _parseClientUNIX, - } - - - -def clientFromString(reactor, description): - """ - Construct a client endpoint from a description string. - - Client description strings are much like server description strings, - although they take all of their arguments as keywords, aside from host and - port. - - You can create a TCP client endpoint with the 'host' and 'port' arguments, - like so:: - - clientFromString(reactor, "tcp:host=www.example.com:port=80") - - or, without specifying host and port keywords:: - - clientFromString(reactor, "tcp:www.example.com:80") - - Or you can specify only one or the other, as in the following 2 examples:: - - clientFromString(reactor, "tcp:host=www.example.com:80") - clientFromString(reactor, "tcp:www.example.com:port=80") - - or an SSL client endpoint with those arguments, plus the arguments used by - the server SSL, for a client certificate:: - - clientFromString(reactor, "ssl:web.example.com:443:" - "privateKey=foo.pem:certKey=foo.pem") - - to specify your certificate trust roots, you can identify a directory with - PEM files in it with the C{caCertsDir} argument:: - - clientFromString(reactor, "ssl:host=web.example.com:port=443:" - "caCertsDir=/etc/ssl/certs") - - You can create a UNIX client endpoint with the 'path' argument and optional - 'lockfile' and 'timeout' arguments:: - - clientFromString(reactor, "unix:path=/var/foo/bar:lockfile=1:timeout=9") - - or, with the path as a positional argument with or without optional - arguments as in the following 2 examples:: - - clientFromString(reactor, "unix:/var/foo/bar") - clientFromString(reactor, "unix:/var/foo/bar:lockfile=1:timeout=9") - - This function is also extensible; new endpoint types may be registered as - L{IStreamClientEndpointStringParser} plugins. See that interface for more - information. - - @param reactor: The client endpoint will be constructed with this reactor. - - @param description: The strports description to parse. - - @return: A new endpoint which can be used to connect with the parameters - given by by C{description}. - @rtype: L{IStreamClientEndpoint} - - @since: 10.2 - """ - args, kwargs = _parse(description) - aname = args.pop(0) - name = aname.upper() - for plugin in getPlugins(IStreamClientEndpointStringParser): - if plugin.prefix.upper() == name: - return plugin.parseStreamClient(*args, **kwargs) - if name not in _clientParsers: - raise ValueError("Unknown endpoint type: %r" % (aname,)) - kwargs = _clientParsers[name](*args, **kwargs) - return _endpointClientFactories[name](reactor, **kwargs) - - - -def connectProtocol(endpoint, protocol): - """ - Connect a protocol instance to an endpoint. - - This allows using a client endpoint without having to create a factory. - - @param endpoint: A client endpoint to connect to. - - @param protocol: A protocol instance. - - @return: The result of calling C{connect} on the endpoint, i.e. a - L{Deferred} that will fire with the protocol when connected, or an - appropriate error. - """ - class OneShotFactory(Factory): - def buildProtocol(self, addr): - return protocol - return endpoint.connect(OneShotFactory()) -}}} ''' diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py index 7b2a78632..fdcbf3977 100644 --- a/scrapy/xlib/tx/interfaces.py +++ b/scrapy/xlib/tx/interfaces.py @@ -9,8 +9,6 @@ Maintainer: Itamar Shtull-Trauring from __future__ import division, absolute_import -from zope.interface import Interface, Attribute - from twisted.internet.interfaces import ( IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL, IReactorWin32Events, IReactorUDP, IReactorMulticast, IReactorProcess, @@ -28,2434 +26,3 @@ from twisted.internet.interfaces import ( IStreamServerEndpointStringParser, IStreamClientEndpointStringParser, IReactorUNIX, IReactorUNIXDatagram, IReactorSocket, IResolver ) - -''' {{{ -class IAddress(Interface): - """ - An address, e.g. a TCP C{(host, port)}. - - Default implementations are in L{twisted.internet.address}. - """ - -### Reactor Interfaces - -class IConnector(Interface): - """ - Object used to interface between connections and protocols. - - Each L{IConnector} manages one connection. - """ - - def stopConnecting(): - """ - Stop attempting to connect. - """ - - def disconnect(): - """ - Disconnect regardless of the connection state. - - If we are connected, disconnect, if we are trying to connect, - stop trying. - """ - - def connect(): - """ - Try to connect to remote address. - """ - - def getDestination(): - """ - Return destination this will try to connect to. - - @return: An object which provides L{IAddress}. - """ - - - -class IResolverSimple(Interface): - def getHostByName(name, timeout = (1, 3, 11, 45)): - """ - Resolve the domain name C{name} into an IP address. - - @type name: C{str} - @type timeout: C{tuple} - @rtype: L{twisted.internet.defer.Deferred} - @return: The callback of the Deferred that is returned will be - passed a string that represents the IP address of the specified - name, or the errback will be called if the lookup times out. If - multiple types of address records are associated with the name, - A6 records will be returned in preference to AAAA records, which - will be returned in preference to A records. If there are multiple - records of the type to be returned, one will be selected at random. - - @raise twisted.internet.defer.TimeoutError: Raised (asynchronously) - if the name cannot be resolved within the specified timeout period. - """ - - - -class IResolver(IResolverSimple): - def query(query, timeout=None): - """ - Dispatch C{query} to the method which can handle its type. - - @type query: L{twisted.names.dns.Query} - @param query: The DNS query being issued, to which a response is to be - generated. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAddress(name, timeout=None): - """ - Perform an A record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAddress6(name, timeout=None): - """ - Perform an A6 record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupIPV6Address(name, timeout=None): - """ - Perform an AAAA record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailExchange(name, timeout=None): - """ - Perform an MX record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupNameservers(name, timeout=None): - """ - Perform an NS record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupCanonicalName(name, timeout=None): - """ - Perform a CNAME record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailBox(name, timeout=None): - """ - Perform an MB record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailGroup(name, timeout=None): - """ - Perform an MG record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailRename(name, timeout=None): - """ - Perform an MR record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupPointer(name, timeout=None): - """ - Perform a PTR record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAuthority(name, timeout=None): - """ - Perform an SOA record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupNull(name, timeout=None): - """ - Perform a NULL record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupWellKnownServices(name, timeout=None): - """ - Perform a WKS record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupHostInfo(name, timeout=None): - """ - Perform a HINFO record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailboxInfo(name, timeout=None): - """ - Perform an MINFO record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupText(name, timeout=None): - """ - Perform a TXT record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupResponsibility(name, timeout=None): - """ - Perform an RP record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAFSDatabase(name, timeout=None): - """ - Perform an AFSDB record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupService(name, timeout=None): - """ - Perform an SRV record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAllRecords(name, timeout=None): - """ - Perform an ALL_RECORD lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupSenderPolicy(name, timeout= 10): - """ - Perform a SPF record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupNamingAuthorityPointer(name, timeout=None): - """ - Perform a NAPTR record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupZone(name, timeout=None): - """ - Perform an AXFR record lookup. - - NB This is quite different from other DNS requests. See - U{http://cr.yp.to/djbdns/axfr-notes.html} for more - information. - - NB Unlike other C{lookup*} methods, the timeout here is not a - list of ints, it is a single int. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: C{int} - @param timeout: When this timeout expires, the query is - considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. - The first element of the tuple gives answers. - The second and third elements are always empty. - The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - -class IReactorTCP(Interface): - - def listenTCP(port, factory, backlog=50, interface=''): - """ - Connects a given protocol factory to the given numeric TCP/IP port. - - @param port: a port number on which to listen - - @param factory: a L{twisted.internet.protocol.ServerFactory} instance - - @param backlog: size of the listen queue - - @param interface: The local IPv4 or IPv6 address to which to bind; - defaults to '', ie all IPv4 addresses. To bind to all IPv4 and IPv6 - addresses, you must call this method twice. - - @return: an object that provides L{IListeningPort}. - - @raise CannotListenError: as defined here - L{twisted.internet.error.CannotListenError}, - if it cannot listen on this port (e.g., it - cannot bind to the required port number) - """ - - def connectTCP(host, port, factory, timeout=30, bindAddress=None): - """ - Connect a TCP client. - - @param host: a host name - - @param port: a port number - - @param factory: a L{twisted.internet.protocol.ClientFactory} instance - - @param timeout: number of seconds to wait before assuming the - connection has failed. - - @param bindAddress: a (host, port) tuple of local address to bind - to, or None. - - @return: An object which provides L{IConnector}. This connector will - call various callbacks on the factory when a connection is - made, failed, or lost - see - L{ClientFactory} - docs for details. - """ - -class IReactorSSL(Interface): - - def connectSSL(host, port, factory, contextFactory, timeout=30, bindAddress=None): - """ - Connect a client Protocol to a remote SSL socket. - - @param host: a host name - - @param port: a port number - - @param factory: a L{twisted.internet.protocol.ClientFactory} instance - - @param contextFactory: a L{twisted.internet.ssl.ClientContextFactory} object. - - @param timeout: number of seconds to wait before assuming the - connection has failed. - - @param bindAddress: a (host, port) tuple of local address to bind to, - or C{None}. - - @return: An object which provides L{IConnector}. - """ - - def listenSSL(port, factory, contextFactory, backlog=50, interface=''): - """ - Connects a given protocol factory to the given numeric TCP/IP port. - The connection is a SSL one, using contexts created by the context - factory. - - @param port: a port number on which to listen - - @param factory: a L{twisted.internet.protocol.ServerFactory} instance - - @param contextFactory: a L{twisted.internet.ssl.ContextFactory} instance - - @param backlog: size of the listen queue - - @param interface: the hostname to bind to, defaults to '' (all) - """ - - - -class IReactorUNIX(Interface): - """ - UNIX socket methods. - """ - - def connectUNIX(address, factory, timeout=30, checkPID=0): - """ - Connect a client protocol to a UNIX socket. - - @param address: a path to a unix socket on the filesystem. - - @param factory: a L{twisted.internet.protocol.ClientFactory} instance - - @param timeout: number of seconds to wait before assuming the connection - has failed. - - @param checkPID: if True, check for a pid file to verify that a server - is listening. If C{address} is a Linux abstract namespace path, - this must be C{False}. - - @return: An object which provides L{IConnector}. - """ - - - def listenUNIX(address, factory, backlog=50, mode=0o666, wantPID=0): - """ - Listen on a UNIX socket. - - @param address: a path to a unix socket on the filesystem. - - @param factory: a L{twisted.internet.protocol.Factory} instance. - - @param backlog: number of connections to allow in backlog. - - @param mode: The mode (B{not} umask) to set on the unix socket. See - platform specific documentation for information about how this - might affect connection attempts. - @type mode: C{int} - - @param wantPID: if True, create a pidfile for the socket. If C{address} - is a Linux abstract namespace path, this must be C{False}. - - @return: An object which provides L{IListeningPort}. - """ - - - -class IReactorUNIXDatagram(Interface): - """ - Datagram UNIX socket methods. - """ - - def connectUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666, bindAddress=None): - """ - Connect a client protocol to a datagram UNIX socket. - - @param address: a path to a unix socket on the filesystem. - - @param protocol: a L{twisted.internet.protocol.ConnectedDatagramProtocol} instance - - @param maxPacketSize: maximum packet size to accept - - @param mode: The mode (B{not} umask) to set on the unix socket. See - platform specific documentation for information about how this - might affect connection attempts. - @type mode: C{int} - - @param bindAddress: address to bind to - - @return: An object which provides L{IConnector}. - """ - - - def listenUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666): - """ - Listen on a datagram UNIX socket. - - @param address: a path to a unix socket on the filesystem. - - @param protocol: a L{twisted.internet.protocol.DatagramProtocol} instance. - - @param maxPacketSize: maximum packet size to accept - - @param mode: The mode (B{not} umask) to set on the unix socket. See - platform specific documentation for information about how this - might affect connection attempts. - @type mode: C{int} - - @return: An object which provides L{IListeningPort}. - """ - - - -class IReactorWin32Events(Interface): - """ - Win32 Event API methods - - @since: 10.2 - """ - - def addEvent(event, fd, action): - """ - Add a new win32 event to the event loop. - - @param event: a Win32 event object created using win32event.CreateEvent() - - @param fd: an instance of L{twisted.internet.abstract.FileDescriptor} - - @param action: a string that is a method name of the fd instance. - This method is called in response to the event. - - @return: None - """ - - - def removeEvent(event): - """ - Remove an event. - - @param event: a Win32 event object added using L{IReactorWin32Events.addEvent} - - @return: None - """ - - - -class IReactorUDP(Interface): - """ - UDP socket methods. - """ - - def listenUDP(port, protocol, interface='', maxPacketSize=8192): - """ - Connects a given DatagramProtocol to the given numeric UDP port. - - @return: object which provides L{IListeningPort}. - """ - - - -class IReactorMulticast(Interface): - """ - UDP socket methods that support multicast. - - IMPORTANT: This is an experimental new interface. It may change - without backwards compatibility. Suggestions are welcome. - """ - - def listenMulticast(port, protocol, interface='', maxPacketSize=8192, - listenMultiple=False): - """ - Connects a given - L{DatagramProtocol} to the - given numeric UDP port. - - @param listenMultiple: If set to True, allows multiple sockets to - bind to the same address and port number at the same time. - @type listenMultiple: C{bool} - - @returns: An object which provides L{IListeningPort}. - - @see: L{twisted.internet.interfaces.IMulticastTransport} - @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html} - """ - - - -class IReactorSocket(Interface): - """ - Methods which allow a reactor to use externally created sockets. - - For example, to use C{adoptStreamPort} to implement behavior equivalent - to that of L{IReactorTCP.listenTCP}, you might write code like this:: - - from socket import SOMAXCONN, AF_INET, SOCK_STREAM, socket - portSocket = socket(AF_INET, SOCK_STREAM) - # Set FD_CLOEXEC on port, left as an exercise. Then make it into a - # non-blocking listening port: - portSocket.setblocking(False) - portSocket.bind(('192.168.1.2', 12345)) - portSocket.listen(SOMAXCONN) - - # Now have the reactor use it as a TCP port - port = reactor.adoptStreamPort( - portSocket.fileno(), AF_INET, YourFactory()) - - # portSocket itself is no longer necessary, and needs to be cleaned - # up by us. - portSocket.close() - - # Whenever the server is no longer needed, stop it as usual. - stoppedDeferred = port.stopListening() - - Another potential use is to inherit a listening descriptor from a parent - process (for example, systemd or launchd), or to receive one over a UNIX - domain socket. - - Some plans for extending this interface exist. See: - - - U{http://twistedmatrix.com/trac/ticket/5570}: established connections - - U{http://twistedmatrix.com/trac/ticket/5573}: AF_UNIX ports - - U{http://twistedmatrix.com/trac/ticket/5574}: SOCK_DGRAM sockets - """ - - def adoptStreamPort(fileDescriptor, addressFamily, factory): - """ - Add an existing listening I{SOCK_STREAM} socket to the reactor to - monitor for new connections to accept and handle. - - @param fileDescriptor: A file descriptor associated with a socket which - is already bound to an address and marked as listening. The socket - must be set non-blocking. Any additional flags (for example, - close-on-exec) must also be set by application code. Application - code is responsible for closing the file descriptor, which may be - done as soon as C{adoptStreamPort} returns. - @type fileDescriptor: C{int} - - @param addressFamily: The address family (or I{domain}) of the socket. - For example, L{socket.AF_INET6}. - - @param factory: A L{ServerFactory} instance to use to create new - protocols to handle connections accepted via this socket. - - @return: An object providing L{IListeningPort}. - - @raise UnsupportedAddressFamily: If the given address family is not - supported by this reactor, or not supported with the given socket - type. - - @raise UnsupportedSocketType: If the given socket type is not supported - by this reactor, or not supported with the given socket type. - """ - - - def adoptStreamConnection(fileDescriptor, addressFamily, factory): - """ - Add an existing connected I{SOCK_STREAM} socket to the reactor to - monitor for data. - - Note that the given factory won't have its C{startFactory} and - C{stopFactory} methods called, as there is no sensible time to call - them in this situation. - - @param fileDescriptor: A file descriptor associated with a socket which - is already connected. The socket must be set non-blocking. Any - additional flags (for example, close-on-exec) must also be set by - application code. Application code is responsible for closing the - file descriptor, which may be done as soon as - C{adoptStreamConnection} returns. - @type fileDescriptor: C{int} - - @param addressFamily: The address family (or I{domain}) of the socket. - For example, L{socket.AF_INET6}. - - @param factory: A L{ServerFactory} instance to use to create a new - protocol to handle the connection via this socket. - - @raise UnsupportedAddressFamily: If the given address family is not - supported by this reactor, or not supported with the given socket - type. - - @raise UnsupportedSocketType: If the given socket type is not supported - by this reactor, or not supported with the given socket type. - """ - - - -class IReactorProcess(Interface): - - def spawnProcess(processProtocol, executable, args=(), env={}, path=None, - uid=None, gid=None, usePTY=0, childFDs=None): - """ - Spawn a process, with a process protocol. - - @type processProtocol: L{IProcessProtocol} provider - @param processProtocol: An object which will be notified of all - events related to the created process. - - @param executable: the file name to spawn - the full path should be - used. - - @param args: the command line arguments to pass to the process; a - sequence of strings. The first string should be the - executable's name. - - @type env: a C{dict} mapping C{str} to C{str}, or C{None}. - @param env: the environment variables to pass to the child process. The - resulting behavior varies between platforms. If - - C{env} is not set: - - On POSIX: pass an empty environment. - - On Windows: pass C{os.environ}. - - C{env} is C{None}: - - On POSIX: pass C{os.environ}. - - On Windows: pass C{os.environ}. - - C{env} is a C{dict}: - - On POSIX: pass the key/value pairs in C{env} as the - complete environment. - - On Windows: update C{os.environ} with the key/value - pairs in the C{dict} before passing it. As a - consequence of U{bug #1640 - }, passing - keys with empty values in an effort to unset - environment variables I{won't} unset them. - - @param path: the path to run the subprocess in - defaults to the - current directory. - - @param uid: user ID to run the subprocess as. (Only available on - POSIX systems.) - - @param gid: group ID to run the subprocess as. (Only available on - POSIX systems.) - - @param usePTY: if true, run this process in a pseudo-terminal. - optionally a tuple of C{(masterfd, slavefd, ttyname)}, - in which case use those file descriptors. - (Not available on all systems.) - - @param childFDs: A dictionary mapping file descriptors in the new child - process to an integer or to the string 'r' or 'w'. - - If the value is an integer, it specifies a file - descriptor in the parent process which will be mapped - to a file descriptor (specified by the key) in the - child process. This is useful for things like inetd - and shell-like file redirection. - - If it is the string 'r', a pipe will be created and - attached to the child at that file descriptor: the - child will be able to write to that file descriptor - and the parent will receive read notification via the - L{IProcessProtocol.childDataReceived} callback. This - is useful for the child's stdout and stderr. - - If it is the string 'w', similar setup to the previous - case will occur, with the pipe being readable by the - child instead of writeable. The parent process can - write to that file descriptor using - L{IProcessTransport.writeToChild}. This is useful for - the child's stdin. - - If childFDs is not passed, the default behaviour is to - use a mapping that opens the usual stdin/stdout/stderr - pipes. - - @see: L{twisted.internet.protocol.ProcessProtocol} - - @return: An object which provides L{IProcessTransport}. - - @raise OSError: Raised with errno C{EAGAIN} or C{ENOMEM} if there are - insufficient system resources to create a new process. - """ - -class IReactorTime(Interface): - """ - Time methods that a Reactor should implement. - """ - - def seconds(): - """ - Get the current time in seconds. - - @return: A number-like object of some sort. - """ - - - def callLater(delay, callable, *args, **kw): - """ - Call a function later. - - @type delay: C{float} - @param delay: the number of seconds to wait. - - @param callable: the callable object to call later. - - @param args: the arguments to call it with. - - @param kw: the keyword arguments to call it with. - - @return: An object which provides L{IDelayedCall} and can be used to - cancel the scheduled call, by calling its C{cancel()} method. - It also may be rescheduled by calling its C{delay()} or - C{reset()} methods. - """ - - - def getDelayedCalls(): - """ - Retrieve all currently scheduled delayed calls. - - @return: A tuple of all L{IDelayedCall} providers representing all - currently scheduled calls. This is everything that has been - returned by C{callLater} but not yet called or canceled. - """ - - -class IDelayedCall(Interface): - """ - A scheduled call. - - There are probably other useful methods we can add to this interface; - suggestions are welcome. - """ - - def getTime(): - """ - Get time when delayed call will happen. - - @return: time in seconds since epoch (a float). - """ - - def cancel(): - """ - Cancel the scheduled call. - - @raises twisted.internet.error.AlreadyCalled: if the call has already - happened. - @raises twisted.internet.error.AlreadyCancelled: if the call has already - been cancelled. - """ - - def delay(secondsLater): - """ - Delay the scheduled call. - - @param secondsLater: how many seconds from its current firing time to delay - - @raises twisted.internet.error.AlreadyCalled: if the call has already - happened. - @raises twisted.internet.error.AlreadyCancelled: if the call has already - been cancelled. - """ - - def reset(secondsFromNow): - """ - Reset the scheduled call's timer. - - @param secondsFromNow: how many seconds from now it should fire, - equivalent to C{.cancel()} and then doing another - C{reactor.callLater(secondsLater, ...)} - - @raises twisted.internet.error.AlreadyCalled: if the call has already - happened. - @raises twisted.internet.error.AlreadyCancelled: if the call has already - been cancelled. - """ - - def active(): - """ - @return: True if this call is still active, False if it has been - called or cancelled. - """ - -class IReactorThreads(Interface): - """ - Dispatch methods to be run in threads. - - Internally, this should use a thread pool and dispatch methods to them. - """ - - def getThreadPool(): - """ - Return the threadpool used by L{callInThread}. Create it first if - necessary. - - @rtype: L{twisted.python.threadpool.ThreadPool} - """ - - - def callInThread(callable, *args, **kwargs): - """ - Run the callable object in a separate thread. - """ - - - def callFromThread(callable, *args, **kw): - """ - Cause a function to be executed by the reactor thread. - - Use this method when you want to run a function in the reactor's thread - from another thread. Calling L{callFromThread} should wake up the main - thread (where L{reactor.run()} is executing) and run the - given callable in that thread. - - If you're writing a multi-threaded application the C{callable} may need - to be thread safe, but this method doesn't require it as such. If you - want to call a function in the next mainloop iteration, but you're in - the same thread, use L{callLater} with a delay of 0. - """ - - - def suggestThreadPoolSize(size): - """ - Suggest the size of the internal threadpool used to dispatch functions - passed to L{callInThread}. - """ - - -class IReactorCore(Interface): - """ - Core methods that a Reactor must implement. - """ - - running = Attribute( - "A C{bool} which is C{True} from I{during startup} to " - "I{during shutdown} and C{False} the rest of the time.") - - - def resolve(name, timeout=10): - """ - Return a L{twisted.internet.defer.Deferred} that will resolve a hostname. - """ - - def run(): - """ - Fire 'startup' System Events, move the reactor to the 'running' - state, then run the main loop until it is stopped with C{stop()} or - C{crash()}. - """ - - def stop(): - """ - Fire 'shutdown' System Events, which will move the reactor to the - 'stopped' state and cause C{reactor.run()} to exit. - """ - - def crash(): - """ - Stop the main loop *immediately*, without firing any system events. - - This is named as it is because this is an extremely "rude" thing to do; - it is possible to lose data and put your system in an inconsistent - state by calling this. However, it is necessary, as sometimes a system - can become wedged in a pre-shutdown call. - """ - - def iterate(delay=0): - """ - Run the main loop's I/O polling function for a period of time. - - This is most useful in applications where the UI is being drawn "as - fast as possible", such as games. All pending L{IDelayedCall}s will - be called. - - The reactor must have been started (via the C{run()} method) prior to - any invocations of this method. It must also be stopped manually - after the last call to this method (via the C{stop()} method). This - method is not re-entrant: you must not call it recursively; in - particular, you must not call it while the reactor is running. - """ - - def fireSystemEvent(eventType): - """ - Fire a system-wide event. - - System-wide events are things like 'startup', 'shutdown', and - 'persist'. - """ - - def addSystemEventTrigger(phase, eventType, callable, *args, **kw): - """ - Add a function to be called when a system event occurs. - - Each "system event" in Twisted, such as 'startup', 'shutdown', and - 'persist', has 3 phases: 'before', 'during', and 'after' (in that - order, of course). These events will be fired internally by the - Reactor. - - An implementor of this interface must only implement those events - described here. - - Callbacks registered for the "before" phase may return either None or a - Deferred. The "during" phase will not execute until all of the - Deferreds from the "before" phase have fired. - - Once the "during" phase is running, all of the remaining triggers must - execute; their return values must be ignored. - - @param phase: a time to call the event -- either the string 'before', - 'after', or 'during', describing when to call it - relative to the event's execution. - - @param eventType: this is a string describing the type of event. - - @param callable: the object to call before shutdown. - - @param args: the arguments to call it with. - - @param kw: the keyword arguments to call it with. - - @return: an ID that can be used to remove this call with - removeSystemEventTrigger. - """ - - def removeSystemEventTrigger(triggerID): - """ - Removes a trigger added with addSystemEventTrigger. - - @param triggerID: a value returned from addSystemEventTrigger. - - @raise KeyError: If there is no system event trigger for the given - C{triggerID}. - - @raise ValueError: If there is no system event trigger for the given - C{triggerID}. - - @raise TypeError: If there is no system event trigger for the given - C{triggerID}. - """ - - def callWhenRunning(callable, *args, **kw): - """ - Call a function when the reactor is running. - - If the reactor has not started, the callable will be scheduled - to run when it does start. Otherwise, the callable will be invoked - immediately. - - @param callable: the callable object to call later. - - @param args: the arguments to call it with. - - @param kw: the keyword arguments to call it with. - - @return: None if the callable was invoked, otherwise a system - event id for the scheduled call. - """ - - -class IReactorPluggableResolver(Interface): - """ - A reactor with a pluggable name resolver interface. - """ - - def installResolver(resolver): - """ - Set the internal resolver to use to for name lookups. - - @type resolver: An object implementing the L{IResolverSimple} interface - @param resolver: The new resolver to use. - - @return: The previously installed resolver. - """ - - -class IReactorDaemonize(Interface): - """ - A reactor which provides hooks that need to be called before and after - daemonization. - - Notes: - - This interface SHOULD NOT be called by applications. - - This interface should only be implemented by reactors as a workaround - (in particular, it's implemented currently only by kqueue()). - For details please see the comments on ticket #1918. - """ - - def beforeDaemonize(): - """ - Hook to be called immediately before daemonization. No reactor methods - may be called until L{afterDaemonize} is called. - - @return: C{None}. - """ - - - def afterDaemonize(): - """ - Hook to be called immediately after daemonization. This may only be - called after L{beforeDaemonize} had been called previously. - - @return: C{None}. - """ - - - -class IReactorFDSet(Interface): - """ - Implement me to be able to use L{IFileDescriptor} type resources. - - This assumes that your main-loop uses UNIX-style numeric file descriptors - (or at least similarly opaque IDs returned from a .fileno() method) - """ - - def addReader(reader): - """ - I add reader to the set of file descriptors to get read events for. - - @param reader: An L{IReadDescriptor} provider that will be checked for - read events until it is removed from the reactor with - L{removeReader}. - - @return: C{None}. - """ - - def addWriter(writer): - """ - I add writer to the set of file descriptors to get write events for. - - @param writer: An L{IWriteDescriptor} provider that will be checked for - write events until it is removed from the reactor with - L{removeWriter}. - - @return: C{None}. - """ - - def removeReader(reader): - """ - Removes an object previously added with L{addReader}. - - @return: C{None}. - """ - - def removeWriter(writer): - """ - Removes an object previously added with L{addWriter}. - - @return: C{None}. - """ - - def removeAll(): - """ - Remove all readers and writers. - - Should not remove reactor internal reactor connections (like a waker). - - @return: A list of L{IReadDescriptor} and L{IWriteDescriptor} providers - which were removed. - """ - - def getReaders(): - """ - Return the list of file descriptors currently monitored for input - events by the reactor. - - @return: the list of file descriptors monitored for input events. - @rtype: C{list} of C{IReadDescriptor} - """ - - def getWriters(): - """ - Return the list file descriptors currently monitored for output events - by the reactor. - - @return: the list of file descriptors monitored for output events. - @rtype: C{list} of C{IWriteDescriptor} - """ - - -class IListeningPort(Interface): - """ - A listening port. - """ - - def startListening(): - """ - Start listening on this port. - - @raise CannotListenError: If it cannot listen on this port (e.g., it is - a TCP port and it cannot bind to the required - port number). - """ - - def stopListening(): - """ - Stop listening on this port. - - If it does not complete immediately, will return Deferred that fires - upon completion. - """ - - def getHost(): - """ - Get the host that this port is listening for. - - @return: An L{IAddress} provider. - """ - - -class ILoggingContext(Interface): - """ - Give context information that will be used to log events generated by - this item. - """ - - def logPrefix(): - """ - @return: Prefix used during log formatting to indicate context. - @rtype: C{str} - """ - - - -class IFileDescriptor(ILoggingContext): - """ - An interface representing a UNIX-style numeric file descriptor. - """ - - def fileno(): - """ - @raise: If the descriptor no longer has a valid file descriptor - number associated with it. - - @return: The platform-specified representation of a file descriptor - number. Or C{-1} if the descriptor no longer has a valid file - descriptor number associated with it. As long as the descriptor - is valid, calls to this method on a particular instance must - return the same value. - """ - - - def connectionLost(reason): - """ - Called when the connection was lost. - - This is called when the connection on a selectable object has been - lost. It will be called whether the connection was closed explicitly, - an exception occurred in an event handler, or the other end of the - connection closed it first. - - See also L{IHalfCloseableDescriptor} if your descriptor wants to be - notified separately of the two halves of the connection being closed. - - @param reason: A failure instance indicating the reason why the - connection was lost. L{error.ConnectionLost} and - L{error.ConnectionDone} are of special note, but the - failure may be of other classes as well. - """ - - - -class IReadDescriptor(IFileDescriptor): - """ - An L{IFileDescriptor} that can read. - - This interface is generally used in conjunction with L{IReactorFDSet}. - """ - - def doRead(): - """ - Some data is available for reading on your descriptor. - - @return: If an error is encountered which causes the descriptor to - no longer be valid, a L{Failure} should be returned. Otherwise, - C{None}. - """ - - -class IWriteDescriptor(IFileDescriptor): - """ - An L{IFileDescriptor} that can write. - - This interface is generally used in conjunction with L{IReactorFDSet}. - """ - - def doWrite(): - """ - Some data can be written to your descriptor. - - @return: If an error is encountered which causes the descriptor to - no longer be valid, a L{Failure} should be returned. Otherwise, - C{None}. - """ - - -class IReadWriteDescriptor(IReadDescriptor, IWriteDescriptor): - """ - An L{IFileDescriptor} that can both read and write. - """ - - -class IHalfCloseableDescriptor(Interface): - """ - A descriptor that can be half-closed. - """ - - def writeConnectionLost(reason): - """ - Indicates write connection was lost. - """ - - def readConnectionLost(reason): - """ - Indicates read connection was lost. - """ - - -class ISystemHandle(Interface): - """ - An object that wraps a networking OS-specific handle. - """ - - def getHandle(): - """ - Return a system- and reactor-specific handle. - - This might be a socket.socket() object, or some other type of - object, depending on which reactor is being used. Use and - manipulate at your own risk. - - This might be used in cases where you want to set specific - options not exposed by the Twisted APIs. - """ - - -class IConsumer(Interface): - """ - A consumer consumes data from a producer. - """ - - def registerProducer(producer, streaming): - """ - Register to receive data from a producer. - - This sets self to be a consumer for a producer. When this object runs - out of data (as when a send(2) call on a socket succeeds in moving the - last data from a userspace buffer into a kernelspace buffer), it will - ask the producer to resumeProducing(). - - For L{IPullProducer} providers, C{resumeProducing} will be called once - each time data is required. - - For L{IPushProducer} providers, C{pauseProducing} will be called - whenever the write buffer fills up and C{resumeProducing} will only be - called when it empties. - - @type producer: L{IProducer} provider - - @type streaming: C{bool} - @param streaming: C{True} if C{producer} provides L{IPushProducer}, - C{False} if C{producer} provides L{IPullProducer}. - - @raise RuntimeError: If a producer is already registered. - - @return: C{None} - """ - - - def unregisterProducer(): - """ - Stop consuming data from a producer, without disconnecting. - """ - - - def write(data): - """ - The producer will write data by calling this method. - - The implementation must be non-blocking and perform whatever - buffering is necessary. If the producer has provided enough data - for now and it is a L{IPushProducer}, the consumer may call its - C{pauseProducing} method. - """ - - - -class IProducer(Interface): - """ - A producer produces data for a consumer. - - Typically producing is done by calling the write method of an class - implementing L{IConsumer}. - """ - - def stopProducing(): - """ - Stop producing data. - - This tells a producer that its consumer has died, so it must stop - producing data for good. - """ - - -class IPushProducer(IProducer): - """ - A push producer, also known as a streaming producer is expected to - produce (write to this consumer) data on a continuous basis, unless - it has been paused. A paused push producer will resume producing - after its resumeProducing() method is called. For a push producer - which is not pauseable, these functions may be noops. - """ - - def pauseProducing(): - """ - Pause producing data. - - Tells a producer that it has produced too much data to process for - the time being, and to stop until resumeProducing() is called. - """ - def resumeProducing(): - """ - Resume producing data. - - This tells a producer to re-add itself to the main loop and produce - more data for its consumer. - """ - -class IPullProducer(IProducer): - """ - A pull producer, also known as a non-streaming producer, is - expected to produce data each time resumeProducing() is called. - """ - - def resumeProducing(): - """ - Produce data for the consumer a single time. - - This tells a producer to produce data for the consumer once - (not repeatedly, once only). Typically this will be done - by calling the consumer's write() method a single time with - produced data. - """ - -class IProtocol(Interface): - - def dataReceived(data): - """ - Called whenever data is received. - - Use this method to translate to a higher-level message. Usually, some - callback will be made upon the receipt of each complete protocol - message. - - @param data: a string of indeterminate length. Please keep in mind - that you will probably need to buffer some data, as partial - (or multiple) protocol messages may be received! I recommend - that unit tests for protocols call through to this method with - differing chunk sizes, down to one byte at a time. - """ - - def connectionLost(reason): - """ - Called when the connection is shut down. - - Clear any circular references here, and any external references - to this Protocol. The connection has been closed. The C{reason} - Failure wraps a L{twisted.internet.error.ConnectionDone} or - L{twisted.internet.error.ConnectionLost} instance (or a subclass - of one of those). - - @type reason: L{twisted.python.failure.Failure} - """ - - def makeConnection(transport): - """ - Make a connection to a transport and a server. - """ - - def connectionMade(): - """ - Called when a connection is made. - - This may be considered the initializer of the protocol, because - it is called when the connection is completed. For clients, - this is called once the connection to the server has been - established; for servers, this is called after an accept() call - stops blocking and a socket has been received. If you need to - send any greeting or initial message, do it here. - """ - - -class IProcessProtocol(Interface): - """ - Interface for process-related event handlers. - """ - - def makeConnection(process): - """ - Called when the process has been created. - - @type process: L{IProcessTransport} provider - @param process: An object representing the process which has been - created and associated with this protocol. - """ - - - def childDataReceived(childFD, data): - """ - Called when data arrives from the child process. - - @type childFD: C{int} - @param childFD: The file descriptor from which the data was - received. - - @type data: C{str} - @param data: The data read from the child's file descriptor. - """ - - - def childConnectionLost(childFD): - """ - Called when a file descriptor associated with the child process is - closed. - - @type childFD: C{int} - @param childFD: The file descriptor which was closed. - """ - - - def processExited(reason): - """ - Called when the child process exits. - - @type reason: L{twisted.python.failure.Failure} - @param reason: A failure giving the reason the child process - terminated. The type of exception for this failure is either - L{twisted.internet.error.ProcessDone} or - L{twisted.internet.error.ProcessTerminated}. - - @since: 8.2 - """ - - - def processEnded(reason): - """ - Called when the child process exits and all file descriptors associated - with it have been closed. - - @type reason: L{twisted.python.failure.Failure} - @param reason: A failure giving the reason the child process - terminated. The type of exception for this failure is either - L{twisted.internet.error.ProcessDone} or - L{twisted.internet.error.ProcessTerminated}. - """ - - - -class IHalfCloseableProtocol(Interface): - """ - Implemented to indicate they want notification of half-closes. - - TCP supports the notion of half-closing the connection, e.g. - closing the write side but still not stopping reading. A protocol - that implements this interface will be notified of such events, - instead of having connectionLost called. - """ - - def readConnectionLost(): - """ - Notification of the read connection being closed. - - This indicates peer did half-close of write side. It is now - the responsibility of the this protocol to call - loseConnection(). In addition, the protocol MUST make sure a - reference to it still exists (i.e. by doing a callLater with - one of its methods, etc.) as the reactor will only have a - reference to it if it is writing. - - If the protocol does not do so, it might get garbage collected - without the connectionLost method ever being called. - """ - - def writeConnectionLost(): - """ - Notification of the write connection being closed. - - This will never be called for TCP connections as TCP does not - support notification of this type of half-close. - """ - - - -class IFileDescriptorReceiver(Interface): - """ - Protocols may implement L{IFileDescriptorReceiver} to receive file - descriptors sent to them. This is useful in conjunction with - L{IUNIXTransport}, which allows file descriptors to be sent between - processes on a single host. - """ - def fileDescriptorReceived(descriptor): - """ - Called when a file descriptor is received over the connection. - - @param descriptor: The descriptor which was received. - @type descriptor: C{int} - - @return: C{None} - """ - - - -class IProtocolFactory(Interface): - """ - Interface for protocol factories. - """ - - def buildProtocol(addr): - """ - Called when a connection has been established to addr. - - If None is returned, the connection is assumed to have been refused, - and the Port will close the connection. - - @type addr: (host, port) - @param addr: The address of the newly-established connection - - @return: None if the connection was refused, otherwise an object - providing L{IProtocol}. - """ - - def doStart(): - """ - Called every time this is connected to a Port or Connector. - """ - - def doStop(): - """ - Called every time this is unconnected from a Port or Connector. - """ - - -class ITransport(Interface): - """ - I am a transport for bytes. - - I represent (and wrap) the physical connection and synchronicity - of the framework which is talking to the network. I make no - representations about whether calls to me will happen immediately - or require returning to a control loop, or whether they will happen - in the same or another thread. Consider methods of this class - (aside from getPeer) to be 'thrown over the wall', to happen at some - indeterminate time. - """ - - def write(data): - """ - Write some data to the physical connection, in sequence, in a - non-blocking fashion. - - If possible, make sure that it is all written. No data will - ever be lost, although (obviously) the connection may be closed - before it all gets through. - """ - - def writeSequence(data): - """ - Write a list of strings to the physical connection. - - If possible, make sure that all of the data is written to - the socket at once, without first copying it all into a - single string. - """ - - def loseConnection(): - """ - Close my connection, after writing all pending data. - - Note that if there is a registered producer on a transport it - will not be closed until the producer has been unregistered. - """ - - def getPeer(): - """ - Get the remote address of this connection. - - Treat this method with caution. It is the unfortunate result of the - CGI and Jabber standards, but should not be considered reliable for - the usual host of reasons; port forwarding, proxying, firewalls, IP - masquerading, etc. - - @return: An L{IAddress} provider. - """ - - def getHost(): - """ - Similar to getPeer, but returns an address describing this side of the - connection. - - @return: An L{IAddress} provider. - """ - - -class ITCPTransport(ITransport): - """ - A TCP based transport. - """ - - def loseWriteConnection(): - """ - Half-close the write side of a TCP connection. - - If the protocol instance this is attached to provides - IHalfCloseableProtocol, it will get notified when the operation is - done. When closing write connection, as with loseConnection this will - only happen when buffer has emptied and there is no registered - producer. - """ - - - def abortConnection(): - """ - Close the connection abruptly. - - Discards any buffered data, stops any registered producer, - and, if possible, notifies the other end of the unclean - closure. - - @since: 11.1 - """ - - - def getTcpNoDelay(): - """ - Return if C{TCP_NODELAY} is enabled. - """ - - def setTcpNoDelay(enabled): - """ - Enable/disable C{TCP_NODELAY}. - - Enabling C{TCP_NODELAY} turns off Nagle's algorithm. Small packets are - sent sooner, possibly at the expense of overall throughput. - """ - - def getTcpKeepAlive(): - """ - Return if C{SO_KEEPALIVE} is enabled. - """ - - def setTcpKeepAlive(enabled): - """ - Enable/disable C{SO_KEEPALIVE}. - - Enabling C{SO_KEEPALIVE} sends packets periodically when the connection - is otherwise idle, usually once every two hours. They are intended - to allow detection of lost peers in a non-infinite amount of time. - """ - - def getHost(): - """ - Returns L{IPv4Address} or L{IPv6Address}. - """ - - def getPeer(): - """ - Returns L{IPv4Address} or L{IPv6Address}. - """ - - - -class IUNIXTransport(ITransport): - """ - Transport for stream-oriented unix domain connections. - """ - def sendFileDescriptor(descriptor): - """ - Send a duplicate of this (file, socket, pipe, etc) descriptor to the - other end of this connection. - - The send is non-blocking and will be queued if it cannot be performed - immediately. The send will be processed in order with respect to other - C{sendFileDescriptor} calls on this transport, but not necessarily with - respect to C{write} calls on this transport. The send can only be - processed if there are also bytes in the normal connection-oriented send - buffer (ie, you must call C{write} at least as many times as you call - C{sendFileDescriptor}). - - @param descriptor: An C{int} giving a valid file descriptor in this - process. Note that a I{file descriptor} may actually refer to a - socket, a pipe, or anything else POSIX tries to treat in the same - way as a file. - - @return: C{None} - """ - - - -class ITLSTransport(ITCPTransport): - """ - A TCP transport that supports switching to TLS midstream. - - Once TLS mode is started the transport will implement L{ISSLTransport}. - """ - - def startTLS(contextFactory): - """ - Initiate TLS negotiation. - - @param contextFactory: A context factory (see L{ssl.py}) - """ - -class ISSLTransport(ITCPTransport): - """ - A SSL/TLS based transport. - """ - - def getPeerCertificate(): - """ - Return an object with the peer's certificate info. - """ - - -class IProcessTransport(ITransport): - """ - A process transport. - """ - - pid = Attribute( - "From before L{IProcessProtocol.makeConnection} is called to before " - "L{IProcessProtocol.processEnded} is called, C{pid} is an L{int} " - "giving the platform process ID of this process. C{pid} is L{None} " - "at all other times.") - - def closeStdin(): - """ - Close stdin after all data has been written out. - """ - - def closeStdout(): - """ - Close stdout. - """ - - def closeStderr(): - """ - Close stderr. - """ - - def closeChildFD(descriptor): - """ - Close a file descriptor which is connected to the child process, identified - by its FD in the child process. - """ - - def writeToChild(childFD, data): - """ - Similar to L{ITransport.write} but also allows the file descriptor in - the child process which will receive the bytes to be specified. - - @type childFD: C{int} - @param childFD: The file descriptor to which to write. - - @type data: C{str} - @param data: The bytes to write. - - @return: C{None} - - @raise KeyError: If C{childFD} is not a file descriptor that was mapped - in the child when L{IReactorProcess.spawnProcess} was used to create - it. - """ - - def loseConnection(): - """ - Close stdin, stderr and stdout. - """ - - def signalProcess(signalID): - """ - Send a signal to the process. - - @param signalID: can be - - one of C{"KILL"}, C{"TERM"}, or C{"INT"}. - These will be implemented in a - cross-platform manner, and so should be used - if possible. - - an integer, where it represents a POSIX - signal ID. - - @raise twisted.internet.error.ProcessExitedAlready: If the process has - already exited. - @raise OSError: If the C{os.kill} call fails with an errno different - from C{ESRCH}. - """ - - -class IServiceCollection(Interface): - """ - An object which provides access to a collection of services. - """ - - def getServiceNamed(serviceName): - """ - Retrieve the named service from this application. - - Raise a C{KeyError} if there is no such service name. - """ - - def addService(service): - """ - Add a service to this collection. - """ - - def removeService(service): - """ - Remove a service from this collection. - """ - - -class IUDPTransport(Interface): - """ - Transport for UDP DatagramProtocols. - """ - - def write(packet, addr=None): - """ - Write packet to given address. - - @param addr: a tuple of (ip, port). For connected transports must - be the address the transport is connected to, or None. - In non-connected mode this is mandatory. - - @raise twisted.internet.error.MessageLengthError: C{packet} was too - long. - """ - - def connect(host, port): - """ - Connect the transport to an address. - - This changes it to connected mode. Datagrams can only be sent to - this address, and will only be received from this address. In addition - the protocol's connectionRefused method might get called if destination - is not receiving datagrams. - - @param host: an IP address, not a domain name ('127.0.0.1', not 'localhost') - @param port: port to connect to. - """ - - def getHost(): - """ - Returns L{IPv4Address}. - """ - - def stopListening(): - """ - Stop listening on this port. - - If it does not complete immediately, will return L{Deferred} that fires - upon completion. - """ - - - -class IUNIXDatagramTransport(Interface): - """ - Transport for UDP PacketProtocols. - """ - - def write(packet, address): - """ - Write packet to given address. - """ - - def getHost(): - """ - Returns L{UNIXAddress}. - """ - - -class IUNIXDatagramConnectedTransport(Interface): - """ - Transport for UDP ConnectedPacketProtocols. - """ - - def write(packet): - """ - Write packet to address we are connected to. - """ - - def getHost(): - """ - Returns L{UNIXAddress}. - """ - - def getPeer(): - """ - Returns L{UNIXAddress}. - """ - - -class IMulticastTransport(Interface): - """ - Additional functionality for multicast UDP. - """ - - def getOutgoingInterface(): - """ - Return interface of outgoing multicast packets. - """ - - def setOutgoingInterface(addr): - """ - Set interface for outgoing multicast packets. - - Returns Deferred of success. - """ - - def getLoopbackMode(): - """ - Return if loopback mode is enabled. - """ - - def setLoopbackMode(mode): - """ - Set if loopback mode is enabled. - """ - - def getTTL(): - """ - Get time to live for multicast packets. - """ - - def setTTL(ttl): - """ - Set time to live on multicast packets. - """ - - def joinGroup(addr, interface=""): - """ - Join a multicast group. Returns L{Deferred} of success or failure. - - If an error occurs, the returned L{Deferred} will fail with - L{error.MulticastJoinError}. - """ - - def leaveGroup(addr, interface=""): - """ - Leave multicast group, return L{Deferred} of success. - """ - - -class IStreamClientEndpoint(Interface): - """ - A stream client endpoint is a place that L{ClientFactory} can connect to. - For example, a remote TCP host/port pair would be a TCP client endpoint. - - @since: 10.1 - """ - - def connect(protocolFactory): - """ - Connect the C{protocolFactory} to the location specified by this - L{IStreamClientEndpoint} provider. - - @param protocolFactory: A provider of L{IProtocolFactory} - @return: A L{Deferred} that results in an L{IProtocol} upon successful - connection otherwise a L{ConnectError} - """ - - - -class IStreamServerEndpoint(Interface): - """ - A stream server endpoint is a place that a L{Factory} can listen for - incoming connections. - - @since: 10.1 - """ - - def listen(protocolFactory): - """ - Listen with C{protocolFactory} at the location specified by this - L{IStreamServerEndpoint} provider. - - @param protocolFactory: A provider of L{IProtocolFactory} - @return: A L{Deferred} that results in an L{IListeningPort} or an - L{CannotListenError} - """ - - - -class IStreamServerEndpointStringParser(Interface): - """ - An L{IStreamServerEndpointStringParser} is like an - L{IStreamClientEndpointStringParser}, except for L{IStreamServerEndpoint}s - instead of clients. It integrates with L{endpoints.serverFromString} in - much the same way. - """ - - prefix = Attribute( - """ - @see: L{IStreamClientEndpointStringParser.prefix} - """ - ) - - - def parseStreamServer(reactor, *args, **kwargs): - """ - Parse a stream server endpoint from a reactor and string-only arguments - and keyword arguments. - - @see: L{IStreamClientEndpointStringParser.parseStreamClient} - - @return: a stream server endpoint - @rtype: L{IStreamServerEndpoint} - """ - - - -class IStreamClientEndpointStringParser(Interface): - """ - An L{IStreamClientEndpointStringParser} is a parser which can convert - a set of string C{*args} and C{**kwargs} into an L{IStreamClientEndpoint} - provider. - - This interface is really only useful in the context of the plugin system - for L{endpoints.clientFromString}. See the document entitled "I{The - Twisted Plugin System}" for more details on how to write a plugin. - - If you place an L{IStreamClientEndpointStringParser} plugin in the - C{twisted.plugins} package, that plugin's C{parseStreamClient} method will - be used to produce endpoints for any description string that begins with - the result of that L{IStreamClientEndpointStringParser}'s prefix attribute. - """ - - prefix = Attribute( - """ - A C{str}, the description prefix to respond to. For example, an - L{IStreamClientEndpointStringParser} plugin which had C{"foo"} for its - C{prefix} attribute would be called for endpoint descriptions like - C{"foo:bar:baz"} or C{"foo:"}. - """ - ) - - - def parseStreamClient(*args, **kwargs): - """ - This method is invoked by L{endpoints.clientFromString}, if the type of - endpoint matches the return value from this - L{IStreamClientEndpointStringParser}'s C{prefix} method. - - @param args: The string arguments, minus the endpoint type, in the - endpoint description string, parsed according to the rules - described in L{endpoints.quoteStringArgument}. For example, if the - description were C{"my-type:foo:bar:baz=qux"}, C{args} would be - C{('foo','bar')} - - @param kwargs: The string arguments from the endpoint description - passed as keyword arguments. For example, if the description were - C{"my-type:foo:bar:baz=qux"}, C{kwargs} would be - C{dict(baz='qux')}. - - @return: a client endpoint - @rtype: L{IStreamClientEndpoint} - """ -}}} ''' diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py index 32c88ff2d..fd814dc22 100644 --- a/scrapy/xlib/tx/iweb.py +++ b/scrapy/xlib/tx/iweb.py @@ -10,580 +10,11 @@ Interface definitions for L{twisted.web}. body is not known in advance. """ -from zope.interface import Interface, Attribute - -#from twisted.internet.interfaces import IPushProducer - from twisted.web.iweb import ( IRequest, ICredentialFactory, IBodyProducer, IRenderable, ITemplateLoader, IResponse, _IRequestEncoder, _IRequestEncoderFactory, UNKNOWN_LENGTH, ) -''' {{{ -class IRequest(Interface): - """ - An HTTP request. - - @since: 9.0 - """ - - method = Attribute("A C{str} giving the HTTP method that was used.") - uri = Attribute( - "A C{str} giving the full encoded URI which was requested (including " - "query arguments).") - path = Attribute( - "A C{str} giving the encoded query path of the request URI.") - args = Attribute( - "A mapping of decoded query argument names as C{str} to " - "corresponding query argument values as C{list}s of C{str}. " - "For example, for a URI with C{'foo=bar&foo=baz&quux=spam'} " - "for its query part, C{args} will be C{{'foo': ['bar', 'baz'], " - "'quux': ['spam']}}.") - - received_headers = Attribute( - "Backwards-compatibility access to C{requestHeaders}. Use " - "C{requestHeaders} instead. C{received_headers} behaves mostly " - "like a C{dict} and does not provide access to all header values.") - - requestHeaders = Attribute( - "A L{http_headers.Headers} instance giving all received HTTP request " - "headers.") - - content = Attribute( - "A file-like object giving the request body. This may be a file on " - "disk, a C{StringIO}, or some other type. The implementation is free " - "to decide on a per-request basis.") - - headers = Attribute( - "Backwards-compatibility access to C{responseHeaders}. Use" - "C{responseHeaders} instead. C{headers} behaves mostly like a " - "C{dict} and does not provide access to all header values nor " - "does it allow multiple values for one header to be set.") - - responseHeaders = Attribute( - "A L{http_headers.Headers} instance holding all HTTP response " - "headers to be sent.") - - def getHeader(key): - """ - Get an HTTP request header. - - @type key: C{str} - @param key: The name of the header to get the value of. - - @rtype: C{str} or C{NoneType} - @return: The value of the specified header, or C{None} if that header - was not present in the request. - """ - - - def getCookie(key): - """ - Get a cookie that was sent from the network. - """ - - - def getAllHeaders(): - """ - Return dictionary mapping the names of all received headers to the last - value received for each. - - Since this method does not return all header information, - C{requestHeaders.getAllRawHeaders()} may be preferred. - """ - - - def getRequestHostname(): - """ - Get the hostname that the user passed in to the request. - - This will either use the Host: header (if it is available) or the - host we are listening on if the header is unavailable. - - @returns: the requested hostname - @rtype: C{str} - """ - - - def getHost(): - """ - Get my originally requesting transport's host. - - @return: An L{IAddress}. - """ - - - def getClientIP(): - """ - Return the IP address of the client who submitted this request. - - @returns: the client IP address or C{None} if the request was submitted - over a transport where IP addresses do not make sense. - @rtype: L{str} or C{NoneType} - """ - - - def getClient(): - """ - Return the hostname of the IP address of the client who submitted this - request, if possible. - - This method is B{deprecated}. See L{getClientIP} instead. - - @rtype: C{NoneType} or L{str} - @return: The canonical hostname of the client, as determined by - performing a name lookup on the IP address of the client. - """ - - - def getUser(): - """ - Return the HTTP user sent with this request, if any. - - If no user was supplied, return the empty string. - - @returns: the HTTP user, if any - @rtype: C{str} - """ - - - def getPassword(): - """ - Return the HTTP password sent with this request, if any. - - If no password was supplied, return the empty string. - - @returns: the HTTP password, if any - @rtype: C{str} - """ - - - def isSecure(): - """ - Return True if this request is using a secure transport. - - Normally this method returns True if this request's HTTPChannel - instance is using a transport that implements ISSLTransport. - - This will also return True if setHost() has been called - with ssl=True. - - @returns: True if this request is secure - @rtype: C{bool} - """ - - - def getSession(sessionInterface=None): - """ - Look up the session associated with this request or create a new one if - there is not one. - - @return: The L{Session} instance identified by the session cookie in - the request, or the C{sessionInterface} component of that session - if C{sessionInterface} is specified. - """ - - - def URLPath(): - """ - @return: A L{URLPath} instance which identifies the URL for which this - request is. - """ - - - def prePathURL(): - """ - @return: At any time during resource traversal, a L{str} giving an - absolute URL to the most nested resource which has yet been - reached. - """ - - - def rememberRootURL(): - """ - Remember the currently-processed part of the URL for later - recalling. - """ - - - def getRootURL(): - """ - Get a previously-remembered URL. - """ - - - # Methods for outgoing response - def finish(): - """ - Indicate that the response to this request is complete. - """ - - - def write(data): - """ - Write some data to the body of the response to this request. Response - headers are written the first time this method is called, after which - new response headers may not be added. - """ - - - def addCookie(k, v, expires=None, domain=None, path=None, max_age=None, comment=None, secure=None): - """ - Set an outgoing HTTP cookie. - - In general, you should consider using sessions instead of cookies, see - L{twisted.web.server.Request.getSession} and the - L{twisted.web.server.Session} class for details. - """ - - - def setResponseCode(code, message=None): - """ - Set the HTTP response code. - """ - - - def setHeader(k, v): - """ - Set an HTTP response header. Overrides any previously set values for - this header. - - @type name: C{str} - @param name: The name of the header for which to set the value. - - @type value: C{str} - @param value: The value to set for the named header. - """ - - - def redirect(url): - """ - Utility function that does a redirect. - - The request should have finish() called after this. - """ - - - def setLastModified(when): - """ - Set the C{Last-Modified} time for the response to this request. - - If I am called more than once, I ignore attempts to set Last-Modified - earlier, only replacing the Last-Modified time if it is to a later - value. - - If I am a conditional request, I may modify my response code to - L{NOT_MODIFIED} if appropriate for the time given. - - @param when: The last time the resource being returned was modified, in - seconds since the epoch. - @type when: L{int}, L{long} or L{float} - - @return: If I am a C{If-Modified-Since} conditional request and the time - given is not newer than the condition, I return - L{CACHED} to indicate that you should write no body. - Otherwise, I return a false value. - """ - - - def setETag(etag): - """ - Set an C{entity tag} for the outgoing response. - - That's "entity tag" as in the HTTP/1.1 I{ETag} header, "used for - comparing two or more entities from the same requested resource." - - If I am a conditional request, I may modify my response code to - L{NOT_MODIFIED} or - L{PRECONDITION_FAILED}, if appropriate for the - tag given. - - @param etag: The entity tag for the resource being returned. - @type etag: C{str} - - @return: If I am a C{If-None-Match} conditional request and the tag - matches one in the request, I return L{CACHED} to - indicate that you should write no body. Otherwise, I return a - false value. - """ - - - def setHost(host, port, ssl=0): - """ - Change the host and port the request thinks it's using. - - This method is useful for working with reverse HTTP proxies (e.g. both - Squid and Apache's mod_proxy can do this), when the address the HTTP - client is using is different than the one we're listening on. - - For example, Apache may be listening on https://www.example.com, and - then forwarding requests to http://localhost:8080, but we don't want - HTML produced by Twisted to say 'http://localhost:8080', they should - say 'https://www.example.com', so we do:: - - request.setHost('www.example.com', 443, ssl=1) - """ - - - -class ICredentialFactory(Interface): - """ - A credential factory defines a way to generate a particular kind of - authentication challenge and a way to interpret the responses to these - challenges. It creates - L{ICredentials} providers from - responses. These objects will be used with L{twisted.cred} to authenticate - an authorize requests. - """ - scheme = Attribute( - "A C{str} giving the name of the authentication scheme with which " - "this factory is associated. For example, C{'basic'} or C{'digest'}.") - - - def getChallenge(request): - """ - Generate a new challenge to be sent to a client. - - @type peer: L{twisted.web.http.Request} - @param peer: The request the response to which this challenge will be - included. - - @rtype: C{dict} - @return: A mapping from C{str} challenge fields to associated C{str} - values. - """ - - - def decode(response, request): - """ - Create a credentials object from the given response. - - @type response: C{str} - @param response: scheme specific response string - - @type request: L{twisted.web.http.Request} - @param request: The request being processed (from which the response - was taken). - - @raise twisted.cred.error.LoginFailed: If the response is invalid. - - @rtype: L{twisted.cred.credentials.ICredentials} provider - @return: The credentials represented by the given response. - """ - - - -class IBodyProducer(IPushProducer): - """ - Objects which provide L{IBodyProducer} write bytes to an object which - provides L{IConsumer} by calling its - C{write} method repeatedly. - - L{IBodyProducer} providers may start producing as soon as they have an - L{IConsumer} provider. That is, they - should not wait for a C{resumeProducing} call to begin writing data. - - L{IConsumer.unregisterProducer} - must not be called. Instead, the - L{Deferred} returned from C{startProducing} - must be fired when all bytes have been written. - - L{IConsumer.write} may - synchronously invoke any of C{pauseProducing}, C{resumeProducing}, or - C{stopProducing}. These methods must be implemented with this in mind. - - @since: 9.0 - """ - - # Despite the restrictions above and the additional requirements of - # stopProducing documented below, this interface still needs to be an - # IPushProducer subclass. Providers of it will be passed to IConsumer - # providers which only know about IPushProducer and IPullProducer, not - # about this interface. This interface needs to remain close enough to one - # of those interfaces for consumers to work with it. - - length = Attribute( - """ - C{length} is a C{int} indicating how many bytes in total this - L{IBodyProducer} will write to the consumer or L{UNKNOWN_LENGTH} - if this is not known in advance. - """) - - def startProducing(consumer): - """ - Start producing to the given - L{IConsumer} provider. - - @return: A L{Deferred} which fires with - C{None} when all bytes have been produced or with a - L{Failure} if there is any problem - before all bytes have been produced. - """ - - - def stopProducing(): - """ - In addition to the standard behavior of - L{IProducer.stopProducing} - (stop producing data), make sure the - L{Deferred} returned by - C{startProducing} is never fired. - """ - - - -class IRenderable(Interface): - """ - An L{IRenderable} is an object that may be rendered by the - L{twisted.web.template} templating system. - """ - - def lookupRenderMethod(name): - """ - Look up and return the render method associated with the given name. - - @type name: C{str} - @param name: The value of a render directive encountered in the - document returned by a call to L{IRenderable.render}. - - @return: A two-argument callable which will be invoked with the request - being responded to and the tag object on which the render directive - was encountered. - """ - - - def render(request): - """ - Get the document for this L{IRenderable}. - - @type request: L{IRequest} provider or C{NoneType} - @param request: The request in response to which this method is being - invoked. - - @return: An object which can be flattened. - """ - - - -class ITemplateLoader(Interface): - """ - A loader for templates; something usable as a value for - L{twisted.web.template.Element}'s C{loader} attribute. - """ - - def load(): - """ - Load a template suitable for rendering. - - @return: a C{list} of C{list}s, C{unicode} objects, C{Element}s and - other L{IRenderable} providers. - """ - - - -class IResponse(Interface): - """ - An object representing an HTTP response received from an HTTP server. - - @since: 11.1 - """ - - version = Attribute( - "A three-tuple describing the protocol and protocol version " - "of the response. The first element is of type C{str}, the second " - "and third are of type C{int}. For example, C{('HTTP', 1, 1)}.") - - - code = Attribute("The HTTP status code of this response, as a C{int}.") - - - phrase = Attribute( - "The HTTP reason phrase of this response, as a C{str}.") - - - headers = Attribute("The HTTP response L{Headers} of this response.") - - - length = Attribute( - "The C{int} number of bytes expected to be in the body of this " - "response or L{UNKNOWN_LENGTH} if the server did not indicate how " - "many bytes to expect. For I{HEAD} responses, this will be 0; if " - "the response includes a I{Content-Length} header, it will be " - "available in C{headers}.") - - - def deliverBody(protocol): - """ - Register an L{IProtocol} provider - to receive the response body. - - The protocol will be connected to a transport which provides - L{IPushProducer}. The protocol's C{connectionLost} method will be - called with: - - - ResponseDone, which indicates that all bytes from the response - have been successfully delivered. - - - PotentialDataLoss, which indicates that it cannot be determined - if the entire response body has been delivered. This only occurs - when making requests to HTTP servers which do not set - I{Content-Length} or a I{Transfer-Encoding} in the response. - - - ResponseFailed, which indicates that some bytes from the response - were lost. The C{reasons} attribute of the exception may provide - more specific indications as to why. - """ - - - -class _IRequestEncoder(Interface): - """ - An object encoding data passed to L{IRequest.write}, for example for - compression purpose. - - @since: 12.3 - """ - - def encode(data): - """ - Encode the data given and return the result. - - @param data: The content to encode. - @type data: C{str} - - @return: The encoded data. - @rtype: C{str} - """ - - - def finish(): - """ - Callback called when the request is closing. - - @return: If necessary, the pending data accumulated from previous - C{encode} calls. - @rtype: C{str} - """ - - - -class _IRequestEncoderFactory(Interface): - """ - A factory for returing L{_IRequestEncoder} instances. - - @since: 12.3 - """ - - def encoderForRequest(request): - """ - If applicable, returns a L{_IRequestEncoder} instance which will encode - the request. - """ - - - -UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH" -}}} ''' __all__ = [ "ICredentialFactory", "IRequest", "IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder",