diff --git a/requirements.txt b/requirements.txt index cfa907050..64b6e771c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Twisted>=10.0.0 +Twisted>=13.1.0 lxml pyOpenSSL cssselect>=0.9 diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py index e902d6683..d20eda34f 100644 --- a/scrapy/xlib/tx/_newclient.py +++ b/scrapy/xlib/tx/_newclient.py @@ -38,26 +38,21 @@ from twisted.internet.error import ConnectionDone from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred from twisted.internet.defer import CancelledError from twisted.internet.protocol import Protocol -from twisted.protocols.basic import LineReceiver -from twisted.web.iweb import UNKNOWN_LENGTH -from twisted.web.http_headers import Headers +#from twisted.protocols.basic import LineReceiver +from twisted.web.iweb import UNKNOWN_LENGTH, IResponse +#from twisted.web.http_headers import Headers from twisted.web.http import NO_CONTENT, NOT_MODIFIED from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder from twisted.web._newclient import ( BadHeaders, ExcessWrite, ParseError, BadResponseVersion, _WrapperException, - RequestGenerationFailed, RequestTransmissionFailed, - WrongBodyLength, ResponseDone, RequestNotSent, - LengthEnforcingConsumer, makeStatefulDispatcher, ChunkedEncoder, - TransportProxyProducer, + RequestGenerationFailed, RequestTransmissionFailed, ConnectionAborted, + WrongBodyLength, ResponseDone, ResponseFailed, RequestNotSent, + ResponseNeverReceived, HTTPParser, HTTPClientParser, Request, + LengthEnforcingConsumer, makeStatefulDispatcher, Response, ChunkedEncoder, + TransportProxyProducer, HTTP11ClientProtocol ) -# newer than 10.0.0 -#from twisted.web._newclient import ( -# ConnectionAborted, ResponseFailed, ResponseNeverReceived, HTTPParser, -# HTTPClientParser, Request, Response, HTTP11ClientProtocol, -#) -from .iweb import IResponse # States HTTPParser can be in STATUS = 'STATUS' @@ -130,7 +125,7 @@ class RequestTransmissionFailed(_WrapperException): @ivar reasons: A C{list} of one or more L{Failure} instances giving the reasons the request transmission was considered to have failed. """ -}}} ''' + class ConnectionAborted(Exception): @@ -139,7 +134,7 @@ class ConnectionAborted(Exception): """ -''' {{{ + class WrongBodyLength(Exception): """ An L{IBodyProducer} declared the number of bytes it was going to @@ -155,7 +150,7 @@ class ResponseDone(Exception): protocol passed to L{Response.deliverBody} and indicates that the entire response has been delivered. """ -}}} ''' + class ResponseFailed(_WrapperException): @@ -182,7 +177,7 @@ class ResponseNeverReceived(ResponseFailed): """ -''' {{{ + class RequestNotSent(Exception): """ L{RequestNotSent} indicates that an attempt was made to issue a request but @@ -191,7 +186,7 @@ class RequestNotSent(Exception): to send a request using a protocol which is no longer connected to a server. """ -}}} ''' + def _callAppFunction(function): @@ -777,7 +772,7 @@ class Request: _callAppFunction(self.bodyProducer.stopProducing) -''' {{{ + class LengthEnforcingConsumer: """ An L{IConsumer} proxy which enforces an exact length requirement on the @@ -1201,7 +1196,7 @@ class TransportProxyProducer: """ if self._producer is not None: self._producer.pauseProducing() -}}} ''' + class HTTP11ClientProtocol(Protocol): @@ -1527,3 +1522,4 @@ class HTTP11ClientProtocol(Protocol): d = Deferred() self._abortDeferreds.append(d) return d +}}} ''' diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py index 396115985..8e0b1df8b 100644 --- a/scrapy/xlib/tx/client.py +++ b/scrapy/xlib/tx/client.py @@ -29,23 +29,18 @@ from twisted.python.failure import Failure from twisted.web import http from twisted.internet import defer, protocol, task, reactor from twisted.internet.interfaces import IProtocol +from twisted.internet.endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint from twisted.python import failure from twisted.python.components import proxyForInterface from twisted.web import error -from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponse from twisted.web.http_headers import Headers from twisted.web.client import ( - PartialDownloadError, + PartialDownloadError, FileBodyProducer, + CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, + Agent, ProxyAgent, HTTPConnectionPool, readBody, ) -# newer than 10.0.0 -#from twisted.web.client import ( -# CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, FileBodyProducer, -# HTTPConnectionPool, Agent, ProxyAgent, -#) - -from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint -from .iweb import IResponse ''' {{{ class PartialDownloadError(error.Error): @@ -54,7 +49,7 @@ class PartialDownloadError(error.Error): @ivar response: All of the response body which was downloaded. """ -}}} ''' + class _URL(tuple): """ @@ -138,22 +133,21 @@ def _makeGetterFactory(url, factoryFactory, contextFactory=None, else: reactor.connectTCP(host, port, factory) return factory - +}}} ''' # The code which follows is based on the new HTTP client implementation. It # should be significantly better than anything above, though it is not yet # feature equivalent. -from twisted.web.error import SchemeNotSupported -from ._newclient import Request, Response, HTTP11ClientProtocol -from twisted.web._newclient import ResponseDone -from ._newclient import ResponseFailed -from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed -from twisted.web._newclient import ( - PotentialDataLoss, _WrapperException) -from ._newclient import ( - ResponseNeverReceived) +#from twisted.web.error import SchemeNotSupported +from twisted.web._newclient import Response +#from twisted.web._newclient import Request, HTTP11ClientProtocol +from twisted.web._newclient import ResponseDone, ResponseFailed +#from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed +#from twisted.web._newclient import ( +# ResponseNeverReceived, PotentialDataLoss, _WrapperException) +''' {{{ try: from twisted.internet.ssl import ClientContextFactory except ImportError: @@ -1170,7 +1164,7 @@ def readBody(response): d = defer.Deferred() response.deliverBody(_ReadBodyProtocol(response.code, response.phrase, d)) return d - +}}} ''' __all__ = [ diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py index 21a467433..3f4704064 100644 --- a/scrapy/xlib/tx/endpoints.py +++ b/scrapy/xlib/tx/endpoints.py @@ -21,38 +21,33 @@ from zope.interface import implementer, directlyProvides import warnings from twisted.internet import interfaces, defer, error, fdesc -from twisted.internet.protocol import ( - ClientFactory, Protocol, Factory) +#from twisted.internet.protocol import ( +# ClientFactory, Protocol) +from twisted.internet.protocol import Factory #from twisted.internet import threads, ProcessProtocol from twisted.internet.interfaces import IStreamServerEndpointStringParser -from twisted.internet.interfaces import IStreamClientEndpointStringParser +#from twisted.internet.interfaces import IStreamClientEndpointStringParser from twisted.python.filepath import FilePath #from twisted.python.failure import Failure #from twisted.python import log -from twisted.python.components import proxyForInterface +#from twisted.python.components import proxyForInterface from twisted.plugin import IPlugin, getPlugins #from twisted.internet import stdio -# newer than 10.0.0 -#from twisted.internet.endpoints import ( -# TCP4ServerEndpoint, TCP6ServerEndpoint, TCP4ClientEndpoint, SSL4ServerEndpoint, SSL4ClientEndpoint, -# UNIXServerEndpoint, UNIXClientEndpoint, AdoptedStreamServerEndpoint, connectProtocol, -# quoteStringArgument, -# serverFromString, #> using newer _parseSSL, _tokenize in _serverParsers -# clientFromString, #> using newer _clientParsers -# _WrappingProtocol, _WrappingFactory, _TCPServerEndpoint, -# _parseTCP, _parseUNIX, _loadCAsFromDir, -# _parseSSL, _tokenize, -# _parseClientTCP, _parseClientSSL, _parseClientUNIX, -#) - -from .interfaces import IFileDescriptorReceiver +from twisted.internet.endpoints import ( + clientFromString, serverFromString, quoteStringArgument, + TCP4ServerEndpoint, TCP6ServerEndpoint, + TCP4ClientEndpoint, TCP6ClientEndpoint, + UNIXServerEndpoint, UNIXClientEndpoint, + SSL4ServerEndpoint, SSL4ClientEndpoint, + AdoptedStreamServerEndpoint, connectProtocol, +) __all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"] - +''' {{{ class _WrappingProtocol(Protocol): """ Wrap another protocol in order to notify my user when a connection has @@ -71,7 +66,7 @@ class _WrappingProtocol(Protocol): self._wrappedProtocol = wrappedProtocol for iface in [interfaces.IHalfCloseableProtocol, - IFileDescriptorReceiver]: + interfaces.IFileDescriptorReceiver]: if iface.providedBy(self._wrappedProtocol): directlyProvides(self, iface) @@ -609,6 +604,7 @@ class AdoptedStreamServerEndpoint(object): + def _parseTCP(factory, port, interface="", backlog=50): """ Internal parser function for L{_parseServer} to convert the string @@ -1280,4 +1276,4 @@ def connectProtocol(endpoint, protocol): def buildProtocol(self, addr): return protocol return endpoint.connect(OneShotFactory()) - +}}} ''' diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py index a715d4a05..7b2a78632 100644 --- a/scrapy/xlib/tx/interfaces.py +++ b/scrapy/xlib/tx/interfaces.py @@ -13,24 +13,21 @@ from zope.interface import Interface, Attribute from twisted.internet.interfaces import ( IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL, - IReactorUDP, IReactorMulticast, IReactorProcess, + IReactorWin32Events, IReactorUDP, IReactorMulticast, IReactorProcess, IReactorTime, IDelayedCall, IReactorThreads, IReactorCore, - IReactorPluggableResolver, IReactorFDSet, + IReactorPluggableResolver, IReactorDaemonize, IReactorFDSet, IListeningPort, ILoggingContext, IFileDescriptor, IReadDescriptor, IWriteDescriptor, IReadWriteDescriptor, IHalfCloseableDescriptor, ISystemHandle, IConsumer, IProducer, IPushProducer, IPullProducer, IProtocol, IProcessProtocol, IHalfCloseableProtocol, - IProtocolFactory, ITransport, IProcessTransport, IServiceCollection, + IFileDescriptorReceiver, IProtocolFactory, ITransport, ITCPTransport, + IUNIXTransport, + ITLSTransport, ISSLTransport, IProcessTransport, IServiceCollection, IUDPTransport, IUNIXDatagramTransport, IUNIXDatagramConnectedTransport, - IMulticastTransport, + IMulticastTransport, IStreamClientEndpoint, IStreamServerEndpoint, + IStreamServerEndpointStringParser, IStreamClientEndpointStringParser, + IReactorUNIX, IReactorUNIXDatagram, IReactorSocket, IResolver ) -# newer than 10.0.0 -#from twisted.internet.interfaces import ( -# IResolver, IReactorUNIX, IReactorUNIXDatagram, IReactorWin32Events, IReactorSocket, -# IReactorDaemonize, IFileDescriptorReceiver, ITCPTransport, IUNIXTransport, -# ITLSTransport, ISSLTransport, IStreamClientEndpoint, IStreamServerEndpoint, -# IStreamServerEndpointStringParser, IStreamClientEndpointStringParser, -#) ''' {{{ class IAddress(Interface): @@ -95,7 +92,7 @@ class IResolverSimple(Interface): @raise twisted.internet.defer.TimeoutError: Raised (asynchronously) if the name cannot be resolved within the specified timeout period. """ -}}} ''' + class IResolver(IResolverSimple): @@ -635,7 +632,7 @@ class IResolver(IResolverSimple): """ -''' {{{ + class IReactorTCP(Interface): def listenTCP(port, factory, backlog=50, interface=''): @@ -722,7 +719,7 @@ class IReactorSSL(Interface): @param interface: the hostname to bind to, defaults to '' (all) """ -}}} ''' + class IReactorUNIX(Interface): @@ -850,7 +847,7 @@ class IReactorWin32Events(Interface): """ -''' {{{ + class IReactorUDP(Interface): """ UDP socket methods. @@ -889,7 +886,7 @@ class IReactorMulticast(Interface): @see: L{twisted.internet.interfaces.IMulticastTransport} @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html} """ -}}} ''' + class IReactorSocket(Interface): @@ -991,7 +988,7 @@ class IReactorSocket(Interface): """ -''' {{{ + class IReactorProcess(Interface): def spawnProcess(processProtocol, executable, args=(), env={}, path=None, @@ -1368,7 +1365,7 @@ class IReactorPluggableResolver(Interface): @return: The previously installed resolver. """ -}}} ''' + class IReactorDaemonize(Interface): """ @@ -1400,7 +1397,7 @@ class IReactorDaemonize(Interface): """ -''' {{{ + class IReactorFDSet(Interface): """ Implement me to be able to use L{IFileDescriptor} type resources. @@ -1884,7 +1881,7 @@ class IHalfCloseableProtocol(Interface): This will never be called for TCP connections as TCP does not support notification of this type of half-close. """ -}}} ''' + class IFileDescriptorReceiver(Interface): @@ -1905,7 +1902,7 @@ class IFileDescriptorReceiver(Interface): """ -''' {{{ + class IProtocolFactory(Interface): """ Interface for protocol factories. @@ -1995,7 +1992,7 @@ class ITransport(Interface): @return: An L{IAddress} provider. """ -}}} ''' + class ITCPTransport(ITransport): """ @@ -2116,7 +2113,7 @@ class ISSLTransport(ITCPTransport): Return an object with the peer's certificate info. """ -''' {{{ + class IProcessTransport(ITransport): """ A process transport. @@ -2345,7 +2342,7 @@ class IMulticastTransport(Interface): """ Leave multicast group, return L{Deferred} of success. """ -}}} ''' + class IStreamClientEndpoint(Interface): """ @@ -2461,3 +2458,4 @@ class IStreamClientEndpointStringParser(Interface): @return: a client endpoint @rtype: L{IStreamClientEndpoint} """ +}}} ''' diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py index 57a111411..32c88ff2d 100644 --- a/scrapy/xlib/tx/iweb.py +++ b/scrapy/xlib/tx/iweb.py @@ -15,14 +15,11 @@ from zope.interface import Interface, Attribute #from twisted.internet.interfaces import IPushProducer from twisted.web.iweb import ( - ICredentialFactory, IBodyProducer, - UNKNOWN_LENGTH, + IRequest, ICredentialFactory, IBodyProducer, IRenderable, ITemplateLoader, + IResponse, _IRequestEncoder, _IRequestEncoderFactory, UNKNOWN_LENGTH, ) -# newer than 10.0.0 -#from twisted.web.iweb import ( -# IRequest, IRenderable, ITemplateLoader, IResponse, _IRequestEncoder, _IRequestEncoderFactory, -#) +''' {{{ class IRequest(Interface): """ An HTTP request. @@ -328,7 +325,7 @@ class IRequest(Interface): """ -''' {{{ + class ICredentialFactory(Interface): """ A credential factory defines a way to generate a particular kind of @@ -432,7 +429,7 @@ class IBodyProducer(IPushProducer): L{Deferred} returned by C{startProducing} is never fired. """ -}}} ''' + class IRenderable(Interface): @@ -584,7 +581,7 @@ class _IRequestEncoderFactory(Interface): """ -''' {{{ + UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH" }}} ''' __all__ = [ diff --git a/setup.py b/setup.py index 92c114a7a..5e32d4240 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( 'Topic :: Software Development :: Libraries :: Python Modules', ], install_requires=[ - 'Twisted>=10.0.0', + 'Twisted>=13.1.0', 'w3lib>=1.15.0', 'queuelib', 'lxml',