Bump Twisted dependency to 13.1.0 (released June 2013)

requirements.txt

@@ -1,4 +1,4 @@
-Twisted>=10.0.0
+Twisted>=13.1.0
 lxml
 pyOpenSSL
 cssselect>=0.9

scrapy/xlib/tx/_newclient.py

@@ -38,26 +38,21 @@ from twisted.internet.error import ConnectionDone
 from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred
 from twisted.internet.defer import CancelledError
 from twisted.internet.protocol import Protocol
-from twisted.protocols.basic import LineReceiver
-from twisted.web.iweb import UNKNOWN_LENGTH
-from twisted.web.http_headers import Headers
+#from twisted.protocols.basic import LineReceiver
+from twisted.web.iweb import UNKNOWN_LENGTH, IResponse
+#from twisted.web.http_headers import Headers
 from twisted.web.http import NO_CONTENT, NOT_MODIFIED
 from twisted.web.http import _DataLoss, PotentialDataLoss
 from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder
 
 from twisted.web._newclient import (
     BadHeaders, ExcessWrite, ParseError, BadResponseVersion, _WrapperException,
-    RequestGenerationFailed, RequestTransmissionFailed,
-    WrongBodyLength, ResponseDone, RequestNotSent,
-    LengthEnforcingConsumer, makeStatefulDispatcher, ChunkedEncoder,
-    TransportProxyProducer,
+    RequestGenerationFailed, RequestTransmissionFailed, ConnectionAborted,
+    WrongBodyLength, ResponseDone, ResponseFailed, RequestNotSent,
+    ResponseNeverReceived, HTTPParser, HTTPClientParser, Request,
+    LengthEnforcingConsumer, makeStatefulDispatcher, Response, ChunkedEncoder,
+    TransportProxyProducer, HTTP11ClientProtocol
 )
-# newer than 10.0.0
-#from twisted.web._newclient import (
-#    ConnectionAborted, ResponseFailed, ResponseNeverReceived, HTTPParser,
-#    HTTPClientParser, Request, Response, HTTP11ClientProtocol,
-#)
-from .iweb import IResponse
 
 # States HTTPParser can be in
 STATUS = 'STATUS'
@@ -130,7 +125,7 @@ class RequestTransmissionFailed(_WrapperException):
     @ivar reasons: A C{list} of one or more L{Failure} instances giving the
         reasons the request transmission was considered to have failed.
     """
-}}} '''
+
 
 
 class ConnectionAborted(Exception):
@@ -139,7 +134,7 @@ class ConnectionAborted(Exception):
     """
 
 
-''' {{{
+
 class WrongBodyLength(Exception):
     """
     An L{IBodyProducer} declared the number of bytes it was going to
@@ -155,7 +150,7 @@ class ResponseDone(Exception):
     protocol passed to L{Response.deliverBody} and indicates that the entire
     response has been delivered.
     """
-}}} '''
+
 
 
 class ResponseFailed(_WrapperException):
@@ -182,7 +177,7 @@ class ResponseNeverReceived(ResponseFailed):
     """
 
 
-''' {{{
+
 class RequestNotSent(Exception):
     """
     L{RequestNotSent} indicates that an attempt was made to issue a request but
@@ -191,7 +186,7 @@ class RequestNotSent(Exception):
     to send a request using a protocol which is no longer connected to a
     server.
     """
-}}} '''
+
 
 
 def _callAppFunction(function):
@@ -777,7 +772,7 @@ class Request:
         _callAppFunction(self.bodyProducer.stopProducing)
 
 
-''' {{{
+
 class LengthEnforcingConsumer:
     """
     An L{IConsumer} proxy which enforces an exact length requirement on the
@@ -1201,7 +1196,7 @@ class TransportProxyProducer:
         """
         if self._producer is not None:
             self._producer.pauseProducing()
-}}} '''
+
 
 
 class HTTP11ClientProtocol(Protocol):
@@ -1527,3 +1522,4 @@ class HTTP11ClientProtocol(Protocol):
         d = Deferred()
         self._abortDeferreds.append(d)
         return d
+}}} '''

scrapy/xlib/tx/client.py

@@ -29,23 +29,18 @@ from twisted.python.failure import Failure
 from twisted.web import http
 from twisted.internet import defer, protocol, task, reactor
 from twisted.internet.interfaces import IProtocol
+from twisted.internet.endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint
 from twisted.python import failure
 from twisted.python.components import proxyForInterface
 from twisted.web import error
-from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer
+from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponse
 from twisted.web.http_headers import Headers
 
 from twisted.web.client import (
-    PartialDownloadError,
+    PartialDownloadError, FileBodyProducer,
+    CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent,
+    Agent, ProxyAgent, HTTPConnectionPool, readBody,
 )
-# newer than 10.0.0
-#from twisted.web.client import (
-#    CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, FileBodyProducer,
-#    HTTPConnectionPool, Agent, ProxyAgent,
-#)
-
-from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint
-from .iweb import IResponse
 
 ''' {{{
 class PartialDownloadError(error.Error):
@@ -54,7 +49,7 @@ class PartialDownloadError(error.Error):
 
     @ivar response: All of the response body which was downloaded.
     """
-}}} '''
+
 
 class _URL(tuple):
     """
@@ -138,22 +133,21 @@ def _makeGetterFactory(url, factoryFactory, contextFactory=None,
     else:
         reactor.connectTCP(host, port, factory)
     return factory
-
+}}} '''
 
 # The code which follows is based on the new HTTP client implementation.  It
 # should be significantly better than anything above, though it is not yet
 # feature equivalent.
 
-from twisted.web.error import SchemeNotSupported
-from ._newclient import Request, Response, HTTP11ClientProtocol
-from twisted.web._newclient import ResponseDone
-from ._newclient import ResponseFailed
-from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed
-from twisted.web._newclient import (
-    PotentialDataLoss, _WrapperException)
-from ._newclient import (
-    ResponseNeverReceived)
+#from twisted.web.error import SchemeNotSupported
+from twisted.web._newclient import Response
+#from twisted.web._newclient import Request, HTTP11ClientProtocol
+from twisted.web._newclient import ResponseDone, ResponseFailed
+#from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed
+#from twisted.web._newclient import (
+#    ResponseNeverReceived, PotentialDataLoss, _WrapperException)
 
+''' {{{
 try:
     from twisted.internet.ssl import ClientContextFactory
 except ImportError:
@@ -1170,7 +1164,7 @@ def readBody(response):
     d = defer.Deferred()
     response.deliverBody(_ReadBodyProtocol(response.code, response.phrase, d))
     return d
-
+}}} '''
 
 
 __all__ = [

scrapy/xlib/tx/endpoints.py

@@ -21,38 +21,33 @@ from zope.interface import implementer, directlyProvides
 import warnings
 
 from twisted.internet import interfaces, defer, error, fdesc
-from twisted.internet.protocol import (
-        ClientFactory, Protocol, Factory)
+#from twisted.internet.protocol import (
+#        ClientFactory, Protocol)
+from twisted.internet.protocol import Factory
 #from twisted.internet import threads, ProcessProtocol
 from twisted.internet.interfaces import IStreamServerEndpointStringParser
-from twisted.internet.interfaces import IStreamClientEndpointStringParser
+#from twisted.internet.interfaces import IStreamClientEndpointStringParser
 from twisted.python.filepath import FilePath
 #from twisted.python.failure import Failure
 #from twisted.python import log
-from twisted.python.components import proxyForInterface
+#from twisted.python.components import proxyForInterface
 
 from twisted.plugin import IPlugin, getPlugins
 #from twisted.internet import stdio
 
-# newer than 10.0.0
-#from twisted.internet.endpoints import (
-#    TCP4ServerEndpoint, TCP6ServerEndpoint, TCP4ClientEndpoint, SSL4ServerEndpoint, SSL4ClientEndpoint,
-#    UNIXServerEndpoint, UNIXClientEndpoint, AdoptedStreamServerEndpoint, connectProtocol,
-#    quoteStringArgument,
-#    serverFromString, #> using newer _parseSSL, _tokenize in _serverParsers
-#    clientFromString, #> using newer _clientParsers
-#    _WrappingProtocol, _WrappingFactory, _TCPServerEndpoint,
-#    _parseTCP, _parseUNIX, _loadCAsFromDir,
-#    _parseSSL, _tokenize,
-#    _parseClientTCP, _parseClientSSL, _parseClientUNIX,
-#)
-
-from .interfaces import IFileDescriptorReceiver
+from twisted.internet.endpoints import (
+    clientFromString, serverFromString, quoteStringArgument,
+    TCP4ServerEndpoint, TCP6ServerEndpoint,
+    TCP4ClientEndpoint, TCP6ClientEndpoint,
+    UNIXServerEndpoint, UNIXClientEndpoint,
+    SSL4ServerEndpoint, SSL4ClientEndpoint,
+    AdoptedStreamServerEndpoint, connectProtocol,
+)
 
 
 __all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"]
 
-
+''' {{{
 class _WrappingProtocol(Protocol):
     """
     Wrap another protocol in order to notify my user when a connection has
@@ -71,7 +66,7 @@ class _WrappingProtocol(Protocol):
         self._wrappedProtocol = wrappedProtocol
 
         for iface in [interfaces.IHalfCloseableProtocol,
-                      IFileDescriptorReceiver]:
+                      interfaces.IFileDescriptorReceiver]:
             if iface.providedBy(self._wrappedProtocol):
                 directlyProvides(self, iface)
 
@@ -609,6 +604,7 @@ class AdoptedStreamServerEndpoint(object):
 
 
 
+
 def _parseTCP(factory, port, interface="", backlog=50):
     """
     Internal parser function for L{_parseServer} to convert the string
@@ -1280,4 +1276,4 @@ def connectProtocol(endpoint, protocol):
         def buildProtocol(self, addr):
             return protocol
     return endpoint.connect(OneShotFactory())
-
+}}} '''

scrapy/xlib/tx/interfaces.py

@@ -13,24 +13,21 @@ from zope.interface import Interface, Attribute
 
 from twisted.internet.interfaces import (
     IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL,
-    IReactorUDP, IReactorMulticast, IReactorProcess,
+    IReactorWin32Events, IReactorUDP, IReactorMulticast, IReactorProcess,
     IReactorTime, IDelayedCall, IReactorThreads, IReactorCore,
-    IReactorPluggableResolver, IReactorFDSet,
+    IReactorPluggableResolver, IReactorDaemonize, IReactorFDSet,
     IListeningPort, ILoggingContext, IFileDescriptor, IReadDescriptor,
     IWriteDescriptor, IReadWriteDescriptor, IHalfCloseableDescriptor,
     ISystemHandle, IConsumer, IProducer, IPushProducer, IPullProducer,
     IProtocol, IProcessProtocol, IHalfCloseableProtocol,
-    IProtocolFactory, ITransport, IProcessTransport, IServiceCollection,
+    IFileDescriptorReceiver, IProtocolFactory, ITransport, ITCPTransport,
+    IUNIXTransport,
+    ITLSTransport, ISSLTransport, IProcessTransport, IServiceCollection,
     IUDPTransport, IUNIXDatagramTransport, IUNIXDatagramConnectedTransport,
-    IMulticastTransport,
+    IMulticastTransport, IStreamClientEndpoint, IStreamServerEndpoint,
+    IStreamServerEndpointStringParser, IStreamClientEndpointStringParser,
+    IReactorUNIX, IReactorUNIXDatagram, IReactorSocket, IResolver
 )
-# newer than 10.0.0
-#from twisted.internet.interfaces import (
-#    IResolver, IReactorUNIX, IReactorUNIXDatagram, IReactorWin32Events, IReactorSocket,
-#    IReactorDaemonize, IFileDescriptorReceiver, ITCPTransport, IUNIXTransport,
-#    ITLSTransport, ISSLTransport, IStreamClientEndpoint, IStreamServerEndpoint,
-#    IStreamServerEndpointStringParser, IStreamClientEndpointStringParser,
-#)
 
 ''' {{{
 class IAddress(Interface):
@@ -95,7 +92,7 @@ class IResolverSimple(Interface):
         @raise twisted.internet.defer.TimeoutError: Raised (asynchronously)
         if the name cannot be resolved within the specified timeout period.
         """
-}}} '''
+
 
 
 class IResolver(IResolverSimple):
@@ -635,7 +632,7 @@ class IResolver(IResolverSimple):
         """
 
 
-''' {{{
+
 class IReactorTCP(Interface):
 
     def listenTCP(port, factory, backlog=50, interface=''):
@@ -722,7 +719,7 @@ class IReactorSSL(Interface):
 
         @param interface: the hostname to bind to, defaults to '' (all)
         """
-}}} '''
+
 
 
 class IReactorUNIX(Interface):
@@ -850,7 +847,7 @@ class IReactorWin32Events(Interface):
         """
 
 
-''' {{{
+
 class IReactorUDP(Interface):
     """
     UDP socket methods.
@@ -889,7 +886,7 @@ class IReactorMulticast(Interface):
         @see: L{twisted.internet.interfaces.IMulticastTransport}
         @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html}
         """
-}}} '''
+
 
 
 class IReactorSocket(Interface):
@@ -991,7 +988,7 @@ class IReactorSocket(Interface):
         """
 
 
-''' {{{
+
 class IReactorProcess(Interface):
 
     def spawnProcess(processProtocol, executable, args=(), env={}, path=None,
@@ -1368,7 +1365,7 @@ class IReactorPluggableResolver(Interface):
 
         @return: The previously installed resolver.
         """
-}}} '''
+
 
 class IReactorDaemonize(Interface):
     """
@@ -1400,7 +1397,7 @@ class IReactorDaemonize(Interface):
         """
 
 
-''' {{{
+
 class IReactorFDSet(Interface):
     """
     Implement me to be able to use L{IFileDescriptor} type resources.
@@ -1884,7 +1881,7 @@ class IHalfCloseableProtocol(Interface):
         This will never be called for TCP connections as TCP does not
         support notification of this type of half-close.
         """
-}}} '''
+
 
 
 class IFileDescriptorReceiver(Interface):
@@ -1905,7 +1902,7 @@ class IFileDescriptorReceiver(Interface):
         """
 
 
-''' {{{
+
 class IProtocolFactory(Interface):
     """
     Interface for protocol factories.
@@ -1995,7 +1992,7 @@ class ITransport(Interface):
 
         @return: An L{IAddress} provider.
         """
-}}} '''
+
 
 class ITCPTransport(ITransport):
     """
@@ -2116,7 +2113,7 @@ class ISSLTransport(ITCPTransport):
         Return an object with the peer's certificate info.
         """
 
-''' {{{
+
 class IProcessTransport(ITransport):
     """
     A process transport.
@@ -2345,7 +2342,7 @@ class IMulticastTransport(Interface):
         """
         Leave multicast group, return L{Deferred} of success.
         """
-}}} '''
+
 
 class IStreamClientEndpoint(Interface):
     """
@@ -2461,3 +2458,4 @@ class IStreamClientEndpointStringParser(Interface):
         @return: a client endpoint
         @rtype: L{IStreamClientEndpoint}
         """
+}}} '''

scrapy/xlib/tx/iweb.py

@@ -15,14 +15,11 @@ from zope.interface import Interface, Attribute
 #from twisted.internet.interfaces import IPushProducer
 
 from twisted.web.iweb import (
-    ICredentialFactory, IBodyProducer,
-    UNKNOWN_LENGTH,
+    IRequest, ICredentialFactory, IBodyProducer, IRenderable, ITemplateLoader,
+    IResponse, _IRequestEncoder, _IRequestEncoderFactory, UNKNOWN_LENGTH,
 )
-# newer than 10.0.0
-#from twisted.web.iweb import (
-#    IRequest, IRenderable, ITemplateLoader, IResponse, _IRequestEncoder, _IRequestEncoderFactory,
-#)
 
+''' {{{
 class IRequest(Interface):
     """
     An HTTP request.
@@ -328,7 +325,7 @@ class IRequest(Interface):
         """
 
 
-''' {{{
+
 class ICredentialFactory(Interface):
     """
     A credential factory defines a way to generate a particular kind of
@@ -432,7 +429,7 @@ class IBodyProducer(IPushProducer):
         L{Deferred<twisted.internet.defer.Deferred>} returned by
         C{startProducing} is never fired.
         """
-}}} '''
+
 
 
 class IRenderable(Interface):
@@ -584,7 +581,7 @@ class _IRequestEncoderFactory(Interface):
         """
 
 
-''' {{{
+
 UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH"
 }}} '''
 __all__ = [

setup.py

@@ -41,7 +41,7 @@ setup(
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
     install_requires=[
-        'Twisted>=10.0.0',
+        'Twisted>=13.1.0',
         'w3lib>=1.15.0',
         'queuelib',
         'lxml',