RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Strip xlib.tx code of Twisted 10

This commit is contained in:
nyov 2016-12-01 22:02:10 +00:00
parent a07400ce0a
commit e1ea0c433a
6 changed files with 115 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scrapy/xlib/tx/__init__.py
View File

@ -15,9 +15,9 @@ else:
client = endpoints = _Mock()
Agent = client.Agent
ProxyAgent = client.ProxyAgent
ResponseDone = client.ResponseDone
ResponseFailed = client.ResponseFailed
HTTPConnectionPool = client.HTTPConnectionPool
TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint
Agent = client.Agent # since < 11.1
ProxyAgent = client.ProxyAgent # since 11.1
ResponseDone = client.ResponseDone # since 11.1
ResponseFailed = client.ResponseFailed # since 11.1
HTTPConnectionPool = client.HTTPConnectionPool # since 12.1
TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint # since 10.1

31
scrapy/xlib/tx/_newclient.py
View File

@ -39,12 +39,25 @@ from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred
from twisted.internet.defer import CancelledError
from twisted.internet.protocol import Protocol
from twisted.protocols.basic import LineReceiver
from twisted.web.iweb import UNKNOWN_LENGTH
from twisted.web.http_headers import Headers
from twisted.web.http import NO_CONTENT, NOT_MODIFIED
from twisted.web.http import _DataLoss, PotentialDataLoss
from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder
from .iweb import IResponse, UNKNOWN_LENGTH
from twisted.web._newclient import (
BadHeaders, ExcessWrite, ParseError, BadResponseVersion, _WrapperException,
RequestGenerationFailed, RequestTransmissionFailed,
WrongBodyLength, ResponseDone, RequestNotSent,
LengthEnforcingConsumer, makeStatefulDispatcher, ChunkedEncoder,
TransportProxyProducer,
)
# newer than 10.0.0
#from twisted.web._newclient import (
# ConnectionAborted, ResponseFailed, ResponseNeverReceived, HTTPParser,
# HTTPClientParser, Request, Response, HTTP11ClientProtocol,
#)
from .iweb import IResponse
# States HTTPParser can be in
STATUS = 'STATUS'
@ -52,7 +65,7 @@ HEADER = 'HEADER'
BODY = 'BODY'
DONE = 'DONE'
''' {{{
class BadHeaders(Exception):
"""
Headers passed to L{Request} were in some way invalid.
@ -117,7 +130,7 @@ class RequestTransmissionFailed(_WrapperException):
@ivar reasons: A C{list} of one or more L{Failure} instances giving the
reasons the request transmission was considered to have failed.
"""
}}} '''
class ConnectionAborted(Exception):
@ -126,7 +139,7 @@ class ConnectionAborted(Exception):
"""
''' {{{
class WrongBodyLength(Exception):
"""
An L{IBodyProducer} declared the number of bytes it was going to
@ -142,7 +155,7 @@ class ResponseDone(Exception):
protocol passed to L{Response.deliverBody} and indicates that the entire
response has been delivered.
"""
}}} '''
class ResponseFailed(_WrapperException):
@ -169,7 +182,7 @@ class ResponseNeverReceived(ResponseFailed):
"""
''' {{{
class RequestNotSent(Exception):
"""
L{RequestNotSent} indicates that an attempt was made to issue a request but
@ -178,7 +191,7 @@ class RequestNotSent(Exception):
to send a request using a protocol which is no longer connected to a
server.
"""
}}} '''
def _callAppFunction(function):
@ -764,7 +777,7 @@ class Request:
_callAppFunction(self.bodyProducer.stopProducing)
''' {{{
class LengthEnforcingConsumer:
"""
An L{IConsumer} proxy which enforces an exact length requirement on the
@ -1188,7 +1201,7 @@ class TransportProxyProducer:
"""
if self._producer is not None:
self._producer.pauseProducing()
}}} '''
class HTTP11ClientProtocol(Protocol):

28
scrapy/xlib/tx/client.py
View File

@ -32,19 +32,29 @@ from twisted.internet.interfaces import IProtocol
from twisted.python import failure
from twisted.python.components import proxyForInterface
from twisted.web import error
from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer
from twisted.web.http_headers import Headers
from twisted.web.client import (
PartialDownloadError,
)
# newer than 10.0.0
#from twisted.web.client import (
# CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, FileBodyProducer,
# HTTPConnectionPool, Agent, ProxyAgent,
#)
from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint
from .iweb import IResponse, UNKNOWN_LENGTH, IBodyProducer
from .iweb import IResponse
''' {{{
class PartialDownloadError(error.Error):
"""
Page was only partially downloaded, we got disconnected in middle.
@ivar response: All of the response body which was downloaded.
"""
}}} '''
class _URL(tuple):
"""
@ -136,10 +146,13 @@ def _makeGetterFactory(url, factoryFactory, contextFactory=None,
from twisted.web.error import SchemeNotSupported
from ._newclient import Request, Response, HTTP11ClientProtocol
from ._newclient import ResponseDone, ResponseFailed
from ._newclient import RequestNotSent, RequestTransmissionFailed
from twisted.web._newclient import ResponseDone
from ._newclient import ResponseFailed
from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed
from twisted.web._newclient import (
PotentialDataLoss, _WrapperException)
from ._newclient import (
ResponseNeverReceived, PotentialDataLoss, _WrapperException)
ResponseNeverReceived)
try:
from twisted.internet.ssl import ClientContextFactory
@ -1161,8 +1174,7 @@ def readBody(response):
__all__ = [
'PartialDownloadError', 'HTTPPageGetter', 'HTTPPageDownloader',
'HTTPClientFactory', 'HTTPDownloader', 'getPage', 'downloadPage',
'PartialDownloadError',
'ResponseDone', 'Response', 'ResponseFailed', 'Agent', 'CookieAgent',
'ProxyAgent', 'ContentDecoderAgent', 'GzipDecoder', 'RedirectAgent',
'HTTPConnectionPool', 'readBody']

26
scrapy/xlib/tx/endpoints.py
View File

@ -15,23 +15,37 @@ parsed by the L{clientFromString} and L{serverFromString} functions.
from __future__ import division, absolute_import
import os
import socket
#import socket
from zope.interface import implementer, directlyProvides
import warnings
from twisted.internet import interfaces, defer, error, fdesc, threads
from twisted.internet import interfaces, defer, error, fdesc
from twisted.internet.protocol import (
ClientFactory, Protocol, ProcessProtocol, Factory)
ClientFactory, Protocol, Factory)
#from twisted.internet import threads, ProcessProtocol
from twisted.internet.interfaces import IStreamServerEndpointStringParser
from twisted.internet.interfaces import IStreamClientEndpointStringParser
from twisted.python.filepath import FilePath
from twisted.python.failure import Failure
from twisted.python import log
#from twisted.python.failure import Failure
#from twisted.python import log
from twisted.python.components import proxyForInterface
from twisted.plugin import IPlugin, getPlugins
from twisted.internet import stdio
#from twisted.internet import stdio
# newer than 10.0.0
#from twisted.internet.endpoints import (
# TCP4ServerEndpoint, TCP6ServerEndpoint, TCP4ClientEndpoint, SSL4ServerEndpoint, SSL4ClientEndpoint,
# UNIXServerEndpoint, UNIXClientEndpoint, AdoptedStreamServerEndpoint, connectProtocol,
# quoteStringArgument,
# serverFromString, #> using newer _parseSSL, _tokenize in _serverParsers
# clientFromString, #> using newer _clientParsers
# _WrappingProtocol, _WrappingFactory, _TCPServerEndpoint,
# _parseTCP, _parseUNIX, _loadCAsFromDir,
# _parseSSL, _tokenize,
# _parseClientTCP, _parseClientSSL, _parseClientUNIX,
#)
from .interfaces import IFileDescriptorReceiver

47
scrapy/xlib/tx/interfaces.py
View File

@ -11,7 +11,28 @@ from __future__ import division, absolute_import
from zope.interface import Interface, Attribute
from twisted.internet.interfaces import (
IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL,
IReactorUDP, IReactorMulticast, IReactorProcess,
IReactorTime, IDelayedCall, IReactorThreads, IReactorCore,
IReactorPluggableResolver, IReactorFDSet,
IListeningPort, ILoggingContext, IFileDescriptor, IReadDescriptor,
IWriteDescriptor, IReadWriteDescriptor, IHalfCloseableDescriptor,
ISystemHandle, IConsumer, IProducer, IPushProducer, IPullProducer,
IProtocol, IProcessProtocol, IHalfCloseableProtocol,
IProtocolFactory, ITransport, IProcessTransport, IServiceCollection,
IUDPTransport, IUNIXDatagramTransport, IUNIXDatagramConnectedTransport,
IMulticastTransport,
)
# newer than 10.0.0
#from twisted.internet.interfaces import (
# IResolver, IReactorUNIX, IReactorUNIXDatagram, IReactorWin32Events, IReactorSocket,
# IReactorDaemonize, IFileDescriptorReceiver, ITCPTransport, IUNIXTransport,
# ITLSTransport, ISSLTransport, IStreamClientEndpoint, IStreamServerEndpoint,
# IStreamServerEndpointStringParser, IStreamClientEndpointStringParser,
#)
''' {{{
class IAddress(Interface):
"""
An address, e.g. a TCP C{(host, port)}.
@ -74,7 +95,7 @@ class IResolverSimple(Interface):
@raise twisted.internet.defer.TimeoutError: Raised (asynchronously)
if the name cannot be resolved within the specified timeout period.
"""
}}} '''
class IResolver(IResolverSimple):
@ -614,7 +635,7 @@ class IResolver(IResolverSimple):
"""
''' {{{
class IReactorTCP(Interface):
def listenTCP(port, factory, backlog=50, interface=''):
@ -701,7 +722,7 @@ class IReactorSSL(Interface):
@param interface: the hostname to bind to, defaults to '' (all)
"""
}}} '''
class IReactorUNIX(Interface):
@ -829,7 +850,7 @@ class IReactorWin32Events(Interface):
"""
''' {{{
class IReactorUDP(Interface):
"""
UDP socket methods.
@ -868,7 +889,7 @@ class IReactorMulticast(Interface):
@see: L{twisted.internet.interfaces.IMulticastTransport}
@see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html}
"""
}}} '''
class IReactorSocket(Interface):
@ -970,7 +991,7 @@ class IReactorSocket(Interface):
"""
''' {{{
class IReactorProcess(Interface):
def spawnProcess(processProtocol, executable, args=(), env={}, path=None,
@ -1347,7 +1368,7 @@ class IReactorPluggableResolver(Interface):
@return: The previously installed resolver.
"""
}}} '''
class IReactorDaemonize(Interface):
"""
@ -1379,7 +1400,7 @@ class IReactorDaemonize(Interface):
"""
''' {{{
class IReactorFDSet(Interface):
"""
Implement me to be able to use L{IFileDescriptor} type resources.
@ -1863,7 +1884,7 @@ class IHalfCloseableProtocol(Interface):
This will never be called for TCP connections as TCP does not
support notification of this type of half-close.
"""
}}} '''
class IFileDescriptorReceiver(Interface):
@ -1884,7 +1905,7 @@ class IFileDescriptorReceiver(Interface):
"""
''' {{{
class IProtocolFactory(Interface):
"""
Interface for protocol factories.
@ -1974,7 +1995,7 @@ class ITransport(Interface):
@return: An L{IAddress} provider.
"""
}}} '''
class ITCPTransport(ITransport):
"""
@ -2095,7 +2116,7 @@ class ISSLTransport(ITCPTransport):
Return an object with the peer's certificate info.
"""
''' {{{
class IProcessTransport(ITransport):
"""
A process transport.
@ -2324,7 +2345,7 @@ class IMulticastTransport(Interface):
"""
Leave multicast group, return L{Deferred} of success.
"""
}}} '''
class IStreamClientEndpoint(Interface):
"""

18
scrapy/xlib/tx/iweb.py
View File

@ -12,8 +12,16 @@ Interface definitions for L{twisted.web}.
from zope.interface import Interface, Attribute
from twisted.internet.interfaces import IPushProducer
#from twisted.internet.interfaces import IPushProducer
from twisted.web.iweb import (
ICredentialFactory, IBodyProducer,
UNKNOWN_LENGTH,
)
# newer than 10.0.0
#from twisted.web.iweb import (
# IRequest, IRenderable, ITemplateLoader, IResponse, _IRequestEncoder, _IRequestEncoderFactory,
#)
class IRequest(Interface):
"""
@ -320,7 +328,7 @@ class IRequest(Interface):
"""
''' {{{
class ICredentialFactory(Interface):
"""
A credential factory defines a way to generate a particular kind of
@ -424,7 +432,7 @@ class IBodyProducer(IPushProducer):
L{Deferred<twisted.internet.defer.Deferred>} returned by
C{startProducing} is never fired.
"""
}}} '''
class IRenderable(Interface):
@ -576,9 +584,9 @@ class _IRequestEncoderFactory(Interface):
"""
''' {{{
UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH"
}}} '''
__all__ = [
"ICredentialFactory", "IRequest",
"IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder",