Merge pull request #318 from scrapy/http11

Add HTTP 1.1 download handler

scrapy/__init__.py

@@ -43,3 +43,8 @@ except ImportError:
     pass
 else:
     optional_features.add('django')
+
+from twisted import version as _txv
+twisted_version = (_txv.major, _txv.minor, _txv.micro)
+if twisted_version >= (11, 1, 0):
+    optional_features.add('http11')

scrapy/core/downloader/__init__.py

@@ -70,7 +70,7 @@ class Downloader(object):
         self.signals = crawler.signals
         self.slots = {}
         self.active = set()
-        self.handlers = DownloadHandlers(crawler.settings)
+        self.handlers = DownloadHandlers(crawler)
         self.total_concurrency = self.settings.getint('CONCURRENT_REQUESTS')
         self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
         self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP')

scrapy/core/downloader/contextfactory.py

@@ -0,0 +1,20 @@
+from OpenSSL import SSL
+from twisted.internet.ssl import ClientContextFactory
+
+
+class ScrapyClientContextFactory(ClientContextFactory):
+    "A SSL context factory which is more permissive against SSL bugs."
+    # see https://github.com/scrapy/scrapy/issues/82
+    # and https://github.com/scrapy/scrapy/issues/26
+
+    def __init__(self):
+        # see this issue on why we use TLSv1_METHOD by default
+        # https://github.com/scrapy/scrapy/issues/194
+        self.method = SSL.TLSv1_METHOD
+
+    def getContext(self, hostname=None, port=None):
+        ctx = ClientContextFactory.getContext(self)
+        # Enable all workarounds to SSL bugs as documented by
+        # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html
+        ctx.set_options(SSL.OP_ALL)
+        return ctx

scrapy/core/downloader/handlers/__init__.py

@@ -1,32 +1,42 @@
 """Download handlers for different schemes"""
 
+from twisted.internet import defer
 from scrapy.exceptions import NotSupported, NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
+from scrapy import signals
 
 
 class DownloadHandlers(object):
 
-    def __init__(self, settings):
+    def __init__(self, crawler):
         self._handlers = {}
         self._notconfigured = {}
-        handlers = settings.get('DOWNLOAD_HANDLERS_BASE')
-        handlers.update(settings.get('DOWNLOAD_HANDLERS', {}))
+        handlers = crawler.settings.get('DOWNLOAD_HANDLERS_BASE')
+        handlers.update(crawler.settings.get('DOWNLOAD_HANDLERS', {}))
         for scheme, clspath in handlers.iteritems():
             cls = load_object(clspath)
             try:
-                dh = cls(settings)
+                dh = cls(crawler.settings)
             except NotConfigured, ex:
                 self._notconfigured[scheme] = str(ex)
             else:
-                self._handlers[scheme] = dh.download_request
+                self._handlers[scheme] = dh
+
+        crawler.signals.connect(self._close, signals.engine_stopped)
 
     def download_request(self, request, spider):
         scheme = urlparse_cached(request).scheme
         try:
-            handler = self._handlers[scheme]
+            handler = self._handlers[scheme].download_request
         except KeyError:
             msg = self._notconfigured.get(scheme, \
                     'no handler available for that scheme')
             raise NotSupported("Unsupported URL scheme '%s': %s" % (scheme, msg))
         return handler(request, spider)
+
+    @defer.inlineCallbacks
+    def _close(self, *_a, **_kw):
+        for dh in self._handlers.values():
+            if hasattr(dh, 'close'):
+                yield dh.close()

scrapy/core/downloader/handlers/http.py

@@ -1,25 +1,19 @@
-"""Download handlers for http and https schemes"""
+from scrapy import optional_features
+from .http10 import HTTP10DownloadHandler
 
-from twisted.internet import reactor
+if 'http11' in optional_features:
+    from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
+else:
+    HTTPDownloadHandler = HTTP10DownloadHandler
 
-from scrapy.utils.misc import load_object
 
-class HttpDownloadHandler(object):
+# backwards compatibility
+class HttpDownloadHandler(HTTP10DownloadHandler):
 
-    def __init__(self, settings):
-        self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
-        self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
-
-    def download_request(self, request, spider):
-        """Return a deferred for the HTTP download"""
-        factory = self.HTTPClientFactory(request)
-        self._connect(factory)
-        return factory.deferred
-
-    def _connect(self, factory):
-        host, port = factory.host, factory.port
-        if factory.scheme == 'https':
-            return reactor.connectSSL(host, port, factory, \
-                    self.ClientContextFactory())
-        else:
-            return reactor.connectTCP(host, port, factory)
+    def __init__(self, *args, **kwargs):
+        import warnings
+        from scrapy.exceptions import ScrapyDeprecationWarning
+        warnings.warn('HttpDownloadHandler is deprecated, import scrapy.core.downloader'
+                      '.handlers.http10.HTTP10DownloadHandler instead',
+                      category=ScrapyDeprecationWarning, stacklevel=1)
+        super(HttpDownloadHandler, self).__init__(*args, **kwargs)

scrapy/core/downloader/handlers/http10.py

@@ -0,0 +1,25 @@
+"""Download handlers for http and https schemes
+"""
+from twisted.internet import reactor
+from scrapy.utils.misc import load_object
+
+
+class HTTP10DownloadHandler(object):
+
+    def __init__(self, settings):
+        self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
+        self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+
+    def download_request(self, request, spider):
+        """Return a deferred for the HTTP download"""
+        factory = self.HTTPClientFactory(request)
+        self._connect(factory)
+        return factory.deferred
+
+    def _connect(self, factory):
+        host, port = factory.host, factory.port
+        if factory.scheme == 'https':
+            return reactor.connectSSL(host, port, factory,
+                                      self.ClientContextFactory())
+        else:
+            return reactor.connectTCP(host, port, factory)

scrapy/core/downloader/handlers/http11.py

@@ -0,0 +1,136 @@
+"""Download handlers for http and https schemes"""
+
+from time import time
+from cStringIO import StringIO
+from urlparse import urldefrag
+
+from zope.interface import implements
+from twisted.internet import defer, reactor, protocol
+from twisted.web.http_headers import Headers as TxHeaders
+from twisted.web.http import PotentialDataLoss
+from twisted.web.iweb import IBodyProducer
+from twisted.internet.error import TimeoutError
+from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \
+        ResponseFailed, HTTPConnectionPool, TCP4ClientEndpoint
+
+from scrapy.http import Headers
+from scrapy.responsetypes import responsetypes
+from scrapy.core.downloader.webclient import _parse
+from scrapy.utils.misc import load_object
+from scrapy import log
+
+
+
+class HTTP11DownloadHandler(object):
+
+    def __init__(self, settings):
+        self._pool = HTTPConnectionPool(reactor, persistent=True)
+        self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+        self._contextFactory = self._contextFactoryClass()
+
+    def download_request(self, request, spider):
+        """Return a deferred for the HTTP download"""
+        agent = ScrapyAgent(contextFactory=self._contextFactory, pool=self._pool)
+        return agent.download_request(request)
+
+    def close(self):
+        return self._pool.closeCachedConnections()
+
+
+class ScrapyAgent(object):
+
+    _Agent = Agent
+    _ProxyAgent = ProxyAgent
+
+    def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None):
+        self._contextFactory = contextFactory
+        self._connectTimeout = connectTimeout
+        self._bindAddress = bindAddress
+        self._pool = pool
+
+    def download_request(self, request):
+        timeout = request.meta.get('download_timeout') or self._connectTimeout
+        url = urldefrag(request.url)[0]
+        method = request.method
+        headers = TxHeaders(request.headers)
+        bodyproducer = _RequestBodyProducer(request.body) if request.body else None
+        agent = self._get_agent(request, timeout)
+        start_time = time()
+        d = agent.request(method, url, headers, bodyproducer)
+        d.addBoth(self._both_cb, request, start_time, url, timeout)
+        d.addCallback(self._downloaded, request)
+        self._timeout_cl = reactor.callLater(timeout, d.cancel)
+        return d
+
+    def _both_cb(self, result, request, start_time, url, timeout):
+        request.meta['download_latency'] = time() - start_time
+        if self._timeout_cl.active():
+            self._timeout_cl.cancel()
+            return result
+        raise TimeoutError("Getting %s took longer than %s seconds." % (url, timeout))
+
+    def _get_agent(self, request, timeout):
+        bindaddress = request.meta.get('bindaddress') or self._bindAddress
+        proxy = request.meta.get('proxy')
+        if proxy:
+            scheme, _, host, port, _ = _parse(proxy)
+            endpoint = TCP4ClientEndpoint(reactor, host, port, timeout=timeout,
+                bindAddress=bindaddress)
+            return self._ProxyAgent(endpoint)
+
+        return self._Agent(reactor, contextFactory=self._contextFactory,
+            connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)
+
+    def _downloaded(self, txresponse, request):
+        if txresponse.length == 0:
+            return self._build_response(('', None), txresponse, request)
+        finished = defer.Deferred()
+        finished.addCallback(self._build_response, txresponse, request)
+        txresponse.deliverBody(_ResponseReader(finished))
+        return finished
+
+    def _build_response(self, (body, flag), txresponse, request):
+        if flag is not None:
+            request.meta[flag] = True
+        url = urldefrag(request.url)[0]
+        status = int(txresponse.code)
+        headers = Headers(txresponse.headers.getAllRawHeaders())
+        respcls = responsetypes.from_args(headers=headers, url=url)
+        return respcls(url=url, status=status, headers=headers, body=body)
+
+
+class _RequestBodyProducer(object):
+    implements(IBodyProducer)
+
+    def __init__(self, body):
+        self.body = body
+        self.length = len(body)
+
+    def startProducing(self, consumer):
+        consumer.write(self.body)
+        return defer.succeed(None)
+
+    def pauseProducing(self):
+        pass
+
+    def stopProducing(self):
+        pass
+
+
+class _ResponseReader(protocol.Protocol):
+
+    def __init__(self, finished):
+        self._finished = finished
+        self._bodybuf = StringIO()
+
+    def dataReceived(self, bodyBytes):
+        self._bodybuf.write(bodyBytes)
+
+    def connectionLost(self, reason):
+        body = self._bodybuf.getvalue()
+        if reason.check(ResponseDone):
+            self._finished.callback((body, None))
+        elif reason.check(PotentialDataLoss, ResponseFailed):
+            self._finished.callback((body, 'partial_download'))
+        else:
+            self._finished.errback(reason)

scrapy/core/downloader/webclient.py

@@ -79,7 +79,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
 
 class ScrapyHTTPClientFactory(HTTPClientFactory):
     """Scrapy implementation of the HTTPClientFactory overwriting the
-    serUrl method to make use of our Url object that cache the parse 
+    serUrl method to make use of our Url object that cache the parse
     result.
     """
 
@@ -137,21 +137,3 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
         self.headers_time = time()
         self.response_headers = headers
 
-
-
-class ScrapyClientContextFactory(ClientContextFactory):
-    "A SSL context factory which is more permissive against SSL bugs."
-    # see https://github.com/scrapy/scrapy/issues/82
-    # and https://github.com/scrapy/scrapy/issues/26
-
-    def __init__(self):
-        # see this issue on why we use TLSv1_METHOD by default
-        # https://github.com/scrapy/scrapy/issues/194
-        self.method = SSL.TLSv1_METHOD
-
-    def getContext(self):
-        ctx = ClientContextFactory.getContext(self)
-        # Enable all workarounds to SSL bugs as documented by
-        # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html
-        ctx.set_options(SSL.OP_ALL)
-        return ctx

scrapy/settings/default_settings.py

@@ -53,8 +53,8 @@ DOWNLOAD_DELAY = 0
 DOWNLOAD_HANDLERS = {}
 DOWNLOAD_HANDLERS_BASE = {
     'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
-    'http': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler',
-    'https': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler',
+    'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
+    'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
     's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler',
 }
 
@@ -63,7 +63,7 @@ DOWNLOAD_TIMEOUT = 180      # 3mins
 DOWNLOADER_DEBUG = False
 
 DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
-DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.webclient.ScrapyClientContextFactory'
+DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
 
 DOWNLOADER_MIDDLEWARES = {}
 

scrapy/tests/test_downloader_handlers.py

@@ -1,9 +1,10 @@
 import os
+import twisted
 
 from twisted.trial import unittest
 from twisted.protocols.policies import WrappingFactory
 from twisted.python.filepath import FilePath
-from twisted.internet import reactor, defer
+from twisted.internet import reactor, defer, error
 from twisted.web import server, static, util, resource
 from twisted.web.test.test_webclient import ForeverTakingResource, \
         NoLengthResource, HostHeaderResource, \
@@ -11,7 +12,9 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \
 from w3lib.url import path_to_file_uri
 
 from scrapy.core.downloader.handlers.file import FileDownloadHandler
-from scrapy.core.downloader.handlers.http import HttpDownloadHandler
+from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler
+from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
+from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 from scrapy.spider import BaseSpider
 from scrapy.http import Request
@@ -46,6 +49,8 @@ class FileTestCase(unittest.TestCase):
 
 class HttpTestCase(unittest.TestCase):
 
+    download_handler_cls = HTTPDownloadHandler
+
     def setUp(self):
         name = self.mktemp()
         os.mkdir(name)
@@ -61,10 +66,14 @@ class HttpTestCase(unittest.TestCase):
         self.wrapper = WrappingFactory(self.site)
         self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1')
         self.portno = self.port.getHost().port
-        self.download_request = HttpDownloadHandler(Settings()).download_request
+        self.download_handler = self.download_handler_cls(Settings())
+        self.download_request = self.download_handler.download_request
 
+    @defer.inlineCallbacks
     def tearDown(self):
-        return self.port.stopListening()
+        yield self.port.stopListening()
+        if hasattr(self.download_handler, 'close'):
+            yield self.download_handler.close()
 
     def getURL(self, path):
         return "http://127.0.0.1:%d/%s" % (self.portno, path)
@@ -98,9 +107,9 @@ class HttpTestCase(unittest.TestCase):
         return d
 
     def test_timeout_download_from_spider(self):
-        request = Request(self.getURL('wait'), meta=dict(download_timeout=0.000001))
+        request = Request(self.getURL('wait'), meta=dict(download_timeout=0.1))
         d = self.download_request(request, BaseSpider('foo'))
-        return self.assertFailure(d, defer.TimeoutError)
+        return self.assertFailure(d, defer.TimeoutError, error.TimeoutError)
 
     def test_host_header_not_in_request_headers(self):
         def _test(response):
@@ -132,6 +141,23 @@ class HttpTestCase(unittest.TestCase):
         return d
 
 
+class DeprecatedHttpTestCase(HttpTestCase):
+    """HTTP 1.0 test case"""
+    download_handler_cls = HttpDownloadHandler
+
+
+class Http10TestCase(HttpTestCase):
+    """HTTP 1.0 test case"""
+    download_handler_cls = HTTP10DownloadHandler
+
+
+class Http11TestCase(HttpTestCase):
+    """HTTP 1.1 test case"""
+    download_handler_cls = HTTP11DownloadHandler
+    if 'http11' not in optional_features:
+        skip = 'HTTP1.1 not supported in twisted < 11.1.0'
+
+
 class UriResource(resource.Resource):
     """Return the full uri that was requested"""
 
@@ -143,16 +169,21 @@ class UriResource(resource.Resource):
 
 
 class HttpProxyTestCase(unittest.TestCase):
+    download_handler_cls = HTTPDownloadHandler
 
     def setUp(self):
         site = server.Site(UriResource(), timeout=None)
         wrapper = WrappingFactory(site)
         self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1')
         self.portno = self.port.getHost().port
-        self.download_request = HttpDownloadHandler(Settings()).download_request
+        self.download_handler = self.download_handler_cls(Settings())
+        self.download_request = self.download_handler.download_request
 
+    @defer.inlineCallbacks
     def tearDown(self):
-        return self.port.stopListening()
+        yield self.port.stopListening()
+        if hasattr(self.download_handler, 'close'):
+            yield self.download_handler.close()
 
     def getURL(self, path):
         return "http://127.0.0.1:%d/%s" % (self.portno, path)
@@ -177,6 +208,21 @@ class HttpProxyTestCase(unittest.TestCase):
         return self.download_request(request, BaseSpider('foo')).addCallback(_test)
 
 
+class DeprecatedHttpProxyTestCase(unittest.TestCase):
+    """Old deprecated reference to http10 downloader handler"""
+    download_handler_cls = HttpDownloadHandler
+
+
+class Http10ProxyTestCase(HttpProxyTestCase):
+    download_handler_cls = HTTP10DownloadHandler
+
+
+class Http11ProxyTestCase(HttpProxyTestCase):
+    download_handler_cls = HTTP11DownloadHandler
+    if 'http11' not in optional_features:
+        skip = 'HTTP1.1 not supported in twisted < 11.1.0'
+
+
 class HttpDownloadHandlerMock(object):
     def __init__(self, settings):
         pass
@@ -240,7 +286,7 @@ class S3TestCase(unittest.TestCase):
                 'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
 
     def test_request_signing5(self):
-        # deletes an object from the 'johnsmith' bucket using the 
+        # deletes an object from the 'johnsmith' bucket using the
         # path-style and Date alternative.
         req = Request('s3://johnsmith/photos/puppy.jpg', \
                 method='DELETE', headers={

scrapy/xlib/tx/LICENSE

@@ -0,0 +1,57 @@
+Copyright (c) 2001-2013
+Allen Short
+Andy Gayton
+Andrew Bennetts
+Antoine Pitrou
+Apple Computer, Inc.
+Benjamin Bruheim
+Bob Ippolito
+Canonical Limited
+Christopher Armstrong
+David Reid
+Donovan Preston
+Eric Mangold
+Eyal Lotem
+Itamar Turner-Trauring
+James Knight
+Jason A. Mobarak
+Jean-Paul Calderone
+Jessica McKellar
+Jonathan Jacobs
+Jonathan Lange
+Jonathan D. Simms
+Jürgen Hermann
+Kevin Horn
+Kevin Turner
+Mary Gardiner
+Matthew Lefkowitz
+Massachusetts Institute of Technology
+Moshe Zadka
+Paul Swartz
+Pavel Pergamenshchik
+Ralph Meijer
+Sean Riley
+Software Freedom Conservancy
+Travis B. Hartwell
+Thijs Triemstra
+Thomas Herve
+Timothy Allen
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

scrapy/xlib/tx/README

@@ -0,0 +1,2 @@
+This source files are adapted copies from Twisted trunk to support HTTP1.1
+handler under Twisted >= 11.1 and Twisted <= 13.0.0

scrapy/xlib/tx/__init__.py

@@ -0,0 +1,23 @@
+from scrapy import twisted_version
+if twisted_version > (13, 0, 0):
+    from twisted.web import client
+    from twisted.internet import endpoints
+if twisted_version >= (11, 1, 0):
+    from . import client, endpoints
+else:
+    from scrapy.exceptions import NotSupported
+    class _Mocked(object):
+        def __init__(self, *args, **kw):
+            raise NotSupported('HTTP1.1 not supported')
+    class _Mock(object):
+        def __getattr__(self, name):
+            return _Mocked
+    client = endpoints = _Mock()
+
+
+Agent = client.Agent
+ProxyAgent = client.ProxyAgent
+ResponseDone = client.ResponseDone
+ResponseFailed = client.ResponseFailed
+HTTPConnectionPool = client.HTTPConnectionPool
+TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint

scrapy/xlib/tx/iweb.py

@@ -0,0 +1,587 @@
+# -*- test-case-name: twisted.web.test -*-
+# Copyright (c) Twisted Matrix Laboratories.
+# See LICENSE for details.
+
+"""
+Interface definitions for L{twisted.web}.
+
+@var UNKNOWN_LENGTH: An opaque object which may be used as the value of
+    L{IBodyProducer.length} to indicate that the length of the entity
+    body is not known in advance.
+"""
+
+from zope.interface import Interface, Attribute
+
+from twisted.internet.interfaces import IPushProducer
+
+
+class IRequest(Interface):
+    """
+    An HTTP request.
+
+    @since: 9.0
+    """
+
+    method = Attribute("A C{str} giving the HTTP method that was used.")
+    uri = Attribute(
+        "A C{str} giving the full encoded URI which was requested (including "
+        "query arguments).")
+    path = Attribute(
+        "A C{str} giving the encoded query path of the request URI.")
+    args = Attribute(
+        "A mapping of decoded query argument names as C{str} to "
+        "corresponding query argument values as C{list}s of C{str}.  "
+        "For example, for a URI with C{'foo=bar&foo=baz&quux=spam'} "
+        "for its query part, C{args} will be C{{'foo': ['bar', 'baz'], "
+        "'quux': ['spam']}}.")
+
+    received_headers = Attribute(
+        "Backwards-compatibility access to C{requestHeaders}.  Use "
+        "C{requestHeaders} instead.  C{received_headers} behaves mostly "
+        "like a C{dict} and does not provide access to all header values.")
+
+    requestHeaders = Attribute(
+        "A L{http_headers.Headers} instance giving all received HTTP request "
+        "headers.")
+
+    content = Attribute(
+        "A file-like object giving the request body.  This may be a file on "
+        "disk, a C{StringIO}, or some other type.  The implementation is free "
+        "to decide on a per-request basis.")
+
+    headers = Attribute(
+        "Backwards-compatibility access to C{responseHeaders}.  Use"
+        "C{responseHeaders} instead.  C{headers} behaves mostly like a "
+        "C{dict} and does not provide access to all header values nor "
+        "does it allow multiple values for one header to be set.")
+
+    responseHeaders = Attribute(
+        "A L{http_headers.Headers} instance holding all HTTP response "
+        "headers to be sent.")
+
+    def getHeader(key):
+        """
+        Get an HTTP request header.
+
+        @type key: C{str}
+        @param key: The name of the header to get the value of.
+
+        @rtype: C{str} or C{NoneType}
+        @return: The value of the specified header, or C{None} if that header
+            was not present in the request.
+        """
+
+
+    def getCookie(key):
+        """
+        Get a cookie that was sent from the network.
+        """
+
+
+    def getAllHeaders():
+        """
+        Return dictionary mapping the names of all received headers to the last
+        value received for each.
+
+        Since this method does not return all header information,
+        C{requestHeaders.getAllRawHeaders()} may be preferred.
+        """
+
+
+    def getRequestHostname():
+        """
+        Get the hostname that the user passed in to the request.
+
+        This will either use the Host: header (if it is available) or the
+        host we are listening on if the header is unavailable.
+
+        @returns: the requested hostname
+        @rtype: C{str}
+        """
+
+
+    def getHost():
+        """
+        Get my originally requesting transport's host.
+
+        @return: An L{IAddress<twisted.internet.interfaces.IAddress>}.
+        """
+
+
+    def getClientIP():
+        """
+        Return the IP address of the client who submitted this request.
+
+        @returns: the client IP address or C{None} if the request was submitted
+            over a transport where IP addresses do not make sense.
+        @rtype: L{str} or C{NoneType}
+        """
+
+
+    def getClient():
+        """
+        Return the hostname of the IP address of the client who submitted this
+        request, if possible.
+
+        This method is B{deprecated}.  See L{getClientIP} instead.
+
+        @rtype: C{NoneType} or L{str}
+        @return: The canonical hostname of the client, as determined by
+            performing a name lookup on the IP address of the client.
+        """
+
+
+    def getUser():
+        """
+        Return the HTTP user sent with this request, if any.
+
+        If no user was supplied, return the empty string.
+
+        @returns: the HTTP user, if any
+        @rtype: C{str}
+        """
+
+
+    def getPassword():
+        """
+        Return the HTTP password sent with this request, if any.
+
+        If no password was supplied, return the empty string.
+
+        @returns: the HTTP password, if any
+        @rtype: C{str}
+        """
+
+
+    def isSecure():
+        """
+        Return True if this request is using a secure transport.
+
+        Normally this method returns True if this request's HTTPChannel
+        instance is using a transport that implements ISSLTransport.
+
+        This will also return True if setHost() has been called
+        with ssl=True.
+
+        @returns: True if this request is secure
+        @rtype: C{bool}
+        """
+
+
+    def getSession(sessionInterface=None):
+        """
+        Look up the session associated with this request or create a new one if
+        there is not one.
+
+        @return: The L{Session} instance identified by the session cookie in
+            the request, or the C{sessionInterface} component of that session
+            if C{sessionInterface} is specified.
+        """
+
+
+    def URLPath():
+        """
+        @return: A L{URLPath} instance which identifies the URL for which this
+            request is.
+        """
+
+
+    def prePathURL():
+        """
+        @return: At any time during resource traversal, a L{str} giving an
+            absolute URL to the most nested resource which has yet been
+            reached.
+        """
+
+
+    def rememberRootURL():
+        """
+        Remember the currently-processed part of the URL for later
+        recalling.
+        """
+
+
+    def getRootURL():
+        """
+        Get a previously-remembered URL.
+        """
+
+
+    # Methods for outgoing response
+    def finish():
+        """
+        Indicate that the response to this request is complete.
+        """
+
+
+    def write(data):
+        """
+        Write some data to the body of the response to this request.  Response
+        headers are written the first time this method is called, after which
+        new response headers may not be added.
+        """
+
+
+    def addCookie(k, v, expires=None, domain=None, path=None, max_age=None, comment=None, secure=None):
+        """
+        Set an outgoing HTTP cookie.
+
+        In general, you should consider using sessions instead of cookies, see
+        L{twisted.web.server.Request.getSession} and the
+        L{twisted.web.server.Session} class for details.
+        """
+
+
+    def setResponseCode(code, message=None):
+        """
+        Set the HTTP response code.
+        """
+
+
+    def setHeader(k, v):
+        """
+        Set an HTTP response header.  Overrides any previously set values for
+        this header.
+
+        @type name: C{str}
+        @param name: The name of the header for which to set the value.
+
+        @type value: C{str}
+        @param value: The value to set for the named header.
+        """
+
+
+    def redirect(url):
+        """
+        Utility function that does a redirect.
+
+        The request should have finish() called after this.
+        """
+
+
+    def setLastModified(when):
+        """
+        Set the C{Last-Modified} time for the response to this request.
+
+        If I am called more than once, I ignore attempts to set Last-Modified
+        earlier, only replacing the Last-Modified time if it is to a later
+        value.
+
+        If I am a conditional request, I may modify my response code to
+        L{NOT_MODIFIED<http.NOT_MODIFIED>} if appropriate for the time given.
+
+        @param when: The last time the resource being returned was modified, in
+            seconds since the epoch.
+        @type when: L{int}, L{long} or L{float}
+
+        @return: If I am a C{If-Modified-Since} conditional request and the time
+            given is not newer than the condition, I return
+            L{CACHED<http.CACHED>} to indicate that you should write no body.
+            Otherwise, I return a false value.
+        """
+
+
+    def setETag(etag):
+        """
+        Set an C{entity tag} for the outgoing response.
+
+        That's "entity tag" as in the HTTP/1.1 I{ETag} header, "used for
+        comparing two or more entities from the same requested resource."
+
+        If I am a conditional request, I may modify my response code to
+        L{NOT_MODIFIED<http.NOT_MODIFIED>} or
+        L{PRECONDITION_FAILED<http.PRECONDITION_FAILED>}, if appropriate for the
+        tag given.
+
+        @param etag: The entity tag for the resource being returned.
+        @type etag: C{str}
+
+        @return: If I am a C{If-None-Match} conditional request and the tag
+            matches one in the request, I return L{CACHED<http.CACHED>} to
+            indicate that you should write no body.  Otherwise, I return a
+            false value.
+        """
+
+
+    def setHost(host, port, ssl=0):
+        """
+        Change the host and port the request thinks it's using.
+
+        This method is useful for working with reverse HTTP proxies (e.g.  both
+        Squid and Apache's mod_proxy can do this), when the address the HTTP
+        client is using is different than the one we're listening on.
+
+        For example, Apache may be listening on https://www.example.com, and
+        then forwarding requests to http://localhost:8080, but we don't want
+        HTML produced by Twisted to say 'http://localhost:8080', they should
+        say 'https://www.example.com', so we do::
+
+           request.setHost('www.example.com', 443, ssl=1)
+        """
+
+
+
+class ICredentialFactory(Interface):
+    """
+    A credential factory defines a way to generate a particular kind of
+    authentication challenge and a way to interpret the responses to these
+    challenges.  It creates
+    L{ICredentials<twisted.cred.credentials.ICredentials>} providers from
+    responses.  These objects will be used with L{twisted.cred} to authenticate
+    an authorize requests.
+    """
+    scheme = Attribute(
+        "A C{str} giving the name of the authentication scheme with which "
+        "this factory is associated.  For example, C{'basic'} or C{'digest'}.")
+
+
+    def getChallenge(request):
+        """
+        Generate a new challenge to be sent to a client.
+
+        @type peer: L{twisted.web.http.Request}
+        @param peer: The request the response to which this challenge will be
+            included.
+
+        @rtype: C{dict}
+        @return: A mapping from C{str} challenge fields to associated C{str}
+            values.
+        """
+
+
+    def decode(response, request):
+        """
+        Create a credentials object from the given response.
+
+        @type response: C{str}
+        @param response: scheme specific response string
+
+        @type request: L{twisted.web.http.Request}
+        @param request: The request being processed (from which the response
+            was taken).
+
+        @raise twisted.cred.error.LoginFailed: If the response is invalid.
+
+        @rtype: L{twisted.cred.credentials.ICredentials} provider
+        @return: The credentials represented by the given response.
+        """
+
+
+
+class IBodyProducer(IPushProducer):
+    """
+    Objects which provide L{IBodyProducer} write bytes to an object which
+    provides L{IConsumer<twisted.internet.interfaces.IConsumer>} by calling its
+    C{write} method repeatedly.
+
+    L{IBodyProducer} providers may start producing as soon as they have an
+    L{IConsumer<twisted.internet.interfaces.IConsumer>} provider.  That is, they
+    should not wait for a C{resumeProducing} call to begin writing data.
+
+    L{IConsumer.unregisterProducer<twisted.internet.interfaces.IConsumer.unregisterProducer>}
+    must not be called.  Instead, the
+    L{Deferred<twisted.internet.defer.Deferred>} returned from C{startProducing}
+    must be fired when all bytes have been written.
+
+    L{IConsumer.write<twisted.internet.interfaces.IConsumer.write>} may
+    synchronously invoke any of C{pauseProducing}, C{resumeProducing}, or
+    C{stopProducing}.  These methods must be implemented with this in mind.
+
+    @since: 9.0
+    """
+
+    # Despite the restrictions above and the additional requirements of
+    # stopProducing documented below, this interface still needs to be an
+    # IPushProducer subclass.  Providers of it will be passed to IConsumer
+    # providers which only know about IPushProducer and IPullProducer, not
+    # about this interface.  This interface needs to remain close enough to one
+    # of those interfaces for consumers to work with it.
+
+    length = Attribute(
+        """
+        C{length} is a C{int} indicating how many bytes in total this
+        L{IBodyProducer} will write to the consumer or L{UNKNOWN_LENGTH}
+        if this is not known in advance.
+        """)
+
+    def startProducing(consumer):
+        """
+        Start producing to the given
+        L{IConsumer<twisted.internet.interfaces.IConsumer>} provider.
+
+        @return: A L{Deferred<twisted.internet.defer.Deferred>} which fires with
+            C{None} when all bytes have been produced or with a
+            L{Failure<twisted.python.failure.Failure>} if there is any problem
+            before all bytes have been produced.
+        """
+
+
+    def stopProducing():
+        """
+        In addition to the standard behavior of
+        L{IProducer.stopProducing<twisted.internet.interfaces.IProducer.stopProducing>}
+        (stop producing data), make sure the
+        L{Deferred<twisted.internet.defer.Deferred>} returned by
+        C{startProducing} is never fired.
+        """
+
+
+
+class IRenderable(Interface):
+    """
+    An L{IRenderable} is an object that may be rendered by the
+    L{twisted.web.template} templating system.
+    """
+
+    def lookupRenderMethod(name):
+        """
+        Look up and return the render method associated with the given name.
+
+        @type name: C{str}
+        @param name: The value of a render directive encountered in the
+            document returned by a call to L{IRenderable.render}.
+
+        @return: A two-argument callable which will be invoked with the request
+            being responded to and the tag object on which the render directive
+            was encountered.
+        """
+
+
+    def render(request):
+        """
+        Get the document for this L{IRenderable}.
+
+        @type request: L{IRequest} provider or C{NoneType}
+        @param request: The request in response to which this method is being
+            invoked.
+
+        @return: An object which can be flattened.
+        """
+
+
+
+class ITemplateLoader(Interface):
+    """
+    A loader for templates; something usable as a value for
+    L{twisted.web.template.Element}'s C{loader} attribute.
+    """
+
+    def load():
+        """
+        Load a template suitable for rendering.
+
+        @return: a C{list} of C{list}s, C{unicode} objects, C{Element}s and
+            other L{IRenderable} providers.
+        """
+
+
+
+class IResponse(Interface):
+    """
+    An object representing an HTTP response received from an HTTP server.
+
+    @since: 11.1
+    """
+
+    version = Attribute(
+        "A three-tuple describing the protocol and protocol version "
+        "of the response.  The first element is of type C{str}, the second "
+        "and third are of type C{int}.  For example, C{('HTTP', 1, 1)}.")
+
+
+    code = Attribute("The HTTP status code of this response, as a C{int}.")
+
+
+    phrase = Attribute(
+        "The HTTP reason phrase of this response, as a C{str}.")
+
+
+    headers = Attribute("The HTTP response L{Headers} of this response.")
+
+
+    length = Attribute(
+        "The C{int} number of bytes expected to be in the body of this "
+        "response or L{UNKNOWN_LENGTH} if the server did not indicate how "
+        "many bytes to expect.  For I{HEAD} responses, this will be 0; if "
+        "the response includes a I{Content-Length} header, it will be "
+        "available in C{headers}.")
+
+
+    def deliverBody(protocol):
+        """
+        Register an L{IProtocol<twisted.internet.interfaces.IProtocol>} provider
+        to receive the response body.
+
+        The protocol will be connected to a transport which provides
+        L{IPushProducer}.  The protocol's C{connectionLost} method will be
+        called with:
+
+            - ResponseDone, which indicates that all bytes from the response
+              have been successfully delivered.
+
+            - PotentialDataLoss, which indicates that it cannot be determined
+              if the entire response body has been delivered.  This only occurs
+              when making requests to HTTP servers which do not set
+              I{Content-Length} or a I{Transfer-Encoding} in the response.
+
+            - ResponseFailed, which indicates that some bytes from the response
+              were lost.  The C{reasons} attribute of the exception may provide
+              more specific indications as to why.
+        """
+
+
+
+class _IRequestEncoder(Interface):
+    """
+    An object encoding data passed to L{IRequest.write}, for example for
+    compression purpose.
+
+    @since: 12.3
+    """
+
+    def encode(data):
+        """
+        Encode the data given and return the result.
+
+        @param data: The content to encode.
+        @type data: C{str}
+
+        @return: The encoded data.
+        @rtype: C{str}
+        """
+
+
+    def finish():
+        """
+        Callback called when the request is closing.
+
+        @return: If necessary, the pending data accumulated from previous
+            C{encode} calls.
+        @rtype: C{str}
+        """
+
+
+
+class _IRequestEncoderFactory(Interface):
+    """
+    A factory for returing L{_IRequestEncoder} instances.
+
+    @since: 12.3
+    """
+
+    def encoderForRequest(request):
+        """
+        If applicable, returns a L{_IRequestEncoder} instance which will encode
+        the request.
+        """
+
+
+
+UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH"
+
+__all__ = [
+    "ICredentialFactory", "IRequest",
+    "IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder",
+    "_IRequestEncoderFactory",
+
+    "UNKNOWN_LENGTH"]