some minor performance improvements in downloader handlers, added scrapy.optional_features set

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40746

scrapy/trunk/scrapy/__init__.py

@@ -17,3 +17,13 @@ sys.path.insert(0, os.path.join(os.path.abspath(os.path.dirname(__file__)), "xli
 # monkey patches to fix external library issues
 from scrapy.patches import monkeypatches
 monkeypatches.apply_patches()
+
+# optional_features is a set containing Scrapy optional features
+optional_features = set()
+
+try:
+    import OpenSSL
+except ImportError:
+    pass
+else:
+    optional_features.add('ssl')

scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py

@@ -3,7 +3,6 @@ from __future__ import with_statement
 import os
 import hashlib
 import datetime
-import urlparse
 import cPickle as pickle
 from pydispatch import dispatcher
 
@@ -74,8 +73,7 @@ class CacheMiddleware(object):
             self.cache.store(domain, key, request, exception.response)
 
 def is_cacheable(request):
-    scheme, _, _, _, _ = urlparse.urlsplit(request.url)
-    return scheme in ['http', 'https']
+    return request.url.scheme in ['http', 'https']
 
 
 class Cache(object):
@@ -172,7 +170,7 @@ class Cache(object):
                 'url':request.url,
                 'method': request.method,
                 'status': response.status,
-                'domain': response.domain,
+                'domain': domain,
                 'timestamp': datetime.datetime.utcnow(),
             }
 

scrapy/trunk/scrapy/core/downloader/handlers.py

@@ -10,42 +10,56 @@ from twisted.web.client import HTTPClientFactory
 from twisted.internet import defer, reactor
 from twisted.web import error as web_error
 
+try:
+    from twisted.internet import ssl
+except ImportError:
+    pass
+
+from scrapy import optional_features
 from scrapy.core import signals
 from scrapy.http import Request, Response, Headers
-from scrapy.core.exceptions import UsageError, HttpException
+from scrapy.core.exceptions import UsageError, HttpException, NotSupported
 from scrapy.utils.defer import defer_succeed
 from scrapy.conf import settings
 
 from scrapy.core.downloader.dnscache import DNSCache
 
+default_timeout = settings.getint('DOWNLOAD_TIMEOUT')
+default_agent = settings.get('USER_AGENT')
+ssl_supported = 'ssl' in optional_features
+
 # Cache for dns lookups.
 dnscache = DNSCache()
 
 def download_any(request, spider):
-    u = urlparse.urlparse(request.url)
-    if u.scheme == 'file':
-        return download_file(request, spider)
-    elif u.scheme in ('http', 'https'):
+    scheme = request.url.scheme
+    if scheme == 'http':
         return download_http(request, spider)
+    elif scheme == 'https':
+        if ssl_supported:
+            return download_https(request, spider)
+        else:
+            raise NotSupported("HTTPS not supported: install pyopenssl library")
+    elif request.url.scheme == 'file':
+        return download_file(request, spider)
     else:
-        raise UsageError("Unsupported scheme '%s' in URL: <%s>" % (u.scheme, request.url))
+        raise NotSupported("Unsupported URL scheme '%s' in: <%s>" % (request.url.scheme, request.url))
 
-def download_http(request, spider):
-    """This functions handles http/https downloads"""
+def create_factory(request, spider):
+    """Return HTTPClientFactory for the given Request"""
     url = urlparse.urldefrag(request.url)[0]
-    
-    agent = request.headers.get('user-agent', settings.get('USER_AGENT'))
-    request.headers.pop('user-agent', None)  # remove user-agent if already present
-    factory = HTTPClientFactory(url=str(url), # never pass unicode urls to twisted
+
+    agent = request.headers.pop('user-agent', default_agent)
+    factory = HTTPClientFactory(url=url, # never pass unicode urls to twisted
                                 method=request.method,
                                 postdata=request.body,
                                 headers=request.headers,
                                 agent=agent,
                                 cookies=request.cookies,
-                                timeout=getattr(spider, "download_timeout", None) or settings.getint('DOWNLOAD_TIMEOUT'),
+                                timeout=getattr(spider, "download_timeout", default_timeout),
                                 followRedirect=False)
 
-    def _response(body):
+    def _create_response(body):
         body = body or ''
         status = int(factory.status)
         headers = Headers(factory.response_headers)
@@ -55,26 +69,33 @@ def download_http(request, spider):
         return r
 
     def _on_success(body):
-        return _response(body)
+        return _create_response(body)
 
     def _on_error(_failure):
         ex = _failure.value
         if isinstance(ex, web_error.Error): # HttpException
-            raise HttpException(ex.status, ex.message, _response(ex.response))
+            raise HttpException(ex.status, ex.message, _create_response(ex.response))
         return _failure
 
     factory.noisy = False
     factory.deferred.addCallbacks(_on_success, _on_error)
+    return factory
 
-    u = urlparse.urlparse(request.url)
-    ip = dnscache.get(u.hostname)
-    port = u.port
-    if u.scheme == 'https' :
-        from twisted.internet import ssl
-        contextFactory = ssl.ClientContextFactory()
-        reactor.connectSSL(ip, port or 443, factory, contextFactory)
-    else:
-        reactor.connectTCP(ip, port or 80, factory)
+def download_http(request, spider):
+    """Return a deferred for the HTTP download"""
+    factory = create_factory(request, spider)
+    ip = dnscache.get(request.url.hostname)
+    port = request.url.port
+    reactor.connectTCP(ip, port or 80, factory)
+    return factory.deferred
+
+def download_https(request, spider):
+    """Return a deferred for the HTTPS download"""
+    factory = create_factory(request, spider)
+    ip = dnscache.get(request.url.hostname)
+    port = request.url.port
+    contextFactory = ssl.ClientContextFactory()
+    reactor.connectSSL(ip, port or 443, factory, contextFactory)
     return factory.deferred
 
 def download_file(request, spider) :

scrapy/trunk/scrapy/tests/test_http_request.py

@@ -13,6 +13,9 @@ class RequestTest(unittest.TestCase):
         self.assertEqual(r.url, "http://www.example.com")
         self.assertEqual(r.method, "GET")
 
+        r.url = "http://www.example.com/other"
+        assert isinstance(r.url, Url)
+
         assert isinstance(r.headers, Headers)
         self.assertEqual(r.headers, {})
         self.assertEqual(r.meta, {})