remove Url class and use str instead for Request and Response urls. Also added urlparse_cached function for achieving the same caching functionality provided by old Url class

scrapy/contrib/aws.py

@@ -5,6 +5,8 @@ import hmac
 import base64
 import hashlib
 from urlparse import urlsplit
+
+from scrapy.utils.httpobj import urlparse_cached
 from scrapy.conf import settings
 
 
@@ -82,7 +84,8 @@ class AWSMiddleware(object):
         self.secret_key = settings['AWS_SECRET_ACCESS_KEY'] or os.environ.get('AWS_SECRET_ACCESS_KEY')
 
     def process_request(self, request, spider):
+        hostname = urlparse_cached(request).hostname
         if spider.domain_name == 's3.amazonaws.com' \
-                or (request.url.hostname and request.url.hostname.endswith('s3.amazonaws.com')):
+                or (hostname and hostname.endswith('s3.amazonaws.com')):
             request.headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
             sign_request(request, self.access_key, self.secret_key)

scrapy/contrib/downloadermiddleware/httpcache.py

@@ -9,11 +9,12 @@ from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.core import signals
 from scrapy import log
-from scrapy.http import Response, Headers
+from scrapy.http import Headers
 from scrapy.core.exceptions import NotConfigured, IgnoreRequest
 from scrapy.core.downloader.responsetypes import responsetypes
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.http import headers_dict_to_raw, headers_raw_to_dict
+from scrapy.utils.httpobj import urlparse_cached
 from scrapy.conf import settings
 
 
@@ -55,7 +56,7 @@ class HttpCacheMiddleware(object):
 
 
 def is_cacheable(request):
-    return request.url.scheme in ['http', 'https']
+    return urlparse_cached(request).scheme in ['http', 'https']
 
 
 class Cache(object):

scrapy/contrib/downloadermiddleware/robotstxt.py

@@ -12,6 +12,7 @@ from scrapy.core import signals
 from scrapy.core.engine import scrapyengine
 from scrapy.core.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
+from scrapy.utils.httpobj import urlparse_cached
 from scrapy.conf import settings
 
 class RobotsTxtMiddleware(object):
@@ -48,7 +49,7 @@ class RobotsTxtMiddleware(object):
     def _parse_robots(self, response):
         rp = robotparser.RobotFileParser(response.url)
         rp.parse(response.body.splitlines())
-        self._parsers[response.url.netloc] = rp
+        self._parsers[urlparse_cached(response).netloc] = rp
 
     def domain_open(self, spider):
         self._spider_netlocs[spider] = set()

scrapy/core/downloader/handlers.py

@@ -6,8 +6,6 @@ from __future__ import with_statement
 import urlparse
 
 from twisted.internet import reactor
-from twisted.web import error as web_error
-
 try:
     from twisted.internet import ssl
 except ImportError:
@@ -18,11 +16,12 @@ from scrapy.core import signals
 from scrapy.http import Headers
 from scrapy.core.exceptions import NotSupported
 from scrapy.utils.defer import defer_succeed
-from scrapy.conf import settings
-
+from scrapy.utils.httpobj import urlparse_cached
 from scrapy.core.downloader.dnscache import DNSCache
 from scrapy.core.downloader.responsetypes import responsetypes
-from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory as HTTPClientFactory
+from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory
+from scrapy.conf import settings
+
 
 default_timeout = settings.getint('DOWNLOAD_TIMEOUT')
 ssl_supported = 'ssl' in optional_features
@@ -31,7 +30,7 @@ ssl_supported = 'ssl' in optional_features
 dnscache = DNSCache()
 
 def download_any(request, spider):
-    scheme = request.url.scheme
+    scheme = urlparse_cached(request).scheme
     if scheme == 'http':
         return download_http(request, spider)
     elif scheme == 'https':
@@ -39,21 +38,16 @@ def download_any(request, spider):
             return download_https(request, spider)
         else:
             raise NotSupported("HTTPS not supported: install pyopenssl library")
-    elif request.url.scheme == 'file':
+    elif scheme == 'file':
         return download_file(request, spider)
     else:
-        raise NotSupported("Unsupported URL scheme '%s' in: <%s>" % (request.url.scheme, request.url))
+        raise NotSupported("Unsupported URL scheme '%s' in: <%s>" % (scheme, request.url))
 
 def create_factory(request, spider):
     """Return HTTPClientFactory for the given Request"""
     url = urlparse.urldefrag(request.url)[0]
     timeout = getattr(spider, "download_timeout", None) or default_timeout
-
-    factory = HTTPClientFactory(url=url, # never pass unicode urls to twisted
-                                method=request.method,
-                                body=request.body or None, # see http://dev.scrapy.org/ticket/60
-                                headers=request.headers,
-                                timeout=timeout)
+    factory = ScrapyHTTPClientFactory.from_request(request, timeout)
 
     def _create_response(body):
         body = body or ''
@@ -71,16 +65,18 @@ def create_factory(request, spider):
 def download_http(request, spider):
     """Return a deferred for the HTTP download"""
     factory = create_factory(request, spider)
-    ip = dnscache.get(request.url.hostname)
-    port = request.url.port
+    url = urlparse_cached(request)
+    ip = dnscache.get(url.hostname)
+    port = url.port
     reactor.connectTCP(ip, port or 80, factory)
     return factory.deferred
 
 def download_https(request, spider):
     """Return a deferred for the HTTPS download"""
     factory = create_factory(request, spider)
-    ip = dnscache.get(request.url.hostname)
-    port = request.url.port
+    url = urlparse_cached(request)
+    ip = dnscache.get(url.hostname)
+    port = url.port
     contextFactory = ssl.ClientContextFactory()
     reactor.connectSSL(ip, port or 443, factory, contextFactory)
     return factory.deferred

scrapy/core/downloader/webclient.py

@@ -1,30 +1,26 @@
-from urlparse import urlunparse
+from urlparse import urlparse, urlunparse
 
 from twisted.python import failure
 from twisted.web.client import HTTPClientFactory, PartialDownloadError
 from twisted.web.http import HTTPClient
 from twisted.internet import defer
 
-from scrapy.http import Url, Headers
-
-
-def _parse(url):
-    url = url.strip()
-    try:
-        parsed = url.parsedurl
-    except AttributeError:
-        parsed = Url(url.strip()).parsedurl
+from scrapy.http import Headers
+from scrapy.utils.httpobj import urlparse_cached
 
+def _parsed_url_args(parsed):
     path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
     host = parsed.hostname
     port = parsed.port
     scheme = parsed.scheme
-
     if port is None:
         port = 443 if scheme == 'https' else 80
-
     return scheme, host, port, path
 
+def _parse(url):
+    url = url.strip()
+    parsed = urlparse(url)
+    return _parsed_url_args(parsed)
 
 class ScrapyHTTPPageGetter(HTTPClient):
 
@@ -81,11 +77,15 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
     waiting = 1
     noisy = False
 
-    def __init__(self, url, method='GET', body=None, headers=None, timeout=0):
+    def __init__(self, url, method='GET', body=None, headers=None, timeout=0, parsedurl=None):
         self.url = url
         self.method = method
         self.body = body or None
-        self.scheme, self.host, self.port, self.path = _parse(url)
+        if parsedurl:
+            self.scheme, self.host, self.port, self.path = _parsed_url_args(parsedurl)
+        else:
+            self.scheme, self.host, self.port, self.path = _parse(url)
+
         self.timeout = timeout
         self.headers = Headers(headers or {})
         self.deferred = defer.Deferred()
@@ -99,6 +99,16 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
             # just in case a broken http/1.1 decides to keep connection alive
             self.headers.setdefault("Connection", "close")
 
+    @classmethod
+    def from_request(cls, request, timeout):
+        return cls(request.url,
+            method=request.method,
+            body=request.body or None, # see http://dev.scrapy.org/ticket/60
+            headers=Headers(request.headers or {}),
+            timeout=timeout,
+            parsedurl=urlparse_cached(request),
+            )
+
     def gotHeaders(self, headers):
         self.response_headers = headers
 

scrapy/http/__init__.py

@@ -2,10 +2,9 @@
 Module containing all HTTP related classes
 
 Use this module (instead of the more specific ones) when importing Headers,
-Request, Response and Url outside this module.
+Request and Response outside this module.
 """
 
-from scrapy.http.url import Url
 from scrapy.http.headers import Headers
 
 from scrapy.http.request import Request

scrapy/http/cookies.py

@@ -1,5 +1,6 @@
-from cookielib import CookieJar as _CookieJar, DefaultCookiePolicy, Cookie
+from cookielib import CookieJar as _CookieJar, DefaultCookiePolicy
 
+from scrapy.utils.httpobj import urlparse_cached
 
 class CookieJar(object):
     def __init__(self, policy=None):
@@ -67,10 +68,10 @@ class WrappedRequest(object):
         return self.request.url
 
     def get_host(self):
-        return self.request.url.netloc
+        return urlparse_cached(self.request).netloc
 
     def get_type(self):
-        return self.request.url.scheme
+        return urlparse_cached(self.request).scheme
 
     def is_unverifiable(self):
         """Unverifiable should indicate whether the request is unverifiable, as defined by RFC 2965.

scrapy/http/request/__init__.py

@@ -9,7 +9,6 @@ import copy
 
 from twisted.internet import defer
 
-from scrapy.http.url import Url
 from scrapy.http.headers import Headers
 from scrapy.utils.url import safe_url_string
 
@@ -58,9 +57,7 @@ class Request(object):
     def _set_url(self, url):
         if isinstance(url, basestring):
             decoded_url = url if isinstance(url, unicode) else url.decode(self.encoding)
-            self._url = Url(safe_url_string(decoded_url, self.encoding))
-        elif isinstance(url, Url):
-            self._url = url
+            self._url = safe_url_string(decoded_url, self.encoding)
         else:
             raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
 

scrapy/http/response/__init__.py

@@ -7,7 +7,6 @@ See documentation in docs/ref/request-response.rst
 
 import copy
 
-from scrapy.http.url import Url
 from scrapy.http.headers import Headers
 
 class Response(object):
@@ -16,7 +15,7 @@ class Response(object):
         'flags', '_cache', '__weakref__']
 
     def __init__(self, url, status=200, headers=None, body='', meta=None, flags=None):
-        self.url = Url(url)
+        self.url = url
         self.headers = Headers(headers or {})
         self.status = int(status)
         self._set_body(body)

scrapy/http/url.py

@@ -1,57 +0,0 @@
-"""
-The Url class is similar to the urlparse.ParseResult class (it has the same
-attributes) with the following differences:
-
-- it inherits from str
-- it's lazy, so it only parses the url when needed
-"""
-
-import urlparse
-
-class Url(str):
-
-    @property
-    def parsedurl(self):
-        if not hasattr(self, '_parsedurl'):
-            self._parsedurl = urlparse.urlparse(self)
-        return self._parsedurl
-
-    @property
-    def scheme(self):
-        return self.parsedurl.scheme
-
-    @property
-    def netloc(self):
-        return self.parsedurl.netloc
-
-    @property
-    def path(self):
-        return self.parsedurl.path
-
-    @property
-    def params(self):
-        return self.parsedurl.params
-
-    @property
-    def query(self):
-        return self.parsedurl.query
-
-    @property
-    def fragment(self):
-        return self.parsedurl.fragment
-
-    @property
-    def username(self):
-        return self.parsedurl.username
-
-    @property
-    def password(self):
-        return self.parsedurl.password
-
-    @property
-    def hostname(self):
-        return self.parsedurl.hostname
-
-    @property
-    def port(self):
-        return self.parsedurl.port

scrapy/tests/test_http_request.py

@@ -1,8 +1,9 @@
 import unittest
-from cStringIO import StringIO
 import cgi
+from cStringIO import StringIO
+from urlparse import urlparse
 
-from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, Url, Response
+from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, Response
 
 class RequestTest(unittest.TestCase):
 
@@ -10,17 +11,17 @@ class RequestTest(unittest.TestCase):
         # Request requires url in the constructor
         self.assertRaises(Exception, Request)
 
-        # url argument must be basestring or Url
+        # url argument must be basestring
         self.assertRaises(TypeError, Request, 123)
-        r = Request(Url('http://www.example.com'))
+        r = Request('http://www.example.com')
 
         r = Request("http://www.example.com")
-        assert isinstance(r.url, Url)
+        assert isinstance(r.url, str)
         self.assertEqual(r.url, "http://www.example.com")
         self.assertEqual(r.method, "GET")
 
         r.url = "http://www.example.com/other"
-        assert isinstance(r.url, Url)
+        assert isinstance(r.url, str)
 
         assert isinstance(r.headers, Headers)
         self.assertEqual(r.headers, {})
@@ -28,7 +29,6 @@ class RequestTest(unittest.TestCase):
 
         meta = {"lala": "lolo"}
         headers = {"caca": "coco"}
-        body = "a body"
         r = Request("http://www.example.com", meta=meta, headers=headers, body="a body")
 
         assert r.meta is not meta
@@ -224,9 +224,9 @@ class FormRequestTest(unittest.TestCase):
         response = Response("http://www.example.com/this/list.html", body=respbody)
         r1 = FormRequest.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
         self.assertEqual(r1.method, 'GET')
-        self.assertEqual(r1.url.hostname, "www.example.com")
-        self.assertEqual(r1.url.path, "/this/get.php")
-        urlargs = cgi.parse_qs(r1.url.query)
+        self.assertEqual(urlparse(r1.url).hostname, "www.example.com")
+        self.assertEqual(urlparse(r1.url).path, "/this/get.php")
+        urlargs = cgi.parse_qs(urlparse(r1.url).query)
         self.assertEqual(set(urlargs['test']), set(['val1', 'val2']))
         self.assertEqual(set(urlargs['one']), set(['two', 'three']))
         self.assertEqual(urlargs['test2'], ['xxx'])

scrapy/tests/test_http_response.py

@@ -1,5 +1,5 @@
 import unittest
-from scrapy.http import Response, TextResponse, HtmlResponse, XmlResponse, Headers, Url
+from scrapy.http import Response, TextResponse, HtmlResponse, XmlResponse, Headers
 
 class ResponseTest(unittest.TestCase):
 
@@ -14,7 +14,7 @@ class ResponseTest(unittest.TestCase):
         self.assertTrue(isinstance(Response('http://example.com/', headers={}, status=200, body=''), Response))
 
         r = Response("http://www.example.com")
-        assert isinstance(r.url, Url)
+        assert isinstance(r.url, str)
         self.assertEqual(r.url, "http://www.example.com")
         self.assertEqual(r.status, 200)
 

scrapy/tests/test_http_url.py

@@ -1,35 +0,0 @@
-import unittest
-from scrapy.http import Url
-
-class UrlClassTest(unittest.TestCase):
-
-    def test_url_attributes(self):
-        u = Url("http://scrapy.org/wiki/info")
-        self.assertEqual("scrapy.org", u.hostname)
-        self.assertEqual("/wiki/info", u.path)
-        self.assertEqual(None, u.username)
-        self.assertEqual(None, u.password)
-
-        u = Url("http://scrapy.org:8080/wiki/info")
-        self.assertEqual("scrapy.org", u.hostname)
-        self.assertEqual("/wiki/info", u.path)
-        self.assertEqual(8080, u.port)
-        self.assertEqual(None, u.username)
-        self.assertEqual(None, u.password)
-
-        u = Url("http://someuser:somepass@example.com/ticket/query?owner=pablo")
-        self.assertEqual("someuser", u.username)
-        self.assertEqual("somepass", u.password)
-        self.assertEqual("example.com", u.hostname)
-        self.assertEqual("/ticket/query", u.path)
-        self.assertEqual("owner=pablo", u.query)
-
-        u = Url("http://example.com/somepage.html#fragment-1")
-        self.assertEqual("fragment-1", u.fragment)
-
-        u = Url("file:///home/pablo/file.txt")
-        self.assertEqual("/home/pablo/file.txt", u.path)
-
-if __name__ == "__main__":
-    unittest.main()
-

scrapy/tests/test_utils_httpobj.py

@@ -0,0 +1,26 @@
+import unittest
+import urlparse
+
+from scrapy.http import Request
+from scrapy.utils.httpobj import urlparse_cached
+
+class HttpobjUtilsTest(unittest.TestCase):
+
+    def test_urlparse_cached(self):
+        url = "http://www.example.com/index.html"
+        request1 = Request(url)
+        request2 = Request(url)
+        req1a = urlparse_cached(request1)
+        req1b = urlparse_cached(request1)
+        req2 = urlparse_cached(request2)
+        urlp = urlparse.urlparse(url)
+
+        assert req1a == req2
+        assert req1a == urlp
+        assert req1a is req1b
+        assert req1a is not req2
+        assert req1a is not req2
+
+
+if __name__ == "__main__":
+    unittest.main()

scrapy/tests/test_webclient.py

@@ -13,14 +13,14 @@ from twisted.python.filepath import FilePath
 from twisted.protocols.policies import WrappingFactory
 
 from scrapy.core.downloader import webclient as client
-from scrapy.http import Url, Headers
+from scrapy.http import Headers
 
 
 class ParseUrlTestCase(unittest.TestCase):
     """Test URL parsing facility and defaults values."""
 
     def _parse(self, url):
-        f = client.ScrapyHTTPClientFactory(Url(url))
+        f = client.ScrapyHTTPClientFactory(url)
         return (f.scheme, f.host, f.port, f.path)
 
     def testParse(self):

scrapy/utils/httpobj.py

@@ -0,0 +1,14 @@
+"""Helper functions for scrapy.http objects (Request, Response)"""
+
+import weakref
+
+from urlparse import urlparse
+
+_urlparse_cache = weakref.WeakKeyDictionary()
+def urlparse_cached(request_or_response):
+    """Return urlparse.urlparse caching the result, where the argument can be a
+    Request or Response object
+    """
+    if request_or_response not in _urlparse_cache:
+        _urlparse_cache[request_or_response] = urlparse(request_or_response.url)
+    return _urlparse_cache[request_or_response]

scrapy/utils/request.py

@@ -7,6 +7,7 @@ import hashlib
 from base64 import urlsafe_b64encode
 
 from scrapy.utils.url import canonicalize_url
+from scrapy.utils.httpobj import urlparse_cached
 
 def request_fingerprint(request, include_headers=()):
     """
@@ -78,9 +79,9 @@ def request_httprepr(request):
     bytes that will be send when performing the request (that's controlled
     by Twisted).
     """
-
+    hostname = urlparse_cached(request).hostname
     s  = "%s %s HTTP/1.1\r\n" % (request.method, request.url)
-    s += "Host: %s\r\n" % request.url.hostname
+    s += "Host: %s\r\n" % hostname
     if request.headers:
         s += request.headers.to_string() + "\r\n"
     s += "\r\n"