diff --git a/scrapy/contrib/aws.py b/scrapy/contrib/aws.py index 03ee993bc..5addcb84b 100644 --- a/scrapy/contrib/aws.py +++ b/scrapy/contrib/aws.py @@ -5,6 +5,8 @@ import hmac import base64 import hashlib from urlparse import urlsplit + +from scrapy.utils.httpobj import urlparse_cached from scrapy.conf import settings @@ -82,7 +84,8 @@ class AWSMiddleware(object): self.secret_key = settings['AWS_SECRET_ACCESS_KEY'] or os.environ.get('AWS_SECRET_ACCESS_KEY') def process_request(self, request, spider): + hostname = urlparse_cached(request).hostname if spider.domain_name == 's3.amazonaws.com' \ - or (request.url.hostname and request.url.hostname.endswith('s3.amazonaws.com')): + or (hostname and hostname.endswith('s3.amazonaws.com')): request.headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) sign_request(request, self.access_key, self.secret_key) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 94b65c7c9..18085bea7 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -9,11 +9,12 @@ from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy import log -from scrapy.http import Response, Headers +from scrapy.http import Headers from scrapy.core.exceptions import NotConfigured, IgnoreRequest from scrapy.core.downloader.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.http import headers_dict_to_raw, headers_raw_to_dict +from scrapy.utils.httpobj import urlparse_cached from scrapy.conf import settings @@ -55,7 +56,7 @@ class HttpCacheMiddleware(object): def is_cacheable(request): - return request.url.scheme in ['http', 'https'] + return urlparse_cached(request).scheme in ['http', 'https'] class Cache(object): diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py index 20da35f2c..f0fed6193 100644 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -12,6 +12,7 @@ from scrapy.core import signals from scrapy.core.engine import scrapyengine from scrapy.core.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request +from scrapy.utils.httpobj import urlparse_cached from scrapy.conf import settings class RobotsTxtMiddleware(object): @@ -48,7 +49,7 @@ class RobotsTxtMiddleware(object): def _parse_robots(self, response): rp = robotparser.RobotFileParser(response.url) rp.parse(response.body.splitlines()) - self._parsers[response.url.netloc] = rp + self._parsers[urlparse_cached(response).netloc] = rp def domain_open(self, spider): self._spider_netlocs[spider] = set() diff --git a/scrapy/core/downloader/handlers.py b/scrapy/core/downloader/handlers.py index f7f2883d6..b36b1dc11 100644 --- a/scrapy/core/downloader/handlers.py +++ b/scrapy/core/downloader/handlers.py @@ -6,8 +6,6 @@ from __future__ import with_statement import urlparse from twisted.internet import reactor -from twisted.web import error as web_error - try: from twisted.internet import ssl except ImportError: @@ -18,11 +16,12 @@ from scrapy.core import signals from scrapy.http import Headers from scrapy.core.exceptions import NotSupported from scrapy.utils.defer import defer_succeed -from scrapy.conf import settings - +from scrapy.utils.httpobj import urlparse_cached from scrapy.core.downloader.dnscache import DNSCache from scrapy.core.downloader.responsetypes import responsetypes -from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory as HTTPClientFactory +from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory +from scrapy.conf import settings + default_timeout = settings.getint('DOWNLOAD_TIMEOUT') ssl_supported = 'ssl' in optional_features @@ -31,7 +30,7 @@ ssl_supported = 'ssl' in optional_features dnscache = DNSCache() def download_any(request, spider): - scheme = request.url.scheme + scheme = urlparse_cached(request).scheme if scheme == 'http': return download_http(request, spider) elif scheme == 'https': @@ -39,21 +38,16 @@ def download_any(request, spider): return download_https(request, spider) else: raise NotSupported("HTTPS not supported: install pyopenssl library") - elif request.url.scheme == 'file': + elif scheme == 'file': return download_file(request, spider) else: - raise NotSupported("Unsupported URL scheme '%s' in: <%s>" % (request.url.scheme, request.url)) + raise NotSupported("Unsupported URL scheme '%s' in: <%s>" % (scheme, request.url)) def create_factory(request, spider): """Return HTTPClientFactory for the given Request""" url = urlparse.urldefrag(request.url)[0] timeout = getattr(spider, "download_timeout", None) or default_timeout - - factory = HTTPClientFactory(url=url, # never pass unicode urls to twisted - method=request.method, - body=request.body or None, # see http://dev.scrapy.org/ticket/60 - headers=request.headers, - timeout=timeout) + factory = ScrapyHTTPClientFactory.from_request(request, timeout) def _create_response(body): body = body or '' @@ -71,16 +65,18 @@ def create_factory(request, spider): def download_http(request, spider): """Return a deferred for the HTTP download""" factory = create_factory(request, spider) - ip = dnscache.get(request.url.hostname) - port = request.url.port + url = urlparse_cached(request) + ip = dnscache.get(url.hostname) + port = url.port reactor.connectTCP(ip, port or 80, factory) return factory.deferred def download_https(request, spider): """Return a deferred for the HTTPS download""" factory = create_factory(request, spider) - ip = dnscache.get(request.url.hostname) - port = request.url.port + url = urlparse_cached(request) + ip = dnscache.get(url.hostname) + port = url.port contextFactory = ssl.ClientContextFactory() reactor.connectSSL(ip, port or 443, factory, contextFactory) return factory.deferred diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index b1064b29e..0ad5ab167 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,30 +1,26 @@ -from urlparse import urlunparse +from urlparse import urlparse, urlunparse from twisted.python import failure from twisted.web.client import HTTPClientFactory, PartialDownloadError from twisted.web.http import HTTPClient from twisted.internet import defer -from scrapy.http import Url, Headers - - -def _parse(url): - url = url.strip() - try: - parsed = url.parsedurl - except AttributeError: - parsed = Url(url.strip()).parsedurl +from scrapy.http import Headers +from scrapy.utils.httpobj import urlparse_cached +def _parsed_url_args(parsed): path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, '')) host = parsed.hostname port = parsed.port scheme = parsed.scheme - if port is None: port = 443 if scheme == 'https' else 80 - return scheme, host, port, path +def _parse(url): + url = url.strip() + parsed = urlparse(url) + return _parsed_url_args(parsed) class ScrapyHTTPPageGetter(HTTPClient): @@ -81,11 +77,15 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): waiting = 1 noisy = False - def __init__(self, url, method='GET', body=None, headers=None, timeout=0): + def __init__(self, url, method='GET', body=None, headers=None, timeout=0, parsedurl=None): self.url = url self.method = method self.body = body or None - self.scheme, self.host, self.port, self.path = _parse(url) + if parsedurl: + self.scheme, self.host, self.port, self.path = _parsed_url_args(parsedurl) + else: + self.scheme, self.host, self.port, self.path = _parse(url) + self.timeout = timeout self.headers = Headers(headers or {}) self.deferred = defer.Deferred() @@ -99,6 +99,16 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): # just in case a broken http/1.1 decides to keep connection alive self.headers.setdefault("Connection", "close") + @classmethod + def from_request(cls, request, timeout): + return cls(request.url, + method=request.method, + body=request.body or None, # see http://dev.scrapy.org/ticket/60 + headers=Headers(request.headers or {}), + timeout=timeout, + parsedurl=urlparse_cached(request), + ) + def gotHeaders(self, headers): self.response_headers = headers diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py index e1509d5e8..f04a9d3e5 100644 --- a/scrapy/http/__init__.py +++ b/scrapy/http/__init__.py @@ -2,10 +2,9 @@ Module containing all HTTP related classes Use this module (instead of the more specific ones) when importing Headers, -Request, Response and Url outside this module. +Request and Response outside this module. """ -from scrapy.http.url import Url from scrapy.http.headers import Headers from scrapy.http.request import Request diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index ec3580164..e2655e0c9 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,5 +1,6 @@ -from cookielib import CookieJar as _CookieJar, DefaultCookiePolicy, Cookie +from cookielib import CookieJar as _CookieJar, DefaultCookiePolicy +from scrapy.utils.httpobj import urlparse_cached class CookieJar(object): def __init__(self, policy=None): @@ -67,10 +68,10 @@ class WrappedRequest(object): return self.request.url def get_host(self): - return self.request.url.netloc + return urlparse_cached(self.request).netloc def get_type(self): - return self.request.url.scheme + return urlparse_cached(self.request).scheme def is_unverifiable(self): """Unverifiable should indicate whether the request is unverifiable, as defined by RFC 2965. diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index df577d63d..ddde58ffc 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -9,7 +9,6 @@ import copy from twisted.internet import defer -from scrapy.http.url import Url from scrapy.http.headers import Headers from scrapy.utils.url import safe_url_string @@ -58,9 +57,7 @@ class Request(object): def _set_url(self, url): if isinstance(url, basestring): decoded_url = url if isinstance(url, unicode) else url.decode(self.encoding) - self._url = Url(safe_url_string(decoded_url, self.encoding)) - elif isinstance(url, Url): - self._url = url + self._url = safe_url_string(decoded_url, self.encoding) else: raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index c64f8ecb3..aae1fec18 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -7,7 +7,6 @@ See documentation in docs/ref/request-response.rst import copy -from scrapy.http.url import Url from scrapy.http.headers import Headers class Response(object): @@ -16,7 +15,7 @@ class Response(object): 'flags', '_cache', '__weakref__'] def __init__(self, url, status=200, headers=None, body='', meta=None, flags=None): - self.url = Url(url) + self.url = url self.headers = Headers(headers or {}) self.status = int(status) self._set_body(body) diff --git a/scrapy/http/url.py b/scrapy/http/url.py deleted file mode 100644 index 098150be1..000000000 --- a/scrapy/http/url.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -The Url class is similar to the urlparse.ParseResult class (it has the same -attributes) with the following differences: - -- it inherits from str -- it's lazy, so it only parses the url when needed -""" - -import urlparse - -class Url(str): - - @property - def parsedurl(self): - if not hasattr(self, '_parsedurl'): - self._parsedurl = urlparse.urlparse(self) - return self._parsedurl - - @property - def scheme(self): - return self.parsedurl.scheme - - @property - def netloc(self): - return self.parsedurl.netloc - - @property - def path(self): - return self.parsedurl.path - - @property - def params(self): - return self.parsedurl.params - - @property - def query(self): - return self.parsedurl.query - - @property - def fragment(self): - return self.parsedurl.fragment - - @property - def username(self): - return self.parsedurl.username - - @property - def password(self): - return self.parsedurl.password - - @property - def hostname(self): - return self.parsedurl.hostname - - @property - def port(self): - return self.parsedurl.port diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index d1d31cd33..ac6ee18dc 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -1,8 +1,9 @@ import unittest -from cStringIO import StringIO import cgi +from cStringIO import StringIO +from urlparse import urlparse -from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, Url, Response +from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, Response class RequestTest(unittest.TestCase): @@ -10,17 +11,17 @@ class RequestTest(unittest.TestCase): # Request requires url in the constructor self.assertRaises(Exception, Request) - # url argument must be basestring or Url + # url argument must be basestring self.assertRaises(TypeError, Request, 123) - r = Request(Url('http://www.example.com')) + r = Request('http://www.example.com') r = Request("http://www.example.com") - assert isinstance(r.url, Url) + assert isinstance(r.url, str) self.assertEqual(r.url, "http://www.example.com") self.assertEqual(r.method, "GET") r.url = "http://www.example.com/other" - assert isinstance(r.url, Url) + assert isinstance(r.url, str) assert isinstance(r.headers, Headers) self.assertEqual(r.headers, {}) @@ -28,7 +29,6 @@ class RequestTest(unittest.TestCase): meta = {"lala": "lolo"} headers = {"caca": "coco"} - body = "a body" r = Request("http://www.example.com", meta=meta, headers=headers, body="a body") assert r.meta is not meta @@ -224,9 +224,9 @@ class FormRequestTest(unittest.TestCase): response = Response("http://www.example.com/this/list.html", body=respbody) r1 = FormRequest.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) self.assertEqual(r1.method, 'GET') - self.assertEqual(r1.url.hostname, "www.example.com") - self.assertEqual(r1.url.path, "/this/get.php") - urlargs = cgi.parse_qs(r1.url.query) + self.assertEqual(urlparse(r1.url).hostname, "www.example.com") + self.assertEqual(urlparse(r1.url).path, "/this/get.php") + urlargs = cgi.parse_qs(urlparse(r1.url).query) self.assertEqual(set(urlargs['test']), set(['val1', 'val2'])) self.assertEqual(set(urlargs['one']), set(['two', 'three'])) self.assertEqual(urlargs['test2'], ['xxx']) diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py index c5803e8f2..d60d43328 100644 --- a/scrapy/tests/test_http_response.py +++ b/scrapy/tests/test_http_response.py @@ -1,5 +1,5 @@ import unittest -from scrapy.http import Response, TextResponse, HtmlResponse, XmlResponse, Headers, Url +from scrapy.http import Response, TextResponse, HtmlResponse, XmlResponse, Headers class ResponseTest(unittest.TestCase): @@ -14,7 +14,7 @@ class ResponseTest(unittest.TestCase): self.assertTrue(isinstance(Response('http://example.com/', headers={}, status=200, body=''), Response)) r = Response("http://www.example.com") - assert isinstance(r.url, Url) + assert isinstance(r.url, str) self.assertEqual(r.url, "http://www.example.com") self.assertEqual(r.status, 200) diff --git a/scrapy/tests/test_http_url.py b/scrapy/tests/test_http_url.py deleted file mode 100644 index eb0b0c860..000000000 --- a/scrapy/tests/test_http_url.py +++ /dev/null @@ -1,35 +0,0 @@ -import unittest -from scrapy.http import Url - -class UrlClassTest(unittest.TestCase): - - def test_url_attributes(self): - u = Url("http://scrapy.org/wiki/info") - self.assertEqual("scrapy.org", u.hostname) - self.assertEqual("/wiki/info", u.path) - self.assertEqual(None, u.username) - self.assertEqual(None, u.password) - - u = Url("http://scrapy.org:8080/wiki/info") - self.assertEqual("scrapy.org", u.hostname) - self.assertEqual("/wiki/info", u.path) - self.assertEqual(8080, u.port) - self.assertEqual(None, u.username) - self.assertEqual(None, u.password) - - u = Url("http://someuser:somepass@example.com/ticket/query?owner=pablo") - self.assertEqual("someuser", u.username) - self.assertEqual("somepass", u.password) - self.assertEqual("example.com", u.hostname) - self.assertEqual("/ticket/query", u.path) - self.assertEqual("owner=pablo", u.query) - - u = Url("http://example.com/somepage.html#fragment-1") - self.assertEqual("fragment-1", u.fragment) - - u = Url("file:///home/pablo/file.txt") - self.assertEqual("/home/pablo/file.txt", u.path) - -if __name__ == "__main__": - unittest.main() - diff --git a/scrapy/tests/test_utils_httpobj.py b/scrapy/tests/test_utils_httpobj.py new file mode 100644 index 000000000..e978ae851 --- /dev/null +++ b/scrapy/tests/test_utils_httpobj.py @@ -0,0 +1,26 @@ +import unittest +import urlparse + +from scrapy.http import Request +from scrapy.utils.httpobj import urlparse_cached + +class HttpobjUtilsTest(unittest.TestCase): + + def test_urlparse_cached(self): + url = "http://www.example.com/index.html" + request1 = Request(url) + request2 = Request(url) + req1a = urlparse_cached(request1) + req1b = urlparse_cached(request1) + req2 = urlparse_cached(request2) + urlp = urlparse.urlparse(url) + + assert req1a == req2 + assert req1a == urlp + assert req1a is req1b + assert req1a is not req2 + assert req1a is not req2 + + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/tests/test_webclient.py b/scrapy/tests/test_webclient.py index c58ef150f..e7a8f7dac 100644 --- a/scrapy/tests/test_webclient.py +++ b/scrapy/tests/test_webclient.py @@ -13,14 +13,14 @@ from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory from scrapy.core.downloader import webclient as client -from scrapy.http import Url, Headers +from scrapy.http import Headers class ParseUrlTestCase(unittest.TestCase): """Test URL parsing facility and defaults values.""" def _parse(self, url): - f = client.ScrapyHTTPClientFactory(Url(url)) + f = client.ScrapyHTTPClientFactory(url) return (f.scheme, f.host, f.port, f.path) def testParse(self): diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py new file mode 100644 index 000000000..49d2f9d7b --- /dev/null +++ b/scrapy/utils/httpobj.py @@ -0,0 +1,14 @@ +"""Helper functions for scrapy.http objects (Request, Response)""" + +import weakref + +from urlparse import urlparse + +_urlparse_cache = weakref.WeakKeyDictionary() +def urlparse_cached(request_or_response): + """Return urlparse.urlparse caching the result, where the argument can be a + Request or Response object + """ + if request_or_response not in _urlparse_cache: + _urlparse_cache[request_or_response] = urlparse(request_or_response.url) + return _urlparse_cache[request_or_response] diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 7780ecc44..ce5475c6c 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -7,6 +7,7 @@ import hashlib from base64 import urlsafe_b64encode from scrapy.utils.url import canonicalize_url +from scrapy.utils.httpobj import urlparse_cached def request_fingerprint(request, include_headers=()): """ @@ -78,9 +79,9 @@ def request_httprepr(request): bytes that will be send when performing the request (that's controlled by Twisted). """ - + hostname = urlparse_cached(request).hostname s = "%s %s HTTP/1.1\r\n" % (request.method, request.url) - s += "Host: %s\r\n" % request.url.hostname + s += "Host: %s\r\n" % hostname if request.headers: s += request.headers.to_string() + "\r\n" s += "\r\n"