From 82dc57e59b10b25f259f7dec82c3124bdb53686a Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Wed, 25 Feb 2009 00:29:10 +0000 Subject: [PATCH] headers: complete new Headers behaviour migration. closes #47 --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40916 --- .../contrib/downloadermiddleware/redirect.py | 4 +- .../trunk/scrapy/contrib/pipeline/s3images.py | 4 +- .../core/downloader/responsetypes/__init__.py | 4 +- scrapy/trunk/scrapy/http/headers.py | 47 +++++++++++- scrapy/trunk/scrapy/http/response/text.py | 2 +- .../trunk/scrapy/tests/test_http_headers.py | 76 +++++++++++++++++++ 6 files changed, 129 insertions(+), 8 deletions(-) create mode 100644 scrapy/trunk/scrapy/tests/test_http_headers.py diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py index e828d2e2e..8fd45c75b 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py @@ -21,12 +21,12 @@ class RedirectMiddleware(object): response = exception.response if status in [302, 303]: - redirected_url = urljoin(request.url, response.headers['location'][0]) + redirected_url = urljoin(request.url, response.headers['location']) redirected = request.replace(url=redirected_url, method='GET', body=None) return self._redirect(redirected, request, spider, status) if status in [301, 307]: - redirected_url = urljoin(request.url, response.headers['location'][0]) + redirected_url = urljoin(request.url, response.headers['location']) redirected = request.replace(url=redirected_url) return self._redirect(redirected, request, spider, status) diff --git a/scrapy/trunk/scrapy/contrib/pipeline/s3images.py b/scrapy/trunk/scrapy/contrib/pipeline/s3images.py index 152a5e450..0d0843684 100644 --- a/scrapy/trunk/scrapy/contrib/pipeline/s3images.py +++ b/scrapy/trunk/scrapy/contrib/pipeline/s3images.py @@ -97,7 +97,7 @@ class S3ImagesPipeline(BaseImagesPipeline): return # returning None force download # check if last modified date did not expires - last_modified = response.headers['Last-Modified'][0] + last_modified = response.headers['Last-Modified'] modified_tuple = rfc822.parsedate_tz(last_modified) modified_stamp = int(rfc822.mktime_tz(modified_tuple)) age_seconds = time.time() - modified_stamp @@ -106,7 +106,7 @@ class S3ImagesPipeline(BaseImagesPipeline): if age_days > self.image_refresh_days: return # returning None force download - etag = response.headers['Etag'][0].strip('"') + etag = response.headers['Etag'].strip('"') referer = request.headers.get('Referer') log.msg('Image (uptodate) type=%s at <%s> referred from <%s>' % \ (self.MEDIA_TYPE, request.url, referer), level=log.DEBUG, domain=info.domain) diff --git a/scrapy/trunk/scrapy/core/downloader/responsetypes/__init__.py b/scrapy/trunk/scrapy/core/downloader/responsetypes/__init__.py index c9b5c2cea..fc8e98d33 100644 --- a/scrapy/trunk/scrapy/core/downloader/responsetypes/__init__.py +++ b/scrapy/trunk/scrapy/core/downloader/responsetypes/__init__.py @@ -55,9 +55,9 @@ class ResponseTypes(object): headers""" cls = Response if 'Content-Type' in headers: - cls = self.from_content_type(headers['Content-type'][0]) + cls = self.from_content_type(headers['Content-type']) if cls is Response and 'Content-Disposition' in headers: - cls = self.from_content_disposition(headers['Content-Disposition'][0]) + cls = self.from_content_disposition(headers['Content-Disposition']) return cls def from_filename(self, filename): diff --git a/scrapy/trunk/scrapy/http/headers.py b/scrapy/trunk/scrapy/http/headers.py index f7f395132..8fa59b678 100644 --- a/scrapy/trunk/scrapy/http/headers.py +++ b/scrapy/trunk/scrapy/http/headers.py @@ -19,7 +19,52 @@ class Headers(CaselessDict): """Headers must not be unicode""" if isinstance(value, unicode): value = value.encode(self.encoding) - return value + + if isinstance(value, (list, tuple)): + return list(value) + return [value] + + def __getitem__(self, key): + try: + return super(Headers, self).__getitem__(key)[-1] + except IndexError: + return None + + def get(self, key, def_val=None): + try: + return super(Headers, self).get(key, def_val)[-1] + except IndexError: + return None + + def getlist(self, key, def_val=None): + try: + return super(Headers, self).__getitem__(key) + except KeyError: + if def_val is not None: + return self.normvalue(def_val) + return [] + + def setlist(self, key, list_): + self[key] = list_ + + def setlistdefault(self, key, default_list=()): + self.setdefault(key, default_list) + + def appendlist(self, key, value): + self.getlist(key).append(self.normvalue(value)) + + def items(self): + return list(self.iteritems()) + + def iteritems(self): + return ((k, self[k]) for k in self.keys()) + + def values(self): + return [self[k] for k in self.keys()] + + def lists(self): + return super(Headers, self).items() def to_string(self): return headers_dict_to_raw(self) + diff --git a/scrapy/trunk/scrapy/http/response/text.py b/scrapy/trunk/scrapy/http/response/text.py index 847c9f888..67227ba86 100644 --- a/scrapy/trunk/scrapy/http/response/text.py +++ b/scrapy/trunk/scrapy/http/response/text.py @@ -50,7 +50,7 @@ class TextResponse(Response): headers = self.headers content_type = headers.get('Content-Type') if content_type: - encoding = self._ENCODING_RE.search(content_type[0]) + encoding = self._ENCODING_RE.search(content_type) if encoding: return encoding.group(1) diff --git a/scrapy/trunk/scrapy/tests/test_http_headers.py b/scrapy/trunk/scrapy/tests/test_http_headers.py new file mode 100644 index 000000000..ddf65e005 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_http_headers.py @@ -0,0 +1,76 @@ +import unittest + +from scrapy.http import Headers + +class HeadersTest(unittest.TestCase): + def test_basics(self): + h = Headers({'Content-Type': 'text/html', 'Content-Length': 1234}) + assert h['Content-Type'] + assert h['Content-Length'] + + self.assertRaises(KeyError, h.__getitem__, 'Accept') + self.assertEqual(h.get('Accept'), None) + self.assertEqual(h.getlist('Accept'), []) + + self.assertEqual(h.get('Accept', '*/*'), '*/*') + self.assertEqual(h.getlist('Accept', '*/*'), ['*/*']) + self.assertEqual(h.getlist('Accept', ['text/html', 'images/jpeg']), ['text/html','images/jpeg']) + + def test_single_value(self): + h = Headers() + h['Content-Type'] = 'text/html' + self.assertEqual(h['Content-Type'], 'text/html') + self.assertEqual(h.get('Content-Type'), 'text/html') + self.assertEqual(h.getlist('Content-Type'), ['text/html']) + + def test_multivalue(self): + h = Headers() + + h['X-Forwarded-For'] = hlist = ['ip1', 'ip2'] + self.assertEqual(h['X-Forwarded-For'], 'ip2') + self.assertEqual(h.get('X-Forwarded-For'), 'ip2') + self.assertEqual(h.getlist('X-Forwarded-For'), hlist) + assert h.getlist('X-Forwarded-For') is not hlist + + def test_delete_and_contains(self): + h = Headers() + + h['Content-Type'] = 'text/html' + assert 'Content-Type' in h + + del h['Content-Type'] + assert 'Content-Type' not in h + + def test_setdefault(self): + h = Headers() + hlist = ['ip1', 'ip2'] + olist = h.setdefault('X-Forwarded-For', hlist) + assert h.getlist('X-Forwarded-For') is not hlist + assert h.getlist('X-Forwarded-For') is olist + + h = Headers() + olist = h.setdefault('X-Forwarded-For', 'ip1') + self.assertEqual(h.getlist('X-Forwarded-For'), ['ip1']) + assert h.getlist('X-Forwarded-For') is olist + + def test_iterables(self): + idict = {'Content-Type': 'text/html', 'X-Forwarded-For': ['ip1', 'ip2']} + + h = Headers(idict) + self.assertEqual(dict(h), {'Content-Type': ['text/html'], 'X-Forwarded-For': ['ip1', 'ip2']}) + self.assertEqual(h.keys(), ['X-Forwarded-For', 'Content-Type']) + self.assertEqual(h.items(), [('X-Forwarded-For', 'ip2'), ('Content-Type', 'text/html')]) + self.assertEqual(list(h.iteritems()), + [('X-Forwarded-For', 'ip2'), ('Content-Type', 'text/html')]) + + self.assertEqual(h.values(), ['ip2', 'text/html']) + self.assertEqual(h.lists(), + [('X-Forwarded-For', ['ip1', 'ip2']), ('Content-Type', ['text/html'])]) + + def test_update(self): + h = Headers() + h.update({'Content-Type': 'text/html', 'X-Forwarded-For': ['ip1', 'ip2']}) + self.assertEqual(h.getlist('Content-Type'), ['text/html']) + self.assertEqual(h.getlist('X-Forwarded-For'), ['ip1', 'ip2']) + +