headers: complete new Headers behaviour migration. closes #47

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40916

scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py

@@ -21,12 +21,12 @@ class RedirectMiddleware(object):
         response = exception.response
 
         if status in [302, 303]:
-            redirected_url = urljoin(request.url, response.headers['location'][0])
+            redirected_url = urljoin(request.url, response.headers['location'])
             redirected = request.replace(url=redirected_url, method='GET', body=None)
             return self._redirect(redirected, request, spider, status)
 
         if status in [301, 307]:
-            redirected_url = urljoin(request.url, response.headers['location'][0])
+            redirected_url = urljoin(request.url, response.headers['location'])
             redirected = request.replace(url=redirected_url)
             return self._redirect(redirected, request, spider, status)
 

scrapy/trunk/scrapy/contrib/pipeline/s3images.py

@@ -97,7 +97,7 @@ class S3ImagesPipeline(BaseImagesPipeline):
                 return # returning None force download
 
             # check if last modified date did not expires
-            last_modified = response.headers['Last-Modified'][0]
+            last_modified = response.headers['Last-Modified']
             modified_tuple = rfc822.parsedate_tz(last_modified)
             modified_stamp = int(rfc822.mktime_tz(modified_tuple))
             age_seconds = time.time() - modified_stamp
@@ -106,7 +106,7 @@ class S3ImagesPipeline(BaseImagesPipeline):
             if age_days > self.image_refresh_days:
                 return # returning None force download
 
-            etag = response.headers['Etag'][0].strip('"')
+            etag = response.headers['Etag'].strip('"')
             referer = request.headers.get('Referer')
             log.msg('Image (uptodate) type=%s at <%s> referred from <%s>' % \
                     (self.MEDIA_TYPE, request.url, referer), level=log.DEBUG, domain=info.domain)

scrapy/trunk/scrapy/core/downloader/responsetypes/__init__.py

@@ -55,9 +55,9 @@ class ResponseTypes(object):
         headers"""
         cls = Response
         if 'Content-Type' in headers:
-            cls = self.from_content_type(headers['Content-type'][0])
+            cls = self.from_content_type(headers['Content-type'])
         if cls is Response and 'Content-Disposition' in headers:
-            cls = self.from_content_disposition(headers['Content-Disposition'][0])
+            cls = self.from_content_disposition(headers['Content-Disposition'])
         return cls
 
     def from_filename(self, filename):

scrapy/trunk/scrapy/http/headers.py

@@ -19,7 +19,52 @@ class Headers(CaselessDict):
         """Headers must not be unicode"""
         if isinstance(value, unicode):
             value = value.encode(self.encoding)
-        return value
+
+        if isinstance(value, (list, tuple)):
+            return list(value)
+        return [value]
+
+    def __getitem__(self, key):
+        try:
+            return super(Headers, self).__getitem__(key)[-1]
+        except IndexError:
+            return None
+
+    def get(self, key, def_val=None):
+        try:
+            return super(Headers, self).get(key, def_val)[-1]
+        except IndexError:
+            return None
+
+    def getlist(self, key, def_val=None):
+        try:
+            return super(Headers, self).__getitem__(key)
+        except KeyError:
+            if def_val is not None:
+                return self.normvalue(def_val)
+            return []
+
+    def setlist(self, key, list_):
+        self[key] = list_
+
+    def setlistdefault(self, key, default_list=()):
+        self.setdefault(key, default_list)
+
+    def appendlist(self, key, value):
+        self.getlist(key).append(self.normvalue(value))
+
+    def items(self):
+        return list(self.iteritems())
+
+    def iteritems(self):
+        return ((k, self[k]) for k in self.keys())
+
+    def values(self):
+        return [self[k] for k in self.keys()]
+
+    def lists(self):
+        return super(Headers, self).items()
 
     def to_string(self):
         return headers_dict_to_raw(self)
+

scrapy/trunk/scrapy/http/response/text.py

@@ -50,7 +50,7 @@ class TextResponse(Response):
             headers = self.headers
         content_type = headers.get('Content-Type')
         if content_type:
-            encoding = self._ENCODING_RE.search(content_type[0])
+            encoding = self._ENCODING_RE.search(content_type)
             if encoding:
                 return encoding.group(1)
 

scrapy/trunk/scrapy/tests/test_http_headers.py

@@ -0,0 +1,76 @@
+import unittest
+
+from scrapy.http import Headers
+
+class HeadersTest(unittest.TestCase):
+    def test_basics(self):
+        h = Headers({'Content-Type': 'text/html', 'Content-Length': 1234})
+        assert h['Content-Type']
+        assert h['Content-Length']
+
+        self.assertRaises(KeyError, h.__getitem__, 'Accept')
+        self.assertEqual(h.get('Accept'), None)
+        self.assertEqual(h.getlist('Accept'), [])
+
+        self.assertEqual(h.get('Accept', '*/*'), '*/*')
+        self.assertEqual(h.getlist('Accept', '*/*'), ['*/*'])
+        self.assertEqual(h.getlist('Accept', ['text/html', 'images/jpeg']), ['text/html','images/jpeg'])
+
+    def test_single_value(self):
+        h = Headers()
+        h['Content-Type'] = 'text/html'
+        self.assertEqual(h['Content-Type'], 'text/html')
+        self.assertEqual(h.get('Content-Type'), 'text/html')
+        self.assertEqual(h.getlist('Content-Type'), ['text/html'])
+
+    def test_multivalue(self):
+        h = Headers()
+
+        h['X-Forwarded-For'] = hlist = ['ip1', 'ip2']
+        self.assertEqual(h['X-Forwarded-For'], 'ip2')
+        self.assertEqual(h.get('X-Forwarded-For'), 'ip2')
+        self.assertEqual(h.getlist('X-Forwarded-For'), hlist)
+        assert h.getlist('X-Forwarded-For') is not hlist
+
+    def test_delete_and_contains(self):
+        h = Headers()
+
+        h['Content-Type'] = 'text/html'
+        assert 'Content-Type' in h
+
+        del h['Content-Type']
+        assert 'Content-Type' not in h
+
+    def test_setdefault(self):
+        h = Headers()
+        hlist = ['ip1', 'ip2']
+        olist = h.setdefault('X-Forwarded-For', hlist)
+        assert h.getlist('X-Forwarded-For') is not hlist
+        assert h.getlist('X-Forwarded-For') is olist
+
+        h = Headers()
+        olist = h.setdefault('X-Forwarded-For', 'ip1')
+        self.assertEqual(h.getlist('X-Forwarded-For'), ['ip1'])
+        assert h.getlist('X-Forwarded-For') is olist
+
+    def test_iterables(self):
+        idict = {'Content-Type': 'text/html', 'X-Forwarded-For': ['ip1', 'ip2']}
+
+        h = Headers(idict)
+        self.assertEqual(dict(h), {'Content-Type': ['text/html'], 'X-Forwarded-For': ['ip1', 'ip2']})
+        self.assertEqual(h.keys(), ['X-Forwarded-For', 'Content-Type'])
+        self.assertEqual(h.items(), [('X-Forwarded-For', 'ip2'), ('Content-Type', 'text/html')])
+        self.assertEqual(list(h.iteritems()),
+                [('X-Forwarded-For', 'ip2'), ('Content-Type', 'text/html')])
+
+        self.assertEqual(h.values(), ['ip2', 'text/html'])
+        self.assertEqual(h.lists(),
+                [('X-Forwarded-For', ['ip1', 'ip2']), ('Content-Type', ['text/html'])])
+
+    def test_update(self):
+        h = Headers()
+        h.update({'Content-Type': 'text/html', 'X-Forwarded-For': ['ip1', 'ip2']})
+        self.assertEqual(h.getlist('Content-Type'), ['text/html'])
+        self.assertEqual(h.getlist('X-Forwarded-For'), ['ip1', 'ip2'])
+
+