From 934535740bfdd1c643f875905f919ed6faecd778 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Tue, 24 Feb 2009 06:58:12 +0000 Subject: [PATCH] headers: cleanup Headers class and add normvalue method to CaselessDict --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40912 --- .../contrib/downloadermiddleware/cache.py | 4 +- scrapy/trunk/scrapy/http/headers.py | 83 ++----------------- scrapy/trunk/scrapy/tests/test_datatypes.py | 38 +++++++++ scrapy/trunk/scrapy/utils/datatypes.py | 18 ++-- scrapy/trunk/scrapy/utils/http.py | 58 +++++++++++++ 5 files changed, 119 insertions(+), 82 deletions(-) create mode 100644 scrapy/trunk/scrapy/utils/http.py diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py index c2c4a3e8e..d63c81d32 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py @@ -10,10 +10,10 @@ from pydispatch import dispatcher from scrapy.core import signals from scrapy import log from scrapy.http import Response, Headers -from scrapy.http.headers import headers_dict_to_raw from scrapy.core.exceptions import NotConfigured, HttpException, IgnoreRequest from scrapy.core.downloader.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint +from scrapy.utils.http import headers_dict_to_raw, headers_raw_to_dict from scrapy.conf import settings class CacheMiddleware(object): @@ -149,7 +149,7 @@ class Cache(object): responseheaders = f.read() url = metadata['url'] - headers = Headers(responseheaders) + headers = Headers(headers_raw_to_dict(responseheaders)) status = metadata['status'] respcls = responsetypes.from_args(headers=headers, url=url) diff --git a/scrapy/trunk/scrapy/http/headers.py b/scrapy/trunk/scrapy/http/headers.py index 25eb0cad7..d672f24f0 100644 --- a/scrapy/trunk/scrapy/http/headers.py +++ b/scrapy/trunk/scrapy/http/headers.py @@ -1,90 +1,23 @@ from scrapy.utils.datatypes import CaselessDict - -def headers_raw_to_dict(headers_raw): - """ - Convert raw headers (single multi-line string) - to the dictionary. - - For example: - >>> headers_raw_to_dict("Content-type: text/html\\n\\rAccept: gzip\\n\\n") - {'Content-type': ['text/html'], 'Accept': ['gzip']} - - Incorrect input: - >>> headers_raw_to_dict("Content-typt gzip\\n\\n") - {} - - Argument is None: - >>> headers_raw_to_dict(None) - """ - if headers_raw is None: - return None - return dict([ - (header_item[0].strip(), [header_item[1].strip()]) - for header_item - in [ - header.split(':', 1) - for header - in headers_raw.splitlines()] - if len(header_item) == 2]) - -def headers_dict_to_raw(headers_dict): - """ - Returns a raw HTTP headers representation of headers - - For example: - >>> headers_dict_to_raw({'Content-type': 'text/html', 'Accept': 'gzip'}) - 'Content-type: text/html\\r\\nAccept: gzip' - >>> from twisted.python.util import InsensitiveDict - >>> td = InsensitiveDict({'Content-type': ['text/html'], 'Accept': ['gzip']}) - >>> headers_dict_to_raw(td) - 'Content-type: text/html\\r\\nAccept: gzip' - - Argument is None: - >>> headers_dict_to_raw(None) - - """ - if headers_dict is None: - return None - raw_lines = [] - for key, value in headers_dict.items(): - if isinstance(value, (str, unicode)): - raw_lines.append("%s: %s" % (key, value)) - elif isinstance(value, (list, tuple)): - for v in value: - raw_lines.append("%s: %s" % (key, v)) - return '\r\n'.join(raw_lines) +from scrapy.utils.http import headers_dict_to_raw class Headers(CaselessDict): - def __init__(self, dictorstr=None, fromdict=None, fromstr=None, encoding='utf-8'): + def __init__(self, seq=None, encoding='utf-8'): self.encoding = encoding - - if dictorstr is not None: - if isinstance(dictorstr, dict): - d = dictorstr - elif isinstance(dictorstr, basestring): - d = headers_raw_to_dict(dictorstr) - elif fromdict is not None: - d = fromdict - elif fromstr is not None: - d = headers_raw_to_dict(fromstr) - else: - d = {} - - # can't use CaselessDict.__init__(self, d) because it doesn't call __setitem__ - for k,v in d.iteritems(): - self.__setitem__(k.lower(), v) + super(Headers, self).__init__(seq) def normkey(self, key): - return key.title() - - def __setitem__(self, key, value): """Headers must not be unicode""" if isinstance(key, unicode): key = key.encode(self.encoding) + return key.title() + + def normvalue(self, value): + """Headers must not be unicode""" if isinstance(value, unicode): value = value.encode(self.encoding) - super(Headers, self).__setitem__(key, value) + return value def to_string(self): return headers_dict_to_raw(self) diff --git a/scrapy/trunk/scrapy/tests/test_datatypes.py b/scrapy/trunk/scrapy/tests/test_datatypes.py index a99c564a6..7eaf171c5 100644 --- a/scrapy/trunk/scrapy/tests/test_datatypes.py +++ b/scrapy/trunk/scrapy/tests/test_datatypes.py @@ -81,3 +81,41 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d.pop('A'), 1) self.assertRaises(KeyError, d.pop, 'A') + def test_normkey(self): + class MyDict(CaselessDict): + def normkey(self, key): + return key.title() + + d = MyDict() + d['key-one'] = 2 + self.assertEqual(list(d.keys()), ['Key-One']) + + def test_normvalue(self): + class MyDict(CaselessDict): + def normvalue(self, value): + if value is not None: + return value + 1 + + d = MyDict({'key': 1}) + self.assertEqual(d['key'], 2) + self.assertEqual(d.get('key'), 2) + + d = MyDict() + d['key'] = 1 + self.assertEqual(d['key'], 2) + self.assertEqual(d.get('key'), 2) + + d = MyDict() + d.setdefault('key', 1) + self.assertEqual(d['key'], 2) + self.assertEqual(d.get('key'), 2) + + d = MyDict() + d.update({'key': 1}) + self.assertEqual(d['key'], 2) + self.assertEqual(d.get('key'), 2) + + d = MyDict.fromkeys(('key',), 1) + self.assertEqual(d['key'], 2) + self.assertEqual(d.get('key'), 2) + diff --git a/scrapy/trunk/scrapy/utils/datatypes.py b/scrapy/trunk/scrapy/utils/datatypes.py index 3eae2d3dd..9e94a01d1 100644 --- a/scrapy/trunk/scrapy/utils/datatypes.py +++ b/scrapy/trunk/scrapy/utils/datatypes.py @@ -183,6 +183,7 @@ class SiteNode(object): s += node.to_string(level+1) return s + class CaselessDict(dict): def __init__(self, seq=None): dict.__init__(self) @@ -193,7 +194,7 @@ class CaselessDict(dict): return dict.__getitem__(self, self.normkey(key)) def __setitem__(self, key, value): - dict.__setitem__(self, self.normkey(key), value) + dict.__setitem__(self, self.normkey(key), self.normvalue(value)) def __delitem__(self, key): dict.__delitem__(self, self.normkey(key)) @@ -203,17 +204,23 @@ class CaselessDict(dict): has_key = __contains__ def normkey(self, key): + """Method to normalize dictionary key access""" return key.lower() + def normvalue(self, value): + """Method to normalize values prior to be setted""" + return value + def get(self, key, def_val=None): - return dict.get(self, self.normkey(key), def_val) + return dict.get(self, self.normkey(key), self.normvalue(def_val)) def setdefault(self, key, def_val=None): - return dict.setdefault(self, self.normkey(key), def_val) + return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) def update(self, seq): - items = seq.iteritems() if isinstance(seq, dict) else seq - dict.update(self, ((self.normkey(k), v) for k, v in items)) + seq = seq.iteritems() if isinstance(seq, dict) else seq + iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) + super(CaselessDict, self).update(iseq) @classmethod def fromkeys(cls, keys, value=None): @@ -222,6 +229,7 @@ class CaselessDict(dict): def pop(self, key, *args): return dict.pop(self, self.normkey(key), *args) + class PriorityQueue(object): """A simple priority queue""" diff --git a/scrapy/trunk/scrapy/utils/http.py b/scrapy/trunk/scrapy/utils/http.py new file mode 100644 index 000000000..173b6e6a4 --- /dev/null +++ b/scrapy/trunk/scrapy/utils/http.py @@ -0,0 +1,58 @@ + +def headers_raw_to_dict(headers_raw): + """ + Convert raw headers (single multi-line string) + to the dictionary. + + For example: + >>> headers_raw_to_dict("Content-type: text/html\\n\\rAccept: gzip\\n\\n") + {'Content-type': ['text/html'], 'Accept': ['gzip']} + + Incorrect input: + >>> headers_raw_to_dict("Content-typt gzip\\n\\n") + {} + + Argument is None: + >>> headers_raw_to_dict(None) + """ + if headers_raw is None: + return None + return dict([ + (header_item[0].strip(), [header_item[1].strip()]) + for header_item + in [ + header.split(':', 1) + for header + in headers_raw.splitlines()] + if len(header_item) == 2]) + + +def headers_dict_to_raw(headers_dict): + """ + Returns a raw HTTP headers representation of headers + + For example: + >>> headers_dict_to_raw({'Content-type': 'text/html', 'Accept': 'gzip'}) + 'Content-type: text/html\\r\\nAccept: gzip' + >>> from twisted.python.util import InsensitiveDict + >>> td = InsensitiveDict({'Content-type': ['text/html'], 'Accept': ['gzip']}) + >>> headers_dict_to_raw(td) + 'Content-type: text/html\\r\\nAccept: gzip' + + Argument is None: + >>> headers_dict_to_raw(None) + + """ + if headers_dict is None: + return None + raw_lines = [] + for key, value in headers_dict.items(): + if isinstance(value, (str, unicode)): + raw_lines.append("%s: %s" % (key, value)) + elif isinstance(value, (list, tuple)): + for v in value: + raw_lines.append("%s: %s" % (key, v)) + return '\r\n'.join(raw_lines) + + +