From 7227b65de1ee8bfcab22bc49acf06150a1d3ff42 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Fri, 27 Mar 2009 06:42:24 +0000 Subject: [PATCH] http: fix copy and failing appendlist method of Headers, also add missing tests --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401022 --- scrapy/trunk/scrapy/http/headers.py | 7 +- scrapy/trunk/scrapy/tests/test_datatypes.py | 121 -------------- .../trunk/scrapy/tests/test_http_headers.py | 31 +++- .../scrapy/tests/test_utils_datatypes.py | 151 ++++++++++++++++-- scrapy/trunk/scrapy/utils/datatypes.py | 6 +- 5 files changed, 175 insertions(+), 141 deletions(-) delete mode 100644 scrapy/trunk/scrapy/tests/test_datatypes.py diff --git a/scrapy/trunk/scrapy/http/headers.py b/scrapy/trunk/scrapy/http/headers.py index 8fa59b678..a3344e29f 100644 --- a/scrapy/trunk/scrapy/http/headers.py +++ b/scrapy/trunk/scrapy/http/headers.py @@ -51,7 +51,7 @@ class Headers(CaselessDict): self.setdefault(key, default_list) def appendlist(self, key, value): - self.getlist(key).append(self.normvalue(value)) + self.getlist(key).extend(self.normvalue(value)) def items(self): return list(self.iteritems()) @@ -68,3 +68,8 @@ class Headers(CaselessDict): def to_string(self): return headers_dict_to_raw(self) + def __copy__(self): + return self.__class__(self.lists()) + copy = __copy__ + + diff --git a/scrapy/trunk/scrapy/tests/test_datatypes.py b/scrapy/trunk/scrapy/tests/test_datatypes.py deleted file mode 100644 index 7eaf171c5..000000000 --- a/scrapy/trunk/scrapy/tests/test_datatypes.py +++ /dev/null @@ -1,121 +0,0 @@ -import unittest - -from scrapy.utils.datatypes import CaselessDict - - -class CaselessDictTest(unittest.TestCase): - - def test_init(self): - seq = {'red': 1, 'black': 3} - d = CaselessDict(seq) - self.assertEqual(d['red'], 1) - self.assertEqual(d['black'], 3) - - seq = (('red', 1), ('black', 3)) - d = CaselessDict(seq) - self.assertEqual(d['red'], 1) - self.assertEqual(d['black'], 3) - - def test_caseless(self): - d = CaselessDict() - d['key_Lower'] = 1 - self.assertEqual(d['KEy_loWer'], 1) - self.assertEqual(d.get('KEy_loWer'), 1) - - d['KEY_LOWER'] = 3 - self.assertEqual(d['key_Lower'], 3) - self.assertEqual(d.get('key_Lower'), 3) - - def test_delete(self): - d = CaselessDict({'key_lower': 1}) - del d['key_LOWER'] - self.assertRaises(KeyError, d.__getitem__, 'key_LOWER') - self.assertRaises(KeyError, d.__getitem__, 'key_lower') - - def test_getdefault(self): - d = CaselessDict() - self.assertEqual(d.get('c', 5), 5) - d['c'] = 10 - self.assertEqual(d.get('c', 5), 10) - - def test_setdefault(self): - d = CaselessDict({'a': 1, 'b': 2}) - - r = d.setdefault('A', 5) - self.assertEqual(r, 1) - self.assertEqual(d['A'], 1) - - r = d.setdefault('c', 5) - self.assertEqual(r, 5) - self.assertEqual(d['C'], 5) - - def test_fromkeys(self): - keys = ('a', 'b') - - d = CaselessDict.fromkeys(keys) - self.assertEqual(d['A'], None) - self.assertEqual(d['B'], None) - - d = CaselessDict.fromkeys(keys, 1) - self.assertEqual(d['A'], 1) - self.assertEqual(d['B'], 1) - - instance = CaselessDict() - d = instance.fromkeys(keys) - self.assertEqual(d['A'], None) - self.assertEqual(d['B'], None) - - d = instance.fromkeys(keys, 1) - self.assertEqual(d['A'], 1) - self.assertEqual(d['B'], 1) - - def test_contains(self): - d = CaselessDict() - d['a'] = 1 - assert 'a' in d - assert d.has_key('a') - - def test_pop(self): - d = CaselessDict() - d['a'] = 1 - self.assertEqual(d.pop('A'), 1) - self.assertRaises(KeyError, d.pop, 'A') - - def test_normkey(self): - class MyDict(CaselessDict): - def normkey(self, key): - return key.title() - - d = MyDict() - d['key-one'] = 2 - self.assertEqual(list(d.keys()), ['Key-One']) - - def test_normvalue(self): - class MyDict(CaselessDict): - def normvalue(self, value): - if value is not None: - return value + 1 - - d = MyDict({'key': 1}) - self.assertEqual(d['key'], 2) - self.assertEqual(d.get('key'), 2) - - d = MyDict() - d['key'] = 1 - self.assertEqual(d['key'], 2) - self.assertEqual(d.get('key'), 2) - - d = MyDict() - d.setdefault('key', 1) - self.assertEqual(d['key'], 2) - self.assertEqual(d.get('key'), 2) - - d = MyDict() - d.update({'key': 1}) - self.assertEqual(d['key'], 2) - self.assertEqual(d.get('key'), 2) - - d = MyDict.fromkeys(('key',), 1) - self.assertEqual(d['key'], 2) - self.assertEqual(d.get('key'), 2) - diff --git a/scrapy/trunk/scrapy/tests/test_http_headers.py b/scrapy/trunk/scrapy/tests/test_http_headers.py index fd444d953..3dc9f6670 100644 --- a/scrapy/trunk/scrapy/tests/test_http_headers.py +++ b/scrapy/trunk/scrapy/tests/test_http_headers.py @@ -74,10 +74,29 @@ class HeadersTest(unittest.TestCase): self.assertEqual(h.getlist('Content-Type'), ['text/html']) self.assertEqual(h.getlist('X-Forwarded-For'), ['ip1', 'ip2']) - #def test_copy(self): - # h1 = Headers({'header1': ['value1', 'value2']}) - # h2 = copy.copy(h1) - # self.assertEqual(h1, h2) - # self.assertEqual(h1.getlist('header1'), h2.getlist('header1')) - # assert h1.getlist('header1') is not h2.getlist('header1') + def test_copy(self): + h1 = Headers({'header1': ['value1', 'value2']}) + h2 = copy.copy(h1) + self.assertEqual(h1, h2) + self.assertEqual(h1.getlist('header1'), h2.getlist('header1')) + assert h1.getlist('header1') is not h2.getlist('header1') + assert isinstance(h2, Headers) + + def test_appendlist(self): + h1 = Headers({'header1': 'value1'}) + h1.appendlist('header1', 'value3') + self.assertEqual(h1.getlist('header1'), ['value1', 'value3']) + + def test_setlist(self): + h1 = Headers({'header1': 'value1'}) + self.assertEqual(h1.getlist('header1'), ['value1']) + h1.setlist('header1', ['value2', 'value3']) + self.assertEqual(h1.getlist('header1'), ['value2', 'value3']) + + def test_setlistdefault(self): + h1 = Headers({'header1': 'value1'}) + h1.setlistdefault('header1', ['value2', 'value3']) + h1.setlistdefault('header2', ['value2', 'value3']) + self.assertEqual(h1.getlist('header1'), ['value1']) + self.assertEqual(h1.getlist('header2'), ['value2', 'value3']) diff --git a/scrapy/trunk/scrapy/tests/test_utils_datatypes.py b/scrapy/trunk/scrapy/tests/test_utils_datatypes.py index 8cce73f53..516bbc6a4 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_datatypes.py +++ b/scrapy/trunk/scrapy/tests/test_utils_datatypes.py @@ -1,6 +1,7 @@ +import copy import unittest -from scrapy.utils.datatypes import PriorityQueue, PriorityStack +from scrapy.utils.datatypes import PriorityQueue, PriorityStack, CaselessDict # (ITEM, PRIORITY) INPUT = [(1, -5), (30, -1), (80, -3), (4, 1), (6, 3), (20, 0), (50, -1)] @@ -23,8 +24,8 @@ class PriorityQueueTestCase(unittest.TestCase): for item, pr in INPUT: pq.push(item, pr) result = [x for x in pq] - self.assertEquals(result, self.output) - + self.assertEquals(result, self.output) + def test_nonzero(self): pq = PriorityQueue() pq.push(80, -1) @@ -32,11 +33,11 @@ class PriorityQueueTestCase(unittest.TestCase): pq.push(30, 1) pq.pop() - self.assertEquals(bool(pq), True) + self.assertEquals(bool(pq), True) pq.pop() - self.assertEquals(bool(pq), True) + self.assertEquals(bool(pq), True) pq.pop() - self.assertEquals(bool(pq), False) + self.assertEquals(bool(pq), False) def test_len(self): pq = PriorityQueue() @@ -44,13 +45,13 @@ class PriorityQueueTestCase(unittest.TestCase): pq.push(20, 0) pq.push(30, 1) - self.assertEquals(len(pq), 3) + self.assertEquals(len(pq), 3) pq.pop() - self.assertEquals(len(pq), 2) + self.assertEquals(len(pq), 2) pq.pop() - self.assertEquals(len(pq), 1) + self.assertEquals(len(pq), 1) pq.pop() - self.assertEquals(len(pq), 0) + self.assertEquals(len(pq), 0) class PriorityStackTestCase(unittest.TestCase): @@ -70,7 +71,133 @@ class PriorityStackTestCase(unittest.TestCase): for item, pr in INPUT: pq.push(item, pr) result = [x for x in pq] - self.assertEquals(result, self.output) - + self.assertEquals(result, self.output) + + +class CaselessDictTest(unittest.TestCase): + + def test_init(self): + seq = {'red': 1, 'black': 3} + d = CaselessDict(seq) + self.assertEqual(d['red'], 1) + self.assertEqual(d['black'], 3) + + seq = (('red', 1), ('black', 3)) + d = CaselessDict(seq) + self.assertEqual(d['red'], 1) + self.assertEqual(d['black'], 3) + + def test_caseless(self): + d = CaselessDict() + d['key_Lower'] = 1 + self.assertEqual(d['KEy_loWer'], 1) + self.assertEqual(d.get('KEy_loWer'), 1) + + d['KEY_LOWER'] = 3 + self.assertEqual(d['key_Lower'], 3) + self.assertEqual(d.get('key_Lower'), 3) + + def test_delete(self): + d = CaselessDict({'key_lower': 1}) + del d['key_LOWER'] + self.assertRaises(KeyError, d.__getitem__, 'key_LOWER') + self.assertRaises(KeyError, d.__getitem__, 'key_lower') + + def test_getdefault(self): + d = CaselessDict() + self.assertEqual(d.get('c', 5), 5) + d['c'] = 10 + self.assertEqual(d.get('c', 5), 10) + + def test_setdefault(self): + d = CaselessDict({'a': 1, 'b': 2}) + + r = d.setdefault('A', 5) + self.assertEqual(r, 1) + self.assertEqual(d['A'], 1) + + r = d.setdefault('c', 5) + self.assertEqual(r, 5) + self.assertEqual(d['C'], 5) + + def test_fromkeys(self): + keys = ('a', 'b') + + d = CaselessDict.fromkeys(keys) + self.assertEqual(d['A'], None) + self.assertEqual(d['B'], None) + + d = CaselessDict.fromkeys(keys, 1) + self.assertEqual(d['A'], 1) + self.assertEqual(d['B'], 1) + + instance = CaselessDict() + d = instance.fromkeys(keys) + self.assertEqual(d['A'], None) + self.assertEqual(d['B'], None) + + d = instance.fromkeys(keys, 1) + self.assertEqual(d['A'], 1) + self.assertEqual(d['B'], 1) + + def test_contains(self): + d = CaselessDict() + d['a'] = 1 + assert 'a' in d + assert d.has_key('a') + + def test_pop(self): + d = CaselessDict() + d['a'] = 1 + self.assertEqual(d.pop('A'), 1) + self.assertRaises(KeyError, d.pop, 'A') + + def test_normkey(self): + class MyDict(CaselessDict): + def normkey(self, key): + return key.title() + + d = MyDict() + d['key-one'] = 2 + self.assertEqual(list(d.keys()), ['Key-One']) + + def test_normvalue(self): + class MyDict(CaselessDict): + def normvalue(self, value): + if value is not None: + return value + 1 + + d = MyDict({'key': 1}) + self.assertEqual(d['key'], 2) + self.assertEqual(d.get('key'), 2) + + d = MyDict() + d['key'] = 1 + self.assertEqual(d['key'], 2) + self.assertEqual(d.get('key'), 2) + + d = MyDict() + d.setdefault('key', 1) + self.assertEqual(d['key'], 2) + self.assertEqual(d.get('key'), 2) + + d = MyDict() + d.update({'key': 1}) + self.assertEqual(d['key'], 2) + self.assertEqual(d.get('key'), 2) + + d = MyDict.fromkeys(('key',), 1) + self.assertEqual(d['key'], 2) + self.assertEqual(d.get('key'), 2) + + def test_copy(self): + h1 = CaselessDict({'header1': 'value'}) + h2 = copy.copy(h1) + self.assertEqual(h1, h2) + self.assertEqual(h1.get('header1'), h2.get('header1')) + assert isinstance(h2, CaselessDict) + + if __name__ == "__main__": unittest.main() + diff --git a/scrapy/trunk/scrapy/utils/datatypes.py b/scrapy/trunk/scrapy/utils/datatypes.py index 2f42ba739..e51084034 100644 --- a/scrapy/trunk/scrapy/utils/datatypes.py +++ b/scrapy/trunk/scrapy/utils/datatypes.py @@ -186,7 +186,7 @@ class SiteNode(object): class CaselessDict(dict): def __init__(self, seq=None): - dict.__init__(self) + super(CaselessDict, self).__init__() if seq: self.update(seq) @@ -203,6 +203,10 @@ class CaselessDict(dict): return dict.__contains__(self, self.normkey(key)) has_key = __contains__ + def __copy__(self): + return self.__class__(self) + copy = __copy__ + def normkey(self, key): """Method to normalize dictionary key access""" return key.lower()