RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

http: fix copy and failing appendlist method of Headers, also add missing tests 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401022
This commit is contained in:
Daniel Grana 2009-03-27 06:42:24 +00:00
parent 1d0e2d1202
commit 7227b65de1
5 changed files with 175 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
scrapy/trunk/scrapy/http/headers.py
View File

@ -51,7 +51,7 @@ class Headers(CaselessDict):
self.setdefault(key, default_list)
def appendlist(self, key, value):
self.getlist(key).append(self.normvalue(value))
self.getlist(key).extend(self.normvalue(value))
def items(self):
return list(self.iteritems())
@ -68,3 +68,8 @@ class Headers(CaselessDict):
def to_string(self):
return headers_dict_to_raw(self)
def __copy__(self):
return self.__class__(self.lists())
copy = __copy__

121
scrapy/trunk/scrapy/tests/test_datatypes.py
View File

@ -1,121 +0,0 @@
import unittest
from scrapy.utils.datatypes import CaselessDict
class CaselessDictTest(unittest.TestCase):
def test_init(self):
seq = {'red': 1, 'black': 3}
d = CaselessDict(seq)
self.assertEqual(d['red'], 1)
self.assertEqual(d['black'], 3)
seq = (('red', 1), ('black', 3))
d = CaselessDict(seq)
self.assertEqual(d['red'], 1)
self.assertEqual(d['black'], 3)
def test_caseless(self):
d = CaselessDict()
d['key_Lower'] = 1
self.assertEqual(d['KEy_loWer'], 1)
self.assertEqual(d.get('KEy_loWer'), 1)
d['KEY_LOWER'] = 3
self.assertEqual(d['key_Lower'], 3)
self.assertEqual(d.get('key_Lower'), 3)
def test_delete(self):
d = CaselessDict({'key_lower': 1})
del d['key_LOWER']
self.assertRaises(KeyError, d.__getitem__, 'key_LOWER')
self.assertRaises(KeyError, d.__getitem__, 'key_lower')
def test_getdefault(self):
d = CaselessDict()
self.assertEqual(d.get('c', 5), 5)
d['c'] = 10
self.assertEqual(d.get('c', 5), 10)
def test_setdefault(self):
d = CaselessDict({'a': 1, 'b': 2})
r = d.setdefault('A', 5)
self.assertEqual(r, 1)
self.assertEqual(d['A'], 1)
r = d.setdefault('c', 5)
self.assertEqual(r, 5)
self.assertEqual(d['C'], 5)
def test_fromkeys(self):
keys = ('a', 'b')
d = CaselessDict.fromkeys(keys)
self.assertEqual(d['A'], None)
self.assertEqual(d['B'], None)
d = CaselessDict.fromkeys(keys, 1)
self.assertEqual(d['A'], 1)
self.assertEqual(d['B'], 1)
instance = CaselessDict()
d = instance.fromkeys(keys)
self.assertEqual(d['A'], None)
self.assertEqual(d['B'], None)
d = instance.fromkeys(keys, 1)
self.assertEqual(d['A'], 1)
self.assertEqual(d['B'], 1)
def test_contains(self):
d = CaselessDict()
d['a'] = 1
assert 'a' in d
assert d.has_key('a')
def test_pop(self):
d = CaselessDict()
d['a'] = 1
self.assertEqual(d.pop('A'), 1)
self.assertRaises(KeyError, d.pop, 'A')
def test_normkey(self):
class MyDict(CaselessDict):
def normkey(self, key):
return key.title()
d = MyDict()
d['key-one'] = 2
self.assertEqual(list(d.keys()), ['Key-One'])
def test_normvalue(self):
class MyDict(CaselessDict):
def normvalue(self, value):
if value is not None:
return value + 1
d = MyDict({'key': 1})
self.assertEqual(d['key'], 2)
self.assertEqual(d.get('key'), 2)
d = MyDict()
d['key'] = 1
self.assertEqual(d['key'], 2)
self.assertEqual(d.get('key'), 2)
d = MyDict()
d.setdefault('key', 1)
self.assertEqual(d['key'], 2)
self.assertEqual(d.get('key'), 2)
d = MyDict()
d.update({'key': 1})
self.assertEqual(d['key'], 2)
self.assertEqual(d.get('key'), 2)
d = MyDict.fromkeys(('key',), 1)
self.assertEqual(d['key'], 2)
self.assertEqual(d.get('key'), 2)

31
scrapy/trunk/scrapy/tests/test_http_headers.py
View File

@ -74,10 +74,29 @@ class HeadersTest(unittest.TestCase):
self.assertEqual(h.getlist('Content-Type'), ['text/html'])
self.assertEqual(h.getlist('X-Forwarded-For'), ['ip1', 'ip2'])
#def test_copy(self):
# h1 = Headers({'header1': ['value1', 'value2']})
# h2 = copy.copy(h1)
# self.assertEqual(h1, h2)
# self.assertEqual(h1.getlist('header1'), h2.getlist('header1'))
# assert h1.getlist('header1') is not h2.getlist('header1')
def test_copy(self):
h1 = Headers({'header1': ['value1', 'value2']})
h2 = copy.copy(h1)
self.assertEqual(h1, h2)
self.assertEqual(h1.getlist('header1'), h2.getlist('header1'))
assert h1.getlist('header1') is not h2.getlist('header1')
assert isinstance(h2, Headers)
def test_appendlist(self):
h1 = Headers({'header1': 'value1'})
h1.appendlist('header1', 'value3')
self.assertEqual(h1.getlist('header1'), ['value1', 'value3'])
def test_setlist(self):
h1 = Headers({'header1': 'value1'})
self.assertEqual(h1.getlist('header1'), ['value1'])
h1.setlist('header1', ['value2', 'value3'])
self.assertEqual(h1.getlist('header1'), ['value2', 'value3'])
def test_setlistdefault(self):
h1 = Headers({'header1': 'value1'})
h1.setlistdefault('header1', ['value2', 'value3'])
h1.setlistdefault('header2', ['value2', 'value3'])
self.assertEqual(h1.getlist('header1'), ['value1'])
self.assertEqual(h1.getlist('header2'), ['value2', 'value3'])

151
scrapy/trunk/scrapy/tests/test_utils_datatypes.py
View File

@ -1,6 +1,7 @@
import copy
import unittest
from scrapy.utils.datatypes import PriorityQueue, PriorityStack
from scrapy.utils.datatypes import PriorityQueue, PriorityStack, CaselessDict
# (ITEM, PRIORITY)
INPUT = [(1, -5), (30, -1), (80, -3), (4, 1), (6, 3), (20, 0), (50, -1)]
@ -23,8 +24,8 @@ class PriorityQueueTestCase(unittest.TestCase):
for item, pr in INPUT:
pq.push(item, pr)
result = [x for x in pq]
self.assertEquals(result, self.output)
self.assertEquals(result, self.output)
def test_nonzero(self):
pq = PriorityQueue()
pq.push(80, -1)
@ -32,11 +33,11 @@ class PriorityQueueTestCase(unittest.TestCase):
pq.push(30, 1)
pq.pop()
self.assertEquals(bool(pq), True)
self.assertEquals(bool(pq), True)
pq.pop()
self.assertEquals(bool(pq), True)
self.assertEquals(bool(pq), True)
pq.pop()
self.assertEquals(bool(pq), False)
self.assertEquals(bool(pq), False)
def test_len(self):
pq = PriorityQueue()
@ -44,13 +45,13 @@ class PriorityQueueTestCase(unittest.TestCase):
pq.push(20, 0)
pq.push(30, 1)
self.assertEquals(len(pq), 3)
self.assertEquals(len(pq), 3)
pq.pop()
self.assertEquals(len(pq), 2)
self.assertEquals(len(pq), 2)
pq.pop()
self.assertEquals(len(pq), 1)
self.assertEquals(len(pq), 1)
pq.pop()
self.assertEquals(len(pq), 0)
self.assertEquals(len(pq), 0)
class PriorityStackTestCase(unittest.TestCase):
@ -70,7 +71,133 @@ class PriorityStackTestCase(unittest.TestCase):
for item, pr in INPUT:
pq.push(item, pr)
result = [x for x in pq]
self.assertEquals(result, self.output)
self.assertEquals(result, self.output)
class CaselessDictTest(unittest.TestCase):
def test_init(self):
seq = {'red': 1, 'black': 3}
d = CaselessDict(seq)
self.assertEqual(d['red'], 1)
self.assertEqual(d['black'], 3)
seq = (('red', 1), ('black', 3))
d = CaselessDict(seq)
self.assertEqual(d['red'], 1)
self.assertEqual(d['black'], 3)
def test_caseless(self):
d = CaselessDict()
d['key_Lower'] = 1
self.assertEqual(d['KEy_loWer'], 1)
self.assertEqual(d.get('KEy_loWer'), 1)
d['KEY_LOWER'] = 3
self.assertEqual(d['key_Lower'], 3)
self.assertEqual(d.get('key_Lower'), 3)
def test_delete(self):
d = CaselessDict({'key_lower': 1})
del d['key_LOWER']
self.assertRaises(KeyError, d.__getitem__, 'key_LOWER')
self.assertRaises(KeyError, d.__getitem__, 'key_lower')
def test_getdefault(self):
d = CaselessDict()
self.assertEqual(d.get('c', 5), 5)
d['c'] = 10
self.assertEqual(d.get('c', 5), 10)
def test_setdefault(self):
d = CaselessDict({'a': 1, 'b': 2})
r = d.setdefault('A', 5)
self.assertEqual(r, 1)
self.assertEqual(d['A'], 1)
r = d.setdefault('c', 5)
self.assertEqual(r, 5)
self.assertEqual(d['C'], 5)
def test_fromkeys(self):
keys = ('a', 'b')
d = CaselessDict.fromkeys(keys)
self.assertEqual(d['A'], None)
self.assertEqual(d['B'], None)
d = CaselessDict.fromkeys(keys, 1)
self.assertEqual(d['A'], 1)
self.assertEqual(d['B'], 1)
instance = CaselessDict()
d = instance.fromkeys(keys)
self.assertEqual(d['A'], None)
self.assertEqual(d['B'], None)
d = instance.fromkeys(keys, 1)
self.assertEqual(d['A'], 1)
self.assertEqual(d['B'], 1)
def test_contains(self):
d = CaselessDict()
d['a'] = 1
assert 'a' in d
assert d.has_key('a')
def test_pop(self):
d = CaselessDict()
d['a'] = 1
self.assertEqual(d.pop('A'), 1)
self.assertRaises(KeyError, d.pop, 'A')
def test_normkey(self):
class MyDict(CaselessDict):
def normkey(self, key):
return key.title()
d = MyDict()
d['key-one'] = 2
self.assertEqual(list(d.keys()), ['Key-One'])
def test_normvalue(self):
class MyDict(CaselessDict):
def normvalue(self, value):
if value is not None:
return value + 1
d = MyDict({'key': 1})
self.assertEqual(d['key'], 2)
self.assertEqual(d.get('key'), 2)
d = MyDict()
d['key'] = 1
self.assertEqual(d['key'], 2)
self.assertEqual(d.get('key'), 2)
d = MyDict()
d.setdefault('key', 1)
self.assertEqual(d['key'], 2)
self.assertEqual(d.get('key'), 2)
d = MyDict()
d.update({'key': 1})
self.assertEqual(d['key'], 2)
self.assertEqual(d.get('key'), 2)
d = MyDict.fromkeys(('key',), 1)
self.assertEqual(d['key'], 2)
self.assertEqual(d.get('key'), 2)
def test_copy(self):
h1 = CaselessDict({'header1': 'value'})
h2 = copy.copy(h1)
self.assertEqual(h1, h2)
self.assertEqual(h1.get('header1'), h2.get('header1'))
assert isinstance(h2, CaselessDict)
if __name__ == "__main__":
unittest.main()

6
scrapy/trunk/scrapy/utils/datatypes.py
View File

@ -186,7 +186,7 @@ class SiteNode(object):
class CaselessDict(dict):
def __init__(self, seq=None):
dict.__init__(self)
super(CaselessDict, self).__init__()
if seq:
self.update(seq)
@ -203,6 +203,10 @@ class CaselessDict(dict):
return dict.__contains__(self, self.normkey(key))
has_key = __contains__
def __copy__(self):
return self.__class__(self)
copy = __copy__
def normkey(self, key):
"""Method to normalize dictionary key access"""
return key.lower()