no need to use dict in scrapy.utils.python.unique

This commit is contained in:
Mikhail Korobov 2013-12-19 15:06:27 +06:00
parent 51f340e1b9
commit 84aa7599a4
1 changed files with 4 additions and 4 deletions

View File

@ -53,13 +53,13 @@ def flatten(x):
def unique(list_, key=lambda x: x):
"""efficient function to uniquify a list preserving item order"""
seen = {}
seen = set()
result = []
for item in list_:
seenkey = key(item)
if seenkey in seen:
if seenkey in seen:
continue
seen[seenkey] = 1
seen.add(seenkey)
result.append(item)
return result
@ -70,7 +70,7 @@ def str_to_unicode(text, encoding=None, errors='strict'):
object without the risk of double-decoding problems (which can happen if
you don't use the default 'ascii' encoding)
"""
if encoding is None:
encoding = 'utf-8'
if isinstance(text, str):