RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #497 from kmike/utils-unique 

[MRG] no need to use dict in scrapy.utils.python.unique
This commit is contained in:
Pablo Hoffman 2013-12-19 05:40:34 -08:00
parent 93eee9d1c6 84aa7599a4
commit 80bb9fb5d2
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/utils/python.py
View File

@ -53,13 +53,13 @@ def flatten(x):
def unique(list_, key=lambda x: x): def unique(list_, key=lambda x: x):
"""efficient function to uniquify a list preserving item order""" """efficient function to uniquify a list preserving item order"""
seen = {} seen = set()
result = [] result = []
for item in list_: for item in list_:
seenkey = key(item) seenkey = key(item)
if seenkey in seen: if seenkey in seen:
continue continue
seen[seenkey] = 1 seen.add(seenkey)
result.append(item) result.append(item)
return result return result
@ -70,7 +70,7 @@ def str_to_unicode(text, encoding=None, errors='strict'):
object without the risk of double-decoding problems (which can happen if object without the risk of double-decoding problems (which can happen if
you don't use the default 'ascii' encoding) you don't use the default 'ascii' encoding)
""" """
if encoding is None: if encoding is None:
encoding = 'utf-8' encoding = 'utf-8'
if isinstance(text, str): if isinstance(text, str):