From 1ef2cd400c90ea02975f25c3ee47a054851962c2 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Sat, 25 Sep 2010 21:17:36 -0300
Subject: [PATCH] Fixed issue with unicode keyword arguments bug in earlier
 Python versions (see http://bugs.python.org/issue2646). Closes #250

---
 scrapy/queue.py                   |  2 ++
 scrapy/tests/test_queue.py        | 11 +++++++++++
 scrapy/tests/test_utils_python.py | 24 +++++++++++++++++++++++-
 scrapy/utils/python.py            | 14 ++++++++++++++
 4 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/scrapy/queue.py b/scrapy/queue.py
index 0087f6ccc..577555ece 100644
--- a/scrapy/queue.py
+++ b/scrapy/queue.py
@@ -3,6 +3,7 @@ from twisted.internet import defer
 from scrapy.http import Request
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.spider import create_spider_for_request
+from scrapy.utils.python import stringify_dict
 from scrapy import log
 
 
@@ -23,6 +24,7 @@ class ExecutionQueue(object):
         msg = self._queue.pop()
         if msg:
             name = msg.pop('name')
+            msg = stringify_dict(msg) # see #250
             self.append_spider_name(name, **msg)
 
     def get_next(self):
diff --git a/scrapy/tests/test_queue.py b/scrapy/tests/test_queue.py
index f7a5e1bf2..077dd71f9 100644
--- a/scrapy/tests/test_queue.py
+++ b/scrapy/tests/test_queue.py
@@ -100,6 +100,17 @@ class ExecutionQueueTest(unittest.TestCase):
         self.assert_(spider.name == 'test123')
         self.assert_(spider.arg == '123')
 
+    def test_append_next(self):
+        # the reason for this test: http://dev.scrapy.org/ticket/250
+        class MockQueue(object):
+            def pop(self):
+                return {u'name': u'test123', u'test': u'hello'}
+        self.queue._queue = MockQueue()
+        self.queue._append_next()
+        spider = self.queue.spider_requests[0][0]
+        self.assert_(spider.name == 'test123')
+        self.assert_(spider.test == 'hello')
+
     def _assert_request_urls(self, requests, urls):
         assert all(isinstance(x, Request) for x in requests)
         self.assertEqual([x.url for x in requests], urls)
diff --git a/scrapy/tests/test_utils_python.py b/scrapy/tests/test_utils_python.py
index b34bdbd85..84eebbac1 100644
--- a/scrapy/tests/test_utils_python.py
+++ b/scrapy/tests/test_utils_python.py
@@ -4,7 +4,7 @@ from itertools import count
 
 from scrapy.utils.python import str_to_unicode, unicode_to_str, \
     memoizemethod_noargs, isbinarytext, equal_attributes, \
-    WeakKeyCache
+    WeakKeyCache, stringify_dict
 
 class UtilsPythonTestCase(unittest.TestCase):
     def test_str_to_unicode(self):
@@ -122,7 +122,29 @@ class UtilsPythonTestCase(unittest.TestCase):
         del k
         self.assertFalse(len(wk._weakdict))
 
+    def test_stringify_dict(self):
+        d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
+        d2 = stringify_dict(d, keys_only=False)
+        self.failUnlessEqual(d, d2)
+        self.failIf(d is d2) # shouldn't modify in place
+        self.failIf(any(isinstance(x, unicode) for x in d2.keys()))
+        self.failIf(any(isinstance(x, unicode) for x in d2.values()))
 
+    def test_stringify_dict_tuples(self):
+        tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')]
+        d = dict(tuples)
+        d2 = stringify_dict(tuples, keys_only=False)
+        self.failUnlessEqual(d, d2)
+        self.failIf(d is d2) # shouldn't modify in place
+        self.failIf(any(isinstance(x, unicode) for x in d2.keys()), d2.keys())
+        self.failIf(any(isinstance(x, unicode) for x in d2.values()))
+
+    def test_stringify_dict_keys_only(self):
+        d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
+        d2 = stringify_dict(d)
+        self.failUnlessEqual(d, d2)
+        self.failIf(d is d2) # shouldn't modify in place
+        self.failIf(any(isinstance(x, unicode) for x in d2.keys()))
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index eebdb60fd..24e4476e6 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -191,3 +191,17 @@ class WeakKeyCache(object):
         if key not in self._weakdict:
             self._weakdict[key] = self.default_factory(key)
         return self._weakdict[key]
+
+
+def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
+    """Return a (new) dict with the unicode keys (and values if, keys_only is
+    False) of the given dict converted to strings. `dct_or_tuples` can be a
+    dict or a list of tuples, like any dict constructor supports.
+    """
+    d = {}
+    for k, v in dict(dct_or_tuples).iteritems():
+        k = k.encode(encoding) if isinstance(k, unicode) else k
+        if not keys_only:
+            v = v.encode(encoding) if isinstance(v, unicode) else v
+        d[k] = v
+    return d