Fixed issue with unicode keyword arguments bug in earlier Python versions (see http://bugs.python.org/issue2646). Closes #250

scrapy/queue.py

@@ -3,6 +3,7 @@ from twisted.internet import defer
 from scrapy.http import Request
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.spider import create_spider_for_request
+from scrapy.utils.python import stringify_dict
 from scrapy import log
 
 
@@ -23,6 +24,7 @@ class ExecutionQueue(object):
         msg = self._queue.pop()
         if msg:
             name = msg.pop('name')
+            msg = stringify_dict(msg) # see #250
             self.append_spider_name(name, **msg)
 
     def get_next(self):

scrapy/tests/test_queue.py

@@ -100,6 +100,17 @@ class ExecutionQueueTest(unittest.TestCase):
         self.assert_(spider.name == 'test123')
         self.assert_(spider.arg == '123')
 
+    def test_append_next(self):
+        # the reason for this test: http://dev.scrapy.org/ticket/250
+        class MockQueue(object):
+            def pop(self):
+                return {u'name': u'test123', u'test': u'hello'}
+        self.queue._queue = MockQueue()
+        self.queue._append_next()
+        spider = self.queue.spider_requests[0][0]
+        self.assert_(spider.name == 'test123')
+        self.assert_(spider.test == 'hello')
+
     def _assert_request_urls(self, requests, urls):
         assert all(isinstance(x, Request) for x in requests)
         self.assertEqual([x.url for x in requests], urls)

scrapy/tests/test_utils_python.py

@@ -4,7 +4,7 @@ from itertools import count
 
 from scrapy.utils.python import str_to_unicode, unicode_to_str, \
     memoizemethod_noargs, isbinarytext, equal_attributes, \
-    WeakKeyCache
+    WeakKeyCache, stringify_dict
 
 class UtilsPythonTestCase(unittest.TestCase):
     def test_str_to_unicode(self):
@@ -122,7 +122,29 @@ class UtilsPythonTestCase(unittest.TestCase):
         del k
         self.assertFalse(len(wk._weakdict))
 
+    def test_stringify_dict(self):
+        d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
+        d2 = stringify_dict(d, keys_only=False)
+        self.failUnlessEqual(d, d2)
+        self.failIf(d is d2) # shouldn't modify in place
+        self.failIf(any(isinstance(x, unicode) for x in d2.keys()))
+        self.failIf(any(isinstance(x, unicode) for x in d2.values()))
 
+    def test_stringify_dict_tuples(self):
+        tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')]
+        d = dict(tuples)
+        d2 = stringify_dict(tuples, keys_only=False)
+        self.failUnlessEqual(d, d2)
+        self.failIf(d is d2) # shouldn't modify in place
+        self.failIf(any(isinstance(x, unicode) for x in d2.keys()), d2.keys())
+        self.failIf(any(isinstance(x, unicode) for x in d2.values()))
+
+    def test_stringify_dict_keys_only(self):
+        d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
+        d2 = stringify_dict(d)
+        self.failUnlessEqual(d, d2)
+        self.failIf(d is d2) # shouldn't modify in place
+        self.failIf(any(isinstance(x, unicode) for x in d2.keys()))
 
 if __name__ == "__main__":
     unittest.main()

scrapy/utils/python.py

@@ -191,3 +191,17 @@ class WeakKeyCache(object):
         if key not in self._weakdict:
             self._weakdict[key] = self.default_factory(key)
         return self._weakdict[key]
+
+
+def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
+    """Return a (new) dict with the unicode keys (and values if, keys_only is
+    False) of the given dict converted to strings. `dct_or_tuples` can be a
+    dict or a list of tuples, like any dict constructor supports.
+    """
+    d = {}
+    for k, v in dict(dct_or_tuples).iteritems():
+        k = k.encode(encoding) if isinstance(k, unicode) else k
+        if not keys_only:
+            v = v.encode(encoding) if isinstance(v, unicode) else v
+        d[k] = v
+    return d