Merge pull request #1408 from scrapy/py3-utils-reqser

PY3 port scrapy.utils.reqser

scrapy/utils/reqser.py

@@ -1,8 +1,11 @@
 """
 Helper functions for serializing (and deserializing) requests.
 """
+import six
 
 from scrapy.http import Request
+from scrapy.utils.python import to_unicode, to_native_str
+
 
 def request_to_dict(request, spider=None):
     """Convert Request object to a dict.
@@ -17,7 +20,7 @@ def request_to_dict(request, spider=None):
     if callable(eb):
         eb = _find_method(spider, eb)
     d = {
-        'url': request.url.decode('ascii'), # urls should be safe (safe_string_url)
+        'url': to_unicode(request.url),  # urls should be safe (safe_string_url)
         'callback': cb,
         'errback': eb,
         'method': request.method,
@@ -45,7 +48,7 @@ def request_from_dict(d, spider=None):
     if eb and spider:
         eb = _get_method(spider, eb)
     return Request(
-        url=d['url'].encode('ascii'),
+        url=to_native_str(d['url']),
         callback=cb,
         errback=eb,
         method=d['method'],
@@ -59,10 +62,16 @@ def request_from_dict(d, spider=None):
 
 
 def _find_method(obj, func):
-    if obj and hasattr(func, 'im_self') and func.im_self is obj:
-        return func.im_func.__name__
-    else:
-        raise ValueError("Function %s is not a method of: %s" % (func, obj))
+    if obj:
+        try:
+            func_self = six.get_method_self(func)
+        except AttributeError:  # func has no __self__
+            pass
+        else:
+            if func_self is obj:
+                return six.get_method_function(func).__name__
+    raise ValueError("Function %s is not a method of: %s" % (func, obj))
+
 
 def _get_method(obj, name):
     name = str(name)

tests/py3-ignores.txt

@@ -39,7 +39,6 @@ tests/test_spider.py
 tests/test_stats.py
 tests/test_utils_iterators.py
 tests/test_utils_log.py
-tests/test_utils_reqser.py
 tests/test_utils_template.py
 tests/test_webclient.py
 

tests/test_utils_reqser.py

@@ -1,9 +1,11 @@
+# -*- coding: utf-8 -*-
 import unittest
 
 from scrapy.http import Request
 from scrapy.spiders import Spider
 from scrapy.utils.reqser import request_to_dict, request_from_dict
 
+
 class RequestSerializationTest(unittest.TestCase):
 
     def setUp(self):
@@ -20,18 +22,18 @@ class RequestSerializationTest(unittest.TestCase):
             method="POST",
             body="some body",
             headers={'content-encoding': 'text/html; charset=latin-1'},
-            cookies={'currency': 'usd'},
+            cookies={'currency': u'руб'},
             encoding='latin-1',
             priority=20,
             meta={'a': 'b'})
         self._assert_serializes_ok(r)
 
     def test_latin1_body(self):
-        r = Request("http://www.example.com", body="\xa3")
+        r = Request("http://www.example.com", body=b"\xa3")
         self._assert_serializes_ok(r)
 
     def test_utf8_body(self):
-        r = Request("http://www.example.com", body="\xc2\xa3")
+        r = Request("http://www.example.com", body=b"\xc2\xa3")
         self._assert_serializes_ok(r)
 
     def _assert_serializes_ok(self, request, spider=None):
@@ -53,8 +55,8 @@ class RequestSerializationTest(unittest.TestCase):
         self.assertEqual(r1.dont_filter, r2.dont_filter)
 
     def test_callback_serialization(self):
-        r = Request("http://www.example.com", callback=self.spider.parse_item, \
-            errback=self.spider.handle_error)
+        r = Request("http://www.example.com", callback=self.spider.parse_item,
+                    errback=self.spider.handle_error)
         self._assert_serializes_ok(r, spider=self.spider)
 
     def test_unserializable_callback1(self):
@@ -69,7 +71,9 @@ class RequestSerializationTest(unittest.TestCase):
 
 class TestSpider(Spider):
     name = 'test'
+
     def parse_item(self, response):
         pass
+
     def handle_error(self, failure):
         pass