diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index d02585a32..7e1e99e48 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -1,8 +1,11 @@ """ Helper functions for serializing (and deserializing) requests. """ +import six from scrapy.http import Request +from scrapy.utils.python import to_unicode, to_native_str + def request_to_dict(request, spider=None): """Convert Request object to a dict. @@ -17,7 +20,7 @@ def request_to_dict(request, spider=None): if callable(eb): eb = _find_method(spider, eb) d = { - 'url': request.url.decode('ascii'), # urls should be safe (safe_string_url) + 'url': to_unicode(request.url), # urls should be safe (safe_string_url) 'callback': cb, 'errback': eb, 'method': request.method, @@ -45,7 +48,7 @@ def request_from_dict(d, spider=None): if eb and spider: eb = _get_method(spider, eb) return Request( - url=d['url'].encode('ascii'), + url=to_native_str(d['url']), callback=cb, errback=eb, method=d['method'], @@ -59,10 +62,16 @@ def request_from_dict(d, spider=None): def _find_method(obj, func): - if obj and hasattr(func, 'im_self') and func.im_self is obj: - return func.im_func.__name__ - else: - raise ValueError("Function %s is not a method of: %s" % (func, obj)) + if obj: + try: + func_self = six.get_method_self(func) + except AttributeError: # func has no __self__ + pass + else: + if func_self is obj: + return six.get_method_function(func).__name__ + raise ValueError("Function %s is not a method of: %s" % (func, obj)) + def _get_method(obj, name): name = str(name) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 469d2c5e1..038f715a6 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -39,7 +39,6 @@ tests/test_spider.py tests/test_stats.py tests/test_utils_iterators.py tests/test_utils_log.py -tests/test_utils_reqser.py tests/test_utils_template.py tests/test_webclient.py diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 9139c0ad0..40c44f7d9 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -1,9 +1,11 @@ +# -*- coding: utf-8 -*- import unittest from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.reqser import request_to_dict, request_from_dict + class RequestSerializationTest(unittest.TestCase): def setUp(self): @@ -20,18 +22,18 @@ class RequestSerializationTest(unittest.TestCase): method="POST", body="some body", headers={'content-encoding': 'text/html; charset=latin-1'}, - cookies={'currency': 'usd'}, + cookies={'currency': u'руб'}, encoding='latin-1', priority=20, meta={'a': 'b'}) self._assert_serializes_ok(r) def test_latin1_body(self): - r = Request("http://www.example.com", body="\xa3") + r = Request("http://www.example.com", body=b"\xa3") self._assert_serializes_ok(r) def test_utf8_body(self): - r = Request("http://www.example.com", body="\xc2\xa3") + r = Request("http://www.example.com", body=b"\xc2\xa3") self._assert_serializes_ok(r) def _assert_serializes_ok(self, request, spider=None): @@ -53,8 +55,8 @@ class RequestSerializationTest(unittest.TestCase): self.assertEqual(r1.dont_filter, r2.dont_filter) def test_callback_serialization(self): - r = Request("http://www.example.com", callback=self.spider.parse_item, \ - errback=self.spider.handle_error) + r = Request("http://www.example.com", callback=self.spider.parse_item, + errback=self.spider.handle_error) self._assert_serializes_ok(r, spider=self.spider) def test_unserializable_callback1(self): @@ -69,7 +71,9 @@ class RequestSerializationTest(unittest.TestCase): class TestSpider(Spider): name = 'test' + def parse_item(self, response): pass + def handle_error(self, failure): pass