Merge pull request #2510 from elacuesta/reqser_request_class

[MRG+1] Preserve request class when converting to/from dicts (utils.reqser)

scrapy/utils/reqser.py

@@ -5,6 +5,7 @@ import six
 
 from scrapy.http import Request
 from scrapy.utils.python import to_unicode, to_native_str
+from scrapy.utils.misc import load_object
 
 
 def request_to_dict(request, spider=None):
@@ -32,6 +33,8 @@ def request_to_dict(request, spider=None):
         'priority': request.priority,
         'dont_filter': request.dont_filter,
     }
+    if type(request) is not Request:
+        d['_class'] = request.__module__ + '.' + request.__class__.__name__
     return d
 
 
@@ -47,7 +50,8 @@ def request_from_dict(d, spider=None):
     eb = d['errback']
     if eb and spider:
         eb = _get_method(spider, eb)
-    return Request(
+    request_cls = load_object(d['_class']) if '_class' in d else Request
+    return request_cls(
         url=to_native_str(d['url']),
         callback=cb,
         errback=eb,

tests/test_utils_reqser.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 import unittest
 
-from scrapy.http import Request
+from scrapy.http import Request, FormRequest
 from scrapy.spiders import Spider
 from scrapy.utils.reqser import request_to_dict, request_from_dict
 
@@ -42,6 +42,7 @@ class RequestSerializationTest(unittest.TestCase):
         self._assert_same_request(request, request2)
 
     def _assert_same_request(self, r1, r2):
+        self.assertEqual(r1.__class__, r2.__class__)
         self.assertEqual(r1.url, r2.url)
         self.assertEqual(r1.callback, r2.callback)
         self.assertEqual(r1.errback, r2.errback)
@@ -54,6 +55,12 @@ class RequestSerializationTest(unittest.TestCase):
         self.assertEqual(r1.priority, r2.priority)
         self.assertEqual(r1.dont_filter, r2.dont_filter)
 
+    def test_request_class(self):
+        r = FormRequest("http://www.example.com")
+        self._assert_serializes_ok(r, spider=self.spider)
+        r = CustomRequest("http://www.example.com")
+        self._assert_serializes_ok(r, spider=self.spider)
+
     def test_callback_serialization(self):
         r = Request("http://www.example.com", callback=self.spider.parse_item,
                     errback=self.spider.handle_error)
@@ -77,3 +84,7 @@ class TestSpider(Spider):
 
     def handle_error(self, failure):
         pass
+
+
+class CustomRequest(Request):
+    pass