RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor request to/from dict (#5130)

This commit is contained in:
Eugenio Lacuesta 2021-05-10 13:00:08 -03:00 committed by GitHub
parent 34b216289c
commit cec36a9284
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 183 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
docs/topics/request-response.rst
View File

@ -26,10 +26,6 @@ Request objects
.. autoclass:: Request
A :class:`Request` object represents an HTTP request, which is usually
generated in the Spider and executed by the Downloader, and thus generating
a :class:`Response`.
:param url: the URL of this request
If the URL is invalid, a :exc:`ValueError` exception is raised.
@ -205,6 +201,8 @@ Request objects
``failure.request.cb_kwargs`` in the request's errback. For more information,
see :ref:`errback-cb_kwargs`.
.. autoattribute:: Request.attributes
.. method:: Request.copy()
Return a new Request which is a copy of this Request. See also:
@ -220,6 +218,15 @@ Request objects
.. automethod:: from_curl
.. automethod:: to_dict
Other functions related to requests
-----------------------------------
.. autofunction:: scrapy.utils.request.request_from_dict
.. _topics-request-response-ref-request-callback-arguments:
Passing additional data to callback functions
@ -642,6 +649,8 @@ dealing with JSON requests.
data into JSON format.
:type dumps_kwargs: dict
.. autoattribute:: JsonRequest.attributes
JsonRequest usage example
-------------------------

70
scrapy/http/request/__init__.py
View File

@ -4,17 +4,37 @@ requests in Scrapy.
See documentation in docs/topics/request-response.rst
"""
import inspect
from typing import Optional, Tuple
from w3lib.url import safe_url_string
import scrapy
from scrapy.http.common import obsolete_setter
from scrapy.http.headers import Headers
from scrapy.utils.curl import curl_to_request_kwargs
from scrapy.utils.python import to_bytes
from scrapy.utils.trackref import object_ref
from scrapy.utils.url import escape_ajax
from scrapy.http.common import obsolete_setter
from scrapy.utils.curl import curl_to_request_kwargs
class Request(object_ref):
"""Represents an HTTP request, which is usually generated in a Spider and
executed by the Downloader, thus generating a :class:`Response`.
"""
attributes: Tuple[str, ...] = (
"url", "callback", "method", "headers", "body",
"cookies", "meta", "encoding", "priority",
"dont_filter", "errback", "flags", "cb_kwargs",
)
"""A tuple of :class:`str` objects containing the name of all public
attributes of the class that are also keyword parameters of the
``__init__`` method.
Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and
:func:`~scrapy.utils.request.request_from_dict`.
"""
def __init__(self, url, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding='utf-8', priority=0,
@ -99,11 +119,8 @@ class Request(object_ref):
return self.replace()
def replace(self, *args, **kwargs):
"""Create a new Request with the same attributes except for those
given new values.
"""
for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags',
'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']:
"""Create a new Request with the same attributes except for those given new values"""
for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))
cls = kwargs.pop('cls', self.__class__)
return cls(*args, **kwargs)
@ -136,8 +153,43 @@ class Request(object_ref):
To translate a cURL command into a Scrapy request,
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
"""
"""
request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options)
request_kwargs.update(kwargs)
return cls(**request_kwargs)
def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> dict:
"""Return a dictionary containing the Request's data.
Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.
If a spider is given, this method will try to find out the name of the spider methods used as callback
and errback and include them in the output dict, raising an exception if they cannot be found.
"""
d = {
"url": self.url, # urls are safe (safe_string_url)
"callback": _find_method(spider, self.callback) if callable(self.callback) else self.callback,
"errback": _find_method(spider, self.errback) if callable(self.errback) else self.errback,
"headers": dict(self.headers),
}
for attr in self.attributes:
d.setdefault(attr, getattr(self, attr))
if type(self) is not Request:
d["_class"] = self.__module__ + '.' + self.__class__.__name__
return d
def _find_method(obj, func):
"""Helper function for Request.to_dict"""
# Only instance methods contain ``__func__``
if obj and hasattr(func, '__func__'):
members = inspect.getmembers(obj, predicate=inspect.ismethod)
for name, obj_func in members:
# We need to use __func__ to access the original function object because instance
# method objects are generated each time attribute is retrieved from instance.
#
# Reference: The standard type hierarchy
# https://docs.python.org/3/reference/datamodel.html
if obj_func.__func__ is func.__func__:
return name
raise ValueError(f"Function {func} is not an instance method in: {obj}")

8
scrapy/http/request/json_request.py
View File

@ -8,12 +8,16 @@ See documentation in docs/topics/request-response.rst
import copy
import json
import warnings
from typing import Tuple
from scrapy.http.request import Request
from scrapy.utils.deprecate import create_deprecated_class
class JsonRequest(Request):
attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",)
def __init__(self, *args, **kwargs):
dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {}))
dumps_kwargs.setdefault('sort_keys', True)
@ -36,6 +40,10 @@ class JsonRequest(Request):
self.headers.setdefault('Content-Type', 'application/json')
self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
@property
def dumps_kwargs(self):
return self._dumps_kwargs
def replace(self, *args, **kwargs):
body_passed = kwargs.get('body', None) is not None
data = kwargs.pop('data', None)

8
scrapy/squeues.py
View File

@ -9,7 +9,7 @@ import pickle
from queuelib import queue
from scrapy.utils.deprecate import create_deprecated_class
from scrapy.utils.reqser import request_to_dict, request_from_dict
from scrapy.utils.request import request_from_dict
def _with_mkdir(queue_class):
@ -68,14 +68,14 @@ def _scrapy_serialization_queue(queue_class):
return cls(crawler, key)
def push(self, request):
request = request_to_dict(request, self.spider)
request = request.to_dict(spider=self.spider)
return super().push(request)
def pop(self):
request = super().pop()
if not request:
return None
return request_from_dict(request, self.spider)
return request_from_dict(request, spider=self.spider)
def peek(self):
"""Returns the next object to be returned by :meth:`pop`,
@ -87,7 +87,7 @@ def _scrapy_serialization_queue(queue_class):
request = super().peek()
if not request:
return None
return request_from_dict(request, self.spider)
return request_from_dict(request, spider=self.spider)
return ScrapyRequestQueue

103
scrapy/utils/reqser.py
View File

@ -1,95 +1,22 @@
"""
Helper functions for serializing (and deserializing) requests.
"""
import inspect
import warnings
from typing import Optional
from scrapy.http import Request
from scrapy.utils.python import to_unicode
from scrapy.utils.misc import load_object
import scrapy
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.request import request_from_dict as _from_dict
def request_to_dict(request, spider=None):
"""Convert Request object to a dict.
If a spider is given, it will try to find out the name of the spider method
used in the callback and store that as the callback.
"""
cb = request.callback
if callable(cb):
cb = _find_method(spider, cb)
eb = request.errback
if callable(eb):
eb = _find_method(spider, eb)
d = {
'url': to_unicode(request.url), # urls should be safe (safe_string_url)
'callback': cb,
'errback': eb,
'method': request.method,
'headers': dict(request.headers),
'body': request.body,
'cookies': request.cookies,
'meta': request.meta,
'_encoding': request._encoding,
'priority': request.priority,
'dont_filter': request.dont_filter,
'flags': request.flags,
'cb_kwargs': request.cb_kwargs,
}
if type(request) is not Request:
d['_class'] = request.__module__ + '.' + request.__class__.__name__
return d
warnings.warn(
("Module scrapy.utils.reqser is deprecated, please use request.to_dict method"
" and/or scrapy.utils.request.request_from_dict instead"),
category=ScrapyDeprecationWarning,
stacklevel=2,
)
def request_from_dict(d, spider=None):
"""Create Request object from a dict.
If a spider is given, it will try to resolve the callbacks looking at the
spider for methods with the same name.
"""
cb = d['callback']
if cb and spider:
cb = _get_method(spider, cb)
eb = d['errback']
if eb and spider:
eb = _get_method(spider, eb)
request_cls = load_object(d['_class']) if '_class' in d else Request
return request_cls(
url=to_unicode(d['url']),
callback=cb,
errback=eb,
method=d['method'],
headers=d['headers'],
body=d['body'],
cookies=d['cookies'],
meta=d['meta'],
encoding=d['_encoding'],
priority=d['priority'],
dont_filter=d['dont_filter'],
flags=d.get('flags'),
cb_kwargs=d.get('cb_kwargs'),
)
def request_to_dict(request: "scrapy.Request", spider: Optional["scrapy.Spider"] = None) -> dict:
return request.to_dict(spider=spider)
def _find_method(obj, func):
# Only instance methods contain ``__func__``
if obj and hasattr(func, '__func__'):
members = inspect.getmembers(obj, predicate=inspect.ismethod)
for name, obj_func in members:
# We need to use __func__ to access the original
# function object because instance method objects
# are generated each time attribute is retrieved from
# instance.
#
# Reference: The standard type hierarchy
# https://docs.python.org/3/reference/datamodel.html
if obj_func.__func__ is func.__func__:
return name
raise ValueError(f"Function {func} is not an instance method in: {obj}")
def _get_method(obj, name):
name = str(name)
try:
return getattr(obj, name)
except AttributeError:
raise ValueError(f"Method {name!r} not found in: {obj}")
def request_from_dict(d: dict, spider: Optional["scrapy.Spider"] = None) -> "scrapy.Request":
return _from_dict(d, spider=spider)

27
scrapy/utils/request.py
View File

@ -11,8 +11,9 @@ from weakref import WeakKeyDictionary
from w3lib.http import basic_auth_header
from w3lib.url import canonicalize_url
from scrapy.http import Request
from scrapy import Request, Spider
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
from scrapy.utils.python import to_bytes, to_unicode
@ -106,3 +107,27 @@ def referer_str(request: Request) -> Optional[str]:
if referrer is None:
return referrer
return to_unicode(referrer, errors='replace')
def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request:
"""Create a :class:`~scrapy.Request` object from a dict.
If a spider is given, it will try to resolve the callbacks looking at the
spider for methods with the same name.
"""
request_cls = load_object(d["_class"]) if "_class" in d else Request
kwargs = {key: value for key, value in d.items() if key in request_cls.attributes}
if d.get("callback") and spider:
kwargs["callback"] = _get_method(spider, d["callback"])
if d.get("errback") and spider:
kwargs["errback"] = _get_method(spider, d["errback"])
return request_cls(**kwargs)
def _get_method(obj, name):
"""Helper function for request_from_dict"""
name = str(name)
try:
return getattr(obj, name)
except AttributeError:
raise ValueError(f"Method {name!r} not found in: {obj}")

76
tests/test_utils_reqser.py → tests/test_request_dict.py
View File

@ -1,8 +1,16 @@
import sys
import unittest
import warnings
from contextlib import suppress
from scrapy.http import Request, FormRequest
from scrapy.spiders import Spider
from scrapy.utils.reqser import request_to_dict, request_from_dict
from scrapy import Spider, Request
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import FormRequest, JsonRequest
from scrapy.utils.request import request_from_dict
class CustomRequest(Request):
pass
class RequestSerializationTest(unittest.TestCase):
@ -27,7 +35,8 @@ class RequestSerializationTest(unittest.TestCase):
priority=20,
meta={'a': 'b'},
cb_kwargs={'k': 'v'},
flags=['testFlag'])
flags=['testFlag'],
)
self._assert_serializes_ok(r, spider=self.spider)
def test_latin1_body(self):
@ -39,7 +48,7 @@ class RequestSerializationTest(unittest.TestCase):
self._assert_serializes_ok(r)
def _assert_serializes_ok(self, request, spider=None):
d = request_to_dict(request, spider=spider)
d = request.to_dict(spider=spider)
request2 = request_from_dict(d, spider=spider)
self._assert_same_request(request, request2)
@ -54,16 +63,21 @@ class RequestSerializationTest(unittest.TestCase):
self.assertEqual(r1.cookies, r2.cookies)
self.assertEqual(r1.meta, r2.meta)
self.assertEqual(r1.cb_kwargs, r2.cb_kwargs)
self.assertEqual(r1.encoding, r2.encoding)
self.assertEqual(r1._encoding, r2._encoding)
self.assertEqual(r1.priority, r2.priority)
self.assertEqual(r1.dont_filter, r2.dont_filter)
self.assertEqual(r1.flags, r2.flags)
if isinstance(r1, JsonRequest):
self.assertEqual(r1.dumps_kwargs, r2.dumps_kwargs)
def test_request_class(self):
r = FormRequest("http://www.example.com")
self._assert_serializes_ok(r, spider=self.spider)
r = CustomRequest("http://www.example.com")
self._assert_serializes_ok(r, spider=self.spider)
r1 = FormRequest("http://www.example.com")
self._assert_serializes_ok(r1, spider=self.spider)
r2 = CustomRequest("http://www.example.com")
self._assert_serializes_ok(r2, spider=self.spider)
r3 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})
self._assert_serializes_ok(r3, spider=self.spider)
def test_callback_serialization(self):
r = Request("http://www.example.com", callback=self.spider.parse_item,
@ -75,7 +89,7 @@ class RequestSerializationTest(unittest.TestCase):
callback=self.spider.parse_item_reference,
errback=self.spider.handle_error_reference)
self._assert_serializes_ok(r, spider=self.spider)
request_dict = request_to_dict(r, self.spider)
request_dict = r.to_dict(spider=self.spider)
self.assertEqual(request_dict['callback'], 'parse_item_reference')
self.assertEqual(request_dict['errback'], 'handle_error_reference')
@ -84,7 +98,7 @@ class RequestSerializationTest(unittest.TestCase):
callback=self.spider._TestSpider__parse_item_reference,
errback=self.spider._TestSpider__handle_error_reference)
self._assert_serializes_ok(r, spider=self.spider)
request_dict = request_to_dict(r, self.spider)
request_dict = r.to_dict(spider=self.spider)
self.assertEqual(request_dict['callback'],
'_TestSpider__parse_item_reference')
self.assertEqual(request_dict['errback'],
@ -110,18 +124,16 @@ class RequestSerializationTest(unittest.TestCase):
def test_unserializable_callback1(self):
r = Request("http://www.example.com", callback=lambda x: x)
self.assertRaises(ValueError, request_to_dict, r)
self.assertRaises(ValueError, request_to_dict, r, spider=self.spider)
self.assertRaises(ValueError, r.to_dict, spider=self.spider)
def test_unserializable_callback2(self):
r = Request("http://www.example.com", callback=self.spider.parse_item)
self.assertRaises(ValueError, request_to_dict, r)
self.assertRaises(ValueError, r.to_dict, spider=None)
def test_unserializable_callback3(self):
"""Parser method is removed or replaced dynamically."""
class MySpider(Spider):
name = 'my_spider'
def parse(self, response):
@ -130,7 +142,35 @@ class RequestSerializationTest(unittest.TestCase):
spider = MySpider()
r = Request("http://www.example.com", callback=spider.parse)
setattr(spider, 'parse', None)
self.assertRaises(ValueError, request_to_dict, r, spider=spider)
self.assertRaises(ValueError, r.to_dict, spider=spider)
def test_callback_not_available(self):
"""Callback method is not available in the spider passed to from_dict"""
spider = TestSpiderDelegation()
r = Request("http://www.example.com", callback=spider.delegated_callback)
d = r.to_dict(spider=spider)
self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo"))
class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest):
def _assert_serializes_ok(self, request, spider=None):
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
with suppress(KeyError):
del sys.modules["scrapy.utils.reqser"] # delete module to reset the deprecation warning
from scrapy.utils.reqser import request_from_dict as _from_dict, request_to_dict as _to_dict
request_copy = _from_dict(_to_dict(request, spider), spider)
self._assert_same_request(request, request_copy)
self.assertEqual(len(caught), 1)
self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning))
self.assertEqual(
"Module scrapy.utils.reqser is deprecated, please use request.to_dict method"
" and/or scrapy.utils.request.request_from_dict instead",
str(caught[0].message),
)
class TestSpiderMixin:
@ -177,7 +217,3 @@ class TestSpider(Spider, TestSpiderMixin):
def __parse_item_private(self, response):
pass
class CustomRequest(Request):
pass