mirror of https://github.com/scrapy/scrapy.git
73 lines
2.0 KiB
Python
73 lines
2.0 KiB
Python
"""
|
|
Helper functions for serializing (and deserializing) requests.
|
|
"""
|
|
|
|
from scrapy.http import Request
|
|
|
|
def request_to_dict(request, spider=None):
|
|
"""Convert Request object to a dict.
|
|
|
|
If a spider is given, it will try to find out the name of the spider method
|
|
used in the callback and store that as the callback.
|
|
"""
|
|
cb = request.callback
|
|
if callable(cb):
|
|
cb = _find_method(spider, cb)
|
|
eb = request.errback
|
|
if callable(eb):
|
|
eb = _find_method(spider, eb)
|
|
d = {
|
|
'url': request.url.decode('ascii'), # urls should be safe (safe_string_url)
|
|
'callback': cb,
|
|
'errback': eb,
|
|
'method': request.method,
|
|
'headers': dict(request.headers),
|
|
'body': request.body,
|
|
'cookies': request.cookies,
|
|
'meta': request.meta,
|
|
'_encoding': request._encoding,
|
|
'priority': request.priority,
|
|
'dont_filter': request.dont_filter,
|
|
}
|
|
return d
|
|
|
|
|
|
def request_from_dict(d, spider=None):
|
|
"""Create Request object from a dict.
|
|
|
|
If a spider is given, it will try to resolve the callbacks looking at the
|
|
spider for methods with the same name.
|
|
"""
|
|
cb = d['callback']
|
|
if cb and spider:
|
|
cb = _get_method(spider, cb)
|
|
eb = d['errback']
|
|
if eb and spider:
|
|
eb = _get_method(spider, eb)
|
|
return Request(
|
|
url=d['url'].encode('ascii'),
|
|
callback=cb,
|
|
errback=eb,
|
|
method=d['method'],
|
|
headers=d['headers'],
|
|
body=d['body'],
|
|
cookies=d['cookies'],
|
|
meta=d['meta'],
|
|
encoding=d['_encoding'],
|
|
priority=d['priority'],
|
|
dont_filter=d['dont_filter'])
|
|
|
|
|
|
def _find_method(obj, func):
|
|
if obj and hasattr(func, 'im_self') and func.im_self is obj:
|
|
return func.im_func.__name__
|
|
else:
|
|
raise ValueError("Function %s is not a method of: %s" % (func, obj))
|
|
|
|
def _get_method(obj, name):
|
|
name = str(name)
|
|
try:
|
|
return getattr(obj, name)
|
|
except AttributeError:
|
|
raise ValueError("Method %r not found in: %s" % (name, obj))
|