mirror of https://github.com/scrapy/scrapy.git
Pass callback results as dicts instead of tuples
This commit is contained in:
parent
ac73bcc79c
commit
91a78eef3e
|
|
@ -384,7 +384,13 @@ class ScrapyAgent(object):
|
|||
def _cb_bodyready(self, txresponse, request):
|
||||
# deliverBody hangs for responses without body
|
||||
if txresponse.length == 0:
|
||||
return txresponse, b'', None, None
|
||||
return {
|
||||
"txresponse": txresponse,
|
||||
"body": b"",
|
||||
"flags": None,
|
||||
"certificate": None,
|
||||
"ip_address": None,
|
||||
}
|
||||
|
||||
maxsize = request.meta.get('download_maxsize', self._maxsize)
|
||||
warnsize = request.meta.get('download_warnsize', self._warnsize)
|
||||
|
|
@ -420,12 +426,18 @@ class ScrapyAgent(object):
|
|||
return d
|
||||
|
||||
def _cb_bodydone(self, result, request, url):
|
||||
txresponse, body, flags, certificate, ip_address = result
|
||||
status = int(txresponse.code)
|
||||
headers = Headers(txresponse.headers.getAllRawHeaders())
|
||||
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
|
||||
return respcls(url=url, status=status, headers=headers, body=body,
|
||||
flags=flags, certificate=certificate, ip_address=ip_address)
|
||||
status = int(result["txresponse"].code)
|
||||
headers = Headers(result["txresponse"].headers.getAllRawHeaders())
|
||||
respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
|
||||
return respcls(
|
||||
url=url,
|
||||
status=status,
|
||||
headers=headers,
|
||||
body=result["body"],
|
||||
flags=result["flags"],
|
||||
certificate=result["certificate"],
|
||||
ip_address=result["ip_address"],
|
||||
)
|
||||
|
||||
|
||||
@implementer(IBodyProducer)
|
||||
|
|
@ -501,22 +513,34 @@ class _ResponseReader(protocol.Protocol):
|
|||
|
||||
body = self._bodybuf.getvalue()
|
||||
if reason.check(ResponseDone):
|
||||
self._finished.callback(
|
||||
(self._txresponse, body, None, self._certificate, self._ip_address)
|
||||
)
|
||||
self._finished.callback({
|
||||
"txresponse": self._txresponse,
|
||||
"body": body,
|
||||
"flags": None,
|
||||
"certificate": self._certificate,
|
||||
"ip_address": self._ip_address,
|
||||
})
|
||||
return
|
||||
|
||||
if reason.check(PotentialDataLoss):
|
||||
self._finished.callback(
|
||||
(self._txresponse, body, ['partial'], self._certificate, self._ip_address)
|
||||
)
|
||||
self._finished.callback({
|
||||
"txresponse": self._txresponse,
|
||||
"body": body,
|
||||
"flags": ["partial"],
|
||||
"certificate": self._certificate,
|
||||
"ip_address": self._ip_address,
|
||||
})
|
||||
return
|
||||
|
||||
if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
|
||||
if not self._fail_on_dataloss:
|
||||
self._finished.callback(
|
||||
(self._txresponse, body, ['dataloss'], self._certificate, self._ip_address)
|
||||
)
|
||||
self._finished.callback({
|
||||
"txresponse": self._txresponse,
|
||||
"body": body,
|
||||
"flags": ["dataloss"],
|
||||
"certificate": self._certificate,
|
||||
"ip_address": self._ip_address,
|
||||
})
|
||||
return
|
||||
|
||||
elif not self._fail_on_dataloss_warned:
|
||||
|
|
|
|||
Loading…
Reference in New Issue