Pass callback results as dicts instead of tuples

This commit is contained in:
Eugenio Lacuesta 2020-03-08 22:32:17 -03:00
parent ac73bcc79c
commit 91a78eef3e
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
1 changed files with 40 additions and 16 deletions

View File

@ -384,7 +384,13 @@ class ScrapyAgent(object):
def _cb_bodyready(self, txresponse, request):
# deliverBody hangs for responses without body
if txresponse.length == 0:
return txresponse, b'', None, None
return {
"txresponse": txresponse,
"body": b"",
"flags": None,
"certificate": None,
"ip_address": None,
}
maxsize = request.meta.get('download_maxsize', self._maxsize)
warnsize = request.meta.get('download_warnsize', self._warnsize)
@ -420,12 +426,18 @@ class ScrapyAgent(object):
return d
def _cb_bodydone(self, result, request, url):
txresponse, body, flags, certificate, ip_address = result
status = int(txresponse.code)
headers = Headers(txresponse.headers.getAllRawHeaders())
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
return respcls(url=url, status=status, headers=headers, body=body,
flags=flags, certificate=certificate, ip_address=ip_address)
status = int(result["txresponse"].code)
headers = Headers(result["txresponse"].headers.getAllRawHeaders())
respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
return respcls(
url=url,
status=status,
headers=headers,
body=result["body"],
flags=result["flags"],
certificate=result["certificate"],
ip_address=result["ip_address"],
)
@implementer(IBodyProducer)
@ -501,22 +513,34 @@ class _ResponseReader(protocol.Protocol):
body = self._bodybuf.getvalue()
if reason.check(ResponseDone):
self._finished.callback(
(self._txresponse, body, None, self._certificate, self._ip_address)
)
self._finished.callback({
"txresponse": self._txresponse,
"body": body,
"flags": None,
"certificate": self._certificate,
"ip_address": self._ip_address,
})
return
if reason.check(PotentialDataLoss):
self._finished.callback(
(self._txresponse, body, ['partial'], self._certificate, self._ip_address)
)
self._finished.callback({
"txresponse": self._txresponse,
"body": body,
"flags": ["partial"],
"certificate": self._certificate,
"ip_address": self._ip_address,
})
return
if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
if not self._fail_on_dataloss:
self._finished.callback(
(self._txresponse, body, ['dataloss'], self._certificate, self._ip_address)
)
self._finished.callback({
"txresponse": self._txresponse,
"body": body,
"flags": ["dataloss"],
"certificate": self._certificate,
"ip_address": self._ip_address,
})
return
elif not self._fail_on_dataloss_warned: