mirror of https://github.com/scrapy/scrapy.git
99 lines
3.2 KiB
Python
99 lines
3.2 KiB
Python
from __future__ import annotations
|
|
|
|
from ipaddress import ip_address
|
|
|
|
from scrapy import Request
|
|
from scrapy.http import HtmlResponse, Response, TextResponse
|
|
from scrapy.utils.response import response_from_dict
|
|
|
|
|
|
class CustomResponse(TextResponse):
|
|
attributes: tuple[str, ...] = (*TextResponse.attributes, "custom")
|
|
|
|
def __init__(self, *args, custom: str | None = None, **kwargs):
|
|
self.custom = custom
|
|
super().__init__(*args, **kwargs)
|
|
|
|
|
|
class DynamicResponse(Response):
|
|
"""Response subclass that extends
|
|
:attr:`~scrapy.http.Response.attributes` on instances, as plugins that
|
|
support several Scrapy versions do."""
|
|
|
|
def __init__(self, *args, custom: str | None = None, **kwargs):
|
|
self.custom = custom
|
|
super().__init__(*args, **kwargs)
|
|
self.attributes = (*self.attributes, "custom")
|
|
|
|
|
|
def test_basic() -> None:
|
|
"""The class of plain responses is not stored, it is guessed back from the
|
|
response data."""
|
|
response = Response("https://example.com", body=b"\x00\x01")
|
|
assert "_class" not in response.to_dict()
|
|
response2 = response_from_dict(response.to_dict())
|
|
assert response2.__class__ is Response
|
|
assert response2.url == response.url
|
|
|
|
|
|
def test_all_attributes() -> None:
|
|
response = HtmlResponse(
|
|
url="https://example.com",
|
|
status=201,
|
|
headers={"Content-Type": "text/html; charset=latin-1"},
|
|
body=b"\xa3",
|
|
flags=["testFlag"],
|
|
encoding="latin-1",
|
|
ip_address=ip_address("127.0.0.1"),
|
|
protocol="h2",
|
|
)
|
|
response2 = response_from_dict(response.to_dict())
|
|
assert response2.__class__ is HtmlResponse
|
|
for attribute in HtmlResponse.attributes:
|
|
if attribute in {"request", "certificate"}:
|
|
continue
|
|
assert getattr(response2, attribute) == getattr(response, attribute)
|
|
|
|
|
|
def test_custom_attributes() -> None:
|
|
response = CustomResponse("https://example.com", custom="value")
|
|
response2 = response_from_dict(response.to_dict())
|
|
assert isinstance(response2, CustomResponse)
|
|
assert response2.custom == "value"
|
|
|
|
|
|
def test_custom_instance_attributes() -> None:
|
|
response = DynamicResponse("https://example.com", custom="value")
|
|
response2 = response_from_dict(response.to_dict())
|
|
assert isinstance(response2, DynamicResponse)
|
|
assert response2.custom == "value"
|
|
|
|
|
|
def test_crawl_attributes_left_out() -> None:
|
|
response = Response(
|
|
"https://example.com",
|
|
request=Request("https://example.com"),
|
|
certificate=object(),
|
|
)
|
|
d = response.to_dict()
|
|
assert "request" not in d
|
|
assert "certificate" not in d
|
|
response2 = response_from_dict(d)
|
|
assert response2.request is None
|
|
assert response2.certificate is None
|
|
|
|
|
|
def test_unknown_class() -> None:
|
|
"""Dicts that do not indicate a response class, e.g. cache entries written
|
|
by older Scrapy versions, get a class based on their data."""
|
|
response2 = response_from_dict(
|
|
{
|
|
"url": "https://example.com",
|
|
"status": 200,
|
|
"headers": {b"Content-Type": [b"text/plain"]},
|
|
"body": b"foo",
|
|
}
|
|
)
|
|
assert response2.__class__ is TextResponse
|
|
assert response2.text == "foo"
|