import unittest
import warnings
from itertools import chain
from pathlib import Path
from urllib.parse import urlparse
import pytest
from xtractmime import BINARY_BYTES, RESOURCE_HEADER_BUFFER_LENGTH
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Response, TextResponse, XmlResponse
from scrapy.http.headers import Headers
from scrapy.responsetypes import ResponseTypes
from scrapy.utils.misc import load_object
from scrapy.utils.python import to_bytes
from scrapy.utils.response import (
get_meta_refresh,
get_response_class,
open_in_browser,
response_httprepr,
response_status_message,
)
__doctests__ = ['scrapy.utils.response']
# https://mimesniff.spec.whatwg.org/#interpreting-the-resource-metadata
PRE_XTRACTMIME_HTML_STARTS = (
b' bytes:
"""Make odd bytes lowecase and even bytes uppercase.
>>> odd_capitalize(b'foobar')
b'fOoBaR'
"""
return b"".join(
bytes([byte]).lower() if index % 2 == 0 else bytes([byte]).upper()
for index, byte in enumerate(value)
)
# Scenarios that work the same with the previously-used, deprecated
# scrapy.responsetypes.responsetypes.from_args
PRE_XTRACTMIME_SCENARIOS = (
# Content-Type determines the type for the HTTP protocol.
*(
(
{
"url": f"{protocol}://example.com/foo",
"headers": Headers(
{"Content-Type": content_type + content_type_parameters}
),
},
response_class,
)
for protocol in ("http", "https")
# Make sure that MIME parameters do not break response class choice.
for content_type_parameters in ("", "; foo=bar")
for content_type, response_class in (
("application/octet-stream", Response),
("text/plain", TextResponse),
("text/html", HtmlResponse),
("text/xml", XmlResponse),
*(
(mime_type, load_object(class_path))
for mime_type, class_path in ResponseTypes.CLASSES.items()
if mime_type not in (
# “Note that XHTML is best parsed as XML”
# https://lxml.de/parsing.html
"application/xhtml+xml",
"application/vnd.wap.xhtml+xml",
)
),
# JavaScript MIME types should trigger a TextResponse.
#
# https://mimesniff.spec.whatwg.org/#javascript-mime-type
*(
(mime_type, TextResponse)
for mime_type in (
'application/javascript',
'application/x-javascript',
'text/ecmascript',
'text/javascript',
'text/javascript1.0',
'text/javascript1.1',
'text/javascript1.2',
'text/javascript1.3',
'text/javascript1.4',
'text/javascript1.5',
'text/jscript',
'text/livescript',
'text/x-ecmascript',
'text/x-javascript',
# Unofficial
'application/x-javascript',
)
),
# JSON MIME types should trigger a TextResponse.
#
# https://mimesniff.spec.whatwg.org/#json-mime-type
*(
(mime_type, TextResponse)
for mime_type in (
'application/json',
'text/json',
# Unofficial
'application/json-amazonui-streaming',
'application/x-json',
)
),
# Binary MIME types should trigger a Response.
#
# https://mimesniff.spec.whatwg.org/#json-mime-type
*(
(mime_type, Response)
for mime_type in (
'application/pdf',
)
),
)
),
# Content-Type triumphs body, except for:
#
# - Binary content mislabeled as plain text due to an Apache bug
# https://mimesniff.spec.whatwg.org/#check-for-apache-bug-flag
# https://mimesniff.spec.whatwg.org/#rules-for-text-or-binary
#
# - Feeds mislabeled as HTML
# https://mimesniff.spec.whatwg.org/#rules-for-distinguishing-if-a-resource-is-a-feed-or-html
*(
(
{
'body': body,
'headers': Headers({'Content-Type': [content_type]}),
},
response_class,
)
for body, content_type, response_class in (
*(
(b'\x00\x01\xff', content_type, TextResponse)
for content_type in (
'text/json',
# text/plain variants *not* affected by the Apache bug
'text/plain; charset=Iso-8859-1',
'text/plain; charset=utf-8',
'text/plain; charset=windows-1252',
)
),
)
),
# Compressed content should be of type Response until uncompressed.
(
{
'headers': Headers(
{
'Content-Encoding': ['zip'],
'Content-Type': ['text/html'],
}
)
},
Response,
),
# We take the file extension of URL paths into account, except for HTTP
# responses, because “they are unreliable and easily spoofed”.
#
# https://mimesniff.spec.whatwg.org/#interpreting-the-resource-metadata
*(
(
{'url': f'{protocol}://example.com/a.{extension}'},
response_class,
)
for protocol in ("file", "ftp")
for extension, response_class in (
("gz", Response),
("html", HtmlResponse),
("json", TextResponse),
("pdf", Response),
("txt", TextResponse),
("xml", XmlResponse),
)
),
# Unlike in a web browser, where an attachment Content-Disposition header
# causes the response to be downloaded, and hence MIME sniffing becomes
# irrelevant, in Scrapy those responses are handled the same as any, and
# hence we take the file extension from Content-Disposition into account
# to choose a response class, as a fallback when there is no Content-Type.
*(
(
{
"url": f"{protocol}://example.com/a",
'headers': Headers(
{
'Content-Disposition': [
'attachment; filename="a.xml"',
]
}
),
},
XmlResponse,
)
for protocol in ("http", "https")
),
*(
(
{
"url": f"{protocol}://example.com/a",
'headers': Headers(
{
'Content-Disposition': [
'attachment; filename="a.html"',
],
"Content-Type": "text/xml",
}
),
},
XmlResponse,
)
for protocol in ("http", "https")
),
# Without anything else, the body determines the response class.
*(
({"body": body}, response_class)
for body, response_class in (
(b'Hello', HtmlResponse),
(b'\n.', HtmlResponse),
# https://mimesniff.spec.whatwg.org/#identifying-a-resource-with-an-unknown-mime-type
*(
(prefix + start + b">", HtmlResponse)
for prefix in (
b"",
*(byte for byte in WHITESPACE_BYTES if byte != b"\x0c"),
)
for start in (
set_case(start)
for set_case in (bytes.lower, bytes.upper, odd_capitalize)
for start in PRE_XTRACTMIME_HTML_STARTS
)
),
*(
(prefix + b"", HtmlResponse)
for start in (
set_case(start)
for set_case in (bytes.lower, bytes.upper, odd_capitalize)
for start in POST_XTRACTMIME_HTML_STARTS
)
),
*(
(start + b" ", HtmlResponse)
for start in (
set_case(start)
for set_case in (bytes.lower, bytes.upper, odd_capitalize)
for start in chain(
PRE_XTRACTMIME_HTML_STARTS,
POST_XTRACTMIME_HTML_STARTS,
)
)
),
*(
(b"\x0c" + start + b">", HtmlResponse)
for start in (
set_case(start)
for set_case in (bytes.lower, bytes.upper, odd_capitalize)
for start in PRE_XTRACTMIME_HTML_STARTS
)
),
(b"\x0c', TextResponse),
(b'a test page test body "
def browser_open(burl):
path = urlparse(burl).path
if not path or not Path(path).exists():
path = burl.replace('file://', '')
bbody = Path(path).read_bytes()
self.assertIn(b'', bbody)
return True
response = HtmlResponse(url, body=body)
assert open_in_browser(response, _openfunc=browser_open), "Browser not called"
resp = Response(url, body=body)
self.assertRaises(TypeError, open_in_browser, resp, debug=True)
def test_get_meta_refresh(self):
r1 = HtmlResponse("http://www.example.com", body=b"""
Dummy
blahablsdfsal&
""")
r2 = HtmlResponse("http://www.example.com", body=b"""
Dummy
blahablsdfsal&
""")
r3 = HtmlResponse("http://www.example.com", body=b"""