mirror of https://github.com/scrapy/scrapy.git
360 lines
11 KiB
Python
360 lines
11 KiB
Python
import os
|
|
import unittest
|
|
from urllib.parse import urlparse
|
|
|
|
import pytest
|
|
|
|
from scrapy.http import HtmlResponse, Response, TextResponse, XmlResponse
|
|
from scrapy.http.headers import Headers
|
|
from scrapy.responsetypes import ResponseTypes
|
|
from scrapy.utils.misc import load_object
|
|
from scrapy.utils.python import to_bytes
|
|
from scrapy.utils.response import (
|
|
get_meta_refresh,
|
|
get_response_class,
|
|
open_in_browser,
|
|
response_httprepr,
|
|
response_status_message,
|
|
)
|
|
|
|
|
|
__doctests__ = ['scrapy.utils.response']
|
|
|
|
|
|
# Scenarios that work the same with the previously-used, deprecated
|
|
# scrapy.responsetypes.responsetypes.from_args
|
|
PRE_XTRACTMIME_SCENARIOS = (
|
|
*(
|
|
(
|
|
{
|
|
'headers': Headers(
|
|
{'Content-Type': [mime_type]}
|
|
),
|
|
},
|
|
load_object(class_path),
|
|
)
|
|
for mime_type, class_path in ResponseTypes.CLASSES.items()
|
|
),
|
|
(
|
|
{
|
|
'url': 'http://www.example.com/data.csv',
|
|
},
|
|
TextResponse,
|
|
),
|
|
(
|
|
{
|
|
'url': 'http://www.example.com/item/',
|
|
'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}),
|
|
},
|
|
HtmlResponse,
|
|
),
|
|
(
|
|
{
|
|
'url': 'http://www.example.com/page/',
|
|
'headers': Headers(
|
|
{
|
|
'Content-Disposition': [
|
|
'attachment; filename="data.xml.gz"',
|
|
]
|
|
}
|
|
),
|
|
'body': b'\x01\x02',
|
|
},
|
|
Response,
|
|
),
|
|
(
|
|
{'body': b'Some plain\0 text data with\0 tabs and null bytes\0'},
|
|
TextResponse,
|
|
),
|
|
(
|
|
{
|
|
'body': b'\x03\x02\xdf\xdd\x23',
|
|
'headers': Headers({'Content-Encoding': 'UTF-8'}),
|
|
},
|
|
Response,
|
|
),
|
|
(
|
|
{
|
|
'body': b'\x00\x01\xff',
|
|
'url': '://www.example.com/item/',
|
|
'headers': Headers({'Content-Type': ['text/plain']}),
|
|
},
|
|
TextResponse,
|
|
),
|
|
({'url': 'http://www.example.com/item/file.html'}, HtmlResponse),
|
|
({'body': b'<html><head><title>Hello</title></head>'}, HtmlResponse),
|
|
({'body': b'<?xml version="1.0" encoding="utf-8"'}, XmlResponse),
|
|
(
|
|
{'body': b'Some plain text data\1\2 with tabs and\n null bytes\0'},
|
|
Response,
|
|
),
|
|
# https://codersblock.com/blog/the-smallest-valid-html5-page/
|
|
({'body': b'<!DOCTYPE html>\n<title>.</title>'}, HtmlResponse),
|
|
(
|
|
{
|
|
'body': b'\x01\x02',
|
|
'headers': Headers({'Content-Type': ['application/pdf']}),
|
|
},
|
|
Response,
|
|
),
|
|
(
|
|
{'headers': Headers({'Content-Type': ['application/x-json']})},
|
|
TextResponse,
|
|
),
|
|
(
|
|
{'headers': Headers({'Content-Type': ['application/x-javascript']})},
|
|
TextResponse,
|
|
),
|
|
(
|
|
{
|
|
'headers': Headers(
|
|
{'Content-Type': ['application/json-amazonui-streaming']}
|
|
)
|
|
},
|
|
TextResponse,
|
|
),
|
|
(
|
|
{
|
|
'headers': Headers(
|
|
{'Content-Disposition': ['attachment; filename="data.xml.gz"']}
|
|
),
|
|
'url': 'http://www.example.com/page/',
|
|
},
|
|
Response,
|
|
),
|
|
({'headers': Headers({'Content-Type': ['application/pdf']})}, Response),
|
|
(
|
|
{
|
|
'url': 'http://www.example.com/page/file.html',
|
|
'headers': Headers({'Content-Type': 'application/octet-stream'}),
|
|
},
|
|
HtmlResponse,
|
|
),
|
|
(
|
|
{
|
|
'url': 'http://www.example.com/item/file.xml',
|
|
'headers': Headers({'Content-Type': 'application/octet-stream'}),
|
|
},
|
|
XmlResponse,
|
|
),
|
|
(
|
|
{
|
|
'url': 'http://www.example.com/item/file.html',
|
|
'headers': Headers({'Content-Type': 'application/octet-stream'}),
|
|
},
|
|
HtmlResponse,
|
|
),
|
|
(
|
|
{
|
|
'url': 'http://www.example.com/item/file.xml',
|
|
'headers': Headers(
|
|
{
|
|
'Content-Disposition': [
|
|
'attachment; filename="data.xml.gz"'
|
|
],
|
|
'Content-Type': 'application/octet-stream',
|
|
}
|
|
),
|
|
},
|
|
XmlResponse,
|
|
),
|
|
({'url': 'http://www.example.com/item/file.pdf'}, Response),
|
|
({'filename': 'file.pdf'}, Response),
|
|
(
|
|
{
|
|
'body': b'<!DOCTYPE html>\n<title>.</title>',
|
|
'url': 'http://www.example.com',
|
|
},
|
|
HtmlResponse,
|
|
),
|
|
)
|
|
|
|
# Scenarios that work differently with the previously-used, deprecated
|
|
# scrapy.responsetypes.responsetypes.from_args
|
|
POST_XTRACTMIME_SCENARIOS = (
|
|
(
|
|
{
|
|
'body': b'\x00\x01\xff',
|
|
'url': 'http://www.example.com/item/',
|
|
'headers': Headers({'Content-Type': ['text/plain']}),
|
|
},
|
|
Response,
|
|
),
|
|
({'filename': '/tmp/temp^'}, TextResponse),
|
|
({'body': b'%PDF-1.4'}, Response),
|
|
(
|
|
{'headers': Headers({'Content-Type': ['application/ecmascript']})},
|
|
TextResponse,
|
|
),
|
|
(
|
|
{'headers': Headers({'Content-Type': ['application/ld+json']})},
|
|
TextResponse,
|
|
),
|
|
(
|
|
{
|
|
'headers': Headers(
|
|
{'Content-Encoding': ['zip'], 'Content-Type': ['text/html']}
|
|
)
|
|
},
|
|
HtmlResponse,
|
|
),
|
|
(
|
|
{
|
|
'headers': Headers(
|
|
{'Content-Encoding': ['zip'], 'Content-Type': ['text/plain']}
|
|
)
|
|
},
|
|
TextResponse,
|
|
),
|
|
(
|
|
{
|
|
'body': b'Non HTML',
|
|
'headers': Headers(
|
|
{'Content-Encoding': ['zip'], 'Content-Type': ['text/html']}
|
|
),
|
|
},
|
|
HtmlResponse,
|
|
),
|
|
(
|
|
{
|
|
'body': b'Some plain text',
|
|
'headers': Headers({'Content-Type': 'application/octet-stream'}),
|
|
},
|
|
Response,
|
|
),
|
|
({'body': b'\x0c\x1b'}, TextResponse),
|
|
({'body': b'this is not <html>'}, TextResponse),
|
|
({'body': b'this is not <?xml'}, TextResponse),
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
'kwargs,response_class',
|
|
(
|
|
*PRE_XTRACTMIME_SCENARIOS,
|
|
*POST_XTRACTMIME_SCENARIOS,
|
|
),
|
|
)
|
|
def test_get_response_class_http(kwargs, response_class):
|
|
if 'headers' in kwargs:
|
|
kwargs['http_headers'] = kwargs.pop('headers')
|
|
if 'filename' in kwargs:
|
|
assert 'url' not in kwargs
|
|
kwargs['url'] = kwargs.pop('filename')
|
|
assert get_response_class(**kwargs) == response_class
|
|
|
|
|
|
class ResponseUtilsTest(unittest.TestCase):
|
|
dummy_response = TextResponse(url='http://example.org/', body=b'dummy_response')
|
|
|
|
def test_response_httprepr(self):
|
|
r1 = Response("http://www.example.com")
|
|
self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n')
|
|
|
|
r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body=b"Some body")
|
|
self.assertEqual(response_httprepr(r1), b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')
|
|
|
|
r1 = Response("http://www.example.com", status=6666, headers={"Content-type": "text/html"}, body=b"Some body")
|
|
self.assertEqual(response_httprepr(r1), b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body')
|
|
|
|
def test_open_in_browser(self):
|
|
url = "http:///www.example.com/some/page.html"
|
|
body = b"<html> <head> <title>test page</title> </head> <body>test body</body> </html>"
|
|
|
|
def browser_open(burl):
|
|
path = urlparse(burl).path
|
|
if not os.path.exists(path):
|
|
path = burl.replace('file://', '')
|
|
with open(path, "rb") as f:
|
|
bbody = f.read()
|
|
self.assertIn(b'<base href="' + to_bytes(url) + b'">', bbody)
|
|
return True
|
|
response = HtmlResponse(url, body=body)
|
|
assert open_in_browser(response, _openfunc=browser_open), "Browser not called"
|
|
|
|
resp = Response(url, body=body)
|
|
self.assertRaises(TypeError, open_in_browser, resp, debug=True)
|
|
|
|
def test_get_meta_refresh(self):
|
|
r1 = HtmlResponse("http://www.example.com", body=b"""
|
|
<html>
|
|
<head><title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
|
|
<body>blahablsdfsal&</body>
|
|
</html>""")
|
|
r2 = HtmlResponse("http://www.example.com", body=b"""
|
|
<html>
|
|
<head><title>Dummy</title><noScript>
|
|
<meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
|
|
</noSCRIPT>
|
|
<body>blahablsdfsal&</body>
|
|
</html>""")
|
|
r3 = HtmlResponse("http://www.example.com", body=b"""
|
|
<noscript><meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage</noscript>
|
|
<script type="text/javascript">
|
|
if(!checkCookies()){
|
|
document.write('<meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage">');
|
|
}
|
|
</script>
|
|
""")
|
|
self.assertEqual(get_meta_refresh(r1), (5.0, 'http://example.org/newpage'))
|
|
self.assertEqual(get_meta_refresh(r2), (None, None))
|
|
self.assertEqual(get_meta_refresh(r3), (None, None))
|
|
|
|
def test_response_status_message(self):
|
|
self.assertEqual(response_status_message(200), '200 OK')
|
|
self.assertEqual(response_status_message(404), '404 Not Found')
|
|
self.assertEqual(response_status_message(573), "573 Unknown Status")
|
|
|
|
def test_inject_base_url(self):
|
|
url = "http://www.example.com"
|
|
|
|
def check_base_url(burl):
|
|
path = urlparse(burl).path
|
|
if not os.path.exists(path):
|
|
path = burl.replace('file://', '')
|
|
with open(path, "rb") as f:
|
|
bbody = f.read()
|
|
self.assertEqual(bbody.count(b'<base href="' + to_bytes(url) + b'">'), 1)
|
|
return True
|
|
|
|
r1 = HtmlResponse(url, body=b"""
|
|
<html>
|
|
<head><title>Dummy</title></head>
|
|
<body><p>Hello world.</p></body>
|
|
</html>""")
|
|
r2 = HtmlResponse(url, body=b"""
|
|
<html>
|
|
<head id="foo"><title>Dummy</title></head>
|
|
<body>Hello world.</body>
|
|
</html>""")
|
|
r3 = HtmlResponse(url, body=b"""
|
|
<html>
|
|
<head><title>Dummy</title></head>
|
|
<body>
|
|
<header>Hello header</header>
|
|
<p>Hello world.</p>
|
|
</body>
|
|
</html>""")
|
|
r4 = HtmlResponse(url, body=b"""
|
|
<html>
|
|
<!-- <head>Dummy comment</head> -->
|
|
<head><title>Dummy</title></head>
|
|
<body><p>Hello world.</p></body>
|
|
</html>""")
|
|
r5 = HtmlResponse(url, body=b"""
|
|
<html>
|
|
<!--[if IE]>
|
|
<head><title>IE head</title></head>
|
|
<![endif]-->
|
|
<!--[if !IE]>-->
|
|
<head><title>Standard head</title></head>
|
|
<!--<![endif]-->
|
|
<body><p>Hello world.</p></body>
|
|
</html>""")
|
|
|
|
assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url"
|
|
assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head"
|
|
assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag"
|
|
assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment"
|
|
assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment"
|