mirror of https://github.com/scrapy/scrapy.git
handle whitespace in response.follow; add tests
This commit is contained in:
Mikhail Korobov
parent
71dd5d0bf9
commit
608c3f0c45
|
|
@ -11,6 +11,7 @@ from six.moves.urllib.parse import urljoin
|
|||
import parsel
|
||||
from w3lib.encoding import html_to_unicode, resolve_encoding, \
|
||||
html_body_declared_encoding, http_content_type_encoding
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.link import Link
|
||||
from scrapy.http.request import Request
|
||||
|
|
@ -142,7 +143,7 @@ class TextResponse(Response):
|
|||
if isinstance(url, Link):
|
||||
url = url.url
|
||||
elif isinstance(url, parsel.Selector):
|
||||
url = _url_from_selector(url).strip()
|
||||
url = _url_from_selector(url)
|
||||
elif isinstance(url, parsel.SelectorList):
|
||||
raise ValueError("SelectorList is not supported")
|
||||
|
||||
|
|
@ -164,7 +165,7 @@ def _url_from_selector(sel):
|
|||
# type: (parsel.Selector) -> str
|
||||
if isinstance(sel.root, six.string_types):
|
||||
# e.g. ::attr(href) result
|
||||
return sel.root
|
||||
return strip_html5_whitespace(sel.root)
|
||||
if not hasattr(sel.root, 'tag'):
|
||||
raise ValueError("Unsupported selector: %s" % sel)
|
||||
if sel.root.tag != 'a':
|
||||
|
|
@ -173,4 +174,4 @@ def _url_from_selector(sel):
|
|||
href = sel.root.get('href')
|
||||
if href is None:
|
||||
raise ValueError("<a> element has no href attribute: %s" % sel)
|
||||
return href
|
||||
return strip_html5_whitespace(href)
|
||||
|
|
|
|||
|
|
@ -26,9 +26,12 @@ try:
|
|||
except ImportError:
|
||||
import mock
|
||||
|
||||
tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data')
|
||||
tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
|
||||
'sample_data')
|
||||
|
||||
|
||||
def get_testdata(*paths):
|
||||
"""Return test data"""
|
||||
path = os.path.join(tests_datadir, *paths)
|
||||
return open(path, 'rb').read()
|
||||
with open(path, 'rb') as f:
|
||||
return f.read()
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import unittest
|
||||
|
||||
import six
|
||||
|
|
@ -8,6 +9,8 @@ from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
|
|||
from scrapy.selector import Selector
|
||||
from scrapy.utils.python import to_native_str
|
||||
from scrapy.exceptions import NotSupported
|
||||
from scrapy.link import Link
|
||||
from tests import get_testdata
|
||||
|
||||
|
||||
class BaseResponseTest(unittest.TestCase):
|
||||
|
|
@ -356,6 +359,11 @@ class HtmlResponseTest(TextResponseTest):
|
|||
|
||||
response_class = HtmlResponse
|
||||
|
||||
def _links_response(self):
|
||||
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
|
||||
resp = self.response_class('http://example.com/index', body=body)
|
||||
return resp
|
||||
|
||||
def test_html_encoding(self):
|
||||
|
||||
body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
|
||||
|
|
@ -388,6 +396,103 @@ class HtmlResponseTest(TextResponseTest):
|
|||
r1 = self.response_class("http://www.example.com", body=body)
|
||||
self._assert_response_values(r1, 'gb2312', body)
|
||||
|
||||
def assert_followed_url(self, follow_obj, target_url, response=None):
|
||||
if response is None:
|
||||
response = self._links_response()
|
||||
req = response.follow(follow_obj)
|
||||
self.assertEqual(req.url, target_url)
|
||||
return req
|
||||
|
||||
def test_follow_url_absolute(self):
|
||||
self.assert_followed_url('http://foo.example.com',
|
||||
'http://foo.example.com')
|
||||
|
||||
def test_follow_url_relative(self):
|
||||
self.assert_followed_url('foo',
|
||||
'http://example.com/foo')
|
||||
|
||||
def test_follow_link(self):
|
||||
self.assert_followed_url(Link('http://example.com/foo'),
|
||||
'http://example.com/foo')
|
||||
|
||||
def test_follow_selector(self):
|
||||
resp = self._links_response()
|
||||
urls = [
|
||||
'http://example.com/sample2.html',
|
||||
'http://example.com/sample3.html',
|
||||
'http://example.com/sample3.html',
|
||||
'http://www.google.com/something',
|
||||
'http://example.com/innertag.html'
|
||||
]
|
||||
|
||||
# select <a> elements
|
||||
for sellist in [resp.css('a'), resp.xpath('//a')]:
|
||||
for sel, url in zip(sellist, urls):
|
||||
self.assert_followed_url(sel, url, response=resp)
|
||||
|
||||
# href attributes should work
|
||||
for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]:
|
||||
for sel, url in zip(sellist, urls):
|
||||
self.assert_followed_url(sel, url, response=resp)
|
||||
|
||||
# non-a elements are not supported
|
||||
self.assertRaises(ValueError, resp.follow, resp.css('div')[0])
|
||||
|
||||
def test_follow_selector_list(self):
|
||||
resp = self._links_response()
|
||||
self.assertRaisesRegex(ValueError, 'SelectorList',
|
||||
resp.follow, resp.css('a'))
|
||||
|
||||
def test_follow_selector_attribute(self):
|
||||
resp = self._links_response()
|
||||
for src in resp.css('img::attr(src)'):
|
||||
self.assert_followed_url(src, 'http://example.com/sample2.jpg')
|
||||
|
||||
def test_follow_whitespace_url(self):
|
||||
self.assert_followed_url('foo ',
|
||||
'http://example.com/foo%20')
|
||||
|
||||
def test_follow_whitespace_link(self):
|
||||
self.assert_followed_url(Link('http://example.com/foo '),
|
||||
'http://example.com/foo%20')
|
||||
|
||||
def test_follow_whitespace_selector(self):
|
||||
resp = self.response_class(
|
||||
'http://example.com',
|
||||
body=b'''<html><body><a href=" foo\n">click me</a></body></html>'''
|
||||
)
|
||||
self.assert_followed_url(resp.css('a')[0],
|
||||
'http://example.com/foo',
|
||||
response=resp)
|
||||
self.assert_followed_url(resp.css('a::attr(href)')[0],
|
||||
'http://example.com/foo',
|
||||
response=resp)
|
||||
|
||||
def test_follow_encoding(self):
|
||||
resp1 = self.response_class(
|
||||
'http://example.com',
|
||||
encoding='utf8',
|
||||
body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
|
||||
)
|
||||
req = self.assert_followed_url(
|
||||
resp1.css('a')[0],
|
||||
'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82',
|
||||
response=resp1,
|
||||
)
|
||||
self.assertEqual(req.encoding, 'utf8')
|
||||
|
||||
resp2 = self.response_class(
|
||||
'http://example.com',
|
||||
encoding='cp1251',
|
||||
body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
|
||||
)
|
||||
req = self.assert_followed_url(
|
||||
resp2.css('a')[0],
|
||||
'http://example.com/foo?%EF%F0%E8%E2%E5%F2',
|
||||
response=resp2,
|
||||
)
|
||||
self.assertEqual(req.encoding, 'cp1251')
|
||||
|
||||
|
||||
class XmlResponseTest(TextResponseTest):
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue