diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 5bfd2debb..6eacfbd35 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -11,6 +11,7 @@ from six.moves.urllib.parse import urljoin
import parsel
from w3lib.encoding import html_to_unicode, resolve_encoding, \
html_body_declared_encoding, http_content_type_encoding
+from w3lib.html import strip_html5_whitespace
from scrapy.link import Link
from scrapy.http.request import Request
@@ -142,7 +143,7 @@ class TextResponse(Response):
if isinstance(url, Link):
url = url.url
elif isinstance(url, parsel.Selector):
- url = _url_from_selector(url).strip()
+ url = _url_from_selector(url)
elif isinstance(url, parsel.SelectorList):
raise ValueError("SelectorList is not supported")
@@ -164,7 +165,7 @@ def _url_from_selector(sel):
# type: (parsel.Selector) -> str
if isinstance(sel.root, six.string_types):
# e.g. ::attr(href) result
- return sel.root
+ return strip_html5_whitespace(sel.root)
if not hasattr(sel.root, 'tag'):
raise ValueError("Unsupported selector: %s" % sel)
if sel.root.tag != 'a':
@@ -173,4 +174,4 @@ def _url_from_selector(sel):
href = sel.root.get('href')
if href is None:
raise ValueError(" element has no href attribute: %s" % sel)
- return href
+ return strip_html5_whitespace(href)
diff --git a/tests/__init__.py b/tests/__init__.py
index d940f28ea..c2e4fd2bf 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -26,9 +26,12 @@ try:
except ImportError:
import mock
-tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data')
+tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+ 'sample_data')
+
def get_testdata(*paths):
"""Return test data"""
path = os.path.join(tests_datadir, *paths)
- return open(path, 'rb').read()
+ with open(path, 'rb') as f:
+ return f.read()
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 9df3bf6e7..2a9baf5ed 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
import unittest
import six
@@ -8,6 +9,8 @@ from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
from scrapy.selector import Selector
from scrapy.utils.python import to_native_str
from scrapy.exceptions import NotSupported
+from scrapy.link import Link
+from tests import get_testdata
class BaseResponseTest(unittest.TestCase):
@@ -356,6 +359,11 @@ class HtmlResponseTest(TextResponseTest):
response_class = HtmlResponse
+ def _links_response(self):
+ body = get_testdata('link_extractor', 'sgml_linkextractor.html')
+ resp = self.response_class('http://example.com/index', body=body)
+ return resp
+
def test_html_encoding(self):
body = b"""Some page
@@ -388,6 +396,103 @@ class HtmlResponseTest(TextResponseTest):
r1 = self.response_class("http://www.example.com", body=body)
self._assert_response_values(r1, 'gb2312', body)
+ def assert_followed_url(self, follow_obj, target_url, response=None):
+ if response is None:
+ response = self._links_response()
+ req = response.follow(follow_obj)
+ self.assertEqual(req.url, target_url)
+ return req
+
+ def test_follow_url_absolute(self):
+ self.assert_followed_url('http://foo.example.com',
+ 'http://foo.example.com')
+
+ def test_follow_url_relative(self):
+ self.assert_followed_url('foo',
+ 'http://example.com/foo')
+
+ def test_follow_link(self):
+ self.assert_followed_url(Link('http://example.com/foo'),
+ 'http://example.com/foo')
+
+ def test_follow_selector(self):
+ resp = self._links_response()
+ urls = [
+ 'http://example.com/sample2.html',
+ 'http://example.com/sample3.html',
+ 'http://example.com/sample3.html',
+ 'http://www.google.com/something',
+ 'http://example.com/innertag.html'
+ ]
+
+ # select elements
+ for sellist in [resp.css('a'), resp.xpath('//a')]:
+ for sel, url in zip(sellist, urls):
+ self.assert_followed_url(sel, url, response=resp)
+
+ # href attributes should work
+ for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]:
+ for sel, url in zip(sellist, urls):
+ self.assert_followed_url(sel, url, response=resp)
+
+ # non-a elements are not supported
+ self.assertRaises(ValueError, resp.follow, resp.css('div')[0])
+
+ def test_follow_selector_list(self):
+ resp = self._links_response()
+ self.assertRaisesRegex(ValueError, 'SelectorList',
+ resp.follow, resp.css('a'))
+
+ def test_follow_selector_attribute(self):
+ resp = self._links_response()
+ for src in resp.css('img::attr(src)'):
+ self.assert_followed_url(src, 'http://example.com/sample2.jpg')
+
+ def test_follow_whitespace_url(self):
+ self.assert_followed_url('foo ',
+ 'http://example.com/foo%20')
+
+ def test_follow_whitespace_link(self):
+ self.assert_followed_url(Link('http://example.com/foo '),
+ 'http://example.com/foo%20')
+
+ def test_follow_whitespace_selector(self):
+ resp = self.response_class(
+ 'http://example.com',
+ body=b'''click me'''
+ )
+ self.assert_followed_url(resp.css('a')[0],
+ 'http://example.com/foo',
+ response=resp)
+ self.assert_followed_url(resp.css('a::attr(href)')[0],
+ 'http://example.com/foo',
+ response=resp)
+
+ def test_follow_encoding(self):
+ resp1 = self.response_class(
+ 'http://example.com',
+ encoding='utf8',
+ body='click me'.encode('utf8')
+ )
+ req = self.assert_followed_url(
+ resp1.css('a')[0],
+ 'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82',
+ response=resp1,
+ )
+ self.assertEqual(req.encoding, 'utf8')
+
+ resp2 = self.response_class(
+ 'http://example.com',
+ encoding='cp1251',
+ body='click me'.encode('cp1251')
+ )
+ req = self.assert_followed_url(
+ resp2.css('a')[0],
+ 'http://example.com/foo?%EF%F0%E8%E2%E5%F2',
+ response=resp2,
+ )
+ self.assertEqual(req.encoding, 'cp1251')
+
class XmlResponseTest(TextResponseTest):