RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add Response.urljoin() testcase 

and add evaluation of base-url for HtmlResponse.
This commit is contained in:
nyov 2015-03-24 11:42:33 +00:00
parent cda3922507
commit 84edc2ebc7
4 changed files with 33 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/http/response/__init__.py
View File

@ -55,7 +55,7 @@ class Response(object_ref):
elif body is None:
self._body = ''
else:
raise TypeError("Response body must either str or unicode. Got: '%s'" \
raise TypeError("Response body must either be str or unicode. Got: '%s'" \
% type(body).__name__)
body = property(_get_body, obsolete_setter(_set_body, 'body'))

8
scrapy/http/response/text.py
View File

@ -5,9 +5,12 @@ discovering (through HTTP headers) to base Response class.
See documentation in docs/topics/request-response.rst
"""
from six.moves.urllib.parse import urljoin
from w3lib.encoding import html_to_unicode, resolve_encoding, \
html_body_declared_encoding, http_content_type_encoding
from scrapy.http.response import Response
from scrapy.utils.response import get_base_url
from scrapy.utils.python import memoizemethod_noargs
@ -63,6 +66,11 @@ class TextResponse(Response):
self._cached_ubody = html_to_unicode(charset, self.body)[1]
return self._cached_ubody
def urljoin(self, url):
"""Join this Response's url with a possible relative url to form an
absolute interpretation of the latter."""
return urljoin(get_base_url(self), url)
@memoizemethod_noargs
def _headers_encoding(self):
content_type = self.headers.get('Content-Type')

2
scrapy/utils/response.py
View File

@ -13,7 +13,6 @@ from twisted.web import http
from twisted.web.http import RESPONSES
from w3lib import html
from scrapy.http import HtmlResponse, TextResponse
from scrapy.utils.decorator import deprecated
@ -73,6 +72,7 @@ def open_in_browser(response, _openfunc=webbrowser.open):
"""Open the given response in a local web browser, populating the <base>
tag for external links to work
"""
from scrapy.http import HtmlResponse, TextResponse
# XXX: this implementation is a bit dirty and could be improved
body = response.body
if isinstance(response, HtmlResponse):

23
tests/test_http_response.py
View File

@ -113,6 +113,12 @@ class BaseResponseTest(unittest.TestCase):
self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com')
self.assertRaises(AttributeError, setattr, r, 'body', 'xxx')
def test_urljoin(self):
"""Test urljoin shortcut (only for existence, since behavior equals urljoin)"""
joined = self.response_class('http://www.example.com').urljoin('/test')
absolute = 'http://www.example.com/test'
self.assertEqual(joined, absolute)
class ResponseText(BaseResponseTest):
@ -295,6 +301,23 @@ class TextResponseTest(BaseResponseTest):
response.selector.css("title::text").extract(),
)
def test_urljoin_with_base_url(self):
"""Test urljoin shortcut which also evaluates base-url through get_base_url()."""
body = '<html><body><base href="https://example.net"></body></html>'
joined = self.response_class('http://www.example.com', body=body).urljoin('/test')
absolute = 'https://example.net/test'
self.assertEqual(joined, absolute)
body = '<html><body><base href="/elsewhere"></body></html>'
joined = self.response_class('http://www.example.com', body=body).urljoin('test')
absolute = 'http://www.example.com/test'
self.assertEqual(joined, absolute)
body = '<html><body><base href="/elsewhere/"></body></html>'
joined = self.response_class('http://www.example.com', body=body).urljoin('test')
absolute = 'http://www.example.com/elsewhere/test'
self.assertEqual(joined, absolute)
class HtmlResponseTest(TextResponseTest):