diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 924bb7979..779f5a71c 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -392,6 +392,7 @@ class TextResponseTest(BaseResponseTest): 'http://example.com/sample2.html', 'http://example.com/sample3.html', 'http://example.com/sample3.html', + 'http://example.com/sample3.html#foo', 'http://www.google.com/something', 'http://example.com/innertag.html' ] diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 50484f060..1d7c4f311 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -13,6 +13,7 @@ from tests import get_testdata class Base: class LinkExtractorTestCase(unittest.TestCase): extractor_cls = None + escapes_whitespace = False def setUp(self): body = get_testdata('link_extractor', 'sgml_linkextractor.html') @@ -26,6 +27,11 @@ class Base: def test_extract_all_links(self): lx = self.extractor_cls() + if self.escapes_whitespace: + page4_url = 'http://example.com/page%204.html' + else: + page4_url = 'http://example.com/page 4.html' + self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), @@ -33,7 +39,7 @@ class Base: Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), + Link(url=page4_url, text=u'href with whitespaces'), ]) def test_extract_filter_allow(self): @@ -301,6 +307,11 @@ class Base: def test_attrs(self): lx = self.extractor_cls(attrs="href") + if self.escapes_whitespace: + page4_url = 'http://example.com/page%204.html' + else: + page4_url = 'http://example.com/page 4.html' + self.assertEqual(lx.extract_links(self.response), [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), @@ -308,7 +319,7 @@ class Base: Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), + Link(url=page4_url, text=u'href with whitespaces'), ]) lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=()) @@ -320,7 +331,7 @@ class Base: Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), + Link(url=page4_url, text=u'href with whitespaces'), ]) lx = self.extractor_cls(attrs=None) diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py index 794f85e0f..1366971be 100644 --- a/tests/test_linkextractors_deprecated.py +++ b/tests/test_linkextractors_deprecated.py @@ -143,6 +143,7 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase): class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase): extractor_cls = SgmlLinkExtractor + escapes_whitespace = True def test_deny_extensions(self): html = """asd and """