diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 924bb7979..779f5a71c 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -392,6 +392,7 @@ class TextResponseTest(BaseResponseTest):
'http://example.com/sample2.html',
'http://example.com/sample3.html',
'http://example.com/sample3.html',
+ 'http://example.com/sample3.html#foo',
'http://www.google.com/something',
'http://example.com/innertag.html'
]
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 50484f060..1d7c4f311 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -13,6 +13,7 @@ from tests import get_testdata
class Base:
class LinkExtractorTestCase(unittest.TestCase):
extractor_cls = None
+ escapes_whitespace = False
def setUp(self):
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
@@ -26,6 +27,11 @@ class Base:
def test_extract_all_links(self):
lx = self.extractor_cls()
+ if self.escapes_whitespace:
+ page4_url = 'http://example.com/page%204.html'
+ else:
+ page4_url = 'http://example.com/page 4.html'
+
self.assertEqual([link for link in lx.extract_links(self.response)], [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2'),
@@ -33,7 +39,7 @@ class Base:
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
- Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
+ Link(url=page4_url, text=u'href with whitespaces'),
])
def test_extract_filter_allow(self):
@@ -301,6 +307,11 @@ class Base:
def test_attrs(self):
lx = self.extractor_cls(attrs="href")
+ if self.escapes_whitespace:
+ page4_url = 'http://example.com/page%204.html'
+ else:
+ page4_url = 'http://example.com/page 4.html'
+
self.assertEqual(lx.extract_links(self.response), [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2'),
@@ -308,7 +319,7 @@ class Base:
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
- Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
+ Link(url=page4_url, text=u'href with whitespaces'),
])
lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=())
@@ -320,7 +331,7 @@ class Base:
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
- Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
+ Link(url=page4_url, text=u'href with whitespaces'),
])
lx = self.extractor_cls(attrs=None)
diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py
index 794f85e0f..1366971be 100644
--- a/tests/test_linkextractors_deprecated.py
+++ b/tests/test_linkextractors_deprecated.py
@@ -143,6 +143,7 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase):
class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
extractor_cls = SgmlLinkExtractor
+ escapes_whitespace = True
def test_deny_extensions(self):
html = """asd and """