RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

TST fixed compatibility with new link extractor whitespace handling

This commit is contained in:
Mikhail Korobov 2017-02-21 00:05:40 +05:00
parent df446d167f
commit 2b4d46315f
3 changed files with 16 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tests/test_http_response.py
View File

@ -392,6 +392,7 @@ class TextResponseTest(BaseResponseTest):
'http://example.com/sample2.html',
'http://example.com/sample3.html',
'http://example.com/sample3.html',
'http://example.com/sample3.html#foo',
'http://www.google.com/something',
'http://example.com/innertag.html'
]

17
tests/test_linkextractors.py
View File

@ -13,6 +13,7 @@ from tests import get_testdata
class Base:
class LinkExtractorTestCase(unittest.TestCase):
extractor_cls = None
escapes_whitespace = False
def setUp(self):
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
@ -26,6 +27,11 @@ class Base:
def test_extract_all_links(self):
lx = self.extractor_cls()
if self.escapes_whitespace:
page4_url = 'http://example.com/page%204.html'
else:
page4_url = 'http://example.com/page 4.html'
self.assertEqual([link for link in lx.extract_links(self.response)], [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2'),
@ -33,7 +39,7 @@ class Base:
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
Link(url=page4_url, text=u'href with whitespaces'),
])
def test_extract_filter_allow(self):
@ -301,6 +307,11 @@ class Base:
def test_attrs(self):
lx = self.extractor_cls(attrs="href")
if self.escapes_whitespace:
page4_url = 'http://example.com/page%204.html'
else:
page4_url = 'http://example.com/page 4.html'
self.assertEqual(lx.extract_links(self.response), [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2'),
@ -308,7 +319,7 @@ class Base:
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
Link(url=page4_url, text=u'href with whitespaces'),
])
lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=())
@ -320,7 +331,7 @@ class Base:
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
Link(url=page4_url, text=u'href with whitespaces'),
])
lx = self.extractor_cls(attrs=None)

1
tests/test_linkextractors_deprecated.py
View File

@ -143,6 +143,7 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase):
class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
extractor_cls = SgmlLinkExtractor
escapes_whitespace = True
def test_deny_extensions(self):
html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""