mirror of https://github.com/scrapy/scrapy.git
cleanup tests for selectors and translators
This commit is contained in:
Elias Dorneles
parent
12579b9afa
commit
3a03ef7c08
|
|
@ -25,8 +25,6 @@ tests/test_mail.py
|
|||
tests/test_pipeline_files.py
|
||||
tests/test_pipeline_images.py
|
||||
tests/test_proxy_connect.py
|
||||
tests/test_selector_csstranslator.py
|
||||
tests/test_selector.py
|
||||
tests/test_spidermiddleware_depth.py
|
||||
tests/test_spidermiddleware_httperror.py
|
||||
tests/test_spidermiddleware_offsite.py
|
||||
|
|
|
|||
|
|
@ -12,18 +12,16 @@ from lxml import etree
|
|||
|
||||
class SelectorTestCase(unittest.TestCase):
|
||||
|
||||
sscls = Selector
|
||||
|
||||
def test_simple_selection(self):
|
||||
"""Simple selector tests"""
|
||||
body = "<p><input name='a'value='1'/><input name='b'value='2'/></p>"
|
||||
response = TextResponse(url="http://example.com", body=body)
|
||||
sel = self.sscls(response)
|
||||
body = u"<p><input name='a'value='1'/><input name='b'value='2'/></p>"
|
||||
response = TextResponse(url="http://example.com", body=body, encoding='utf-8')
|
||||
sel = Selector(response)
|
||||
|
||||
xl = sel.xpath('//input')
|
||||
self.assertEqual(2, len(xl))
|
||||
for x in xl:
|
||||
assert isinstance(x, self.sscls)
|
||||
assert isinstance(x, Selector)
|
||||
|
||||
self.assertEqual(sel.xpath('//input').extract(),
|
||||
[x.extract() for x in sel.xpath('//input')])
|
||||
|
|
@ -41,7 +39,7 @@ class SelectorTestCase(unittest.TestCase):
|
|||
def test_deprecated_root_argument(self):
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
root = etree.fromstring(u'<html/>')
|
||||
sel = self.sscls(_root=root)
|
||||
sel = Selector(_root=root)
|
||||
self.assertIs(root, sel.root)
|
||||
self.assertEqual(str(w[-1].message),
|
||||
'Argument `_root` is deprecated, use `root` instead')
|
||||
|
|
@ -50,238 +48,22 @@ class SelectorTestCase(unittest.TestCase):
|
|||
with warnings.catch_warnings(record=True) as w:
|
||||
_root = etree.fromstring(u'<xml/>')
|
||||
root = etree.fromstring(u'<html/>')
|
||||
sel = self.sscls(_root=_root, root=root)
|
||||
sel = Selector(_root=_root, root=root)
|
||||
self.assertIs(root, sel.root)
|
||||
self.assertIn('Ignoring deprecated `_root` argument', str(w[-1].message))
|
||||
|
||||
def test_representation_slice(self):
|
||||
body = u"<p><input name='{}' value='\xa9'/></p>".format(50 * 'b')
|
||||
response = TextResponse(url="http://example.com", body=body, encoding='utf8')
|
||||
sel = self.sscls(response)
|
||||
|
||||
self.assertEqual(
|
||||
map(repr, sel.xpath('//input/@name')),
|
||||
["<Selector xpath='//input/@name' data=u'{}'>".format(40 * 'b')]
|
||||
)
|
||||
|
||||
def test_representation_unicode_query(self):
|
||||
body = u"<p><input name='{}' value='\xa9'/></p>".format(50 * 'b')
|
||||
response = TextResponse(url="http://example.com", body=body, encoding='utf8')
|
||||
sel = self.sscls(response)
|
||||
self.assertEqual(
|
||||
map(repr, sel.xpath(u'//input[@value="\xa9"]/@value')),
|
||||
["<Selector xpath=u'//input[@value=\"\\xa9\"]/@value' data=u'\\xa9'>"]
|
||||
)
|
||||
|
||||
def test_extract_first(self):
|
||||
"""Test if extract_first() returns first element"""
|
||||
body = '<ul><li id="1">1</li><li id="2">2</li></ul>'
|
||||
response = TextResponse(url="http://example.com", body=body)
|
||||
sel = self.sscls(response)
|
||||
|
||||
self.assertEqual(sel.xpath('//ul/li/text()').extract_first(),
|
||||
sel.xpath('//ul/li/text()').extract()[0])
|
||||
|
||||
self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').extract_first(),
|
||||
sel.xpath('//ul/li[@id="1"]/text()').extract()[0])
|
||||
|
||||
self.assertEqual(sel.xpath('//ul/li[2]/text()').extract_first(),
|
||||
sel.xpath('//ul/li/text()').extract()[1])
|
||||
|
||||
self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').extract_first(), None)
|
||||
|
||||
def test_extract_first_default(self):
|
||||
"""Test if extract_first() returns default value when no results found"""
|
||||
body = '<ul><li id="1">1</li><li id="2">2</li></ul>'
|
||||
response = TextResponse(url="http://example.com", body=body)
|
||||
sel = self.sscls(response)
|
||||
|
||||
self.assertEqual(sel.xpath('//div/text()').extract_first(default='missing'), 'missing')
|
||||
|
||||
def test_re_first(self):
|
||||
"""Test if re_first() returns first matched element"""
|
||||
body = '<ul><li id="1">1</li><li id="2">2</li></ul>'
|
||||
response = TextResponse(url="http://example.com", body=body)
|
||||
sel = self.sscls(response)
|
||||
|
||||
self.assertEqual(sel.xpath('//ul/li/text()').re_first('\d'),
|
||||
sel.xpath('//ul/li/text()').re('\d')[0])
|
||||
|
||||
self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').re_first('\d'),
|
||||
sel.xpath('//ul/li[@id="1"]/text()').re('\d')[0])
|
||||
|
||||
self.assertEqual(sel.xpath('//ul/li[2]/text()').re_first('\d'),
|
||||
sel.xpath('//ul/li/text()').re('\d')[1])
|
||||
|
||||
self.assertEqual(sel.xpath('/ul/li/text()').re_first('\w+'), None)
|
||||
self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').re_first('\d'), None)
|
||||
|
||||
def test_select_unicode_query(self):
|
||||
body = u"<p><input name='\xa9' value='1'/></p>"
|
||||
response = TextResponse(url="http://example.com", body=body, encoding='utf8')
|
||||
sel = self.sscls(response)
|
||||
self.assertEqual(sel.xpath(u'//input[@name="\xa9"]/@value').extract(), [u'1'])
|
||||
|
||||
def test_list_elements_type(self):
|
||||
"""Test Selector returning the same type in selection methods"""
|
||||
text = '<p>test<p>'
|
||||
assert isinstance(self.sscls(text=text).xpath("//p")[0], self.sscls)
|
||||
assert isinstance(self.sscls(text=text).css("p")[0], self.sscls)
|
||||
|
||||
def test_boolean_result(self):
|
||||
body = "<p><input name='a'value='1'/><input name='b'value='2'/></p>"
|
||||
response = TextResponse(url="http://example.com", body=body)
|
||||
xs = self.sscls(response)
|
||||
self.assertEquals(xs.xpath("//input[@name='a']/@name='a'").extract(), [u'1'])
|
||||
self.assertEquals(xs.xpath("//input[@name='a']/@name='n'").extract(), [u'0'])
|
||||
|
||||
def test_differences_parsing_xml_vs_html(self):
|
||||
"""Test that XML and HTML Selector's behave differently"""
|
||||
# some text which is parsed differently by XML and HTML flavors
|
||||
text = '<div><img src="a.jpg"><p>Hello</div>'
|
||||
hs = self.sscls(text=text, type='html')
|
||||
self.assertEqual(hs.xpath("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></div>'])
|
||||
|
||||
xs = self.sscls(text=text, type='xml')
|
||||
self.assertEqual(xs.xpath("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
|
||||
|
||||
def test_flavor_detection(self):
|
||||
text = '<div><img src="a.jpg"><p>Hello</div>'
|
||||
sel = self.sscls(XmlResponse('http://example.com', body=text))
|
||||
text = u'<div><img src="a.jpg"><p>Hello</div>'
|
||||
sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
|
||||
self.assertEqual(sel.type, 'xml')
|
||||
self.assertEqual(sel.xpath("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
|
||||
|
||||
sel = self.sscls(HtmlResponse('http://example.com', body=text))
|
||||
sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))
|
||||
self.assertEqual(sel.type, 'html')
|
||||
self.assertEqual(sel.xpath("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></div>'])
|
||||
|
||||
def test_nested_selectors(self):
|
||||
"""Nested selector tests"""
|
||||
body = """<body>
|
||||
<div class='one'>
|
||||
<ul>
|
||||
<li>one</li><li>two</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class='two'>
|
||||
<ul>
|
||||
<li>four</li><li>five</li><li>six</li>
|
||||
</ul>
|
||||
</div>
|
||||
</body>"""
|
||||
|
||||
response = HtmlResponse(url="http://example.com", body=body)
|
||||
x = self.sscls(response)
|
||||
divtwo = x.xpath('//div[@class="two"]')
|
||||
self.assertEqual(divtwo.xpath("//li").extract(),
|
||||
["<li>one</li>", "<li>two</li>", "<li>four</li>", "<li>five</li>", "<li>six</li>"])
|
||||
self.assertEqual(divtwo.xpath("./ul/li").extract(),
|
||||
["<li>four</li>", "<li>five</li>", "<li>six</li>"])
|
||||
self.assertEqual(divtwo.xpath(".//li").extract(),
|
||||
["<li>four</li>", "<li>five</li>", "<li>six</li>"])
|
||||
self.assertEqual(divtwo.xpath("./li").extract(), [])
|
||||
|
||||
def test_mixed_nested_selectors(self):
|
||||
body = '''<body>
|
||||
<div id=1>not<span>me</span></div>
|
||||
<div class="dos"><p>text</p><a href='#'>foo</a></div>
|
||||
</body>'''
|
||||
sel = self.sscls(text=body)
|
||||
self.assertEqual(sel.xpath('//div[@id="1"]').css('span::text').extract(), [u'me'])
|
||||
self.assertEqual(sel.css('#1').xpath('./span/text()').extract(), [u'me'])
|
||||
|
||||
def test_dont_strip(self):
|
||||
sel = self.sscls(text='<div>fff: <a href="#">zzz</a></div>')
|
||||
self.assertEqual(sel.xpath("//text()").extract(), [u'fff: ', u'zzz'])
|
||||
|
||||
def test_namespaces_simple(self):
|
||||
body = """
|
||||
<test xmlns:somens="http://scrapy.org">
|
||||
<somens:a id="foo">take this</a>
|
||||
<a id="bar">found</a>
|
||||
</test>
|
||||
"""
|
||||
|
||||
response = XmlResponse(url="http://example.com", body=body)
|
||||
x = self.sscls(response)
|
||||
|
||||
x.register_namespace("somens", "http://scrapy.org")
|
||||
self.assertEqual(x.xpath("//somens:a/text()").extract(),
|
||||
[u'take this'])
|
||||
|
||||
def test_namespaces_multiple(self):
|
||||
body = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<BrowseNode xmlns="http://webservices.amazon.com/AWSECommerceService/2005-10-05"
|
||||
xmlns:b="http://somens.com"
|
||||
xmlns:p="http://www.scrapy.org/product" >
|
||||
<b:Operation>hello</b:Operation>
|
||||
<TestTag b:att="value"><Other>value</Other></TestTag>
|
||||
<p:SecondTestTag><material>iron</material><price>90</price><p:name>Dried Rose</p:name></p:SecondTestTag>
|
||||
</BrowseNode>
|
||||
"""
|
||||
response = XmlResponse(url="http://example.com", body=body)
|
||||
x = self.sscls(response)
|
||||
x.register_namespace("xmlns", "http://webservices.amazon.com/AWSECommerceService/2005-10-05")
|
||||
x.register_namespace("p", "http://www.scrapy.org/product")
|
||||
x.register_namespace("b", "http://somens.com")
|
||||
self.assertEqual(len(x.xpath("//xmlns:TestTag")), 1)
|
||||
self.assertEqual(x.xpath("//b:Operation/text()").extract()[0], 'hello')
|
||||
self.assertEqual(x.xpath("//xmlns:TestTag/@b:att").extract()[0], 'value')
|
||||
self.assertEqual(x.xpath("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90')
|
||||
self.assertEqual(x.xpath("//p:SecondTestTag").xpath("./xmlns:price/text()")[0].extract(), '90')
|
||||
self.assertEqual(x.xpath("//p:SecondTestTag/xmlns:material/text()").extract()[0], 'iron')
|
||||
|
||||
def test_re(self):
|
||||
body = """<div>Name: Mary
|
||||
<ul>
|
||||
<li>Name: John</li>
|
||||
<li>Age: 10</li>
|
||||
<li>Name: Paul</li>
|
||||
<li>Age: 20</li>
|
||||
</ul>
|
||||
Age: 20
|
||||
</div>"""
|
||||
response = HtmlResponse(url="http://example.com", body=body)
|
||||
x = self.sscls(response)
|
||||
|
||||
name_re = re.compile("Name: (\w+)")
|
||||
self.assertEqual(x.xpath("//ul/li").re(name_re),
|
||||
["John", "Paul"])
|
||||
self.assertEqual(x.xpath("//ul/li").re("Age: (\d+)"),
|
||||
["10", "20"])
|
||||
|
||||
def test_re_intl(self):
|
||||
body = """<div>Evento: cumplea\xc3\xb1os</div>"""
|
||||
response = HtmlResponse(url="http://example.com", body=body, encoding='utf-8')
|
||||
x = self.sscls(response)
|
||||
self.assertEqual(x.xpath("//div").re("Evento: (\w+)"), [u'cumplea\xf1os'])
|
||||
|
||||
def test_selector_over_text(self):
|
||||
hs = self.sscls(text='<root>lala</root>')
|
||||
self.assertEqual(hs.extract(), u'<html><body><root>lala</root></body></html>')
|
||||
xs = self.sscls(text='<root>lala</root>', type='xml')
|
||||
self.assertEqual(xs.extract(), u'<root>lala</root>')
|
||||
self.assertEqual(xs.xpath('.').extract(), [u'<root>lala</root>'])
|
||||
|
||||
def test_invalid_xpath(self):
|
||||
"Test invalid xpath raises ValueError with the invalid xpath"
|
||||
response = XmlResponse(url="http://example.com", body="<html></html>")
|
||||
x = self.sscls(response)
|
||||
xpath = "//test[@foo='bar]"
|
||||
self.assertRaisesRegexp(ValueError, re.escape(xpath), x.xpath, xpath)
|
||||
|
||||
def test_invalid_xpath_unicode(self):
|
||||
"Test *Unicode* invalid xpath raises ValueError with the invalid xpath"
|
||||
response = XmlResponse(url="http://example.com", body="<html></html>")
|
||||
x = self.sscls(response)
|
||||
xpath = u"//test[@foo='\u0431ar]"
|
||||
encoded = xpath if six.PY3 else xpath.encode('unicode_escape')
|
||||
self.assertRaisesRegexp(ValueError, re.escape(encoded), x.xpath, xpath)
|
||||
|
||||
def test_http_header_encoding_precedence(self):
|
||||
# u'\xa3' = pound symbol in unicode
|
||||
# u'\xc2\xa3' = pound symbol in utf-8
|
||||
|
|
@ -297,136 +79,28 @@ class SelectorTestCase(unittest.TestCase):
|
|||
|
||||
headers = {'Content-Type': ['text/html; charset=utf-8']}
|
||||
response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
|
||||
x = self.sscls(response)
|
||||
x = Selector(response)
|
||||
self.assertEquals(x.xpath("//span[@id='blank']/text()").extract(),
|
||||
[u'\xa3'])
|
||||
|
||||
def test_empty_bodies(self):
|
||||
# shouldn't raise errors
|
||||
r1 = TextResponse('http://www.example.com', body='')
|
||||
self.sscls(r1).xpath('//text()').extract()
|
||||
|
||||
def test_null_bytes(self):
|
||||
# shouldn't raise errors
|
||||
r1 = TextResponse('http://www.example.com', \
|
||||
body='<root>pre\x00post</root>', \
|
||||
encoding='utf-8')
|
||||
self.sscls(r1).xpath('//text()').extract()
|
||||
|
||||
def test_badly_encoded_body(self):
|
||||
# \xe9 alone isn't valid utf8 sequence
|
||||
r1 = TextResponse('http://www.example.com', \
|
||||
body='<html><p>an Jos\xe9 de</p><html>', \
|
||||
body=u'<html><p>an Jos\xe9 de</p><html>', \
|
||||
encoding='utf-8')
|
||||
self.sscls(r1).xpath('//text()').extract()
|
||||
|
||||
def test_select_on_unevaluable_nodes(self):
|
||||
r = self.sscls(text=u'<span class="big">some text</span>')
|
||||
# Text node
|
||||
x1 = r.xpath('//text()')
|
||||
self.assertEquals(x1.extract(), [u'some text'])
|
||||
self.assertEquals(x1.xpath('.//b').extract(), [])
|
||||
# Tag attribute
|
||||
x1 = r.xpath('//span/@class')
|
||||
self.assertEquals(x1.extract(), [u'big'])
|
||||
self.assertEquals(x1.xpath('.//text()').extract(), [])
|
||||
|
||||
def test_select_on_text_nodes(self):
|
||||
r = self.sscls(text=u'<div><b>Options:</b>opt1</div><div><b>Other</b>opt2</div>')
|
||||
x1 = r.xpath("//div/descendant::text()[preceding-sibling::b[contains(text(), 'Options')]]")
|
||||
self.assertEquals(x1.extract(), [u'opt1'])
|
||||
|
||||
x1 = r.xpath("//div/descendant::text()/preceding-sibling::b[contains(text(), 'Options')]")
|
||||
self.assertEquals(x1.extract(), [u'<b>Options:</b>'])
|
||||
|
||||
def test_nested_select_on_text_nodes(self):
|
||||
# FIXME: does not work with lxml backend [upstream]
|
||||
r = self.sscls(text=u'<div><b>Options:</b>opt1</div><div><b>Other</b>opt2</div>')
|
||||
x1 = r.xpath("//div/descendant::text()")
|
||||
x2 = x1.xpath("./preceding-sibling::b[contains(text(), 'Options')]")
|
||||
self.assertEquals(x2.extract(), [u'<b>Options:</b>'])
|
||||
test_nested_select_on_text_nodes.skip = "Text nodes lost parent node reference in lxml"
|
||||
Selector(r1).xpath('//text()').extract()
|
||||
|
||||
def test_weakref_slots(self):
|
||||
"""Check that classes are using slots and are weak-referenceable"""
|
||||
x = self.sscls(text='')
|
||||
x = Selector(text='')
|
||||
weakref.ref(x)
|
||||
assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
|
||||
x.__class__.__name__
|
||||
|
||||
def test_remove_namespaces(self):
|
||||
xml = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en-US" xmlns:media="http://search.yahoo.com/mrss/">
|
||||
<link type="text/html">
|
||||
<link type="application/atom+xml">
|
||||
</feed>
|
||||
"""
|
||||
sel = self.sscls(XmlResponse("http://example.com/feed.atom", body=xml))
|
||||
self.assertEqual(len(sel.xpath("//link")), 0)
|
||||
sel.remove_namespaces()
|
||||
self.assertEqual(len(sel.xpath("//link")), 2)
|
||||
|
||||
def test_remove_attributes_namespaces(self):
|
||||
xml = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed xmlns:atom="http://www.w3.org/2005/Atom" xml:lang="en-US" xmlns:media="http://search.yahoo.com/mrss/">
|
||||
<link atom:type="text/html">
|
||||
<link atom:type="application/atom+xml">
|
||||
</feed>
|
||||
"""
|
||||
sel = self.sscls(XmlResponse("http://example.com/feed.atom", body=xml))
|
||||
self.assertEqual(len(sel.xpath("//link/@type")), 0)
|
||||
sel.remove_namespaces()
|
||||
self.assertEqual(len(sel.xpath("//link/@type")), 2)
|
||||
|
||||
def test_smart_strings(self):
|
||||
"""Lxml smart strings return values"""
|
||||
|
||||
class SmartStringsSelector(Selector):
|
||||
_lxml_smart_strings = True
|
||||
|
||||
body = """<body>
|
||||
<div class='one'>
|
||||
<ul>
|
||||
<li>one</li><li>two</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class='two'>
|
||||
<ul>
|
||||
<li>four</li><li>five</li><li>six</li>
|
||||
</ul>
|
||||
</div>
|
||||
</body>"""
|
||||
|
||||
response = HtmlResponse(url="http://example.com", body=body)
|
||||
|
||||
# .getparent() is available for text nodes and attributes
|
||||
# only when smart_strings are on
|
||||
x = self.sscls(response)
|
||||
li_text = x.xpath('//li/text()')
|
||||
self.assertFalse(any(map(lambda e: hasattr(e._root, 'getparent'), li_text)))
|
||||
div_class = x.xpath('//div/@class')
|
||||
self.assertFalse(any(map(lambda e: hasattr(e._root, 'getparent'), div_class)))
|
||||
|
||||
x = SmartStringsSelector(response)
|
||||
li_text = x.xpath('//li/text()')
|
||||
self.assertTrue(all(map(lambda e: hasattr(e._root, 'getparent'), li_text)))
|
||||
div_class = x.xpath('//div/@class')
|
||||
self.assertTrue(all(map(lambda e: hasattr(e._root, 'getparent'), div_class)))
|
||||
|
||||
def test_xml_entity_expansion(self):
|
||||
malicious_xml = '<?xml version="1.0" encoding="ISO-8859-1"?>'\
|
||||
'<!DOCTYPE foo [ <!ELEMENT foo ANY > <!ENTITY xxe SYSTEM '\
|
||||
'"file:///etc/passwd" >]><foo>&xxe;</foo>'
|
||||
|
||||
response = XmlResponse('http://example.com', body=malicious_xml)
|
||||
sel = self.sscls(response=response)
|
||||
|
||||
self.assertEqual(sel.extract(), '<foo>&xxe;</foo>')
|
||||
|
||||
|
||||
class DeprecatedXpathSelectorTest(unittest.TestCase):
|
||||
|
||||
text = '<div><img src="a.jpg"><p>Hello</div>'
|
||||
text = u'<div><img src="a.jpg"><p>Hello</div>'
|
||||
|
||||
def test_warnings_xpathselector(self):
|
||||
cls = XPathSelector
|
||||
|
|
@ -504,147 +178,4 @@ class DeprecatedXpathSelectorTest(unittest.TestCase):
|
|||
self.assertTrue(isinstance(sel, Selector))
|
||||
self.assertTrue(isinstance(usel, Selector))
|
||||
self.assertTrue(isinstance(sel, XPathSelector))
|
||||
self.assertTrue(isinstance(usel, XPathSelector))
|
||||
|
||||
def test_xpathselector(self):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', ScrapyDeprecationWarning)
|
||||
hs = XPathSelector(text=self.text)
|
||||
self.assertEqual(hs.select("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></div>'])
|
||||
self.assertRaises(RuntimeError, hs.css, 'div')
|
||||
|
||||
def test_htmlxpathselector(self):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', ScrapyDeprecationWarning)
|
||||
hs = HtmlXPathSelector(text=self.text)
|
||||
self.assertEqual(hs.select("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></div>'])
|
||||
self.assertRaises(RuntimeError, hs.css, 'div')
|
||||
|
||||
def test_xmlxpathselector(self):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', ScrapyDeprecationWarning)
|
||||
xs = XmlXPathSelector(text=self.text)
|
||||
self.assertEqual(xs.select("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
|
||||
self.assertRaises(RuntimeError, xs.css, 'div')
|
||||
|
||||
|
||||
class ExsltTestCase(unittest.TestCase):
|
||||
|
||||
sscls = Selector
|
||||
|
||||
def test_regexp(self):
|
||||
"""EXSLT regular expression tests"""
|
||||
body = """
|
||||
<p><input name='a' value='1'/><input name='b' value='2'/></p>
|
||||
<div class="links">
|
||||
<a href="/first.html">first link</a>
|
||||
<a href="/second.html">second link</a>
|
||||
<a href="http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml">EXSLT match example</a>
|
||||
</div>
|
||||
"""
|
||||
response = TextResponse(url="http://example.com", body=body)
|
||||
sel = self.sscls(response)
|
||||
|
||||
# re:test()
|
||||
self.assertEqual(
|
||||
sel.xpath(
|
||||
'//input[re:test(@name, "[A-Z]+", "i")]').extract(),
|
||||
[x.extract() for x in sel.xpath('//input[re:test(@name, "[A-Z]+", "i")]')])
|
||||
self.assertEqual(
|
||||
[x.extract()
|
||||
for x in sel.xpath(
|
||||
'//a[re:test(@href, "\.html$")]/text()')],
|
||||
[u'first link', u'second link'])
|
||||
self.assertEqual(
|
||||
[x.extract()
|
||||
for x in sel.xpath(
|
||||
'//a[re:test(@href, "first")]/text()')],
|
||||
[u'first link'])
|
||||
self.assertEqual(
|
||||
[x.extract()
|
||||
for x in sel.xpath(
|
||||
'//a[re:test(@href, "second")]/text()')],
|
||||
[u'second link'])
|
||||
|
||||
|
||||
# re:match() is rather special: it returns a node-set of <match> nodes
|
||||
#[u'<match>http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml</match>',
|
||||
#u'<match>http</match>',
|
||||
#u'<match>www.bayes.co.uk</match>',
|
||||
#u'<match></match>',
|
||||
#u'<match>/xml/index.xml?/xml/utils/rechecker.xml</match>']
|
||||
self.assertEqual(
|
||||
sel.xpath('re:match(//a[re:test(@href, "\.xml$")]/@href,'
|
||||
'"(\w+):\/\/([^/:]+)(:\d*)?([^# ]*)")/text()').extract(),
|
||||
[u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml',
|
||||
u'http',
|
||||
u'www.bayes.co.uk',
|
||||
u'',
|
||||
u'/xml/index.xml?/xml/utils/rechecker.xml'])
|
||||
|
||||
|
||||
|
||||
# re:replace()
|
||||
self.assertEqual(
|
||||
sel.xpath('re:replace(//a[re:test(@href, "\.xml$")]/@href,'
|
||||
'"(\w+)://(.+)(\.xml)", "","https://\\2.html")').extract(),
|
||||
[u'https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html'])
|
||||
|
||||
def test_set(self):
|
||||
"""EXSLT set manipulation tests"""
|
||||
# microdata example from http://schema.org/Event
|
||||
body="""
|
||||
<div itemscope itemtype="http://schema.org/Event">
|
||||
<a itemprop="url" href="nba-miami-philidelphia-game3.html">
|
||||
NBA Eastern Conference First Round Playoff Tickets:
|
||||
<span itemprop="name"> Miami Heat at Philadelphia 76ers - Game 3 (Home Game 1) </span>
|
||||
</a>
|
||||
|
||||
<meta itemprop="startDate" content="2016-04-21T20:00">
|
||||
Thu, 04/21/16
|
||||
8:00 p.m.
|
||||
|
||||
<div itemprop="location" itemscope itemtype="http://schema.org/Place">
|
||||
<a itemprop="url" href="wells-fargo-center.html">
|
||||
Wells Fargo Center
|
||||
</a>
|
||||
<div itemprop="address" itemscope itemtype="http://schema.org/PostalAddress">
|
||||
<span itemprop="addressLocality">Philadelphia</span>,
|
||||
<span itemprop="addressRegion">PA</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div itemprop="offers" itemscope itemtype="http://schema.org/AggregateOffer">
|
||||
Priced from: <span itemprop="lowPrice">$35</span>
|
||||
<span itemprop="offerCount">1938</span> tickets left
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
response = TextResponse(url="http://example.com", body=body)
|
||||
sel = self.sscls(response)
|
||||
|
||||
self.assertEqual(
|
||||
sel.xpath('''//div[@itemtype="http://schema.org/Event"]
|
||||
//@itemprop''').extract(),
|
||||
[u'url',
|
||||
u'name',
|
||||
u'startDate',
|
||||
u'location',
|
||||
u'url',
|
||||
u'address',
|
||||
u'addressLocality',
|
||||
u'addressRegion',
|
||||
u'offers',
|
||||
u'lowPrice',
|
||||
u'offerCount']
|
||||
)
|
||||
|
||||
self.assertEqual(sel.xpath('''
|
||||
set:difference(//div[@itemtype="http://schema.org/Event"]
|
||||
//@itemprop,
|
||||
//div[@itemtype="http://schema.org/Event"]
|
||||
//*[@itemscope]/*/@itemprop)''').extract(),
|
||||
[u'url', u'name', u'startDate', u'location', u'offers'])
|
||||
self.assertTrue(isinstance(usel, XPathSelector))
|
||||
|
|
@ -1,153 +1,22 @@
|
|||
"""
|
||||
Selector tests for cssselect backend
|
||||
"""
|
||||
import warnings
|
||||
from twisted.trial import unittest
|
||||
from scrapy.http import HtmlResponse
|
||||
from scrapy.selector.csstranslator import ScrapyHTMLTranslator
|
||||
from scrapy.selector import Selector
|
||||
from cssselect.parser import SelectorSyntaxError
|
||||
from cssselect.xpath import ExpressionError
|
||||
from scrapy.selector.csstranslator import (
|
||||
ScrapyHTMLTranslator,
|
||||
ScrapyGenericTranslator,
|
||||
ScrapyXPathExpr
|
||||
)
|
||||
|
||||
|
||||
HTMLBODY = b'''
|
||||
<html>
|
||||
<body>
|
||||
<div>
|
||||
<a id="name-anchor" name="foo"></a>
|
||||
<a id="tag-anchor" rel="tag" href="http://localhost/foo">link</a>
|
||||
<a id="nofollow-anchor" rel="nofollow" href="https://example.org"> link</a>
|
||||
<p id="paragraph">
|
||||
lorem ipsum text
|
||||
<b id="p-b">hi</b> <em id="p-em">there</em>
|
||||
<b id="p-b2">guy</b>
|
||||
<input type="checkbox" id="checkbox-unchecked" />
|
||||
<input type="checkbox" id="checkbox-disabled" disabled="" />
|
||||
<input type="text" id="text-checked" checked="checked" />
|
||||
<input type="hidden" />
|
||||
<input type="hidden" disabled="disabled" />
|
||||
<input type="checkbox" id="checkbox-checked" checked="checked" />
|
||||
<input type="checkbox" id="checkbox-disabled-checked"
|
||||
disabled="disabled" checked="checked" />
|
||||
<fieldset id="fieldset" disabled="disabled">
|
||||
<input type="checkbox" id="checkbox-fieldset-disabled" />
|
||||
<input type="hidden" />
|
||||
</fieldset>
|
||||
</p>
|
||||
<map name="dummymap">
|
||||
<area shape="circle" coords="200,250,25" href="foo.html" id="area-href" />
|
||||
<area shape="default" id="area-nohref" />
|
||||
</map>
|
||||
</div>
|
||||
<div class="cool-footer" id="foobar-div" foobar="ab bc cde">
|
||||
<span id="foobar-span">foo ter</span>
|
||||
</div>
|
||||
</body></html>
|
||||
'''
|
||||
class DeprecatedClassesTest(unittest.TestCase):
|
||||
|
||||
def test_deprecated_warnings(self):
|
||||
for cls in [ScrapyHTMLTranslator, ScrapyGenericTranslator, ScrapyXPathExpr]:
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
obj = cls()
|
||||
self.assertIn('%s is deprecated' % cls.__name__, str(w[-1].message),
|
||||
'Missing deprecate warning for %s' % cls.__name__)
|
||||
|
||||
|
||||
class TranslatorMixinTest(unittest.TestCase):
|
||||
|
||||
tr_cls = ScrapyHTMLTranslator
|
||||
|
||||
def setUp(self):
|
||||
self.tr = self.tr_cls()
|
||||
self.c2x = self.tr.css_to_xpath
|
||||
|
||||
def test_attr_function(self):
|
||||
cases = [
|
||||
('::attr(name)', u'descendant-or-self::*/@name'),
|
||||
('a::attr(href)', u'descendant-or-self::a/@href'),
|
||||
('a ::attr(img)', u'descendant-or-self::a/descendant-or-self::*/@img'),
|
||||
('a > ::attr(class)', u'descendant-or-self::a/*/@class'),
|
||||
]
|
||||
for css, xpath in cases:
|
||||
self.assertEqual(self.c2x(css), xpath, css)
|
||||
|
||||
def test_attr_function_exception(self):
|
||||
cases = [
|
||||
('::attr(12)', ExpressionError),
|
||||
('::attr(34test)', ExpressionError),
|
||||
('::attr(@href)', SelectorSyntaxError),
|
||||
]
|
||||
for css, exc in cases:
|
||||
self.assertRaises(exc, self.c2x, css)
|
||||
|
||||
def test_text_pseudo_element(self):
|
||||
cases = [
|
||||
('::text', u'descendant-or-self::text()'),
|
||||
('p::text', u'descendant-or-self::p/text()'),
|
||||
('p ::text', u'descendant-or-self::p/descendant-or-self::text()'),
|
||||
('#id::text', u"descendant-or-self::*[@id = 'id']/text()"),
|
||||
('p#id::text', u"descendant-or-self::p[@id = 'id']/text()"),
|
||||
('p#id ::text', u"descendant-or-self::p[@id = 'id']/descendant-or-self::text()"),
|
||||
('p#id > ::text', u"descendant-or-self::p[@id = 'id']/*/text()"),
|
||||
('p#id ~ ::text', u"descendant-or-self::p[@id = 'id']/following-sibling::*/text()"),
|
||||
('a[href]::text', u'descendant-or-self::a[@href]/text()'),
|
||||
('a[href] ::text', u'descendant-or-self::a[@href]/descendant-or-self::text()'),
|
||||
('p::text, a::text', u"descendant-or-self::p/text() | descendant-or-self::a/text()"),
|
||||
]
|
||||
for css, xpath in cases:
|
||||
self.assertEqual(self.c2x(css), xpath, css)
|
||||
|
||||
def test_pseudo_function_exception(self):
|
||||
cases = [
|
||||
('::attribute(12)', ExpressionError),
|
||||
('::text()', ExpressionError),
|
||||
('::attr(@href)', SelectorSyntaxError),
|
||||
]
|
||||
for css, exc in cases:
|
||||
self.assertRaises(exc, self.c2x, css)
|
||||
|
||||
def test_unknown_pseudo_element(self):
|
||||
cases = [
|
||||
('::text-node', ExpressionError),
|
||||
]
|
||||
for css, exc in cases:
|
||||
self.assertRaises(exc, self.c2x, css)
|
||||
|
||||
def test_unknown_pseudo_class(self):
|
||||
cases = [
|
||||
(':text', ExpressionError),
|
||||
(':attribute(name)', ExpressionError),
|
||||
]
|
||||
for css, exc in cases:
|
||||
self.assertRaises(exc, self.c2x, css)
|
||||
|
||||
|
||||
class CSSSelectorTest(unittest.TestCase):
|
||||
|
||||
sscls = Selector
|
||||
|
||||
def setUp(self):
|
||||
self.htmlresponse = HtmlResponse('http://example.com', body=HTMLBODY)
|
||||
self.sel = self.sscls(self.htmlresponse)
|
||||
|
||||
def x(self, *a, **kw):
|
||||
return [v.strip() for v in self.sel.css(*a, **kw).extract() if v.strip()]
|
||||
|
||||
def test_selector_simple(self):
|
||||
for x in self.sel.css('input'):
|
||||
self.assertTrue(isinstance(x, self.sel.__class__), x)
|
||||
self.assertEqual(self.sel.css('input').extract(),
|
||||
[x.extract() for x in self.sel.css('input')])
|
||||
|
||||
def test_text_pseudo_element(self):
|
||||
self.assertEqual(self.x('#p-b2'), [u'<b id="p-b2">guy</b>'])
|
||||
self.assertEqual(self.x('#p-b2::text'), [u'guy'])
|
||||
self.assertEqual(self.x('#p-b2 ::text'), [u'guy'])
|
||||
self.assertEqual(self.x('#paragraph::text'), [u'lorem ipsum text'])
|
||||
self.assertEqual(self.x('#paragraph ::text'), [u'lorem ipsum text', u'hi', u'there', u'guy'])
|
||||
self.assertEqual(self.x('p::text'), [u'lorem ipsum text'])
|
||||
self.assertEqual(self.x('p ::text'), [u'lorem ipsum text', u'hi', u'there', u'guy'])
|
||||
|
||||
def test_attribute_function(self):
|
||||
self.assertEqual(self.x('#p-b2::attr(id)'), [u'p-b2'])
|
||||
self.assertEqual(self.x('.cool-footer::attr(class)'), [u'cool-footer'])
|
||||
self.assertEqual(self.x('.cool-footer ::attr(id)'), [u'foobar-div', u'foobar-span'])
|
||||
self.assertEqual(self.x('map[name="dummymap"] ::attr(shape)'), [u'circle', u'default'])
|
||||
|
||||
def test_nested_selector(self):
|
||||
self.assertEqual(self.sel.css('p').css('b::text').extract(),
|
||||
[u'hi', u'guy'])
|
||||
self.assertEqual(self.sel.css('div').css('area:last-child').extract(),
|
||||
[u'<area shape="default" id="area-nohref">'])
|
||||
|
|
|
|||
Loading…
Reference in New Issue