RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

TST update tests to use get/getall/attrib instead of extract

This commit is contained in:
Mikhail Korobov 2018-08-22 04:17:55 +05:00
parent 2c48d156db
commit 53da56c8dc
7 changed files with 66 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/test_command_shell.py
View File

@ -35,7 +35,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
@defer.inlineCallbacks
def test_response_selector_html(self):
xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]'
xpath = 'response.xpath("//p[@class=\'one\']/text()").get()'
_, out, _ = yield self.execute([self.url('/html'), '-c', xpath])
self.assertEqual(out.strip(), b'Works')

34
tests/test_http_response.py
View File

@ -336,11 +336,11 @@ class TextResponseTest(BaseResponseTest):
self.assertIs(response.selector.response, response)
self.assertEqual(
response.selector.xpath("//title/text()").extract(),
response.selector.xpath("//title/text()").getall(),
[u'Some page']
)
self.assertEqual(
response.selector.css("title::text").extract(),
response.selector.css("title::text").getall(),
[u'Some page']
)
self.assertEqual(
@ -353,12 +353,12 @@ class TextResponseTest(BaseResponseTest):
response = self.response_class("http://www.example.com", body=body)
self.assertEqual(
response.xpath("//title/text()").extract(),
response.selector.xpath("//title/text()").extract(),
response.xpath("//title/text()").getall(),
response.selector.xpath("//title/text()").getall(),
)
self.assertEqual(
response.css("title::text").extract(),
response.selector.css("title::text").extract(),
response.css("title::text").getall(),
response.selector.css("title::text").getall(),
)
def test_selector_shortcuts_kwargs(self):
@ -366,13 +366,13 @@ class TextResponseTest(BaseResponseTest):
response = self.response_class("http://www.example.com", body=body)
self.assertEqual(
response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").extract(),
response.xpath("normalize-space(//p[@class=\"content\"])").extract(),
response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").getall(),
response.xpath("normalize-space(//p[@class=\"content\"])").getall(),
)
self.assertEqual(
response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()",
pclass="content", pcount=1).extract(),
response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").extract(),
pclass="content", pcount=1).getall(),
response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(),
)
def test_urljoin_with_base_url(self):
@ -562,7 +562,7 @@ class XmlResponseTest(TextResponseTest):
self.assertIs(response.selector.response, response)
self.assertEqual(
response.selector.xpath("//elem/text()").extract(),
response.selector.xpath("//elem/text()").getall(),
[u'value']
)
@ -571,8 +571,8 @@ class XmlResponseTest(TextResponseTest):
response = self.response_class("http://www.example.com", body=body)
self.assertEqual(
response.xpath("//elem/text()").extract(),
response.selector.xpath("//elem/text()").extract(),
response.xpath("//elem/text()").getall(),
response.selector.xpath("//elem/text()").getall(),
)
def test_selector_shortcuts_kwargs(self):
@ -583,12 +583,12 @@ class XmlResponseTest(TextResponseTest):
response = self.response_class("http://www.example.com", body=body)
self.assertEqual(
response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(),
response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(),
)
response.selector.register_namespace('s2', 'http://scrapy.org')
self.assertEqual(
response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).extract(),
response.selector.xpath("//s2:elem/text()").extract(),
response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(),
response.selector.xpath("//s2:elem/text()").getall(),
)

4
tests/test_loader.py
View File

@ -634,7 +634,7 @@ class SubselectorLoaderTest(unittest.TestCase):
nl = l.nested_xpath("//header")
nl.add_xpath('name', 'div/text()')
nl.add_css('name_div', '#id')
nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
self.assertEqual(l.get_output_value('name'), [u'marta'])
self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
@ -649,7 +649,7 @@ class SubselectorLoaderTest(unittest.TestCase):
nl = l.nested_css("header")
nl.add_xpath('name', 'div/text()')
nl.add_css('name_div', '#id')
nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
self.assertEqual(l.get_output_value('name'), [u'marta'])
self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])

2
tests/test_pipeline_crawl.py
View File

@ -29,7 +29,7 @@ class MediaDownloadSpider(SimpleSpider):
for href in response.xpath('''
//table[thead/tr/th="Filename"]
/tbody//a/@href
''').extract()],
''').getall()],
}
yield item

24
tests/test_selector.py
View File

@ -20,17 +20,17 @@ class SelectorTestCase(unittest.TestCase):
for x in xl:
assert isinstance(x, Selector)
self.assertEqual(sel.xpath('//input').extract(),
[x.extract() for x in sel.xpath('//input')])
self.assertEqual(sel.xpath('//input').getall(),
[x.get() for x in sel.xpath('//input')])
self.assertEqual([x.extract() for x in sel.xpath("//input[@name='a']/@name")],
self.assertEqual([x.get() for x in sel.xpath("//input[@name='a']/@name")],
[u'a'])
self.assertEqual([x.extract() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
self.assertEqual([x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
[u'12.0'])
self.assertEqual(sel.xpath("concat('xpath', 'rules')").extract(),
self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(),
[u'xpathrules'])
self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
self.assertEqual([x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
[u'12'])
def test_root_base_url(self):
@ -60,12 +60,12 @@ class SelectorTestCase(unittest.TestCase):
text = b'<div><img src="a.jpg"><p>Hello</div>'
sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
self.assertEqual(sel.type, 'xml')
self.assertEqual(sel.xpath("//div").extract(),
self.assertEqual(sel.xpath("//div").getall(),
[u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))
self.assertEqual(sel.type, 'html')
self.assertEqual(sel.xpath("//div").extract(),
self.assertEqual(sel.xpath("//div").getall(),
[u'<div><img src="a.jpg"><p>Hello</p></div>'])
def test_http_header_encoding_precedence(self):
@ -84,15 +84,15 @@ class SelectorTestCase(unittest.TestCase):
headers = {'Content-Type': ['text/html; charset=utf-8']}
response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
x = Selector(response)
self.assertEqual(x.xpath("//span[@id='blank']/text()").extract(),
self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(),
[u'\xa3'])
def test_badly_encoded_body(self):
# \xe9 alone isn't valid utf8 sequence
r1 = TextResponse('http://www.example.com', \
body=b'<html><p>an Jos\xe9 de</p><html>', \
r1 = TextResponse('http://www.example.com',
body=b'<html><p>an Jos\xe9 de</p><html>',
encoding='utf-8')
Selector(r1).xpath('//text()').extract()
Selector(r1).xpath('//text()').getall()
def test_weakref_slots(self):
"""Check that classes are using slots and are weak-referenceable"""

8
tests/test_spider.py
View File

@ -147,10 +147,10 @@ class XMLFeedSpiderTest(SpiderTest):
def parse_node(self, response, selector):
yield {
'loc': selector.xpath('a:loc/text()').extract(),
'updated': selector.xpath('b:updated/text()').extract(),
'other': selector.xpath('other/@value').extract(),
'custom': selector.xpath('other/@b:custom').extract(),
'loc': selector.xpath('a:loc/text()').getall(),
'updated': selector.xpath('b:updated/text()').getall(),
'other': selector.xpath('other/@value').getall(),
'custom': selector.xpath('other/@b:custom').getall(),
}
for iterator in ('iternodes', 'xml'):

55
tests/test_utils_iterators.py
View File

@ -30,10 +30,13 @@ class XmliterTestCase(unittest.TestCase):
response = XmlResponse(url="http://example.com", body=body)
attrs = []
for x in self.xmliter(response, 'product'):
attrs.append((x.xpath("@id").extract(), x.xpath("name/text()").extract(), x.xpath("./type/text()").extract()))
attrs.append((
x.attrib['id'],
x.xpath("name/text()").extract(),
x.xpath("./type/text()").extract()))
self.assertEqual(attrs,
[(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])])
[('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])])
def test_xmliter_unusual_node(self):
body = b"""<?xml version="1.0" encoding="UTF-8"?>
@ -43,7 +46,7 @@ class XmliterTestCase(unittest.TestCase):
</root>
"""
response = XmlResponse(url="http://example.com", body=body)
nodenames = [e.xpath('name()').extract()
nodenames = [e.xpath('name()').getall()
for e in self.xmliter(response, 'matchme...')]
self.assertEqual(nodenames, [['matchme...']])
@ -93,19 +96,19 @@ class XmliterTestCase(unittest.TestCase):
attrs = []
for x in self.xmliter(r, u'þingflokkur'):
attrs.append((x.xpath('@id').extract(),
x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(),
x.xpath(u'./tímabil/fyrstaþing/text()').extract()))
attrs.append((x.attrib['id'],
x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(),
x.xpath(u'./tímabil/fyrstaþing/text()').getall()))
self.assertEqual(attrs,
[([u'26'], [u'-'], [u'80']),
([u'21'], [u'Ab'], [u'76']),
([u'27'], [u'A'], [u'27'])])
[(u'26', [u'-'], [u'80']),
(u'21', [u'Ab'], [u'76']),
(u'27', [u'A'], [u'27'])])
def test_xmliter_text(self):
body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
self.assertEqual([x.xpath("text()").extract() for x in self.xmliter(body, 'product')],
self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],
[[u'one'], [u'two']])
def test_xmliter_namespaces(self):
@ -132,15 +135,15 @@ class XmliterTestCase(unittest.TestCase):
node = next(my_iter)
node.register_namespace('g', 'http://base.google.com/ns/1.0')
self.assertEqual(node.xpath('title/text()').extract(), ['Item 1'])
self.assertEqual(node.xpath('description/text()').extract(), ['This is item 1'])
self.assertEqual(node.xpath('link/text()').extract(), ['http://www.mydummycompany.com/items/1'])
self.assertEqual(node.xpath('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
self.assertEqual(node.xpath('g:id/text()').extract(), ['ITEM_1'])
self.assertEqual(node.xpath('g:price/text()').extract(), ['400'])
self.assertEqual(node.xpath('image_link/text()').extract(), [])
self.assertEqual(node.xpath('id/text()').extract(), [])
self.assertEqual(node.xpath('price/text()').extract(), [])
self.assertEqual(node.xpath('title/text()').getall(), ['Item 1'])
self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1'])
self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1'])
self.assertEqual(node.xpath('g:image_link/text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])
self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1'])
self.assertEqual(node.xpath('g:price/text()').getall(), ['400'])
self.assertEqual(node.xpath('image_link/text()').getall(), [])
self.assertEqual(node.xpath('id/text()').getall(), [])
self.assertEqual(node.xpath('price/text()').getall(), [])
def test_xmliter_exception(self):
body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
@ -159,7 +162,7 @@ class XmliterTestCase(unittest.TestCase):
body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
response = XmlResponse('http://www.example.com', body=body)
self.assertEqual(
next(self.xmliter(response, 'item')).extract(),
next(self.xmliter(response, 'item')).get(),
u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
)
@ -192,9 +195,9 @@ class LxmlXmliterTestCase(XmliterTestCase):
namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0')
node = next(namespace_iter)
self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])
node = next(namespace_iter)
self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg'])
self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg'])
def test_xmliter_namespaces_prefix(self):
body = b"""\
@ -219,14 +222,14 @@ class LxmlXmliterTestCase(XmliterTestCase):
my_iter = self.xmliter(response, 'table', 'http://www.w3.org/TR/html4/', 'h')
node = next(my_iter)
self.assertEqual(len(node.xpath('h:tr/h:td').extract()), 2)
self.assertEqual(node.xpath('h:tr/h:td[1]/text()').extract(), ['Apples'])
self.assertEqual(node.xpath('h:tr/h:td[2]/text()').extract(), ['Bananas'])
self.assertEqual(len(node.xpath('h:tr/h:td').getall()), 2)
self.assertEqual(node.xpath('h:tr/h:td[1]/text()').getall(), ['Apples'])
self.assertEqual(node.xpath('h:tr/h:td[2]/text()').getall(), ['Bananas'])
my_iter = self.xmliter(response, 'table', 'http://www.w3schools.com/furniture', 'f')
node = next(my_iter)
self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table'])
self.assertEqual(node.xpath('f:name/text()').getall(), ['African Coffee Table'])
def test_xmliter_objtype_exception(self):
i = self.xmliter(42, 'product')