diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 3e27d6abd..36baacfbd 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -35,7 +35,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_response_selector_html(self): - xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]' + xpath = 'response.xpath("//p[@class=\'one\']/text()").get()' _, out, _ = yield self.execute([self.url('/html'), '-c', xpath]) self.assertEqual(out.strip(), b'Works') diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 820758dc9..3b90e3dac 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -336,11 +336,11 @@ class TextResponseTest(BaseResponseTest): self.assertIs(response.selector.response, response) self.assertEqual( - response.selector.xpath("//title/text()").extract(), + response.selector.xpath("//title/text()").getall(), [u'Some page'] ) self.assertEqual( - response.selector.css("title::text").extract(), + response.selector.css("title::text").getall(), [u'Some page'] ) self.assertEqual( @@ -353,12 +353,12 @@ class TextResponseTest(BaseResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("//title/text()").extract(), - response.selector.xpath("//title/text()").extract(), + response.xpath("//title/text()").getall(), + response.selector.xpath("//title/text()").getall(), ) self.assertEqual( - response.css("title::text").extract(), - response.selector.css("title::text").extract(), + response.css("title::text").getall(), + response.selector.css("title::text").getall(), ) def test_selector_shortcuts_kwargs(self): @@ -366,13 +366,13 @@ class TextResponseTest(BaseResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").extract(), - response.xpath("normalize-space(//p[@class=\"content\"])").extract(), + response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").getall(), + response.xpath("normalize-space(//p[@class=\"content\"])").getall(), ) self.assertEqual( response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()", - pclass="content", pcount=1).extract(), - response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").extract(), + pclass="content", pcount=1).getall(), + response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(), ) def test_urljoin_with_base_url(self): @@ -562,7 +562,7 @@ class XmlResponseTest(TextResponseTest): self.assertIs(response.selector.response, response) self.assertEqual( - response.selector.xpath("//elem/text()").extract(), + response.selector.xpath("//elem/text()").getall(), [u'value'] ) @@ -571,8 +571,8 @@ class XmlResponseTest(TextResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("//elem/text()").extract(), - response.selector.xpath("//elem/text()").extract(), + response.xpath("//elem/text()").getall(), + response.selector.xpath("//elem/text()").getall(), ) def test_selector_shortcuts_kwargs(self): @@ -583,12 +583,12 @@ class XmlResponseTest(TextResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(), - response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(), + response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(), + response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(), ) response.selector.register_namespace('s2', 'http://scrapy.org') self.assertEqual( - response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).extract(), - response.selector.xpath("//s2:elem/text()").extract(), + response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(), + response.selector.xpath("//s2:elem/text()").getall(), ) diff --git a/tests/test_loader.py b/tests/test_loader.py index 3b5714058..8b58e4dbd 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -634,7 +634,7 @@ class SubselectorLoaderTest(unittest.TestCase): nl = l.nested_xpath("//header") nl.add_xpath('name', 'div/text()') nl.add_css('name_div', '#id') - nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract()) + nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall()) self.assertEqual(l.get_output_value('name'), [u'marta']) self.assertEqual(l.get_output_value('name_div'), [u'
marta
']) @@ -649,7 +649,7 @@ class SubselectorLoaderTest(unittest.TestCase): nl = l.nested_css("header") nl.add_xpath('name', 'div/text()') nl.add_css('name_div', '#id') - nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract()) + nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall()) self.assertEqual(l.get_output_value('name'), [u'marta']) self.assertEqual(l.get_output_value('name_div'), [u'
marta
']) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 5985a6f3e..fb72c9d6d 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -29,7 +29,7 @@ class MediaDownloadSpider(SimpleSpider): for href in response.xpath(''' //table[thead/tr/th="Filename"] /tbody//a/@href - ''').extract()], + ''').getall()], } yield item diff --git a/tests/test_selector.py b/tests/test_selector.py index 526660cc8..bc4baf7ea 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -20,17 +20,17 @@ class SelectorTestCase(unittest.TestCase): for x in xl: assert isinstance(x, Selector) - self.assertEqual(sel.xpath('//input').extract(), - [x.extract() for x in sel.xpath('//input')]) + self.assertEqual(sel.xpath('//input').getall(), + [x.get() for x in sel.xpath('//input')]) - self.assertEqual([x.extract() for x in sel.xpath("//input[@name='a']/@name")], + self.assertEqual([x.get() for x in sel.xpath("//input[@name='a']/@name")], [u'a']) - self.assertEqual([x.extract() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], + self.assertEqual([x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], [u'12.0']) - self.assertEqual(sel.xpath("concat('xpath', 'rules')").extract(), + self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(), [u'xpathrules']) - self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], + self.assertEqual([x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], [u'12']) def test_root_base_url(self): @@ -60,12 +60,12 @@ class SelectorTestCase(unittest.TestCase): text = b'

Hello

' sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8')) self.assertEqual(sel.type, 'xml') - self.assertEqual(sel.xpath("//div").extract(), + self.assertEqual(sel.xpath("//div").getall(), [u'

Hello

']) sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8')) self.assertEqual(sel.type, 'html') - self.assertEqual(sel.xpath("//div").extract(), + self.assertEqual(sel.xpath("//div").getall(), [u'

Hello

']) def test_http_header_encoding_precedence(self): @@ -84,15 +84,15 @@ class SelectorTestCase(unittest.TestCase): headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) x = Selector(response) - self.assertEqual(x.xpath("//span[@id='blank']/text()").extract(), + self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3']) def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence - r1 = TextResponse('http://www.example.com', \ - body=b'

an Jos\xe9 de

', \ + r1 = TextResponse('http://www.example.com', + body=b'

an Jos\xe9 de

', encoding='utf-8') - Selector(r1).xpath('//text()').extract() + Selector(r1).xpath('//text()').getall() def test_weakref_slots(self): """Check that classes are using slots and are weak-referenceable""" diff --git a/tests/test_spider.py b/tests/test_spider.py index 929e0fea8..f26da2334 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -147,10 +147,10 @@ class XMLFeedSpiderTest(SpiderTest): def parse_node(self, response, selector): yield { - 'loc': selector.xpath('a:loc/text()').extract(), - 'updated': selector.xpath('b:updated/text()').extract(), - 'other': selector.xpath('other/@value').extract(), - 'custom': selector.xpath('other/@b:custom').extract(), + 'loc': selector.xpath('a:loc/text()').getall(), + 'updated': selector.xpath('b:updated/text()').getall(), + 'other': selector.xpath('other/@value').getall(), + 'custom': selector.xpath('other/@b:custom').getall(), } for iterator in ('iternodes', 'xml'): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index f953076b8..00eb78068 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -30,10 +30,13 @@ class XmliterTestCase(unittest.TestCase): response = XmlResponse(url="http://example.com", body=body) attrs = [] for x in self.xmliter(response, 'product'): - attrs.append((x.xpath("@id").extract(), x.xpath("name/text()").extract(), x.xpath("./type/text()").extract())) + attrs.append(( + x.attrib['id'], + x.xpath("name/text()").extract(), + x.xpath("./type/text()").extract())) self.assertEqual(attrs, - [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])]) + [('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])]) def test_xmliter_unusual_node(self): body = b""" @@ -43,7 +46,7 @@ class XmliterTestCase(unittest.TestCase): """ response = XmlResponse(url="http://example.com", body=body) - nodenames = [e.xpath('name()').extract() + nodenames = [e.xpath('name()').getall() for e in self.xmliter(response, 'matchme...')] self.assertEqual(nodenames, [['matchme...']]) @@ -93,19 +96,19 @@ class XmliterTestCase(unittest.TestCase): attrs = [] for x in self.xmliter(r, u'þingflokkur'): - attrs.append((x.xpath('@id').extract(), - x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), - x.xpath(u'./tímabil/fyrstaþing/text()').extract())) + attrs.append((x.attrib['id'], + x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(), + x.xpath(u'./tímabil/fyrstaþing/text()').getall())) self.assertEqual(attrs, - [([u'26'], [u'-'], [u'80']), - ([u'21'], [u'Ab'], [u'76']), - ([u'27'], [u'A'], [u'27'])]) + [(u'26', [u'-'], [u'80']), + (u'21', [u'Ab'], [u'76']), + (u'27', [u'A'], [u'27'])]) def test_xmliter_text(self): body = u"""onetwo""" - self.assertEqual([x.xpath("text()").extract() for x in self.xmliter(body, 'product')], + self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')], [[u'one'], [u'two']]) def test_xmliter_namespaces(self): @@ -132,15 +135,15 @@ class XmliterTestCase(unittest.TestCase): node = next(my_iter) node.register_namespace('g', 'http://base.google.com/ns/1.0') - self.assertEqual(node.xpath('title/text()').extract(), ['Item 1']) - self.assertEqual(node.xpath('description/text()').extract(), ['This is item 1']) - self.assertEqual(node.xpath('link/text()').extract(), ['http://www.mydummycompany.com/items/1']) - self.assertEqual(node.xpath('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) - self.assertEqual(node.xpath('g:id/text()').extract(), ['ITEM_1']) - self.assertEqual(node.xpath('g:price/text()').extract(), ['400']) - self.assertEqual(node.xpath('image_link/text()').extract(), []) - self.assertEqual(node.xpath('id/text()').extract(), []) - self.assertEqual(node.xpath('price/text()').extract(), []) + self.assertEqual(node.xpath('title/text()').getall(), ['Item 1']) + self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1']) + self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1']) + self.assertEqual(node.xpath('g:image_link/text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg']) + self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1']) + self.assertEqual(node.xpath('g:price/text()').getall(), ['400']) + self.assertEqual(node.xpath('image_link/text()').getall(), []) + self.assertEqual(node.xpath('id/text()').getall(), []) + self.assertEqual(node.xpath('price/text()').getall(), []) def test_xmliter_exception(self): body = u"""onetwo""" @@ -159,7 +162,7 @@ class XmliterTestCase(unittest.TestCase): body = b'\n\n Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6\n\n\n' response = XmlResponse('http://www.example.com', body=body) self.assertEqual( - next(self.xmliter(response, 'item')).extract(), + next(self.xmliter(response, 'item')).get(), u'Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6' ) @@ -192,9 +195,9 @@ class LxmlXmliterTestCase(XmliterTestCase): namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0') node = next(namespace_iter) - self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg']) node = next(namespace_iter) - self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg']) + self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg']) def test_xmliter_namespaces_prefix(self): body = b"""\ @@ -219,14 +222,14 @@ class LxmlXmliterTestCase(XmliterTestCase): my_iter = self.xmliter(response, 'table', 'http://www.w3.org/TR/html4/', 'h') node = next(my_iter) - self.assertEqual(len(node.xpath('h:tr/h:td').extract()), 2) - self.assertEqual(node.xpath('h:tr/h:td[1]/text()').extract(), ['Apples']) - self.assertEqual(node.xpath('h:tr/h:td[2]/text()').extract(), ['Bananas']) + self.assertEqual(len(node.xpath('h:tr/h:td').getall()), 2) + self.assertEqual(node.xpath('h:tr/h:td[1]/text()').getall(), ['Apples']) + self.assertEqual(node.xpath('h:tr/h:td[2]/text()').getall(), ['Bananas']) my_iter = self.xmliter(response, 'table', 'http://www.w3schools.com/furniture', 'f') node = next(my_iter) - self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table']) + self.assertEqual(node.xpath('f:name/text()').getall(), ['African Coffee Table']) def test_xmliter_objtype_exception(self): i = self.xmliter(42, 'product')