diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 3e27d6abd..36baacfbd 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -35,7 +35,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_response_selector_html(self): - xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]' + xpath = 'response.xpath("//p[@class=\'one\']/text()").get()' _, out, _ = yield self.execute([self.url('/html'), '-c', xpath]) self.assertEqual(out.strip(), b'Works') diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 820758dc9..3b90e3dac 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -336,11 +336,11 @@ class TextResponseTest(BaseResponseTest): self.assertIs(response.selector.response, response) self.assertEqual( - response.selector.xpath("//title/text()").extract(), + response.selector.xpath("//title/text()").getall(), [u'Some page'] ) self.assertEqual( - response.selector.css("title::text").extract(), + response.selector.css("title::text").getall(), [u'Some page'] ) self.assertEqual( @@ -353,12 +353,12 @@ class TextResponseTest(BaseResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("//title/text()").extract(), - response.selector.xpath("//title/text()").extract(), + response.xpath("//title/text()").getall(), + response.selector.xpath("//title/text()").getall(), ) self.assertEqual( - response.css("title::text").extract(), - response.selector.css("title::text").extract(), + response.css("title::text").getall(), + response.selector.css("title::text").getall(), ) def test_selector_shortcuts_kwargs(self): @@ -366,13 +366,13 @@ class TextResponseTest(BaseResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").extract(), - response.xpath("normalize-space(//p[@class=\"content\"])").extract(), + response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").getall(), + response.xpath("normalize-space(//p[@class=\"content\"])").getall(), ) self.assertEqual( response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()", - pclass="content", pcount=1).extract(), - response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").extract(), + pclass="content", pcount=1).getall(), + response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(), ) def test_urljoin_with_base_url(self): @@ -562,7 +562,7 @@ class XmlResponseTest(TextResponseTest): self.assertIs(response.selector.response, response) self.assertEqual( - response.selector.xpath("//elem/text()").extract(), + response.selector.xpath("//elem/text()").getall(), [u'value'] ) @@ -571,8 +571,8 @@ class XmlResponseTest(TextResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("//elem/text()").extract(), - response.selector.xpath("//elem/text()").extract(), + response.xpath("//elem/text()").getall(), + response.selector.xpath("//elem/text()").getall(), ) def test_selector_shortcuts_kwargs(self): @@ -583,12 +583,12 @@ class XmlResponseTest(TextResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(), - response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(), + response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(), + response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(), ) response.selector.register_namespace('s2', 'http://scrapy.org') self.assertEqual( - response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).extract(), - response.selector.xpath("//s2:elem/text()").extract(), + response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(), + response.selector.xpath("//s2:elem/text()").getall(), ) diff --git a/tests/test_loader.py b/tests/test_loader.py index 3b5714058..8b58e4dbd 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -634,7 +634,7 @@ class SubselectorLoaderTest(unittest.TestCase): nl = l.nested_xpath("//header") nl.add_xpath('name', 'div/text()') nl.add_css('name_div', '#id') - nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract()) + nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall()) self.assertEqual(l.get_output_value('name'), [u'marta']) self.assertEqual(l.get_output_value('name_div'), [u'

Hello

Hello

Hello
an Jos\xe9 de
', \ + r1 = TextResponse('http://www.example.com', + body=b'an Jos\xe9 de
', encoding='utf-8') - Selector(r1).xpath('//text()').extract() + Selector(r1).xpath('//text()').getall() def test_weakref_slots(self): """Check that classes are using slots and are weak-referenceable""" diff --git a/tests/test_spider.py b/tests/test_spider.py index 929e0fea8..f26da2334 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -147,10 +147,10 @@ class XMLFeedSpiderTest(SpiderTest): def parse_node(self, response, selector): yield { - 'loc': selector.xpath('a:loc/text()').extract(), - 'updated': selector.xpath('b:updated/text()').extract(), - 'other': selector.xpath('other/@value').extract(), - 'custom': selector.xpath('other/@b:custom').extract(), + 'loc': selector.xpath('a:loc/text()').getall(), + 'updated': selector.xpath('b:updated/text()').getall(), + 'other': selector.xpath('other/@value').getall(), + 'custom': selector.xpath('other/@b:custom').getall(), } for iterator in ('iternodes', 'xml'): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index f953076b8..00eb78068 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -30,10 +30,13 @@ class XmliterTestCase(unittest.TestCase): response = XmlResponse(url="http://example.com", body=body) attrs = [] for x in self.xmliter(response, 'product'): - attrs.append((x.xpath("@id").extract(), x.xpath("name/text()").extract(), x.xpath("./type/text()").extract())) + attrs.append(( + x.attrib['id'], + x.xpath("name/text()").extract(), + x.xpath("./type/text()").extract())) self.assertEqual(attrs, - [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])]) + [('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])]) def test_xmliter_unusual_node(self): body = b""" @@ -43,7 +46,7 @@ class XmliterTestCase(unittest.TestCase): """ response = XmlResponse(url="http://example.com", body=body) - nodenames = [e.xpath('name()').extract() + nodenames = [e.xpath('name()').getall() for e in self.xmliter(response, 'matchme...')] self.assertEqual(nodenames, [['matchme...']]) @@ -93,19 +96,19 @@ class XmliterTestCase(unittest.TestCase): attrs = [] for x in self.xmliter(r, u'þingflokkur'): - attrs.append((x.xpath('@id').extract(), - x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), - x.xpath(u'./tímabil/fyrstaþing/text()').extract())) + attrs.append((x.attrib['id'], + x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(), + x.xpath(u'./tímabil/fyrstaþing/text()').getall())) self.assertEqual(attrs, - [([u'26'], [u'-'], [u'80']), - ([u'21'], [u'Ab'], [u'76']), - ([u'27'], [u'A'], [u'27'])]) + [(u'26', [u'-'], [u'80']), + (u'21', [u'Ab'], [u'76']), + (u'27', [u'A'], [u'27'])]) def test_xmliter_text(self): body = u"""