From e625a8d56e765e3ef31971ac331b0e94e9bb003a Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 27 Oct 2010 08:54:32 -0200 Subject: [PATCH] moved all similar selector tests to common selector tests, to reuse them among all backends --- scrapy/tests/test_selector.py | 52 ++++++++++++++++++++++++++ scrapy/tests/test_selector_libxml2.py | 45 +---------------------- scrapy/tests/test_selector_lxml.py | 53 +-------------------------- 3 files changed, 56 insertions(+), 94 deletions(-) diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index 9fe3aa6ed..375ef3464 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -52,6 +52,18 @@ class XPathSelectorTestCase(unittest.TestCase): assert isinstance(self.hxs_cls(text=text).select("//p")[0], self.hxs_cls) + @libxml2debug + def test_selector_boolean_result(self): + body = "

" + response = TextResponse(url="http://example.com", body=body) + xs = self.hxs_cls(response) + true = xs.select("//input[@name='a']/@name='a'").extract()[0] + false = xs.select("//input[@name='a']/@name='n'").extract()[0] + + # the actual result depends on the backend used + assert true in [u'1', u'True'], true + assert false in [u'0', u'False'], false + @libxml2debug def test_selector_xml_html(self): """Test that XML and HTML XPathSelector's behave differently""" @@ -94,6 +106,46 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEqual(divtwo.select("./li").extract(), []) + @libxml2debug + def test_selector_namespaces_simple(self): + body = """ + + take this + found + + """ + + response = XmlResponse(url="http://example.com", body=body) + x = self.xxs_cls(response) + + x.register_namespace("somens", "http://scrapy.org") + self.assertEqual(x.select("//somens:a/text()").extract(), + [u'take this']) + + @libxml2debug + def test_selector_namespaces_multiple(self): + body = """ + + hello + value + iron90Dried Rose + + """ + response = XmlResponse(url="http://example.com", body=body) + x = self.xxs_cls(response) + + x.register_namespace("xmlns", "http://webservices.amazon.com/AWSECommerceService/2005-10-05") + x.register_namespace("p", "http://www.scrapy.org/product") + x.register_namespace("b", "http://somens.com") + self.assertEqual(len(x.select("//xmlns:TestTag")), 1) + self.assertEqual(x.select("//b:Operation/text()").extract()[0], 'hello') + self.assertEqual(x.select("//xmlns:TestTag/@b:att").extract()[0], 'value') + self.assertEqual(x.select("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90') + self.assertEqual(x.select("//p:SecondTestTag").select("./xmlns:price/text()")[0].extract(), '90') + self.assertEqual(x.select("//p:SecondTestTag/xmlns:material/text()").extract()[0], 'iron') + @libxml2debug def test_selector_re(self): body = """
Name: Mary diff --git a/scrapy/tests/test_selector_libxml2.py b/scrapy/tests/test_selector_libxml2.py index e4e6a2f18..917ddaf96 100644 --- a/scrapy/tests/test_selector_libxml2.py +++ b/scrapy/tests/test_selector_libxml2.py @@ -9,55 +9,14 @@ from scrapy.selector.libxml2sel import XmlXPathSelector, HtmlXPathSelector, \ XPathSelector from scrapy.selector.document import Libxml2Document from scrapy.utils.test import libxml2debug -from scrapy.tests.test_selector import XPathSelectorTestCase +from scrapy.tests import test_selector -class XPathSelectorTestCase(XPathSelectorTestCase): +class Libxml2XPathSelectorTestCase(test_selector.XPathSelectorTestCase): xs_cls = XPathSelector hxs_cls = HtmlXPathSelector xxs_cls = XmlXPathSelector - @libxml2debug - def test_selector_namespaces_simple(self): - body = """ - - - found - - """ - - response = XmlResponse(url="http://example.com", body=body) - x = XmlXPathSelector(response) - - x.register_namespace("somens", "http://scrapy.org") - self.assertEqual(x.select("//somens:a").extract(), - ['']) - - - @libxml2debug - def test_selector_namespaces_multiple(self): - body = """ - - hello - value - 90Dried Rose - - """ - response = XmlResponse(url="http://example.com", body=body) - x = XmlXPathSelector(response) - - x.register_namespace("xmlns", "http://webservices.amazon.com/AWSECommerceService/2005-10-05") - x.register_namespace("p", "http://www.scrapy.org/product") - x.register_namespace("b", "http://somens.com") - self.assertEqual(len(x.select("//xmlns:TestTag")), 1) - self.assertEqual(x.select("//b:Operation/text()").extract()[0], 'hello') - self.assertEqual(x.select("//xmlns:TestTag/@b:att").extract()[0], 'value') - self.assertEqual(x.select("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90') - self.assertEqual(x.select("//p:SecondTestTag").select("./xmlns:price/text()")[0].extract(), '90') - self.assertEqual(x.select("//p:SecondTestTag/xmlns:material").extract()[0], '') - @libxml2debug def test_null_bytes(self): hxs = HtmlXPathSelector(text='la\x00la') diff --git a/scrapy/tests/test_selector_lxml.py b/scrapy/tests/test_selector_lxml.py index 9e3bf5cb5..8050a2758 100644 --- a/scrapy/tests/test_selector_lxml.py +++ b/scrapy/tests/test_selector_lxml.py @@ -10,9 +10,9 @@ try: except ImportError: has_lxml = False from scrapy.utils.test import libxml2debug -from scrapy.tests.test_selector import XPathSelectorTestCase +from scrapy.tests import test_selector -class XPathSelectorTestCase(XPathSelectorTestCase): +class LxmlXPathSelectorTestCase(test_selector.XPathSelectorTestCase): if has_lxml: xs_cls = XPathSelector @@ -21,55 +21,6 @@ class XPathSelectorTestCase(XPathSelectorTestCase): else: skip = "lxml not available" - @libxml2debug - def test_selector_boolean_result(self): - body = "

" - response = TextResponse(url="http://example.com", body=body) - xs = HtmlXPathSelector(response) - self.assertEqual(xs.select("//input[@name='a']/@name='a'").extract(), [u'True']) - self.assertEqual(xs.select("//input[@name='a']/@name='n'").extract(), [u'False']) - - @libxml2debug - def test_selector_namespaces_simple(self): - body = """ - - take this - found - - """ - - response = XmlResponse(url="http://example.com", body=body) - x = XmlXPathSelector(response) - - x.register_namespace("somens", "http://scrapy.org") - self.assertEqual(x.select("//somens:a/text()").extract(), - [u'take this']) - - - @libxml2debug - def test_selector_namespaces_multiple(self): - body = """ - - hello - value - iron90Dried Rose - - """ - response = XmlResponse(url="http://example.com", body=body) - x = XmlXPathSelector(response) - - x.register_namespace("xmlns", "http://webservices.amazon.com/AWSECommerceService/2005-10-05") - x.register_namespace("p", "http://www.scrapy.org/product") - x.register_namespace("b", "http://somens.com") - self.assertEqual(len(x.select("//xmlns:TestTag")), 1) - self.assertEqual(x.select("//b:Operation/text()").extract()[0], 'hello') - self.assertEqual(x.select("//xmlns:TestTag/@b:att").extract()[0], 'value') - self.assertEqual(x.select("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90') - self.assertEqual(x.select("//p:SecondTestTag").select("./xmlns:price/text()")[0].extract(), '90') - self.assertEqual(x.select("//p:SecondTestTag/xmlns:material/text()").extract()[0], 'iron') - # XXX: this test was disabled because lxml behaves inconsistently when # handling null bytes between different 2.2.x versions, but it may be due # to differences in libxml2 too. it's also unclear what should be the