diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py index 2052f0eaa..bc655b523 100644 --- a/scrapy/selector/lxmlsel.py +++ b/scrapy/selector/lxmlsel.py @@ -12,6 +12,19 @@ class XPathSelector(Selector): __slots__ = () default_contenttype = 'html' + def __init__(self, *a, **kw): + import warnings + from scrapy.exceptions import ScrapyDeprecationWarning + warnings.warn('%s is deprecated, instanciate scrapy.selector.Selector ' + 'instead' % type(self).__name__, + category=ScrapyDeprecationWarning, stacklevel=1) + super(XPathSelector, self).__init__(*a, **kw) + + def css(self, *a, **kw): + raise RuntimeError('.css() method not available for %s, ' + 'instanciate scrapy.selector.Selector ' + 'instead' % type(self).__name__) + class XmlXPathSelector(XPathSelector): __slots__ = () @@ -28,7 +41,7 @@ class XPathSelectorList(SelectorList): def __init__(self, *a, **kw): import warnings from scrapy.exceptions import ScrapyDeprecationWarning - warnings.warn('XPathSelectorList is deprecated, use ' + warnings.warn('XPathSelectorList is deprecated, instanciate ' 'scrapy.selector.SelectorList instead', category=ScrapyDeprecationWarning, stacklevel=1) super(XPathSelectorList, self).__init__(*a, **kw) diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index fc732a694..4149b837c 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -1,81 +1,85 @@ -""" -Selectors tests, common for all backends -""" - import re +import warnings import weakref - from twisted.trial import unittest - +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import TextResponse, HtmlResponse, XmlResponse -from scrapy.selector import XmlXPathSelector, HtmlXPathSelector, \ - XPathSelector +from scrapy.selector import Selector +from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector -class XPathSelectorTestCase(unittest.TestCase): +class SelectorTestCase(unittest.TestCase): - xs_cls = XPathSelector - hxs_cls = HtmlXPathSelector - xxs_cls = XmlXPathSelector + sscls = Selector - def test_selector_simple(self): + def test_simple_selection(self): """Simple selector tests""" body = "

" response = TextResponse(url="http://example.com", body=body) - xpath = self.hxs_cls(response) + ss = self.sscls(response) - xl = xpath.select('//input') + xl = ss.xpath('//input') self.assertEqual(2, len(xl)) for x in xl: - assert isinstance(x, self.hxs_cls) + assert isinstance(x, self.sscls) - self.assertEqual(xpath.select('//input').extract(), - [x.extract() for x in xpath.select('//input')]) + self.assertEqual(ss.xpath('//input').extract(), + [x.extract() for x in ss.xpath('//input')]) - self.assertEqual([x.extract() for x in xpath.select("//input[@name='a']/@name")], + self.assertEqual([x.extract() for x in ss.xpath("//input[@name='a']/@name")], [u'a']) - self.assertEqual([x.extract() for x in xpath.select("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], + self.assertEqual([x.extract() for x in ss.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], [u'12.0']) - self.assertEqual(xpath.select("concat('xpath', 'rules')").extract(), + self.assertEqual(ss.xpath("concat('xpath', 'rules')").extract(), [u'xpathrules']) - self.assertEqual([x.extract() for x in xpath.select("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], + self.assertEqual([x.extract() for x in ss.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], [u'12']) - def test_selector_unicode_query(self): + def test_select_unicode_query(self): body = u"

" response = TextResponse(url="http://example.com", body=body, encoding='utf8') - xpath = self.hxs_cls(response) - self.assertEqual(xpath.select(u'//input[@name="\xa9"]/@value').extract(), [u'1']) + ss = self.sscls(response) + self.assertEqual(ss.xpath(u'//input[@name="\xa9"]/@value').extract(), [u'1']) - def test_selector_same_type(self): - """Test XPathSelector returning the same type in x() method""" + def test_list_elements_type(self): + """Test Selector returning the same type in selection methods""" text = '

test

' - assert isinstance(self.xxs_cls(text=text).select("//p")[0], - self.xxs_cls) - assert isinstance(self.hxs_cls(text=text).select("//p")[0], - self.hxs_cls) + assert isinstance(self.sscls(text=text).xpath("//p")[0], self.sscls) + assert isinstance(self.sscls(text=text).css("p")[0], self.sscls) - def test_selector_boolean_result(self): + def test_boolean_result(self): body = "

" response = TextResponse(url="http://example.com", body=body) - xs = self.hxs_cls(response) - self.assertEquals(xs.select("//input[@name='a']/@name='a'").extract(), [u'1']) - self.assertEquals(xs.select("//input[@name='a']/@name='n'").extract(), [u'0']) - - def test_selector_xml_html(self): - """Test that XML and HTML XPathSelector's behave differently""" + xs = self.sscls(response) + self.assertEquals(xs.xpath("//input[@name='a']/@name='a'").extract(), [u'1']) + self.assertEquals(xs.xpath("//input[@name='a']/@name='n'").extract(), [u'0']) + def test_differences_parsing_xml_vs_html(self): + """Test that XML and HTML Selector's behave differently""" # some text which is parsed differently by XML and HTML flavors text = '

Hello

' - - self.assertEqual(self.xxs_cls(text=text).select("//div").extract(), - [u'

Hello

']) - - self.assertEqual(self.hxs_cls(text=text).select("//div").extract(), + hs = self.sscls(text=text, contenttype='html') + self.assertEqual(hs.xpath("//div").extract(), [u'

Hello

']) - def test_selector_nested(self): + xs = self.sscls(text=text, contenttype='xml') + self.assertEqual(xs.xpath("//div").extract(), + [u'

Hello

']) + + def test_flavor_detection(self): + text = '

Hello

' + ss = self.sscls(XmlResponse('http://example.com', body=text)) + self.assertEqual(ss.contenttype, 'xml') + self.assertEqual(ss.xpath("//div").extract(), + [u'

Hello

']) + + ss = self.sscls(HtmlResponse('http://example.com', body=text)) + self.assertEqual(ss.contenttype, 'html') + self.assertEqual(ss.xpath("//div").extract(), + [u'

Hello

']) + + def test_nested_selectors(self): """Nested selector tests""" body = """
@@ -91,24 +95,30 @@ class XPathSelectorTestCase(unittest.TestCase): """ response = HtmlResponse(url="http://example.com", body=body) - x = self.hxs_cls(response) - - divtwo = x.select('//div[@class="two"]') - self.assertEqual(map(unicode.strip, divtwo.select("//li").extract()), + x = self.sscls(response) + divtwo = x.xpath('//div[@class="two"]') + self.assertEqual(divtwo.xpath("//li").extract(), ["
  • one
  • ", "
  • two
  • ", "
  • four
  • ", "
  • five
  • ", "
  • six
  • "]) - self.assertEqual(map(unicode.strip, divtwo.select("./ul/li").extract()), + self.assertEqual(divtwo.xpath("./ul/li").extract(), ["
  • four
  • ", "
  • five
  • ", "
  • six
  • "]) - self.assertEqual(map(unicode.strip, divtwo.select(".//li").extract()), + self.assertEqual(divtwo.xpath(".//li").extract(), ["
  • four
  • ", "
  • five
  • ", "
  • six
  • "]) - self.assertEqual(divtwo.select("./li").extract(), - []) + self.assertEqual(divtwo.xpath("./li").extract(), []) + + def test_mixed_nested_selectors(self): + body = ''' +
    notme
    +

    text

    foo
    + ''' + ss = self.sscls(text=body) + self.assertEqual(ss.xpath('//div[@id="1"]').css('span::text').extract(), [u'me']) + self.assertEqual(ss.css('#1').xpath('./span/text()').extract(), [u'me']) def test_dont_strip(self): - hxs = self.hxs_cls(text='
    fff: zzz
    ') - self.assertEqual(hxs.select("//text()").extract(), - [u'fff: ', u'zzz']) + hxs = self.sscls(text='
    fff: zzz
    ') + self.assertEqual(hxs.xpath("//text()").extract(), [u'fff: ', u'zzz']) - def test_selector_namespaces_simple(self): + def test_namespaces_simple(self): body = """ take this @@ -117,13 +127,13 @@ class XPathSelectorTestCase(unittest.TestCase): """ response = XmlResponse(url="http://example.com", body=body) - x = self.xxs_cls(response) + x = self.sscls(response) x.register_namespace("somens", "http://scrapy.org") - self.assertEqual(x.select("//somens:a/text()").extract(), + self.assertEqual(x.xpath("//somens:a/text()").extract(), [u'take this']) - def test_selector_namespaces_multiple(self): + def test_namespaces_multiple(self): body = """ """ response = XmlResponse(url="http://example.com", body=body) - x = self.xxs_cls(response) - + x = self.sscls(response) x.register_namespace("xmlns", "http://webservices.amazon.com/AWSECommerceService/2005-10-05") x.register_namespace("p", "http://www.scrapy.org/product") x.register_namespace("b", "http://somens.com") - self.assertEqual(len(x.select("//xmlns:TestTag")), 1) - self.assertEqual(x.select("//b:Operation/text()").extract()[0], 'hello') - self.assertEqual(x.select("//xmlns:TestTag/@b:att").extract()[0], 'value') - self.assertEqual(x.select("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90') - self.assertEqual(x.select("//p:SecondTestTag").select("./xmlns:price/text()")[0].extract(), '90') - self.assertEqual(x.select("//p:SecondTestTag/xmlns:material/text()").extract()[0], 'iron') + self.assertEqual(len(x.xpath("//xmlns:TestTag")), 1) + self.assertEqual(x.xpath("//b:Operation/text()").extract()[0], 'hello') + self.assertEqual(x.xpath("//xmlns:TestTag/@b:att").extract()[0], 'value') + self.assertEqual(x.xpath("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90') + self.assertEqual(x.xpath("//p:SecondTestTag").xpath("./xmlns:price/text()")[0].extract(), '90') + self.assertEqual(x.xpath("//p:SecondTestTag/xmlns:material/text()").extract()[0], 'iron') - def test_selector_re(self): + def test_re(self): body = """
    Name: Mary
    • Name: John
    • @@ -155,44 +164,35 @@ class XPathSelectorTestCase(unittest.TestCase):
    • Age: 20
    Age: 20 -
    - - """ +
    """ response = HtmlResponse(url="http://example.com", body=body) - x = self.hxs_cls(response) + x = self.sscls(response) name_re = re.compile("Name: (\w+)") - self.assertEqual(x.select("//ul/li").re(name_re), + self.assertEqual(x.xpath("//ul/li").re(name_re), ["John", "Paul"]) - self.assertEqual(x.select("//ul/li").re("Age: (\d+)"), + self.assertEqual(x.xpath("//ul/li").re("Age: (\d+)"), ["10", "20"]) - def test_selector_re_intl(self): + def test_re_intl(self): body = """
    Evento: cumplea\xc3\xb1os
    """ response = HtmlResponse(url="http://example.com", body=body, encoding='utf-8') - x = self.hxs_cls(response) - self.assertEqual(x.select("//div").re("Evento: (\w+)"), [u'cumplea\xf1os']) + x = self.sscls(response) + self.assertEqual(x.xpath("//div").re("Evento: (\w+)"), [u'cumplea\xf1os']) def test_selector_over_text(self): - hxs = self.hxs_cls(text='lala') - self.assertEqual(hxs.extract(), - u'lala') + hs = self.sscls(text='lala') + self.assertEqual(hs.extract(), u'lala') + xs = self.sscls(text='lala', contenttype='xml') + self.assertEqual(xs.extract(), u'lala') + self.assertEqual(xs.xpath('.').extract(), [u'lala']) - xxs = self.xxs_cls(text='lala') - self.assertEqual(xxs.extract(), - u'lala') - - xxs = self.xxs_cls(text='lala') - self.assertEqual(xxs.select('.').extract(), - [u'lala']) - - - def test_selector_invalid_xpath(self): + def test_invalid_xpath(self): response = XmlResponse(url="http://example.com", body="") - x = self.hxs_cls(response) + x = self.sscls(response) xpath = "//test[@foo='bar]" try: - x.select(xpath) + x.xpath(xpath) except ValueError, e: assert xpath in str(e), "Exception message does not contain invalid xpath" except Exception: @@ -215,64 +215,121 @@ class XPathSelectorTestCase(unittest.TestCase): headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) - x = self.hxs_cls(response) - self.assertEquals(x.select("//span[@id='blank']/text()").extract(), + x = self.sscls(response) + self.assertEquals(x.xpath("//span[@id='blank']/text()").extract(), [u'\xa3']) def test_empty_bodies(self): # shouldn't raise errors r1 = TextResponse('http://www.example.com', body='') - self.hxs_cls(r1).select('//text()').extract() - self.xxs_cls(r1).select('//text()').extract() + self.sscls(r1).xpath('//text()').extract() def test_null_bytes(self): # shouldn't raise errors r1 = TextResponse('http://www.example.com', \ body='pre\x00post', \ encoding='utf-8') - self.hxs_cls(r1).select('//text()').extract() - self.xxs_cls(r1).select('//text()').extract() + self.sscls(r1).xpath('//text()').extract() def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence r1 = TextResponse('http://www.example.com', \ body='

    an Jos\xe9 de

    ', \ encoding='utf-8') - self.hxs_cls(r1).select('//text()').extract() - self.xxs_cls(r1).select('//text()').extract() + self.sscls(r1).xpath('//text()').extract() def test_select_on_unevaluable_nodes(self): - r = self.hxs_cls(text=u'some text') + r = self.sscls(text=u'some text') # Text node - x1 = r.select('//text()') + x1 = r.xpath('//text()') self.assertEquals(x1.extract(), [u'some text']) - self.assertEquals(x1.select('.//b').extract(), []) + self.assertEquals(x1.xpath('.//b').extract(), []) # Tag attribute - x1 = r.select('//span/@class') + x1 = r.xpath('//span/@class') self.assertEquals(x1.extract(), [u'big']) - self.assertEquals(x1.select('.//text()').extract(), []) + self.assertEquals(x1.xpath('.//text()').extract(), []) def test_select_on_text_nodes(self): - r = self.hxs_cls(text=u'
    Options:opt1
    Otheropt2
    ') - x1 = r.select("//div/descendant::text()[preceding-sibling::b[contains(text(), 'Options')]]") + r = self.sscls(text=u'
    Options:opt1
    Otheropt2
    ') + x1 = r.xpath("//div/descendant::text()[preceding-sibling::b[contains(text(), 'Options')]]") self.assertEquals(x1.extract(), [u'opt1']) - x1 = r.select("//div/descendant::text()/preceding-sibling::b[contains(text(), 'Options')]") + x1 = r.xpath("//div/descendant::text()/preceding-sibling::b[contains(text(), 'Options')]") self.assertEquals(x1.extract(), [u'Options:']) def test_nested_select_on_text_nodes(self): # FIXME: does not work with lxml backend [upstream] - r = self.hxs_cls(text=u'
    Options:opt1
    Otheropt2
    ') - x1 = r.select("//div/descendant::text()") - x2 = x1.select("./preceding-sibling::b[contains(text(), 'Options')]") - + r = self.sscls(text=u'
    Options:opt1
    Otheropt2
    ') + x1 = r.xpath("//div/descendant::text()") + x2 = x1.xpath("./preceding-sibling::b[contains(text(), 'Options')]") self.assertEquals(x2.extract(), [u'Options:']) - test_nested_select_on_text_nodes.skip = True + test_nested_select_on_text_nodes.skip = "Text nodes lost parent node reference in lxml" def test_weakref_slots(self): """Check that classes are using slots and are weak-referenceable""" - for cls in [self.xs_cls, self.hxs_cls, self.xxs_cls]: - x = cls() - weakref.ref(x) - assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ - x.__class__.__name__ + x = self.sscls() + weakref.ref(x) + assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ + x.__class__.__name__ + + def test_remove_namespaces(self): + xml = """ + + + + +""" + xxs = self.sscls(XmlResponse("http://example.com/feed.atom", body=xml)) + self.assertEqual(len(xxs.xpath("//link")), 0) + xxs.remove_namespaces() + self.assertEqual(len(xxs.xpath("//link")), 2) + + def test_remove_attributes_namespaces(self): + xml = """ + + + + +""" + xxs = self.sscls(XmlResponse("http://example.com/feed.atom", body=xml)) + self.assertEqual(len(xxs.xpath("//link/@type")), 0) + xxs.remove_namespaces() + self.assertEqual(len(xxs.xpath("//link/@type")), 2) + + +class DeprecatedXpathSelectorTest(unittest.TestCase): + + text = '

    Hello

    ' + + def test_warnings(self): + for cls in XPathSelector, HtmlXPathSelector, XPathSelector: + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + hs = cls(text=self.text) + assert len(w) == 1, w + assert issubclass(w[0].category, ScrapyDeprecationWarning) + assert 'deprecated' in str(w[-1].message) + hs.select("//div").extract() + assert issubclass(w[1].category, ScrapyDeprecationWarning) + assert 'deprecated' in str(w[-1].message) + + def test_xpathselector(self): + with warnings.catch_warnings(record=True): + hs = XPathSelector(text=self.text) + self.assertEqual(hs.select("//div").extract(), + [u'

    Hello

    ']) + self.assertRaises(RuntimeError, hs.css, 'div') + + def test_htmlxpathselector(self): + with warnings.catch_warnings(record=True): + hs = HtmlXPathSelector(text=self.text) + self.assertEqual(hs.select("//div").extract(), + [u'

    Hello

    ']) + self.assertRaises(RuntimeError, hs.css, 'div') + + def test_xmlxpathselector(self): + with warnings.catch_warnings(record=True): + xs = XmlXPathSelector(text=self.text) + self.assertEqual(xs.select("//div").extract(), + [u'

    Hello

    ']) + self.assertRaises(RuntimeError, xs.css, 'div') diff --git a/scrapy/tests/test_selector_cssselect.py b/scrapy/tests/test_selector_csstranslator.py similarity index 90% rename from scrapy/tests/test_selector_cssselect.py rename to scrapy/tests/test_selector_csstranslator.py index dd2e292b6..45ef91c19 100644 --- a/scrapy/tests/test_selector_cssselect.py +++ b/scrapy/tests/test_selector_csstranslator.py @@ -114,22 +114,22 @@ class TranslatorMixinTest(unittest.TestCase): self.assertRaises(exc, self.c2x, css) -class HTMLCSSSelectorTest(unittest.TestCase): +class CSSSelectorTest(unittest.TestCase): - hcs_cls = Selector + sscls = Selector def setUp(self): self.htmlresponse = HtmlResponse('http://example.com', body=HTMLBODY) - self.hcs = self.hcs_cls(self.htmlresponse) + self.ss = self.sscls(self.htmlresponse) def x(self, *a, **kw): - return [v.strip() for v in self.hcs.css(*a, **kw).extract() if v.strip()] + return [v.strip() for v in self.ss.css(*a, **kw).extract() if v.strip()] def test_selector_simple(self): - for x in self.hcs.css('input'): - self.assertTrue(isinstance(x, self.hcs.__class__), x) - self.assertEqual(self.hcs.css('input').extract(), - [x.extract() for x in self.hcs.css('input')]) + for x in self.ss.css('input'): + self.assertTrue(isinstance(x, self.ss.__class__), x) + self.assertEqual(self.ss.css('input').extract(), + [x.extract() for x in self.ss.css('input')]) def test_text_pseudo_element(self): self.assertEqual(self.x('#p-b2'), [u'guy']) @@ -147,7 +147,7 @@ class HTMLCSSSelectorTest(unittest.TestCase): self.assertEqual(self.x('map[name="dummymap"] ::attr(shape)'), [u'circle', u'default']) def test_nested_selector(self): - self.assertEqual(self.hcs.css('p').css('b::text').extract(), + self.assertEqual(self.ss.css('p').css('b::text').extract(), [u'hi', u'guy']) - self.assertEqual(self.hcs.css('div').css('area:last-child').extract(), + self.assertEqual(self.ss.css('div').css('area:last-child').extract(), [u'']) diff --git a/scrapy/tests/test_selector_lxml.py b/scrapy/tests/test_selector_lxml.py deleted file mode 100644 index 7ea674d62..000000000 --- a/scrapy/tests/test_selector_lxml.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -Selectors tests, specific for lxml backend -""" - -import unittest -from scrapy.tests import test_selector -from scrapy.http import TextResponse, HtmlResponse, XmlResponse -from scrapy.selector.lxmldocument import LxmlDocument -from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector - - -class LxmlXPathSelectorTestCase(test_selector.XPathSelectorTestCase): - - xs_cls = XPathSelector - hxs_cls = HtmlXPathSelector - xxs_cls = XmlXPathSelector - - def test_remove_namespaces(self): - xml = """ - - - - -""" - xxs = XmlXPathSelector(XmlResponse("http://example.com/feed.atom", body=xml)) - self.assertEqual(len(xxs.select("//link")), 0) - xxs.remove_namespaces() - self.assertEqual(len(xxs.select("//link")), 2) - - def test_remove_attributes_namespaces(self): - xml = """ - - - - -""" - xxs = XmlXPathSelector(XmlResponse("http://example.com/feed.atom", body=xml)) - self.assertEqual(len(xxs.select("//link/@type")), 0) - xxs.remove_namespaces() - self.assertEqual(len(xxs.select("//link/@type")), 2) - - -class Libxml2DocumentTest(unittest.TestCase): - - def test_caching(self): - r1 = HtmlResponse('http://www.example.com', body='') - r2 = r1.copy() - - doc1 = LxmlDocument(r1) - doc2 = LxmlDocument(r1) - doc3 = LxmlDocument(r2) - - # make sure it's cached - assert doc1 is doc2 - assert doc1 is not doc3 - - def test_null_char(self): - # make sure bodies with null char ('\x00') don't raise a TypeError exception - self.body_content = 'test problematic \x00 body' - response = TextResponse('http://example.com/catalog/product/blabla-123', - headers={'Content-Type': 'text/plain; charset=utf-8'}, body=self.body_content) - LxmlDocument(response) diff --git a/scrapy/tests/test_selector_lxmldocument.py b/scrapy/tests/test_selector_lxmldocument.py new file mode 100644 index 000000000..e544613d5 --- /dev/null +++ b/scrapy/tests/test_selector_lxmldocument.py @@ -0,0 +1,26 @@ +import unittest +from scrapy.selector.lxmldocument import LxmlDocument +from scrapy.http import TextResponse, HtmlResponse + + +class Libxml2DocumentTest(unittest.TestCase): + + def test_caching(self): + r1 = HtmlResponse('http://www.example.com', body='') + r2 = r1.copy() + + doc1 = LxmlDocument(r1) + doc2 = LxmlDocument(r1) + doc3 = LxmlDocument(r2) + + # make sure it's cached + assert doc1 is doc2 + assert doc1 is not doc3 + + def test_null_char(self): + # make sure bodies with null char ('\x00') don't raise a TypeError exception + body = 'test problematic \x00 body' + response = TextResponse('http://example.com/catalog/product/blabla-123', + headers={'Content-Type': 'text/plain; charset=utf-8'}, + body=body) + LxmlDocument(response)