diff --git a/scrapy/contrib/linkextractors/image.py b/scrapy/contrib/linkextractors/image.py index 88dd78577..1f29a6797 100644 --- a/scrapy/contrib/linkextractors/image.py +++ b/scrapy/contrib/linkextractors/image.py @@ -7,7 +7,7 @@ image links only. from scrapy.link import Link from scrapy.utils.url import canonicalize_url, urljoin_rfc from scrapy.utils.python import unicode_to_str, flatten -from scrapy.selector import XPathSelectorList, HtmlXPathSelector +from scrapy.selector.libxml2sel import XPathSelectorList, HtmlXPathSelector class HTMLImageLinkExtractor(object): '''HTMLImageLinkExtractor objects are intended to extract image links from HTML pages diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py new file mode 100644 index 000000000..9fe3aa6ed --- /dev/null +++ b/scrapy/tests/test_selector.py @@ -0,0 +1,182 @@ +""" +Selectors tests, common for all backends +""" + +import re +import weakref + +from twisted.trial import unittest + +from scrapy.http import TextResponse, HtmlResponse, XmlResponse +from scrapy.selector import XmlXPathSelector, HtmlXPathSelector, \ + XPathSelector +from scrapy.utils.test import libxml2debug + +class XPathSelectorTestCase(unittest.TestCase): + + xs_cls = XPathSelector + hxs_cls = HtmlXPathSelector + xxs_cls = XmlXPathSelector + + @libxml2debug + def test_selector_simple(self): + """Simple selector tests""" + body = "

" + response = TextResponse(url="http://example.com", body=body) + xpath = self.hxs_cls(response) + + xl = xpath.select('//input') + self.assertEqual(2, len(xl)) + for x in xl: + assert isinstance(x, self.hxs_cls) + + self.assertEqual(xpath.select('//input').extract(), + [x.extract() for x in xpath.select('//input')]) + + self.assertEqual([x.extract() for x in xpath.select("//input[@name='a']/@name")], + [u'a']) + self.assertEqual([x.extract() for x in xpath.select("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], + [u'12.0']) + + self.assertEqual(xpath.select("concat('xpath', 'rules')").extract(), + [u'xpathrules']) + self.assertEqual([x.extract() for x in xpath.select("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], + [u'12']) + + @libxml2debug + def test_selector_same_type(self): + """Test XPathSelector returning the same type in x() method""" + text = '

test

' + assert isinstance(self.xxs_cls(text=text).select("//p")[0], + self.xxs_cls) + assert isinstance(self.hxs_cls(text=text).select("//p")[0], + self.hxs_cls) + + @libxml2debug + def test_selector_xml_html(self): + """Test that XML and HTML XPathSelector's behave differently""" + + # some text which is parsed differently by XML and HTML flavors + text = '

Hello

' + + self.assertEqual(self.xxs_cls(text=text).select("//div").extract(), + [u'

Hello

']) + + self.assertEqual(self.hxs_cls(text=text).select("//div").extract(), + [u'

Hello

']) + + @libxml2debug + def test_selector_nested(self): + """Nested selector tests""" + body = """ +
+ +
+
+ +
+ """ + + response = HtmlResponse(url="http://example.com", body=body) + x = self.hxs_cls(response) + + divtwo = x.select('//div[@class="two"]') + self.assertEqual(divtwo.select("//li").extract(), + ["
  • one
    • ", "
  • two
    • ", "
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) + self.assertEqual(divtwo.select("./ul/li").extract(), + ["
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) + self.assertEqual(divtwo.select(".//li").extract(), + ["
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) + self.assertEqual(divtwo.select("./li").extract(), + []) + + @libxml2debug + def test_selector_re(self): + body = """
    Name: Mary + + Age: 20 +
    + + """ + response = HtmlResponse(url="http://example.com", body=body) + x = self.hxs_cls(response) + + name_re = re.compile("Name: (\w+)") + self.assertEqual(x.select("//ul/li").re(name_re), + ["John", "Paul"]) + self.assertEqual(x.select("//ul/li").re("Age: (\d+)"), + ["10", "20"]) + + @libxml2debug + def test_selector_over_text(self): + hxs = self.hxs_cls(text='lala') + self.assertEqual(hxs.extract(), + u'lala') + + xxs = self.xxs_cls(text='lala') + self.assertEqual(xxs.extract(), + u'lala') + + xxs = self.xxs_cls(text='lala') + self.assertEqual(xxs.select('.').extract(), + [u'lala']) + + + @libxml2debug + def test_selector_invalid_xpath(self): + response = XmlResponse(url="http://example.com", body="") + x = self.hxs_cls(response) + xpath = "//test[@foo='bar]" + try: + x.select(xpath) + except ValueError, e: + assert xpath in str(e), "Exception message does not contain invalid xpath" + except Exception: + raise AssertionError("A invalid XPath does not raise ValueError") + else: + raise AssertionError("A invalid XPath does not raise an exception") + + @libxml2debug + def test_http_header_encoding_precedence(self): + # u'\xa3' = pound symbol in unicode + # u'\xc2\xa3' = pound symbol in utf-8 + # u'\xa3' = pound symbol in latin-1 (iso-8859-1) + + meta = u'' + head = u'' + meta + u'' + body_content = u'\xa3' + body = u'' + body_content + u'' + html = u'' + head + body + u'' + encoding = 'utf-8' + html_utf8 = html.encode(encoding) + + headers = {'Content-Type': ['text/html; charset=utf-8']} + response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) + x = self.hxs_cls(response) + self.assertEquals(x.select("//span[@id='blank']/text()").extract(), + [u'\xa3']) + + @libxml2debug + def test_empty_bodies(self): + r1 = TextResponse('http://www.example.com', body='') + self.hxs_cls(r1) # shouldn't raise error + self.xxs_cls(r1) # shouldn't raise error + + @libxml2debug + def test_weakref_slots(self): + """Check that classes are using slots and are weak-referenceable""" + for cls in [self.xs_cls, self.hxs_cls, self.xxs_cls]: + x = cls() + weakref.ref(x) + assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ + x.__class__.__name__ + diff --git a/scrapy/tests/test_selector_libxml2.py b/scrapy/tests/test_selector_libxml2.py index b40d0697d..e4e6a2f18 100644 --- a/scrapy/tests/test_selector_libxml2.py +++ b/scrapy/tests/test_selector_libxml2.py @@ -1,127 +1,21 @@ -import re +""" +Selectors tests, specific for libxml2 backend +""" + import unittest -import weakref from scrapy.http import TextResponse, HtmlResponse, XmlResponse -from scrapy.selector import XmlXPathSelector, HtmlXPathSelector, \ +from scrapy.selector.libxml2sel import XmlXPathSelector, HtmlXPathSelector, \ XPathSelector from scrapy.selector.document import Libxml2Document from scrapy.utils.test import libxml2debug +from scrapy.tests.test_selector import XPathSelectorTestCase -class XPathSelectorTestCase(unittest.TestCase): - - @libxml2debug - def test_selector_simple(self): - """Simple selector tests""" - body = "

    " - response = TextResponse(url="http://example.com", body=body) - xpath = HtmlXPathSelector(response) - - xl = xpath.select('//input') - self.assertEqual(2, len(xl)) - for x in xl: - assert isinstance(x, HtmlXPathSelector) - - self.assertEqual(xpath.select('//input').extract(), - [x.extract() for x in xpath.select('//input')]) - - self.assertEqual([x.extract() for x in xpath.select("//input[@name='a']/@name")], - [u'a']) - self.assertEqual([x.extract() for x in xpath.select("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], - [u'12.0']) - - self.assertEqual(xpath.select("concat('xpath', 'rules')").extract(), - [u'xpathrules']) - self.assertEqual([x.extract() for x in xpath.select("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], - [u'12']) - - @libxml2debug - def test_selector_same_type(self): - """Test XPathSelector returning the same type in x() method""" - text = '

    test

    ' - assert isinstance(XmlXPathSelector(text=text).select("//p")[0], - XmlXPathSelector) - assert isinstance(HtmlXPathSelector(text=text).select("//p")[0], - HtmlXPathSelector) - - @libxml2debug - def test_selector_xml_html(self): - """Test that XML and HTML XPathSelector's behave differently""" - - # some text which is parsed differently by XML and HTML flavors - text = '

    Hello

    ' - - self.assertEqual(XmlXPathSelector(text=text).select("//div").extract(), - [u'

    Hello

    ']) - - self.assertEqual(HtmlXPathSelector(text=text).select("//div").extract(), - [u'

    Hello

    ']) - - @libxml2debug - def test_selector_nested(self): - """Nested selector tests""" - body = """ -
    - -
    -
    - -
    - """ - - response = HtmlResponse(url="http://example.com", body=body) - x = HtmlXPathSelector(response) - - divtwo = x.select('//div[@class="two"]') - self.assertEqual(divtwo.select("//li").extract(), - ["
  • one
    • ", "
  • two
    • ", "
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) - self.assertEqual(divtwo.select("./ul/li").extract(), - ["
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) - self.assertEqual(divtwo.select(".//li").extract(), - ["
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) - self.assertEqual(divtwo.select("./li").extract(), - []) - - @libxml2debug - def test_selector_re(self): - body = """
    Name: Mary - - Age: 20 -
    - - """ - response = HtmlResponse(url="http://example.com", body=body) - x = HtmlXPathSelector(response) - - name_re = re.compile("Name: (\w+)") - self.assertEqual(x.select("//ul/li").re(name_re), - ["John", "Paul"]) - self.assertEqual(x.select("//ul/li").re("Age: (\d+)"), - ["10", "20"]) - - @libxml2debug - def test_selector_over_text(self): - hxs = HtmlXPathSelector(text='lala') - self.assertEqual(hxs.extract(), - u'lala') - - xxs = XmlXPathSelector(text='lala') - self.assertEqual(xxs.extract(), - u'lala') - - xxs = XmlXPathSelector(text='lala') - self.assertEqual(xxs.select('.').extract(), - [u'lala']) +class XPathSelectorTestCase(XPathSelectorTestCase): + xs_cls = XPathSelector + hxs_cls = HtmlXPathSelector + xxs_cls = XmlXPathSelector @libxml2debug def test_selector_namespaces_simple(self): @@ -164,40 +58,6 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEqual(x.select("//p:SecondTestTag").select("./xmlns:price/text()")[0].extract(), '90') self.assertEqual(x.select("//p:SecondTestTag/xmlns:material").extract()[0], '') - @libxml2debug - def test_selector_invalid_xpath(self): - response = XmlResponse(url="http://example.com", body="") - x = HtmlXPathSelector(response) - xpath = "//test[@foo='bar]" - try: - x.select(xpath) - except ValueError, e: - assert xpath in str(e), "Exception message does not contain invalid xpath" - except Exception: - raise AssertionError("A invalid XPath does not raise ValueError") - else: - raise AssertionError("A invalid XPath does not raise an exception") - - @libxml2debug - def test_http_header_encoding_precedence(self): - # u'\xa3' = pound symbol in unicode - # u'\xc2\xa3' = pound symbol in utf-8 - # u'\xa3' = pound symbol in latin-1 (iso-8859-1) - - meta = u'' - head = u'' + meta + u'' - body_content = u'\xa3' - body = u'' + body_content + u'' - html = u'' + head + body + u'' - encoding = 'utf-8' - html_utf8 = html.encode(encoding) - - headers = {'Content-Type': ['text/html; charset=utf-8']} - response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) - x = HtmlXPathSelector(response) - self.assertEquals(x.select("//span[@id='blank']/text()").extract(), - [u'\xa3']) - @libxml2debug def test_null_bytes(self): hxs = HtmlXPathSelector(text='la\x00la') @@ -239,20 +99,6 @@ class XPathSelectorTestCase(unittest.TestCase): u'\n ', u'\n pff\n']) - @libxml2debug - def test_empty_bodies(self): - r1 = TextResponse('http://www.example.com', body='') - hxs = HtmlXPathSelector(r1) # shouldn't raise error - xxs = XmlXPathSelector(r1) # shouldn't raise error - - @libxml2debug - def test_weakref_slots(self): - """Check that classes are using slots and are weak-referenceable""" - for cls in [XPathSelector, HtmlXPathSelector, XmlXPathSelector]: - x = cls() - weakref.ref(x) - assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ - x.__class__.__name__ class Libxml2DocumentTest(unittest.TestCase): diff --git a/scrapy/tests/test_selector_lxml.py b/scrapy/tests/test_selector_lxml.py index 75375a57f..9e3bf5cb5 100644 --- a/scrapy/tests/test_selector_lxml.py +++ b/scrapy/tests/test_selector_lxml.py @@ -1,50 +1,26 @@ -# TODO: we should merge these tests with test_selector_libxml2.py +""" +Selectors tests, specific for lxml backend +""" -import re -import weakref - -from twisted.trial import unittest - -from scrapy.http import TextResponse, HtmlResponse, XmlResponse -nolxml = False +from scrapy.http import TextResponse, XmlResponse +has_lxml = True try: from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, \ XPathSelector except ImportError: - nolxml = True - + has_lxml = False from scrapy.utils.test import libxml2debug +from scrapy.tests.test_selector import XPathSelectorTestCase -class XPathSelectorTestCase(unittest.TestCase): +class XPathSelectorTestCase(XPathSelectorTestCase): - if nolxml: + if has_lxml: + xs_cls = XPathSelector + hxs_cls = HtmlXPathSelector + xxs_cls = XmlXPathSelector + else: skip = "lxml not available" - @libxml2debug - def test_selector_simple(self): - """Simple selector tests""" - body = "

    " - response = TextResponse(url="http://example.com", body=body) - xpath = HtmlXPathSelector(response) - - xl = xpath.select('//input') - self.assertEqual(2, len(xl)) - for x in xl: - assert isinstance(x, HtmlXPathSelector) - - self.assertEqual(xpath.select('//input').extract(), - [x.extract() for x in xpath.select('//input')]) - - self.assertEqual([x.extract() for x in xpath.select("//input[@name='a']/@name")], - [u'a']) - self.assertEqual([x.extract() for x in xpath.select("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], - [u'12.0']) - - self.assertEqual(xpath.select("concat('xpath', 'rules')").extract(), - [u'xpathrules']) - self.assertEqual([x.extract() for x in xpath.select("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], - [u'12']) - @libxml2debug def test_selector_boolean_result(self): body = "

    " @@ -53,94 +29,6 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEqual(xs.select("//input[@name='a']/@name='a'").extract(), [u'True']) self.assertEqual(xs.select("//input[@name='a']/@name='n'").extract(), [u'False']) - @libxml2debug - def test_selector_same_type(self): - """Test XPathSelector returning the same type in x() method""" - text = '

    test

    ' - assert isinstance(XmlXPathSelector(text=text).select("//p")[0], - XmlXPathSelector) - assert isinstance(HtmlXPathSelector(text=text).select("//p")[0], - HtmlXPathSelector) - - @libxml2debug - def test_selector_xml_html(self): - """Test that XML and HTML XPathSelector's behave differently""" - - # some text which is parsed differently by XML and HTML flavors - text = '

    Hello

    ' - - self.assertEqual(XmlXPathSelector(text=text).select("//div").extract(), - [u'

    Hello

    ']) - - self.assertEqual(HtmlXPathSelector(text=text).select("//div").extract(), - [u'

    Hello

    ']) - - @libxml2debug - def test_selector_nested(self): - """Nested selector tests""" - body = """ -
    - -
    -
    - -
    - """ - - response = HtmlResponse(url="http://example.com", body=body) - x = HtmlXPathSelector(response) - - divtwo = x.select('//div[@class="two"]') - self.assertEqual(divtwo.select("//li").extract(), - ["
  • one
    • ", "
  • two
    • ", "
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) - self.assertEqual(divtwo.select("./ul/li").extract(), - ["
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) - self.assertEqual(divtwo.select(".//li").extract(), - ["
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) - self.assertEqual(divtwo.select("./li").extract(), - []) - - @libxml2debug - def test_selector_re(self): - body = """
    Name: Mary - - Age: 20 -
    - - """ - response = HtmlResponse(url="http://example.com", body=body) - x = HtmlXPathSelector(response) - - name_re = re.compile("Name: (\w+)") - self.assertEqual(x.select("//ul/li").re(name_re), - ["John", "Paul"]) - self.assertEqual(x.select("//ul/li").re("Age: (\d+)"), - ["10", "20"]) - - @libxml2debug - def test_selector_over_text(self): - hxs = HtmlXPathSelector(text='lala') - self.assertEqual(hxs.extract(), - u'lala') - - xxs = XmlXPathSelector(text='lala') - self.assertEqual(xxs.extract(), - u'lala') - - xxs = XmlXPathSelector(text='lala') - self.assertEqual(xxs.select('.').extract(), - [u'lala']) - - @libxml2debug def test_selector_namespaces_simple(self): body = """ @@ -182,40 +70,6 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEqual(x.select("//p:SecondTestTag").select("./xmlns:price/text()")[0].extract(), '90') self.assertEqual(x.select("//p:SecondTestTag/xmlns:material/text()").extract()[0], 'iron') - @libxml2debug - def test_selector_invalid_xpath(self): - response = XmlResponse(url="http://example.com", body="") - x = HtmlXPathSelector(response) - xpath = "//test[@foo='bar]" - try: - x.select(xpath) - except ValueError, e: - assert xpath in str(e), "Exception message does not contain invalid xpath" - except Exception: - raise AssertionError("A invalid XPath does not raise ValueError") - else: - raise AssertionError("A invalid XPath does not raise an exception") - - @libxml2debug - def test_http_header_encoding_precedence(self): - # u'\xa3' = pound symbol in unicode - # u'\xc2\xa3' = pound symbol in utf-8 - # u'\xa3' = pound symbol in latin-1 (iso-8859-1) - - meta = u'' - head = u'' + meta + u'' - body_content = u'\xa3' - body = u'' + body_content + u'' - html = u'' + head + body + u'' - encoding = 'utf-8' - html_utf8 = html.encode(encoding) - - headers = {'Content-Type': ['text/html; charset=utf-8']} - response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) - x = HtmlXPathSelector(response) - self.assertEquals(x.select("//span[@id='blank']/text()").extract(), - [u'\xa3']) - # XXX: this test was disabled because lxml behaves inconsistently when # handling null bytes between different 2.2.x versions, but it may be due # to differences in libxml2 too. it's also unclear what should be the @@ -229,51 +83,3 @@ class XPathSelectorTestCase(unittest.TestCase): # xxs = XmlXPathSelector(text='la\x00la') # self.assertEqual(xxs.extract(), # u'la') - - @libxml2debug - def test_unquote(self): - xmldoc = '\n'.join(( - '', - ' lala', - ' ', - ' blabla&moreatestoh', - ' PPPPppp&la]]>', - ' ', - ' pff', - '')) - xxs = XmlXPathSelector(text=xmldoc) - - # this tests were commented out because they make no sense (pablo) - #self.assertEqual(xxs.extract_unquoted(), u'') - #self.assertEqual(xxs.select('/root').extract_unquoted(), [u'']) - #self.assertEqual(xxs.select('//*').extract_unquoted(), [u'', u'', u'']) - - self.assertEqual(xxs.select('/root/text()').extract_unquoted(), [ - u'lala', - u'pff']) - - self.assertEqual(xxs.select('//text()').extract_unquoted(), [ - u'lala', - u'blabla&more', - u'a', - u'test', - u'oh\n lalalal&pppppPPPPppp&la', - u'pff']) - - @libxml2debug - def test_empty_bodies(self): - r1 = TextResponse('http://www.example.com', body='') - hxs = HtmlXPathSelector(r1) # shouldn't raise error - xxs = XmlXPathSelector(r1) # shouldn't raise error - - @libxml2debug - def test_weakref_slots(self): - """Check that classes are using slots and are weak-referenceable""" - for cls in [XPathSelector, HtmlXPathSelector, XmlXPathSelector]: - x = cls() - weakref.ref(x) - assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ - x.__class__.__name__ - -if __name__ == "__main__": - unittest.main()