import re import warnings import weakref import six from twisted.trial import unittest from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import TextResponse, HtmlResponse, XmlResponse from scrapy.selector import Selector from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector from lxml import etree class SelectorTestCase(unittest.TestCase): def test_simple_selection(self): """Simple selector tests""" body = u"

" response = TextResponse(url="http://example.com", body=body, encoding='utf-8') sel = Selector(response) xl = sel.xpath('//input') self.assertEqual(2, len(xl)) for x in xl: assert isinstance(x, Selector) self.assertEqual(sel.xpath('//input').extract(), [x.extract() for x in sel.xpath('//input')]) self.assertEqual([x.extract() for x in sel.xpath("//input[@name='a']/@name")], [u'a']) self.assertEqual([x.extract() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], [u'12.0']) self.assertEqual(sel.xpath("concat('xpath', 'rules')").extract(), [u'xpathrules']) self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], [u'12']) def test_deprecated_root_argument(self): with warnings.catch_warnings(record=True) as w: root = etree.fromstring(u'') sel = Selector(_root=root) self.assertIs(root, sel.root) self.assertEqual(str(w[-1].message), 'Argument `_root` is deprecated, use `root` instead') def test_deprecated_root_argument_ambiguous(self): with warnings.catch_warnings(record=True) as w: _root = etree.fromstring(u'') root = etree.fromstring(u'') sel = Selector(_root=_root, root=root) self.assertIs(root, sel.root) self.assertIn('Ignoring deprecated `_root` argument', str(w[-1].message)) def test_flavor_detection(self): text = u'

Hello

' sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8')) self.assertEqual(sel.type, 'xml') self.assertEqual(sel.xpath("//div").extract(), [u'

Hello

']) sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8')) self.assertEqual(sel.type, 'html') self.assertEqual(sel.xpath("//div").extract(), [u'

Hello

']) def test_http_header_encoding_precedence(self): # u'\xa3' = pound symbol in unicode # u'\xc2\xa3' = pound symbol in utf-8 # u'\xa3' = pound symbol in latin-1 (iso-8859-1) meta = u'' head = u'' + meta + u'' body_content = u'\xa3' body = u'' + body_content + u'' html = u'' + head + body + u'' encoding = 'utf-8' html_utf8 = html.encode(encoding) headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) x = Selector(response) self.assertEquals(x.xpath("//span[@id='blank']/text()").extract(), [u'\xa3']) def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence r1 = TextResponse('http://www.example.com', \ body=u'

an Jos\xe9 de

', \ encoding='utf-8') Selector(r1).xpath('//text()').extract() def test_weakref_slots(self): """Check that classes are using slots and are weak-referenceable""" x = Selector(text='') weakref.ref(x) assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ x.__class__.__name__ class DeprecatedXpathSelectorTest(unittest.TestCase): text = u'

Hello

' def test_warnings_xpathselector(self): cls = XPathSelector with warnings.catch_warnings(record=True) as w: class UserClass(cls): pass # subclassing must issue a warning self.assertEqual(len(w), 1, str(cls)) self.assertIn('scrapy.Selector', str(w[0].message)) # subclass instance doesn't issue a warning usel = UserClass(text=self.text) self.assertEqual(len(w), 1) # class instance must issue a warning sel = cls(text=self.text) self.assertEqual(len(w), 2, str((cls, [x.message for x in w]))) self.assertIn('scrapy.Selector', str(w[1].message)) # subclass and instance checks self.assertTrue(issubclass(cls, Selector)) self.assertTrue(isinstance(sel, Selector)) self.assertTrue(isinstance(usel, Selector)) def test_warnings_xmlxpathselector(self): cls = XmlXPathSelector with warnings.catch_warnings(record=True) as w: class UserClass(cls): pass # subclassing must issue a warning self.assertEqual(len(w), 1, str(cls)) self.assertIn('scrapy.Selector', str(w[0].message)) # subclass instance doesn't issue a warning usel = UserClass(text=self.text) self.assertEqual(len(w), 1) # class instance must issue a warning sel = cls(text=self.text) self.assertEqual(len(w), 2, str((cls, [x.message for x in w]))) self.assertIn('scrapy.Selector', str(w[1].message)) # subclass and instance checks self.assertTrue(issubclass(cls, Selector)) self.assertTrue(issubclass(cls, XPathSelector)) self.assertTrue(isinstance(sel, Selector)) self.assertTrue(isinstance(usel, Selector)) self.assertTrue(isinstance(sel, XPathSelector)) self.assertTrue(isinstance(usel, XPathSelector)) def test_warnings_htmlxpathselector(self): cls = HtmlXPathSelector with warnings.catch_warnings(record=True) as w: class UserClass(cls): pass # subclassing must issue a warning self.assertEqual(len(w), 1, str(cls)) self.assertIn('scrapy.Selector', str(w[0].message)) # subclass instance doesn't issue a warning usel = UserClass(text=self.text) self.assertEqual(len(w), 1) # class instance must issue a warning sel = cls(text=self.text) self.assertEqual(len(w), 2, str((cls, [x.message for x in w]))) self.assertIn('scrapy.Selector', str(w[1].message)) # subclass and instance checks self.assertTrue(issubclass(cls, Selector)) self.assertTrue(issubclass(cls, XPathSelector)) self.assertTrue(isinstance(sel, Selector)) self.assertTrue(isinstance(usel, Selector)) self.assertTrue(isinstance(sel, XPathSelector)) self.assertTrue(isinstance(usel, XPathSelector))