From ea579bca0456f4e0da656af7c39d09b793f417ef Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 16 Sep 2013 15:36:35 +0200 Subject: [PATCH] Support only ::text and ::attr(name) + add more CSS selector tests --- scrapy/selector/csssel.py | 24 +++---- scrapy/tests/test_selector_cssselect.py | 92 ++++++++++++++----------- 2 files changed, 60 insertions(+), 56 deletions(-) diff --git a/scrapy/selector/csssel.py b/scrapy/selector/csssel.py index 90a3261df..e8c5538aa 100644 --- a/scrapy/selector/csssel.py +++ b/scrapy/selector/csssel.py @@ -46,18 +46,6 @@ class TranslatorMixin(object): xpath = super(TranslatorMixin, self).xpath_element(selector) return ScrapyXPathExpr.from_xpath(xpath) - def xpath_text_pseudo(self, xpath): - """Support selecting text nodes using :text pseudo-element""" - return ScrapyXPathExpr.from_xpath(xpath, textnode=True) - - def xpath_attribute_function(self, xpath, function): - if function.argument_types() not in (['STRING'], ['IDENT']): - raise ExpressionError( - "Expected a single string or ident for :contains(), got %r" - % function.arguments) - value = function.arguments[0].value - return ScrapyXPathExpr.from_xpath(xpath, attribute=value) - def xpath_pseudo_element(self, xpath, pseudo_element): if isinstance(pseudo_element, FunctionalPseudoElement): method = 'xpath_%s_functional_pseudo_element' % ( @@ -79,11 +67,17 @@ class TranslatorMixin(object): xpath = method(xpath) return xpath - def xpath_attribute_functional_pseudo_element(self, xpath, arguments): - return self.xpath_attribute_function(xpath, arguments) + def xpath_attr_functional_pseudo_element(self, xpath, function): + if function.argument_types() not in (['STRING'], ['IDENT']): + raise ExpressionError( + "Expected a single string or ident for ::attr(), got %r" + % function.arguments) + return ScrapyXPathExpr.from_xpath(xpath, + attribute=function.arguments[0].value) def xpath_text_simple_pseudo_element(self, xpath): - return self.xpath_text_pseudo(xpath) + """Support selecting text nodes using ::text pseudo-element""" + return ScrapyXPathExpr.from_xpath(xpath, textnode=True) class ScrapyGenericTranslator(TranslatorMixin, GenericTranslator): diff --git a/scrapy/tests/test_selector_cssselect.py b/scrapy/tests/test_selector_cssselect.py index 058ebc783..4b69dff15 100644 --- a/scrapy/tests/test_selector_cssselect.py +++ b/scrapy/tests/test_selector_cssselect.py @@ -5,6 +5,9 @@ from twisted.trial import unittest from scrapy.http import TextResponse, HtmlResponse, XmlResponse from scrapy.selector import CSSSelector, XmlCSSSelector, HtmlCSSSelector from scrapy.selector.csssel import ScrapyHTMLTranslator +from cssselect.parser import SelectorSyntaxError +from cssselect.xpath import ExpressionError + HTMLBODY = ''' @@ -50,44 +53,26 @@ class TranslatorMixinTest(unittest.TestCase): self.tr = self.tr_cls() self.c2x = self.tr.css_to_xpath - def test_attribute_function(self): + def test_attr_function(self): cases = [ - (':attribute(name)', u'descendant-or-self::*/@name'), - ('a:attribute(name)', u'descendant-or-self::a/@name'), - ('a :attribute(name)', u'descendant-or-self::a/descendant-or-self::*/@name'), - ('a > :attribute(name)', u'descendant-or-self::a/*/@name'), + ('::attr(name)', u'descendant-or-self::*/@name'), + ('a::attr(href)', u'descendant-or-self::a/@href'), + ('a ::attr(img)', u'descendant-or-self::a/descendant-or-self::*/@img'), + ('a > ::attr(class)', u'descendant-or-self::a/*/@class'), ] for css, xpath in cases: self.assertEqual(self.c2x(css), xpath, css) - def test_attribute_function2(self): + def test_attr_function_exception(self): cases = [ - ('::attribute(name)', u'descendant-or-self::*/@name'), - ('a::attribute(name)', u'descendant-or-self::a/@name'), - ('a ::attribute(name)', u'descendant-or-self::a/descendant-or-self::*/@name'), - ('a > ::attribute(name)', u'descendant-or-self::a/*/@name'), + ('::attr(12)', ExpressionError), + ('::attr(34test)', ExpressionError), + ('::attr(@href)', SelectorSyntaxError), ] - for css, xpath in cases: - self.assertEqual(self.c2x(css), xpath, css) + for css, exc in cases: + self.assertRaises(exc, self.c2x, css) def test_text_pseudo_element(self): - cases = [ - (':text', u'descendant-or-self::text()'), - ('p:text', u'descendant-or-self::p/text()'), - ('p :text', u'descendant-or-self::p/descendant-or-self::text()'), - ('#id:text', u"descendant-or-self::*[@id = 'id']/text()"), - ('p#id:text', u"descendant-or-self::p[@id = 'id']/text()"), - ('p#id :text', u"descendant-or-self::p[@id = 'id']/descendant-or-self::text()"), - ('p#id > :text', u"descendant-or-self::p[@id = 'id']/*/text()"), - ('p#id ~ :text', u"descendant-or-self::p[@id = 'id']/following-sibling::*/text()"), - ('a[href]:text', u'descendant-or-self::a[@href]/text()'), - ('a[href] :text', u'descendant-or-self::a[@href]/descendant-or-self::text()'), - ('p:text, a:text', u"descendant-or-self::p/text() | descendant-or-self::a/text()"), - ] - for css, xpath in cases: - self.assertEqual(self.c2x(css), xpath, css) - - def test_text_pseudo_element2(self): cases = [ ('::text', u'descendant-or-self::text()'), ('p::text', u'descendant-or-self::p/text()'), @@ -99,11 +84,36 @@ class TranslatorMixinTest(unittest.TestCase): ('p#id ~ ::text', u"descendant-or-self::p[@id = 'id']/following-sibling::*/text()"), ('a[href]::text', u'descendant-or-self::a[@href]/text()'), ('a[href] ::text', u'descendant-or-self::a[@href]/descendant-or-self::text()'), - ('p:text, a::text', u"descendant-or-self::p/text() | descendant-or-self::a/text()"), + ('p::text, a::text', u"descendant-or-self::p/text() | descendant-or-self::a/text()"), ] for css, xpath in cases: self.assertEqual(self.c2x(css), xpath, css) + def test_pseudo_function_exception(self): + cases = [ + ('::attribute(12)', ExpressionError), + ('::text()', ExpressionError), + ('::attr(@href)', SelectorSyntaxError), + ] + for css, exc in cases: + self.assertRaises(exc, self.c2x, css) + + def test_unknown_pseudo_element(self): + cases = [ + ('::text-node', ExpressionError), + ] + for css, exc in cases: + self.assertRaises(exc, self.c2x, css) + + def test_unknown_pseudo_class(self): + cases = [ + (':text', ExpressionError), + (':attribute(name)', ExpressionError), + ] + for css, exc in cases: + self.assertRaises(exc, self.c2x, css) + + class HTMLCSSSelectorTest(unittest.TestCase): hcs_cls = HtmlCSSSelector @@ -123,21 +133,21 @@ class HTMLCSSSelectorTest(unittest.TestCase): def test_text_pseudo_element(self): self.assertEqual(self.x('#p-b2'), [u'guy']) - self.assertEqual(self.x('#p-b2:text'), [u'guy']) - self.assertEqual(self.x('#p-b2 :text'), [u'guy']) - self.assertEqual(self.x('#paragraph:text'), [u'lorem ipsum text']) - self.assertEqual(self.x('#paragraph :text'), [u'lorem ipsum text', u'hi', u'there', u'guy']) - self.assertEqual(self.x('p:text'), [u'lorem ipsum text']) - self.assertEqual(self.x('p :text'), [u'lorem ipsum text', u'hi', u'there', u'guy']) + self.assertEqual(self.x('#p-b2::text'), [u'guy']) + self.assertEqual(self.x('#p-b2 ::text'), [u'guy']) + self.assertEqual(self.x('#paragraph::text'), [u'lorem ipsum text']) + self.assertEqual(self.x('#paragraph ::text'), [u'lorem ipsum text', u'hi', u'there', u'guy']) + self.assertEqual(self.x('p::text'), [u'lorem ipsum text']) + self.assertEqual(self.x('p ::text'), [u'lorem ipsum text', u'hi', u'there', u'guy']) def test_attribute_function(self): - self.assertEqual(self.x('#p-b2:attribute(id)'), [u'p-b2']) - self.assertEqual(self.x('.cool-footer:attribute(class)'), [u'cool-footer']) - self.assertEqual(self.x('.cool-footer :attribute(id)'), [u'foobar-div', u'foobar-span']) - self.assertEqual(self.x('map[name="dummymap"] :attribute(shape)'), [u'circle', u'default']) + self.assertEqual(self.x('#p-b2::attr(id)'), [u'p-b2']) + self.assertEqual(self.x('.cool-footer::attr(class)'), [u'cool-footer']) + self.assertEqual(self.x('.cool-footer ::attr(id)'), [u'foobar-div', u'foobar-span']) + self.assertEqual(self.x('map[name="dummymap"] ::attr(shape)'), [u'circle', u'default']) def test_nested_selector(self): - self.assertEqual(self.hcs.select('p').select('b:text').extract(), + self.assertEqual(self.hcs.select('p').select('b::text').extract(), [u'hi', u'guy']) self.assertEqual(self.hcs.select('div').select('area:last-child').extract(), [u''])