diff --git a/scrapy/selector/csssel.py b/scrapy/selector/csssel.py index d5b571bae..90a3261df 100644 --- a/scrapy/selector/csssel.py +++ b/scrapy/selector/csssel.py @@ -1,5 +1,6 @@ from cssselect import GenericTranslator, HTMLTranslator -from cssselect.xpath import XPathExpr, ExpressionError +from cssselect.xpath import _unicode_safe_getattr, XPathExpr, ExpressionError +from cssselect.parser import FunctionalPseudoElement from scrapy.selector import XPathSelector, HtmlXPathSelector, XmlXPathSelector @@ -57,6 +58,33 @@ class TranslatorMixin(object): value = function.arguments[0].value return ScrapyXPathExpr.from_xpath(xpath, attribute=value) + def xpath_pseudo_element(self, xpath, pseudo_element): + if isinstance(pseudo_element, FunctionalPseudoElement): + method = 'xpath_%s_functional_pseudo_element' % ( + pseudo_element.name.replace('-', '_')) + method = _unicode_safe_getattr(self, method, None) + if not method: + raise ExpressionError( + "The functional pseudo-element ::%s() is unknown" + % pseudo_element.name) + xpath = method(xpath, pseudo_element) + else: + method = 'xpath_%s_simple_pseudo_element' % ( + pseudo_element.replace('-', '_')) + method = _unicode_safe_getattr(self, method, None) + if not method: + raise ExpressionError( + "The pseudo-element ::%s is unknown" + % pseudo_element) + xpath = method(xpath) + return xpath + + def xpath_attribute_functional_pseudo_element(self, xpath, arguments): + return self.xpath_attribute_function(xpath, arguments) + + def xpath_text_simple_pseudo_element(self, xpath): + return self.xpath_text_pseudo(xpath) + class ScrapyGenericTranslator(TranslatorMixin, GenericTranslator): pass diff --git a/scrapy/tests/test_selector_cssselect.py b/scrapy/tests/test_selector_cssselect.py index f406a80ee..058ebc783 100644 --- a/scrapy/tests/test_selector_cssselect.py +++ b/scrapy/tests/test_selector_cssselect.py @@ -60,6 +60,16 @@ class TranslatorMixinTest(unittest.TestCase): for css, xpath in cases: self.assertEqual(self.c2x(css), xpath, css) + def test_attribute_function2(self): + cases = [ + ('::attribute(name)', u'descendant-or-self::*/@name'), + ('a::attribute(name)', u'descendant-or-self::a/@name'), + ('a ::attribute(name)', u'descendant-or-self::a/descendant-or-self::*/@name'), + ('a > ::attribute(name)', u'descendant-or-self::a/*/@name'), + ] + for css, xpath in cases: + self.assertEqual(self.c2x(css), xpath, css) + def test_text_pseudo_element(self): cases = [ (':text', u'descendant-or-self::text()'), @@ -77,6 +87,22 @@ class TranslatorMixinTest(unittest.TestCase): for css, xpath in cases: self.assertEqual(self.c2x(css), xpath, css) + def test_text_pseudo_element2(self): + cases = [ + ('::text', u'descendant-or-self::text()'), + ('p::text', u'descendant-or-self::p/text()'), + ('p ::text', u'descendant-or-self::p/descendant-or-self::text()'), + ('#id::text', u"descendant-or-self::*[@id = 'id']/text()"), + ('p#id::text', u"descendant-or-self::p[@id = 'id']/text()"), + ('p#id ::text', u"descendant-or-self::p[@id = 'id']/descendant-or-self::text()"), + ('p#id > ::text', u"descendant-or-self::p[@id = 'id']/*/text()"), + ('p#id ~ ::text', u"descendant-or-self::p[@id = 'id']/following-sibling::*/text()"), + ('a[href]::text', u'descendant-or-self::a[@href]/text()'), + ('a[href] ::text', u'descendant-or-self::a[@href]/descendant-or-self::text()'), + ('p:text, a::text', u"descendant-or-self::p/text() | descendant-or-self::a/text()"), + ] + for css, xpath in cases: + self.assertEqual(self.c2x(css), xpath, css) class HTMLCSSSelectorTest(unittest.TestCase):