RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adapt to latest cssselect API supporting pseudo-elements

This commit is contained in:
Paul Tremberth 2013-09-16 13:26:34 +02:00 committed by Daniel Graña
parent 4e6967b854
commit 52e2eb6b70
2 changed files with 55 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
scrapy/selector/csssel.py
View File

@ -1,5 +1,6 @@
from cssselect import GenericTranslator, HTMLTranslator
from cssselect.xpath import XPathExpr, ExpressionError
from cssselect.xpath import _unicode_safe_getattr, XPathExpr, ExpressionError
from cssselect.parser import FunctionalPseudoElement
from scrapy.selector import XPathSelector, HtmlXPathSelector, XmlXPathSelector
@ -57,6 +58,33 @@ class TranslatorMixin(object):
value = function.arguments[0].value
return ScrapyXPathExpr.from_xpath(xpath, attribute=value)
def xpath_pseudo_element(self, xpath, pseudo_element):
if isinstance(pseudo_element, FunctionalPseudoElement):
method = 'xpath_%s_functional_pseudo_element' % (
pseudo_element.name.replace('-', '_'))
method = _unicode_safe_getattr(self, method, None)
if not method:
raise ExpressionError(
"The functional pseudo-element ::%s() is unknown"
% pseudo_element.name)
xpath = method(xpath, pseudo_element)
else:
method = 'xpath_%s_simple_pseudo_element' % (
pseudo_element.replace('-', '_'))
method = _unicode_safe_getattr(self, method, None)
if not method:
raise ExpressionError(
"The pseudo-element ::%s is unknown"
% pseudo_element)
xpath = method(xpath)
return xpath
def xpath_attribute_functional_pseudo_element(self, xpath, arguments):
return self.xpath_attribute_function(xpath, arguments)
def xpath_text_simple_pseudo_element(self, xpath):
return self.xpath_text_pseudo(xpath)
class ScrapyGenericTranslator(TranslatorMixin, GenericTranslator):
pass

26
scrapy/tests/test_selector_cssselect.py
View File

@ -60,6 +60,16 @@ class TranslatorMixinTest(unittest.TestCase):
for css, xpath in cases:
self.assertEqual(self.c2x(css), xpath, css)
def test_attribute_function2(self):
cases = [
('::attribute(name)', u'descendant-or-self::*/@name'),
('a::attribute(name)', u'descendant-or-self::a/@name'),
('a ::attribute(name)', u'descendant-or-self::a/descendant-or-self::*/@name'),
('a > ::attribute(name)', u'descendant-or-self::a/*/@name'),
]
for css, xpath in cases:
self.assertEqual(self.c2x(css), xpath, css)
def test_text_pseudo_element(self):
cases = [
(':text', u'descendant-or-self::text()'),
@ -77,6 +87,22 @@ class TranslatorMixinTest(unittest.TestCase):
for css, xpath in cases:
self.assertEqual(self.c2x(css), xpath, css)
def test_text_pseudo_element2(self):
cases = [
('::text', u'descendant-or-self::text()'),
('p::text', u'descendant-or-self::p/text()'),
('p ::text', u'descendant-or-self::p/descendant-or-self::text()'),
('#id::text', u"descendant-or-self::*[@id = 'id']/text()"),
('p#id::text', u"descendant-or-self::p[@id = 'id']/text()"),
('p#id ::text', u"descendant-or-self::p[@id = 'id']/descendant-or-self::text()"),
('p#id > ::text', u"descendant-or-self::p[@id = 'id']/*/text()"),
('p#id ~ ::text', u"descendant-or-self::p[@id = 'id']/following-sibling::*/text()"),
('a[href]::text', u'descendant-or-self::a[@href]/text()'),
('a[href] ::text', u'descendant-or-self::a[@href]/descendant-or-self::text()'),
('p:text, a::text', u"descendant-or-self::p/text() | descendant-or-self::a/text()"),
]
for css, xpath in cases:
self.assertEqual(self.c2x(css), xpath, css)
class HTMLCSSSelectorTest(unittest.TestCase):