mirror of https://github.com/scrapy/scrapy.git
Support only ::text and ::attr(name) + add more CSS selector tests
This commit is contained in:
Paul Tremberth
Daniel Graña
parent
52e2eb6b70
commit
ea579bca04
|
|
@ -46,18 +46,6 @@ class TranslatorMixin(object):
|
|||
xpath = super(TranslatorMixin, self).xpath_element(selector)
|
||||
return ScrapyXPathExpr.from_xpath(xpath)
|
||||
|
||||
def xpath_text_pseudo(self, xpath):
|
||||
"""Support selecting text nodes using :text pseudo-element"""
|
||||
return ScrapyXPathExpr.from_xpath(xpath, textnode=True)
|
||||
|
||||
def xpath_attribute_function(self, xpath, function):
|
||||
if function.argument_types() not in (['STRING'], ['IDENT']):
|
||||
raise ExpressionError(
|
||||
"Expected a single string or ident for :contains(), got %r"
|
||||
% function.arguments)
|
||||
value = function.arguments[0].value
|
||||
return ScrapyXPathExpr.from_xpath(xpath, attribute=value)
|
||||
|
||||
def xpath_pseudo_element(self, xpath, pseudo_element):
|
||||
if isinstance(pseudo_element, FunctionalPseudoElement):
|
||||
method = 'xpath_%s_functional_pseudo_element' % (
|
||||
|
|
@ -79,11 +67,17 @@ class TranslatorMixin(object):
|
|||
xpath = method(xpath)
|
||||
return xpath
|
||||
|
||||
def xpath_attribute_functional_pseudo_element(self, xpath, arguments):
|
||||
return self.xpath_attribute_function(xpath, arguments)
|
||||
def xpath_attr_functional_pseudo_element(self, xpath, function):
|
||||
if function.argument_types() not in (['STRING'], ['IDENT']):
|
||||
raise ExpressionError(
|
||||
"Expected a single string or ident for ::attr(), got %r"
|
||||
% function.arguments)
|
||||
return ScrapyXPathExpr.from_xpath(xpath,
|
||||
attribute=function.arguments[0].value)
|
||||
|
||||
def xpath_text_simple_pseudo_element(self, xpath):
|
||||
return self.xpath_text_pseudo(xpath)
|
||||
"""Support selecting text nodes using ::text pseudo-element"""
|
||||
return ScrapyXPathExpr.from_xpath(xpath, textnode=True)
|
||||
|
||||
|
||||
class ScrapyGenericTranslator(TranslatorMixin, GenericTranslator):
|
||||
|
|
|
|||
|
|
@ -5,6 +5,9 @@ from twisted.trial import unittest
|
|||
from scrapy.http import TextResponse, HtmlResponse, XmlResponse
|
||||
from scrapy.selector import CSSSelector, XmlCSSSelector, HtmlCSSSelector
|
||||
from scrapy.selector.csssel import ScrapyHTMLTranslator
|
||||
from cssselect.parser import SelectorSyntaxError
|
||||
from cssselect.xpath import ExpressionError
|
||||
|
||||
|
||||
HTMLBODY = '''
|
||||
<html>
|
||||
|
|
@ -50,44 +53,26 @@ class TranslatorMixinTest(unittest.TestCase):
|
|||
self.tr = self.tr_cls()
|
||||
self.c2x = self.tr.css_to_xpath
|
||||
|
||||
def test_attribute_function(self):
|
||||
def test_attr_function(self):
|
||||
cases = [
|
||||
(':attribute(name)', u'descendant-or-self::*/@name'),
|
||||
('a:attribute(name)', u'descendant-or-self::a/@name'),
|
||||
('a :attribute(name)', u'descendant-or-self::a/descendant-or-self::*/@name'),
|
||||
('a > :attribute(name)', u'descendant-or-self::a/*/@name'),
|
||||
('::attr(name)', u'descendant-or-self::*/@name'),
|
||||
('a::attr(href)', u'descendant-or-self::a/@href'),
|
||||
('a ::attr(img)', u'descendant-or-self::a/descendant-or-self::*/@img'),
|
||||
('a > ::attr(class)', u'descendant-or-self::a/*/@class'),
|
||||
]
|
||||
for css, xpath in cases:
|
||||
self.assertEqual(self.c2x(css), xpath, css)
|
||||
|
||||
def test_attribute_function2(self):
|
||||
def test_attr_function_exception(self):
|
||||
cases = [
|
||||
('::attribute(name)', u'descendant-or-self::*/@name'),
|
||||
('a::attribute(name)', u'descendant-or-self::a/@name'),
|
||||
('a ::attribute(name)', u'descendant-or-self::a/descendant-or-self::*/@name'),
|
||||
('a > ::attribute(name)', u'descendant-or-self::a/*/@name'),
|
||||
('::attr(12)', ExpressionError),
|
||||
('::attr(34test)', ExpressionError),
|
||||
('::attr(@href)', SelectorSyntaxError),
|
||||
]
|
||||
for css, xpath in cases:
|
||||
self.assertEqual(self.c2x(css), xpath, css)
|
||||
for css, exc in cases:
|
||||
self.assertRaises(exc, self.c2x, css)
|
||||
|
||||
def test_text_pseudo_element(self):
|
||||
cases = [
|
||||
(':text', u'descendant-or-self::text()'),
|
||||
('p:text', u'descendant-or-self::p/text()'),
|
||||
('p :text', u'descendant-or-self::p/descendant-or-self::text()'),
|
||||
('#id:text', u"descendant-or-self::*[@id = 'id']/text()"),
|
||||
('p#id:text', u"descendant-or-self::p[@id = 'id']/text()"),
|
||||
('p#id :text', u"descendant-or-self::p[@id = 'id']/descendant-or-self::text()"),
|
||||
('p#id > :text', u"descendant-or-self::p[@id = 'id']/*/text()"),
|
||||
('p#id ~ :text', u"descendant-or-self::p[@id = 'id']/following-sibling::*/text()"),
|
||||
('a[href]:text', u'descendant-or-self::a[@href]/text()'),
|
||||
('a[href] :text', u'descendant-or-self::a[@href]/descendant-or-self::text()'),
|
||||
('p:text, a:text', u"descendant-or-self::p/text() | descendant-or-self::a/text()"),
|
||||
]
|
||||
for css, xpath in cases:
|
||||
self.assertEqual(self.c2x(css), xpath, css)
|
||||
|
||||
def test_text_pseudo_element2(self):
|
||||
cases = [
|
||||
('::text', u'descendant-or-self::text()'),
|
||||
('p::text', u'descendant-or-self::p/text()'),
|
||||
|
|
@ -99,11 +84,36 @@ class TranslatorMixinTest(unittest.TestCase):
|
|||
('p#id ~ ::text', u"descendant-or-self::p[@id = 'id']/following-sibling::*/text()"),
|
||||
('a[href]::text', u'descendant-or-self::a[@href]/text()'),
|
||||
('a[href] ::text', u'descendant-or-self::a[@href]/descendant-or-self::text()'),
|
||||
('p:text, a::text', u"descendant-or-self::p/text() | descendant-or-self::a/text()"),
|
||||
('p::text, a::text', u"descendant-or-self::p/text() | descendant-or-self::a/text()"),
|
||||
]
|
||||
for css, xpath in cases:
|
||||
self.assertEqual(self.c2x(css), xpath, css)
|
||||
|
||||
def test_pseudo_function_exception(self):
|
||||
cases = [
|
||||
('::attribute(12)', ExpressionError),
|
||||
('::text()', ExpressionError),
|
||||
('::attr(@href)', SelectorSyntaxError),
|
||||
]
|
||||
for css, exc in cases:
|
||||
self.assertRaises(exc, self.c2x, css)
|
||||
|
||||
def test_unknown_pseudo_element(self):
|
||||
cases = [
|
||||
('::text-node', ExpressionError),
|
||||
]
|
||||
for css, exc in cases:
|
||||
self.assertRaises(exc, self.c2x, css)
|
||||
|
||||
def test_unknown_pseudo_class(self):
|
||||
cases = [
|
||||
(':text', ExpressionError),
|
||||
(':attribute(name)', ExpressionError),
|
||||
]
|
||||
for css, exc in cases:
|
||||
self.assertRaises(exc, self.c2x, css)
|
||||
|
||||
|
||||
class HTMLCSSSelectorTest(unittest.TestCase):
|
||||
|
||||
hcs_cls = HtmlCSSSelector
|
||||
|
|
@ -123,21 +133,21 @@ class HTMLCSSSelectorTest(unittest.TestCase):
|
|||
|
||||
def test_text_pseudo_element(self):
|
||||
self.assertEqual(self.x('#p-b2'), [u'<b id="p-b2">guy</b>'])
|
||||
self.assertEqual(self.x('#p-b2:text'), [u'guy'])
|
||||
self.assertEqual(self.x('#p-b2 :text'), [u'guy'])
|
||||
self.assertEqual(self.x('#paragraph:text'), [u'lorem ipsum text'])
|
||||
self.assertEqual(self.x('#paragraph :text'), [u'lorem ipsum text', u'hi', u'there', u'guy'])
|
||||
self.assertEqual(self.x('p:text'), [u'lorem ipsum text'])
|
||||
self.assertEqual(self.x('p :text'), [u'lorem ipsum text', u'hi', u'there', u'guy'])
|
||||
self.assertEqual(self.x('#p-b2::text'), [u'guy'])
|
||||
self.assertEqual(self.x('#p-b2 ::text'), [u'guy'])
|
||||
self.assertEqual(self.x('#paragraph::text'), [u'lorem ipsum text'])
|
||||
self.assertEqual(self.x('#paragraph ::text'), [u'lorem ipsum text', u'hi', u'there', u'guy'])
|
||||
self.assertEqual(self.x('p::text'), [u'lorem ipsum text'])
|
||||
self.assertEqual(self.x('p ::text'), [u'lorem ipsum text', u'hi', u'there', u'guy'])
|
||||
|
||||
def test_attribute_function(self):
|
||||
self.assertEqual(self.x('#p-b2:attribute(id)'), [u'p-b2'])
|
||||
self.assertEqual(self.x('.cool-footer:attribute(class)'), [u'cool-footer'])
|
||||
self.assertEqual(self.x('.cool-footer :attribute(id)'), [u'foobar-div', u'foobar-span'])
|
||||
self.assertEqual(self.x('map[name="dummymap"] :attribute(shape)'), [u'circle', u'default'])
|
||||
self.assertEqual(self.x('#p-b2::attr(id)'), [u'p-b2'])
|
||||
self.assertEqual(self.x('.cool-footer::attr(class)'), [u'cool-footer'])
|
||||
self.assertEqual(self.x('.cool-footer ::attr(id)'), [u'foobar-div', u'foobar-span'])
|
||||
self.assertEqual(self.x('map[name="dummymap"] ::attr(shape)'), [u'circle', u'default'])
|
||||
|
||||
def test_nested_selector(self):
|
||||
self.assertEqual(self.hcs.select('p').select('b:text').extract(),
|
||||
self.assertEqual(self.hcs.select('p').select('b::text').extract(),
|
||||
[u'hi', u'guy'])
|
||||
self.assertEqual(self.hcs.select('div').select('area:last-child').extract(),
|
||||
[u'<area shape="default" id="area-nohref">'])
|
||||
|
|
|
|||
Loading…
Reference in New Issue