diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index 7dd420d1d..a1203f987 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -1,17 +1,6 @@ """ Selectors """ +from scrapy.selector.unified import * from scrapy.selector.lxmlsel import * from scrapy.selector.csssel import * -from scrapy.selector.list import SelectorList - - -class XPathSelectorList(SelectorList): - - def __init__(self, *a, **kw): - import warnings - from scrapy.exceptions import ScrapyDeprecationWarning - warnings.warn('XPathSelectorList is deprecated, use ' - 'scrapy.selector.SelectorList instead', - category=ScrapyDeprecationWarning, stacklevel=1) - super(XPathSelectorList, self).__init__(*a, **kw) diff --git a/scrapy/selector/csssel.py b/scrapy/selector/csssel.py index e8c5538aa..a2fc12e8d 100644 --- a/scrapy/selector/csssel.py +++ b/scrapy/selector/csssel.py @@ -1,110 +1,16 @@ -from cssselect import GenericTranslator, HTMLTranslator -from cssselect.xpath import _unicode_safe_getattr, XPathExpr, ExpressionError -from cssselect.parser import FunctionalPseudoElement -from scrapy.selector import XPathSelector, HtmlXPathSelector, XmlXPathSelector +from .unified import Selector -class ScrapyXPathExpr(XPathExpr): +class CSSSelector(Selector): - textnode = False - attribute = None - - @classmethod - def from_xpath(cls, xpath, textnode=False, attribute=None): - x = cls(path=xpath.path, element=xpath.element, condition=xpath.condition) - x.textnode = textnode - x.attribute = attribute - return x - - def __str__(self): - path = super(ScrapyXPathExpr, self).__str__() - if self.textnode: - if path == '*': - path = 'text()' - elif path.endswith('::*/*'): - path = path[:-3] + 'text()' - else: - path += '/text()' - - if self.attribute is not None: - if path.endswith('::*/*'): - path = path[:-2] - path += '/@%s' % self.attribute - - return path - - def join(self, combiner, other): - super(ScrapyXPathExpr, self).join(combiner, other) - self.textnode = other.textnode - self.attribute = other.attribute - return self - - -class TranslatorMixin(object): - - def xpath_element(self, selector): - xpath = super(TranslatorMixin, self).xpath_element(selector) - return ScrapyXPathExpr.from_xpath(xpath) - - def xpath_pseudo_element(self, xpath, pseudo_element): - if isinstance(pseudo_element, FunctionalPseudoElement): - method = 'xpath_%s_functional_pseudo_element' % ( - pseudo_element.name.replace('-', '_')) - method = _unicode_safe_getattr(self, method, None) - if not method: - raise ExpressionError( - "The functional pseudo-element ::%s() is unknown" - % pseudo_element.name) - xpath = method(xpath, pseudo_element) - else: - method = 'xpath_%s_simple_pseudo_element' % ( - pseudo_element.replace('-', '_')) - method = _unicode_safe_getattr(self, method, None) - if not method: - raise ExpressionError( - "The pseudo-element ::%s is unknown" - % pseudo_element) - xpath = method(xpath) - return xpath - - def xpath_attr_functional_pseudo_element(self, xpath, function): - if function.argument_types() not in (['STRING'], ['IDENT']): - raise ExpressionError( - "Expected a single string or ident for ::attr(), got %r" - % function.arguments) - return ScrapyXPathExpr.from_xpath(xpath, - attribute=function.arguments[0].value) - - def xpath_text_simple_pseudo_element(self, xpath): - """Support selecting text nodes using ::text pseudo-element""" - return ScrapyXPathExpr.from_xpath(xpath, textnode=True) - - -class ScrapyGenericTranslator(TranslatorMixin, GenericTranslator): - pass - - -class ScrapyHTMLTranslator(TranslatorMixin, HTMLTranslator): - pass - - -class CSSSelectorMixin(object): + default_contenttype = 'html' def select(self, css): - xpath = self._css2xpath(css) - return super(CSSSelectorMixin, self).select(xpath) - - def _css2xpath(self, css): - return self.translator.css_to_xpath(css) + return self.css(css) -class CSSSelector(CSSSelectorMixin, XPathSelector): - translator = ScrapyHTMLTranslator() +HtmlCSSSelector = CSSSelector -class HtmlCSSSelector(CSSSelectorMixin, HtmlXPathSelector): - translator = ScrapyHTMLTranslator() - - -class XmlCSSSelector(CSSSelectorMixin, XmlXPathSelector): - translator = ScrapyGenericTranslator() +class XmlCSSSelector(CSSSelector): + default_contenttype = 'xml' diff --git a/scrapy/selector/csstranslator.py b/scrapy/selector/csstranslator.py new file mode 100644 index 000000000..7482837a0 --- /dev/null +++ b/scrapy/selector/csstranslator.py @@ -0,0 +1,88 @@ +from cssselect import GenericTranslator, HTMLTranslator +from cssselect.xpath import _unicode_safe_getattr, XPathExpr, ExpressionError +from cssselect.parser import FunctionalPseudoElement + + +class ScrapyXPathExpr(XPathExpr): + + textnode = False + attribute = None + + @classmethod + def from_xpath(cls, xpath, textnode=False, attribute=None): + x = cls(path=xpath.path, element=xpath.element, condition=xpath.condition) + x.textnode = textnode + x.attribute = attribute + return x + + def __str__(self): + path = super(ScrapyXPathExpr, self).__str__() + if self.textnode: + if path == '*': + path = 'text()' + elif path.endswith('::*/*'): + path = path[:-3] + 'text()' + else: + path += '/text()' + + if self.attribute is not None: + if path.endswith('::*/*'): + path = path[:-2] + path += '/@%s' % self.attribute + + return path + + def join(self, combiner, other): + super(ScrapyXPathExpr, self).join(combiner, other) + self.textnode = other.textnode + self.attribute = other.attribute + return self + + +class TranslatorMixin(object): + + def xpath_element(self, selector): + xpath = super(TranslatorMixin, self).xpath_element(selector) + return ScrapyXPathExpr.from_xpath(xpath) + + def xpath_pseudo_element(self, xpath, pseudo_element): + if isinstance(pseudo_element, FunctionalPseudoElement): + method = 'xpath_%s_functional_pseudo_element' % ( + pseudo_element.name.replace('-', '_')) + method = _unicode_safe_getattr(self, method, None) + if not method: + raise ExpressionError( + "The functional pseudo-element ::%s() is unknown" + % pseudo_element.name) + xpath = method(xpath, pseudo_element) + else: + method = 'xpath_%s_simple_pseudo_element' % ( + pseudo_element.replace('-', '_')) + method = _unicode_safe_getattr(self, method, None) + if not method: + raise ExpressionError( + "The pseudo-element ::%s is unknown" + % pseudo_element) + xpath = method(xpath) + return xpath + + def xpath_attr_functional_pseudo_element(self, xpath, function): + if function.argument_types() not in (['STRING'], ['IDENT']): + raise ExpressionError( + "Expected a single string or ident for ::attr(), got %r" + % function.arguments) + return ScrapyXPathExpr.from_xpath(xpath, + attribute=function.arguments[0].value) + + def xpath_text_simple_pseudo_element(self, xpath): + """Support selecting text nodes using ::text pseudo-element""" + return ScrapyXPathExpr.from_xpath(xpath, textnode=True) + + +class ScrapyGenericTranslator(TranslatorMixin, GenericTranslator): + pass + + +class ScrapyHTMLTranslator(TranslatorMixin, HTMLTranslator): + pass + diff --git a/scrapy/selector/list.py b/scrapy/selector/list.py deleted file mode 100644 index 774be1155..000000000 --- a/scrapy/selector/list.py +++ /dev/null @@ -1,25 +0,0 @@ -from scrapy.utils.python import flatten -from scrapy.utils.decorator import deprecated - - -class SelectorList(list): - - def __getslice__(self, i, j): - return self.__class__(list.__getslice__(self, i, j)) - - def select(self, xpath): - return self.__class__(flatten([x.select(xpath) for x in self])) - - def re(self, regex): - return flatten([x.re(regex) for x in self]) - - def extract(self): - return [x.extract() for x in self] - - @deprecated(use_instead='SelectorList.extract') - def extract_unquoted(self): - return [x.extract_unquoted() for x in self] - - @deprecated(use_instead='SelectorList.select') - def x(self, xpath): - return self.select(xpath) diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py index 18b3e5044..b37b5d74c 100644 --- a/scrapy/selector/lxmlsel.py +++ b/scrapy/selector/lxmlsel.py @@ -1,106 +1,34 @@ """ XPath selectors based on lxml """ - -from lxml import etree - -from scrapy.utils.misc import extract_regex -from scrapy.utils.trackref import object_ref -from scrapy.utils.python import unicode_to_str -from scrapy.utils.decorator import deprecated -from scrapy.http import TextResponse -from .lxmldocument import LxmlDocument -from .list import SelectorList +from .unified import Selector, SelectorList -__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector'] +__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', + 'XPathSelectorList'] -class XPathSelector(object_ref): - - __slots__ = ['response', 'text', 'namespaces', '_expr', '_root', '__weakref__'] - _parser = etree.HTMLParser - _tostring_method = 'html' - - def __init__(self, response=None, text=None, namespaces=None, _root=None, _expr=None): - if text is not None: - response = TextResponse(url='about:blank', encoding='utf-8', - body=unicode_to_str(text, 'utf-8')) - if response is not None: - _root = LxmlDocument(response, self._parser) - - self.namespaces = namespaces - self.response = response - self._root = _root - self._expr = _expr - - def select(self, xpath): - try: - xpathev = self._root.xpath - except AttributeError: - return SelectorList([]) - - try: - result = xpathev(xpath, namespaces=self.namespaces) - except etree.XPathError: - raise ValueError("Invalid XPath: %s" % xpath) - - if type(result) is not list: - result = [result] - - result = [self.__class__(_root=x, _expr=xpath, namespaces=self.namespaces) - for x in result] - return SelectorList(result) - - def re(self, regex): - return extract_regex(regex, self.extract()) - - def extract(self): - try: - return etree.tostring(self._root, method=self._tostring_method, \ - encoding=unicode, with_tail=False) - except (AttributeError, TypeError): - if self._root is True: - return u'1' - elif self._root is False: - return u'0' - else: - return unicode(self._root) - - def register_namespace(self, prefix, uri): - if self.namespaces is None: - self.namespaces = {} - self.namespaces[prefix] = uri - - def remove_namespaces(self): - for el in self._root.iter('*'): - if el.tag.startswith('{'): - el.tag = el.tag.split('}', 1)[1] - # loop on element attributes also - for an in el.attrib.keys(): - if an.startswith('{'): - el.attrib[an.split('}', 1)[1]] = el.attrib.pop(an) - - def __nonzero__(self): - return bool(self.extract()) - - def __str__(self): - data = repr(self.extract()[:40]) - return "<%s xpath=%r data=%s>" % (type(self).__name__, self._expr, data) - __repr__ = __str__ - - @deprecated(use_instead='XPathSelector.extract') - def extract_unquoted(self): - return self.extract() +class XPathSelector(Selector): + __slots__ = () + default_contenttype = 'xml' class XmlXPathSelector(XPathSelector): __slots__ = () - _parser = etree.XMLParser - _tostring_method = 'xml' + default_contenttype = 'xml' class HtmlXPathSelector(XPathSelector): __slots__ = () - _parser = etree.HTMLParser - _tostring_method = 'html' + default_contenttype = 'html' + + +class XPathSelectorList(SelectorList): + + def __init__(self, *a, **kw): + import warnings + from scrapy.exceptions import ScrapyDeprecationWarning + warnings.warn('XPathSelectorList is deprecated, use ' + 'scrapy.selector.SelectorList instead', + category=ScrapyDeprecationWarning, stacklevel=1) + super(XPathSelectorList, self).__init__(*a, **kw) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py new file mode 100644 index 000000000..12d0188ab --- /dev/null +++ b/scrapy/selector/unified.py @@ -0,0 +1,164 @@ +""" +XPath selectors based on lxml +""" + +from lxml import etree + +from scrapy.utils.misc import extract_regex +from scrapy.utils.trackref import object_ref +from scrapy.utils.python import unicode_to_str, flatten +from scrapy.utils.decorator import deprecated +from scrapy.http import HtmlResponse, XmlResponse +from .lxmldocument import LxmlDocument +from .csstranslator import ScrapyHTMLTranslator, ScrapyGenericTranslator + + +__all__ = ['Selector', 'SelectorList'] + +_ctgroup = { + 'html': {'_parser': etree.HTMLParser, + '_csstranslator': ScrapyHTMLTranslator(), + '_tostring_method': 'html'}, + 'xml': {'_parser': etree.XMLParser, + '_csstranslator': ScrapyGenericTranslator(), + '_tostring_method': 'xml'}, +} + + +def _ct(response, ct): + if ct is None: + return 'xml' if isinstance(response, XmlResponse) else 'html' + elif ct in ('xml', 'html'): + return ct + else: + raise ValueError('Invalid contenttype: %s' % ct) + + +def _response_from_text(text, ct): + rt = XmlResponse if ct == 'xml' else HtmlResponse + return rt(url='about:blank', encoding='utf-8', + body=unicode_to_str(text, 'utf-8')) + + +class Selector(object_ref): + + __slots__ = ['response', 'text', 'namespaces', 'contenttype', '_expr', '_root', + '__weakref__', '_parser', '_csstranslator', '_tostring_method'] + + default_contenttype = None + + def __init__(self, response=None, text=None, namespaces=None, contenttype=None, + _root=None, _expr=None): + + self.contenttype = ct = self.default_contenttype or _ct(contenttype) + self._parser = _ctgroup[ct]['_parser'] + self._csstranslator = _ctgroup[ct]['_csstranslator'] + self._tostring_method = _ctgroup[ct]['_tostring_method'] + + if text is not None: + response = _response_from_text(text, ct) + + if response is not None: + _root = LxmlDocument(response, self._parser) + + self.response = response + self.namespaces = namespaces + self._root = _root + self._expr = _expr + + def xpath(self, query): + try: + xpathev = self._root.xpath + except AttributeError: + return SelectorList([]) + + try: + result = xpathev(query, namespaces=self.namespaces) + except etree.XPathError: + raise ValueError("Invalid XPath: %s" % query) + + if type(result) is not list: + result = [result] + + result = [self.__class__(_root=x, _expr=query, + namespaces=self.namespaces, + contenttype=self.contenttype) + for x in result] + return SelectorList(result) + + def css(self, query): + return self.xpath(self._css2xpath(query)) + + def _css2xpath(self, query): + return self._csstranslator.css_to_xpath(query) + + def re(self, regex): + return extract_regex(regex, self.extract()) + + def extract(self): + try: + return etree.tostring(self._root, + method=self._tostring_method, + encoding=unicode, + with_tail=False) + except (AttributeError, TypeError): + if self._root is True: + return u'1' + elif self._root is False: + return u'0' + else: + return unicode(self._root) + + def register_namespace(self, prefix, uri): + if self.namespaces is None: + self.namespaces = {} + self.namespaces[prefix] = uri + + def remove_namespaces(self): + for el in self._root.iter('*'): + if el.tag.startswith('{'): + el.tag = el.tag.split('}', 1)[1] + # loop on element attributes also + for an in el.attrib.keys(): + if an.startswith('{'): + el.attrib[an.split('}', 1)[1]] = el.attrib.pop(an) + + def __nonzero__(self): + return bool(self.extract()) + + def __str__(self): + data = repr(self.extract()[:40]) + return "<%s xpath=%r data=%s>" % (type(self).__name__, self._expr, data) + __repr__ = __str__ + + # Deprecated api + @deprecated(use_instead='.xpath()') + def select(self, xpath): + return self.xpath(xpath) + + @deprecated(use_instead='.extract()') + def extract_unquoted(self): + return self.extract() + + +class SelectorList(list): + + def __getslice__(self, i, j): + return self.__class__(list.__getslice__(self, i, j)) + + def select(self, xpath): + return self.__class__(flatten([x.select(xpath) for x in self])) + + def re(self, regex): + return flatten([x.re(regex) for x in self]) + + def extract(self): + return [x.extract() for x in self] + + @deprecated(use_instead='SelectorList.extract') + def extract_unquoted(self): + return [x.extract_unquoted() for x in self] + + @deprecated(use_instead='SelectorList.select') + def x(self, xpath): + return self.select(xpath) diff --git a/scrapy/tests/test_selector_cssselect.py b/scrapy/tests/test_selector_cssselect.py index 4b69dff15..399820240 100644 --- a/scrapy/tests/test_selector_cssselect.py +++ b/scrapy/tests/test_selector_cssselect.py @@ -4,7 +4,7 @@ Selector tests for cssselect backend from twisted.trial import unittest from scrapy.http import TextResponse, HtmlResponse, XmlResponse from scrapy.selector import CSSSelector, XmlCSSSelector, HtmlCSSSelector -from scrapy.selector.csssel import ScrapyHTMLTranslator +from scrapy.selector.csstranslator import ScrapyHTMLTranslator from cssselect.parser import SelectorSyntaxError from cssselect.xpath import ExpressionError