RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

working implementaion of unified api

This commit is contained in:
Daniel Graña 2013-10-11 18:06:27 -02:00
parent bf37f78572
commit c3d28cc412
7 changed files with 280 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
scrapy/selector/__init__.py
View File

@ -1,17 +1,6 @@
"""
Selectors
"""
from scrapy.selector.unified import *
from scrapy.selector.lxmlsel import *
from scrapy.selector.csssel import *
from scrapy.selector.list import SelectorList
class XPathSelectorList(SelectorList):
def __init__(self, *a, **kw):
import warnings
from scrapy.exceptions import ScrapyDeprecationWarning
warnings.warn('XPathSelectorList is deprecated, use '
'scrapy.selector.SelectorList instead',
category=ScrapyDeprecationWarning, stacklevel=1)
super(XPathSelectorList, self).__init__(*a, **kw)

108
scrapy/selector/csssel.py
View File

@ -1,110 +1,16 @@
from cssselect import GenericTranslator, HTMLTranslator
from cssselect.xpath import _unicode_safe_getattr, XPathExpr, ExpressionError
from cssselect.parser import FunctionalPseudoElement
from scrapy.selector import XPathSelector, HtmlXPathSelector, XmlXPathSelector
from .unified import Selector
class ScrapyXPathExpr(XPathExpr):
class CSSSelector(Selector):
textnode = False
attribute = None
@classmethod
def from_xpath(cls, xpath, textnode=False, attribute=None):
x = cls(path=xpath.path, element=xpath.element, condition=xpath.condition)
x.textnode = textnode
x.attribute = attribute
return x
def __str__(self):
path = super(ScrapyXPathExpr, self).__str__()
if self.textnode:
if path == '*':
path = 'text()'
elif path.endswith('::*/*'):
path = path[:-3] + 'text()'
else:
path += '/text()'
if self.attribute is not None:
if path.endswith('::*/*'):
path = path[:-2]
path += '/@%s' % self.attribute
return path
def join(self, combiner, other):
super(ScrapyXPathExpr, self).join(combiner, other)
self.textnode = other.textnode
self.attribute = other.attribute
return self
class TranslatorMixin(object):
def xpath_element(self, selector):
xpath = super(TranslatorMixin, self).xpath_element(selector)
return ScrapyXPathExpr.from_xpath(xpath)
def xpath_pseudo_element(self, xpath, pseudo_element):
if isinstance(pseudo_element, FunctionalPseudoElement):
method = 'xpath_%s_functional_pseudo_element' % (
pseudo_element.name.replace('-', '_'))
method = _unicode_safe_getattr(self, method, None)
if not method:
raise ExpressionError(
"The functional pseudo-element ::%s() is unknown"
% pseudo_element.name)
xpath = method(xpath, pseudo_element)
else:
method = 'xpath_%s_simple_pseudo_element' % (
pseudo_element.replace('-', '_'))
method = _unicode_safe_getattr(self, method, None)
if not method:
raise ExpressionError(
"The pseudo-element ::%s is unknown"
% pseudo_element)
xpath = method(xpath)
return xpath
def xpath_attr_functional_pseudo_element(self, xpath, function):
if function.argument_types() not in (['STRING'], ['IDENT']):
raise ExpressionError(
"Expected a single string or ident for ::attr(), got %r"
% function.arguments)
return ScrapyXPathExpr.from_xpath(xpath,
attribute=function.arguments[0].value)
def xpath_text_simple_pseudo_element(self, xpath):
"""Support selecting text nodes using ::text pseudo-element"""
return ScrapyXPathExpr.from_xpath(xpath, textnode=True)
class ScrapyGenericTranslator(TranslatorMixin, GenericTranslator):
pass
class ScrapyHTMLTranslator(TranslatorMixin, HTMLTranslator):
pass
class CSSSelectorMixin(object):
default_contenttype = 'html'
def select(self, css):
xpath = self._css2xpath(css)
return super(CSSSelectorMixin, self).select(xpath)
def _css2xpath(self, css):
return self.translator.css_to_xpath(css)
return self.css(css)
class CSSSelector(CSSSelectorMixin, XPathSelector):
translator = ScrapyHTMLTranslator()
HtmlCSSSelector = CSSSelector
class HtmlCSSSelector(CSSSelectorMixin, HtmlXPathSelector):
translator = ScrapyHTMLTranslator()
class XmlCSSSelector(CSSSelectorMixin, XmlXPathSelector):
translator = ScrapyGenericTranslator()
class XmlCSSSelector(CSSSelector):
default_contenttype = 'xml'

88
scrapy/selector/csstranslator.py Normal file
View File

@ -0,0 +1,88 @@
from cssselect import GenericTranslator, HTMLTranslator
from cssselect.xpath import _unicode_safe_getattr, XPathExpr, ExpressionError
from cssselect.parser import FunctionalPseudoElement
class ScrapyXPathExpr(XPathExpr):
textnode = False
attribute = None
@classmethod
def from_xpath(cls, xpath, textnode=False, attribute=None):
x = cls(path=xpath.path, element=xpath.element, condition=xpath.condition)
x.textnode = textnode
x.attribute = attribute
return x
def __str__(self):
path = super(ScrapyXPathExpr, self).__str__()
if self.textnode:
if path == '*':
path = 'text()'
elif path.endswith('::*/*'):
path = path[:-3] + 'text()'
else:
path += '/text()'
if self.attribute is not None:
if path.endswith('::*/*'):
path = path[:-2]
path += '/@%s' % self.attribute
return path
def join(self, combiner, other):
super(ScrapyXPathExpr, self).join(combiner, other)
self.textnode = other.textnode
self.attribute = other.attribute
return self
class TranslatorMixin(object):
def xpath_element(self, selector):
xpath = super(TranslatorMixin, self).xpath_element(selector)
return ScrapyXPathExpr.from_xpath(xpath)
def xpath_pseudo_element(self, xpath, pseudo_element):
if isinstance(pseudo_element, FunctionalPseudoElement):
method = 'xpath_%s_functional_pseudo_element' % (
pseudo_element.name.replace('-', '_'))
method = _unicode_safe_getattr(self, method, None)
if not method:
raise ExpressionError(
"The functional pseudo-element ::%s() is unknown"
% pseudo_element.name)
xpath = method(xpath, pseudo_element)
else:
method = 'xpath_%s_simple_pseudo_element' % (
pseudo_element.replace('-', '_'))
method = _unicode_safe_getattr(self, method, None)
if not method:
raise ExpressionError(
"The pseudo-element ::%s is unknown"
% pseudo_element)
xpath = method(xpath)
return xpath
def xpath_attr_functional_pseudo_element(self, xpath, function):
if function.argument_types() not in (['STRING'], ['IDENT']):
raise ExpressionError(
"Expected a single string or ident for ::attr(), got %r"
% function.arguments)
return ScrapyXPathExpr.from_xpath(xpath,
attribute=function.arguments[0].value)
def xpath_text_simple_pseudo_element(self, xpath):
"""Support selecting text nodes using ::text pseudo-element"""
return ScrapyXPathExpr.from_xpath(xpath, textnode=True)
class ScrapyGenericTranslator(TranslatorMixin, GenericTranslator):
pass
class ScrapyHTMLTranslator(TranslatorMixin, HTMLTranslator):
pass

25
scrapy/selector/list.py
View File

@ -1,25 +0,0 @@
from scrapy.utils.python import flatten
from scrapy.utils.decorator import deprecated
class SelectorList(list):
def __getslice__(self, i, j):
return self.__class__(list.__getslice__(self, i, j))
def select(self, xpath):
return self.__class__(flatten([x.select(xpath) for x in self]))
def re(self, regex):
return flatten([x.re(regex) for x in self])
def extract(self):
return [x.extract() for x in self]
@deprecated(use_instead='SelectorList.extract')
def extract_unquoted(self):
return [x.extract_unquoted() for x in self]
@deprecated(use_instead='SelectorList.select')
def x(self, xpath):
return self.select(xpath)

110
scrapy/selector/lxmlsel.py
View File

@ -1,106 +1,34 @@
"""
XPath selectors based on lxml
"""
from lxml import etree
from scrapy.utils.misc import extract_regex
from scrapy.utils.trackref import object_ref
from scrapy.utils.python import unicode_to_str
from scrapy.utils.decorator import deprecated
from scrapy.http import TextResponse
from .lxmldocument import LxmlDocument
from .list import SelectorList
from .unified import Selector, SelectorList
__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector']
__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector',
'XPathSelectorList']
class XPathSelector(object_ref):
__slots__ = ['response', 'text', 'namespaces', '_expr', '_root', '__weakref__']
_parser = etree.HTMLParser
_tostring_method = 'html'
def __init__(self, response=None, text=None, namespaces=None, _root=None, _expr=None):
if text is not None:
response = TextResponse(url='about:blank', encoding='utf-8',
body=unicode_to_str(text, 'utf-8'))
if response is not None:
_root = LxmlDocument(response, self._parser)
self.namespaces = namespaces
self.response = response
self._root = _root
self._expr = _expr
def select(self, xpath):
try:
xpathev = self._root.xpath
except AttributeError:
return SelectorList([])
try:
result = xpathev(xpath, namespaces=self.namespaces)
except etree.XPathError:
raise ValueError("Invalid XPath: %s" % xpath)
if type(result) is not list:
result = [result]
result = [self.__class__(_root=x, _expr=xpath, namespaces=self.namespaces)
for x in result]
return SelectorList(result)
def re(self, regex):
return extract_regex(regex, self.extract())
def extract(self):
try:
return etree.tostring(self._root, method=self._tostring_method, \
encoding=unicode, with_tail=False)
except (AttributeError, TypeError):
if self._root is True:
return u'1'
elif self._root is False:
return u'0'
else:
return unicode(self._root)
def register_namespace(self, prefix, uri):
if self.namespaces is None:
self.namespaces = {}
self.namespaces[prefix] = uri
def remove_namespaces(self):
for el in self._root.iter('*'):
if el.tag.startswith('{'):
el.tag = el.tag.split('}', 1)[1]
# loop on element attributes also
for an in el.attrib.keys():
if an.startswith('{'):
el.attrib[an.split('}', 1)[1]] = el.attrib.pop(an)
def __nonzero__(self):
return bool(self.extract())
def __str__(self):
data = repr(self.extract()[:40])
return "<%s xpath=%r data=%s>" % (type(self).__name__, self._expr, data)
__repr__ = __str__
@deprecated(use_instead='XPathSelector.extract')
def extract_unquoted(self):
return self.extract()
class XPathSelector(Selector):
__slots__ = ()
default_contenttype = 'xml'
class XmlXPathSelector(XPathSelector):
__slots__ = ()
_parser = etree.XMLParser
_tostring_method = 'xml'
default_contenttype = 'xml'
class HtmlXPathSelector(XPathSelector):
__slots__ = ()
_parser = etree.HTMLParser
_tostring_method = 'html'
default_contenttype = 'html'
class XPathSelectorList(SelectorList):
def __init__(self, *a, **kw):
import warnings
from scrapy.exceptions import ScrapyDeprecationWarning
warnings.warn('XPathSelectorList is deprecated, use '
'scrapy.selector.SelectorList instead',
category=ScrapyDeprecationWarning, stacklevel=1)
super(XPathSelectorList, self).__init__(*a, **kw)

164
scrapy/selector/unified.py Normal file
View File

@ -0,0 +1,164 @@
"""
XPath selectors based on lxml
"""
from lxml import etree
from scrapy.utils.misc import extract_regex
from scrapy.utils.trackref import object_ref
from scrapy.utils.python import unicode_to_str, flatten
from scrapy.utils.decorator import deprecated
from scrapy.http import HtmlResponse, XmlResponse
from .lxmldocument import LxmlDocument
from .csstranslator import ScrapyHTMLTranslator, ScrapyGenericTranslator
__all__ = ['Selector', 'SelectorList']
_ctgroup = {
'html': {'_parser': etree.HTMLParser,
'_csstranslator': ScrapyHTMLTranslator(),
'_tostring_method': 'html'},
'xml': {'_parser': etree.XMLParser,
'_csstranslator': ScrapyGenericTranslator(),
'_tostring_method': 'xml'},
}
def _ct(response, ct):
if ct is None:
return 'xml' if isinstance(response, XmlResponse) else 'html'
elif ct in ('xml', 'html'):
return ct
else:
raise ValueError('Invalid contenttype: %s' % ct)
def _response_from_text(text, ct):
rt = XmlResponse if ct == 'xml' else HtmlResponse
return rt(url='about:blank', encoding='utf-8',
body=unicode_to_str(text, 'utf-8'))
class Selector(object_ref):
__slots__ = ['response', 'text', 'namespaces', 'contenttype', '_expr', '_root',
'__weakref__', '_parser', '_csstranslator', '_tostring_method']
default_contenttype = None
def __init__(self, response=None, text=None, namespaces=None, contenttype=None,
_root=None, _expr=None):
self.contenttype = ct = self.default_contenttype or _ct(contenttype)
self._parser = _ctgroup[ct]['_parser']
self._csstranslator = _ctgroup[ct]['_csstranslator']
self._tostring_method = _ctgroup[ct]['_tostring_method']
if text is not None:
response = _response_from_text(text, ct)
if response is not None:
_root = LxmlDocument(response, self._parser)
self.response = response
self.namespaces = namespaces
self._root = _root
self._expr = _expr
def xpath(self, query):
try:
xpathev = self._root.xpath
except AttributeError:
return SelectorList([])
try:
result = xpathev(query, namespaces=self.namespaces)
except etree.XPathError:
raise ValueError("Invalid XPath: %s" % query)
if type(result) is not list:
result = [result]
result = [self.__class__(_root=x, _expr=query,
namespaces=self.namespaces,
contenttype=self.contenttype)
for x in result]
return SelectorList(result)
def css(self, query):
return self.xpath(self._css2xpath(query))
def _css2xpath(self, query):
return self._csstranslator.css_to_xpath(query)
def re(self, regex):
return extract_regex(regex, self.extract())
def extract(self):
try:
return etree.tostring(self._root,
method=self._tostring_method,
encoding=unicode,
with_tail=False)
except (AttributeError, TypeError):
if self._root is True:
return u'1'
elif self._root is False:
return u'0'
else:
return unicode(self._root)
def register_namespace(self, prefix, uri):
if self.namespaces is None:
self.namespaces = {}
self.namespaces[prefix] = uri
def remove_namespaces(self):
for el in self._root.iter('*'):
if el.tag.startswith('{'):
el.tag = el.tag.split('}', 1)[1]
# loop on element attributes also
for an in el.attrib.keys():
if an.startswith('{'):
el.attrib[an.split('}', 1)[1]] = el.attrib.pop(an)
def __nonzero__(self):
return bool(self.extract())
def __str__(self):
data = repr(self.extract()[:40])
return "<%s xpath=%r data=%s>" % (type(self).__name__, self._expr, data)
__repr__ = __str__
# Deprecated api
@deprecated(use_instead='.xpath()')
def select(self, xpath):
return self.xpath(xpath)
@deprecated(use_instead='.extract()')
def extract_unquoted(self):
return self.extract()
class SelectorList(list):
def __getslice__(self, i, j):
return self.__class__(list.__getslice__(self, i, j))
def select(self, xpath):
return self.__class__(flatten([x.select(xpath) for x in self]))
def re(self, regex):
return flatten([x.re(regex) for x in self])
def extract(self):
return [x.extract() for x in self]
@deprecated(use_instead='SelectorList.extract')
def extract_unquoted(self):
return [x.extract_unquoted() for x in self]
@deprecated(use_instead='SelectorList.select')
def x(self, xpath):
return self.select(xpath)

2
scrapy/tests/test_selector_cssselect.py
View File

@ -4,7 +4,7 @@ Selector tests for cssselect backend
from twisted.trial import unittest
from scrapy.http import TextResponse, HtmlResponse, XmlResponse
from scrapy.selector import CSSSelector, XmlCSSSelector, HtmlCSSSelector
from scrapy.selector.csssel import ScrapyHTMLTranslator
from scrapy.selector.csstranslator import ScrapyHTMLTranslator
from cssselect.parser import SelectorSyntaxError
from cssselect.xpath import ExpressionError