From 17a6adde1f91e08abe898578a7498b01148cfd6a Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 27 Oct 2010 06:19:49 -0200 Subject: [PATCH] some refactoring to selectors code, to reuse more code between lxml and libxml2 backends (refs #147). also added tests for dummy backend --- scrapy/selector/dummysel.py | 12 ++------ scrapy/selector/libxml2sel.py | 46 ++++------------------------- scrapy/selector/list.py | 23 +++++++++++++++ scrapy/selector/lxmlsel.py | 22 +------------- scrapy/tests/test_selector_dummy.py | 17 +++++++++++ 5 files changed, 49 insertions(+), 71 deletions(-) create mode 100644 scrapy/selector/list.py create mode 100644 scrapy/tests/test_selector_dummy.py diff --git a/scrapy/selector/dummysel.py b/scrapy/selector/dummysel.py index 7930835b6..8718f3311 100644 --- a/scrapy/selector/dummysel.py +++ b/scrapy/selector/dummysel.py @@ -2,6 +2,8 @@ Dummy selectors """ +from .list import XPathSelectorList as XPathSelectorList + __all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \ 'XPathSelectorList'] @@ -14,15 +16,7 @@ class XPathSelector(object): raise RuntimeError("No selectors backend available. " \ "Please install libxml2 or lxml") - select = re = exract = register_namespace = __nonzero__ = _raise - -class XPathSelectorList(list): - - def _raise(self, *a, **kw): - raise RuntimeError("No selectors backend available. " \ - "Please install libxml2 or lxml") - - __getslice__ = select = re = extract = extract_unquoted = _raise + select = re = extract = register_namespace = __nonzero__ = _raise XmlXPathSelector = XPathSelector HtmlXPathSelector = XPathSelector diff --git a/scrapy/selector/libxml2sel.py b/scrapy/selector/libxml2sel.py index d6ef7ebb1..15ae3c584 100644 --- a/scrapy/selector/libxml2sel.py +++ b/scrapy/selector/libxml2sel.py @@ -5,12 +5,13 @@ XPath selectors based on libxml2 import libxml2 from scrapy.http import TextResponse -from scrapy.utils.python import flatten, unicode_to_str +from scrapy.utils.python import unicode_to_str from scrapy.utils.misc import extract_regex from scrapy.utils.trackref import object_ref from scrapy.utils.decorator import deprecated from .factories import xmlDoc_from_html, xmlDoc_from_xml from .document import Libxml2Document +from .list import XPathSelectorList __all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \ 'XPathSelectorList'] @@ -34,8 +35,6 @@ class XPathSelector(object_ref): self.expr = expr def select(self, xpath): - """Perform the given XPath query on the current XPathSelector and - return a XPathSelectorList of the result""" if hasattr(self.xmlNode, 'xpathEval'): self.doc.xpathContext.setContextNode(self.xmlNode) try: @@ -52,12 +51,9 @@ class XPathSelector(object_ref): return XPathSelectorList([]) def re(self, regex): - """Return a list of unicode strings by applying the regex over all - current XPath selections, and flattening the results""" return extract_regex(regex, self.extract()) def extract(self): - """Return a unicode string of the content referenced by the XPathSelector""" if isinstance(self.xmlNode, basestring): text = unicode(self.xmlNode, 'utf-8', errors='ignore') elif hasattr(self.xmlNode, 'serialize'): @@ -79,25 +75,24 @@ class XPathSelector(object_ref): def extract_unquoted(self): """Get unescaped contents from the text node (no entities, no CDATA)""" + # TODO: this function should be deprecated. but what would be use instead? if self.select('self::text()'): return unicode(self.xmlNode.getContent(), 'utf-8', errors='ignore') else: return u'' def register_namespace(self, prefix, uri): - """Register namespace so that it can be used in XPath queries""" self.doc.xpathContext.xpathRegisterNs(prefix, uri) def _get_libxml2_doc(self, response): - """Return libxml2 document (xmlDoc) from response""" return xmlDoc_from_html(response) def __nonzero__(self): return bool(self.extract()) def __str__(self): - return "<%s (%s) xpath=%s>" % (type(self).__name__, getattr(self.xmlNode, \ - 'name', type(self.xmlNode).__name__), self.expr) + data = repr(self.extract()[:40]) + return "<%s xpath=%r data=%s>" % (type(self).__name__, self.expr, data) __repr__ = __str__ @@ -110,42 +105,11 @@ class XPathSelector(object_ref): return self.select(xpath) -class XPathSelectorList(list): - """List of XPathSelector objects""" - - def __getslice__(self, i, j): - return XPathSelectorList(list.__getslice__(self, i, j)) - - def select(self, xpath): - """Perform the given XPath query on each XPathSelector of the list and - return a new (flattened) XPathSelectorList of the results""" - return XPathSelectorList(flatten([x.select(xpath) for x in self])) - - def re(self, regex): - """Perform the re() method on each XPathSelector of the list, and - return the result as a flattened list of unicode strings""" - return flatten([x.re(regex) for x in self]) - - def extract(self): - """Return a list of unicode strings with the content referenced by each - XPathSelector of the list""" - return [x.extract() if isinstance(x, XPathSelector) else x for x in self] - - def extract_unquoted(self): - return [x.extract_unquoted() if isinstance(x, XPathSelector) else x for x in self] - - @deprecated(use_instead='XPathSelectorList.select') - def x(self, xpath): - return self.select(xpath) - - class XmlXPathSelector(XPathSelector): - """XPathSelector for XML content""" __slots__ = () _get_libxml2_doc = staticmethod(xmlDoc_from_xml) class HtmlXPathSelector(XPathSelector): - """XPathSelector for HTML content""" __slots__ = () _get_libxml2_doc = staticmethod(xmlDoc_from_html) diff --git a/scrapy/selector/list.py b/scrapy/selector/list.py new file mode 100644 index 000000000..a6977fd99 --- /dev/null +++ b/scrapy/selector/list.py @@ -0,0 +1,23 @@ +from scrapy.utils.python import flatten +from scrapy.utils.decorator import deprecated + +class XPathSelectorList(list): + + def __getslice__(self, i, j): + return self.__class__(list.__getslice__(self, i, j)) + + def select(self, xpath): + return self.__class__(flatten([x.select(xpath) for x in self])) + + def re(self, regex): + return flatten([x.re(regex) for x in self]) + + def extract(self): + return [x.extract() for x in self] + + def extract_unquoted(self): + return [x.extract_unquoted() for x in self] + + @deprecated(use_instead='XPathSelectorList.select') + def x(self, xpath): + return self.select(xpath) diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py index ae7166ccb..af5f765b4 100644 --- a/scrapy/selector/lxmlsel.py +++ b/scrapy/selector/lxmlsel.py @@ -10,6 +10,7 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.python import unicode_to_str from scrapy.utils.decorator import deprecated from scrapy.http import TextResponse +from .list import XPathSelectorList __all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \ 'XPathSelectorList'] @@ -88,34 +89,13 @@ class XPathSelector(object_ref): return self.extract() -class XPathSelectorList(list): - - def __getslice__(self, i, j): - return XPathSelectorList(list.__getslice__(self, i, j)) - - def select(self, xpath): - return XPathSelectorList(flatten([x.select(xpath) for x in self])) - - def re(self, regex): - return flatten([x.re(regex) for x in self]) - - def extract(self): - return [x.extract() if isinstance(x, XPathSelector) else x for x in self] - - @deprecated(use_instead='XPathSelectorList.extract_unquoted') - def extract_unquoted(self): - return [x.extract_unquoted() if isinstance(x, XPathSelector) else x for x in self] - - class XmlXPathSelector(XPathSelector): - """XPathSelector for XML content""" __slots__ = () _parser = etree.XMLParser _tostring_method = 'xml' class HtmlXPathSelector(XPathSelector): - """XPathSelector for HTML content""" __slots__ = () _parser = etree.HTMLParser _tostring_method = 'html' diff --git a/scrapy/tests/test_selector_dummy.py b/scrapy/tests/test_selector_dummy.py new file mode 100644 index 000000000..00ac0c152 --- /dev/null +++ b/scrapy/tests/test_selector_dummy.py @@ -0,0 +1,17 @@ +import unittest + +from scrapy.http import TextResponse +from scrapy.selector.dummysel import XmlXPathSelector, HtmlXPathSelector, \ + XPathSelector + +class XPathSelectorTestCase(unittest.TestCase): + + def test_raises(self): + response = TextResponse(url="http://example.com", body='test') + for cls in [XmlXPathSelector, HtmlXPathSelector, XPathSelector]: + sel = cls(response) + self.assertRaises(RuntimeError, sel.select, '//h2') + self.assertRaises(RuntimeError, sel.re, 'lala') + self.assertRaises(RuntimeError, sel.extract) + self.assertRaises(RuntimeError, sel.register_namespace, 'a', 'b') + self.assertRaises(RuntimeError, sel.__nonzero__, 'a', 'b')