some refactoring to selectors code, to reuse more code between lxml and libxml2 backends (refs #147). also added tests for dummy backend

scrapy/selector/dummysel.py

@@ -2,6 +2,8 @@
 Dummy selectors
 """
 
+from .list import XPathSelectorList as XPathSelectorList
+
 __all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \
     'XPathSelectorList']
 
@@ -14,15 +16,7 @@ class XPathSelector(object):
         raise RuntimeError("No selectors backend available. " \
             "Please install libxml2 or lxml")
 
-    select = re = exract = register_namespace = __nonzero__ = _raise
-
-class XPathSelectorList(list):
-
-    def _raise(self, *a, **kw):
-        raise RuntimeError("No selectors backend available. " \
-            "Please install libxml2 or lxml")
-
-    __getslice__ = select = re = extract = extract_unquoted = _raise
+    select = re = extract = register_namespace = __nonzero__ = _raise
 
 XmlXPathSelector = XPathSelector
 HtmlXPathSelector = XPathSelector

scrapy/selector/libxml2sel.py

@@ -5,12 +5,13 @@ XPath selectors based on libxml2
 import libxml2
 
 from scrapy.http import TextResponse
-from scrapy.utils.python import flatten, unicode_to_str
+from scrapy.utils.python import unicode_to_str
 from scrapy.utils.misc import extract_regex
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.decorator import deprecated
 from .factories import xmlDoc_from_html, xmlDoc_from_xml
 from .document import Libxml2Document
+from .list import XPathSelectorList
 
 __all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \
     'XPathSelectorList']
@@ -34,8 +35,6 @@ class XPathSelector(object_ref):
         self.expr = expr
 
     def select(self, xpath):
-        """Perform the given XPath query on the current XPathSelector and
-        return a XPathSelectorList of the result"""
         if hasattr(self.xmlNode, 'xpathEval'):
             self.doc.xpathContext.setContextNode(self.xmlNode)
             try:
@@ -52,12 +51,9 @@ class XPathSelector(object_ref):
             return XPathSelectorList([])
 
     def re(self, regex):
-        """Return a list of unicode strings by applying the regex over all
-        current XPath selections, and flattening the results"""
         return extract_regex(regex, self.extract())
 
     def extract(self):
-        """Return a unicode string of the content referenced by the XPathSelector"""
         if isinstance(self.xmlNode, basestring):
             text = unicode(self.xmlNode, 'utf-8', errors='ignore')
         elif hasattr(self.xmlNode, 'serialize'):
@@ -79,25 +75,24 @@ class XPathSelector(object_ref):
 
     def extract_unquoted(self):
         """Get unescaped contents from the text node (no entities, no CDATA)"""
+        # TODO: this function should be deprecated. but what would be use instead?
         if self.select('self::text()'):
             return unicode(self.xmlNode.getContent(), 'utf-8', errors='ignore')
         else:
             return u''
 
     def register_namespace(self, prefix, uri):
-        """Register namespace so that it can be used in XPath queries"""
         self.doc.xpathContext.xpathRegisterNs(prefix, uri)
 
     def _get_libxml2_doc(self, response):
-        """Return libxml2 document (xmlDoc) from response"""
         return xmlDoc_from_html(response)
 
     def __nonzero__(self):
         return bool(self.extract())
 
     def __str__(self):
-        return "<%s (%s) xpath=%s>" % (type(self).__name__, getattr(self.xmlNode, \
-            'name', type(self.xmlNode).__name__), self.expr)
+        data = repr(self.extract()[:40])
+        return "<%s xpath=%r data=%s>" % (type(self).__name__, self.expr, data)
 
     __repr__ = __str__
 
@@ -110,42 +105,11 @@ class XPathSelector(object_ref):
         return self.select(xpath)
 
 
-class XPathSelectorList(list):
-    """List of XPathSelector objects"""
-
-    def __getslice__(self, i, j):
-        return XPathSelectorList(list.__getslice__(self, i, j))
-
-    def select(self, xpath):
-        """Perform the given XPath query on each XPathSelector of the list and
-        return a new (flattened) XPathSelectorList of the results"""
-        return XPathSelectorList(flatten([x.select(xpath) for x in self]))
-
-    def re(self, regex):
-        """Perform the re() method on each XPathSelector of the list, and
-        return the result as a flattened list of unicode strings"""
-        return flatten([x.re(regex) for x in self])
-
-    def extract(self):
-        """Return a list of unicode strings with the content referenced by each
-        XPathSelector of the list"""
-        return [x.extract() if isinstance(x, XPathSelector) else x for x in self]
-
-    def extract_unquoted(self):
-        return [x.extract_unquoted() if isinstance(x, XPathSelector) else x for x in self]
-
-    @deprecated(use_instead='XPathSelectorList.select')
-    def x(self, xpath):
-        return self.select(xpath)
-
-
 class XmlXPathSelector(XPathSelector):
-    """XPathSelector for XML content"""
     __slots__ = ()
     _get_libxml2_doc = staticmethod(xmlDoc_from_xml)
 
 
 class HtmlXPathSelector(XPathSelector):
-    """XPathSelector for HTML content"""
     __slots__ = ()
     _get_libxml2_doc = staticmethod(xmlDoc_from_html)

scrapy/selector/list.py

@@ -0,0 +1,23 @@
+from scrapy.utils.python import flatten
+from scrapy.utils.decorator import deprecated
+
+class XPathSelectorList(list):
+
+    def __getslice__(self, i, j):
+        return self.__class__(list.__getslice__(self, i, j))
+
+    def select(self, xpath):
+        return self.__class__(flatten([x.select(xpath) for x in self]))
+
+    def re(self, regex):
+        return flatten([x.re(regex) for x in self])
+
+    def extract(self):
+        return [x.extract() for x in self]
+
+    def extract_unquoted(self):
+        return [x.extract_unquoted() for x in self]
+
+    @deprecated(use_instead='XPathSelectorList.select')
+    def x(self, xpath):
+        return self.select(xpath)

scrapy/selector/lxmlsel.py

@@ -10,6 +10,7 @@ from scrapy.utils.trackref import object_ref
 from scrapy.utils.python import unicode_to_str
 from scrapy.utils.decorator import deprecated
 from scrapy.http import TextResponse
+from .list import XPathSelectorList
 
 __all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \
     'XPathSelectorList']
@@ -88,34 +89,13 @@ class XPathSelector(object_ref):
         return self.extract()
 
 
-class XPathSelectorList(list):
-
-    def __getslice__(self, i, j):
-        return XPathSelectorList(list.__getslice__(self, i, j))
-
-    def select(self, xpath):
-        return XPathSelectorList(flatten([x.select(xpath) for x in self]))
-
-    def re(self, regex):
-        return flatten([x.re(regex) for x in self])
-
-    def extract(self):
-        return [x.extract() if isinstance(x, XPathSelector) else x for x in self]
-
-    @deprecated(use_instead='XPathSelectorList.extract_unquoted')
-    def extract_unquoted(self):
-        return [x.extract_unquoted() if isinstance(x, XPathSelector) else x for x in self]
-
-
 class XmlXPathSelector(XPathSelector):
-    """XPathSelector for XML content"""
     __slots__ = ()
     _parser = etree.XMLParser
     _tostring_method = 'xml'
 
 
 class HtmlXPathSelector(XPathSelector):
-    """XPathSelector for HTML content"""
     __slots__ = ()
     _parser = etree.HTMLParser
     _tostring_method = 'html'

scrapy/tests/test_selector_dummy.py

@@ -0,0 +1,17 @@
+import unittest
+
+from scrapy.http import TextResponse
+from scrapy.selector.dummysel import XmlXPathSelector, HtmlXPathSelector, \
+    XPathSelector
+
+class XPathSelectorTestCase(unittest.TestCase):
+
+    def test_raises(self):
+        response = TextResponse(url="http://example.com", body='test')
+        for cls in [XmlXPathSelector, HtmlXPathSelector, XPathSelector]:
+            sel = cls(response)
+            self.assertRaises(RuntimeError, sel.select, '//h2')
+            self.assertRaises(RuntimeError, sel.re, 'lala')
+            self.assertRaises(RuntimeError, sel.extract)
+            self.assertRaises(RuntimeError, sel.register_namespace, 'a', 'b')
+            self.assertRaises(RuntimeError, sel.__nonzero__, 'a', 'b')