additional cleanup to scrapy.xpath module

scrapy/core/engine.py

@@ -236,7 +236,8 @@ class ExecutionEngine(object):
             assert isinstance(response, (Response, Request))
             if isinstance(response, Response):
                 response.request = request # tie request to response received
-                log.msg("Crawled %s from <%s>" % (response, referer), level=log.DEBUG, domain=domain)
+                log.msg("Crawled %s from <%s>" % (response, referer), level=log.DEBUG, \
+                    domain=domain)
                 return response
             elif isinstance(response, Request):
                 newrequest = response

scrapy/core/scraper.py

@@ -127,7 +127,8 @@ class Scraper(object):
 
     def handle_spider_error(self, _failure, request, spider, propagated_failure=None):
         referer = request.headers.get('Referer', None)
-        msg = "SPIDER BUG processing <%s> from <%s>: %s" % (request.url, referer, _failure)
+        msg = "Spider exception caught while processing <%s> (referer: <%s>): %s" % \
+            (request.url, referer, _failure)
         log.msg(msg, log.ERROR, domain=spider.domain_name)
         stats.inc_value("spider_exceptions/%s" % _failure.value.__class__.__name__, \
             domain=spider.domain_name)

scrapy/xpath/factories.py

@@ -14,8 +14,10 @@ html_parser_options = libxml2.HTML_PARSE_RECOVER + \
                       libxml2.HTML_PARSE_NOERROR + \
                       libxml2.HTML_PARSE_NOWARNING
 
+utf8_encodings = set(('utf-8', 'UTF-8', 'utf8', 'UTF8'))
+
 def body_as_utf8(response):
-    if response.encoding in ('utf-8', 'utf8'):
+    if response.encoding in utf8_encodings:
         return response.body
     else:
         return response.body_as_unicode().encode('utf-8')
@@ -24,16 +26,20 @@ def xmlDoc_from_html(response):
     """Return libxml2 doc for HTMLs"""
     utf8body = body_as_utf8(response)
     try:
-        lxdoc = libxml2.htmlReadDoc(utf8body, response.url, 'utf-8', html_parser_options)
+        lxdoc = libxml2.htmlReadDoc(utf8body, response.url, 'utf-8', \
+            html_parser_options)
     except TypeError:  # libxml2 doesn't parse text with null bytes
-        lxdoc = libxml2.htmlReadDoc(utf8body.replace("\x00", ""), response.url, 'utf-8', html_parser_options)
+        lxdoc = libxml2.htmlReadDoc(utf8body.replace("\x00", ""), response.url, \
+            'utf-8', html_parser_options)
     return lxdoc
 
 def xmlDoc_from_xml(response):
     """Return libxml2 doc for XMLs"""
     utf8body = body_as_utf8(response)
     try:
-        lxdoc = libxml2.readDoc(utf8body, response.url, 'utf-8', xml_parser_options)
+        lxdoc = libxml2.readDoc(utf8body, response.url, 'utf-8', \
+            xml_parser_options)
     except TypeError:  # libxml2 doesn't parse text with null bytes
-        lxdoc = libxml2.readDoc(utf8body.replace("\x00", ""), response.url, 'utf-8', xml_parser_options)
+        lxdoc = libxml2.readDoc(utf8body.replace("\x00", ""), response.url, \
+            'utf-8', xml_parser_options)
     return lxdoc

scrapy/xpath/selector.py

@@ -13,17 +13,6 @@ from scrapy.utils.python import flatten, unicode_to_str
 from scrapy.utils.misc import extract_regex
 
 class XPathSelector(object):
-    """The XPathSelector class provides a convenient way for selecting document
-    parts using XPaths and regexs, with support for nested queries.
-
-    Although this is not an abstract class, you usually instantiate one of its
-    children:
-    
-    - XmlXPathSelector (for XML content)
-    - HtmlXPathSelector (for HTML content)
-    """
-
-    xmlDoc_factory = staticmethod(xmlDoc_from_html)
 
     def __init__(self, response=None, text=None, node=None, parent=None, expr=None):
         if parent:
@@ -32,14 +21,14 @@ class XPathSelector(object):
         elif response:
             try:
                 # try with cached version first
-                self.doc = response.getlibxml2doc(factory=self.xmlDoc_factory)
+                self.doc = response.getlibxml2doc(factory=self._get_libxml2_doc)
             except AttributeError:
-                self.doc = Libxml2Document(response, factory=self.xmlDoc_factory)
+                self.doc = Libxml2Document(response, factory=self._get_libxml2_doc)
             self.xmlNode = self.doc.xmlDoc
         elif text:
             response = TextResponse(url=None, body=unicode_to_str(text), \
                 encoding='utf-8')
-            self.doc = Libxml2Document(response, factory=self.xmlDoc_factory)
+            self.doc = Libxml2Document(response, factory=self._get_libxml2_doc)
             self.xmlNode = self.doc.xmlDoc
         self.expr = expr
         self.response = response
@@ -101,11 +90,16 @@ class XPathSelector(object):
         """Register namespace so that it can be used in XPath queries"""
         self.doc.xpathContext.xpathRegisterNs(prefix, uri)
 
+    def _get_libxml2_doc(self, response):
+        """Return libxml2 document (xmlDoc) from response"""
+        return xmlDoc_from_html(response)
+
     def __nonzero__(self):
         return bool(self.extract())
 
     def __str__(self):
-        return "<%s (%s) xpath=%s>" % (type(self).__name__, getattr(self.xmlNode, 'name', type(self.xmlNode).__name__), self.expr)
+        return "<%s (%s) xpath=%s>" % (type(self).__name__, getattr(self.xmlNode, \
+            'name', type(self.xmlNode).__name__), self.expr)
 
     __repr__ = __str__
 
@@ -137,9 +131,9 @@ class XPathSelectorList(list):
 
 class XmlXPathSelector(XPathSelector):
     """XPathSelector for XML content"""
-    xmlDoc_factory = staticmethod(xmlDoc_from_xml)
+    _get_libxml2_doc = staticmethod(xmlDoc_from_xml)
 
 
 class HtmlXPathSelector(XPathSelector):
     """XPathSelector for HTML content"""
-    xmlDoc_factory = staticmethod(xmlDoc_from_html)
+    _get_libxml2_doc = staticmethod(xmlDoc_from_html)