diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 0524c74f2..0f81e6a64 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -236,7 +236,8 @@ class ExecutionEngine(object): assert isinstance(response, (Response, Request)) if isinstance(response, Response): response.request = request # tie request to response received - log.msg("Crawled %s from <%s>" % (response, referer), level=log.DEBUG, domain=domain) + log.msg("Crawled %s from <%s>" % (response, referer), level=log.DEBUG, \ + domain=domain) return response elif isinstance(response, Request): newrequest = response diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index eac4176cf..4fb61ac81 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -127,7 +127,8 @@ class Scraper(object): def handle_spider_error(self, _failure, request, spider, propagated_failure=None): referer = request.headers.get('Referer', None) - msg = "SPIDER BUG processing <%s> from <%s>: %s" % (request.url, referer, _failure) + msg = "Spider exception caught while processing <%s> (referer: <%s>): %s" % \ + (request.url, referer, _failure) log.msg(msg, log.ERROR, domain=spider.domain_name) stats.inc_value("spider_exceptions/%s" % _failure.value.__class__.__name__, \ domain=spider.domain_name) diff --git a/scrapy/xpath/factories.py b/scrapy/xpath/factories.py index 3105155ad..ec9d2bfe8 100644 --- a/scrapy/xpath/factories.py +++ b/scrapy/xpath/factories.py @@ -14,8 +14,10 @@ html_parser_options = libxml2.HTML_PARSE_RECOVER + \ libxml2.HTML_PARSE_NOERROR + \ libxml2.HTML_PARSE_NOWARNING +utf8_encodings = set(('utf-8', 'UTF-8', 'utf8', 'UTF8')) + def body_as_utf8(response): - if response.encoding in ('utf-8', 'utf8'): + if response.encoding in utf8_encodings: return response.body else: return response.body_as_unicode().encode('utf-8') @@ -24,16 +26,20 @@ def xmlDoc_from_html(response): """Return libxml2 doc for HTMLs""" utf8body = body_as_utf8(response) try: - lxdoc = libxml2.htmlReadDoc(utf8body, response.url, 'utf-8', html_parser_options) + lxdoc = libxml2.htmlReadDoc(utf8body, response.url, 'utf-8', \ + html_parser_options) except TypeError: # libxml2 doesn't parse text with null bytes - lxdoc = libxml2.htmlReadDoc(utf8body.replace("\x00", ""), response.url, 'utf-8', html_parser_options) + lxdoc = libxml2.htmlReadDoc(utf8body.replace("\x00", ""), response.url, \ + 'utf-8', html_parser_options) return lxdoc def xmlDoc_from_xml(response): """Return libxml2 doc for XMLs""" utf8body = body_as_utf8(response) try: - lxdoc = libxml2.readDoc(utf8body, response.url, 'utf-8', xml_parser_options) + lxdoc = libxml2.readDoc(utf8body, response.url, 'utf-8', \ + xml_parser_options) except TypeError: # libxml2 doesn't parse text with null bytes - lxdoc = libxml2.readDoc(utf8body.replace("\x00", ""), response.url, 'utf-8', xml_parser_options) + lxdoc = libxml2.readDoc(utf8body.replace("\x00", ""), response.url, \ + 'utf-8', xml_parser_options) return lxdoc diff --git a/scrapy/xpath/selector.py b/scrapy/xpath/selector.py index 4fb38d54a..69a4d9f20 100644 --- a/scrapy/xpath/selector.py +++ b/scrapy/xpath/selector.py @@ -13,17 +13,6 @@ from scrapy.utils.python import flatten, unicode_to_str from scrapy.utils.misc import extract_regex class XPathSelector(object): - """The XPathSelector class provides a convenient way for selecting document - parts using XPaths and regexs, with support for nested queries. - - Although this is not an abstract class, you usually instantiate one of its - children: - - - XmlXPathSelector (for XML content) - - HtmlXPathSelector (for HTML content) - """ - - xmlDoc_factory = staticmethod(xmlDoc_from_html) def __init__(self, response=None, text=None, node=None, parent=None, expr=None): if parent: @@ -32,14 +21,14 @@ class XPathSelector(object): elif response: try: # try with cached version first - self.doc = response.getlibxml2doc(factory=self.xmlDoc_factory) + self.doc = response.getlibxml2doc(factory=self._get_libxml2_doc) except AttributeError: - self.doc = Libxml2Document(response, factory=self.xmlDoc_factory) + self.doc = Libxml2Document(response, factory=self._get_libxml2_doc) self.xmlNode = self.doc.xmlDoc elif text: response = TextResponse(url=None, body=unicode_to_str(text), \ encoding='utf-8') - self.doc = Libxml2Document(response, factory=self.xmlDoc_factory) + self.doc = Libxml2Document(response, factory=self._get_libxml2_doc) self.xmlNode = self.doc.xmlDoc self.expr = expr self.response = response @@ -101,11 +90,16 @@ class XPathSelector(object): """Register namespace so that it can be used in XPath queries""" self.doc.xpathContext.xpathRegisterNs(prefix, uri) + def _get_libxml2_doc(self, response): + """Return libxml2 document (xmlDoc) from response""" + return xmlDoc_from_html(response) + def __nonzero__(self): return bool(self.extract()) def __str__(self): - return "<%s (%s) xpath=%s>" % (type(self).__name__, getattr(self.xmlNode, 'name', type(self.xmlNode).__name__), self.expr) + return "<%s (%s) xpath=%s>" % (type(self).__name__, getattr(self.xmlNode, \ + 'name', type(self.xmlNode).__name__), self.expr) __repr__ = __str__ @@ -137,9 +131,9 @@ class XPathSelectorList(list): class XmlXPathSelector(XPathSelector): """XPathSelector for XML content""" - xmlDoc_factory = staticmethod(xmlDoc_from_xml) + _get_libxml2_doc = staticmethod(xmlDoc_from_xml) class HtmlXPathSelector(XPathSelector): """XPathSelector for HTML content""" - xmlDoc_factory = staticmethod(xmlDoc_from_html) + _get_libxml2_doc = staticmethod(xmlDoc_from_html)