mirror of https://github.com/scrapy/scrapy.git
* Added lxml backend for XPath selectors. Closes #147
* Added new setting (SELECTORS_BACKEND) to choose which backend to use * Deprecated the extract_unquoted() function from selectors * Made libxml2 optional by adding a dummy selector backend. Closes #260 --HG-- rename : scrapy/tests/test_selector.py => scrapy/tests/test_selector_libxml2.py
This commit is contained in:
Pablo Hoffman
parent
7640e99979
commit
a59bfb539d
|
|
@ -255,14 +255,6 @@ XPathSelector objects
|
|||
Return a unicode string with the content of this :class:`XPathSelector`
|
||||
object.
|
||||
|
||||
.. method:: extract_unquoted()
|
||||
|
||||
Return a unicode string with the content of this :class:`XPathSelector`
|
||||
without entities or CDATA. This method is intended to be use for text-only
|
||||
selectors, like ``//h1/text()`` (but not ``//h1``). If it's used for
|
||||
:class:`XPathSelector` objects which don't select a textual content (ie. if
|
||||
they contain tags), the output of this method is undefined.
|
||||
|
||||
.. method:: register_namespace(prefix, uri)
|
||||
|
||||
Register the given namespace to be used in this :class:`XPathSelector`.
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ See documentation in docs/topics/extensions.rst
|
|||
import gc
|
||||
import socket
|
||||
|
||||
import libxml2
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
|
||||
from scrapy import signals
|
||||
|
|
@ -19,6 +18,11 @@ from scrapy import log
|
|||
class MemoryDebugger(object):
|
||||
|
||||
def __init__(self):
|
||||
try:
|
||||
import libxml2
|
||||
self.libxml2 = libxml2
|
||||
except ImportError:
|
||||
raise NotConfigured
|
||||
if not settings.getbool('MEMDEBUG_ENABLED'):
|
||||
raise NotConfigured
|
||||
|
||||
|
|
@ -29,7 +33,7 @@ class MemoryDebugger(object):
|
|||
dispatcher.connect(self.engine_stopped, signals.engine_stopped)
|
||||
|
||||
def engine_started(self):
|
||||
libxml2.debugMemory(1)
|
||||
self.libxml2.debugMemory(1)
|
||||
|
||||
def engine_stopped(self):
|
||||
figures = self.collect_figures()
|
||||
|
|
@ -37,12 +41,12 @@ class MemoryDebugger(object):
|
|||
self.log_or_send_report(report)
|
||||
|
||||
def collect_figures(self):
|
||||
libxml2.cleanupParser()
|
||||
self.libxml2.cleanupParser()
|
||||
gc.collect()
|
||||
|
||||
figures = []
|
||||
figures.append(("Objects in gc.garbage", len(gc.garbage), ""))
|
||||
figures.append(("libxml2 memory leak", libxml2.debugMemory(1), "bytes"))
|
||||
figures.append(("libxml2 memory leak", self.libxml2.debugMemory(1), "bytes"))
|
||||
return figures
|
||||
|
||||
def create_report(self, figures):
|
||||
|
|
|
|||
|
|
@ -1,153 +1,30 @@
|
|||
"""
|
||||
XPath selectors
|
||||
XPath selectors
|
||||
|
||||
See documentation in docs/topics/selectors.rst
|
||||
Two backends are currently available: libxml2 and lxml
|
||||
|
||||
To select the backend explicitly use the SELECTORS_BACKEND variable in your
|
||||
project. Otherwise, libxml2 will be tried first. If libxml2 is not available,
|
||||
lxml will be used.
|
||||
"""
|
||||
|
||||
import libxml2
|
||||
from scrapy.conf import settings
|
||||
|
||||
from scrapy.http import TextResponse
|
||||
from scrapy.utils.python import flatten, unicode_to_str
|
||||
from scrapy.utils.misc import extract_regex
|
||||
from scrapy.utils.trackref import object_ref
|
||||
from scrapy.utils.decorator import deprecated
|
||||
from .factories import xmlDoc_from_html, xmlDoc_from_xml
|
||||
from .document import Libxml2Document
|
||||
|
||||
__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \
|
||||
'XPathSelectorList']
|
||||
|
||||
class XPathSelector(object_ref):
|
||||
|
||||
__slots__ = ['doc', 'xmlNode', 'expr', '__weakref__']
|
||||
|
||||
def __init__(self, response=None, text=None, node=None, parent=None, expr=None):
|
||||
if parent:
|
||||
self.doc = parent.doc
|
||||
self.xmlNode = node
|
||||
elif response:
|
||||
self.doc = Libxml2Document(response, factory=self._get_libxml2_doc)
|
||||
self.xmlNode = self.doc.xmlDoc
|
||||
elif text:
|
||||
response = TextResponse(url='about:blank', \
|
||||
body=unicode_to_str(text, 'utf-8'), encoding='utf-8')
|
||||
self.doc = Libxml2Document(response, factory=self._get_libxml2_doc)
|
||||
self.xmlNode = self.doc.xmlDoc
|
||||
self.expr = expr
|
||||
|
||||
def select(self, xpath):
|
||||
"""Perform the given XPath query on the current XPathSelector and
|
||||
return a XPathSelectorList of the result"""
|
||||
if hasattr(self.xmlNode, 'xpathEval'):
|
||||
self.doc.xpathContext.setContextNode(self.xmlNode)
|
||||
try:
|
||||
xpath_result = self.doc.xpathContext.xpathEval(xpath)
|
||||
except libxml2.xpathError:
|
||||
raise ValueError("Invalid XPath: %s" % xpath)
|
||||
if hasattr(xpath_result, '__iter__'):
|
||||
return XPathSelectorList([self.__class__(node=node, parent=self, \
|
||||
expr=xpath) for node in xpath_result])
|
||||
else:
|
||||
return XPathSelectorList([self.__class__(node=xpath_result, \
|
||||
parent=self, expr=xpath)])
|
||||
if settings['SELECTORS_BACKEND'] == 'lxml':
|
||||
from .lxmlsel import *
|
||||
elif settings['SELECTORS_BACKEND'] == 'libxml2':
|
||||
from .libxml2sel import *
|
||||
elif settings['SELECTORS_BACKEND'] == 'dummy':
|
||||
from .dummysel import *
|
||||
else:
|
||||
try:
|
||||
import libxml2
|
||||
except ImportError:
|
||||
try:
|
||||
import lxml
|
||||
except ImportError:
|
||||
from .dummysel import *
|
||||
else:
|
||||
return XPathSelectorList([])
|
||||
|
||||
def re(self, regex):
|
||||
"""Return a list of unicode strings by applying the regex over all
|
||||
current XPath selections, and flattening the results"""
|
||||
return extract_regex(regex, self.extract(), 'utf-8')
|
||||
|
||||
def extract(self):
|
||||
"""Return a unicode string of the content referenced by the XPathSelector"""
|
||||
if isinstance(self.xmlNode, basestring):
|
||||
text = unicode(self.xmlNode, 'utf-8', errors='ignore')
|
||||
elif hasattr(self.xmlNode, 'serialize'):
|
||||
if isinstance(self.xmlNode, libxml2.xmlDoc):
|
||||
data = self.xmlNode.getRootElement().serialize('utf-8')
|
||||
text = unicode(data, 'utf-8', errors='ignore') if data else u''
|
||||
elif isinstance(self.xmlNode, libxml2.xmlAttr):
|
||||
# serialization doesn't work sometimes for xmlAttr types
|
||||
text = unicode(self.xmlNode.content, 'utf-8', errors='ignore')
|
||||
else:
|
||||
data = self.xmlNode.serialize('utf-8')
|
||||
text = unicode(data, 'utf-8', errors='ignore') if data else u''
|
||||
else:
|
||||
try:
|
||||
text = unicode(self.xmlNode, 'utf-8', errors='ignore')
|
||||
except TypeError: # catched when self.xmlNode is a float - see tests
|
||||
text = unicode(self.xmlNode)
|
||||
return text
|
||||
|
||||
def extract_unquoted(self):
|
||||
"""Get unescaped contents from the text node (no entities, no CDATA)"""
|
||||
if self.select('self::text()'):
|
||||
return unicode(self.xmlNode.getContent(), 'utf-8', errors='ignore')
|
||||
else:
|
||||
return u''
|
||||
|
||||
def register_namespace(self, prefix, uri):
|
||||
"""Register namespace so that it can be used in XPath queries"""
|
||||
self.doc.xpathContext.xpathRegisterNs(prefix, uri)
|
||||
|
||||
def _get_libxml2_doc(self, response):
|
||||
"""Return libxml2 document (xmlDoc) from response"""
|
||||
return xmlDoc_from_html(response)
|
||||
|
||||
def __nonzero__(self):
|
||||
return bool(self.extract())
|
||||
|
||||
def __str__(self):
|
||||
return "<%s (%s) xpath=%s>" % (type(self).__name__, getattr(self.xmlNode, \
|
||||
'name', type(self.xmlNode).__name__), self.expr)
|
||||
|
||||
__repr__ = __str__
|
||||
|
||||
@deprecated(use_instead='XPathSelector.select')
|
||||
def __call__(self, xpath):
|
||||
return self.select(xpath)
|
||||
|
||||
@deprecated(use_instead='XPathSelector.select')
|
||||
def x(self, xpath):
|
||||
return self.select(xpath)
|
||||
|
||||
|
||||
class XPathSelectorList(list):
|
||||
"""List of XPathSelector objects"""
|
||||
|
||||
def __getslice__(self, i, j):
|
||||
return XPathSelectorList(list.__getslice__(self, i, j))
|
||||
|
||||
def select(self, xpath):
|
||||
"""Perform the given XPath query on each XPathSelector of the list and
|
||||
return a new (flattened) XPathSelectorList of the results"""
|
||||
return XPathSelectorList(flatten([x.select(xpath) for x in self]))
|
||||
|
||||
def re(self, regex):
|
||||
"""Perform the re() method on each XPathSelector of the list, and
|
||||
return the result as a flattened list of unicode strings"""
|
||||
return flatten([x.re(regex) for x in self])
|
||||
|
||||
def extract(self):
|
||||
"""Return a list of unicode strings with the content referenced by each
|
||||
XPathSelector of the list"""
|
||||
return [x.extract() if isinstance(x, XPathSelector) else x for x in self]
|
||||
|
||||
def extract_unquoted(self):
|
||||
return [x.extract_unquoted() if isinstance(x, XPathSelector) else x for x in self]
|
||||
|
||||
@deprecated(use_instead='XPathSelectorList.select')
|
||||
def x(self, xpath):
|
||||
return self.select(xpath)
|
||||
|
||||
|
||||
class XmlXPathSelector(XPathSelector):
|
||||
"""XPathSelector for XML content"""
|
||||
__slots__ = ()
|
||||
_get_libxml2_doc = staticmethod(xmlDoc_from_xml)
|
||||
|
||||
|
||||
class HtmlXPathSelector(XPathSelector):
|
||||
"""XPathSelector for HTML content"""
|
||||
__slots__ = ()
|
||||
_get_libxml2_doc = staticmethod(xmlDoc_from_html)
|
||||
from .lxmlsel import *
|
||||
else:
|
||||
from .libxml2sel import *
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
"""
|
||||
Dummy selectors
|
||||
"""
|
||||
|
||||
__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \
|
||||
'XPathSelectorList']
|
||||
|
||||
class XPathSelector(object):
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
pass
|
||||
|
||||
def _raise(self, *a, **kw):
|
||||
raise RuntimeError("No selectors backend available. " \
|
||||
"Please install libxml2 or lxml")
|
||||
|
||||
select = re = exract = register_namespace = __nonzero__ = _raise
|
||||
|
||||
class XPathSelectorList(list):
|
||||
|
||||
def _raise(self, *a, **kw):
|
||||
raise RuntimeError("No selectors backend available. " \
|
||||
"Please install libxml2 or lxml")
|
||||
|
||||
__getslice__ = select = re = extract = extract_unquoted = _raise
|
||||
|
||||
XmlXPathSelector = XPathSelector
|
||||
HtmlXPathSelector = XPathSelector
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
"""
|
||||
XPath selectors based on libxml2
|
||||
"""
|
||||
|
||||
import libxml2
|
||||
|
||||
from scrapy.http import TextResponse
|
||||
from scrapy.utils.python import flatten, unicode_to_str
|
||||
from scrapy.utils.misc import extract_regex
|
||||
from scrapy.utils.trackref import object_ref
|
||||
from scrapy.utils.decorator import deprecated
|
||||
from .factories import xmlDoc_from_html, xmlDoc_from_xml
|
||||
from .document import Libxml2Document
|
||||
|
||||
__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \
|
||||
'XPathSelectorList']
|
||||
|
||||
class XPathSelector(object_ref):
|
||||
|
||||
__slots__ = ['doc', 'xmlNode', 'expr', '__weakref__']
|
||||
|
||||
def __init__(self, response=None, text=None, node=None, parent=None, expr=None):
|
||||
if parent:
|
||||
self.doc = parent.doc
|
||||
self.xmlNode = node
|
||||
elif response:
|
||||
self.doc = Libxml2Document(response, factory=self._get_libxml2_doc)
|
||||
self.xmlNode = self.doc.xmlDoc
|
||||
elif text:
|
||||
response = TextResponse(url='about:blank', \
|
||||
body=unicode_to_str(text, 'utf-8'), encoding='utf-8')
|
||||
self.doc = Libxml2Document(response, factory=self._get_libxml2_doc)
|
||||
self.xmlNode = self.doc.xmlDoc
|
||||
self.expr = expr
|
||||
|
||||
def select(self, xpath):
|
||||
"""Perform the given XPath query on the current XPathSelector and
|
||||
return a XPathSelectorList of the result"""
|
||||
if hasattr(self.xmlNode, 'xpathEval'):
|
||||
self.doc.xpathContext.setContextNode(self.xmlNode)
|
||||
try:
|
||||
xpath_result = self.doc.xpathContext.xpathEval(xpath)
|
||||
except libxml2.xpathError:
|
||||
raise ValueError("Invalid XPath: %s" % xpath)
|
||||
if hasattr(xpath_result, '__iter__'):
|
||||
return XPathSelectorList([self.__class__(node=node, parent=self, \
|
||||
expr=xpath) for node in xpath_result])
|
||||
else:
|
||||
return XPathSelectorList([self.__class__(node=xpath_result, \
|
||||
parent=self, expr=xpath)])
|
||||
else:
|
||||
return XPathSelectorList([])
|
||||
|
||||
def re(self, regex):
|
||||
"""Return a list of unicode strings by applying the regex over all
|
||||
current XPath selections, and flattening the results"""
|
||||
return extract_regex(regex, self.extract(), 'utf-8')
|
||||
|
||||
def extract(self):
|
||||
"""Return a unicode string of the content referenced by the XPathSelector"""
|
||||
if isinstance(self.xmlNode, basestring):
|
||||
text = unicode(self.xmlNode, 'utf-8', errors='ignore')
|
||||
elif hasattr(self.xmlNode, 'serialize'):
|
||||
if isinstance(self.xmlNode, libxml2.xmlDoc):
|
||||
data = self.xmlNode.getRootElement().serialize('utf-8')
|
||||
text = unicode(data, 'utf-8', errors='ignore') if data else u''
|
||||
elif isinstance(self.xmlNode, libxml2.xmlAttr):
|
||||
# serialization doesn't work sometimes for xmlAttr types
|
||||
text = unicode(self.xmlNode.content, 'utf-8', errors='ignore')
|
||||
else:
|
||||
data = self.xmlNode.serialize('utf-8')
|
||||
text = unicode(data, 'utf-8', errors='ignore') if data else u''
|
||||
else:
|
||||
try:
|
||||
text = unicode(self.xmlNode, 'utf-8', errors='ignore')
|
||||
except TypeError: # catched when self.xmlNode is a float - see tests
|
||||
text = unicode(self.xmlNode)
|
||||
return text
|
||||
|
||||
def extract_unquoted(self):
|
||||
"""Get unescaped contents from the text node (no entities, no CDATA)"""
|
||||
if self.select('self::text()'):
|
||||
return unicode(self.xmlNode.getContent(), 'utf-8', errors='ignore')
|
||||
else:
|
||||
return u''
|
||||
|
||||
def register_namespace(self, prefix, uri):
|
||||
"""Register namespace so that it can be used in XPath queries"""
|
||||
self.doc.xpathContext.xpathRegisterNs(prefix, uri)
|
||||
|
||||
def _get_libxml2_doc(self, response):
|
||||
"""Return libxml2 document (xmlDoc) from response"""
|
||||
return xmlDoc_from_html(response)
|
||||
|
||||
def __nonzero__(self):
|
||||
return bool(self.extract())
|
||||
|
||||
def __str__(self):
|
||||
return "<%s (%s) xpath=%s>" % (type(self).__name__, getattr(self.xmlNode, \
|
||||
'name', type(self.xmlNode).__name__), self.expr)
|
||||
|
||||
__repr__ = __str__
|
||||
|
||||
@deprecated(use_instead='XPathSelector.select')
|
||||
def __call__(self, xpath):
|
||||
return self.select(xpath)
|
||||
|
||||
@deprecated(use_instead='XPathSelector.select')
|
||||
def x(self, xpath):
|
||||
return self.select(xpath)
|
||||
|
||||
|
||||
class XPathSelectorList(list):
|
||||
"""List of XPathSelector objects"""
|
||||
|
||||
def __getslice__(self, i, j):
|
||||
return XPathSelectorList(list.__getslice__(self, i, j))
|
||||
|
||||
def select(self, xpath):
|
||||
"""Perform the given XPath query on each XPathSelector of the list and
|
||||
return a new (flattened) XPathSelectorList of the results"""
|
||||
return XPathSelectorList(flatten([x.select(xpath) for x in self]))
|
||||
|
||||
def re(self, regex):
|
||||
"""Perform the re() method on each XPathSelector of the list, and
|
||||
return the result as a flattened list of unicode strings"""
|
||||
return flatten([x.re(regex) for x in self])
|
||||
|
||||
def extract(self):
|
||||
"""Return a list of unicode strings with the content referenced by each
|
||||
XPathSelector of the list"""
|
||||
return [x.extract() if isinstance(x, XPathSelector) else x for x in self]
|
||||
|
||||
def extract_unquoted(self):
|
||||
return [x.extract_unquoted() if isinstance(x, XPathSelector) else x for x in self]
|
||||
|
||||
@deprecated(use_instead='XPathSelectorList.select')
|
||||
def x(self, xpath):
|
||||
return self.select(xpath)
|
||||
|
||||
|
||||
class XmlXPathSelector(XPathSelector):
|
||||
"""XPathSelector for XML content"""
|
||||
__slots__ = ()
|
||||
_get_libxml2_doc = staticmethod(xmlDoc_from_xml)
|
||||
|
||||
|
||||
class HtmlXPathSelector(XPathSelector):
|
||||
"""XPathSelector for HTML content"""
|
||||
__slots__ = ()
|
||||
_get_libxml2_doc = staticmethod(xmlDoc_from_html)
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
"""
|
||||
XPath selectors based on lxml
|
||||
"""
|
||||
|
||||
from lxml import etree
|
||||
|
||||
from scrapy.utils.python import flatten
|
||||
from scrapy.utils.misc import extract_regex
|
||||
from scrapy.utils.trackref import object_ref
|
||||
from scrapy.utils.python import unicode_to_str
|
||||
from scrapy.utils.decorator import deprecated
|
||||
from scrapy.http import TextResponse
|
||||
|
||||
__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \
|
||||
'XPathSelectorList']
|
||||
|
||||
class XPathSelector(object_ref):
|
||||
|
||||
__slots__ = ['response', 'text', 'expr', 'namespaces', '_root', '__weakref__']
|
||||
_parser = etree.HTMLParser
|
||||
_tostring_method = 'html'
|
||||
|
||||
def __init__(self, response=None, text=None, root=None, expr=None, namespaces=None):
|
||||
if text:
|
||||
self.response = TextResponse(url='about:blank', \
|
||||
body=unicode_to_str(text, 'utf-8'), encoding='utf-8')
|
||||
else:
|
||||
self.response = response
|
||||
self._root = root
|
||||
self.namespaces = namespaces
|
||||
self.expr = expr
|
||||
|
||||
@property
|
||||
def root(self):
|
||||
if self._root is None:
|
||||
parser = self._parser(encoding=self.response.encoding, recover=True)
|
||||
self._root = etree.fromstring(self.response.body, parser=parser, \
|
||||
base_url=self.response.url)
|
||||
return self._root
|
||||
|
||||
def select(self, xpath):
|
||||
xpatheval = etree.XPathEvaluator(self.root, namespaces=self.namespaces)
|
||||
try:
|
||||
result = xpatheval(xpath)
|
||||
except etree.XPathError:
|
||||
raise ValueError("Invalid XPath: %s" % xpath)
|
||||
if hasattr(result, '__iter__'):
|
||||
result = [self.__class__(root=x, expr=xpath, namespaces=self.namespaces) \
|
||||
for x in result]
|
||||
elif result:
|
||||
result = [self.__class__(root=result, expr=xpath, namespaces=self.namespaces)]
|
||||
return XPathSelectorList(result)
|
||||
|
||||
def re(self, regex):
|
||||
return extract_regex(regex, self.extract(), 'utf-8')
|
||||
|
||||
def extract(self):
|
||||
try:
|
||||
return etree.tostring(self.root, method=self._tostring_method, \
|
||||
encoding=unicode).strip()
|
||||
except (AttributeError, TypeError):
|
||||
return unicode(self.root).strip()
|
||||
|
||||
def register_namespace(self, prefix, uri):
|
||||
if self.namespaces is None:
|
||||
self.namespaces = {}
|
||||
self.namespaces[prefix] = uri
|
||||
|
||||
def __nonzero__(self):
|
||||
return bool(self.extract())
|
||||
|
||||
def __str__(self):
|
||||
data = repr(self.extract()[:40])
|
||||
return "<%s xpath=%r data=%s>" % (type(self).__name__, self.expr, data)
|
||||
|
||||
__repr__ = __str__
|
||||
|
||||
|
||||
@deprecated(use_instead='XPathSelector.extract')
|
||||
def extract_unquoted(self):
|
||||
return self.extract()
|
||||
|
||||
|
||||
class XPathSelectorList(list):
|
||||
|
||||
def __getslice__(self, i, j):
|
||||
return XPathSelectorList(list.__getslice__(self, i, j))
|
||||
|
||||
def select(self, xpath):
|
||||
return XPathSelectorList(flatten([x.select(xpath) for x in self]))
|
||||
|
||||
def re(self, regex):
|
||||
return flatten([x.re(regex) for x in self])
|
||||
|
||||
def extract(self):
|
||||
return [x.extract() if isinstance(x, XPathSelector) else x for x in self]
|
||||
|
||||
@deprecated(use_instead='XPathSelectorList.extract_unquoted')
|
||||
def extract_unquoted(self):
|
||||
return [x.extract_unquoted() if isinstance(x, XPathSelector) else x for x in self]
|
||||
|
||||
|
||||
class XmlXPathSelector(XPathSelector):
|
||||
"""XPathSelector for XML content"""
|
||||
__slots__ = ()
|
||||
_parser = etree.XMLParser
|
||||
_tostring_method = 'xml'
|
||||
|
||||
|
||||
class HtmlXPathSelector(XPathSelector):
|
||||
"""XPathSelector for HTML content"""
|
||||
__slots__ = ()
|
||||
_parser = etree.HTMLParser
|
||||
_tostring_method = 'html'
|
||||
|
|
@ -212,6 +212,8 @@ SCHEDULER_MIDDLEWARES_BASE = {
|
|||
|
||||
SCHEDULER_ORDER = 'DFO'
|
||||
|
||||
SELECTORS_BACKEND = None # possible values: libxml2, lxml
|
||||
|
||||
SPIDER_MANAGER_CLASS = 'scrapy.spidermanager.SpiderManager'
|
||||
|
||||
SPIDER_MIDDLEWARES = {}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
from twisted.trial import unittest
|
||||
|
||||
from scrapy.utils.test import libxml2debug
|
||||
|
||||
class Libxml2Test(unittest.TestCase):
|
||||
|
||||
try:
|
||||
import libxml2
|
||||
except ImportError, e:
|
||||
skip = str(e)
|
||||
|
||||
@libxml2debug
|
||||
def test_libxml2_bug_2_6_27(self):
|
||||
# this test will fail in version 2.6.27 but passes on 2.6.29+
|
||||
html = "<td>1<b>2</b>3</td>"
|
||||
node = self.libxml2.htmlParseDoc(html, 'utf-8')
|
||||
result = [str(r) for r in node.xpathEval('//text()')]
|
||||
self.assertEquals(result, ['1', '2', '3'])
|
||||
node.freeDoc()
|
||||
|
||||
|
|
@ -2,8 +2,6 @@ import re
|
|||
import unittest
|
||||
import weakref
|
||||
|
||||
import libxml2
|
||||
|
||||
from scrapy.http import TextResponse, HtmlResponse, XmlResponse
|
||||
from scrapy.selector import XmlXPathSelector, HtmlXPathSelector, \
|
||||
XPathSelector
|
||||
|
|
@ -284,16 +282,5 @@ class Libxml2DocumentTest(unittest.TestCase):
|
|||
headers={'Content-Type': 'text/plain; charset=utf-8'}, body=self.body_content)
|
||||
Libxml2Document(response)
|
||||
|
||||
class Libxml2Test(unittest.TestCase):
|
||||
|
||||
@libxml2debug
|
||||
def test_libxml2_bug_2_6_27(self):
|
||||
# this test will fail in version 2.6.27 but passes on 2.6.29+
|
||||
html = "<td>1<b>2</b>3</td>"
|
||||
node = libxml2.htmlParseDoc(html, 'utf-8')
|
||||
result = [str(r) for r in node.xpathEval('//text()')]
|
||||
self.assertEquals(result, ['1', '2', '3'])
|
||||
node.freeDoc()
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -0,0 +1,258 @@
|
|||
# TODO: we should merge these tests with test_selector_libxml2.py
|
||||
|
||||
import re
|
||||
import unittest
|
||||
import weakref
|
||||
|
||||
from scrapy.http import TextResponse, HtmlResponse, XmlResponse
|
||||
from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, \
|
||||
XPathSelector
|
||||
from scrapy.utils.test import libxml2debug
|
||||
|
||||
class XPathSelectorTestCase(unittest.TestCase):
|
||||
|
||||
@libxml2debug
|
||||
def test_selector_simple(self):
|
||||
"""Simple selector tests"""
|
||||
body = "<p><input name='a'value='1'/><input name='b'value='2'/></p>"
|
||||
response = TextResponse(url="http://example.com", body=body)
|
||||
xpath = HtmlXPathSelector(response)
|
||||
|
||||
xl = xpath.select('//input')
|
||||
self.assertEqual(2, len(xl))
|
||||
for x in xl:
|
||||
assert isinstance(x, HtmlXPathSelector)
|
||||
|
||||
self.assertEqual(xpath.select('//input').extract(),
|
||||
[x.extract() for x in xpath.select('//input')])
|
||||
|
||||
self.assertEqual([x.extract() for x in xpath.select("//input[@name='a']/@name")],
|
||||
[u'a'])
|
||||
self.assertEqual([x.extract() for x in xpath.select("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
|
||||
[u'12.0'])
|
||||
|
||||
self.assertEqual(xpath.select("concat('xpath', 'rules')").extract(),
|
||||
[u'xpathrules'])
|
||||
self.assertEqual([x.extract() for x in xpath.select("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
|
||||
[u'12'])
|
||||
|
||||
@libxml2debug
|
||||
def test_selector_same_type(self):
|
||||
"""Test XPathSelector returning the same type in x() method"""
|
||||
text = '<p>test<p>'
|
||||
assert isinstance(XmlXPathSelector(text=text).select("//p")[0],
|
||||
XmlXPathSelector)
|
||||
assert isinstance(HtmlXPathSelector(text=text).select("//p")[0],
|
||||
HtmlXPathSelector)
|
||||
|
||||
@libxml2debug
|
||||
def test_selector_xml_html(self):
|
||||
"""Test that XML and HTML XPathSelector's behave differently"""
|
||||
|
||||
# some text which is parsed differently by XML and HTML flavors
|
||||
text = '<div><img src="a.jpg"><p>Hello</div>'
|
||||
|
||||
self.assertEqual(XmlXPathSelector(text=text).select("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
|
||||
|
||||
self.assertEqual(HtmlXPathSelector(text=text).select("//div").extract(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></div>'])
|
||||
|
||||
@libxml2debug
|
||||
def test_selector_nested(self):
|
||||
"""Nested selector tests"""
|
||||
body = """<body>
|
||||
<div class='one'>
|
||||
<ul>
|
||||
<li>one</li><li>two</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class='two'>
|
||||
<ul>
|
||||
<li>four</li><li>five</li><li>six</li>
|
||||
</ul>
|
||||
</div>
|
||||
</body>"""
|
||||
|
||||
response = HtmlResponse(url="http://example.com", body=body)
|
||||
x = HtmlXPathSelector(response)
|
||||
|
||||
divtwo = x.select('//div[@class="two"]')
|
||||
self.assertEqual(divtwo.select("//li").extract(),
|
||||
["<li>one</li>", "<li>two</li>", "<li>four</li>", "<li>five</li>", "<li>six</li>"])
|
||||
self.assertEqual(divtwo.select("./ul/li").extract(),
|
||||
["<li>four</li>", "<li>five</li>", "<li>six</li>"])
|
||||
self.assertEqual(divtwo.select(".//li").extract(),
|
||||
["<li>four</li>", "<li>five</li>", "<li>six</li>"])
|
||||
self.assertEqual(divtwo.select("./li").extract(),
|
||||
[])
|
||||
|
||||
@libxml2debug
|
||||
def test_selector_re(self):
|
||||
body = """<div>Name: Mary
|
||||
<ul>
|
||||
<li>Name: John</li>
|
||||
<li>Age: 10</li>
|
||||
<li>Name: Paul</li>
|
||||
<li>Age: 20</li>
|
||||
</ul>
|
||||
Age: 20
|
||||
</div>
|
||||
|
||||
"""
|
||||
response = HtmlResponse(url="http://example.com", body=body)
|
||||
x = HtmlXPathSelector(response)
|
||||
|
||||
name_re = re.compile("Name: (\w+)")
|
||||
self.assertEqual(x.select("//ul/li").re(name_re),
|
||||
["John", "Paul"])
|
||||
self.assertEqual(x.select("//ul/li").re("Age: (\d+)"),
|
||||
["10", "20"])
|
||||
|
||||
@libxml2debug
|
||||
def test_selector_over_text(self):
|
||||
hxs = HtmlXPathSelector(text='<root>lala</root>')
|
||||
self.assertEqual(hxs.extract(),
|
||||
u'<html><body><root>lala</root></body></html>')
|
||||
|
||||
xxs = XmlXPathSelector(text='<root>lala</root>')
|
||||
self.assertEqual(xxs.extract(),
|
||||
u'<root>lala</root>')
|
||||
|
||||
xxs = XmlXPathSelector(text='<root>lala</root>')
|
||||
self.assertEqual(xxs.select('.').extract(),
|
||||
[u'<root>lala</root>'])
|
||||
|
||||
|
||||
@libxml2debug
|
||||
def test_selector_namespaces_simple(self):
|
||||
body = """
|
||||
<test xmlns:somens="http://scrapy.org">
|
||||
<somens:a id="foo">take this</a>
|
||||
<a id="bar">found</a>
|
||||
</test>
|
||||
"""
|
||||
|
||||
response = XmlResponse(url="http://example.com", body=body)
|
||||
x = XmlXPathSelector(response)
|
||||
|
||||
x.register_namespace("somens", "http://scrapy.org")
|
||||
self.assertEqual(x.select("//somens:a/text()").extract(),
|
||||
[u'take this'])
|
||||
|
||||
|
||||
@libxml2debug
|
||||
def test_selector_namespaces_multiple(self):
|
||||
body = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<BrowseNode xmlns="http://webservices.amazon.com/AWSECommerceService/2005-10-05"
|
||||
xmlns:b="http://somens.com"
|
||||
xmlns:p="http://www.scrapy.org/product" >
|
||||
<b:Operation>hello</b:Operation>
|
||||
<TestTag b:att="value"><Other>value</Other></TestTag>
|
||||
<p:SecondTestTag><material>iron</material><price>90</price><p:name>Dried Rose</p:name></p:SecondTestTag>
|
||||
</BrowseNode>
|
||||
"""
|
||||
response = XmlResponse(url="http://example.com", body=body)
|
||||
x = XmlXPathSelector(response)
|
||||
|
||||
x.register_namespace("xmlns", "http://webservices.amazon.com/AWSECommerceService/2005-10-05")
|
||||
x.register_namespace("p", "http://www.scrapy.org/product")
|
||||
x.register_namespace("b", "http://somens.com")
|
||||
self.assertEqual(len(x.select("//xmlns:TestTag")), 1)
|
||||
self.assertEqual(x.select("//b:Operation/text()").extract()[0], 'hello')
|
||||
self.assertEqual(x.select("//xmlns:TestTag/@b:att").extract()[0], 'value')
|
||||
self.assertEqual(x.select("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90')
|
||||
self.assertEqual(x.select("//p:SecondTestTag").select("./xmlns:price/text()")[0].extract(), '90')
|
||||
self.assertEqual(x.select("//p:SecondTestTag/xmlns:material/text()").extract()[0], 'iron')
|
||||
|
||||
@libxml2debug
|
||||
def test_selector_invalid_xpath(self):
|
||||
response = XmlResponse(url="http://example.com", body="<html></html>")
|
||||
x = HtmlXPathSelector(response)
|
||||
xpath = "//test[@foo='bar]"
|
||||
try:
|
||||
x.select(xpath)
|
||||
except ValueError, e:
|
||||
assert xpath in str(e), "Exception message does not contain invalid xpath"
|
||||
except Exception:
|
||||
raise AssertionError("A invalid XPath does not raise ValueError")
|
||||
else:
|
||||
raise AssertionError("A invalid XPath does not raise an exception")
|
||||
|
||||
@libxml2debug
|
||||
def test_http_header_encoding_precedence(self):
|
||||
# u'\xa3' = pound symbol in unicode
|
||||
# u'\xc2\xa3' = pound symbol in utf-8
|
||||
# u'\xa3' = pound symbol in latin-1 (iso-8859-1)
|
||||
|
||||
meta = u'<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">'
|
||||
head = u'<head>' + meta + u'</head>'
|
||||
body_content = u'<span id="blank">\xa3</span>'
|
||||
body = u'<body>' + body_content + u'</body>'
|
||||
html = u'<html>' + head + body + u'</html>'
|
||||
encoding = 'utf-8'
|
||||
html_utf8 = html.encode(encoding)
|
||||
|
||||
headers = {'Content-Type': ['text/html; charset=utf-8']}
|
||||
response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
|
||||
x = HtmlXPathSelector(response)
|
||||
self.assertEquals(x.select("//span[@id='blank']/text()").extract(),
|
||||
[u'\xa3'])
|
||||
|
||||
@libxml2debug
|
||||
def test_null_bytes(self):
|
||||
hxs = HtmlXPathSelector(text='<root>la\x00la</root>')
|
||||
self.assertEqual(hxs.extract(),
|
||||
u'<html><body><root>la la</root></body></html>')
|
||||
|
||||
xxs = XmlXPathSelector(text='<root>la\x00la</root>')
|
||||
self.assertEqual(xxs.extract(),
|
||||
u'<root>la</root>')
|
||||
|
||||
@libxml2debug
|
||||
def test_unquote(self):
|
||||
xmldoc = '\n'.join((
|
||||
'<root>',
|
||||
' lala',
|
||||
' <node>',
|
||||
' blabla&more<!--comment-->a<b>test</b>oh',
|
||||
' <![CDATA[lalalal&ppppp<b>PPPP</b>ppp&la]]>',
|
||||
' </node>',
|
||||
' pff',
|
||||
'</root>'))
|
||||
xxs = XmlXPathSelector(text=xmldoc)
|
||||
|
||||
# this tests were commented out because they make no sense (pablo)
|
||||
#self.assertEqual(xxs.extract_unquoted(), u'')
|
||||
#self.assertEqual(xxs.select('/root').extract_unquoted(), [u''])
|
||||
#self.assertEqual(xxs.select('//*').extract_unquoted(), [u'', u'', u''])
|
||||
|
||||
self.assertEqual(xxs.select('/root/text()').extract_unquoted(), [
|
||||
u'lala',
|
||||
u'pff'])
|
||||
|
||||
self.assertEqual(xxs.select('//text()').extract_unquoted(), [
|
||||
u'lala',
|
||||
u'blabla&more',
|
||||
u'a',
|
||||
u'test',
|
||||
u'oh\n lalalal&ppppp<b>PPPP</b>ppp&la',
|
||||
u'pff'])
|
||||
|
||||
@libxml2debug
|
||||
def test_empty_bodies(self):
|
||||
r1 = TextResponse('http://www.example.com', body='')
|
||||
hxs = HtmlXPathSelector(r1) # shouldn't raise error
|
||||
xxs = XmlXPathSelector(r1) # shouldn't raise error
|
||||
|
||||
@libxml2debug
|
||||
def test_weakref_slots(self):
|
||||
"""Check that classes are using slots and are weak-referenceable"""
|
||||
for cls in [XPathSelector, HtmlXPathSelector, XmlXPathSelector]:
|
||||
x = cls()
|
||||
weakref.ref(x)
|
||||
assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
|
||||
x.__class__.__name__
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
import os
|
||||
import libxml2
|
||||
from twisted.trial import unittest
|
||||
|
||||
from scrapy.utils.iterators import csviter, xmliter
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ This module contains some assorted functions used in tests
|
|||
|
||||
import os
|
||||
|
||||
import libxml2
|
||||
from twisted.trial.unittest import SkipTest
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
|
|
@ -19,6 +18,10 @@ def libxml2debug(testfunction):
|
|||
LIBXML2_DEBUGLEAKS is set.
|
||||
|
||||
"""
|
||||
try:
|
||||
import libxml2
|
||||
except ImportError:
|
||||
return testfunction
|
||||
def newfunc(*args, **kwargs):
|
||||
libxml2.debugMemory(1)
|
||||
testfunction(*args, **kwargs)
|
||||
|
|
|
|||
Loading…
Reference in New Issue