diff --git a/scrapy/selector/lxmldocument.py b/scrapy/selector/lxmldocument.py
new file mode 100644
index 000000000..065ce222b
--- /dev/null
+++ b/scrapy/selector/lxmldocument.py
@@ -0,0 +1,31 @@
+"""
+This module contains a simple class (LxmlDocument) which provides cache and
+garbage collection to lxml element tree documents.
+"""
+
+import weakref
+from lxml import etree
+from scrapy.utils.trackref import object_ref
+
+
+def _factory(response, parser_cls):
+ url = response.url
+ body = response.body_as_unicode().strip().encode('utf8') or '
'
+ parser = parser_cls(recover=True, encoding='utf8')
+ return etree.fromstring(body, parser=parser, base_url=url)
+
+
+class LxmlDocument(object_ref):
+
+ cache = weakref.WeakKeyDictionary()
+ __slots__ = ['xmlDoc', 'xpathContext', '__weakref__']
+
+ def __new__(cls, response, parser=etree.HTMLParser):
+ cache = cls.cache.setdefault(response, {})
+ if parser not in cache:
+ obj = object_ref.__new__(cls)
+ cache[parser] = _factory(response, parser)
+ return cache[parser]
+
+ def __str__(self):
+ return "" % self.root.tag
diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py
index bb0a4e1ec..af6839474 100644
--- a/scrapy/selector/lxmlsel.py
+++ b/scrapy/selector/lxmlsel.py
@@ -9,42 +9,35 @@ from scrapy.utils.trackref import object_ref
from scrapy.utils.python import unicode_to_str
from scrapy.utils.decorator import deprecated
from scrapy.http import TextResponse
+from .lxmldocument import LxmlDocument
from .list import XPathSelectorList
+
__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \
- 'XPathSelectorList']
+ 'XPathSelectorList']
+
class XPathSelector(object_ref):
- __slots__ = ['response', 'text', 'expr', 'namespaces', '_root', '_xpathev', \
- '__weakref__']
+ __slots__ = ['response', 'text', 'namespaces', '_expr', '_root', '__weakref__']
_parser = etree.HTMLParser
_tostring_method = 'html'
- def __init__(self, response=None, text=None, root=None, expr=None, namespaces=None):
- if text:
- self.response = TextResponse(url='about:blank', \
+ def __init__(self, response=None, text=None, namespaces=None, _root=None, _expr=None):
+ if text is not None:
+ response = TextResponse(url='about:blank', \
body=unicode_to_str(text, 'utf-8'), encoding='utf-8')
- else:
- self.response = response
- self._root = root
- self._xpathev = None
- self.namespaces = namespaces
- self.expr = expr
+ if response is not None:
+ _root = LxmlDocument(response, self._parser)
- @property
- def root(self):
- if self._root is None:
- url = self.response.url
- body = self.response.body_as_unicode().strip().encode('utf8') or ''
- parser = self._parser(recover=True, encoding='utf8')
- self._root = etree.fromstring(body, parser=parser, base_url=url)
- assert self._root is not None, 'BUG lxml selector with None root'
- return self._root
+ self.namespaces = namespaces
+ self.response = response
+ self._root = _root
+ self._expr = _expr
def select(self, xpath):
try:
- xpathev = self.root.xpath
+ xpathev = self._root.xpath
except AttributeError:
return XPathSelectorList([])
@@ -56,7 +49,7 @@ class XPathSelector(object_ref):
if type(result) is not list:
result = [result]
- result = [self.__class__(root=x, expr=xpath, namespaces=self.namespaces)
+ result = [self.__class__(_root=x, _expr=xpath, namespaces=self.namespaces)
for x in result]
return XPathSelectorList(result)
@@ -65,15 +58,15 @@ class XPathSelector(object_ref):
def extract(self):
try:
- return etree.tostring(self.root, method=self._tostring_method, \
+ return etree.tostring(self._root, method=self._tostring_method, \
encoding=unicode)
except (AttributeError, TypeError):
- if self.root is True:
+ if self._root is True:
return u'1'
- elif self.root is False:
+ elif self._root is False:
return u'0'
else:
- return unicode(self.root)
+ return unicode(self._root)
def register_namespace(self, prefix, uri):
if self.namespaces is None:
@@ -85,7 +78,7 @@ class XPathSelector(object_ref):
def __str__(self):
data = repr(self.extract()[:40])
- return "<%s xpath=%r data=%s>" % (type(self).__name__, self.expr, data)
+ return "<%s xpath=%r data=%s>" % (type(self).__name__, self._expr, data)
__repr__ = __str__
diff --git a/scrapy/tests/test_selector_lxml.py b/scrapy/tests/test_selector_lxml.py
index 8050a2758..c8eca9679 100644
--- a/scrapy/tests/test_selector_lxml.py
+++ b/scrapy/tests/test_selector_lxml.py
@@ -2,35 +2,40 @@
Selectors tests, specific for lxml backend
"""
-from scrapy.http import TextResponse, XmlResponse
-has_lxml = True
-try:
- from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, \
- XPathSelector
-except ImportError:
- has_lxml = False
-from scrapy.utils.test import libxml2debug
+import unittest
from scrapy.tests import test_selector
+from scrapy.http import TextResponse, HtmlResponse
+from scrapy.selector.lxmldocument import LxmlDocument
+from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector
+
class LxmlXPathSelectorTestCase(test_selector.XPathSelectorTestCase):
- if has_lxml:
- xs_cls = XPathSelector
- hxs_cls = HtmlXPathSelector
- xxs_cls = XmlXPathSelector
- else:
- skip = "lxml not available"
+ xs_cls = XPathSelector
+ hxs_cls = HtmlXPathSelector
+ xxs_cls = XmlXPathSelector
- # XXX: this test was disabled because lxml behaves inconsistently when
- # handling null bytes between different 2.2.x versions, but it may be due
- # to differences in libxml2 too. it's also unclear what should be the
- # proper behaviour (pablo - 26 oct 2010)
- #@libxml2debug
- #def test_null_bytes(self):
- # hxs = HtmlXPathSelector(text='la\x00la')
- # self.assertEqual(hxs.extract(),
- # u'la')
- #
- # xxs = XmlXPathSelector(text='la\x00la')
- # self.assertEqual(xxs.extract(),
- # u'la')
+
+class Libxml2DocumentTest(unittest.TestCase):
+
+ def test_caching(self):
+ r1 = HtmlResponse('http://www.example.com', body='')
+ r2 = r1.copy()
+
+ doc1 = LxmlDocument(r1)
+ doc2 = LxmlDocument(r1)
+ doc3 = LxmlDocument(r2)
+
+ # make sure it's cached
+ assert doc1 is doc2
+ assert doc1 is not doc3
+
+ # don't leave documents in memory to avoid wrong libxml2 leaks reports
+ del doc1, doc2, doc3
+
+ def test_null_char(self):
+ # make sure bodies with null char ('\x00') don't raise a TypeError exception
+ self.body_content = 'test problematic \x00 body'
+ response = TextResponse('http://example.com/catalog/product/blabla-123',
+ headers={'Content-Type': 'text/plain; charset=utf-8'}, body=self.body_content)
+ LxmlDocument(response)