RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

remove deprecated module lxmldocument

This commit is contained in:
Elias Dorneles 2015-08-06 22:35:43 -03:00
parent 6287fc3109
commit 94c3a345b7
3 changed files with 0 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
scrapy/selector/lxmldocument.py
View File

@ -1,31 +0,0 @@
"""
This module contains a simple class (LxmlDocument) which provides cache and
garbage collection to lxml element tree documents.
"""
import weakref
from lxml import etree
from scrapy.utils.trackref import object_ref
def _factory(response, parser_cls):
url = response.url
body = response.body_as_unicode().strip().encode('utf8') or '<html/>'
parser = parser_cls(recover=True, encoding='utf8')
return etree.fromstring(body, parser=parser, base_url=url)
class LxmlDocument(object_ref):
cache = weakref.WeakKeyDictionary()
__slots__ = ['__weakref__']
def __new__(cls, response, parser=etree.HTMLParser):
cache = cls.cache.setdefault(response, {})
if parser not in cache:
obj = object_ref.__new__(cls)
cache[parser] = _factory(response, parser)
return cache[parser]
def __str__(self):
return "<LxmlDocument %s>" % self.root.tag

1
tests/py3-ignores.txt
View File

@ -26,7 +26,6 @@ tests/test_pipeline_files.py
tests/test_pipeline_images.py
tests/test_proxy_connect.py
tests/test_selector_csstranslator.py
tests/test_selector_lxmldocument.py
tests/test_selector.py
tests/test_spidermiddleware_depth.py
tests/test_spidermiddleware_httperror.py

26
tests/test_selector_lxmldocument.py
View File

@ -1,26 +0,0 @@
import unittest
from scrapy.selector.lxmldocument import LxmlDocument
from scrapy.http import TextResponse, HtmlResponse
class LxmlDocumentTest(unittest.TestCase):
def test_caching(self):
r1 = HtmlResponse('http://www.example.com', body=b'<html><head></head><body></body></html>')
r2 = r1.copy()
doc1 = LxmlDocument(r1)
doc2 = LxmlDocument(r1)
doc3 = LxmlDocument(r2)
# make sure it's cached
assert doc1 is doc2
assert doc1 is not doc3
def test_null_char(self):
# make sure bodies with null char ('\x00') don't raise a TypeError exception
body = b'test problematic \x00 body'
response = TextResponse('http://example.com/catalog/product/blabla-123',
headers={'Content-Type': 'text/plain; charset=utf-8'},
body=body)
LxmlDocument(response)