From 10cdc70fc4c2ce441fb3c0a7a9ee9621c0d2eb63 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 8 Aug 2009 06:03:46 -0300 Subject: [PATCH] some cleanup to scrapy.xpath module --HG-- rename : scrapy/xpath/constructors.py => scrapy/xpath/factories.py --- scrapy/xpath/document.py | 6 ++-- scrapy/xpath/extension.py | 8 ++--- .../xpath/{constructors.py => factories.py} | 0 scrapy/xpath/selector.py | 32 ++++++++++--------- 4 files changed, 24 insertions(+), 22 deletions(-) rename scrapy/xpath/{constructors.py => factories.py} (100%) diff --git a/scrapy/xpath/document.py b/scrapy/xpath/document.py index 291a1f99b..4e25bbc20 100644 --- a/scrapy/xpath/document.py +++ b/scrapy/xpath/document.py @@ -3,12 +3,12 @@ This module contains a simple class (Libxml2Document) to wrap libxml2 documents (xmlDoc) for proper garbage collection. """ -from scrapy.xpath.constructors import xmlDoc_from_html +from scrapy.xpath.factories import xmlDoc_from_html class Libxml2Document(object): - def __init__(self, response, constructor=xmlDoc_from_html): - self.xmlDoc = constructor(response) + def __init__(self, response, factory=xmlDoc_from_html): + self.xmlDoc = factory(response) self.xpathContext = self.xmlDoc.xpathNewContext() def __del__(self): diff --git a/scrapy/xpath/extension.py b/scrapy/xpath/extension.py index fc07689c2..30674657a 100644 --- a/scrapy/xpath/extension.py +++ b/scrapy/xpath/extension.py @@ -5,16 +5,16 @@ The ResponseLibxml2 extension causes the Response objects to grow a new method from scrapy.http import Response from scrapy.xpath.document import Libxml2Document -from scrapy.xpath.constructors import xmlDoc_from_html +from scrapy.xpath.factories import xmlDoc_from_html class ResponseLibxml2(object): def __init__(self): setattr(Response, 'getlibxml2doc', getlibxml2doc) -def getlibxml2doc(response, constructor=xmlDoc_from_html): - cachekey = 'lx2doc_%s' % constructor.__name__ +def getlibxml2doc(response, factory=xmlDoc_from_html): + cachekey = 'lx2doc_%s' % factory.__name__ if cachekey not in response.cache: - lx2doc = Libxml2Document(response, constructor=constructor) + lx2doc = Libxml2Document(response, factory=factory) response.cache[cachekey] = lx2doc return response.cache[cachekey] diff --git a/scrapy/xpath/constructors.py b/scrapy/xpath/factories.py similarity index 100% rename from scrapy/xpath/constructors.py rename to scrapy/xpath/factories.py diff --git a/scrapy/xpath/selector.py b/scrapy/xpath/selector.py index f1882a37c..4fb38d54a 100644 --- a/scrapy/xpath/selector.py +++ b/scrapy/xpath/selector.py @@ -8,7 +8,7 @@ import libxml2 from scrapy.http import TextResponse from scrapy.xpath.extension import Libxml2Document -from scrapy.xpath.constructors import xmlDoc_from_html, xmlDoc_from_xml +from scrapy.xpath.factories import xmlDoc_from_html, xmlDoc_from_xml from scrapy.utils.python import flatten, unicode_to_str from scrapy.utils.misc import extract_regex @@ -23,19 +23,23 @@ class XPathSelector(object): - HtmlXPathSelector (for HTML content) """ - def __init__(self, response=None, text=None, node=None, parent=None, expr=None, constructor=xmlDoc_from_html): + xmlDoc_factory = staticmethod(xmlDoc_from_html) + + def __init__(self, response=None, text=None, node=None, parent=None, expr=None): if parent: self.doc = parent.doc self.xmlNode = node elif response: try: - self.doc = response.getlibxml2doc(constructor=constructor) # try with cached version first + # try with cached version first + self.doc = response.getlibxml2doc(factory=self.xmlDoc_factory) except AttributeError: - self.doc = Libxml2Document(response, constructor=constructor) + self.doc = Libxml2Document(response, factory=self.xmlDoc_factory) self.xmlNode = self.doc.xmlDoc elif text: - response = TextResponse(url=None, body=unicode_to_str(text), encoding='utf-8') - self.doc = Libxml2Document(response, constructor=constructor) + response = TextResponse(url=None, body=unicode_to_str(text), \ + encoding='utf-8') + self.doc = Libxml2Document(response, factory=self.xmlDoc_factory) self.xmlNode = self.doc.xmlDoc self.expr = expr self.response = response @@ -51,10 +55,11 @@ class XPathSelector(object): raise ValueError("Invalid XPath: %s" % xpath) cls = type(self) if hasattr(xpath_result, '__iter__'): - return XPathSelectorList([cls(node=node, parent=self, expr=xpath, response=self.response) - for node in xpath_result]) + return XPathSelectorList([cls(node=node, parent=self, expr=xpath, \ + response=self.response) for node in xpath_result]) else: - return XPathSelectorList([cls(node=xpath_result, parent=self, expr=xpath, response=self.response)]) + return XPathSelectorList([cls(node=xpath_result, parent=self, \ + expr=xpath, response=self.response)]) else: return XPathSelectorList([]) __call__ = x @@ -132,12 +137,9 @@ class XPathSelectorList(list): class XmlXPathSelector(XPathSelector): """XPathSelector for XML content""" - def __init__(self, *args, **kwargs): - kwargs['constructor'] = xmlDoc_from_xml - XPathSelector.__init__(self, *args, **kwargs) + xmlDoc_factory = staticmethod(xmlDoc_from_xml) + class HtmlXPathSelector(XPathSelector): """XPathSelector for HTML content""" - def __init__(self, *args, **kwargs): - kwargs['constructor'] = xmlDoc_from_html - XPathSelector.__init__(self, *args, **kwargs) + xmlDoc_factory = staticmethod(xmlDoc_from_html)