From 74f706b3562a714b59e5c55bb53d86fd080ce88c Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 24 Aug 2009 10:54:34 -0300 Subject: [PATCH] renamed "parse_item" method of XMLFeedSpider to "parse_node", keeping backwards compatibility --- docs/topics/spiders.rst | 23 ++++++++--------------- scrapy/contrib/spiders/feed.py | 10 ++++++---- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 2e7f7dd48..0e7f1e510 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -256,10 +256,9 @@ Let's now take a look at an example CrawlSpider with rules:: This spider would start crawling example.com's home page, collecting category -links, and item links, parsing the latter with the -:meth:`XMLFeedSpider.parse_item` method. For each item response, some data will -be extracted from the HTML using XPath, and a :class:`~scrapy.item.Item` will -be filled with it. +links, and item links, parsing the latter with the ``parse_item`` method. For +each item response, some data will be extracted from the HTML using XPath, and +a :class:`~scrapy.item.Item` will be filled with it. XMLFeedSpider ------------- @@ -327,7 +326,7 @@ XMLFeedSpider the response body before parsing it. This method receives a response and returns response (it could be the same or another one). - .. method:: parse_item(response, selector) + .. method:: parse_node(response, selector) This method is called for the nodes matching the provided tag name (``itertag``). Receives the response and an XPathSelector for each node. @@ -336,8 +335,6 @@ XMLFeedSpider :class:`~scrapy.http.Request` object, or an iterable containing any of them. - .. warning:: This method will soon change its name to ``parse_node`` - .. method:: process_results(response, results) This method is called for each result (item or request) returned by the @@ -362,7 +359,7 @@ These spiders are pretty easy to use, let's have at one example:: iterator = 'iternodes' # This is actually unnecesary, since it's the default value itertag = 'item' - def parse_item(self, response, node): + def parse_node(self, response, node): log.msg('Hi, this is a <%s> node!: %s' % (self.itertag, ''.join(node.extract()))) item = Item() @@ -382,10 +379,9 @@ CSVFeedSpider .. class:: CSVFeedSpider - .. warning:: The API of the CSVFeedSpider is not yet stable. Use with caution. - - This spider is very similar to the XMLFeedSpider, although it iterates through - rows, instead of nodes. It also has other two different attributes: + This spider is very similar to the XMLFeedSpider, except that it iterates + over rows, instead of nodes. The method that gets called in each iteration + is :meth:`parse_row`. .. attribute:: CSVFeedSpider.delimiter @@ -397,9 +393,6 @@ CSVFeedSpider A list of the rows contained in the file CSV feed which will be used for extracting fields from it. - In this spider, the method that gets called in each row iteration ``parse_row`` - instead of ``parse_item`` (like in :class:`XMLFeedSpider`). - .. method:: CSVFeedSpider.parse_row(response, row) Receives a response and a dict (representing each row) with a key for each diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/contrib/spiders/feed.py index b6c05d7bf..a80108c5c 100644 --- a/scrapy/contrib/spiders/feed.py +++ b/scrapy/contrib/spiders/feed.py @@ -43,8 +43,10 @@ class XMLFeedSpider(InitSpider): """ return response - def parse_item(self, response, selector): + def parse_node(self, response, selector): """This method must be overriden with your custom spider functionality""" + if hasattr(self, 'parse_item'): # backward compatibility + return self.parse_item(response, selector) raise NotImplemented def parse_nodes(self, response, nodes): @@ -56,7 +58,7 @@ class XMLFeedSpider(InitSpider): """ for selector in nodes: - ret = self.parse_item(response, selector) + ret = self.parse_node(response, selector) if isinstance(ret, (BaseItem, Request)): ret = [ret] if not isinstance(ret, (list, tuple)): @@ -65,8 +67,8 @@ class XMLFeedSpider(InitSpider): yield result_item def parse(self, response): - if not hasattr(self, 'parse_item'): - raise NotConfigured('You must define parse_item method in order to scrape this XML feed') + if not hasattr(self, 'parse_node'): + raise NotConfigured('You must define parse_node method in order to scrape this XML feed') response = self.adapt_response(response) if self.iterator == 'iternodes':