From 4ee3626763434cd6943dbf6d387be0fb718f41ef Mon Sep 17 00:00:00 2001 From: elpolilla Date: Fri, 26 Dec 2008 11:51:09 +0000 Subject: [PATCH] Modified XMLFeedSpider in order to support parsing with HtmlXPathSelector --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40548 --- scrapy/trunk/scrapy/contrib/spiders/feed.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/spiders/feed.py b/scrapy/trunk/scrapy/contrib/spiders/feed.py index 5e2a55c33..9da11c267 100644 --- a/scrapy/trunk/scrapy/contrib/spiders/feed.py +++ b/scrapy/trunk/scrapy/contrib/spiders/feed.py @@ -3,18 +3,20 @@ from scrapy.spider import BaseSpider from scrapy.item import ScrapedItem from scrapy.http import Request from scrapy.utils.iterators import xmliter, csviter -from scrapy.xpath.selector import XmlXPathSelector -from scrapy.core.exceptions import UsageError, NotConfigured +from scrapy.xpath.selector import XmlXPathSelector, HtmlXPathSelector +from scrapy.core.exceptions import UsageError, NotConfigured, NotSupported class XMLFeedSpider(BaseSpider): """ This class intends to be the base class for spiders that scrape from XML feeds. - You can choose whether to parse the file using the iternodes tool, - or not using it (which just splits the tags using xpath) + You can choose whether to parse the file using the 'iternodes' iterator, + an 'xml' selector, or an 'html' selector. + In most cases, it's convenient to use iternodes, since it's a faster and + cleaner. """ - iternodes = True + iterator = 'iternodes' itertag = 'item' def process_results(self, results, response): @@ -46,10 +48,14 @@ class XMLFeedSpider(BaseSpider): raise NotConfigured('You must define parse_item method in order to scrape this XML feed') response = self.adapt_response(response) - if self.iternodes: + if self.iterator == 'iternodes': nodes = xmliter(response, self.itertag) - else: + elif self.iterator == 'xml': nodes = XmlXPathSelector(response).x('//%s' % self.itertag) + elif self.iterator == 'html': + nodes = HtmlXPathSelector(response).x('//%s' % self.itertag) + else: + raise NotSupported('Unsupported node iterator') return self.parse_nodes(response, nodes)