diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 99114d3bb..ad6649134 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -145,7 +145,7 @@ class Scraper(object): def call_spider(self, result, request, spider): result.request = request dfd = defer_result(result) - callback = request.callback or spider.parse + callback = request.callback or spider._parse warn_on_generator_with_return_value(spider, callback) warn_on_generator_with_return_value(spider, request.errback) dfd.addCallbacks(callback=callback, diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 9429f6cb2..1011eb870 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -80,6 +80,9 @@ class Spider(object_ref): """ This method is deprecated. """ return Request(url, dont_filter=True) + def _parse(self, response): + return self.parse(response) + def parse(self, response): raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__)) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index a2c364c0e..e28d17dcd 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -74,7 +74,7 @@ class CrawlSpider(Spider): super(CrawlSpider, self).__init__(*a, **kw) self._compile_rules() - def parse(self, response): + def _parse(self, response): return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True) def parse_start_url(self, response): diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index c566f0236..11bd17db4 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -61,7 +61,7 @@ class XMLFeedSpider(Spider): for result_item in self.process_results(response, ret): yield result_item - def parse(self, response): + def _parse(self, response): if not hasattr(self, 'parse_node'): raise NotConfigured('You must define parse_node method in order to scrape this XML feed') @@ -128,7 +128,7 @@ class CSVFeedSpider(Spider): for result_item in self.process_results(response, ret): yield result_item - def parse(self, response): + def _parse(self, response): if not hasattr(self, 'parse_row'): raise NotConfigured('You must define parse_row method in order to scrape this CSV feed') response = self.adapt_response(response) diff --git a/tests/test_spider.py b/tests/test_spider.py index 317a27076..6fbec7e58 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -142,7 +142,7 @@ class XMLFeedSpiderTest(SpiderTest): for iterator in ('iternodes', 'xml'): spider = _XMLSpider('example', iterator=iterator) - output = list(spider.parse(response)) + output = list(spider._parse(response)) self.assertEqual(len(output), 2, iterator) self.assertEqual(output, [ {'loc': [u'http://www.example.com/Special-Offers.html'],