From be71f98e92688c759d3af48101617229dcdfe05f Mon Sep 17 00:00:00 2001 From: kirankoduru Date: Sat, 29 Jul 2017 20:51:54 -0400 Subject: [PATCH 1/4] Explicit message for scrapy parse callback The scrapy parse method raises a NotImplementedError when not defined, but for new comers it can be hard to debug what might be going wrong. Adding an explicit message for NotImplementedError will help new users. --- scrapy/spiders/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 30cb7590a..c6b92f8eb 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -87,7 +87,7 @@ class Spider(object_ref): return Request(url, dont_filter=True) def parse(self, response): - raise NotImplementedError + raise NotImplementedError('Spider.parse callback is not defined') @classmethod def update_settings(cls, settings): From 7adab61a7a5f88c78311cc44a468c7b8d0a4c954 Mon Sep 17 00:00:00 2001 From: kirankoduru Date: Tue, 1 Aug 2017 22:42:26 -0400 Subject: [PATCH 2/4] Added test for NotImplemented Spider.parse method --- tests/test_spider.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_spider.py b/tests/test_spider.py index e55f0fa6d..6c845d826 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -429,3 +429,17 @@ class DeprecationTest(unittest.TestCase): self.assertEqual(len(requests), 1) self.assertEqual(requests[0].url, 'http://example.com/foo') self.assertEqual(len(w), 1) + + +class NoParseMethodSpiderTest(unittest.TestCase): + + spider_class = Spider + + def test_undefined_parse_method(self): + spider = self.spider_class('example.com') + text = 'Random text response' + resp = TextResponse(url="http://www.example.com/random_url", body=text) + + exc_msg = 'Spider.parse callback is not defined' + with self.assertRaisesRegexp(NotImplementedError, exc_msg): + spider.parse(resp) From 12409a0cf6c37ff5c19588bb064690549798bb37 Mon Sep 17 00:00:00 2001 From: Kiran Koduru Date: Wed, 2 Aug 2017 08:32:38 -0400 Subject: [PATCH 3/4] Fix broken encoding on text for py 3 --- tests/test_spider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index 6c845d826..6a52b3ea7 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -437,7 +437,7 @@ class NoParseMethodSpiderTest(unittest.TestCase): def test_undefined_parse_method(self): spider = self.spider_class('example.com') - text = 'Random text response' + text = b'Random text' resp = TextResponse(url="http://www.example.com/random_url", body=text) exc_msg = 'Spider.parse callback is not defined' From 2960c9b5683dceb149823e7f927d8c86ee83deb8 Mon Sep 17 00:00:00 2001 From: Kiran Koduru Date: Sat, 5 Aug 2017 16:29:41 -0400 Subject: [PATCH 4/4] Use self.__class__.__name__ instead of showing generic Spider class name --- scrapy/spiders/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index c6b92f8eb..e9c131e3b 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -87,7 +87,7 @@ class Spider(object_ref): return Request(url, dont_filter=True) def parse(self, response): - raise NotImplementedError('Spider.parse callback is not defined') + raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__)) @classmethod def update_settings(cls, settings):