From 1718e450ef9549a4fc71b01dba1e6faf7a63238a Mon Sep 17 00:00:00 2001 From: Mabel Villalba Date: Mon, 18 Nov 2019 12:33:55 +0100 Subject: [PATCH] [start_url] Fixes #4133: Raise AttributeError error when empty 'start_urls' and 'start_url' found. Added test. --- scrapy/spiders/__init__.py | 5 +++++ tests/test_spider.py | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index e9c131e3b..5a35fcdb6 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -68,6 +68,11 @@ class Spider(object_ref): def start_requests(self): cls = self.__class__ + if not self.start_urls and hasattr(self, 'start_url'): + raise AttributeError( + "Crawling could not start: 'start_urls' not found " + "or empty (but found 'start_url' attribute instead, " + "did you miss an 's'?)") if method_is_overridden(cls, Spider, 'make_requests_from_url'): warnings.warn( "Spider.make_requests_from_url method is deprecated; it " diff --git a/tests/test_spider.py b/tests/test_spider.py index 83fb68c2f..0a6640cec 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -391,6 +391,13 @@ class CrawlSpiderTest(SpiderTest): self.assertTrue(hasattr(spider, '_follow_links')) self.assertFalse(spider._follow_links) + def test_start_url(self): + spider = self.spider_class("example.com") + spider.start_url = 'https://www.example.com' + + with self.assertRaisesRegex(AttributeError, + r'^Crawling could not start.*$'): + list(spider.start_requests()) class SitemapSpiderTest(SpiderTest):