fix make_requests_from_url deprcation implementation, add tests

scrapy/spiders/__init__.py

@@ -66,11 +66,14 @@ class Spider(object_ref):
         crawler.signals.connect(self.close, signals.spider_closed)
 
     def start_requests(self):
-        if self.make_requests_from_url is not Spider.make_requests_from_url:
+        cls = self.__class__
+        if cls.make_requests_from_url is not Spider.make_requests_from_url:
             warnings.warn(
-                "Spider.make_requests_from_url method is deprecated; "
-                "it won't be called in future Scrapy releases. "
-                "Please override start_requests method instead."
+                "Spider.make_requests_from_url method is deprecated; it "
+                "won't be called in future Scrapy releases. Please "
+                "override Spider.start_requests method instead (see %s.%s)." % (
+                    cls.__module__, cls.__name__
+                ),
             )
             for url in self.start_urls:
                 yield self.make_requests_from_url(url)

tests/test_spider.py

@@ -345,7 +345,7 @@ Sitemap: /sitemap-relative-url.xml
                           'http://www.example.com/sitemap-relative-url.xml'])
 
 
-class BaseSpiderDeprecationTest(unittest.TestCase):
+class DeprecationTest(unittest.TestCase):
 
     def test_basespider_is_deprecated(self):
         with warnings.catch_warnings(record=True) as w:
@@ -399,6 +399,29 @@ class BaseSpiderDeprecationTest(unittest.TestCase):
         assert isinstance(CrawlSpider(name='foo'), Spider)
         assert isinstance(CrawlSpider(name='foo'), BaseSpider)
 
+    def test_make_requests_from_url_deprecated(self):
+        class MySpider4(Spider):
+            name = 'spider1'
+            start_urls = ['http://example.com']
 
-if __name__ == '__main__':
-    unittest.main()
+        class MySpider5(Spider):
+            name = 'spider2'
+            start_urls = ['http://example.com']
+
+            def make_requests_from_url(self, url):
+                return Request(url + "/foo", dont_filter=True)
+
+        with warnings.catch_warnings(record=True) as w:
+            # spider without overridden make_requests_from_url method
+            # doesn't issue a warning
+            spider1 = MySpider4()
+            self.assertEqual(len(list(spider1.start_requests())), 1)
+            self.assertEqual(len(w), 0)
+
+            # spider with overridden make_requests_from_url issues a warning,
+            # but the method still works
+            spider2 = MySpider5()
+            requests = list(spider2.start_requests())
+            self.assertEqual(len(requests), 1)
+            self.assertEqual(requests[0].url, 'http://example.com/foo')
+            self.assertEqual(len(w), 1)