Improve and simplify tests.

tests/test_addons.py

@@ -3,6 +3,8 @@ import unittest
 from typing import Any, Dict
 from unittest.mock import patch
 
+from twisted.internet.defer import inlineCallbacks
+
 from scrapy import Spider
 from scrapy.crawler import Crawler, CrawlerRunner
 from scrapy.exceptions import NotConfigured
@@ -177,3 +179,22 @@ class AddonManagerTest(unittest.TestCase):
                     {"addons": [addon]},
                     extra={"crawler": crawler},
                 )
+
+    @inlineCallbacks
+    def test_enable_addon_in_spider(self):
+        class MySpider(Spider):
+            name = "myspider"
+
+            @classmethod
+            def from_crawler(cls, crawler, *args, **kwargs):
+                spider = super().from_crawler(crawler, *args, **kwargs)
+                addon_config = {"KEY": "addon"}
+                addon_cls = get_addon_cls(addon_config)
+                spider.settings.set("ADDONS", {addon_cls: 1}, priority="spider")
+                return spider
+
+        runner = CrawlerRunner({"KEY": "project"})
+        crawler = runner.create_crawler(MySpider)
+        self.assertEqual(crawler.settings.get("KEY"), "project")
+        yield crawler.crawl()
+        self.assertEqual(crawler.settings.get("KEY"), "addon")

tests/test_crawl.py

@@ -1,7 +1,6 @@
 import json
 import logging
 import unittest
-import warnings
 from ipaddress import IPv4Address
 from socket import gethostbyname
 from urllib.parse import urlparse
@@ -14,14 +13,11 @@ from twisted.python.failure import Failure
 from twisted.trial.unittest import TestCase
 
 from scrapy import signals
-from scrapy.crawler import Crawler, CrawlerRunner
-from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
-from scrapy.extensions.throttle import AutoThrottle
+from scrapy.crawler import CrawlerRunner
+from scrapy.exceptions import StopDownload
 from scrapy.http import Request
 from scrapy.http.response import Response
-from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
-from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
 from tests import NON_EXISTING_RESOLVABLE
 from tests.mockserver import MockServer
@@ -414,38 +410,6 @@ with multiples lines
         self._assert_retried(log)
         self.assertIn("Got response 200", str(log))
 
-    @defer.inlineCallbacks
-    def test_populate_spidercls_settings(self):
-        spider_settings = {
-            "TEST1": "spider",
-            "TEST2": "spider",
-            "AUTOTHROTTLE_ENABLED": True,
-        }
-        project_settings = {"TEST1": "project", "TEST3": "project"}
-
-        class CustomSettingsSpider(DefaultSpider):
-            custom_settings = spider_settings
-
-            def parse(self, response):
-                return
-
-        settings = Settings()
-        settings.setdict(project_settings, priority="project")
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore", ScrapyDeprecationWarning)
-            crawler = Crawler(CustomSettingsSpider, settings)
-        yield crawler.crawl()
-
-        self.assertEqual(crawler.settings.get("TEST1"), "spider")
-        self.assertEqual(crawler.settings.get("TEST2"), "spider")
-        self.assertEqual(crawler.settings.get("TEST3"), "project")
-
-        enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
-        self.assertIn(AutoThrottle, enabled_exts)
-
-        self.assertFalse(settings.frozen)
-        self.assertTrue(crawler.settings.frozen)
-
 
 class CrawlSpiderTestCase(TestCase):
     def setUp(self):

tests/test_crawler.py

@@ -17,6 +17,7 @@ import scrapy
 from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extensions import telnet
+from scrapy.extensions.throttle import AutoThrottle
 from scrapy.settings import Settings, default_settings
 from scrapy.spiderloader import SpiderLoader
 from scrapy.utils.log import configure_logging, get_scrapy_root_handler
@@ -32,6 +33,25 @@ class BaseCrawlerTest(unittest.TestCase):
 
 
 class CrawlerTestCase(BaseCrawlerTest):
+    def test_populate_spidercls_settings(self):
+        spider_settings = {"TEST1": "spider", "TEST2": "spider"}
+        project_settings = {"TEST1": "project", "TEST3": "project"}
+
+        class CustomSettingsSpider(DefaultSpider):
+            custom_settings = spider_settings
+
+        settings = Settings()
+        settings.setdict(project_settings, priority="project")
+        crawler = Crawler(CustomSettingsSpider, settings)
+        crawler._load_settings()
+
+        self.assertEqual(crawler.settings.get("TEST1"), "spider")
+        self.assertEqual(crawler.settings.get("TEST2"), "spider")
+        self.assertEqual(crawler.settings.get("TEST3"), "project")
+
+        self.assertFalse(settings.frozen)
+        self.assertTrue(crawler.settings.frozen)
+
     def test_crawler_accepts_dict(self):
         crawler = get_crawler(DefaultSpider, {"foo": "bar"})
         self.assertEqual(crawler.settings["foo"], "bar")
@@ -58,6 +78,17 @@ class CrawlerTestCase(BaseCrawlerTest):
             yield crawler.crawl()
 
 
+class SpiderSettingsTestCase(unittest.TestCase):
+    def test_spider_custom_settings(self):
+        class MySpider(scrapy.Spider):
+            name = "spider"
+            custom_settings = {"AUTOTHROTTLE_ENABLED": True}
+
+        crawler = get_crawler(MySpider)
+        enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
+        self.assertIn(AutoThrottle, enabled_exts)
+
+
 class CrawlerLoggingTestCase(unittest.TestCase):
     def test_no_root_handler_installed(self):
         handler = get_scrapy_root_handler()