diff --git a/tests/test_addons.py b/tests/test_addons.py index aa1b760c2..8375a6495 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -3,6 +3,8 @@ import unittest from typing import Any, Dict from unittest.mock import patch +from twisted.internet.defer import inlineCallbacks + from scrapy import Spider from scrapy.crawler import Crawler, CrawlerRunner from scrapy.exceptions import NotConfigured @@ -177,3 +179,22 @@ class AddonManagerTest(unittest.TestCase): {"addons": [addon]}, extra={"crawler": crawler}, ) + + @inlineCallbacks + def test_enable_addon_in_spider(self): + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + addon_config = {"KEY": "addon"} + addon_cls = get_addon_cls(addon_config) + spider.settings.set("ADDONS", {addon_cls: 1}, priority="spider") + return spider + + runner = CrawlerRunner({"KEY": "project"}) + crawler = runner.create_crawler(MySpider) + self.assertEqual(crawler.settings.get("KEY"), "project") + yield crawler.crawl() + self.assertEqual(crawler.settings.get("KEY"), "addon") diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 496ab77a5..96d43b2b9 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,7 +1,6 @@ import json import logging import unittest -import warnings from ipaddress import IPv4Address from socket import gethostbyname from urllib.parse import urlparse @@ -14,14 +13,11 @@ from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from scrapy import signals -from scrapy.crawler import Crawler, CrawlerRunner -from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload -from scrapy.extensions.throttle import AutoThrottle +from scrapy.crawler import CrawlerRunner +from scrapy.exceptions import StopDownload from scrapy.http import Request from scrapy.http.response import Response -from scrapy.settings import Settings from scrapy.utils.python import to_unicode -from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests import NON_EXISTING_RESOLVABLE from tests.mockserver import MockServer @@ -414,38 +410,6 @@ with multiples lines self._assert_retried(log) self.assertIn("Got response 200", str(log)) - @defer.inlineCallbacks - def test_populate_spidercls_settings(self): - spider_settings = { - "TEST1": "spider", - "TEST2": "spider", - "AUTOTHROTTLE_ENABLED": True, - } - project_settings = {"TEST1": "project", "TEST3": "project"} - - class CustomSettingsSpider(DefaultSpider): - custom_settings = spider_settings - - def parse(self, response): - return - - settings = Settings() - settings.setdict(project_settings, priority="project") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - crawler = Crawler(CustomSettingsSpider, settings) - yield crawler.crawl() - - self.assertEqual(crawler.settings.get("TEST1"), "spider") - self.assertEqual(crawler.settings.get("TEST2"), "spider") - self.assertEqual(crawler.settings.get("TEST3"), "project") - - enabled_exts = [e.__class__ for e in crawler.extensions.middlewares] - self.assertIn(AutoThrottle, enabled_exts) - - self.assertFalse(settings.frozen) - self.assertTrue(crawler.settings.frozen) - class CrawlSpiderTestCase(TestCase): def setUp(self): diff --git a/tests/test_crawler.py b/tests/test_crawler.py index bfae6c690..08149725c 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -17,6 +17,7 @@ import scrapy from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions import telnet +from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler @@ -32,6 +33,25 @@ class BaseCrawlerTest(unittest.TestCase): class CrawlerTestCase(BaseCrawlerTest): + def test_populate_spidercls_settings(self): + spider_settings = {"TEST1": "spider", "TEST2": "spider"} + project_settings = {"TEST1": "project", "TEST3": "project"} + + class CustomSettingsSpider(DefaultSpider): + custom_settings = spider_settings + + settings = Settings() + settings.setdict(project_settings, priority="project") + crawler = Crawler(CustomSettingsSpider, settings) + crawler._load_settings() + + self.assertEqual(crawler.settings.get("TEST1"), "spider") + self.assertEqual(crawler.settings.get("TEST2"), "spider") + self.assertEqual(crawler.settings.get("TEST3"), "project") + + self.assertFalse(settings.frozen) + self.assertTrue(crawler.settings.frozen) + def test_crawler_accepts_dict(self): crawler = get_crawler(DefaultSpider, {"foo": "bar"}) self.assertEqual(crawler.settings["foo"], "bar") @@ -58,6 +78,17 @@ class CrawlerTestCase(BaseCrawlerTest): yield crawler.crawl() +class SpiderSettingsTestCase(unittest.TestCase): + def test_spider_custom_settings(self): + class MySpider(scrapy.Spider): + name = "spider" + custom_settings = {"AUTOTHROTTLE_ENABLED": True} + + crawler = get_crawler(MySpider) + enabled_exts = [e.__class__ for e in crawler.extensions.middlewares] + self.assertIn(AutoThrottle, enabled_exts) + + class CrawlerLoggingTestCase(unittest.TestCase): def test_no_root_handler_installed(self): handler = get_scrapy_root_handler()