diff --git a/scrapy/addons.py b/scrapy/addons.py index 9060d4f3f..65d7a0310 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, List from scrapy.exceptions import NotConfigured from scrapy.settings import Settings from scrapy.utils.conf import build_component_list -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler @@ -32,9 +32,7 @@ class AddonManager: for clspath in build_component_list(settings["ADDONS"]): try: addoncls = load_object(clspath) - addon = create_instance( - addoncls, settings=settings, crawler=self.crawler - ) + addon = build_from_crawler(addoncls, self.crawler) addon.update_settings(settings) self.addons.append(addon) except NotConfigured as e: diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 909cc273f..dba4d8cdc 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -20,7 +20,7 @@ from scrapy.core.downloader.tls import ( openssl_methods, ) from scrapy.settings import BaseSettings -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from twisted.internet._sslverify import ClientTLSOptions @@ -165,18 +165,16 @@ def load_context_factory_from_settings(settings, crawler): context_factory_cls = load_object(settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]) # try method-aware context factory try: - context_factory = create_instance( - objcls=context_factory_cls, - settings=settings, - crawler=crawler, + context_factory = build_from_crawler( + context_factory_cls, + crawler, method=ssl_method, ) except TypeError: # use context factory defaults - context_factory = create_instance( - objcls=context_factory_cls, - settings=settings, - crawler=crawler, + context_factory = build_from_crawler( + context_factory_cls, + crawler, ) msg = ( f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept " diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 6a211aafa..416669b7f 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -9,7 +9,7 @@ from twisted.internet.defer import Deferred from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured, NotSupported from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values if TYPE_CHECKING: @@ -55,10 +55,9 @@ class DownloadHandlers: dhcls = load_object(path) if skip_lazy and getattr(dhcls, "lazy", True): return None - dh = create_instance( - objcls=dhcls, - settings=self._crawler.settings, - crawler=self._crawler, + dh = build_from_crawler( + dhcls, + self._crawler, ) except NotConfigured as ex: self._notconfigured[scheme] = str(ex) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 6c1dac4a5..b6ac7a251 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -1,6 +1,6 @@ """Download handlers for http and https schemes """ -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode @@ -30,10 +30,9 @@ class HTTP10DownloadHandler: host, port = to_unicode(factory.host), factory.port if factory.scheme == b"https": - client_context_factory = create_instance( - objcls=self.ClientContextFactory, - settings=self._settings, - crawler=self._crawler, + client_context_factory = build_from_crawler( + self.ClientContextFactory, + self._crawler, ) return reactor.connectSSL(host, port, factory, client_context_factory) return reactor.connectTCP(host, port, factory) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 81d8e8115..1f7533759 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -2,7 +2,7 @@ from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.exceptions import NotConfigured from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.misc import create_instance +from scrapy.utils.misc import build_from_crawler class S3DownloadHandler: @@ -50,10 +50,9 @@ class S3DownloadHandler: ) ) - _http_handler = create_instance( - objcls=httpdownloadhandler, - settings=settings, - crawler=crawler, + _http_handler = build_from_crawler( + httpdownloadhandler, + crawler, ) self._download_http = _http_handler.download_request diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index dd1f56f8c..545cd401f 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -34,7 +34,7 @@ from scrapy.settings import BaseSettings, Settings from scrapy.signalmanager import SignalManager from scrapy.spiders import Spider from scrapy.utils.log import failure_to_exc_info, logformatter_adapter -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: @@ -358,9 +358,7 @@ class ExecutionEngine: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={"spider": spider}) nextcall = CallLaterOnce(self._next_request) - scheduler = create_instance( - self.scheduler_cls, settings=None, crawler=self.crawler - ) + scheduler = build_from_crawler(self.scheduler_cls, self.crawler) start_requests = yield self.scraper.spidermw.process_start_requests( start_requests, spider ) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 17c95f1ea..f41b83a67 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -14,7 +14,7 @@ from scrapy.http.request import Request from scrapy.spiders import Spider from scrapy.statscollectors import StatsCollector from scrapy.utils.job import job_dir -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -202,7 +202,7 @@ class Scheduler(BaseScheduler): """ dupefilter_cls = load_object(crawler.settings["DUPEFILTER_CLASS"]) return cls( - dupefilter=create_instance(dupefilter_cls, crawler.settings, crawler), + dupefilter=build_from_crawler(dupefilter_cls, crawler), jobdir=job_dir(crawler.settings), dqclass=load_object(crawler.settings["SCHEDULER_DISK_QUEUE"]), mqclass=load_object(crawler.settings["SCHEDULER_MEMORY_QUEUE"]), @@ -322,10 +322,9 @@ class Scheduler(BaseScheduler): def _mq(self): """Create a new priority queue instance, with in-memory storage""" - return create_instance( + return build_from_crawler( self.pqclass, - settings=None, - crawler=self.crawler, + self.crawler, downstream_queue_cls=self.mqclass, key="", ) @@ -334,10 +333,9 @@ class Scheduler(BaseScheduler): """Create a new priority queue instance, with disk storage""" assert self.dqdir state = self._read_dqs_state(self.dqdir) - q = create_instance( + q = build_from_crawler( self.pqclass, - settings=None, - crawler=self.crawler, + self.crawler, downstream_queue_cls=self.dqclass, key=self.dqdir, startprios=state, diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1d3a11208..844d5f759 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -39,7 +39,7 @@ from scrapy.utils.log import ( log_reactor_info, log_scrapy_info, ) -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.reactor import ( install_reactor, @@ -109,10 +109,9 @@ class Crawler: lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) self.logformatter = lf_cls.from_crawler(self) - self.request_fingerprinter = create_instance( + self.request_fingerprinter = build_from_crawler( load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]), - settings=self.settings, - crawler=self, + self, ) reactor_class: str = self.settings["TWISTED_REACTOR"] @@ -404,7 +403,7 @@ class CrawlerProcess(CrawlerRunner): d.addBoth(self._stop_reactor) resolver_class = load_object(self.settings["DNS_RESOLVER"]) - resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) + resolver = build_from_crawler(resolver_class, self, reactor=reactor) resolver.install_on_reactor() tp = reactor.getThreadPool() tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index fadbbb582..e5e363b52 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -28,7 +28,7 @@ from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values logger = logging.getLogger(__name__) @@ -371,7 +371,7 @@ class FeedSlot: self._exporting = True def _get_instance(self, objcls, *args, **kwargs): - return create_instance(objcls, self.settings, self.crawler, *args, **kwargs) + return build_from_crawler(objcls, self.crawler, *args, **kwargs) def _get_exporter(self, file, format, *args, **kwargs): return self._get_instance(self.exporters[format], file, *args, **kwargs) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 090588130..f60c726f9 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -23,7 +23,7 @@ from scrapy import Spider from scrapy.exceptions import NotConfigured from scrapy.settings import Settings from scrapy.utils.defer import process_chain, process_parallel -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -64,7 +64,10 @@ class MiddlewareManager: for clspath in mwlist: try: mwcls = load_object(clspath) - mw = create_instance(mwcls, settings, crawler) + if crawler is not None: + mw = build_from_crawler(mwcls, crawler) + else: + mw = build_from_settings(mwcls, settings) middlewares.append(mw) enabled.append(clspath) except NotConfigured as e: diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 62a9af477..b62d2fe58 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,7 +1,7 @@ import hashlib import logging -from scrapy.utils.misc import create_instance +from scrapy.utils.misc import build_from_crawler logger = logging.getLogger(__name__) @@ -72,9 +72,8 @@ class ScrapyPriorityQueue: self.curprio = min(startprios) def qfactory(self, key): - return create_instance( + return build_from_crawler( self.downstream_queue_cls, - None, self.crawler, self.key + "/" + str(key), ) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index a9364bea2..b38190cb3 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -25,6 +25,7 @@ from typing import ( cast, ) +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item from scrapy.utils.datatypes import LocalWeakReferencedCache @@ -142,6 +143,13 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an extension has not been implemented correctly). """ + warnings.warn( + "The create_instance() function is deprecated. " + "Please use build_from_crawler() or build_from_settings() instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if settings is None: if crawler is None: raise ValueError("Specify at least one of settings and crawler.") @@ -160,6 +168,45 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): return instance +def build_from_crawler(objcls, crawler, /, *args, **kwargs): + """Construct a class instance using its ``from_crawler`` constructor. + + ``*args`` and ``**kwargs`` are forwarded to the constructor. + + Raises ``TypeError`` if the resulting instance is ``None``. + """ + if hasattr(objcls, "from_crawler"): + instance = objcls.from_crawler(crawler, *args, **kwargs) + method_name = "from_crawler" + elif hasattr(objcls, "from_settings"): + instance = objcls.from_settings(crawler.settings, *args, **kwargs) + method_name = "from_settings" + else: + instance = objcls(*args, **kwargs) + method_name = "__new__" + if instance is None: + raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") + return instance + + +def build_from_settings(objcls, settings, /, *args, **kwargs): + """Construct a class instance using its ``from_settings`` constructor. + + ``*args`` and ``**kwargs`` are forwarded to the constructor. + + Raises ``TypeError`` if the resulting instance is ``None``. + """ + if hasattr(objcls, "from_settings"): + instance = objcls.from_settings(settings, *args, **kwargs) + method_name = "from_settings" + else: + instance = objcls(*args, **kwargs) + method_name = "__new__" + if instance is None: + raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") + return instance + + @contextmanager def set_environ(**kwargs: str) -> Generator[None, Any, None]: """Temporarily set environment variables inside the context manager and diff --git a/tests/test_addons.py b/tests/test_addons.py index 0f4f2e5b8..f1b01bc5c 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -89,7 +89,7 @@ class AddonManagerTest(unittest.TestCase): self.assertEqual([a.number for a in manager.addons], expected_order) self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1]) - def test_create_instance(self): + def test_build_from_crawler(self): settings_dict = { "ADDONS": {"tests.test_addons.CreateInstanceAddon": 0}, "MYADDON": {"MYADDON_KEY": "val"}, @@ -167,12 +167,12 @@ class AddonManagerTest(unittest.TestCase): pass with patch("scrapy.addons.logger") as logger_mock: - with patch("scrapy.addons.create_instance") as create_instance_mock: + with patch("scrapy.addons.build_from_crawler") as build_from_crawler_mock: settings_dict = { "ADDONS": {LoggedAddon: 1}, } addon = LoggedAddon() - create_instance_mock.return_value = addon + build_from_crawler_mock.return_value = addon crawler = get_crawler(settings_dict=settings_dict) logger_mock.info.assert_called_once_with( "Enabled addons:\n%(addons)s", diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 924ece6f9..dd07d33f1 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -29,7 +29,7 @@ from scrapy.http import Headers, HtmlResponse, Request from scrapy.http.response.text import TextResponse from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider -from scrapy.utils.misc import create_instance +from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, skip_if_no_boto from tests import NON_EXISTING_RESOLVABLE @@ -109,7 +109,7 @@ class FileTestCase(unittest.TestCase): # add a special char to check that they are handled correctly self.tmpname = Path(self.mktemp() + "^") Path(self.tmpname).write_text("0123456789", encoding="utf-8") - handler = create_instance(FileDownloadHandler, None, get_crawler()) + handler = build_from_crawler(FileDownloadHandler, get_crawler()) self.download_request = handler.download_request def tearDown(self): @@ -257,8 +257,8 @@ class HttpTestCase(unittest.TestCase): else: self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) self.portno = self.port.getHost().port - self.download_handler = create_instance( - self.download_handler_cls, None, get_crawler() + self.download_handler = build_from_crawler( + self.download_handler_cls, get_crawler() ) self.download_request = self.download_handler.download_request @@ -557,7 +557,7 @@ class Http11TestCase(HttpTestCase): def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"): crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False}) - download_handler = create_instance(self.download_handler_cls, None, crawler) + download_handler = build_from_crawler(self.download_handler_cls, crawler) request = Request(self.getURL(url)) d = download_handler.download_request(request, Spider("foo")) d.addCallback(lambda r: r.flags) @@ -590,7 +590,7 @@ class Https11TestCase(Http11TestCase): crawler = get_crawler( settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True} ) - download_handler = create_instance(self.download_handler_cls, None, crawler) + download_handler = build_from_crawler(self.download_handler_cls, crawler) try: with LogCapture() as log_capture: request = Request(self.getURL("file")) @@ -669,9 +669,7 @@ class Https11CustomCiphers(unittest.TestCase): crawler = get_crawler( settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": "CAMELLIA256-SHA"} ) - self.download_handler = create_instance( - self.download_handler_cls, None, crawler - ) + self.download_handler = build_from_crawler(self.download_handler_cls, crawler) self.download_request = self.download_handler.download_request @defer.inlineCallbacks @@ -751,8 +749,8 @@ class HttpProxyTestCase(unittest.TestCase): wrapper = WrappingFactory(site) self.port = reactor.listenTCP(0, wrapper, interface="127.0.0.1") self.portno = self.port.getHost().port - self.download_handler = create_instance( - self.download_handler_cls, None, get_crawler() + self.download_handler = build_from_crawler( + self.download_handler_cls, get_crawler() ) self.download_request = self.download_handler.download_request @@ -830,10 +828,9 @@ class S3AnonTestCase(unittest.TestCase): def setUp(self): skip_if_no_boto() crawler = get_crawler() - self.s3reqh = create_instance( - objcls=S3DownloadHandler, - settings=None, - crawler=crawler, + self.s3reqh = build_from_crawler( + S3DownloadHandler, + crawler, httpdownloadhandler=HttpDownloadHandlerMock, # anon=True, # implicit ) @@ -861,10 +858,9 @@ class S3TestCase(unittest.TestCase): def setUp(self): skip_if_no_boto() crawler = get_crawler() - s3reqh = create_instance( - objcls=S3DownloadHandler, - settings=None, - crawler=crawler, + s3reqh = build_from_crawler( + S3DownloadHandler, + crawler, aws_access_key_id=self.AWS_ACCESS_KEY_ID, aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, httpdownloadhandler=HttpDownloadHandlerMock, @@ -889,10 +885,9 @@ class S3TestCase(unittest.TestCase): def test_extra_kw(self): try: crawler = get_crawler() - create_instance( - objcls=S3DownloadHandler, - settings=None, - crawler=crawler, + build_from_crawler( + S3DownloadHandler, + crawler, extra_kw=True, ) except Exception as e: @@ -1039,9 +1034,7 @@ class BaseFTPTestCase(unittest.TestCase): self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") self.portNum = self.port.getHost().port crawler = get_crawler() - self.download_handler = create_instance( - FTPDownloadHandler, crawler.settings, crawler - ) + self.download_handler = build_from_crawler(FTPDownloadHandler, crawler) self.addCleanup(self.port.stopListening) def tearDown(self): @@ -1185,9 +1178,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase): self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") self.portNum = self.port.getHost().port crawler = get_crawler() - self.download_handler = create_instance( - FTPDownloadHandler, crawler.settings, crawler - ) + self.download_handler = build_from_crawler(FTPDownloadHandler, crawler) self.addCleanup(self.port.stopListening) def tearDown(self): @@ -1197,9 +1188,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase): class DataURITestCase(unittest.TestCase): def setUp(self): crawler = get_crawler() - self.download_handler = create_instance( - DataURIDownloadHandler, crawler.settings, crawler - ) + self.download_handler = build_from_crawler(DataURIDownloadHandler, crawler) self.download_request = self.download_handler.download_request self.spider = Spider("foo") diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 31fa1430d..322075043 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -11,7 +11,7 @@ from twisted.web.http import H2_ENABLED from scrapy.http import Request from scrapy.spiders import Spider -from scrapy.utils.misc import create_instance +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler from tests.mockserver import ssl_context_factory from tests.test_downloader_handlers import ( @@ -240,8 +240,8 @@ class Https2ProxyTestCase(Http11ProxyTestCase): interface=self.host, ) self.portno = self.port.getHost().port - self.download_handler = create_instance( - self.download_handler_cls, None, get_crawler() + self.download_handler = build_from_crawler( + self.download_handler_cls, get_crawler() ) self.download_request = self.download_handler.download_request diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index e7799737f..3fde5e8c5 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -440,7 +440,7 @@ class SettingsTest(unittest.TestCase): def test_passing_objects_as_values(self): from scrapy.core.downloader.handlers.file import FileDownloadHandler - from scrapy.utils.misc import create_instance + from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler class TestPipeline: @@ -468,7 +468,7 @@ class SettingsTest(unittest.TestCase): myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp") self.assertEqual(myhandler, FileDownloadHandler) - myhandler_instance = create_instance(myhandler, None, get_crawler()) + myhandler_instance = build_from_crawler(myhandler, get_crawler()) self.assertIsInstance(myhandler_instance, FileDownloadHandler) self.assertTrue(hasattr(myhandler_instance, "download_request")) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 69793ee75..ee3314d8e 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -7,6 +7,8 @@ from unittest import mock from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, + build_from_crawler, + build_from_settings, create_instance, load_object, rel_has_nofollow, @@ -153,6 +155,78 @@ class UtilsMiscTestCase(unittest.TestCase): with self.assertRaises(TypeError): create_instance(m, settings, None) + def test_build_from_crawler(self): + settings = mock.MagicMock() + crawler = mock.MagicMock(spec_set=["settings"]) + args = (True, 100.0) + kwargs = {"key": "val"} + + def _test_with_crawler(mock, settings, crawler): + build_from_crawler(mock, crawler, *args, **kwargs) + if hasattr(mock, "from_crawler"): + mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) + if hasattr(mock, "from_settings"): + self.assertEqual(mock.from_settings.call_count, 0) + self.assertEqual(mock.call_count, 0) + elif hasattr(mock, "from_settings"): + mock.from_settings.assert_called_once_with(settings, *args, **kwargs) + self.assertEqual(mock.call_count, 0) + else: + mock.assert_called_once_with(*args, **kwargs) + + # Check usage of correct constructor using three mocks: + # 1. with no alternative constructors + # 2. with from_crawler() constructor + # 3. with from_settings() and from_crawler() constructor + spec_sets = ( + ["__qualname__"], + ["__qualname__", "from_crawler"], + ["__qualname__", "from_settings", "from_crawler"], + ) + for specs in spec_sets: + m = mock.MagicMock(spec_set=specs) + _test_with_crawler(m, settings, crawler) + m.reset_mock() + + # Check adoption of crawler + m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"]) + m.from_crawler.return_value = None + with self.assertRaises(TypeError): + build_from_crawler(m, crawler, *args, **kwargs) + + def test_build_from_settings(self): + settings = mock.MagicMock() + args = (True, 100.0) + kwargs = {"key": "val"} + + def _test_with_settings(mock, settings): + build_from_settings(mock, settings, *args, **kwargs) + if hasattr(mock, "from_settings"): + mock.from_settings.assert_called_once_with(settings, *args, **kwargs) + self.assertEqual(mock.call_count, 0) + else: + mock.assert_called_once_with(*args, **kwargs) + + # Check usage of correct constructor using three mocks: + # 1. with no alternative constructors + # 2. with from_settings() constructor + # 3. with from_settings() and from_crawler() constructor + spec_sets = ( + ["__qualname__"], + ["__qualname__", "from_settings"], + ["__qualname__", "from_settings", "from_crawler"], + ) + for specs in spec_sets: + m = mock.MagicMock(spec_set=specs) + _test_with_settings(m, settings) + m.reset_mock() + + # Check adoption of crawler settings + m = mock.MagicMock(spec_set=["__qualname__", "from_settings"]) + m.from_settings.return_value = None + with self.assertRaises(TypeError): + build_from_settings(m, settings, *args, **kwargs) + def test_set_environ(self): assert os.environ.get("some_test_environ") is None with set_environ(some_test_environ="test_value"): diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 0042fe8f0..d4b6ba15b 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -24,7 +24,7 @@ from scrapy.core.downloader import webclient as client from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory from scrapy.http import Headers, Request from scrapy.settings import Settings -from scrapy.utils.misc import create_instance +from scrapy.utils.misc import build_from_settings from scrapy.utils.python import to_bytes, to_unicode from tests.mockserver import ( BrokenDownloadResource, @@ -470,8 +470,8 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): def testPayload(self): s = "0123456789" * 10 settings = Settings({"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers}) - client_context_factory = create_instance( - ScrapyClientContextFactory, settings=settings, crawler=None + client_context_factory = build_from_settings( + ScrapyClientContextFactory, settings ) return getPage( self.getURL("payload"), body=s, contextFactory=client_context_factory @@ -482,8 +482,8 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): settings = Settings( {"DOWNLOADER_CLIENT_TLS_CIPHERS": "ECDHE-RSA-AES256-GCM-SHA384"} ) - client_context_factory = create_instance( - ScrapyClientContextFactory, settings=settings, crawler=None + client_context_factory = build_from_settings( + ScrapyClientContextFactory, settings ) d = getPage( self.getURL("payload"), body=s, contextFactory=client_context_factory