RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

create_instance → build_from_crawler, build_from_settings (#6169)

This commit is contained in:
Swati Anshu 2023-12-18 05:51:02 -05:00 committed by GitHub
parent 70ba3a0868
commit c67f730695
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 197 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/addons.py
View File

@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, List
from scrapy.exceptions import NotConfigured
from scrapy.settings import Settings
from scrapy.utils.conf import build_component_list
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
from scrapy.crawler import Crawler
@ -32,9 +32,7 @@ class AddonManager:
for clspath in build_component_list(settings["ADDONS"]):
try:
addoncls = load_object(clspath)
addon = create_instance(
addoncls, settings=settings, crawler=self.crawler
)
addon = build_from_crawler(addoncls, self.crawler)
addon.update_settings(settings)
self.addons.append(addon)
except NotConfigured as e:

16
scrapy/core/downloader/contextfactory.py
View File

@ -20,7 +20,7 @@ from scrapy.core.downloader.tls import (
openssl_methods,
)
from scrapy.settings import BaseSettings
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
from twisted.internet._sslverify import ClientTLSOptions
@ -165,18 +165,16 @@ def load_context_factory_from_settings(settings, crawler):
context_factory_cls = load_object(settings["DOWNLOADER_CLIENTCONTEXTFACTORY"])
# try method-aware context factory
try:
context_factory = create_instance(
objcls=context_factory_cls,
settings=settings,
crawler=crawler,
context_factory = build_from_crawler(
context_factory_cls,
crawler,
method=ssl_method,
)
except TypeError:
# use context factory defaults
context_factory = create_instance(
objcls=context_factory_cls,
settings=settings,
crawler=crawler,
context_factory = build_from_crawler(
context_factory_cls,
crawler,
)
msg = (
f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept "

9
scrapy/core/downloader/handlers/__init__.py
View File

@ -9,7 +9,7 @@ from twisted.internet.defer import Deferred
from scrapy import Request, Spider, signals
from scrapy.exceptions import NotConfigured, NotSupported
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
@ -55,10 +55,9 @@ class DownloadHandlers:
dhcls = load_object(path)
if skip_lazy and getattr(dhcls, "lazy", True):
return None
dh = create_instance(
objcls=dhcls,
settings=self._crawler.settings,
crawler=self._crawler,
dh = build_from_crawler(
dhcls,
self._crawler,
)
except NotConfigured as ex:
self._notconfigured[scheme] = str(ex)

9
scrapy/core/downloader/handlers/http10.py
View File

@ -1,6 +1,6 @@
"""Download handlers for http and https schemes
"""
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import to_unicode
@ -30,10 +30,9 @@ class HTTP10DownloadHandler:
host, port = to_unicode(factory.host), factory.port
if factory.scheme == b"https":
client_context_factory = create_instance(
objcls=self.ClientContextFactory,
settings=self._settings,
crawler=self._crawler,
client_context_factory = build_from_crawler(
self.ClientContextFactory,
self._crawler,
)
return reactor.connectSSL(host, port, factory, client_context_factory)
return reactor.connectTCP(host, port, factory)

9
scrapy/core/downloader/handlers/s3.py
View File

@ -2,7 +2,7 @@ from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
from scrapy.exceptions import NotConfigured
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import create_instance
from scrapy.utils.misc import build_from_crawler
class S3DownloadHandler:
@ -50,10 +50,9 @@ class S3DownloadHandler:
)
)
_http_handler = create_instance(
objcls=httpdownloadhandler,
settings=settings,
crawler=crawler,
_http_handler = build_from_crawler(
httpdownloadhandler,
crawler,
)
self._download_http = _http_handler.download_request

6
scrapy/core/engine.py
View File

@ -34,7 +34,7 @@ from scrapy.settings import BaseSettings, Settings
from scrapy.signalmanager import SignalManager
from scrapy.spiders import Spider
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.reactor import CallLaterOnce
if TYPE_CHECKING:
@ -358,9 +358,7 @@ class ExecutionEngine:
raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
logger.info("Spider opened", extra={"spider": spider})
nextcall = CallLaterOnce(self._next_request)
scheduler = create_instance(
self.scheduler_cls, settings=None, crawler=self.crawler
)
scheduler = build_from_crawler(self.scheduler_cls, self.crawler)
start_requests = yield self.scraper.spidermw.process_start_requests(
start_requests, spider
)

14
scrapy/core/scheduler.py
View File

@ -14,7 +14,7 @@ from scrapy.http.request import Request
from scrapy.spiders import Spider
from scrapy.statscollectors import StatsCollector
from scrapy.utils.job import job_dir
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
# typing.Self requires Python 3.11
@ -202,7 +202,7 @@ class Scheduler(BaseScheduler):
"""
dupefilter_cls = load_object(crawler.settings["DUPEFILTER_CLASS"])
return cls(
dupefilter=create_instance(dupefilter_cls, crawler.settings, crawler),
dupefilter=build_from_crawler(dupefilter_cls, crawler),
jobdir=job_dir(crawler.settings),
dqclass=load_object(crawler.settings["SCHEDULER_DISK_QUEUE"]),
mqclass=load_object(crawler.settings["SCHEDULER_MEMORY_QUEUE"]),
@ -322,10 +322,9 @@ class Scheduler(BaseScheduler):
def _mq(self):
"""Create a new priority queue instance, with in-memory storage"""
return create_instance(
return build_from_crawler(
self.pqclass,
settings=None,
crawler=self.crawler,
self.crawler,
downstream_queue_cls=self.mqclass,
key="",
)
@ -334,10 +333,9 @@ class Scheduler(BaseScheduler):
"""Create a new priority queue instance, with disk storage"""
assert self.dqdir
state = self._read_dqs_state(self.dqdir)
q = create_instance(
q = build_from_crawler(
self.pqclass,
settings=None,
crawler=self.crawler,
self.crawler,
downstream_queue_cls=self.dqclass,
key=self.dqdir,
startprios=state,

9
scrapy/crawler.py
View File

@ -39,7 +39,7 @@ from scrapy.utils.log import (
log_reactor_info,
log_scrapy_info,
)
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
from scrapy.utils.reactor import (
install_reactor,
@ -109,10 +109,9 @@ class Crawler:
lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
self.logformatter = lf_cls.from_crawler(self)
self.request_fingerprinter = create_instance(
self.request_fingerprinter = build_from_crawler(
load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),
settings=self.settings,
crawler=self,
self,
)
reactor_class: str = self.settings["TWISTED_REACTOR"]
@ -404,7 +403,7 @@ class CrawlerProcess(CrawlerRunner):
d.addBoth(self._stop_reactor)
resolver_class = load_object(self.settings["DNS_RESOLVER"])
resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
resolver = build_from_crawler(resolver_class, self, reactor=reactor)
resolver.install_on_reactor()
tp = reactor.getThreadPool()
tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))

4
scrapy/extensions/feedexport.py
View File

@ -28,7 +28,7 @@ from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.deprecate import create_deprecated_class
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import without_none_values
logger = logging.getLogger(__name__)
@ -371,7 +371,7 @@ class FeedSlot:
self._exporting = True
def _get_instance(self, objcls, *args, **kwargs):
return create_instance(objcls, self.settings, self.crawler, *args, **kwargs)
return build_from_crawler(objcls, self.crawler, *args, **kwargs)
def _get_exporter(self, file, format, *args, **kwargs):
return self._get_instance(self.exporters[format], file, *args, **kwargs)

7
scrapy/middleware.py
View File

@ -23,7 +23,7 @@ from scrapy import Spider
from scrapy.exceptions import NotConfigured
from scrapy.settings import Settings
from scrapy.utils.defer import process_chain, process_parallel
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object
if TYPE_CHECKING:
# typing.Self requires Python 3.11
@ -64,7 +64,10 @@ class MiddlewareManager:
for clspath in mwlist:
try:
mwcls = load_object(clspath)
mw = create_instance(mwcls, settings, crawler)
if crawler is not None:
mw = build_from_crawler(mwcls, crawler)
else:
mw = build_from_settings(mwcls, settings)
middlewares.append(mw)
enabled.append(clspath)
except NotConfigured as e:

5
scrapy/pqueues.py
View File

@ -1,7 +1,7 @@
import hashlib
import logging
from scrapy.utils.misc import create_instance
from scrapy.utils.misc import build_from_crawler
logger = logging.getLogger(__name__)
@ -72,9 +72,8 @@ class ScrapyPriorityQueue:
self.curprio = min(startprios)
def qfactory(self, key):
return create_instance(
return build_from_crawler(
self.downstream_queue_cls,
None,
self.crawler,
self.key + "/" + str(key),
)

47
scrapy/utils/misc.py
View File

@ -25,6 +25,7 @@ from typing import (
cast,
)
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import Item
from scrapy.utils.datatypes import LocalWeakReferencedCache
@ -142,6 +143,13 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
extension has not been implemented correctly).
"""
warnings.warn(
"The create_instance() function is deprecated. "
"Please use build_from_crawler() or build_from_settings() instead.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
if settings is None:
if crawler is None:
raise ValueError("Specify at least one of settings and crawler.")
@ -160,6 +168,45 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
return instance
def build_from_crawler(objcls, crawler, /, *args, **kwargs):
"""Construct a class instance using its ``from_crawler`` constructor.
``*args`` and ``**kwargs`` are forwarded to the constructor.
Raises ``TypeError`` if the resulting instance is ``None``.
"""
if hasattr(objcls, "from_crawler"):
instance = objcls.from_crawler(crawler, *args, **kwargs)
method_name = "from_crawler"
elif hasattr(objcls, "from_settings"):
instance = objcls.from_settings(crawler.settings, *args, **kwargs)
method_name = "from_settings"
else:
instance = objcls(*args, **kwargs)
method_name = "__new__"
if instance is None:
raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
return instance
def build_from_settings(objcls, settings, /, *args, **kwargs):
"""Construct a class instance using its ``from_settings`` constructor.
``*args`` and ``**kwargs`` are forwarded to the constructor.
Raises ``TypeError`` if the resulting instance is ``None``.
"""
if hasattr(objcls, "from_settings"):
instance = objcls.from_settings(settings, *args, **kwargs)
method_name = "from_settings"
else:
instance = objcls(*args, **kwargs)
method_name = "__new__"
if instance is None:
raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
return instance
@contextmanager
def set_environ(**kwargs: str) -> Generator[None, Any, None]:
"""Temporarily set environment variables inside the context manager and

6
tests/test_addons.py
View File

@ -89,7 +89,7 @@ class AddonManagerTest(unittest.TestCase):
self.assertEqual([a.number for a in manager.addons], expected_order)
self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1])
def test_create_instance(self):
def test_build_from_crawler(self):
settings_dict = {
"ADDONS": {"tests.test_addons.CreateInstanceAddon": 0},
"MYADDON": {"MYADDON_KEY": "val"},
@ -167,12 +167,12 @@ class AddonManagerTest(unittest.TestCase):
pass
with patch("scrapy.addons.logger") as logger_mock:
with patch("scrapy.addons.create_instance") as create_instance_mock:
with patch("scrapy.addons.build_from_crawler") as build_from_crawler_mock:
settings_dict = {
"ADDONS": {LoggedAddon: 1},
}
addon = LoggedAddon()
create_instance_mock.return_value = addon
build_from_crawler_mock.return_value = addon
crawler = get_crawler(settings_dict=settings_dict)
logger_mock.info.assert_called_once_with(
"Enabled addons:\n%(addons)s",

53
tests/test_downloader_handlers.py
View File

@ -29,7 +29,7 @@ from scrapy.http import Headers, HtmlResponse, Request
from scrapy.http.response.text import TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
from scrapy.utils.misc import create_instance
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler, skip_if_no_boto
from tests import NON_EXISTING_RESOLVABLE
@ -109,7 +109,7 @@ class FileTestCase(unittest.TestCase):
# add a special char to check that they are handled correctly
self.tmpname = Path(self.mktemp() + "^")
Path(self.tmpname).write_text("0123456789", encoding="utf-8")
handler = create_instance(FileDownloadHandler, None, get_crawler())
handler = build_from_crawler(FileDownloadHandler, get_crawler())
self.download_request = handler.download_request
def tearDown(self):
@ -257,8 +257,8 @@ class HttpTestCase(unittest.TestCase):
else:
self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
self.portno = self.port.getHost().port
self.download_handler = create_instance(
self.download_handler_cls, None, get_crawler()
self.download_handler = build_from_crawler(
self.download_handler_cls, get_crawler()
)
self.download_request = self.download_handler.download_request
@ -557,7 +557,7 @@ class Http11TestCase(HttpTestCase):
def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"):
crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False})
download_handler = create_instance(self.download_handler_cls, None, crawler)
download_handler = build_from_crawler(self.download_handler_cls, crawler)
request = Request(self.getURL(url))
d = download_handler.download_request(request, Spider("foo"))
d.addCallback(lambda r: r.flags)
@ -590,7 +590,7 @@ class Https11TestCase(Http11TestCase):
crawler = get_crawler(
settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True}
)
download_handler = create_instance(self.download_handler_cls, None, crawler)
download_handler = build_from_crawler(self.download_handler_cls, crawler)
try:
with LogCapture() as log_capture:
request = Request(self.getURL("file"))
@ -669,9 +669,7 @@ class Https11CustomCiphers(unittest.TestCase):
crawler = get_crawler(
settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": "CAMELLIA256-SHA"}
)
self.download_handler = create_instance(
self.download_handler_cls, None, crawler
)
self.download_handler = build_from_crawler(self.download_handler_cls, crawler)
self.download_request = self.download_handler.download_request
@defer.inlineCallbacks
@ -751,8 +749,8 @@ class HttpProxyTestCase(unittest.TestCase):
wrapper = WrappingFactory(site)
self.port = reactor.listenTCP(0, wrapper, interface="127.0.0.1")
self.portno = self.port.getHost().port
self.download_handler = create_instance(
self.download_handler_cls, None, get_crawler()
self.download_handler = build_from_crawler(
self.download_handler_cls, get_crawler()
)
self.download_request = self.download_handler.download_request
@ -830,10 +828,9 @@ class S3AnonTestCase(unittest.TestCase):
def setUp(self):
skip_if_no_boto()
crawler = get_crawler()
self.s3reqh = create_instance(
objcls=S3DownloadHandler,
settings=None,
crawler=crawler,
self.s3reqh = build_from_crawler(
S3DownloadHandler,
crawler,
httpdownloadhandler=HttpDownloadHandlerMock,
# anon=True, # implicit
)
@ -861,10 +858,9 @@ class S3TestCase(unittest.TestCase):
def setUp(self):
skip_if_no_boto()
crawler = get_crawler()
s3reqh = create_instance(
objcls=S3DownloadHandler,
settings=None,
crawler=crawler,
s3reqh = build_from_crawler(
S3DownloadHandler,
crawler,
aws_access_key_id=self.AWS_ACCESS_KEY_ID,
aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
httpdownloadhandler=HttpDownloadHandlerMock,
@ -889,10 +885,9 @@ class S3TestCase(unittest.TestCase):
def test_extra_kw(self):
try:
crawler = get_crawler()
create_instance(
objcls=S3DownloadHandler,
settings=None,
crawler=crawler,
build_from_crawler(
S3DownloadHandler,
crawler,
extra_kw=True,
)
except Exception as e:
@ -1039,9 +1034,7 @@ class BaseFTPTestCase(unittest.TestCase):
self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")
self.portNum = self.port.getHost().port
crawler = get_crawler()
self.download_handler = create_instance(
FTPDownloadHandler, crawler.settings, crawler
)
self.download_handler = build_from_crawler(FTPDownloadHandler, crawler)
self.addCleanup(self.port.stopListening)
def tearDown(self):
@ -1185,9 +1178,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")
self.portNum = self.port.getHost().port
crawler = get_crawler()
self.download_handler = create_instance(
FTPDownloadHandler, crawler.settings, crawler
)
self.download_handler = build_from_crawler(FTPDownloadHandler, crawler)
self.addCleanup(self.port.stopListening)
def tearDown(self):
@ -1197,9 +1188,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
class DataURITestCase(unittest.TestCase):
def setUp(self):
crawler = get_crawler()
self.download_handler = create_instance(
DataURIDownloadHandler, crawler.settings, crawler
)
self.download_handler = build_from_crawler(DataURIDownloadHandler, crawler)
self.download_request = self.download_handler.download_request
self.spider = Spider("foo")

6
tests/test_downloader_handlers_http2.py
View File

@ -11,7 +11,7 @@ from twisted.web.http import H2_ENABLED
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.misc import create_instance
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.mockserver import ssl_context_factory
from tests.test_downloader_handlers import (
@ -240,8 +240,8 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
interface=self.host,
)
self.portno = self.port.getHost().port
self.download_handler = create_instance(
self.download_handler_cls, None, get_crawler()
self.download_handler = build_from_crawler(
self.download_handler_cls, get_crawler()
)
self.download_request = self.download_handler.download_request

4
tests/test_settings/__init__.py
View File

@ -440,7 +440,7 @@ class SettingsTest(unittest.TestCase):
def test_passing_objects_as_values(self):
from scrapy.core.downloader.handlers.file import FileDownloadHandler
from scrapy.utils.misc import create_instance
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
class TestPipeline:
@ -468,7 +468,7 @@ class SettingsTest(unittest.TestCase):
myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp")
self.assertEqual(myhandler, FileDownloadHandler)
myhandler_instance = create_instance(myhandler, None, get_crawler())
myhandler_instance = build_from_crawler(myhandler, get_crawler())
self.assertIsInstance(myhandler_instance, FileDownloadHandler)
self.assertTrue(hasattr(myhandler_instance, "download_request"))

74
tests/test_utils_misc/__init__.py
View File

@ -7,6 +7,8 @@ from unittest import mock
from scrapy.item import Field, Item
from scrapy.utils.misc import (
arg_to_iter,
build_from_crawler,
build_from_settings,
create_instance,
load_object,
rel_has_nofollow,
@ -153,6 +155,78 @@ class UtilsMiscTestCase(unittest.TestCase):
with self.assertRaises(TypeError):
create_instance(m, settings, None)
def test_build_from_crawler(self):
settings = mock.MagicMock()
crawler = mock.MagicMock(spec_set=["settings"])
args = (True, 100.0)
kwargs = {"key": "val"}
def _test_with_crawler(mock, settings, crawler):
build_from_crawler(mock, crawler, *args, **kwargs)
if hasattr(mock, "from_crawler"):
mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
if hasattr(mock, "from_settings"):
self.assertEqual(mock.from_settings.call_count, 0)
self.assertEqual(mock.call_count, 0)
elif hasattr(mock, "from_settings"):
mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
self.assertEqual(mock.call_count, 0)
else:
mock.assert_called_once_with(*args, **kwargs)
# Check usage of correct constructor using three mocks:
# 1. with no alternative constructors
# 2. with from_crawler() constructor
# 3. with from_settings() and from_crawler() constructor
spec_sets = (
["__qualname__"],
["__qualname__", "from_crawler"],
["__qualname__", "from_settings", "from_crawler"],
)
for specs in spec_sets:
m = mock.MagicMock(spec_set=specs)
_test_with_crawler(m, settings, crawler)
m.reset_mock()
# Check adoption of crawler
m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"])
m.from_crawler.return_value = None
with self.assertRaises(TypeError):
build_from_crawler(m, crawler, *args, **kwargs)
def test_build_from_settings(self):
settings = mock.MagicMock()
args = (True, 100.0)
kwargs = {"key": "val"}
def _test_with_settings(mock, settings):
build_from_settings(mock, settings, *args, **kwargs)
if hasattr(mock, "from_settings"):
mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
self.assertEqual(mock.call_count, 0)
else:
mock.assert_called_once_with(*args, **kwargs)
# Check usage of correct constructor using three mocks:
# 1. with no alternative constructors
# 2. with from_settings() constructor
# 3. with from_settings() and from_crawler() constructor
spec_sets = (
["__qualname__"],
["__qualname__", "from_settings"],
["__qualname__", "from_settings", "from_crawler"],
)
for specs in spec_sets:
m = mock.MagicMock(spec_set=specs)
_test_with_settings(m, settings)
m.reset_mock()
# Check adoption of crawler settings
m = mock.MagicMock(spec_set=["__qualname__", "from_settings"])
m.from_settings.return_value = None
with self.assertRaises(TypeError):
build_from_settings(m, settings, *args, **kwargs)
def test_set_environ(self):
assert os.environ.get("some_test_environ") is None
with set_environ(some_test_environ="test_value"):

10
tests/test_webclient.py
View File

@ -24,7 +24,7 @@ from scrapy.core.downloader import webclient as client
from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
from scrapy.http import Headers, Request
from scrapy.settings import Settings
from scrapy.utils.misc import create_instance
from scrapy.utils.misc import build_from_settings
from scrapy.utils.python import to_bytes, to_unicode
from tests.mockserver import (
BrokenDownloadResource,
@ -470,8 +470,8 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
def testPayload(self):
s = "0123456789" * 10
settings = Settings({"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers})
client_context_factory = create_instance(
ScrapyClientContextFactory, settings=settings, crawler=None
client_context_factory = build_from_settings(
ScrapyClientContextFactory, settings
)
return getPage(
self.getURL("payload"), body=s, contextFactory=client_context_factory
@ -482,8 +482,8 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
settings = Settings(
{"DOWNLOADER_CLIENT_TLS_CIPHERS": "ECDHE-RSA-AES256-GCM-SHA384"}
)
client_context_factory = create_instance(
ScrapyClientContextFactory, settings=settings, crawler=None
client_context_factory = build_from_settings(
ScrapyClientContextFactory, settings
)
d = getPage(
self.getURL("payload"), body=s, contextFactory=client_context_factory