mirror of https://github.com/scrapy/scrapy.git
Andrey Rakhmatullin
GitHub
commit
06ebdee35d
|
|
@ -222,6 +222,7 @@ Extending Scrapy
|
|||
:hidden:
|
||||
|
||||
topics/architecture
|
||||
topics/addons
|
||||
topics/downloader-middleware
|
||||
topics/spider-middleware
|
||||
topics/extensions
|
||||
|
|
@ -235,6 +236,9 @@ Extending Scrapy
|
|||
:doc:`topics/architecture`
|
||||
Understand the Scrapy architecture.
|
||||
|
||||
:doc:`topics/addons`
|
||||
Enable and configure third-party extensions.
|
||||
|
||||
:doc:`topics/downloader-middleware`
|
||||
Customize how pages get requested and downloaded.
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,193 @@
|
|||
.. _topics-addons:
|
||||
|
||||
=======
|
||||
Add-ons
|
||||
=======
|
||||
|
||||
Scrapy's add-on system is a framework which unifies managing and configuring
|
||||
components that extend Scrapy's core functionality, such as middlewares,
|
||||
extensions, or pipelines. It provides users with a plug-and-play experience in
|
||||
Scrapy extension management, and grants extensive configuration control to
|
||||
developers.
|
||||
|
||||
|
||||
Activating and configuring add-ons
|
||||
==================================
|
||||
|
||||
During :class:`~scrapy.crawler.Crawler` initialization, the list of enabled
|
||||
add-ons is read from your ``ADDONS`` setting.
|
||||
|
||||
The ``ADDONS`` setting is a dict in which every key is an add-on class or its
|
||||
import path and the value is its priority.
|
||||
|
||||
This is an example where two add-ons are enabled in a project's
|
||||
``settings.py``::
|
||||
|
||||
ADDONS = {
|
||||
'path.to.someaddon': 0,
|
||||
SomeAddonClass: 1,
|
||||
}
|
||||
|
||||
|
||||
Writing your own add-ons
|
||||
========================
|
||||
|
||||
Add-ons are Python classes that include the following method:
|
||||
|
||||
.. method:: update_settings(settings)
|
||||
|
||||
This method is called during the initialization of the
|
||||
:class:`~scrapy.crawler.Crawler`. Here, you should perform dependency checks
|
||||
(e.g. for external Python libraries) and update the
|
||||
:class:`~scrapy.settings.Settings` object as wished, e.g. enable components
|
||||
for this add-on or set required configuration of other extensions.
|
||||
|
||||
:param settings: The settings object storing Scrapy/component configuration
|
||||
:type settings: :class:`~scrapy.settings.Settings`
|
||||
|
||||
They can also have the following method:
|
||||
|
||||
.. classmethod:: from_crawler(cls, crawler)
|
||||
:noindex:
|
||||
|
||||
If present, this class method is called to create an add-on instance
|
||||
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
|
||||
of the add-on. The crawler object provides access to all Scrapy core
|
||||
components like settings and signals; it is a way for the add-on to access
|
||||
them and hook its functionality into Scrapy.
|
||||
|
||||
:param crawler: The crawler that uses this add-on
|
||||
:type crawler: :class:`~scrapy.crawler.Crawler`
|
||||
|
||||
The settings set by the add-on should use the ``addon`` priority (see
|
||||
:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`)::
|
||||
|
||||
class MyAddon:
|
||||
def update_settings(self, settings):
|
||||
settings.set("DNSCACHE_ENABLED", True, "addon")
|
||||
|
||||
This allows users to override these settings in the project or spider
|
||||
configuration. This is not possible with settings that are mutable objects,
|
||||
such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases
|
||||
you can provide an add-on-specific setting that governs whether the add-on will
|
||||
modify :setting:`ITEM_PIPELINES`::
|
||||
|
||||
class MyAddon:
|
||||
def update_settings(self, settings):
|
||||
if settings.getbool("MYADDON_ENABLE_PIPELINE"):
|
||||
settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200
|
||||
|
||||
If the ``update_settings`` method raises
|
||||
:exc:`scrapy.exceptions.NotConfigured`, the add-on will be skipped. This makes
|
||||
it easy to enable an add-on only when some conditions are met.
|
||||
|
||||
Fallbacks
|
||||
---------
|
||||
|
||||
Some components provided by add-ons need to fall back to "default"
|
||||
implementations, e.g. a custom download handler needs to send the request that
|
||||
it doesn't handle via the default download handler, or a stats collector that
|
||||
includes some additional processing but otherwise uses the default stats
|
||||
collector. And it's possible that a project needs to use several custom
|
||||
components of the same type, e.g. two custom download handlers that support
|
||||
different kinds of custom requests and still need to use the default download
|
||||
handler for other requests. To make such use cases easier to configure, we
|
||||
recommend that such custom components should be written in the following way:
|
||||
|
||||
1. The custom component (e.g. ``MyDownloadHandler``) shouldn't inherit from the
|
||||
default Scrapy one (e.g.
|
||||
``scrapy.core.downloader.handlers.http.HTTPDownloadHandler``), but instead
|
||||
be able to load the class of the fallback component from a special setting
|
||||
(e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use
|
||||
it.
|
||||
2. The add-ons that include these components should read the current value of
|
||||
the default setting (e.g. ``DOWNLOAD_HANDLERS``) in their
|
||||
``update_settings()`` methods, save that value into the fallback setting
|
||||
(``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default
|
||||
setting to the component provided by the add-on (e.g.
|
||||
``MyDownloadHandler``). If the fallback setting is already set by the user,
|
||||
they shouldn't change it.
|
||||
3. This way, if there are several add-ons that want to modify the same setting,
|
||||
all of them will fallback to the component from the previous one and then to
|
||||
the Scrapy default. The order of that depends on the priority order in the
|
||||
``ADDONS`` setting.
|
||||
|
||||
|
||||
Add-on examples
|
||||
===============
|
||||
|
||||
Set some basic configuration:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
class MyAddon:
|
||||
def update_settings(self, settings):
|
||||
settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200
|
||||
settings.set("DNSCACHE_ENABLED", True, "addon")
|
||||
|
||||
Check dependencies:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
class MyAddon:
|
||||
def update_settings(self, settings):
|
||||
try:
|
||||
import boto
|
||||
except ImportError:
|
||||
raise NotConfigured("MyAddon requires the boto library")
|
||||
...
|
||||
|
||||
Access the crawler instance:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
class MyAddon:
|
||||
def __init__(self, crawler) -> None:
|
||||
super().__init__()
|
||||
self.crawler = crawler
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(crawler)
|
||||
|
||||
def update_settings(self, settings):
|
||||
...
|
||||
|
||||
Use a fallback component:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
|
||||
|
||||
|
||||
FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER"
|
||||
|
||||
|
||||
class MyHandler:
|
||||
lazy = False
|
||||
|
||||
def __init__(self, settings, crawler):
|
||||
dhcls = load_object(settings.get(FALLBACK_SETTING))
|
||||
self._fallback_handler = create_instance(
|
||||
dhcls,
|
||||
settings=None,
|
||||
crawler=crawler,
|
||||
)
|
||||
|
||||
def download_request(self, request, spider):
|
||||
if request.meta.get("my_params"):
|
||||
# handle the request
|
||||
...
|
||||
else:
|
||||
return self._fallback_handler.download_request(request, spider)
|
||||
|
||||
|
||||
class MyAddon:
|
||||
def update_settings(self, settings):
|
||||
if not settings.get(FALLBACK_SETTING):
|
||||
settings.set(
|
||||
FALLBACK_SETTING,
|
||||
settings.getwithbase("DOWNLOAD_HANDLERS")["https"],
|
||||
"addon",
|
||||
)
|
||||
settings["DOWNLOAD_HANDLERS"]["https"] = MyHandler
|
||||
|
|
@ -137,6 +137,7 @@ Settings API
|
|||
SETTINGS_PRIORITIES = {
|
||||
"default": 0,
|
||||
"command": 10,
|
||||
"addon": 15,
|
||||
"project": 20,
|
||||
"spider": 30,
|
||||
"cmdline": 40,
|
||||
|
|
|
|||
|
|
@ -40,8 +40,9 @@ precedence:
|
|||
1. Command line options (most precedence)
|
||||
2. Settings per-spider
|
||||
3. Project settings module
|
||||
4. Default settings per-command
|
||||
5. Default global settings (less precedence)
|
||||
4. Settings set by add-ons
|
||||
5. Default settings per-command
|
||||
6. Default global settings (less precedence)
|
||||
|
||||
The population of these settings sources is taken care of internally, but a
|
||||
manual handling is possible using API calls. See the
|
||||
|
|
@ -89,7 +90,13 @@ project, it's where most of your custom settings will be populated. For a
|
|||
standard Scrapy project, this means you'll be adding or changing the settings
|
||||
in the ``settings.py`` file created for your project.
|
||||
|
||||
4. Default settings per-command
|
||||
4. Settings set by add-ons
|
||||
--------------------------
|
||||
|
||||
:ref:`Add-ons <topics-addons>` can modify settings. They should do this with
|
||||
this priority, though this is not enforced.
|
||||
|
||||
5. Default settings per-command
|
||||
-------------------------------
|
||||
|
||||
Each :doc:`Scrapy tool </topics/commands>` command can have its own default
|
||||
|
|
@ -97,7 +104,7 @@ settings, which override the global default settings. Those custom command
|
|||
settings are specified in the ``default_settings`` attribute of the command
|
||||
class.
|
||||
|
||||
5. Default global settings
|
||||
6. Default global settings
|
||||
--------------------------
|
||||
|
||||
The global defaults are located in the ``scrapy.settings.default_settings``
|
||||
|
|
@ -201,6 +208,16 @@ to any particular component. In that case the module of that component will be
|
|||
shown, typically an extension, middleware or pipeline. It also means that the
|
||||
component must be enabled in order for the setting to have any effect.
|
||||
|
||||
.. setting:: ADDONS
|
||||
|
||||
ADDONS
|
||||
------
|
||||
|
||||
Default: ``{}``
|
||||
|
||||
A dict containing paths to the add-ons enabled in your project and their
|
||||
priorities. For more information, see :ref:`topics-addons`.
|
||||
|
||||
.. setting:: AWS_ACCESS_KEY_ID
|
||||
|
||||
AWS_ACCESS_KEY_ID
|
||||
|
|
@ -964,7 +981,6 @@ some of them need to be enabled through a setting.
|
|||
For more information See the :ref:`extensions user guide <topics-extensions>`
|
||||
and the :ref:`list of available extensions <topics-extensions-ref>`.
|
||||
|
||||
|
||||
.. setting:: FEED_TEMPDIR
|
||||
|
||||
FEED_TEMPDIR
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
import logging
|
||||
from typing import TYPE_CHECKING, Any, List
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.conf import build_component_list
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy.crawler import Crawler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AddonManager:
|
||||
"""This class facilitates loading and storing :ref:`topics-addons`."""
|
||||
|
||||
def __init__(self, crawler: "Crawler") -> None:
|
||||
self.crawler: "Crawler" = crawler
|
||||
self.addons: List[Any] = []
|
||||
|
||||
def load_settings(self, settings: Settings) -> None:
|
||||
"""Load add-ons and configurations from a settings object.
|
||||
|
||||
This will load the add-on for every add-on path in the
|
||||
``ADDONS`` setting and execute their ``update_settings`` methods.
|
||||
|
||||
:param settings: The :class:`~scrapy.settings.Settings` object from \
|
||||
which to read the add-on configuration
|
||||
:type settings: :class:`~scrapy.settings.Settings`
|
||||
"""
|
||||
enabled: List[Any] = []
|
||||
for clspath in build_component_list(settings["ADDONS"]):
|
||||
try:
|
||||
addoncls = load_object(clspath)
|
||||
addon = create_instance(
|
||||
addoncls, settings=settings, crawler=self.crawler
|
||||
)
|
||||
addon.update_settings(settings)
|
||||
self.addons.append(addon)
|
||||
except NotConfigured as e:
|
||||
if e.args:
|
||||
logger.warning(
|
||||
"Disabled %(clspath)s: %(eargs)s",
|
||||
{"clspath": clspath, "eargs": e.args[0]},
|
||||
extra={"crawler": self.crawler},
|
||||
)
|
||||
logger.info(
|
||||
"Enabled addons:\n%(addons)s",
|
||||
{
|
||||
"addons": enabled,
|
||||
},
|
||||
extra={"crawler": self.crawler},
|
||||
)
|
||||
|
|
@ -18,6 +18,7 @@ except ImportError:
|
|||
from zope.interface.verify import verifyClass
|
||||
|
||||
from scrapy import Spider, signals
|
||||
from scrapy.addons import AddonManager
|
||||
from scrapy.core.engine import ExecutionEngine
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.extension import ExtensionManager
|
||||
|
|
@ -68,6 +69,9 @@ class Crawler:
|
|||
self.settings: Settings = settings.copy()
|
||||
self.spidercls.update_settings(self.settings)
|
||||
|
||||
self.addons: AddonManager = AddonManager(self)
|
||||
self.addons.load_settings(self.settings)
|
||||
|
||||
self.signals: SignalManager = SignalManager(self)
|
||||
|
||||
self.stats: StatsCollector = load_object(self.settings["STATS_CLASS"])(self)
|
||||
|
|
|
|||
|
|
@ -46,10 +46,9 @@ class MiddlewareManager:
|
|||
enabled.append(clspath)
|
||||
except NotConfigured as e:
|
||||
if e.args:
|
||||
clsname = clspath.split(".")[-1]
|
||||
logger.warning(
|
||||
"Disabled %(clsname)s: %(eargs)s",
|
||||
{"clsname": clsname, "eargs": e.args[0]},
|
||||
"Disabled %(clspath)s: %(eargs)s",
|
||||
{"clspath": clspath, "eargs": e.args[0]},
|
||||
extra={"crawler": crawler},
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ if TYPE_CHECKING:
|
|||
SETTINGS_PRIORITIES: Dict[str, int] = {
|
||||
"default": 0,
|
||||
"command": 10,
|
||||
"addon": 15,
|
||||
"project": 20,
|
||||
"spider": 30,
|
||||
"cmdline": 40,
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@ import sys
|
|||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
|
||||
ADDONS = {}
|
||||
|
||||
AJAXCRAWL_ENABLED = False
|
||||
|
||||
ASYNCIO_EVENT_LOOP = None
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ def load_object(path: Union[str, Callable]) -> Any:
|
|||
if callable(path):
|
||||
return path
|
||||
raise TypeError(
|
||||
"Unexpected argument type, expected string " f"or object, got: {type(path)}"
|
||||
f"Unexpected argument type, expected string or object, got: {type(path)}"
|
||||
)
|
||||
|
||||
try:
|
||||
|
|
|
|||
113
sep/sep-021.rst
113
sep/sep-021.rst
|
|
@ -1,113 +0,0 @@
|
|||
======= ===================
|
||||
SEP 21
|
||||
Title Add-ons
|
||||
Author Pablo Hoffman
|
||||
Created 2014-02-14
|
||||
Status Draft
|
||||
======= ===================
|
||||
|
||||
================
|
||||
SEP-021: Add-ons
|
||||
================
|
||||
|
||||
This proposal introduces add-ons, a unified way to manage Scrapy extensions,
|
||||
middlewares and pipelines.
|
||||
|
||||
Scrapy currently supports many hooks and mechanisms for extending its
|
||||
functionality, but no single entry point for enabling and configuring them.
|
||||
Instead, the hooks are spread over:
|
||||
|
||||
* Spider middlewares (SPIDER_MIDDLEWARES)
|
||||
* Downloader middlewares (DOWNLOADER_MIDDLEWARES)
|
||||
* Downloader handlers (DOWNLOADER_HANDLERS)
|
||||
* Item pipelines (ITEM_PIPELINES)
|
||||
* Feed exporters and storages (FEED_EXPORTERS, FEED_STORAGES)
|
||||
* Overridable components (DUPEFILTER_CLASS, STATS_CLASS, SCHEDULER, SPIDER_MANAGER_CLASS, ITEM_PROCESSOR, etc)
|
||||
* Generic extensions (EXTENSIONS)
|
||||
* CLI commands (COMMANDS_MODULE)
|
||||
|
||||
One problem of this approach is that enabling an extension often requires
|
||||
modifying many settings, often in a coordinated way, which is complex and error
|
||||
prone. Add-ons are meant to fix this by providing a simple mechanism for
|
||||
enabling extensions.
|
||||
|
||||
Design goals and non-goals
|
||||
==========================
|
||||
|
||||
Goals:
|
||||
|
||||
* simple to manage: adding or removing extensions should be just a matter of
|
||||
adding or removing lines in a ``scrapy.cfg`` file
|
||||
* backward compatibility with enabling extension the "old way" (i.e. modifying
|
||||
settings directly)
|
||||
|
||||
Non-goals:
|
||||
|
||||
* a way to publish, distribute or discover extensions (use pypi for that)
|
||||
|
||||
|
||||
Managing add-ons
|
||||
================
|
||||
|
||||
Add-ons are defined in the ``scrapy.cfg`` file, inside the ``[addons]``
|
||||
section.
|
||||
|
||||
To enable the "httpcache" addon, either shipped with Scrapy or in the Python
|
||||
search path, create an entry for it in your ``scrapy.cfg``, like this::
|
||||
|
||||
[addons]
|
||||
httpcache =
|
||||
|
||||
You may also specify the full path to an add-on (which may be either a .py file
|
||||
or a folder containing __init__.py)::
|
||||
|
||||
[addons]
|
||||
mongodb_pipeline = /path/to/mongodb_pipeline.py
|
||||
|
||||
|
||||
Writing add-ons
|
||||
===============
|
||||
|
||||
Add-ons are Python modules that implement the following callbacks.
|
||||
|
||||
addon_configure
|
||||
---------------
|
||||
|
||||
Receives the Settings object and modifies it to enable the required components.
|
||||
If it raises an exception, Scrapy will print it and exit.
|
||||
|
||||
Examples:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def addon_configure(settings):
|
||||
settings.overrides["DOWNLOADER_MIDDLEWARES"].update(
|
||||
{
|
||||
"scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware": 900,
|
||||
}
|
||||
)
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def addon_configure(settings):
|
||||
try:
|
||||
import boto
|
||||
except ImportError:
|
||||
raise RuntimeError("boto library is required")
|
||||
|
||||
|
||||
crawler_ready
|
||||
-------------
|
||||
|
||||
``crawler_ready`` receives a Crawler object after it has been initialized and
|
||||
is meant to be used to perform post-initialization checks like making sure the
|
||||
extension and its dependencies were configured properly. If it raises an
|
||||
exception, Scrapy will print and exit.
|
||||
|
||||
Examples:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def crawler_ready(crawler):
|
||||
if "some.other.addon" not in crawler.extensions.enabled:
|
||||
raise RuntimeError("Some other addon is required to use this addon")
|
||||
|
|
@ -0,0 +1,158 @@
|
|||
import itertools
|
||||
import unittest
|
||||
from typing import Any, Dict
|
||||
|
||||
from scrapy import Spider
|
||||
from scrapy.crawler import Crawler, CrawlerRunner
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.settings import BaseSettings, Settings
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class SimpleAddon:
|
||||
def update_settings(self, settings):
|
||||
pass
|
||||
|
||||
|
||||
def get_addon_cls(config: Dict[str, Any]) -> type:
|
||||
class AddonWithConfig:
|
||||
def update_settings(self, settings: BaseSettings):
|
||||
settings.update(config, priority="addon")
|
||||
|
||||
return AddonWithConfig
|
||||
|
||||
|
||||
class CreateInstanceAddon:
|
||||
def __init__(self, crawler: Crawler) -> None:
|
||||
super().__init__()
|
||||
self.crawler = crawler
|
||||
self.config = crawler.settings.getdict("MYADDON")
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler):
|
||||
return cls(crawler)
|
||||
|
||||
def update_settings(self, settings):
|
||||
settings.update(self.config, "addon")
|
||||
|
||||
|
||||
class AddonTest(unittest.TestCase):
|
||||
def test_update_settings(self):
|
||||
settings = BaseSettings()
|
||||
settings.set("KEY1", "default", priority="default")
|
||||
settings.set("KEY2", "project", priority="project")
|
||||
addon_config = {"KEY1": "addon", "KEY2": "addon", "KEY3": "addon"}
|
||||
testaddon = get_addon_cls(addon_config)()
|
||||
testaddon.update_settings(settings)
|
||||
self.assertEqual(settings["KEY1"], "addon")
|
||||
self.assertEqual(settings["KEY2"], "project")
|
||||
self.assertEqual(settings["KEY3"], "addon")
|
||||
|
||||
|
||||
class AddonManagerTest(unittest.TestCase):
|
||||
def test_load_settings(self):
|
||||
settings_dict = {
|
||||
"ADDONS": {"tests.test_addons.SimpleAddon": 0},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
manager = crawler.addons
|
||||
self.assertIsInstance(manager.addons[0], SimpleAddon)
|
||||
|
||||
def test_notconfigured(self):
|
||||
class NotConfiguredAddon:
|
||||
def update_settings(self, settings):
|
||||
raise NotConfigured()
|
||||
|
||||
settings_dict = {
|
||||
"ADDONS": {NotConfiguredAddon: 0},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
manager = crawler.addons
|
||||
self.assertFalse(manager.addons)
|
||||
|
||||
def test_load_settings_order(self):
|
||||
# Get three addons with different settings
|
||||
addonlist = []
|
||||
for i in range(3):
|
||||
addon = get_addon_cls({"KEY1": i})
|
||||
addon.number = i
|
||||
addonlist.append(addon)
|
||||
# Test for every possible ordering
|
||||
for ordered_addons in itertools.permutations(addonlist):
|
||||
expected_order = [a.number for a in ordered_addons]
|
||||
settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
manager = crawler.addons
|
||||
self.assertEqual([a.number for a in manager.addons], expected_order)
|
||||
self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1])
|
||||
|
||||
def test_create_instance(self):
|
||||
settings_dict = {
|
||||
"ADDONS": {"tests.test_addons.CreateInstanceAddon": 0},
|
||||
"MYADDON": {"MYADDON_KEY": "val"},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
manager = crawler.addons
|
||||
self.assertIsInstance(manager.addons[0], CreateInstanceAddon)
|
||||
self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val")
|
||||
|
||||
def test_settings_priority(self):
|
||||
config = {
|
||||
"KEY": 15, # priority=addon
|
||||
}
|
||||
settings_dict = {
|
||||
"ADDONS": {get_addon_cls(config): 1},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
self.assertEqual(crawler.settings.getint("KEY"), 15)
|
||||
|
||||
settings = Settings(settings_dict)
|
||||
settings.set("KEY", 0, priority="default")
|
||||
runner = CrawlerRunner(settings)
|
||||
crawler = runner.create_crawler(Spider)
|
||||
self.assertEqual(crawler.settings.getint("KEY"), 15)
|
||||
|
||||
settings_dict = {
|
||||
"KEY": 20, # priority=project
|
||||
"ADDONS": {get_addon_cls(config): 1},
|
||||
}
|
||||
settings = Settings(settings_dict)
|
||||
settings.set("KEY", 0, priority="default")
|
||||
runner = CrawlerRunner(settings)
|
||||
crawler = runner.create_crawler(Spider)
|
||||
self.assertEqual(crawler.settings.getint("KEY"), 20)
|
||||
|
||||
def test_fallback_workflow(self):
|
||||
FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER"
|
||||
|
||||
class AddonWithFallback:
|
||||
def update_settings(self, settings):
|
||||
if not settings.get(FALLBACK_SETTING):
|
||||
settings.set(
|
||||
FALLBACK_SETTING,
|
||||
settings.getwithbase("DOWNLOAD_HANDLERS")["https"],
|
||||
"addon",
|
||||
)
|
||||
settings["DOWNLOAD_HANDLERS"]["https"] = "AddonHandler"
|
||||
|
||||
settings_dict = {
|
||||
"ADDONS": {AddonWithFallback: 1},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
self.assertEqual(
|
||||
crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler"
|
||||
)
|
||||
self.assertEqual(
|
||||
crawler.settings.get(FALLBACK_SETTING),
|
||||
"scrapy.core.downloader.handlers.http.HTTPDownloadHandler",
|
||||
)
|
||||
|
||||
settings_dict = {
|
||||
"ADDONS": {AddonWithFallback: 1},
|
||||
"DOWNLOAD_HANDLERS": {"https": "UserHandler"},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
self.assertEqual(
|
||||
crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler"
|
||||
)
|
||||
self.assertEqual(crawler.settings.get(FALLBACK_SETTING), "UserHandler")
|
||||
|
|
@ -39,7 +39,7 @@ class MOff:
|
|||
pass
|
||||
|
||||
def __init__(self):
|
||||
raise NotConfigured
|
||||
raise NotConfigured("foo")
|
||||
|
||||
|
||||
class TestMiddlewareManager(MiddlewareManager):
|
||||
|
|
|
|||
|
|
@ -296,3 +296,7 @@ class UpdateClassPathTest(unittest.TestCase):
|
|||
output = update_classpath("scrapy.unmatched.Path")
|
||||
self.assertEqual(output, "scrapy.unmatched.Path")
|
||||
self.assertEqual(len(w), 0)
|
||||
|
||||
def test_returns_nonstring(self):
|
||||
for notastring in [None, True, [1, 2, 3], object()]:
|
||||
self.assertEqual(update_classpath(notastring), notastring)
|
||||
|
|
|
|||
Loading…
Reference in New Issue