mirror of https://github.com/scrapy/scrapy.git
Deprecate _BASE settings, unify _BASE backwards-compatibility
This commit is contained in:
Jakob de Maeyer
parent
a769a1ef78
commit
26586ef5a6
|
|
@ -23,22 +23,20 @@ Here's an example::
|
|||
'myproject.middlewares.CustomDownloaderMiddleware': 543,
|
||||
}
|
||||
|
||||
The :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the
|
||||
:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting defined in Scrapy (and not meant to
|
||||
be overridden) and then sorted by order to get the final sorted list of enabled
|
||||
middlewares: the first middleware is the one closer to the engine and the last
|
||||
is the one closer to the downloader.
|
||||
The specified :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the
|
||||
default one (i.e. it does not overwrite it) and then sorted by order to get the
|
||||
final sorted list of enabled middlewares: the first middleware is the one
|
||||
closer to the engine and the last is the one closer to the downloader.
|
||||
|
||||
To decide which order to assign to your middleware see the
|
||||
:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting and pick a value according to
|
||||
To decide which order to assign to your middleware see the default
|
||||
:setting:`DOWNLOADER_MIDDLEWARES` setting and pick a value according to
|
||||
where you want to insert the middleware. The order does matter because each
|
||||
middleware performs a different action and your middleware could depend on some
|
||||
previous (or subsequent) middleware being applied.
|
||||
|
||||
If you want to disable a built-in middleware (the ones defined in
|
||||
:setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it
|
||||
in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign `None`
|
||||
as its value. For example, if you want to disable the user-agent middleware::
|
||||
If you want to disable a built-in middleware you must define it in your
|
||||
project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None`` as its
|
||||
value. For example, if you want to disable the user-agent middleware::
|
||||
|
||||
DOWNLOADER_MIDDLEWARES = {
|
||||
'myproject.middlewares.CustomDownloaderMiddleware': 543,
|
||||
|
|
@ -164,7 +162,7 @@ middleware, see the :ref:`downloader middleware usage guide
|
|||
<topics-downloader-middleware>`.
|
||||
|
||||
For a list of the components enabled by default (and their orders) see the
|
||||
:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting.
|
||||
:setting:`DOWNLOADER_MIDDLEWARES` setting.
|
||||
|
||||
.. _cookies-mw:
|
||||
|
||||
|
|
|
|||
|
|
@ -42,17 +42,15 @@ by a string: the full Python path to the extension's class name. For example::
|
|||
|
||||
As you can see, the :setting:`EXTENSIONS` setting is a dict where the keys are
|
||||
the extension paths, and their values are the orders, which define the
|
||||
extension *loading* order. Extensions orders are not as important as middleware
|
||||
orders though, and they are typically irrelevant, ie. it doesn't matter in
|
||||
which order the extensions are loaded because they don't depend on each other
|
||||
[1].
|
||||
extension *loading* order. The specified :setting:`EXTENSIONS` setting is merged
|
||||
with the default one (i.e. it does not overwrite it) and then sorted by order
|
||||
to get the final sorted list of enabled extensions.
|
||||
|
||||
However, this feature can be exploited if you need to add an extension which
|
||||
depends on other extensions already loaded.
|
||||
|
||||
[1] This is is why the :setting:`EXTENSIONS_BASE` setting in Scrapy (which
|
||||
contains all built-in extensions enabled by default) defines all the extensions
|
||||
with the same order (``500``).
|
||||
As extensions typically do not depend on each other, their loading order is
|
||||
irrelevant in most cases. This is why the default :setting:`EXTENSIONS` setting
|
||||
defines all extensions with the same order (``500``). However, this feature can
|
||||
be exploited if you need to add an extension which depends on other extensions
|
||||
already loaded.
|
||||
|
||||
Available, enabled and disabled extensions
|
||||
==========================================
|
||||
|
|
@ -65,7 +63,7 @@ Disabling an extension
|
|||
======================
|
||||
|
||||
In order to disable an extension that comes enabled by default (ie. those
|
||||
included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to
|
||||
included in the default :setting:`EXTENSIONS` setting) you must set its order to
|
||||
``None``. For example::
|
||||
|
||||
EXTENSIONS = {
|
||||
|
|
|
|||
|
|
@ -265,16 +265,6 @@ Whether to export empty feeds (ie. feeds with no items).
|
|||
FEED_STORAGES
|
||||
-------------
|
||||
|
||||
Default:: ``{}``
|
||||
|
||||
A dict containing additional feed storage backends supported by your project.
|
||||
The keys are URI schemes and the values are paths to storage classes.
|
||||
|
||||
.. setting:: FEED_STORAGES_BASE
|
||||
|
||||
FEED_STORAGES_BASE
|
||||
------------------
|
||||
|
||||
Default::
|
||||
|
||||
{
|
||||
|
|
@ -285,36 +275,39 @@ Default::
|
|||
'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
|
||||
}
|
||||
|
||||
A dict containing the built-in feed storage backends supported by Scrapy.
|
||||
A dict containing all feed storage backends supported by your project. The keys
|
||||
are URI schemes and the values are paths to storage classes.
|
||||
|
||||
When you set :setting:`FEED_STORAGES` manually, e.g. in your project's settings
|
||||
module, it will be merged with the default, not overwrite it. If you want to
|
||||
disable any of the default feed storage backends, you must assign ``None`` as
|
||||
their value.
|
||||
|
||||
.. setting:: FEED_EXPORTERS
|
||||
|
||||
FEED_EXPORTERS
|
||||
--------------
|
||||
|
||||
Default:: ``{}``
|
||||
|
||||
A dict containing additional exporters supported by your project. The keys are
|
||||
URI schemes and the values are paths to :ref:`Item exporter <topics-exporters>`
|
||||
classes.
|
||||
|
||||
.. setting:: FEED_EXPORTERS_BASE
|
||||
|
||||
FEED_EXPORTERS_BASE
|
||||
-------------------
|
||||
|
||||
Default::
|
||||
|
||||
FEED_EXPORTERS_BASE = {
|
||||
{
|
||||
'json': 'scrapy.exporters.JsonItemExporter',
|
||||
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
|
||||
'jl': 'scrapy.exporters.JsonLinesItemExporter',
|
||||
'csv': 'scrapy.exporters.CsvItemExporter',
|
||||
'xml': 'scrapy.exporters.XmlItemExporter',
|
||||
'marshal': 'scrapy.exporters.MarshalItemExporter',
|
||||
'pickle': 'scrapy.exporters.PickleItemExporter',
|
||||
}
|
||||
|
||||
A dict containing the built-in feed exporters supported by Scrapy.
|
||||
A dict containing all feed exporters supported by your project. The keys are
|
||||
URI schemes and the values are paths to :ref:`Item exporter <topics-exporters>`
|
||||
classes.
|
||||
|
||||
When you set :setting:`FEED_EXPORTERS` manually, e.g. in your project's settings
|
||||
module, it will be merged with the default, not overwrite it. If you want to
|
||||
disable any of the default feed exporters, you must assign ``None`` as their
|
||||
value.
|
||||
|
||||
.. _URI: http://en.wikipedia.org/wiki/Uniform_Resource_Identifier
|
||||
.. _Amazon S3: http://aws.amazon.com/s3/
|
||||
|
|
|
|||
|
|
@ -269,6 +269,11 @@ Default::
|
|||
The default headers used for Scrapy HTTP Requests. They're populated in the
|
||||
:class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`.
|
||||
|
||||
When you set :setting:`DEFAULT_REQUEST_HEADERS` manually, e.g. in your
|
||||
project's settings module, it will be merged with the default, not overwrite it.
|
||||
If you want to disable any of the default request headers (and not replace them)
|
||||
you must assign ``None`` as their value.
|
||||
|
||||
.. setting:: DEPTH_LIMIT
|
||||
|
||||
DEPTH_LIMIT
|
||||
|
|
@ -350,16 +355,6 @@ The downloader to use for crawling.
|
|||
DOWNLOADER_MIDDLEWARES
|
||||
----------------------
|
||||
|
||||
Default:: ``{}``
|
||||
|
||||
A dict containing the downloader middlewares enabled in your project, and their
|
||||
orders. For more info see :ref:`topics-downloader-middleware-setting`.
|
||||
|
||||
.. setting:: DOWNLOADER_MIDDLEWARES_BASE
|
||||
|
||||
DOWNLOADER_MIDDLEWARES_BASE
|
||||
---------------------------
|
||||
|
||||
Default::
|
||||
|
||||
{
|
||||
|
|
@ -369,6 +364,7 @@ Default::
|
|||
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400,
|
||||
'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,
|
||||
'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550,
|
||||
'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
|
||||
'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
|
||||
'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,
|
||||
'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
|
||||
|
|
@ -379,10 +375,16 @@ Default::
|
|||
'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
|
||||
}
|
||||
|
||||
A dict containing the downloader middlewares enabled by default in Scrapy. You
|
||||
should never modify this setting in your project, modify
|
||||
:setting:`DOWNLOADER_MIDDLEWARES` instead. For more info see
|
||||
:ref:`topics-downloader-middleware-setting`.
|
||||
A dict containing the downloader middlewares enabled in your project, and their
|
||||
orders. Low orders are closer to the engine, high orders are closer to the
|
||||
downloader.
|
||||
|
||||
When you set :setting:`DOWNLOADER_MIDDLEWARES` manually, e.g. in your project's
|
||||
settings module, it will be merged with the default, not overwrite it. If you
|
||||
want to disable any of the default downloader middlewares you must assign
|
||||
``None`` as their value.
|
||||
|
||||
For more info see :ref:`topics-downloader-middleware-setting`.
|
||||
|
||||
.. setting:: DOWNLOADER_STATS
|
||||
|
||||
|
|
@ -423,33 +425,23 @@ spider attribute.
|
|||
DOWNLOAD_HANDLERS
|
||||
-----------------
|
||||
|
||||
Default: ``{}``
|
||||
|
||||
A dict containing the request downloader handlers enabled in your project.
|
||||
See `DOWNLOAD_HANDLERS_BASE` for example format.
|
||||
|
||||
.. setting:: DOWNLOAD_HANDLERS_BASE
|
||||
|
||||
DOWNLOAD_HANDLERS_BASE
|
||||
----------------------
|
||||
|
||||
Default::
|
||||
|
||||
{
|
||||
'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
|
||||
'http': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler',
|
||||
'https': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler',
|
||||
'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
|
||||
'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
|
||||
's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler',
|
||||
'ftp': 'scrapy.core.downloader.handlers.ftp.FTPDownloadHandler',
|
||||
}
|
||||
|
||||
A dict containing the request download handlers enabled by default in Scrapy.
|
||||
You should never modify this setting in your project, modify
|
||||
:setting:`DOWNLOAD_HANDLERS` instead.
|
||||
|
||||
If you want to disable any of the above download handlers you must define them
|
||||
in your project's :setting:`DOWNLOAD_HANDLERS` setting and assign `None`
|
||||
as their value. For example, if you want to disable the file download
|
||||
handler::
|
||||
A dict containing the request downloader handlers enabled in your project.
|
||||
|
||||
When you set :setting:`DOWNLOAD_HANDLERS` manually, e.g. in your project's
|
||||
settings module, it will be merged with the default, not overwrite it. If you
|
||||
want to disable any of the default download handlers you must assign ``None``
|
||||
as their value. For example, if you want to disable the file download handler::
|
||||
|
||||
DOWNLOAD_HANDLERS = {
|
||||
'file': None,
|
||||
|
|
@ -552,15 +544,6 @@ to ``vi`` (on Unix systems) or the IDLE editor (on Windows).
|
|||
EXTENSIONS
|
||||
----------
|
||||
|
||||
Default:: ``{}``
|
||||
|
||||
A dict containing the extensions enabled in your project, and their orders.
|
||||
|
||||
.. setting:: EXTENSIONS_BASE
|
||||
|
||||
EXTENSIONS_BASE
|
||||
---------------
|
||||
|
||||
Default::
|
||||
|
||||
{
|
||||
|
|
@ -575,13 +558,19 @@ Default::
|
|||
'scrapy.extensions.throttle.AutoThrottle': 0,
|
||||
}
|
||||
|
||||
The list of available extensions. Keep in mind that some of them need to
|
||||
be enabled through a setting. By default, this setting contains all stable
|
||||
built-in extensions.
|
||||
A dict containing the extensions enabled in your project, and their orders. By
|
||||
default, this setting contains all stable built-in extensions. Keep in mind that
|
||||
some of them need to be enabled through a setting.
|
||||
|
||||
When you set :setting:`EXTENSIONS` manually, e.g. in your project's settings
|
||||
module, it will be merged with the default, not overwrite it. If you want to
|
||||
disable any of the default enabled extensions you must assign ``None`` as their
|
||||
value.
|
||||
|
||||
For more information See the :ref:`extensions user guide <topics-extensions>`
|
||||
and the :ref:`list of available extensions <topics-extensions-ref>`.
|
||||
|
||||
|
||||
.. setting:: ITEM_PIPELINES
|
||||
|
||||
ITEM_PIPELINES
|
||||
|
|
@ -589,12 +578,9 @@ ITEM_PIPELINES
|
|||
|
||||
Default: ``{}``
|
||||
|
||||
A dict containing the item pipelines to use, and their orders. The dict is
|
||||
empty by default order values are arbitrary but it's customary to define them
|
||||
in the 0-1000 range.
|
||||
|
||||
Lists are supported in :setting:`ITEM_PIPELINES` for backwards compatibility,
|
||||
but they are deprecated.
|
||||
A dict containing the item pipelines to use, and their orders. Order values are
|
||||
arbitrary, but it is customary to define them in the 0-1000 range. Lower orders
|
||||
process before higher orders.
|
||||
|
||||
Example::
|
||||
|
||||
|
|
@ -603,16 +589,6 @@ Example::
|
|||
'mybot.pipelines.validate.StoreMyItem': 800,
|
||||
}
|
||||
|
||||
.. setting:: ITEM_PIPELINES_BASE
|
||||
|
||||
ITEM_PIPELINES_BASE
|
||||
-------------------
|
||||
|
||||
Default: ``{}``
|
||||
|
||||
A dict containing the pipelines enabled by default in Scrapy. You should never
|
||||
modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead.
|
||||
|
||||
.. setting:: LOG_ENABLED
|
||||
|
||||
LOG_ENABLED
|
||||
|
|
@ -638,7 +614,7 @@ LOG_FILE
|
|||
|
||||
Default: ``None``
|
||||
|
||||
File name to use for logging output. If None, standard error will be used.
|
||||
File name to use for logging output. If ``None``, standard error will be used.
|
||||
|
||||
.. setting:: LOG_FORMAT
|
||||
|
||||
|
|
@ -902,16 +878,6 @@ The scheduler to use for crawling.
|
|||
SPIDER_CONTRACTS
|
||||
----------------
|
||||
|
||||
Default:: ``{}``
|
||||
|
||||
A dict containing the scrapy contracts enabled in your project, used for
|
||||
testing spiders. For more info see :ref:`topics-contracts`.
|
||||
|
||||
.. setting:: SPIDER_CONTRACTS_BASE
|
||||
|
||||
SPIDER_CONTRACTS_BASE
|
||||
---------------------
|
||||
|
||||
Default::
|
||||
|
||||
{
|
||||
|
|
@ -920,9 +886,13 @@ Default::
|
|||
'scrapy.contracts.default.ScrapesContract': 3,
|
||||
}
|
||||
|
||||
A dict containing the scrapy contracts enabled by default in Scrapy. You should
|
||||
never modify this setting in your project, modify :setting:`SPIDER_CONTRACTS`
|
||||
instead. For more info see :ref:`topics-contracts`.
|
||||
A dict containing the scrapy contracts enabled in your project, used for
|
||||
testing spiders. For more info see :ref:`topics-contracts`.
|
||||
|
||||
When you set :setting:`SPIDER_CONTRACTS` manually, e.g. in your project's
|
||||
settings module, it will be merged with the default, not overwrite it. If you
|
||||
want to disable any of the default contracts you must assign ``None`` as their
|
||||
value.
|
||||
|
||||
.. setting:: SPIDER_LOADER_CLASS
|
||||
|
||||
|
|
@ -939,16 +909,6 @@ The class that will be used for loading spiders, which must implement the
|
|||
SPIDER_MIDDLEWARES
|
||||
------------------
|
||||
|
||||
Default:: ``{}``
|
||||
|
||||
A dict containing the spider middlewares enabled in your project, and their
|
||||
orders. For more info see :ref:`topics-spider-middleware-setting`.
|
||||
|
||||
.. setting:: SPIDER_MIDDLEWARES_BASE
|
||||
|
||||
SPIDER_MIDDLEWARES_BASE
|
||||
-----------------------
|
||||
|
||||
Default::
|
||||
|
||||
{
|
||||
|
|
@ -959,10 +919,14 @@ Default::
|
|||
'scrapy.spidermiddlewares.depth.DepthMiddleware': 900,
|
||||
}
|
||||
|
||||
A dict containing the spider middlewares enabled by default in Scrapy. You
|
||||
should never modify this setting in your project, modify
|
||||
:setting:`SPIDER_MIDDLEWARES` instead. For more info see
|
||||
:ref:`topics-spider-middleware-setting`.
|
||||
A dict containing the spider middlewares enabled in your project, and their
|
||||
orders. Low orders are closer to the engine, high orders are closer to the
|
||||
spider. For more info see :ref:`topics-spider-middleware-setting`.
|
||||
|
||||
When you set :setting:`SPIDER_MIDDLEWARES` manually, e.g. in your project's
|
||||
settings module, it will be merged with the default, not overwrite it. If you
|
||||
want to disable any of the default spider middlewares you must assign ``None``
|
||||
as their value.
|
||||
|
||||
.. setting:: SPIDER_MODULES
|
||||
|
||||
|
|
|
|||
|
|
@ -24,22 +24,20 @@ Here's an example::
|
|||
'myproject.middlewares.CustomSpiderMiddleware': 543,
|
||||
}
|
||||
|
||||
The :setting:`SPIDER_MIDDLEWARES` setting is merged with the
|
||||
:setting:`SPIDER_MIDDLEWARES_BASE` setting defined in Scrapy (and not meant to
|
||||
be overridden) and then sorted by order to get the final sorted list of enabled
|
||||
middlewares: the first middleware is the one closer to the engine and the last
|
||||
is the one closer to the spider.
|
||||
The specified :setting:`SPIDER_MIDDLEWARES` setting is merged with the default
|
||||
one (i.e. it does not overwrite it) and then sorted by order to get the final
|
||||
sorted list of enabled middlewares: the first middleware is the one closer to
|
||||
the engine and the last is the one closer to the spider.
|
||||
|
||||
To decide which order to assign to your middleware see the
|
||||
:setting:`SPIDER_MIDDLEWARES_BASE` setting and pick a value according to where
|
||||
To decide which order to assign to your middleware see the default
|
||||
:setting:`SPIDER_MIDDLEWARES` setting and pick a value according to where
|
||||
you want to insert the middleware. The order does matter because each
|
||||
middleware performs a different action and your middleware could depend on some
|
||||
previous (or subsequent) middleware being applied.
|
||||
|
||||
If you want to disable a builtin middleware (the ones defined in
|
||||
:setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
|
||||
in your project :setting:`SPIDER_MIDDLEWARES` setting and assign `None` as its
|
||||
value. For example, if you want to disable the off-site middleware::
|
||||
If you want to disable a builtin middleware you must define it in your project's
|
||||
:setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its value. For
|
||||
example, if you want to disable the off-site middleware::
|
||||
|
||||
SPIDER_MIDDLEWARES = {
|
||||
'myproject.middlewares.CustomSpiderMiddleware': 543,
|
||||
|
|
@ -173,7 +171,7 @@ information on how to use them and how to write your own spider middleware, see
|
|||
the :ref:`spider middleware usage guide <topics-spider-middleware>`.
|
||||
|
||||
For a list of the components enabled by default (and their orders) see the
|
||||
:setting:`SPIDER_MIDDLEWARES_BASE` setting.
|
||||
:setting:`SPIDER_MIDDLEWARES` setting.
|
||||
|
||||
DepthMiddleware
|
||||
---------------
|
||||
|
|
|
|||
|
|
@ -58,10 +58,7 @@ class Command(ScrapyCommand):
|
|||
|
||||
def run(self, args, opts):
|
||||
# load contracts
|
||||
contracts = build_component_list(
|
||||
self.settings['SPIDER_CONTRACTS_BASE'],
|
||||
self.settings['SPIDER_CONTRACTS'],
|
||||
)
|
||||
contracts = build_component_list(self.settings._getcomposite('SPIDER_CONTRACTS'))
|
||||
conman = ContractsManager(load_object(c) for c in contracts)
|
||||
runner = TextTestRunner(verbosity=2 if opts.verbose else 1)
|
||||
result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import os
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.utils.conf import arglist_to_dict
|
||||
from scrapy.utils.conf import arglist_to_dict, remove_none_values
|
||||
from scrapy.exceptions import UsageError
|
||||
|
||||
|
||||
|
|
@ -34,10 +34,8 @@ class Command(ScrapyCommand):
|
|||
self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
|
||||
else:
|
||||
self.settings.set('FEED_URI', opts.output, priority='cmdline')
|
||||
valid_output_formats = (
|
||||
list(self.settings.getdict('FEED_EXPORTERS').keys()) +
|
||||
list(self.settings.getdict('FEED_EXPORTERS_BASE').keys())
|
||||
)
|
||||
feed_exporters = remove_none_values(self.settings._getcomposite('FEED_EXPORTERS'))
|
||||
valid_output_formats = feed_exporters.keys()
|
||||
if not opts.output_format:
|
||||
opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
|
||||
if opts.output_format not in valid_output_formats:
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from importlib import import_module
|
|||
from scrapy.utils.spider import iter_spider_classes
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.utils.conf import arglist_to_dict
|
||||
from scrapy.utils.conf import arglist_to_dict, remove_none_values
|
||||
|
||||
|
||||
def _import_file(filepath):
|
||||
|
|
@ -57,10 +57,8 @@ class Command(ScrapyCommand):
|
|||
self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
|
||||
else:
|
||||
self.settings.set('FEED_URI', opts.output, priority='cmdline')
|
||||
valid_output_formats = (
|
||||
list(self.settings.getdict('FEED_EXPORTERS').keys()) +
|
||||
list(self.settings.getdict('FEED_EXPORTERS_BASE').keys())
|
||||
)
|
||||
feed_exporters = remove_none_values(self.settings._getcomposite('FEED_EXPORTERS'))
|
||||
valid_output_formats = feed_exporters.keys()
|
||||
if not opts.output_format:
|
||||
opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
|
||||
if opts.output_format not in valid_output_formats:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import logging
|
|||
from twisted.internet import defer
|
||||
import six
|
||||
from scrapy.exceptions import NotSupported, NotConfigured
|
||||
from scrapy.utils.conf import remove_none_values
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy import signals
|
||||
|
|
@ -19,13 +20,8 @@ class DownloadHandlers(object):
|
|||
self._schemes = {} # stores acceptable schemes on instancing
|
||||
self._handlers = {} # stores instanced handlers for schemes
|
||||
self._notconfigured = {} # remembers failed handlers
|
||||
handlers = crawler.settings.get('DOWNLOAD_HANDLERS_BASE')
|
||||
handlers.update(crawler.settings.get('DOWNLOAD_HANDLERS', {}))
|
||||
handlers = remove_none_values(crawler.settings._getcomposite('DOWNLOAD_HANDLERS'))
|
||||
for scheme, clspath in six.iteritems(handlers):
|
||||
# Allow to disable a handler just like any other
|
||||
# component (extension, middleware, etc).
|
||||
if clspath is None:
|
||||
continue
|
||||
self._schemes[scheme] = clspath
|
||||
|
||||
crawler.signals.connect(self._close, signals.engine_stopped)
|
||||
|
|
|
|||
|
|
@ -19,8 +19,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
|
||||
@classmethod
|
||||
def _get_mwlist_from_settings(cls, settings):
|
||||
return build_component_list(settings['DOWNLOADER_MIDDLEWARES_BASE'], \
|
||||
settings['DOWNLOADER_MIDDLEWARES'])
|
||||
return build_component_list(settings._getcomposite('DOWNLOADER_MIDDLEWARES'))
|
||||
|
||||
def _add_middleware(self, mw):
|
||||
if hasattr(mw, 'process_request'):
|
||||
|
|
|
|||
|
|
@ -18,8 +18,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
|
||||
@classmethod
|
||||
def _get_mwlist_from_settings(cls, settings):
|
||||
return build_component_list(settings['SPIDER_MIDDLEWARES_BASE'], \
|
||||
settings['SPIDER_MIDDLEWARES'])
|
||||
return build_component_list(settings._getcomposite('SPIDER_MIDDLEWARES'))
|
||||
|
||||
def _add_middleware(self, mw):
|
||||
super(SpiderMiddlewareManager, self)._add_middleware(mw)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ DefaultHeaders downloader middleware
|
|||
See documentation in docs/topics/downloader-middleware.rst
|
||||
"""
|
||||
|
||||
from scrapy.utils.conf import remove_none_values
|
||||
|
||||
|
||||
class DefaultHeadersMiddleware(object):
|
||||
|
||||
|
|
@ -12,7 +14,8 @@ class DefaultHeadersMiddleware(object):
|
|||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(crawler.settings.get('DEFAULT_REQUEST_HEADERS').items())
|
||||
headers = remove_none_values(crawler.settings['DEFAULT_REQUEST_HEADERS'])
|
||||
return cls(headers.items())
|
||||
|
||||
def process_request(self, request, spider):
|
||||
for k, v in self._headers:
|
||||
|
|
|
|||
|
|
@ -12,5 +12,4 @@ class ExtensionManager(MiddlewareManager):
|
|||
|
||||
@classmethod
|
||||
def _get_mwlist_from_settings(cls, settings):
|
||||
return build_component_list(settings['EXTENSIONS_BASE'], \
|
||||
settings['EXTENSIONS'])
|
||||
return build_component_list(settings._getcomposite('EXTENSIONS'))
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from twisted.internet import defer, threads
|
|||
from w3lib.url import file_uri_to_path
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.utils.conf import remove_none_values
|
||||
from scrapy.utils.ftp import ftp_makedirs_cwd
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.misc import load_object
|
||||
|
|
@ -195,8 +196,7 @@ class FeedExporter(object):
|
|||
return item
|
||||
|
||||
def _load_components(self, setting_prefix):
|
||||
conf = dict(self.settings['%s_BASE' % setting_prefix])
|
||||
conf.update(self.settings[setting_prefix])
|
||||
conf = remove_none_values(self.settings._getcomposite(setting_prefix))
|
||||
d = {}
|
||||
for k, v in conf.items():
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -13,15 +13,7 @@ class ItemPipelineManager(MiddlewareManager):
|
|||
|
||||
@classmethod
|
||||
def _get_mwlist_from_settings(cls, settings):
|
||||
item_pipelines = settings['ITEM_PIPELINES']
|
||||
if isinstance(item_pipelines, (tuple, list, set, frozenset)):
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
import warnings
|
||||
warnings.warn('ITEM_PIPELINES defined as a list or a set is deprecated, switch to a dict',
|
||||
category=ScrapyDeprecationWarning, stacklevel=1)
|
||||
# convert old ITEM_PIPELINE list to a dict with order 500
|
||||
item_pipelines = dict(zip(item_pipelines, range(500, 500+len(item_pipelines))))
|
||||
return build_component_list(settings['ITEM_PIPELINES_BASE'], item_pipelines)
|
||||
return build_component_list(settings._getcomposite('ITEM_PIPELINES'))
|
||||
|
||||
def _add_middleware(self, pipe):
|
||||
super(ItemPipelineManager, self)._add_middleware(pipe)
|
||||
|
|
|
|||
|
|
@ -36,13 +36,21 @@ class SettingsAttribute(object):
|
|||
|
||||
def __init__(self, value, priority):
|
||||
self.value = value
|
||||
self.priority = priority
|
||||
if isinstance(self.value, BaseSettings):
|
||||
self.priority = max(self.value.maxpriority(), priority)
|
||||
else:
|
||||
self.priority = priority
|
||||
|
||||
def set(self, value, priority):
|
||||
"""Sets value if priority is higher or equal than current priority."""
|
||||
if priority >= self.priority:
|
||||
self.value = value
|
||||
self.priority = priority
|
||||
if isinstance(self.value, BaseSettings):
|
||||
# Ignore self.priority if self.value has per-key priorities
|
||||
self.value.update(value, priority)
|
||||
self.priority = max(self.value.maxpriority(), priority)
|
||||
else:
|
||||
if priority >= self.priority:
|
||||
self.value = value
|
||||
self.priority = priority
|
||||
|
||||
def __str__(self):
|
||||
return "<SettingsAttribute value={self.value!r} " \
|
||||
|
|
@ -95,6 +103,20 @@ class BaseSettings(MutableMapping):
|
|||
value = json.loads(value)
|
||||
return dict(value)
|
||||
|
||||
def _getcomposite(self, name):
|
||||
# DO NOT USE THIS FUNCTION IN YOUR CUSTOM PROJECTS
|
||||
# It's for internal use in the transition away from the _BASE settings and
|
||||
# will be removed along with _BASE support in a future release
|
||||
basename = name + "_BASE"
|
||||
if basename in self:
|
||||
warnings.warn('_BASE settings are deprecated.',
|
||||
category=ScrapyDeprecationWarning)
|
||||
compsett = BaseSettings(self[name + "_BASE"], priority='default')
|
||||
compsett.update(self[name])
|
||||
return compsett
|
||||
else:
|
||||
return self[name]
|
||||
|
||||
def getpriority(self, name):
|
||||
prio = None
|
||||
if name in self:
|
||||
|
|
@ -232,16 +254,25 @@ class _DictProxy(MutableMapping):
|
|||
class Settings(BaseSettings):
|
||||
|
||||
def __init__(self, values=None, priority='project'):
|
||||
# Do not pass kwarg values here. We don't want to promote user-defined
|
||||
# dicts, and we want to update, not replace, default dicts with the
|
||||
# values given by the user
|
||||
super(Settings, self).__init__()
|
||||
self.setmodule(default_settings, 'default')
|
||||
# Promote default dictionaries to BaseSettings instances for per-key
|
||||
# priorities
|
||||
for name in self:
|
||||
val = self[name]
|
||||
if isinstance(val, dict):
|
||||
self.set(name, BaseSettings(val, 'default'), 'default')
|
||||
self.update(values, priority)
|
||||
|
||||
|
||||
class CrawlerSettings(Settings):
|
||||
|
||||
def __init__(self, settings_module=None, **kw):
|
||||
Settings.__init__(self, **kw)
|
||||
self.settings_module = settings_module
|
||||
Settings.__init__(self, **kw)
|
||||
|
||||
def __getitem__(self, opt_name):
|
||||
if opt_name in self.overrides:
|
||||
|
|
|
|||
|
|
@ -63,8 +63,7 @@ DNS_TIMEOUT = 60
|
|||
|
||||
DOWNLOAD_DELAY = 0
|
||||
|
||||
DOWNLOAD_HANDLERS = {}
|
||||
DOWNLOAD_HANDLERS_BASE = {
|
||||
DOWNLOAD_HANDLERS = {
|
||||
'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
|
||||
'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
|
||||
'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
|
||||
|
|
@ -82,9 +81,7 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader'
|
|||
DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
|
||||
DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
|
||||
|
||||
DOWNLOADER_MIDDLEWARES = {}
|
||||
|
||||
DOWNLOADER_MIDDLEWARES_BASE = {
|
||||
DOWNLOADER_MIDDLEWARES = {
|
||||
# Engine side
|
||||
'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
|
||||
'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
|
||||
|
|
@ -116,9 +113,7 @@ except KeyError:
|
|||
else:
|
||||
EDITOR = 'vi'
|
||||
|
||||
EXTENSIONS = {}
|
||||
|
||||
EXTENSIONS_BASE = {
|
||||
EXTENSIONS = {
|
||||
'scrapy.extensions.corestats.CoreStats': 0,
|
||||
'scrapy.telnet.TelnetConsole': 0,
|
||||
'scrapy.extensions.memusage.MemoryUsage': 0,
|
||||
|
|
@ -135,16 +130,14 @@ FEED_URI_PARAMS = None # a function to extend uri arguments
|
|||
FEED_FORMAT = 'jsonlines'
|
||||
FEED_STORE_EMPTY = False
|
||||
FEED_EXPORT_FIELDS = None
|
||||
FEED_STORAGES = {}
|
||||
FEED_STORAGES_BASE = {
|
||||
FEED_STORAGES = {
|
||||
'': 'scrapy.extensions.feedexport.FileFeedStorage',
|
||||
'file': 'scrapy.extensions.feedexport.FileFeedStorage',
|
||||
'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
|
||||
's3': 'scrapy.extensions.feedexport.S3FeedStorage',
|
||||
'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
|
||||
}
|
||||
FEED_EXPORTERS = {}
|
||||
FEED_EXPORTERS_BASE = {
|
||||
FEED_EXPORTERS = {
|
||||
'json': 'scrapy.exporters.JsonItemExporter',
|
||||
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
|
||||
'jl': 'scrapy.exporters.JsonLinesItemExporter',
|
||||
|
|
@ -170,7 +163,6 @@ HTTPCACHE_GZIP = False
|
|||
ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
|
||||
|
||||
ITEM_PIPELINES = {}
|
||||
ITEM_PIPELINES_BASE = {}
|
||||
|
||||
LOG_ENABLED = True
|
||||
LOG_ENCODING = 'utf-8'
|
||||
|
|
@ -229,9 +221,7 @@ SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
|
|||
|
||||
SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
|
||||
|
||||
SPIDER_MIDDLEWARES = {}
|
||||
|
||||
SPIDER_MIDDLEWARES_BASE = {
|
||||
SPIDER_MIDDLEWARES = {
|
||||
# Engine side
|
||||
'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
|
||||
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500,
|
||||
|
|
@ -258,8 +248,7 @@ TELNETCONSOLE_ENABLED = 1
|
|||
TELNETCONSOLE_PORT = [6023, 6073]
|
||||
TELNETCONSOLE_HOST = '127.0.0.1'
|
||||
|
||||
SPIDER_CONTRACTS = {}
|
||||
SPIDER_CONTRACTS_BASE = {
|
||||
SPIDER_CONTRACTS = {
|
||||
'scrapy.contracts.default.UrlContract': 1,
|
||||
'scrapy.contracts.default.ReturnsContract': 2,
|
||||
'scrapy.contracts.default.ScrapesContract': 3,
|
||||
|
|
|
|||
|
|
@ -1,36 +1,49 @@
|
|||
import os
|
||||
import sys
|
||||
import warnings
|
||||
from operator import itemgetter
|
||||
|
||||
import six
|
||||
from six.moves.configparser import SafeConfigParser
|
||||
|
||||
from scrapy.settings import BaseSettings
|
||||
from scrapy.utils.deprecate import update_classpath
|
||||
|
||||
|
||||
def build_component_list(base, custom, convert=update_classpath):
|
||||
"""Compose a component list based on a custom and base dict of components
|
||||
(typically middlewares or extensions), unless custom is already a list, in
|
||||
which case it's returned.
|
||||
"""
|
||||
def build_component_list(compdict, convert=update_classpath):
|
||||
"""Compose a component list from a { class: order } dictionary."""
|
||||
|
||||
def _check_components(complist):
|
||||
if len({convert(c) for c in complist}) != len(complist):
|
||||
raise ValueError('Some paths in {!r} convert to the same object, '
|
||||
'please update your settings'.format(complist))
|
||||
|
||||
if isinstance(custom, (list, tuple)):
|
||||
_check_components(custom)
|
||||
return type(custom)(convert(c) for c in custom)
|
||||
|
||||
def _map_keys(compdict):
|
||||
_check_components(compdict)
|
||||
return {convert(k): v for k, v in six.iteritems(compdict)}
|
||||
if isinstance(compdict, BaseSettings):
|
||||
compbs = BaseSettings()
|
||||
for k, v in six.iteritems(compdict):
|
||||
prio = compdict.getpriority(k)
|
||||
if compbs.getpriority(convert(k)) == prio:
|
||||
raise ValueError('Some paths in {!r} convert to the same '
|
||||
'object, please update your settings'
|
||||
''.format(list(compdict.keys())))
|
||||
else:
|
||||
compbs.set(convert(k), v, priority=prio)
|
||||
return compbs
|
||||
else:
|
||||
_check_components(compdict)
|
||||
return {convert(k): v for k, v in six.iteritems(compdict)}
|
||||
|
||||
compdict = _map_keys(base)
|
||||
compdict.update(_map_keys(custom))
|
||||
items = (x for x in six.iteritems(compdict) if x[1] is not None)
|
||||
return [x[0] for x in sorted(items, key=itemgetter(1))]
|
||||
if isinstance(compdict, (list, tuple)):
|
||||
_check_components(compdict)
|
||||
return type(compdict)(convert(c) for c in compdict)
|
||||
compdict = remove_none_values(_map_keys(compdict))
|
||||
return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]
|
||||
|
||||
|
||||
def remove_none_values(compdict):
|
||||
"""Return dict with all pairs that have value 'None' removed"""
|
||||
return {k: v for k, v in six.iteritems(compdict) if v is not None}
|
||||
|
||||
|
||||
def arglist_to_dict(arglist):
|
||||
|
|
|
|||
|
|
@ -28,11 +28,22 @@ class SettingsAttributeTest(unittest.TestCase):
|
|||
self.assertEqual(self.attribute.value, 'value')
|
||||
self.assertEqual(self.attribute.priority, 10)
|
||||
|
||||
def test_set_per_key_priorities(self):
|
||||
attribute = SettingsAttribute(
|
||||
BaseSettings({'one': 10, 'two': 20}, 0),
|
||||
0)
|
||||
|
||||
class SettingsTest(unittest.TestCase):
|
||||
new_dict = {'one': 11, 'two': 21}
|
||||
attribute.set(new_dict, 10)
|
||||
self.assertEqual(attribute.value['one'], 11)
|
||||
self.assertEqual(attribute.value['two'], 21)
|
||||
|
||||
if six.PY3:
|
||||
assertItemsEqual = unittest.TestCase.assertCountEqual
|
||||
new_settings = BaseSettings()
|
||||
new_settings.set('one', 12, 20)
|
||||
new_settings.set('two', 12, 0)
|
||||
attribute.set(new_settings, 0)
|
||||
self.assertEqual(attribute.value['one'], 12)
|
||||
self.assertEqual(attribute.value['two'], 21)
|
||||
|
||||
|
||||
class BaseSettingsTest(unittest.TestCase):
|
||||
|
|
@ -239,6 +250,20 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
self.assertEqual(settings.getpriority('key'), 99)
|
||||
self.assertEqual(settings.getpriority('nonexistentkey'), None)
|
||||
|
||||
def test_getcomposite(self):
|
||||
s = BaseSettings({'TEST_BASE': {1: 1, 2: 2},
|
||||
'TEST': BaseSettings({1: 10}),
|
||||
'HASNOBASE': BaseSettings({1: 1})})
|
||||
cs = s._getcomposite('TEST')
|
||||
self.assertEqual(len(cs), 2)
|
||||
self.assertEqual(cs[1], 10)
|
||||
self.assertEqual(cs[2], 2)
|
||||
cs = s._getcomposite('HASNOBASE')
|
||||
self.assertEqual(len(cs), 1)
|
||||
self.assertEqual(cs[1], 1)
|
||||
cs = s._getcomposite('NONEXISTENT')
|
||||
self.assertIsNone(cs)
|
||||
|
||||
def test_maxpriority(self):
|
||||
# Empty settings should return 'default'
|
||||
self.assertEqual(self.settings.maxpriority(), 0)
|
||||
|
|
@ -342,6 +367,24 @@ class SettingsTest(unittest.TestCase):
|
|||
self.assertEqual(attr.value, 'value')
|
||||
self.assertEqual(attr.priority, 10)
|
||||
|
||||
@mock.patch('scrapy.settings.default_settings', default_settings)
|
||||
def test_autopromote_dicts(self):
|
||||
settings = Settings()
|
||||
mydict = settings.get('TEST_DICT')
|
||||
self.assertIsInstance(mydict, BaseSettings)
|
||||
self.assertIn('key', mydict)
|
||||
self.assertEqual(mydict['key'], 'val')
|
||||
self.assertEqual(mydict.getpriority('key'), 0)
|
||||
|
||||
@mock.patch('scrapy.settings.default_settings', default_settings)
|
||||
def test_getdict_autodegrade_basesettings(self):
|
||||
settings = Settings()
|
||||
mydict = settings.getdict('TEST_DICT')
|
||||
self.assertIsInstance(mydict, dict)
|
||||
self.assertEqual(len(mydict), 1)
|
||||
self.assertIn('key', mydict)
|
||||
self.assertEqual(mydict['key'], 'val')
|
||||
|
||||
|
||||
class CrawlerSettingsTest(unittest.TestCase):
|
||||
|
||||
|
|
|
|||
|
|
@ -1,43 +1,64 @@
|
|||
import unittest
|
||||
|
||||
from scrapy.utils.conf import build_component_list, arglist_to_dict
|
||||
from scrapy.settings import BaseSettings
|
||||
from scrapy.utils.conf import (build_component_list, arglist_to_dict,
|
||||
remove_none_values)
|
||||
|
||||
|
||||
class BuildComponentListTest(unittest.TestCase):
|
||||
|
||||
def test_build_dict(self):
|
||||
base = {'one': 1, 'two': 2, 'three': 3, 'five': 5, 'six': None}
|
||||
custom = {'two': None, 'three': 8, 'four': 4}
|
||||
self.assertEqual(build_component_list(base, custom, lambda x: x),
|
||||
['one', 'four', 'five', 'three'])
|
||||
d = {'one': 1, 'two': None, 'three': 8, 'four': 4}
|
||||
self.assertEqual(build_component_list(d, lambda x: x),
|
||||
['one', 'four', 'three'])
|
||||
|
||||
def test_return_list(self):
|
||||
custom = ['a', 'b', 'c']
|
||||
self.assertEqual(build_component_list(None, custom, lambda x: x),
|
||||
custom)
|
||||
self.assertEqual(build_component_list(custom, lambda x: x), custom)
|
||||
|
||||
def test_map_dict(self):
|
||||
custom = {'one': 1, 'two': 2, 'three': 3}
|
||||
self.assertEqual(build_component_list({}, custom, lambda x: x.upper()),
|
||||
self.assertEqual(build_component_list(custom, lambda x: x.upper()),
|
||||
['ONE', 'TWO', 'THREE'])
|
||||
|
||||
def test_map_list(self):
|
||||
custom = ['a', 'b', 'c']
|
||||
self.assertEqual(build_component_list(None, custom, lambda x: x.upper()),
|
||||
self.assertEqual(build_component_list(custom, lambda x: x.upper()),
|
||||
['A', 'B', 'C'])
|
||||
|
||||
def test_duplicate_components_in_dict(self):
|
||||
duplicate_dict = {'one': 1, 'two': 2, 'ONE': 4}
|
||||
self.assertRaises(ValueError,
|
||||
build_component_list, {}, duplicate_dict, lambda x: x.lower())
|
||||
build_component_list, duplicate_dict, lambda x: x.lower())
|
||||
|
||||
def test_duplicate_components_in_list(self):
|
||||
duplicate_list = ['a', 'b', 'a']
|
||||
self.assertRaises(ValueError,
|
||||
build_component_list, None, duplicate_list, lambda x: x)
|
||||
build_component_list, duplicate_list, lambda x: x)
|
||||
|
||||
def test_duplicate_components_in_basesettings(self):
|
||||
# Higher priority takes precedence
|
||||
duplicate_bs = BaseSettings({'one': 1, 'two': 2}, priority=0)
|
||||
duplicate_bs.set('ONE', 4, priority=10)
|
||||
self.assertEqual(build_component_list(duplicate_bs, convert=lambda x: x.lower()),
|
||||
['two', 'one'])
|
||||
duplicate_bs.set('one', duplicate_bs['one'], priority=20)
|
||||
self.assertEqual(build_component_list(duplicate_bs, convert=lambda x: x.lower()),
|
||||
['one', 'two'])
|
||||
# Same priority raises ValueError
|
||||
duplicate_bs.set('ONE', duplicate_bs['ONE'], priority=20)
|
||||
self.assertRaises(ValueError,
|
||||
build_component_list, duplicate_bs, convert=lambda x: x.lower())
|
||||
|
||||
|
||||
class UtilsConfTestCase(unittest.TestCase):
|
||||
|
||||
def test_remove_none_values(self):
|
||||
comps = {'one': 1, 'none': None, 'three': 3, 'four': 4}
|
||||
compscopy = dict(comps)
|
||||
del compscopy['none']
|
||||
self.assertEqual(remove_none_values(comps), compscopy)
|
||||
|
||||
def test_arglist_to_dict(self):
|
||||
self.assertEqual(arglist_to_dict(['arg1=val1', 'arg2=val2']),
|
||||
{'arg1': 'val1', 'arg2': 'val2'})
|
||||
|
|
|
|||
Loading…
Reference in New Issue