From aadf7ff143f2caeecc8567b7395d35a9705c4319 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 19 Sep 2012 01:46:46 -0300 Subject: [PATCH] - removed scrapy.conf singleton from scrapy.log, scrapy.responsetypes, scrapy.http.response.text, scrapy.selector - fixed bug with scrapy.conf.settings backwards compatibility support - added facility to notify (and provide some guidelines) about deprecated/obsolete settings --- docs/topics/settings.rst | 13 +---------- scrapy/cmdline.py | 12 ++++++++-- scrapy/command.py | 2 +- scrapy/conf.py | 6 ++--- scrapy/http/response/text.py | 3 +-- scrapy/log.py | 35 ++++++++++++++--------------- scrapy/responsetypes.py | 2 -- scrapy/selector/dummysel.py | 22 ------------------ scrapy/settings/default_settings.py | 4 +--- scrapy/settings/deprecated.py | 20 +++++++++++++++++ scrapy/tests/test_selector_dummy.py | 17 -------------- 11 files changed, 53 insertions(+), 83 deletions(-) delete mode 100644 scrapy/selector/dummysel.py create mode 100644 scrapy/settings/deprecated.py delete mode 100644 scrapy/tests/test_selector_dummy.py diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5a8be53ff..54d01d518 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -225,17 +225,6 @@ Default:: The default headers used for Scrapy HTTP Requests. They're populated in the :class:`~scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware`. -.. setting:: DEFAULT_RESPONSE_ENCODING - -DEFAULT_RESPONSE_ENCODING -------------------------- - -Default: ``'ascii'`` - -The default encoding to use for :class:`~scrapy.http.TextResponse` objects (and -subclasses) when no encoding is declared and no encoding could be inferred from -the body. - .. setting:: DEPTH_LIMIT DEPTH_LIMIT @@ -827,7 +816,7 @@ the default value for this setting see: http://www.boutell.com/newfaq/misc/urlle USER_AGENT ---------- -Default: ``"Scrapy/0.15 (+http://scrapy.org)"`` +Default: ``"Scrapy/VERSION (+http://scrapy.org)"`` The default User-Agent to use when crawling, unless overridden. diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 9f6297cb5..a0f5901bb 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -10,6 +10,7 @@ from scrapy.command import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules from scrapy.utils.project import inside_project, get_project_settings +from scrapy.settings.deprecated import check_deprecated_settings def _iter_command_classes(module_name): # TODO: add `name` attribute to commands and and merge this function with @@ -80,10 +81,17 @@ def _run_print_help(parser, func, *a, **kw): parser.print_help() sys.exit(2) -def execute(argv=None): +def execute(argv=None, settings=None): if argv is None: argv = sys.argv - settings = get_project_settings() + if settings is None: + settings = get_project_settings() + check_deprecated_settings(settings) + + # backwards compatibility to support scrapy.conf.settings + from scrapy import conf + conf.settings = settings + crawler = CrawlerProcess(settings) crawler.install() inproject = inside_project() diff --git a/scrapy/command.py b/scrapy/command.py index be92d9780..3bfbe2a24 100644 --- a/scrapy/command.py +++ b/scrapy/command.py @@ -28,7 +28,7 @@ class ScrapyCommand(object): @property def crawler(self): if not log.started: - log.start() + log.start_from_settings(self.settings) self._crawler.configure() return self._crawler diff --git a/scrapy/conf.py b/scrapy/conf.py index b720ce60c..bd8ac3498 100644 --- a/scrapy/conf.py +++ b/scrapy/conf.py @@ -1,7 +1,5 @@ -# This module is kept for backwards compatibility. +# This module is kept for backwards compatibility, so users can import +# scrapy.conf.settings and get the settings they expect # # TODO: Add deprecation warning once all scrapy.conf instances have been # removed from Scrapy codebase. - -from scrapy.utils.project import get_project_settings -settings = get_project_settings() diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index db77e1f06..618d3970e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -9,12 +9,11 @@ from w3lib.encoding import html_to_unicode, resolve_encoding, \ html_body_declared_encoding, http_content_type_encoding from scrapy.http.response import Response from scrapy.utils.python import memoizemethod_noargs -from scrapy.conf import settings class TextResponse(Response): - _DEFAULT_ENCODING = settings['DEFAULT_RESPONSE_ENCODING'] + _DEFAULT_ENCODING = 'ascii' def __init__(self, *args, **kwargs): self._encoding = kwargs.pop('encoding', None) diff --git a/scrapy/log.py b/scrapy/log.py index 3b3f01fa3..64e1e88cb 100644 --- a/scrapy/log.py +++ b/scrapy/log.py @@ -10,7 +10,6 @@ import warnings from twisted.python import log import scrapy -from scrapy.conf import settings from scrapy.utils.python import unicode_to_str from scrapy.utils.misc import load_object @@ -32,6 +31,7 @@ level_names = { } started = False +formatter = None class ScrapyFileLogObserver(log.FileLogObserver): @@ -94,35 +94,22 @@ def _adapt_eventdict(eventDict, log_level=INFO, encoding='utf-8', prepend_level= return ev def _get_log_level(level_name_or_id=None): - if level_name_or_id is None: - lvlname = settings['LOG_LEVEL'] - return globals()[lvlname] - elif isinstance(level_name_or_id, int): + if isinstance(level_name_or_id, int): return level_name_or_id elif isinstance(level_name_or_id, basestring): return globals()[level_name_or_id] else: raise ValueError("Unknown log level: %r" % level_name_or_id) -def start(logfile=None, loglevel=None, logstdout=None): - global started - if started or not settings.getbool('LOG_ENABLED'): - return - started = True - +def start(logfile=None, loglevel='INFO', logstdout=True, logencoding='utf-8'): if log.defaultObserver: # check twisted log not already started loglevel = _get_log_level(loglevel) - logfile = logfile or settings['LOG_FILE'] file = open(logfile, 'a') if logfile else sys.stderr - if logstdout is None: - logstdout = settings.getbool('LOG_STDOUT') - sflo = ScrapyFileLogObserver(file, loglevel, settings['LOG_ENCODING']) + sflo = ScrapyFileLogObserver(file, loglevel, logencoding) _oldshowwarning = warnings.showwarning log.startLoggingWithObserver(sflo.emit, setStdout=logstdout) # restore warnings, wrongly silenced by Twisted warnings.showwarning = _oldshowwarning - msg("Scrapy %s started (bot: %s)" % (scrapy.__version__, \ - settings['BOT_NAME'])) def msg(message=None, _level=INFO, **kw): kw['logLevel'] = kw.pop('level', _level) @@ -137,4 +124,16 @@ def err(_stuff=None, _why=None, **kw): kw.setdefault('system', 'scrapy') log.err(_stuff, _why, **kw) -formatter = load_object(settings['LOG_FORMATTER'])() +def start_from_settings(settings): + global started, formatter + if started or not settings.getbool('LOG_ENABLED'): + return + started = True + formatter = load_object(settings['LOG_FORMATTER'])() + + if not settings.getbool('LOG_ENABLED'): + return + start(settings['LOG_FILE'], settings['LOG_LEVEL'], settings['LOG_STDOUT'], + settings['LOG_ENCODING']) + msg("Scrapy %s started (bot: %s)" % (scrapy.__version__, \ + settings['BOT_NAME'])) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 6cb4fc04b..efb11a267 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -11,7 +11,6 @@ from cStringIO import StringIO from scrapy.http import Response from scrapy.utils.misc import load_object from scrapy.utils.python import isbinarytext -from scrapy.conf import settings class ResponseTypes(object): @@ -31,7 +30,6 @@ class ResponseTypes(object): } def __init__(self): - self.CLASSES.update(settings.get('RESPONSE_CLASSES', {})) self.classes = {} self.mimetypes = MimeTypes() mimedata = get_data('scrapy', 'mime.types') diff --git a/scrapy/selector/dummysel.py b/scrapy/selector/dummysel.py deleted file mode 100644 index 8718f3311..000000000 --- a/scrapy/selector/dummysel.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Dummy selectors -""" - -from .list import XPathSelectorList as XPathSelectorList - -__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \ - 'XPathSelectorList'] - -class XPathSelector(object): - - def __init__(self, *a, **kw): - pass - - def _raise(self, *a, **kw): - raise RuntimeError("No selectors backend available. " \ - "Please install libxml2 or lxml") - - select = re = extract = register_namespace = __nonzero__ = _raise - -XmlXPathSelector = XPathSelector -HtmlXPathSelector = XPathSelector diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 4f2730b98..2df78214e 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -41,8 +41,6 @@ DEFAULT_REQUEST_HEADERS = { 'Accept-Language': 'en', } -DEFAULT_RESPONSE_ENCODING = 'ascii' - DEPTH_LIMIT = 0 DEPTH_STATS = True DEPTH_PRIORITY = 0 @@ -224,7 +222,7 @@ TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates')) URLLENGTH_LIMIT = 2083 -USER_AGENT = 'Scrapy/0.15 (+http://scrapy.org)' +USER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % __import__('scrapy').__version__ TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] diff --git a/scrapy/settings/deprecated.py b/scrapy/settings/deprecated.py new file mode 100644 index 000000000..7bf2d8523 --- /dev/null +++ b/scrapy/settings/deprecated.py @@ -0,0 +1,20 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning + +DEPRECATED_SETTINGS = [ + ('TRACK_REFS', 'no longer needed (trackref is always enabled)'), + ('RESPONSE_CLASSES', 'no longer supported'), + ('DEFAULT_RESPONSE_ENCODING', 'no longer supported'), + ('BOT_VERSION', 'no longer used (user agent defaults to Scrapy now)'), + ('ENCODING_ALIASES', 'no longer needed (encoding discovery uses w3lib now)'), + ('STATS_ENABLED', 'no longer supported (change STATS_CLASS instead)'), + ('SQLITE_DB', 'no longer supported'), +] + +def check_deprecated_settings(settings): + deprecated = [x for x in DEPRECATED_SETTINGS if settings[x[0]] is not None] + if deprecated: + msg = "You are using the following settings which are deprecated or obsolete" + msg += " (ask scrapy-users@googlegroups.com for alternatives):" + msg = msg + "\n " + "\n ".join("%s: %s" % x for x in deprecated) + warnings.warn(msg, ScrapyDeprecationWarning) diff --git a/scrapy/tests/test_selector_dummy.py b/scrapy/tests/test_selector_dummy.py deleted file mode 100644 index 00ac0c152..000000000 --- a/scrapy/tests/test_selector_dummy.py +++ /dev/null @@ -1,17 +0,0 @@ -import unittest - -from scrapy.http import TextResponse -from scrapy.selector.dummysel import XmlXPathSelector, HtmlXPathSelector, \ - XPathSelector - -class XPathSelectorTestCase(unittest.TestCase): - - def test_raises(self): - response = TextResponse(url="http://example.com", body='test') - for cls in [XmlXPathSelector, HtmlXPathSelector, XPathSelector]: - sel = cls(response) - self.assertRaises(RuntimeError, sel.select, '//h2') - self.assertRaises(RuntimeError, sel.re, 'lala') - self.assertRaises(RuntimeError, sel.extract) - self.assertRaises(RuntimeError, sel.register_namespace, 'a', 'b') - self.assertRaises(RuntimeError, sel.__nonzero__, 'a', 'b')