- removed scrapy.conf singleton from scrapy.log, scrapy.responsetypes,

scrapy.http.response.text, scrapy.selector
- fixed bug with scrapy.conf.settings backwards compatibility support
- added facility to notify (and provide some guidelines) about deprecated/obsolete settings

docs/topics/settings.rst

@@ -225,17 +225,6 @@ Default::
 The default headers used for Scrapy HTTP Requests. They're populated in the
 :class:`~scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware`.
 
-.. setting:: DEFAULT_RESPONSE_ENCODING
-
-DEFAULT_RESPONSE_ENCODING
--------------------------
-
-Default: ``'ascii'``
-
-The default encoding to use for :class:`~scrapy.http.TextResponse` objects (and
-subclasses) when no encoding is declared and no encoding could be inferred from
-the body.
-
 .. setting:: DEPTH_LIMIT
 
 DEPTH_LIMIT
@@ -827,7 +816,7 @@ the default value for this setting see: http://www.boutell.com/newfaq/misc/urlle
 USER_AGENT
 ----------
 
-Default: ``"Scrapy/0.15 (+http://scrapy.org)"``
+Default: ``"Scrapy/VERSION (+http://scrapy.org)"``
 
 The default User-Agent to use when crawling, unless overridden. 
 

scrapy/cmdline.py

@@ -10,6 +10,7 @@ from scrapy.command import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.project import inside_project, get_project_settings
+from scrapy.settings.deprecated import check_deprecated_settings
 
 def _iter_command_classes(module_name):
     # TODO: add `name` attribute to commands and and merge this function with
@@ -80,10 +81,17 @@ def _run_print_help(parser, func, *a, **kw):
             parser.print_help()
         sys.exit(2)
 
-def execute(argv=None):
+def execute(argv=None, settings=None):
     if argv is None:
         argv = sys.argv
-    settings = get_project_settings()
+    if settings is None:
+        settings = get_project_settings()
+    check_deprecated_settings(settings)
+
+    # backwards compatibility to support scrapy.conf.settings
+    from scrapy import conf
+    conf.settings = settings
+
     crawler = CrawlerProcess(settings)
     crawler.install()
     inproject = inside_project()

scrapy/command.py

@@ -28,7 +28,7 @@ class ScrapyCommand(object):
     @property
     def crawler(self):
         if not log.started:
-            log.start()
+            log.start_from_settings(self.settings)
         self._crawler.configure()
         return self._crawler
 

scrapy/conf.py

@@ -1,7 +1,5 @@
-# This module is kept for backwards compatibility.
+# This module is kept for backwards compatibility, so users can import
+# scrapy.conf.settings and get the settings they expect
 #
 # TODO: Add deprecation warning once all scrapy.conf instances have been
 # removed from Scrapy codebase.
-
-from scrapy.utils.project import get_project_settings
-settings = get_project_settings()

scrapy/http/response/text.py

@@ -9,12 +9,11 @@ from w3lib.encoding import html_to_unicode, resolve_encoding, \
     html_body_declared_encoding, http_content_type_encoding
 from scrapy.http.response import Response
 from scrapy.utils.python import memoizemethod_noargs
-from scrapy.conf import settings
 
 
 class TextResponse(Response):
 
-    _DEFAULT_ENCODING = settings['DEFAULT_RESPONSE_ENCODING']
+    _DEFAULT_ENCODING = 'ascii'
 
     def __init__(self, *args, **kwargs):
         self._encoding = kwargs.pop('encoding', None)

scrapy/log.py

@@ -10,7 +10,6 @@ import warnings
 from twisted.python import log
 
 import scrapy
-from scrapy.conf import settings
 from scrapy.utils.python import unicode_to_str
 from scrapy.utils.misc import load_object
  
@@ -32,6 +31,7 @@ level_names = {
 }
 
 started = False
+formatter = None
 
 class ScrapyFileLogObserver(log.FileLogObserver):
 
@@ -94,35 +94,22 @@ def _adapt_eventdict(eventDict, log_level=INFO, encoding='utf-8', prepend_level=
     return ev
 
 def _get_log_level(level_name_or_id=None):
-    if level_name_or_id is None:
-        lvlname = settings['LOG_LEVEL']
-        return globals()[lvlname]
-    elif isinstance(level_name_or_id, int):
+    if isinstance(level_name_or_id, int):
         return level_name_or_id
     elif isinstance(level_name_or_id, basestring):
         return globals()[level_name_or_id]
     else:
         raise ValueError("Unknown log level: %r" % level_name_or_id)
 
-def start(logfile=None, loglevel=None, logstdout=None):
-    global started
-    if started or not settings.getbool('LOG_ENABLED'):
-        return
-    started = True
-
+def start(logfile=None, loglevel='INFO', logstdout=True, logencoding='utf-8'):
     if log.defaultObserver: # check twisted log not already started
         loglevel = _get_log_level(loglevel)
-        logfile = logfile or settings['LOG_FILE']
         file = open(logfile, 'a') if logfile else sys.stderr
-        if logstdout is None:
-            logstdout = settings.getbool('LOG_STDOUT')
-        sflo = ScrapyFileLogObserver(file, loglevel, settings['LOG_ENCODING'])
+        sflo = ScrapyFileLogObserver(file, loglevel, logencoding)
         _oldshowwarning = warnings.showwarning
         log.startLoggingWithObserver(sflo.emit, setStdout=logstdout)
         # restore warnings, wrongly silenced by Twisted
         warnings.showwarning = _oldshowwarning
-        msg("Scrapy %s started (bot: %s)" % (scrapy.__version__, \
-            settings['BOT_NAME']))
 
 def msg(message=None, _level=INFO, **kw):
     kw['logLevel'] = kw.pop('level', _level)
@@ -137,4 +124,16 @@ def err(_stuff=None, _why=None, **kw):
     kw.setdefault('system', 'scrapy')
     log.err(_stuff, _why, **kw)
 
-formatter = load_object(settings['LOG_FORMATTER'])()
+def start_from_settings(settings):
+    global started, formatter
+    if started or not settings.getbool('LOG_ENABLED'):
+        return
+    started = True
+    formatter = load_object(settings['LOG_FORMATTER'])()
+
+    if not settings.getbool('LOG_ENABLED'):
+        return
+    start(settings['LOG_FILE'], settings['LOG_LEVEL'], settings['LOG_STDOUT'],
+        settings['LOG_ENCODING'])
+    msg("Scrapy %s started (bot: %s)" % (scrapy.__version__, \
+        settings['BOT_NAME']))

scrapy/responsetypes.py

@@ -11,7 +11,6 @@ from cStringIO import StringIO
 from scrapy.http import Response
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import isbinarytext
-from scrapy.conf import settings
 
 class ResponseTypes(object):
 
@@ -31,7 +30,6 @@ class ResponseTypes(object):
     }
 
     def __init__(self):
-        self.CLASSES.update(settings.get('RESPONSE_CLASSES', {}))
         self.classes = {}
         self.mimetypes = MimeTypes()
         mimedata = get_data('scrapy', 'mime.types')

scrapy/selector/dummysel.py

@@ -1,22 +0,0 @@
-"""
-Dummy selectors
-"""
-
-from .list import XPathSelectorList as XPathSelectorList
-
-__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \
-    'XPathSelectorList']
-
-class XPathSelector(object):
-
-    def __init__(self, *a, **kw):
-        pass
-
-    def _raise(self, *a, **kw):
-        raise RuntimeError("No selectors backend available. " \
-            "Please install libxml2 or lxml")
-
-    select = re = extract = register_namespace = __nonzero__ = _raise
-
-XmlXPathSelector = XPathSelector
-HtmlXPathSelector = XPathSelector

scrapy/settings/default_settings.py

@@ -41,8 +41,6 @@ DEFAULT_REQUEST_HEADERS = {
     'Accept-Language': 'en',
 }
 
-DEFAULT_RESPONSE_ENCODING = 'ascii'
-
 DEPTH_LIMIT = 0
 DEPTH_STATS = True
 DEPTH_PRIORITY = 0
@@ -224,7 +222,7 @@ TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates'))
 
 URLLENGTH_LIMIT = 2083
 
-USER_AGENT = 'Scrapy/0.15 (+http://scrapy.org)'
+USER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % __import__('scrapy').__version__
 
 TELNETCONSOLE_ENABLED = 1
 TELNETCONSOLE_PORT = [6023, 6073]

scrapy/settings/deprecated.py

@@ -0,0 +1,20 @@
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+
+DEPRECATED_SETTINGS = [
+    ('TRACK_REFS', 'no longer needed (trackref is always enabled)'),
+    ('RESPONSE_CLASSES', 'no longer supported'),
+    ('DEFAULT_RESPONSE_ENCODING', 'no longer supported'),
+    ('BOT_VERSION', 'no longer used (user agent defaults to Scrapy now)'),
+    ('ENCODING_ALIASES', 'no longer needed (encoding discovery uses w3lib now)'),
+    ('STATS_ENABLED', 'no longer supported (change STATS_CLASS instead)'),
+    ('SQLITE_DB', 'no longer supported'),
+]
+
+def check_deprecated_settings(settings):
+    deprecated = [x for x in DEPRECATED_SETTINGS if settings[x[0]] is not None]
+    if deprecated:
+        msg = "You are using the following settings which are deprecated or obsolete"
+        msg += " (ask scrapy-users@googlegroups.com for alternatives):"
+        msg = msg + "\n    " + "\n    ".join("%s: %s" % x for x in deprecated)
+        warnings.warn(msg, ScrapyDeprecationWarning)

scrapy/tests/test_selector_dummy.py

@@ -1,17 +0,0 @@
-import unittest
-
-from scrapy.http import TextResponse
-from scrapy.selector.dummysel import XmlXPathSelector, HtmlXPathSelector, \
-    XPathSelector
-
-class XPathSelectorTestCase(unittest.TestCase):
-
-    def test_raises(self):
-        response = TextResponse(url="http://example.com", body='test')
-        for cls in [XmlXPathSelector, HtmlXPathSelector, XPathSelector]:
-            sel = cls(response)
-            self.assertRaises(RuntimeError, sel.select, '//h2')
-            self.assertRaises(RuntimeError, sel.re, 'lala')
-            self.assertRaises(RuntimeError, sel.extract)
-            self.assertRaises(RuntimeError, sel.register_namespace, 'a', 'b')
-            self.assertRaises(RuntimeError, sel.__nonzero__, 'a', 'b')