diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 52c4fe446..16af52eea 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -16,6 +16,8 @@ accepts a different set of arguments and options. (The ``scrapy deploy`` command has been removed in 1.0 in favor of the standalone ``scrapyd-deploy``. See `Deploying your project`_.) +.. _topics-config-settings: + Configuration settings ====================== @@ -34,8 +36,9 @@ and project-wide settings will override all others, when defined. Scrapy also understands, and can be configured through, a number of environment variables. Currently these are: -* ``SCRAPY_SETTINGS_MODULE`` (See :ref:`topics-settings-module-envvar`) +* ``SCRAPY_SETTINGS_MODULE`` (see :ref:`topics-settings-module-envvar`) * ``SCRAPY_PROJECT`` +* ``SCRAPY_PYTHON_SHELL`` (see :ref:`topics-shell`) .. _topics-project-structure: diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 9c9411d6d..327e22e90 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -17,6 +17,9 @@ spider, without having to run the spider to test every change. Once you get familiarized with the Scrapy shell, you'll see that it's an invaluable tool for developing and debugging your spiders. +Configuring the shell +===================== + If you have `IPython`_ installed, the Scrapy shell will use it (instead of the standard Python console). The `IPython`_ console is much more powerful and provides smart auto-completion and colorized output, among other things. @@ -25,8 +28,20 @@ We highly recommend you install `IPython`_, specially if you're working on Unix systems (where `IPython`_ excels). See the `IPython installation guide`_ for more info. +Scrapy also has support for `bpython`_, and will try to use it where `IPython`_ +is unavailable. + +Through scrapy's settings you can configure it to use any one of +``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which +are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment +variable; or by defining it in your :ref:`scrapy.cfg `:: + + [settings] + shell = bpython + .. _IPython: http://ipython.org/ .. _IPython installation guide: http://ipython.org/install.html +.. _bpython: http://www.bpython-interpreter.org/ Launch the shell ================ diff --git a/scrapy/shell.py b/scrapy/shell.py index 4142396ab..a60b45dbb 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -5,6 +5,7 @@ See documentation in docs/topics/shell.rst """ from __future__ import print_function +import os import signal import warnings @@ -21,6 +22,8 @@ from scrapy.spiders import Spider from scrapy.utils.console import start_python_console from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser +from scrapy.utils.conf import get_config +from scrapy.utils.console import DEFAULT_PYTHON_SHELLS class Shell(object): @@ -52,7 +55,28 @@ class Shell(object): if self.code: print(eval(self.code, globals(), self.vars)) else: - start_python_console(self.vars) + """ + Detect interactive shell setting in scrapy.cfg + e.g.: ~/.config/scrapy.cfg or ~/.scrapy.cfg + [settings] + # shell can be one of ipython, bpython or python; + # to be used as the interactive python console, if available. + # (default is ipython, fallbacks in the order listed above) + shell = python + """ + cfg = get_config() + section, option = 'settings', 'shell' + env = os.environ.get('SCRAPY_PYTHON_SHELL') + shells = [] + if env: + shells += env.strip().lower().split(',') + elif cfg.has_option(section, option): + shells += [cfg.get(section, option).strip().lower()] + else: # try all by default + shells += DEFAULT_PYTHON_SHELLS.keys() + # always add standard shell as fallback + shells += ['python'] + start_python_console(self.vars, shells=shells) def _schedule(self, request, spider): spider = self._open_spider(request, spider) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 924501002..a712df30e 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,37 +1,79 @@ +from functools import wraps +from collections import OrderedDict -def start_python_console(namespace=None, noipython=False, banner=''): - """Start Python console binded to the given namespace. If IPython is - available, an IPython console will be started instead, unless `noipython` - is True. Also, tab completion will be used on Unix systems. +def _embed_ipython_shell(namespace={}, banner=''): + """Start an IPython Shell""" + try: + from IPython.terminal.embed import InteractiveShellEmbed + from IPython.terminal.ipapp import load_default_config + except ImportError: + from IPython.frontend.terminal.embed import InteractiveShellEmbed + from IPython.frontend.terminal.ipapp import load_default_config + + @wraps(_embed_ipython_shell) + def wrapper(namespace=namespace, banner=''): + config = load_default_config() + shell = InteractiveShellEmbed( + banner1=banner, user_ns=namespace, config=config) + shell() + return wrapper + +def _embed_bpython_shell(namespace={}, banner=''): + """Start a bpython shell""" + import bpython + @wraps(_embed_bpython_shell) + def wrapper(namespace=namespace, banner=''): + bpython.embed(locals_=namespace, banner=banner) + return wrapper + +def _embed_standard_shell(namespace={}, banner=''): + """Start a standard python shell""" + import code + try: # readline module is only available on unix systems + import readline + except ImportError: + pass + else: + import rlcompleter + readline.parse_and_bind("tab:complete") + @wraps(_embed_standard_shell) + def wrapper(namespace=namespace, banner=''): + code.interact(banner=banner, local=namespace) + return wrapper + +DEFAULT_PYTHON_SHELLS = OrderedDict([ + ('ipython', _embed_ipython_shell), + ('bpython', _embed_bpython_shell), + ( 'python', _embed_standard_shell), +]) + +def get_shell_embed_func(shells=None, known_shells=None): + """Return the first acceptable shell-embed function + from a given list of shell names. + """ + if shells is None: # list, preference order of shells + shells = DEFAULT_PYTHON_SHELLS.keys() + if known_shells is None: # available embeddable shells + known_shells = DEFAULT_PYTHON_SHELLS.copy() + for shell in shells: + if shell in known_shells: + try: + # function test: run all setup code (imports), + # but dont fall into the shell + return known_shells[shell]() + except ImportError: + continue + +def start_python_console(namespace=None, banner='', shells=None): + """Start Python console bound to the given namespace. + Readline support and tab completion will be used on Unix, if available. """ if namespace is None: namespace = {} try: - try: # use IPython if available - if noipython: - raise ImportError() - - try: - from IPython.terminal.embed import InteractiveShellEmbed - from IPython.terminal.ipapp import load_default_config - except ImportError: - from IPython.frontend.terminal.embed import InteractiveShellEmbed - from IPython.frontend.terminal.ipapp import load_default_config - - config = load_default_config() - shell = InteractiveShellEmbed( - banner1=banner, user_ns=namespace, config=config) - shell() - except ImportError: - import code - try: # readline module is only available on unix systems - import readline - except ImportError: - pass - else: - import rlcompleter - readline.parse_and_bind("tab:complete") - code.interact(banner=banner, local=namespace) + shell = get_shell_embed_func(shells) + if shell is not None: + shell(namespace=namespace, banner=banner) except SystemExit: # raised when using exit() in python code.interact pass diff --git a/tests/requirements.txt b/tests/requirements.txt index 452a884e0..f88d17538 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,3 +6,6 @@ pytest-twisted pytest-cov jmespath testfixtures +# optional for shell wrapper tests +bpython +ipython diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py new file mode 100644 index 000000000..65782747b --- /dev/null +++ b/tests/test_utils_console.py @@ -0,0 +1,44 @@ +import unittest + +from scrapy.utils.console import get_shell_embed_func +try: + import bpython + bpy = True + del bpython +except ImportError: + bpy = False +try: + import IPython + ipy = True + del IPython +except ImportError: + ipy = False + +class UtilsConsoleTestCase(unittest.TestCase): + + def test_get_shell_embed_func(self): + + shell = get_shell_embed_func(['invalid']) + self.assertEqual(shell, None) + + shell = get_shell_embed_func(['invalid','python']) + self.assertTrue(callable(shell)) + self.assertEqual(shell.__name__, '_embed_standard_shell') + + @unittest.skipIf(not bpy, 'bpython not available in testenv') + def test_get_shell_embed_func2(self): + + shell = get_shell_embed_func(['bpython']) + self.assertTrue(callable(shell)) + self.assertEqual(shell.__name__, '_embed_bpython_shell') + + @unittest.skipIf(not ipy, 'IPython not available in testenv') + def test_get_shell_embed_func3(self): + + # default shell should be 'ipython' + shell = get_shell_embed_func() + self.assertEqual(shell.__name__, '_embed_ipython_shell') + + +if __name__ == "__main__": + unittest.main()