diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 0ea9e13db..8f50b45a9 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -107,6 +107,7 @@ Global commands: * :command:`shell` * :command:`fetch` * :command:`view` +* :command:`version` Project-only commands: @@ -116,6 +117,7 @@ Project-only commands: * :command:`genspider` * :command:`runserver` * :command:`queue` +* :command:`deploy` .. command:: startproject @@ -399,6 +401,8 @@ And clear the queue:: $ scrapy queue clear +.. command:: version + version ------- @@ -407,6 +411,17 @@ version Prints the Scrapy version. +.. command:: deploy + +deploy +------ + +.. versionadded:: 0.11 + +* Syntax: ``scrapy deploy [ | -l | -L ]`` +* Requires project: *yes* + +Deploy the project into a Scrapyd server. See :ref:`topics-deploying`. Custom project commands ======================= diff --git a/docs/topics/scrapyd.rst b/docs/topics/scrapyd.rst index effedfde4..56c75f40c 100644 --- a/docs/topics/scrapyd.rst +++ b/docs/topics/scrapyd.rst @@ -193,35 +193,94 @@ Here is an example configuration file with all the defaults: .. literalinclude:: ../../scrapyd/default_scrapyd.conf -Eggifying your project +.. _topics-deploying: + +Deploying your project ====================== -In order to upload your project to Scrapyd, you must first build a `Python -egg`_ of it. This is called "eggifying" your project. You'll need to install -`setuptools`_ for this. +Deploying your project into a Scrapyd server typically involves two steps: -To eggify your project add a `setup.py`_ file to the root directory of your -project (where the ``scrapy.cfg`` resides) with the following contents:: +1. building a `Python egg`_ of your project. This is called "eggifying" your + project. You'll need to install `setuptools`_ for this. See + :ref:`topics-egg-caveats` below. - #!/usr/bin/env python +2. uploading the egg to the Scrapyd server - from setuptools import setup, find_packages +The simplest way to deploy your project is by using the :command:`deploy` +command, which automates the process of building the egg uploading it using the +Scrapyd HTTP JSON API. - setup( - name = 'myproject', - version = '1.0', - packages = find_packages(), - entry_points = {'scrapy': ['settings = myproject.settings']}, - ) +The :command:`deploy` command supports multiple targets (Scrapyd servers that +can host your project) and each target supports multiple projects. -And then the run the following command:: +Each time you deploy a new version of a project, you can name it for later +reference. - python setup.py bdist_egg +Show and define targets +----------------------- -This will generate an egg file and leave it in the ``dist`` directory, for -example:: +To see all available targets type:: - dist/myproject-1.0-py2.6.egg + scrapy deploy -L + +This will return a list of available targets and their URLs:: + + scrapyd http://localhost:6800/ + +The ``scrapyd`` target is available by default. You can define more targets by +adding them to the ``scrapy.cfg`` file in your project or any other supported +location like ``~/.scrapy.cfg``, ``/etc/scrapy.cfg``, or +``c:\scrapy\scrapy.cfg``. + +Here's an example of defining a new target ``scrapyd2`` with restricted access +through HTTP basic authentication:: + + [deploy_scrapyd2] + url = http://scrapyd.mydomain.com/api/scrapy/ + username = john + password = secret + +.. note:: The :command:`deploy` command also supports netrc for getting the + credentials. + +Now, if you type ``scrapy deploy -L`` you'd see:: + + scrapyd http://localhost:6800/ + scrapyd2 http://scrapyd.mydomain.com/api/scrapy/ + +See available projects +---------------------- + +To see all available projets in certain target use:: + + scrapy deploy -l scrapyd + +It would return something like this:: + + project1 + project2 + +Deploying a project +------------------- + +Finally, to deploy your project use:: + + scrapy deploy scrapyd:project1 + +This will eggify your project and upload it to the target, printing the JSON +response returned from the Scrapyd server. If you have a ``setup.py`` file in +your project, that one will be used. Otherwise a ``setup.py`` file will be +created automatically (based on a simple template) that you can edit later. + +If you don't want to specify the target and project every time you run ``scrapy +deploy`` you can define the default target and project in the ``scrapy.cfg`` +file, like this:: + + [deploy] + target = scrapyd + project = project1 + +.. _topics-egg-caveats: Egg caveats ----------- @@ -242,19 +301,6 @@ project: have write access to certain directories. If you can, avoid writing to disk and always use `tempfile`_ for temporary files. -Uploading your project -====================== - -In these examples we'll be using `curl`_ for the web service interaction -examples, but you can use any command or library that speaks HTTP. - -Once you've built the egg, you can upload your project to Scrapyd, like this:: - - $ curl http://localhost:6800/addversion.json -F project=myproject -F version=r23 -F egg=@dist/myproject-1.0-py2.6.egg - {"status": "ok", "spiders": ["spider1", "spider2", "spider3"]} - -You'll see that the JSON response contains the spiders found in your project. - Scheduling a spider run ======================= diff --git a/extras/scrapy_bash_completion b/extras/scrapy_bash_completion index f311a5a28..b0654ee6a 100644 --- a/extras/scrapy_bash_completion +++ b/extras/scrapy_bash_completion @@ -11,7 +11,7 @@ _scrapy_completion() { ;; *) if [ $COMP_CWORD -eq 1 ]; then - commands="crawl fetch genspider list parse queue runserver runspider settings shell startproject view" + commands="crawl deploy fetch genspider list parse queue runserver runspider settings shell startproject view" COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W "$commands" -- "$cmd")) fi ;; diff --git a/scrapy/commands/deploy.py b/scrapy/commands/deploy.py new file mode 100644 index 000000000..d6ebd16df --- /dev/null +++ b/scrapy/commands/deploy.py @@ -0,0 +1,186 @@ + +import sys +import os +import glob +import tempfile +import shutil +import time +import urllib2 +import netrc +from urlparse import urlparse, urljoin +from subprocess import Popen, PIPE, check_call + +from scrapy.command import ScrapyCommand +from scrapy.exceptions import UsageError +from scrapy.utils.py26 import json +from scrapy.utils.multipart import encode_multipart +from scrapy.utils.http import basic_auth_header +from scrapy.utils.conf import get_config, closest_scrapy_cfg + +_DEFAULT_TARGETS = { + 'scrapyd': { + 'url': 'http://localhost:6800/', + }, +} + +_SETUP_PY_TEMPLATE = \ +"""# Automatically created by: scrapy deploy + +from setuptools import setup, find_packages + +setup( + name = 'project', + version = '1.0', + packages = find_packages(), + entry_points = {'scrapy': ['settings = %(settings)s']}, +) +""" + +class Command(ScrapyCommand): + + def syntax(self): + return "[options] [ | -l | -L ]" + + def short_desc(self): + return "Deploy project in Scrapyd server" + + def long_desc(self): + return "Deploy the current project into the given Scrapyd server " \ + "(aka target) and project." + + def add_options(self, parser): + ScrapyCommand.add_options(self, parser) + parser.add_option("-v", "--version", + help="the version to deploy. Defaults to current timestamp") + parser.add_option("-L", "--list-targets", action="store_true", \ + help="list available targets") + parser.add_option("-l", "--list-projects", metavar="TARGET", \ + help="list available projects on TARGET") + + def run(self, args, opts): + try: + import setuptools + except ImportError: + raise UsageError("setuptools not installed") + if opts.list_targets: + for name, target in _get_targets().items(): + print "%-20s %s" % (name, target['url']) + return + if opts.list_projects: + target = _get_target(opts.list_projects) + req = urllib2.Request(_url(target, 'listprojects.json')) + _add_auth_header(req, target) + f = urllib2.urlopen(req) + projects = json.loads(f.read())['projects'] + print os.linesep.join(projects) + return + target, project = _get_target_project(args) + version = _get_version(opts) + egg = _build_egg() + _upload_egg(target, egg, project, version) + +def _log(message): + sys.stderr.write("%s\n" % message) + +def _get_target_project(args): + if len(args) >= 1 and ':' in args[0]: + target_name, project = args[0].split(':', 1) + elif len(args) < 1: + target_name = _get_option('deploy', 'target') + project = _get_option('deploy', 'project') + if not target_name or not project: + raise UsageError(" not given and defaults not found") + else: + raise UsageError("%r is not a " % args[0]) + target = _get_target(target_name) + return target, project + +def _get_option(section, option, default=None): + cfg = get_config() + return cfg.get(section, option) if cfg.has_option(section, option) \ + else default + +def _get_targets(): + cfg = get_config() + targets = _DEFAULT_TARGETS.copy() + for x in cfg.sections(): + if x.startswith('deploy_'): + targets[x[7:]] = dict(cfg.items(x)) + return targets + +def _get_target(name): + try: + return _get_targets()[name] + except KeyError: + raise UsageError("Unknown target: %s" % name) + +def _url(target, action): + return urljoin(target['url'], action) + +def _get_version(opts): + if opts.version == 'HG': + p = Popen(['hg', 'tip', '--template', '{rev}'], stdout=PIPE) + return 'r%s' % p.communicate()[0] + elif opts.version: + return opts.version + else: + return str(int(time.time())) + +def _upload_egg(target, eggfile, project, version): + data = { + 'project': project, + 'version': version, + 'egg': ('project.egg', eggfile.read()), + } + body, boundary = encode_multipart(data) + url = _url(target, 'addversion.json') + headers = { + 'Content-Type': 'multipart/form-data; boundary=%s' % boundary, + 'Content-Length': str(len(body)), + } + req = urllib2.Request(url, body, headers) + _add_auth_header(req, target) + _log("Deploying %s-%s to %s" % (project, version, url)) + _http_post(req) + +def _add_auth_header(request, target): + if 'username' in target: + u, p = target.get('username'), target.get('password', '') + request.add_header('Authorization', basic_auth_header(u, p)) + else: # try netrc + try: + host = urlparse(target['url']).hostname + a = netrc.netrc().authenticators(host) + request.add_header('Authorization', basic_auth_header(a[0], a[2])) + except (netrc.NetrcParseError, TypeError): + pass + +def _http_post(request): + try: + f = urllib2.urlopen(request) + _log("Server response (%s):" % f.getcode()) + print f.read() + except urllib2.HTTPError, e: + _log("Deploy failed (%s):" % e.getcode()) + print e.read() + except urllib2.URLError, e: + _log("Deploy failed: %s" % e) + +def _build_egg(): + closest = closest_scrapy_cfg() + os.chdir(os.path.dirname(closest)) + if not os.path.exists('setup.py'): + settings = get_config().get('settings', 'default') + _create_default_setup_py(settings=settings) + d = tempfile.mkdtemp() + try: + f = tempfile.TemporaryFile(dir=d) + check_call([sys.executable, 'setup.py', 'bdist_egg', '-d', d], stdout=f) + egg = glob.glob(os.path.join(d, '*.egg'))[0] + return open(egg, 'rb') + finally: + shutil.rmtree(d) + +def _create_default_setup_py(**kwargs): + with open('setup.py', 'w') as f: + f.write(_SETUP_PY_TEMPLATE % kwargs) diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.py new file mode 100644 index 000000000..f1ed235ae --- /dev/null +++ b/scrapy/utils/multipart.py @@ -0,0 +1,34 @@ +from cStringIO import StringIO + +def encode_multipart(data): + """Encode the given data to be used in a multipart HTTP POST. Data is a + where keys are the field name, and values are either strings or tuples + (filename, content) for file uploads. + + This code is based on distutils.command.upload + """ + + # Build up the MIME payload for the POST data + boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' + sep_boundary = '\r\n--' + boundary + end_boundary = sep_boundary + '--' + body = StringIO() + for key, value in data.items(): + # handle multiple entries for the same name + if type(value) != type([]): + value = [value] + for value in value: + if type(value) is tuple: + fn = '; filename="%s"' % value[0] + value = value[1] + else: + fn = "" + + body.write(sep_boundary) + body.write('\r\nContent-Disposition: form-data; name="%s"' % key) + body.write(fn) + body.write("\r\n\r\n") + body.write(value) + body.write(end_boundary) + body.write("\r\n") + return body.getvalue(), boundary