diff --git a/docs/topics/scrapyd.rst b/docs/topics/scrapyd.rst index e55cf80fe..e78b5c76e 100644 --- a/docs/topics/scrapyd.rst +++ b/docs/topics/scrapyd.rst @@ -221,39 +221,38 @@ Show and define targets To see all available targets type:: - scrapy deploy -L + scrapy deploy -l -This will return a list of available targets and their URLs:: +This will return a list of available targets and their URLs. For example:: scrapyd http://localhost:6800/ -The ``scrapyd`` target is available by default. You can define more targets by -adding them to the ``scrapy.cfg`` file in your project or any other supported -location like ``~/.scrapy.cfg``, ``/etc/scrapy.cfg``, or -``c:\scrapy\scrapy.cfg``. +You can define targets by adding them to your project's ``scrapy.cfg`` file, +or any other supported location like ``~/.scrapy.cfg``, ``/etc/scrapy.cfg``, +or ``c:\scrapy\scrapy.cfg`` (in Windows). Here's an example of defining a new target ``scrapyd2`` with restricted access through HTTP basic authentication:: - [deploy_scrapyd2] - url = http://scrapyd.mydomain.com/api/scrapy/ + [deploy:scrapyd2] + url = http://scrapyd.mydomain.com/api/scrapyd/ username = john password = secret .. note:: The :command:`deploy` command also supports netrc for getting the credentials. -Now, if you type ``scrapy deploy -L`` you'd see:: +Now, if you type ``scrapy deploy -l`` you'll see:: scrapyd http://localhost:6800/ - scrapyd2 http://scrapyd.mydomain.com/api/scrapy/ + scrapyd2 http://scrapyd.mydomain.com/api/scrapyd/ See available projects ---------------------- -To see all available projets in certain target use:: +To see all available projets in a specific target use:: - scrapy deploy -l scrapyd + scrapy deploy -L scrapyd It would return something like this:: @@ -265,14 +264,14 @@ Deploying a project Finally, to deploy your project use:: - scrapy deploy scrapyd:project1 + scrapy deploy scrapyd -p project1 This will eggify your project and upload it to the target, printing the JSON response returned from the Scrapyd server. If you have a ``setup.py`` file in your project, that one will be used. Otherwise a ``setup.py`` file will be created automatically (based on a simple template) that you can edit later. -After running that command you will see something like this meaning your +After running that command you will see something like this, meaning your project was uploaded successfully:: Deploying myproject-1287453519 to http://localhost:6800/addversion.json @@ -283,23 +282,30 @@ By default ``scrapy deploy`` uses the current timestamp for generating the project version, as you can see in the output above. However, you can pass a custom version with the ``--version`` option:: - scrapy deploy scrapyd:project1 --version 54 + scrapy deploy scrapyd -p project1 --version 54 Also, if you use Mercurial for tracking your project source code, you can use ``HG`` for the version which will be replaced by the current Mercurial revision, for example ``r382``:: - scrapy deploy scrapyd:project1 --version HG + scrapy deploy scrapyd -p project1 --version HG Support for other version discovery sources may be added in the future. -Finally, if you don't want to specify the target and project every time you run -``scrapy deploy`` you can define the default ones in the ``scrapy.cfg`` file, -like this:: +Finally, if you don't want to specify the target, project and version every +time you run ``scrapy deploy`` you can define the defaults in the +``scrapy.cfg`` file. For example:: [deploy] - target = scrapyd + url = http://scrapyd.mydomain.com/api/scrapyd/ + username = john + password = secret project = project1 + version = HG + +This way, you can deploy your project just by using:: + + scrapy deploy .. _topics-egg-caveats: diff --git a/scrapy/commands/deploy.py b/scrapy/commands/deploy.py index aa2a4e9c5..a8c546de1 100644 --- a/scrapy/commands/deploy.py +++ b/scrapy/commands/deploy.py @@ -18,12 +18,6 @@ from scrapy.utils.multipart import encode_multipart from scrapy.utils.http import basic_auth_header from scrapy.utils.conf import get_config, closest_scrapy_cfg -_DEFAULT_TARGETS = { - 'scrapyd': { - 'url': 'http://localhost:6800/', - }, -} - _SETUP_PY_TEMPLATE = \ """# Automatically created by: scrapy deploy @@ -42,22 +36,24 @@ class Command(ScrapyCommand): requires_project = True def syntax(self): - return "[options] [ | -l | -L ]" + return "[options] [ [target] | -l | -L ]" def short_desc(self): - return "Deploy project in Scrapyd server" + return "Deploy project in Scrapyd target" def long_desc(self): return "Deploy the current project into the given Scrapyd server " \ - "(aka target) and project." + "(known as target)" def add_options(self, parser): ScrapyCommand.add_options(self, parser) + parser.add_option("-p", "--project", + help="the project name in the target") parser.add_option("-v", "--version", help="the version to deploy. Defaults to current timestamp") - parser.add_option("-L", "--list-targets", action="store_true", \ + parser.add_option("-l", "--list-targets", action="store_true", \ help="list available targets") - parser.add_option("-l", "--list-projects", metavar="TARGET", \ + parser.add_option("-L", "--list-projects", metavar="TARGET", \ help="list available projects on TARGET") parser.add_option("--egg", metavar="FILE", help="use the given egg, instead of building it") @@ -79,13 +75,16 @@ class Command(ScrapyCommand): projects = json.loads(f.read())['projects'] print os.linesep.join(projects) return - target, project = _get_target_project(args) - version = _get_version(opts) + target_name = _get_target_name(args) + target = _get_target(target_name) + project = _get_project(target, opts) + version = _get_version(target, opts) tmpdir = None if opts.egg: + _log("Using egg: %s" % opts.egg) egg = open(opts.egg, 'rb') else: - _log("Bulding egg of %s-%s" % (project, version)) + _log("Building egg of %s-%s" % (project, version)) egg, tmpdir = _build_egg() _upload_egg(target, egg, project, version) egg.close() @@ -93,20 +92,21 @@ class Command(ScrapyCommand): shutil.rmtree(tmpdir) def _log(message): - sys.stderr.write("%s\n" % message) + sys.stderr.write(message + os.linesep) -def _get_target_project(args): - if len(args) >= 1 and ':' in args[0]: - target_name, project = args[0].split(':', 1) +def _get_target_name(args): + if len(args) > 1: + raise UsageError("Too many arguments: %s" % ' '.join(args)) + elif args: + return args[0] elif len(args) < 1: - target_name = _get_option('deploy', 'target') - project = _get_option('deploy', 'project') - if not target_name or not project: - raise UsageError(" not given and defaults not found") - else: - raise UsageError("%r is not a " % args[0]) - target = _get_target(target_name) - return target, project + return 'default' + +def _get_project(target, opts): + project = opts.project or target.get('project') + if not project: + raise UsageError("Missing project") + return project def _get_option(section, option, default=None): cfg = get_config() @@ -115,10 +115,15 @@ def _get_option(section, option, default=None): def _get_targets(): cfg = get_config() - targets = _DEFAULT_TARGETS.copy() + baset = dict(cfg.items('deploy')) if cfg.has_section('deploy') else {} + targets = {} + if 'url' in baset: + targets['default'] = baset for x in cfg.sections(): - if x.startswith('deploy_'): - targets[x[7:]] = dict(cfg.items(x)) + if x.startswith('deploy:'): + t = baset.copy() + t.update(cfg.items(x)) + targets[x[7:]] = t return targets def _get_target(name): @@ -130,12 +135,13 @@ def _get_target(name): def _url(target, action): return urljoin(target['url'], action) -def _get_version(opts): - if opts.version == 'HG': +def _get_version(target, opts): + version = opts.version or target.get('version') + if version == 'HG': p = Popen(['hg', 'tip', '--template', '{rev}'], stdout=PIPE) return 'r%s' % p.communicate()[0] - elif opts.version: - return opts.version + elif version: + return version else: return str(int(time.time())) diff --git a/scrapy/templates/project/scrapy.cfg b/scrapy/templates/project/scrapy.cfg index 7c357dc0d..889516a42 100644 --- a/scrapy/templates/project/scrapy.cfg +++ b/scrapy/templates/project/scrapy.cfg @@ -1,2 +1,11 @@ +# Automatically created by: scrapy startproject +# +# For more information about the [deploy] section see: +# http://doc.scrapy.org/topics/scrapyd.html + [settings] default = ${project_name}.settings + +[deploy] +#url = http://localhost:6800/ +project = ${project_name}