diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 066447c4b..178dd3f88 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -184,8 +184,13 @@ BOT_NAME Default: ``scrapybot`` -The name of the bot implemented by this Scrapy project. This will be used to -construct the User-Agent by default, and also for logging. +The name of the bot implemented by this Scrapy project (also known as the +project name). This will be used to construct the User-Agent by default, and +also for logging. + +It's automatically populated with your project name when you create your +project with the :doc:`scrapy-ctl.py ` ``startproject`` +command. .. setting:: BOT_VERSION @@ -696,17 +701,6 @@ Example:: NEWSPIDER_MODULE = 'mybot.spiders_dev' -.. setting:: PROJECT_NAME - -PROJECT_NAME ------------- - -Default: ``Not Defined`` - -The name of the current project. It matches the project module name as created -by :ref:`scrapy-ctl.py startproject ` command, -and is only defined by project settings file. - .. setting:: REDIRECT_MAX_TIMES REDIRECT_MAX_TIMES diff --git a/examples/googledir/googledir/settings.py b/examples/googledir/googledir/settings.py index 640530d5a..b920cd46e 100644 --- a/examples/googledir/googledir/settings.py +++ b/examples/googledir/googledir/settings.py @@ -2,14 +2,11 @@ import googledir -PROJECT_NAME = 'googledir' - -BOT_NAME = PROJECT_NAME +BOT_NAME = 'googledir' BOT_VERSION = '1.0' SPIDER_MODULES = ['googledir.spiders'] NEWSPIDER_MODULE = 'googledir.spiders' -TEMPLATES_DIR = '%s/templates' % googledir.__path__[0] DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem' USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) diff --git a/scrapy/command/commands/genspider.py b/scrapy/command/commands/genspider.py index 218adb67e..f39d52d14 100644 --- a/scrapy/command/commands/genspider.py +++ b/scrapy/command/commands/genspider.py @@ -72,8 +72,8 @@ class Command(ScrapyCommand): def _genspider(self, module, domain, template_name, template_file): """Generate the spider module, based on the given template""" tvars = { - 'project_name': settings.get('PROJECT_NAME'), - 'ProjectName': string_camelcase(settings.get('PROJECT_NAME')), + 'project_name': settings.get('BOT_NAME'), + 'ProjectName': string_camelcase(settings.get('BOT_NAME')), 'module': module, 'site': domain, 'classname': '%sSpider' % ''.join([s.capitalize() \ diff --git a/scrapy/command/commands/startproject.py b/scrapy/command/commands/startproject.py index b30b91ce4..eed8ae260 100644 --- a/scrapy/command/commands/startproject.py +++ b/scrapy/command/commands/startproject.py @@ -1,20 +1,17 @@ -#!/usr/bin/env python - import sys -import os import string import re +import shutil +from os.path import join, exists import scrapy from scrapy.command import ScrapyCommand from scrapy.utils.template import render_templatefile, string_camelcase from scrapy.utils.python import ignore_patterns, copytree -PROJECT_TEMPLATES_PATH = os.path.join(scrapy.__path__[0], 'templates/project') +TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'project') -# This is the list of templatefile's path that are rendered *after copying* to -# the new project directory. -TEMPLATES = ( +TEMPLATES_TO_RENDER = ( ('scrapy-ctl.py',), ('${project_name}', 'settings.py.tmpl'), ('${project_name}', 'items.py.tmpl'), @@ -36,33 +33,21 @@ class Command(ScrapyCommand): def run(self, args, opts): if len(args) != 1: return False - project_name = args[0] - if not re.search(r'^[_a-zA-Z]\w*$', project_name): # If it's not a valid directory name. - # Provide a smart error message, depending on the error. - if not re.search(r'^[a-zA-Z]', project_name): - message = 'Project names must begin with a letter' - else: - message = 'Project names must contain only letters, numbers and underscores' - print "Invalid project name: %s\n\n%s" % (project_name, message) + if not re.search(r'^[_a-zA-Z]\w*$', project_name): + print 'Error: Project names must begin with a letter and contain only\n' \ + 'letters, numbers and underscores' + sys.exit(1) + elif exists(project_name): + print "Error: directory %r already exists" % project_name sys.exit(1) - else: - if os.path.exists(project_name): - print "%s dir already exists" % project_name - sys.exit(1) - project_root_path = project_name - - roottpl = os.path.join(PROJECT_TEMPLATES_PATH, 'root') - copytree(roottpl, project_name, ignore=IGNORE) - - moduletpl = os.path.join(PROJECT_TEMPLATES_PATH, 'module') - copytree(moduletpl, '%s/%s' % (project_name, project_name), - ignore=IGNORE) - - for paths in TEMPLATES: - path = os.path.join(*paths) - tplfile = os.path.join(project_root_path, - string.Template(path).substitute(project_name=project_name)) - render_templatefile(tplfile, project_name=project_name, - ProjectName=string_camelcase(project_name)) + moduletpl = join(TEMPLATES_PATH, 'module') + copytree(moduletpl, join(project_name, project_name), ignore=IGNORE) + shutil.copy(join(TEMPLATES_PATH, 'scrapy-ctl.py'), project_name) + for paths in TEMPLATES_TO_RENDER: + path = join(*paths) + tplfile = join(project_name, + string.Template(path).substitute(project_name=project_name)) + render_templatefile(tplfile, project_name=project_name, + ProjectName=string_camelcase(project_name)) diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl index 9bbe421a5..3cab2c1b8 100644 --- a/scrapy/templates/project/module/items.py.tmpl +++ b/scrapy/templates/project/module/items.py.tmpl @@ -1,4 +1,7 @@ # Define here the models for your scraped items +# +# See documentation in: +# http://doc.scrapy.org/topics/items.html from scrapy.item import Item, Field diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index 0bdacc044..fa6f5ea6f 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -1,7 +1,7 @@ # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting -# See: http://doc.scrapy.org/ref/settings.html#item-pipelines +# See: http://doc.scrapy.org/topics/item-pipeline.html class ${ProjectName}Pipeline(object): def process_item(self, domain, item): diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index b51e74cbe..8c5180fc8 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -1,9 +1,9 @@ # Scrapy settings for $project_name project # # For simplicity, this file contains only the most important settings by -# default. All the other settings are documented here +# default. All the other settings are documented here: # -# http://doc.scrapy.org/ref/settings.html +# http://doc.scrapy.org/topics/settings.html # # Or you can copy and paste them from where they're defined in Scrapy: # @@ -12,14 +12,11 @@ import $project_name -PROJECT_NAME = '$project_name' - -BOT_NAME = PROJECT_NAME +BOT_NAME = '$project_name' BOT_VERSION = '1.0' SPIDER_MODULES = ['$project_name.spiders'] NEWSPIDER_MODULE = '$project_name.spiders' -TEMPLATES_DIR = '%s/templates' % $project_name.__path__[0] DEFAULT_ITEM_CLASS = '$project_name.items.${ProjectName}Item' USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) diff --git a/scrapy/templates/project/module/spiders/__init__.py b/scrapy/templates/project/module/spiders/__init__.py index 59d02a55f..5065ccba5 100644 --- a/scrapy/templates/project/module/spiders/__init__.py +++ b/scrapy/templates/project/module/spiders/__init__.py @@ -1 +1,8 @@ -# Place here all your scrapy spiders +# This package will contain the spiders of your Scrapy project +# +# To create the first spider for your project use this command: +# +# scrapy-ctl.py genspider myspider myspider-domain.com +# +# For more info see: +# http://doc.scrapy.org/topics/spiders.html diff --git a/scrapy/templates/project/root/scrapy-ctl.py b/scrapy/templates/project/scrapy-ctl.py similarity index 100% rename from scrapy/templates/project/root/scrapy-ctl.py rename to scrapy/templates/project/scrapy-ctl.py