more cleanups to startproject and project templates

--HG--
rename : scrapy/templates/project/root/scrapy-ctl.py => scrapy/templates/project/scrapy-ctl.py

docs/topics/settings.rst

@@ -184,8 +184,13 @@ BOT_NAME
 
 Default: ``scrapybot``
 
-The name of the bot implemented by this Scrapy project. This will be used to
-construct the User-Agent by default, and also for logging.
+The name of the bot implemented by this Scrapy project (also known as the
+project name). This will be used to construct the User-Agent by default, and
+also for logging.
+
+It's automatically populated with your project name when you create your
+project with the :doc:`scrapy-ctl.py </topics/scrapy-ctl>` ``startproject``
+command.
 
 .. setting:: BOT_VERSION
 
@@ -696,17 +701,6 @@ Example::
 
     NEWSPIDER_MODULE = 'mybot.spiders_dev'
 
-.. setting:: PROJECT_NAME
-
-PROJECT_NAME
-------------
-
-Default: ``Not Defined``
-
-The name of the current project. It matches the project module name as created
-by :ref:`scrapy-ctl.py startproject <topics-scripts-scrapy-ctl-startproject>` command,
-and is only defined by project settings file.
-
 .. setting:: REDIRECT_MAX_TIMES
 
 REDIRECT_MAX_TIMES

examples/googledir/googledir/settings.py

@@ -2,14 +2,11 @@
 
 import googledir
 
-PROJECT_NAME = 'googledir'
-
-BOT_NAME = PROJECT_NAME
+BOT_NAME = 'googledir'
 BOT_VERSION = '1.0'
 
 SPIDER_MODULES = ['googledir.spiders']
 NEWSPIDER_MODULE = 'googledir.spiders'
-TEMPLATES_DIR = '%s/templates' % googledir.__path__[0]
 DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem'
 USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
 

scrapy/command/commands/genspider.py

@@ -72,8 +72,8 @@ class Command(ScrapyCommand):
     def _genspider(self, module, domain, template_name, template_file):
         """Generate the spider module, based on the given template"""
         tvars = {
-            'project_name': settings.get('PROJECT_NAME'),
-            'ProjectName': string_camelcase(settings.get('PROJECT_NAME')),
+            'project_name': settings.get('BOT_NAME'),
+            'ProjectName': string_camelcase(settings.get('BOT_NAME')),
             'module': module,
             'site': domain,
             'classname': '%sSpider' % ''.join([s.capitalize() \

scrapy/command/commands/startproject.py

@@ -1,20 +1,17 @@
-#!/usr/bin/env python
-
 import sys
-import os
 import string
 import re
+import shutil
+from os.path import join, exists
 
 import scrapy
 from scrapy.command import ScrapyCommand
 from scrapy.utils.template import render_templatefile, string_camelcase
 from scrapy.utils.python import ignore_patterns, copytree
 
-PROJECT_TEMPLATES_PATH = os.path.join(scrapy.__path__[0], 'templates/project')
+TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'project')
 
-# This is the list of templatefile's path that are rendered *after copying* to
-# the new project directory.
-TEMPLATES = (
+TEMPLATES_TO_RENDER = (
     ('scrapy-ctl.py',),
     ('${project_name}', 'settings.py.tmpl'),
     ('${project_name}', 'items.py.tmpl'),
@@ -36,33 +33,21 @@ class Command(ScrapyCommand):
     def run(self, args, opts):
         if len(args) != 1:
             return False
-
         project_name = args[0]
-        if not re.search(r'^[_a-zA-Z]\w*$', project_name): # If it's not a valid directory name.
-            # Provide a smart error message, depending on the error.
-            if not re.search(r'^[a-zA-Z]', project_name):
-                message = 'Project names must begin with a letter'
-            else:
-                message = 'Project names must contain only letters, numbers and underscores'
-            print "Invalid project name: %s\n\n%s" % (project_name, message)
+        if not re.search(r'^[_a-zA-Z]\w*$', project_name):
+            print 'Error: Project names must begin with a letter and contain only\n' \
+                'letters, numbers and underscores'
+            sys.exit(1)
+        elif exists(project_name):
+            print "Error: directory %r already exists" % project_name
             sys.exit(1)
-        else:
-            if os.path.exists(project_name):
-                print "%s dir already exists" % project_name
-                sys.exit(1)
 
-            project_root_path = project_name
-
-            roottpl = os.path.join(PROJECT_TEMPLATES_PATH, 'root')
-            copytree(roottpl, project_name, ignore=IGNORE)
-
-            moduletpl = os.path.join(PROJECT_TEMPLATES_PATH, 'module')
-            copytree(moduletpl, '%s/%s' % (project_name, project_name),
-                ignore=IGNORE)
-
-            for paths in TEMPLATES:
-                path = os.path.join(*paths)
-                tplfile = os.path.join(project_root_path,
-                    string.Template(path).substitute(project_name=project_name))
-                render_templatefile(tplfile, project_name=project_name,
-                    ProjectName=string_camelcase(project_name))
+        moduletpl = join(TEMPLATES_PATH, 'module')
+        copytree(moduletpl, join(project_name, project_name), ignore=IGNORE)
+        shutil.copy(join(TEMPLATES_PATH, 'scrapy-ctl.py'), project_name)
+        for paths in TEMPLATES_TO_RENDER:
+            path = join(*paths)
+            tplfile = join(project_name,
+                string.Template(path).substitute(project_name=project_name))
+            render_templatefile(tplfile, project_name=project_name,
+                ProjectName=string_camelcase(project_name))

scrapy/templates/project/module/items.py.tmpl

@@ -1,4 +1,7 @@
 # Define here the models for your scraped items
+#
+# See documentation in:
+# http://doc.scrapy.org/topics/items.html
 
 from scrapy.item import Item, Field
 

scrapy/templates/project/module/pipelines.py.tmpl

@@ -1,7 +1,7 @@
 # Define your item pipelines here
 #
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
-# See: http://doc.scrapy.org/ref/settings.html#item-pipelines
+# See: http://doc.scrapy.org/topics/item-pipeline.html
 
 class ${ProjectName}Pipeline(object):
     def process_item(self, domain, item):

scrapy/templates/project/module/settings.py.tmpl

@@ -1,9 +1,9 @@
 # Scrapy settings for $project_name project
 #
 # For simplicity, this file contains only the most important settings by
-# default. All the other settings are documented here
+# default. All the other settings are documented here:
 #
-#     http://doc.scrapy.org/ref/settings.html
+#     http://doc.scrapy.org/topics/settings.html
 #
 # Or you can copy and paste them from where they're defined in Scrapy:
 # 
@@ -12,14 +12,11 @@
 
 import $project_name
 
-PROJECT_NAME = '$project_name'
-
-BOT_NAME = PROJECT_NAME
+BOT_NAME = '$project_name'
 BOT_VERSION = '1.0'
 
 SPIDER_MODULES = ['$project_name.spiders']
 NEWSPIDER_MODULE = '$project_name.spiders'
-TEMPLATES_DIR = '%s/templates' % $project_name.__path__[0]
 DEFAULT_ITEM_CLASS = '$project_name.items.${ProjectName}Item'
 USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
 

scrapy/templates/project/module/spiders/__init__.py

@@ -1 +1,8 @@
-# Place here all your scrapy spiders
+# This package will contain the spiders of your Scrapy project
+#
+# To create the first spider for your project use this command:
+#
+#   scrapy-ctl.py genspider myspider myspider-domain.com
+#
+# For more info see:
+# http://doc.scrapy.org/topics/spiders.html