removed spider templates from project, addeded sumcommands to manage templates in genspider

scrapy/command/commands/genspider.py

@@ -1,12 +1,18 @@
 import shutil
 import string
+from os import listdir
 from os.path import join, dirname, abspath, exists
 
+import scrapy
 from scrapy.spider import spiders
 from scrapy.command import ScrapyCommand
 from scrapy.conf import settings
 from scrapy.utils.template import render_templatefile, string_camelcase
 
+
+SPIDER_TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'spiders')
+
+
 def sanitize_module_name(module_name):
     """Sanitize the given module name, by replacing dashes with underscores and
     prefixing it with a letter if it doesn't start with one
@@ -22,26 +28,32 @@ class Command(ScrapyCommand):
         return "[options] <spider_module_name> <spider_domain_name>"
 
     def short_desc(self):
-        return "Generate new spider based on template passed with --template"
+        return "Generate new spider based on template passed with -t or --template"
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_option("--template", dest="template", default="crawl",
+        parser.add_option("--list", dest="list", action="store_true")
+        parser.add_option("--dump", dest="dump", action="store_true")
+        parser.add_option("-t", "--template", dest="template", default="crawl",
             help="Uses a custom template.")
         parser.add_option("--force", dest="force", action="store_true",
             help="If the spider already exists, overwrite it with the template")
 
     def run(self, args, opts):
+        if opts.list:
+            self._list_templates()
+            return
+
+        if opts.dump:
+            template_file = self._find_template(opts.template)
+            if template_file:
+                template = open(template_file, 'r')
+                print template.read() 
+            return
+
         if len(args) < 2:
             return False
 
-        template_file = join(settings['TEMPLATES_DIR'], 'spider_%s.tmpl' % \
-            opts.template)
-        if not exists(template_file):
-            print "Unable to create spider: template %r not found." % opts.template
-            print "Use genspider --list to see all available templates."
-            return
-
         module = sanitize_module_name(args[0])
         domain = args[1]
         spider = spiders.fromdomain(domain)
@@ -49,7 +61,10 @@ class Command(ScrapyCommand):
             print "Spider '%s' already exists in module:" % domain
             print "  %s" % spider.__module__
             return
-        self._genspider(module, domain, opts.template, template_file)
+
+        template_file = self._find_template(opts.template)
+        if template_file:
+            self._genspider(module, domain, opts.template, template_file)
 
     def _genspider(self, module, domain, template_name, template_file):
         """Generate the spider module, based on the given template"""
@@ -71,3 +86,25 @@ class Command(ScrapyCommand):
         print "Created spider %r using template %r in module:" % (domain, \
             template_name)
         print "  %s.%s" % (spiders_module.__name__, module)
+
+    def _find_template(self, template):
+        template_file = join(settings['TEMPLATES_DIR'], '%s.tmpl' % template)
+        if not exists(template_file):
+            template_file = join(SPIDER_TEMPLATES_PATH, '%s.tmpl' % template)
+            if not exists(template_file):
+                print "Unable to find template %r." \
+                        % template
+                print "Use genspider --list to see all available templates."
+                return None
+        return template_file
+
+    def _list_templates(self):
+        files = set()
+        if exists(settings['TEMPLATES_DIR']):
+            files.update(listdir(settings['TEMPLATES_DIR']))
+        files.update(listdir(SPIDER_TEMPLATES_PATH))
+
+        for filename in sorted(files):
+            if filename.endswith('.tmpl'):
+                print filename
+

scrapy/templates/spiders/xmlfeed.tmpl

@@ -6,11 +6,11 @@ class $classname(XMLFeedSpider):
     domain_name = '$site'
     start_urls = ['http://www.$site/feed.xml']
 
-    def parse_item(self, response, xSel):
+    def parse_item(self, response, selector):
         i = ${ProjectName}Item()
-        #i['url'] = xSel('url'.extract()
-        #i['name'] = xSel('name').extract()
-        #i['description'] = xSel('description').extract()
+        #i['url'] = selector.select('url').extract()
+        #i['name'] = selector.select('name').extract()
+        #i['description'] = selector.select('description').extract()
         return i
 
 SPIDER = $classname()