Improved genspider command

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40384

scrapy/trunk/scrapy/command/commands/genspider.py

@@ -1,64 +1,66 @@
+from __future__ import with_statement
+
 import os
 import string
+import shutil
 
 from scrapy.spider import spiders
 from scrapy.command import ScrapyCommand
 from scrapy.conf import settings
+from scrapy.utils.misc import render_templatefile
 
 class Command(ScrapyCommand):
-
-    """ Default template file name """
-    template_name = 'spider.tmpl'
-
     """ Childs can define custom tvars """
     custom_tvars = {}
 
     def syntax(self):
-        return "<spider_name> <spider_domain_name>"
+        return "<spider_name> <spider_domain_name> [--template=template_name]"
 
     def short_desc(self):
-        return "Generate new spider based on predefined template"
+        return "Generate new spider based on a predefined template"
+
+    def add_options(self, parser):
+        ScrapyCommand.add_options(self, parser)
+        parser.add_option("--template", dest="template", help="Uses a custom template.", default="crawl")
 
     def run(self, args, opts):
-        if len(args) != 2:
+        if len(args) < 2:
             return False
 
+        template_file = os.path.join(settings['TEMPLATES_DIR'], 'spider_%s.tmpl' % opts.template)
+        if not os.path.exists(template_file):
+            print "Template named %s.tmpl does not exist" % opts.template
+            return
+
         name = args[0]
         site = args[1]
         spiders_dict = spiders.asdict()
-        if not name in spiders_dict.keys():
-            self._genspider(name, site)
+        if not site in spiders_dict.keys():
+            name = self.normalize_name(name)
+            self._genspider(name, site, template_file)
         else:
-            print "Spider '%s' exist" % name
+            print "Spider '%s' already exists" % name
 
-    def _genspider(self, name, site):
+    def normalize_name(self, name):
+        name = name.replace('-', '_') # - are replaced by _, for valid python modules
+        if name[0] not in string.letters: # name must start with a letter, for valid python modules
+            name = "a" + name
+            print "Spider names must start with a letter; converted to %s." % name
+        return name
+        
+    def _genspider(self, name, site, template_file):
         """ Generate spider """
         tvars = {
             'name': name,
             'site': site,
-            'classname': '%sSpider' % ''.join([s.capitalize() for s in name.split('-')])
+            'classname': '%sSpider' % ''.join([s.capitalize() for s in name.split('_')])
         }
         tvars.update(self.custom_tvars)
 
         spiders_module = __import__(settings['NEWSPIDER_MODULE'], {}, {}, [''])
-        spidersdir = os.path.abspath(os.path.dirname(spiders_module.__file__))
-        if name[0] not in string.letters: # must start with a letter, for valid python modules
-            name = "a" + name
-        name = name.replace('-', '_') # - are replaced by _, for valid python modules
-        self._genfiles(self.template_name, '%s/%s.py' % (spidersdir, name), tvars)
+        spiders_dir = os.path.abspath(os.path.dirname(spiders_module.__file__))
+        spider_file = '%s/%s.py' % (spiders_dir, name)
+
+        shutil.copyfile(template_file, spider_file)
+        render_templatefile(spider_file, **tvars)
 
-    def _genfiles(self, template_name, source_name, tvars):
-        """ Generate source from template, substitute variables """
-        template_file = os.path.join(settings['TEMPLATES_DIR'], template_name)
-        tmpl = open(template_file)
-        clines = []
-        for l in tmpl.readlines():
-            for key, val in tvars.items():
-                l = l.replace('@%s@' % key, val)
-            clines.append(l)
-        tmpl.close()
-        source = ''.join(clines)
-        if not os.path.exists(source_name):
-            sfile = open(source_name, "w")
-            sfile.write(source)
-            sfile.close()

scrapy/trunk/scrapy/conf/project_template/templates/spider_crawl.tmpl

@@ -0,0 +1,25 @@
+# -*- coding: utf8 -*-
+import re
+
+from scrapy.xpath import HtmlXPathSelector
+from scrapy.link.extractors import RegexLinkExtractor
+from scrapy.contrib.spiders import CrawlSpider
+from scrapy.utils.url import url_query_parameter
+
+class $classname(CrawlSpider):
+    domain_name = "$site"
+    start_urls = ['http://www.$site/']
+
+    links_category = RegexLinkExtractor(allow=(re.compile(r'ProductList'), ))
+    links_product = RegexLinkExtractor(allow=(re.compile(r'ProductDetail'), ))
+
+    def parse_product(self, response):
+        #xs = HtmlXPathSelector(response)
+        #p = self.create_product(response)
+        #p.attribute('site_id', url_query_parameter(response.url,'productid'))
+        #p.attribute('site_id', xs.x("//input[@name='productid']/@value"))
+        #p.attribute('site_id', xs.x(""))
+        #p.attribute('name', xs.x(""))
+        #p.attribute('description', xs.x(""))
+
+SPIDER = $classname()

scrapy/trunk/scrapy/conf/project_template/templates/spider_xmlfeed.tmpl

@@ -0,0 +1,21 @@
+# -*- coding: utf8 -*-
+from scrapy.xpath import HtmlXPathSelector
+from scrapy.contrib.spiders import XMLFeedSpider
+
+class $classname(XMLFeedSpider):
+    domain_name = "$site"
+    start_urls = ['http://www.$site/feed.xml']
+
+    def parse_item(self, response, xSel):
+        p = self.create_product(response)
+        #p.attribute('url', xSel(""))
+        #p.attribute('supplier', self.domain_name)
+        #p.attribute('site_id', xSel(""))
+        #p.attribute('name', xSel(""))
+        #p.attribute('description', xSel(""))
+        #p.attribute('image_urls', xSel(""))
+        #p.attribute('price', xSel(""))
+        #p.attribute('dimensions', xSel(""))
+        return p
+
+SPIDER = $classname()