diff --git a/scrapy/command/commands/genspider.py b/scrapy/command/commands/genspider.py index 64ca7ae55..4ab23f6f8 100644 --- a/scrapy/command/commands/genspider.py +++ b/scrapy/command/commands/genspider.py @@ -1,12 +1,18 @@ import shutil import string +from os import listdir from os.path import join, dirname, abspath, exists +import scrapy from scrapy.spider import spiders from scrapy.command import ScrapyCommand from scrapy.conf import settings from scrapy.utils.template import render_templatefile, string_camelcase + +SPIDER_TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'spiders') + + def sanitize_module_name(module_name): """Sanitize the given module name, by replacing dashes with underscores and prefixing it with a letter if it doesn't start with one @@ -22,26 +28,32 @@ class Command(ScrapyCommand): return "[options] " def short_desc(self): - return "Generate new spider based on template passed with --template" + return "Generate new spider based on template passed with -t or --template" def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--template", dest="template", default="crawl", + parser.add_option("--list", dest="list", action="store_true") + parser.add_option("--dump", dest="dump", action="store_true") + parser.add_option("-t", "--template", dest="template", default="crawl", help="Uses a custom template.") parser.add_option("--force", dest="force", action="store_true", help="If the spider already exists, overwrite it with the template") def run(self, args, opts): + if opts.list: + self._list_templates() + return + + if opts.dump: + template_file = self._find_template(opts.template) + if template_file: + template = open(template_file, 'r') + print template.read() + return + if len(args) < 2: return False - template_file = join(settings['TEMPLATES_DIR'], 'spider_%s.tmpl' % \ - opts.template) - if not exists(template_file): - print "Unable to create spider: template %r not found." % opts.template - print "Use genspider --list to see all available templates." - return - module = sanitize_module_name(args[0]) domain = args[1] spider = spiders.fromdomain(domain) @@ -49,7 +61,10 @@ class Command(ScrapyCommand): print "Spider '%s' already exists in module:" % domain print " %s" % spider.__module__ return - self._genspider(module, domain, opts.template, template_file) + + template_file = self._find_template(opts.template) + if template_file: + self._genspider(module, domain, opts.template, template_file) def _genspider(self, module, domain, template_name, template_file): """Generate the spider module, based on the given template""" @@ -71,3 +86,25 @@ class Command(ScrapyCommand): print "Created spider %r using template %r in module:" % (domain, \ template_name) print " %s.%s" % (spiders_module.__name__, module) + + def _find_template(self, template): + template_file = join(settings['TEMPLATES_DIR'], '%s.tmpl' % template) + if not exists(template_file): + template_file = join(SPIDER_TEMPLATES_PATH, '%s.tmpl' % template) + if not exists(template_file): + print "Unable to find template %r." \ + % template + print "Use genspider --list to see all available templates." + return None + return template_file + + def _list_templates(self): + files = set() + if exists(settings['TEMPLATES_DIR']): + files.update(listdir(settings['TEMPLATES_DIR'])) + files.update(listdir(SPIDER_TEMPLATES_PATH)) + + for filename in sorted(files): + if filename.endswith('.tmpl'): + print filename + diff --git a/scrapy/templates/project/module/templates/spider_basic.tmpl b/scrapy/templates/spiders/basic.tmpl similarity index 100% rename from scrapy/templates/project/module/templates/spider_basic.tmpl rename to scrapy/templates/spiders/basic.tmpl diff --git a/scrapy/templates/project/module/templates/spider_crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl similarity index 100% rename from scrapy/templates/project/module/templates/spider_crawl.tmpl rename to scrapy/templates/spiders/crawl.tmpl diff --git a/scrapy/templates/project/module/templates/spider_csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl similarity index 100% rename from scrapy/templates/project/module/templates/spider_csvfeed.tmpl rename to scrapy/templates/spiders/csvfeed.tmpl diff --git a/scrapy/templates/project/module/templates/spider_xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl similarity index 57% rename from scrapy/templates/project/module/templates/spider_xmlfeed.tmpl rename to scrapy/templates/spiders/xmlfeed.tmpl index 22da2fec8..2ca44b752 100644 --- a/scrapy/templates/project/module/templates/spider_xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -6,11 +6,11 @@ class $classname(XMLFeedSpider): domain_name = '$site' start_urls = ['http://www.$site/feed.xml'] - def parse_item(self, response, xSel): + def parse_item(self, response, selector): i = ${ProjectName}Item() - #i['url'] = xSel('url'.extract() - #i['name'] = xSel('name').extract() - #i['description'] = xSel('description').extract() + #i['url'] = selector.select('url').extract() + #i['name'] = selector.select('name').extract() + #i['description'] = selector.select('description').extract() return i SPIDER = $classname()