diff --git a/scrapy/command/commands/genspider.py b/scrapy/command/commands/genspider.py index f25bfa63a..b5a1fde64 100644 --- a/scrapy/command/commands/genspider.py +++ b/scrapy/command/commands/genspider.py @@ -1,8 +1,6 @@ -from __future__ import with_statement - -import os -import string import shutil +import string +from os.path import join, dirname, abspath, exists from scrapy.spider import spiders from scrapy.command import ScrapyCommand @@ -11,8 +9,6 @@ from scrapy.utils.misc import render_templatefile, string_camelcase class Command(ScrapyCommand): - """ Childs can define custom tvars """ - custom_tvars = {} def syntax(self): return "[options] " @@ -22,49 +18,52 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--template", dest="template", help="Uses a custom template.", default="crawl") - parser.add_option("--force", dest="force", help="If the spider already exists, overwrite it with the template", action="store_true") + parser.add_option("--template", dest="template", default="crawl", + help="Uses a custom template.") + parser.add_option("--force", dest="force", action="store_true", + help="If the spider already exists, overwrite it with the template") def run(self, args, opts): if len(args) < 2: return False - template_file = os.path.join(settings['TEMPLATES_DIR'], 'spider_%s.tmpl' % opts.template) - if not os.path.exists(template_file): - print "Template named %s.tmpl does not exist" % opts.template + template_file = join(settings['TEMPLATES_DIR'], 'spider_%s.tmpl' % opts.template) + if not exists(template_file): + print "Template '%s.tmpl' not found" % opts.template return name = self.normalize_name(args[0]) - site = args[1] + domain = args[1] spiders_dict = spiders.asdict() - if site in spiders_dict.keys(): + if domain in spiders_dict.keys(): if opts.force: - print "Spider '%s' already exists. Overwriting it..." % name + print "Spider '%s' already exists. Overwriting it..." % domain else: - print "Spider '%s' already exists" % name + print "Spider '%s' already exists" % domain return - self._genspider(name, site, template_file) + self._genspider(name, domain, template_file) def normalize_name(self, name): - name = name.replace('-', '_') # - are replaced by _, for valid python modules - if name[0] not in string.letters: # name must start with a letter, for valid python modules + # - are replaced by _, for valid python modules + name = name.replace('-', '_') + # name must start with a letter, for valid python modules + if name[0] not in string.letters: name = "a" + name print "Spider names must start with a letter; converted to %s." % name return name - def _genspider(self, name, site, template_file): - """ Generate spider """ + def _genspider(self, name, domain, template_file): + """Generate the spider module, based on the given template""" tvars = { 'project_name': settings.get('PROJECT_NAME'), 'ProjectName': string_camelcase(settings.get('PROJECT_NAME')), 'name': name, - 'site': site, + 'site': domain, 'classname': '%sSpider' % ''.join([s.capitalize() for s in name.split('_')]) } - tvars.update(self.custom_tvars) spiders_module = __import__(settings['NEWSPIDER_MODULE'], {}, {}, ['']) - spiders_dir = os.path.abspath(os.path.dirname(spiders_module.__file__)) + spiders_dir = abspath(dirname(spiders_module.__file__)) spider_file = '%s/%s.py' % (spiders_dir, name) shutil.copyfile(template_file, spider_file) diff --git a/scrapy/conf/commands/genspider.py b/scrapy/conf/commands/genspider.py new file mode 100644 index 000000000..bcee80b55 --- /dev/null +++ b/scrapy/conf/commands/genspider.py @@ -0,0 +1 @@ +LOG_ENABLED = False