diff --git a/scrapy/command/commands/genspider.py b/scrapy/command/commands/genspider.py index b5a1fde64..64ca7ae55 100644 --- a/scrapy/command/commands/genspider.py +++ b/scrapy/command/commands/genspider.py @@ -5,13 +5,21 @@ from os.path import join, dirname, abspath, exists from scrapy.spider import spiders from scrapy.command import ScrapyCommand from scrapy.conf import settings -from scrapy.utils.misc import render_templatefile, string_camelcase +from scrapy.utils.template import render_templatefile, string_camelcase +def sanitize_module_name(module_name): + """Sanitize the given module name, by replacing dashes with underscores and + prefixing it with a letter if it doesn't start with one + """ + module_name = module_name.replace('-', '_') + if module_name[0] not in string.letters: + module_name = "a" + module_name + return module_name class Command(ScrapyCommand): def syntax(self): - return "[options] " + return "[options] " def short_desc(self): return "Generate new spider based on template passed with --template" @@ -27,45 +35,39 @@ class Command(ScrapyCommand): if len(args) < 2: return False - template_file = join(settings['TEMPLATES_DIR'], 'spider_%s.tmpl' % opts.template) + template_file = join(settings['TEMPLATES_DIR'], 'spider_%s.tmpl' % \ + opts.template) if not exists(template_file): - print "Template '%s.tmpl' not found" % opts.template + print "Unable to create spider: template %r not found." % opts.template + print "Use genspider --list to see all available templates." return - name = self.normalize_name(args[0]) + module = sanitize_module_name(args[0]) domain = args[1] - spiders_dict = spiders.asdict() - if domain in spiders_dict.keys(): - if opts.force: - print "Spider '%s' already exists. Overwriting it..." % domain - else: - print "Spider '%s' already exists" % domain - return - self._genspider(name, domain, template_file) + spider = spiders.fromdomain(domain) + if spider and not opts.force: + print "Spider '%s' already exists in module:" % domain + print " %s" % spider.__module__ + return + self._genspider(module, domain, opts.template, template_file) - def normalize_name(self, name): - # - are replaced by _, for valid python modules - name = name.replace('-', '_') - # name must start with a letter, for valid python modules - if name[0] not in string.letters: - name = "a" + name - print "Spider names must start with a letter; converted to %s." % name - return name - - def _genspider(self, name, domain, template_file): + def _genspider(self, module, domain, template_name, template_file): """Generate the spider module, based on the given template""" tvars = { 'project_name': settings.get('PROJECT_NAME'), 'ProjectName': string_camelcase(settings.get('PROJECT_NAME')), - 'name': name, + 'module': module, 'site': domain, - 'classname': '%sSpider' % ''.join([s.capitalize() for s in name.split('_')]) + 'classname': '%sSpider' % ''.join([s.capitalize() \ + for s in module.split('_')]) } spiders_module = __import__(settings['NEWSPIDER_MODULE'], {}, {}, ['']) spiders_dir = abspath(dirname(spiders_module.__file__)) - spider_file = '%s/%s.py' % (spiders_dir, name) + spider_file = "%s.py" % join(spiders_dir, module) shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) - + print "Created spider %r using template %r in module:" % (domain, \ + template_name) + print " %s.%s" % (spiders_module.__name__, module) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index dac86abd1..c1d7214e1 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -7,7 +7,6 @@ from contextlib import closing import os import re import gzip -import string import hashlib import csv @@ -128,19 +127,6 @@ def hash_values(*values): hash.update(value) return hash.hexdigest() - -def render_templatefile(path, **kwargs): - with open(path, 'rb') as file: - raw = file.read() - - content = string.Template(raw).substitute(**kwargs) - - with open(path.rstrip('.tmpl'), 'wb') as file: - file.write(content) - if path.endswith('.tmpl'): - os.remove(path) - - def items_to_csv(file, items, delimiter=';', headers=None): """ This function takes a list of items and stores their attributes @@ -168,20 +154,6 @@ def items_to_csv(file, items, delimiter=';', headers=None): csv_file.writerow(row) -CAMELCASE_INVALID_CHARS = re.compile('[^a-zA-Z]') -def string_camelcase(string): - """ Convert a word to its CamelCase version and remove invalid chars - - >>> string_camelcase('lost-pound') - 'LostPound' - - >>> string_camelcase('missing_images') - 'MissingImages' - - """ - return CAMELCASE_INVALID_CHARS.sub('', string.title()) - - def md5sum(buffer): """Calculate the md5 checksum of a file @@ -194,7 +166,8 @@ def md5sum(buffer): buffer.seek(0) while 1: d = buffer.read(8096) - if not d: break + if not d: + break m.update(d) return m.hexdigest() diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py new file mode 100644 index 000000000..406243459 --- /dev/null +++ b/scrapy/utils/template.py @@ -0,0 +1,29 @@ +"""Helper functions for working with templates""" + +import os +import re +import string + +def render_templatefile(path, **kwargs): + with open(path, 'rb') as file: + raw = file.read() + + content = string.Template(raw).substitute(**kwargs) + + with open(path.rstrip('.tmpl'), 'wb') as file: + file.write(content) + if path.endswith('.tmpl'): + os.remove(path) + +CAMELCASE_INVALID_CHARS = re.compile('[^a-zA-Z]') +def string_camelcase(string): + """ Convert a word to its CamelCase version and remove invalid chars + + >>> string_camelcase('lost-pound') + 'LostPound' + + >>> string_camelcase('missing_images') + 'MissingImages' + + """ + return CAMELCASE_INVALID_CHARS.sub('', string.title())