RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

some cleanup to genspider command, disabled log and fixed inconsistencies reported in #55

This commit is contained in:
Pablo Hoffman 2009-07-21 11:31:14 -03:00
parent 29c92fe818
commit 4a10e1dabd
2 changed files with 23 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
scrapy/command/commands/genspider.py
View File

@ -1,8 +1,6 @@
from __future__ import with_statement
import os
import string
import shutil
import string
from os.path import join, dirname, abspath, exists
from scrapy.spider import spiders
from scrapy.command import ScrapyCommand
@ -11,8 +9,6 @@ from scrapy.utils.misc import render_templatefile, string_camelcase
class Command(ScrapyCommand):
""" Childs can define custom tvars """
custom_tvars = {}
def syntax(self):
return "[options] <spider_name> <spider_domain_name>"
@ -22,49 +18,52 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--template", dest="template", help="Uses a custom template.", default="crawl")
parser.add_option("--force", dest="force", help="If the spider already exists, overwrite it with the template", action="store_true")
parser.add_option("--template", dest="template", default="crawl",
help="Uses a custom template.")
parser.add_option("--force", dest="force", action="store_true",
help="If the spider already exists, overwrite it with the template")
def run(self, args, opts):
if len(args) < 2:
return False
template_file = os.path.join(settings['TEMPLATES_DIR'], 'spider_%s.tmpl' % opts.template)
if not os.path.exists(template_file):
print "Template named %s.tmpl does not exist" % opts.template
template_file = join(settings['TEMPLATES_DIR'], 'spider_%s.tmpl' % opts.template)
if not exists(template_file):
print "Template '%s.tmpl' not found" % opts.template
return
name = self.normalize_name(args[0])
site = args[1]
domain = args[1]
spiders_dict = spiders.asdict()
if site in spiders_dict.keys():
if domain in spiders_dict.keys():
if opts.force:
print "Spider '%s' already exists. Overwriting it..." % name
print "Spider '%s' already exists. Overwriting it..." % domain
else:
print "Spider '%s' already exists" % name
print "Spider '%s' already exists" % domain
return
self._genspider(name, site, template_file)
self._genspider(name, domain, template_file)
def normalize_name(self, name):
name = name.replace('-', '_') # - are replaced by _, for valid python modules
if name[0] not in string.letters: # name must start with a letter, for valid python modules
# - are replaced by _, for valid python modules
name = name.replace('-', '_')
# name must start with a letter, for valid python modules
if name[0] not in string.letters:
name = "a" + name
print "Spider names must start with a letter; converted to %s." % name
return name
def _genspider(self, name, site, template_file):
""" Generate spider """
def _genspider(self, name, domain, template_file):
"""Generate the spider module, based on the given template"""
tvars = {
'project_name': settings.get('PROJECT_NAME'),
'ProjectName': string_camelcase(settings.get('PROJECT_NAME')),
'name': name,
'site': site,
'site': domain,
'classname': '%sSpider' % ''.join([s.capitalize() for s in name.split('_')])
}
tvars.update(self.custom_tvars)
spiders_module = __import__(settings['NEWSPIDER_MODULE'], {}, {}, [''])
spiders_dir = os.path.abspath(os.path.dirname(spiders_module.__file__))
spiders_dir = abspath(dirname(spiders_module.__file__))
spider_file = '%s/%s.py' % (spiders_dir, name)
shutil.copyfile(template_file, spider_file)

1
scrapy/conf/commands/genspider.py Normal file
View File

@ -0,0 +1 @@
LOG_ENABLED = False