diff --git a/scrapy/trunk/scrapy/command/commands/genspider.py b/scrapy/trunk/scrapy/command/commands/genspider.py index d442c500a..272c69b95 100644 --- a/scrapy/trunk/scrapy/command/commands/genspider.py +++ b/scrapy/trunk/scrapy/command/commands/genspider.py @@ -1,64 +1,66 @@ +from __future__ import with_statement + import os import string +import shutil from scrapy.spider import spiders from scrapy.command import ScrapyCommand from scrapy.conf import settings +from scrapy.utils.misc import render_templatefile class Command(ScrapyCommand): - - """ Default template file name """ - template_name = 'spider.tmpl' - """ Childs can define custom tvars """ custom_tvars = {} def syntax(self): - return " " + return " [--template=template_name]" def short_desc(self): - return "Generate new spider based on predefined template" + return "Generate new spider based on a predefined template" + + def add_options(self, parser): + ScrapyCommand.add_options(self, parser) + parser.add_option("--template", dest="template", help="Uses a custom template.", default="crawl") def run(self, args, opts): - if len(args) != 2: + if len(args) < 2: return False + template_file = os.path.join(settings['TEMPLATES_DIR'], 'spider_%s.tmpl' % opts.template) + if not os.path.exists(template_file): + print "Template named %s.tmpl does not exist" % opts.template + return + name = args[0] site = args[1] spiders_dict = spiders.asdict() - if not name in spiders_dict.keys(): - self._genspider(name, site) + if not site in spiders_dict.keys(): + name = self.normalize_name(name) + self._genspider(name, site, template_file) else: - print "Spider '%s' exist" % name + print "Spider '%s' already exists" % name - def _genspider(self, name, site): + def normalize_name(self, name): + name = name.replace('-', '_') # - are replaced by _, for valid python modules + if name[0] not in string.letters: # name must start with a letter, for valid python modules + name = "a" + name + print "Spider names must start with a letter; converted to %s." % name + return name + + def _genspider(self, name, site, template_file): """ Generate spider """ tvars = { 'name': name, 'site': site, - 'classname': '%sSpider' % ''.join([s.capitalize() for s in name.split('-')]) + 'classname': '%sSpider' % ''.join([s.capitalize() for s in name.split('_')]) } tvars.update(self.custom_tvars) spiders_module = __import__(settings['NEWSPIDER_MODULE'], {}, {}, ['']) - spidersdir = os.path.abspath(os.path.dirname(spiders_module.__file__)) - if name[0] not in string.letters: # must start with a letter, for valid python modules - name = "a" + name - name = name.replace('-', '_') # - are replaced by _, for valid python modules - self._genfiles(self.template_name, '%s/%s.py' % (spidersdir, name), tvars) + spiders_dir = os.path.abspath(os.path.dirname(spiders_module.__file__)) + spider_file = '%s/%s.py' % (spiders_dir, name) + + shutil.copyfile(template_file, spider_file) + render_templatefile(spider_file, **tvars) - def _genfiles(self, template_name, source_name, tvars): - """ Generate source from template, substitute variables """ - template_file = os.path.join(settings['TEMPLATES_DIR'], template_name) - tmpl = open(template_file) - clines = [] - for l in tmpl.readlines(): - for key, val in tvars.items(): - l = l.replace('@%s@' % key, val) - clines.append(l) - tmpl.close() - source = ''.join(clines) - if not os.path.exists(source_name): - sfile = open(source_name, "w") - sfile.write(source) - sfile.close() diff --git a/scrapy/trunk/scrapy/conf/project_template/templates/spider_crawl.tmpl b/scrapy/trunk/scrapy/conf/project_template/templates/spider_crawl.tmpl new file mode 100644 index 000000000..44db54daf --- /dev/null +++ b/scrapy/trunk/scrapy/conf/project_template/templates/spider_crawl.tmpl @@ -0,0 +1,25 @@ +# -*- coding: utf8 -*- +import re + +from scrapy.xpath import HtmlXPathSelector +from scrapy.link.extractors import RegexLinkExtractor +from scrapy.contrib.spiders import CrawlSpider +from scrapy.utils.url import url_query_parameter + +class $classname(CrawlSpider): + domain_name = "$site" + start_urls = ['http://www.$site/'] + + links_category = RegexLinkExtractor(allow=(re.compile(r'ProductList'), )) + links_product = RegexLinkExtractor(allow=(re.compile(r'ProductDetail'), )) + + def parse_product(self, response): + #xs = HtmlXPathSelector(response) + #p = self.create_product(response) + #p.attribute('site_id', url_query_parameter(response.url,'productid')) + #p.attribute('site_id', xs.x("//input[@name='productid']/@value")) + #p.attribute('site_id', xs.x("")) + #p.attribute('name', xs.x("")) + #p.attribute('description', xs.x("")) + +SPIDER = $classname() diff --git a/scrapy/trunk/scrapy/conf/project_template/templates/spider_xmlfeed.tmpl b/scrapy/trunk/scrapy/conf/project_template/templates/spider_xmlfeed.tmpl new file mode 100644 index 000000000..d5c9f1182 --- /dev/null +++ b/scrapy/trunk/scrapy/conf/project_template/templates/spider_xmlfeed.tmpl @@ -0,0 +1,21 @@ +# -*- coding: utf8 -*- +from scrapy.xpath import HtmlXPathSelector +from scrapy.contrib.spiders import XMLFeedSpider + +class $classname(XMLFeedSpider): + domain_name = "$site" + start_urls = ['http://www.$site/feed.xml'] + + def parse_item(self, response, xSel): + p = self.create_product(response) + #p.attribute('url', xSel("")) + #p.attribute('supplier', self.domain_name) + #p.attribute('site_id', xSel("")) + #p.attribute('name', xSel("")) + #p.attribute('description', xSel("")) + #p.attribute('image_urls', xSel("")) + #p.attribute('price', xSel("")) + #p.attribute('dimensions', xSel("")) + return p + +SPIDER = $classname()