From 68f9fcffe8030b65b0c01f08e27ba2d8eee1313b Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 21 Aug 2010 04:46:48 -0300 Subject: [PATCH] genspider command refactoring. Also updated tests and doc --- docs/topics/commands.rst | 39 ++++++++++++++++------ scrapy/commands/genspider.py | 61 +++++++++++++---------------------- scrapy/tests/test_commands.py | 38 +++++++++++----------- 3 files changed, 73 insertions(+), 65 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 82c0b4bb8..d2ef2a1a4 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -113,23 +113,44 @@ Usage example:: genspider --------- -+-------------------+--------------------------------------+ -| Syntax: | ``scrapy genspider `` | -+-------------------+--------------------------------------+ -| Requires project: | *yes* | -+-------------------+--------------------------------------+ ++-------------------+----------------------------------------------------+ +| Syntax: | ``scrapy genspider [-t template] `` | ++-------------------+----------------------------------------------------+ +| Requires project: | *yes* | ++-------------------+----------------------------------------------------+ Create a new spider in the current project. This is just a convenient shortcut command for creating spiders based on pre-defined templates, but certainly not the only way to create spiders. You -can just create the spider source code files yourself. +can just create the spider source code files yourself, instead of using this +command. Usage example:: - $ scrapy genspider example example.com - Created spider 'example' using template 'crawl' in module: - jobsbot.spiders.example + $ scrapy genspider -l + Available templates: + basic + crawl + csvfeed + xmlfeed + + $ scrapy genspider -d basic + from scrapy.spider import BaseSpider + + class $classname(BaseSpider): + name = "$name" + allowed_domains = ["$domain"] + start_urls = ( + 'http://www.$domain/', + ) + + def parse(self, response): + pass + + $ scrapy genspider -t basic example example.com + Created spider 'example' using template 'basic' in module: + mybot.spiders.example .. command:: crawl diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 685541fa0..e8abafac1 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -1,8 +1,7 @@ -import sys import shutil import string from os import listdir -from os.path import join, dirname, abspath, exists +from os.path import join, dirname, abspath, exists, splitext import scrapy from scrapy.spider import spiders @@ -10,10 +9,6 @@ from scrapy.command import ScrapyCommand from scrapy.conf import settings from scrapy.utils.template import render_templatefile, string_camelcase - -SPIDER_TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'spiders') - - def sanitize_module_name(module_name): """Sanitize the given module name, by replacing dashes and points with underscores and prefixing it with a letter if it doesn't start @@ -24,10 +19,14 @@ def sanitize_module_name(module_name): module_name = "a" + module_name return module_name +_templates_base_dir = settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], \ + 'templates') + class Command(ScrapyCommand): requires_project = True default_settings = {'LOG_ENABLED': False} + templates_dir = join(_templates_base_dir, 'spiders') def syntax(self): return "[options] " @@ -37,8 +36,10 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--list", dest="list", action="store_true") - parser.add_option("--dump", dest="dump", action="store_true") + parser.add_option("-l", "--list", dest="list", action="store_true", + help="List available templates") + parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE", + help="Dump template to standard output") parser.add_option("-t", "--template", dest="template", default="crawl", help="Uses a custom template.") parser.add_option("--force", dest="force", action="store_true", @@ -48,33 +49,26 @@ class Command(ScrapyCommand): if opts.list: self._list_templates() return - if opts.dump: - template_file = self._find_template(opts.template) + template_file = self._find_template(opts.dump) if template_file: - template = open(template_file, 'r') - print template.read() + print open(template_file, 'r').read() return - if len(args) != 2: return False - name = args[0] - domain = args[1] - + name, domain = args[0:2] module = sanitize_module_name(name) - - # if spider already exists and not force option then halt try: spider = spiders.create(name) except KeyError: pass else: + # if spider already exists and not --force then halt if not opts.force: - print "Spider '%s' already exists in module:" % name + print "Spider %r already exists in module:" % name print " %s" % spider.__module__ - sys.exit(1) - + return template_file = self._find_template(opts.template) if template_file: self._genspider(module, name, domain, opts.template, template_file) @@ -90,11 +84,9 @@ class Command(ScrapyCommand): 'classname': '%sSpider' % ''.join([s.capitalize() \ for s in module.split('_')]) } - spiders_module = __import__(settings['NEWSPIDER_MODULE'], {}, {}, ['']) spiders_dir = abspath(dirname(spiders_module.__file__)) spider_file = "%s.py" % join(spiders_dir, module) - shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) print "Created spider %r using template %r in module:" % (name, \ @@ -102,22 +94,15 @@ class Command(ScrapyCommand): print " %s.%s" % (spiders_module.__name__, module) def _find_template(self, template): - template_file = join(settings['TEMPLATES_DIR'], 'spiders', '%s.tmpl' % template) - if not exists(template_file): - template_file = join(SPIDER_TEMPLATES_PATH, '%s.tmpl' % template) - if not exists(template_file): - print "Unable to find template %r." \ - % template - print "Use genspider --list to see all available templates." - return None - return template_file + template_file = join(self.templates_dir, '%s.tmpl' % template) + if exists(template_file): + return template_file + print "Unable to find template: %s\n" % template + print 'Use "scrapy genspider --list" to see all available templates.' def _list_templates(self): - files = set(listdir(SPIDER_TEMPLATES_PATH)) - if exists(settings['TEMPLATES_DIR']): - files.update(listdir(join(settings['TEMPLATES_DIR'], 'spiders'))) - - for filename in sorted(files): + print "Available templates:" + for filename in sorted(listdir(self.templates_dir)): if filename.endswith('.tmpl'): - print filename + print " %s" % splitext(filename)[0] diff --git a/scrapy/tests/test_commands.py b/scrapy/tests/test_commands.py index 6df70fd2c..7ece8a62b 100644 --- a/scrapy/tests/test_commands.py +++ b/scrapy/tests/test_commands.py @@ -32,8 +32,7 @@ class ProjectTest(unittest.TestCase): return subprocess.call(args, stdout=out, stderr=out, cwd=self.cwd, \ env=self.env, **kwargs) - def call_get_proc(self, *new_args, **kwargs): - out = os.tmpfile() + def proc(self, *new_args, **kwargs): args = (sys.executable, '-m', 'scrapy.cmdline') + new_args return subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, \ cwd=self.cwd, env=self.env, **kwargs) @@ -76,29 +75,32 @@ class GenspiderCommandTest(CommandTest): self.assertEqual(0, self.call('genspider', 'test_name', 'test.com')) assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) - def test_template_default(self, *args): - self.assertEqual(0, self.call('genspider', 'test_spider', 'test.com', *args)) - assert exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')) - self.assertEqual(1, self.call('genspider', 'test_spider', 'test.com')) + def test_template(self, tplname='crawl'): + args = ['--template=%s' % tplname] if tplname else [] + spname = 'test_spider' + p = self.proc('genspider', spname, 'test.com', *args) + out = p.stdout.read() + self.assert_("Created spider %r using template %r in module" % (spname, tplname) in out) + self.assert_(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) + p = self.proc('genspider', spname, 'test.com', *args) + out = p.stdout.read() + self.assert_("Spider %r already exists in module" % spname in out) def test_template_basic(self): - self.test_template_default('--template=basic') + self.test_template('basic') def test_template_csvfeed(self): - self.test_template_default('--template=csvfeed') + self.test_template('csvfeed') def test_template_xmlfeed(self): - self.test_template_default('--template=xmlfeed') - - def test_template_crawl(self): - self.test_template_default('--template=crawl') + self.test_template('xmlfeed') def test_list(self): self.assertEqual(0, self.call('genspider', '--list')) def test_dump(self): - self.assertEqual(0, self.call('genspider', '--dump')) - self.assertEqual(0, self.call('genspider', '--dump', '--template=basic')) + self.assertEqual(0, self.call('genspider', '--dump=basic')) + self.assertEqual(0, self.call('genspider', '-d', 'basic')) class MiscCommandsTest(CommandTest): @@ -127,7 +129,7 @@ class MySpider(BaseSpider): self.log("It Works!") return [] """) - p = self.call_get_proc('runspider', fname) + p = self.proc('runspider', fname) log = p.stderr.read() self.assert_("[myspider] DEBUG: It Works!" in log) self.assert_("[myspider] INFO: Spider opened" in log) @@ -143,12 +145,12 @@ class MySpider(BaseSpider): from scrapy import log from scrapy.spider import BaseSpider """) - p = self.call_get_proc('runspider', fname) + p = self.proc('runspider', fname) log = p.stderr.read() self.assert_("ERROR: No spider found in file" in log) def test_runspider_file_not_found(self): - p = self.call_get_proc('runspider', 'some_non_existent_file') + p = self.proc('runspider', 'some_non_existent_file') log = p.stderr.read() self.assert_("ERROR: File not found: some_non_existent_file" in log) @@ -158,7 +160,7 @@ from scrapy.spider import BaseSpider fname = abspath(join(tmpdir, 'myspider.txt')) with open(fname, 'w') as f: f.write("") - p = self.call_get_proc('runspider', fname) + p = self.proc('runspider', fname) log = p.stderr.read() self.assert_("ERROR: Unable to load" in log)