RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

genspider command refactoring. Also updated tests and doc

This commit is contained in:
Pablo Hoffman 2010-08-21 04:46:48 -03:00
parent 0da6132136
commit 68f9fcffe8
3 changed files with 73 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
docs/topics/commands.rst
View File

@ -113,23 +113,44 @@ Usage example::
genspider
---------
+-------------------+--------------------------------------+
| Syntax: | ``scrapy genspider <name> <domain>`` |
+-------------------+--------------------------------------+
| Requires project: | *yes* |
+-------------------+--------------------------------------+
+-------------------+----------------------------------------------------+
| Syntax: | ``scrapy genspider [-t template] <name> <domain>`` |
+-------------------+----------------------------------------------------+
| Requires project: | *yes* |
+-------------------+----------------------------------------------------+
Create a new spider in the current project.
This is just a convenient shortcut command for creating spiders based on
pre-defined templates, but certainly not the only way to create spiders. You
can just create the spider source code files yourself.
can just create the spider source code files yourself, instead of using this
command.
Usage example::
$ scrapy genspider example example.com
Created spider 'example' using template 'crawl' in module:
jobsbot.spiders.example
$ scrapy genspider -l
Available templates:
basic
crawl
csvfeed
xmlfeed
$ scrapy genspider -d basic
from scrapy.spider import BaseSpider
class $classname(BaseSpider):
name = "$name"
allowed_domains = ["$domain"]
start_urls = (
'http://www.$domain/',
)
def parse(self, response):
pass
$ scrapy genspider -t basic example example.com
Created spider 'example' using template 'basic' in module:
mybot.spiders.example
.. command:: crawl

61
scrapy/commands/genspider.py
View File

@ -1,8 +1,7 @@
import sys
import shutil
import string
from os import listdir
from os.path import join, dirname, abspath, exists
from os.path import join, dirname, abspath, exists, splitext
import scrapy
from scrapy.spider import spiders
@ -10,10 +9,6 @@ from scrapy.command import ScrapyCommand
from scrapy.conf import settings
from scrapy.utils.template import render_templatefile, string_camelcase
SPIDER_TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'spiders')
def sanitize_module_name(module_name):
"""Sanitize the given module name, by replacing dashes and points
with underscores and prefixing it with a letter if it doesn't start
@ -24,10 +19,14 @@ def sanitize_module_name(module_name):
module_name = "a" + module_name
return module_name
_templates_base_dir = settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], \
'templates')
class Command(ScrapyCommand):
requires_project = True
default_settings = {'LOG_ENABLED': False}
templates_dir = join(_templates_base_dir, 'spiders')
def syntax(self):
return "[options] <name> <domain>"
@ -37,8 +36,10 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--list", dest="list", action="store_true")
parser.add_option("--dump", dest="dump", action="store_true")
parser.add_option("-l", "--list", dest="list", action="store_true",
help="List available templates")
parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
help="Dump template to standard output")
parser.add_option("-t", "--template", dest="template", default="crawl",
help="Uses a custom template.")
parser.add_option("--force", dest="force", action="store_true",
@ -48,33 +49,26 @@ class Command(ScrapyCommand):
if opts.list:
self._list_templates()
return
if opts.dump:
template_file = self._find_template(opts.template)
template_file = self._find_template(opts.dump)
if template_file:
template = open(template_file, 'r')
print template.read()
print open(template_file, 'r').read()
return
if len(args) != 2:
return False
name = args[0]
domain = args[1]
name, domain = args[0:2]
module = sanitize_module_name(name)
# if spider already exists and not force option then halt
try:
spider = spiders.create(name)
except KeyError:
pass
else:
# if spider already exists and not --force then halt
if not opts.force:
print "Spider '%s' already exists in module:" % name
print "Spider %r already exists in module:" % name
print " %s" % spider.__module__
sys.exit(1)
return
template_file = self._find_template(opts.template)
if template_file:
self._genspider(module, name, domain, opts.template, template_file)
@ -90,11 +84,9 @@ class Command(ScrapyCommand):
'classname': '%sSpider' % ''.join([s.capitalize() \
for s in module.split('_')])
}
spiders_module = __import__(settings['NEWSPIDER_MODULE'], {}, {}, [''])
spiders_dir = abspath(dirname(spiders_module.__file__))
spider_file = "%s.py" % join(spiders_dir, module)
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
print "Created spider %r using template %r in module:" % (name, \
@ -102,22 +94,15 @@ class Command(ScrapyCommand):
print " %s.%s" % (spiders_module.__name__, module)
def _find_template(self, template):
template_file = join(settings['TEMPLATES_DIR'], 'spiders', '%s.tmpl' % template)
if not exists(template_file):
template_file = join(SPIDER_TEMPLATES_PATH, '%s.tmpl' % template)
if not exists(template_file):
print "Unable to find template %r." \
% template
print "Use genspider --list to see all available templates."
return None
return template_file
template_file = join(self.templates_dir, '%s.tmpl' % template)
if exists(template_file):
return template_file
print "Unable to find template: %s\n" % template
print 'Use "scrapy genspider --list" to see all available templates.'
def _list_templates(self):
files = set(listdir(SPIDER_TEMPLATES_PATH))
if exists(settings['TEMPLATES_DIR']):
files.update(listdir(join(settings['TEMPLATES_DIR'], 'spiders')))
for filename in sorted(files):
print "Available templates:"
for filename in sorted(listdir(self.templates_dir)):
if filename.endswith('.tmpl'):
print filename
print " %s" % splitext(filename)[0]

38
scrapy/tests/test_commands.py
View File

@ -32,8 +32,7 @@ class ProjectTest(unittest.TestCase):
return subprocess.call(args, stdout=out, stderr=out, cwd=self.cwd, \
env=self.env, **kwargs)
def call_get_proc(self, *new_args, **kwargs):
out = os.tmpfile()
def proc(self, *new_args, **kwargs):
args = (sys.executable, '-m', 'scrapy.cmdline') + new_args
return subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, \
cwd=self.cwd, env=self.env, **kwargs)
@ -76,29 +75,32 @@ class GenspiderCommandTest(CommandTest):
self.assertEqual(0, self.call('genspider', 'test_name', 'test.com'))
assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py'))
def test_template_default(self, *args):
self.assertEqual(0, self.call('genspider', 'test_spider', 'test.com', *args))
assert exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))
self.assertEqual(1, self.call('genspider', 'test_spider', 'test.com'))
def test_template(self, tplname='crawl'):
args = ['--template=%s' % tplname] if tplname else []
spname = 'test_spider'
p = self.proc('genspider', spname, 'test.com', *args)
out = p.stdout.read()
self.assert_("Created spider %r using template %r in module" % (spname, tplname) in out)
self.assert_(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
p = self.proc('genspider', spname, 'test.com', *args)
out = p.stdout.read()
self.assert_("Spider %r already exists in module" % spname in out)
def test_template_basic(self):
self.test_template_default('--template=basic')
self.test_template('basic')
def test_template_csvfeed(self):
self.test_template_default('--template=csvfeed')
self.test_template('csvfeed')
def test_template_xmlfeed(self):
self.test_template_default('--template=xmlfeed')
def test_template_crawl(self):
self.test_template_default('--template=crawl')
self.test_template('xmlfeed')
def test_list(self):
self.assertEqual(0, self.call('genspider', '--list'))
def test_dump(self):
self.assertEqual(0, self.call('genspider', '--dump'))
self.assertEqual(0, self.call('genspider', '--dump', '--template=basic'))
self.assertEqual(0, self.call('genspider', '--dump=basic'))
self.assertEqual(0, self.call('genspider', '-d', 'basic'))
class MiscCommandsTest(CommandTest):
@ -127,7 +129,7 @@ class MySpider(BaseSpider):
self.log("It Works!")
return []
""")
p = self.call_get_proc('runspider', fname)
p = self.proc('runspider', fname)
log = p.stderr.read()
self.assert_("[myspider] DEBUG: It Works!" in log)
self.assert_("[myspider] INFO: Spider opened" in log)
@ -143,12 +145,12 @@ class MySpider(BaseSpider):
from scrapy import log
from scrapy.spider import BaseSpider
""")
p = self.call_get_proc('runspider', fname)
p = self.proc('runspider', fname)
log = p.stderr.read()
self.assert_("ERROR: No spider found in file" in log)
def test_runspider_file_not_found(self):
p = self.call_get_proc('runspider', 'some_non_existent_file')
p = self.proc('runspider', 'some_non_existent_file')
log = p.stderr.read()
self.assert_("ERROR: File not found: some_non_existent_file" in log)
@ -158,7 +160,7 @@ from scrapy.spider import BaseSpider
fname = abspath(join(tmpdir, 'myspider.txt'))
with open(fname, 'w') as f:
f.write("")
p = self.call_get_proc('runspider', fname)
p = self.proc('runspider', fname)
log = p.stderr.read()
self.assert_("ERROR: Unable to load" in log)