RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

removed hacky --callback option to crawl command

This commit is contained in:
Pablo Hoffman 2009-09-11 19:36:00 -03:00
parent e854d0d6ef
commit 0c292e3350
1 changed files with 0 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
scrapy/command/commands/crawl.py
View File

@ -21,8 +21,6 @@ class Command(ScrapyCommand):
ScrapyCommand.add_options(self, parser)
parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", \
help="don't follow links (for use with URLs only)")
parser.add_option("-c", "--callback", dest="callback", action="store", \
help="use the provided callback for starting to crawl the given url")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
@ -30,32 +28,4 @@ class Command(ScrapyCommand):
settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False
def run(self, args, opts):
if opts.callback:
requests = []
for a in args:
if is_url(a):
spider = spiders.fromurl(a)
urls = [a]
else:
spider = spiders.fromdomain(a)
urls = spider.start_urls if hasattr(spider.start_urls, \
'__iter__') else [spider.start_urls]
if spider:
if hasattr(spider, opts.callback):
requests.extend(Request(url=url, callback=getattr(spider, \
opts.callback)) for url in urls)
else:
log.msg('Callback %s doesnt exist in spider %s' % \
(opts.callback, spider.domain_name), log.ERROR)
else:
log.msg('Cannot find spider for %s' % a, log.ERROR)
if requests:
args = requests
else:
log.msg('Cannot create any requests with the provided arguments', \
log.ERROR)
return
scrapymanager.runonce(*args)