diff --git a/scrapy/trunk/scrapy/command/commands/crawl.py b/scrapy/trunk/scrapy/command/commands/crawl.py index 9d42e15ac..479a11d22 100644 --- a/scrapy/trunk/scrapy/command/commands/crawl.py +++ b/scrapy/trunk/scrapy/command/commands/crawl.py @@ -2,6 +2,10 @@ from scrapy.command import ScrapyCommand from scrapy.core.manager import scrapymanager from scrapy.replay import Replay from scrapy.conf import settings +from scrapy.utils.url import is_url +from scrapy.spider import spiders +from scrapy.http import Request +from scrapy import log class Command(ScrapyCommand): @@ -19,6 +23,7 @@ class Command(ScrapyCommand): parser.add_option("--record", dest="record", help="use FILE for recording session (see replay command)", metavar="FILE") parser.add_option("--record-dir", dest="recorddir", help="use DIR for recording (instead of file)", metavar="DIR") parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", help="don't follow links (for use with URLs only)") + parser.add_option("-c", "--callback", dest="callback", action="store", help="use the provided callback for starting to crawl the given url") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -41,4 +46,28 @@ class Command(ScrapyCommand): self.replay.record(args=args, opts=opts.__dict__) def run(self, args, opts): + if opts.callback: + requests = [] + for a in args: + if is_url(a): + spider = spiders.fromurl(a) + urls = [a] + else: + spider = spiders.fromdomain(a) + urls = spider.start_urls if hasattr(spider.start_urls, '__iter__') else [spider.start_urls] + + if spider: + if hasattr(spider, opts.callback): + requests.extend(Request(url=url, callback=getattr(spider, opts.callback)) for url in urls) + else: + log.msg('Callback %s doesnt exist in spider %s' % (opts.callback, spider.domain_name), log.ERROR) + else: + log.msg('Could not found spider for %s' % a, log.ERROR) + + if requests: + args = requests + else: + log.msg('Couldnt create any requests from the provided arguments', log.ERROR) + return + scrapymanager.runonce(*args, **opts.__dict__) diff --git a/scrapy/trunk/scrapy/command/commands/parse.py b/scrapy/trunk/scrapy/command/commands/parse.py index a09b56433..bdf8deccc 100644 --- a/scrapy/trunk/scrapy/command/commands/parse.py +++ b/scrapy/trunk/scrapy/command/commands/parse.py @@ -8,17 +8,18 @@ from scrapy import log class Command(ScrapyCommand): def syntax(self): - return "[options] " + return "[options] " def short_desc(self): - return "Parse the URL with the given spider method and print the results" + return "Parse the given URL and print the results" def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("--nolinks", dest="nolinks", action="store_true", help="don't show extracted links") parser.add_option("--noitems", dest="noitems", action="store_true", help="don't show scraped items") parser.add_option("--nocolour", dest="nocolour", action="store_true", help="avoid using pygments to colorize the output") - parser.add_option("--matches", dest="matches", action="store_true", help="try to match and parse the url with the defined rules (if any)") + parser.add_option("-r", "--rules", dest="rules", action="store_true", help="try to match and parse the url with the defined rules (if any)") + parser.add_option("-c", "--callback", dest="callback", action="store", help="use the provided callback for parsing the url") def pipeline_process(self, item, spider, opts): return item @@ -48,8 +49,8 @@ class Command(ScrapyCommand): if not opts.noitems: for item in items: for key in item.__dict__.keys(): - if key.startswith('_'): - item.__dict__.pop(key, None) + if key.startswith('_'): + item.__dict__.pop(key, None) print "# Scraped Items", "-"*60 display.pprint(list(items)) @@ -58,42 +59,32 @@ class Command(ScrapyCommand): display.pprint(list(links)) def run(self, args, opts): - if opts.matches: - url = args[0] - method = None - else: - if len(args) < 2: - print "A URL and method is required" - return - else: - url, method = args[:2] + if not args: + print "An URL is required" + return - items = [] - links = [] - for response in fetch([url]): + ret_items, ret_links = [], [] + for response in fetch(args): spider = spiders.fromurl(response.url) if not spider: log.msg('Couldnt find spider for "%s"' % response.url) continue - if method: - ret_items, ret_links = self.run_method(spider, response, method, args, opts) - items.extend(ret_items) - links.extend(ret_links) - else: - if hasattr(spider, 'rules'): - already_parsed = False - - for rule in spider.rules: - links.extend(Request(url=link.url, link_text=link.text) for link in rule.link_extractor.extract_urls(response)) - if not already_parsed and rule.link_extractor.matches(response.url): - already_parsed = True - ret_items, ret_links = self.run_method(spider, response, rule.callback, args, opts) - items.extend(ret_items) - links.extend(ret_links) + if opts.callback: + items, links = self.run_method(spider, response, opts.callback, args, opts) + elif opts.rules: + for rule in getattr(spider, 'rules', ()): + if rule.link_extractor.matches(response.url): + items, links = self.run_method(spider, response, rule.callback, args, opts) + break else: log.msg('No rules found for spider "%s", please specify a parsing method' % spider.domain_name) continue + else: + items, links = self.run_method(spider, response, 'parse', args, opts) - self.print_results(items, links, opts) + ret_items.extend(items) + ret_links.extend(links) + + self.print_results(ret_items, ret_links, opts) diff --git a/scrapy/trunk/scrapy/contrib/spiders/crawl.py b/scrapy/trunk/scrapy/contrib/spiders/crawl.py index 2fb7ad34c..89414ee38 100644 --- a/scrapy/trunk/scrapy/contrib/spiders/crawl.py +++ b/scrapy/trunk/scrapy/contrib/spiders/crawl.py @@ -2,7 +2,6 @@ import copy from scrapy.http import Request from scrapy.spider import BaseSpider -from scrapy.item import ScrapedItem from scrapy.conf import settings class Rule(object): @@ -61,10 +60,12 @@ class CrawlSpider(BaseSpider): """This function is called by the framework core for all the start_urls. Do not override this function, override parse_start_url instead.""" - for rule in self._rules: - if rule.callback and rule.link_extractor.matches(response.url): - return self._response_downloaded(response, rule.callback, rule.cb_kwargs, follow=True) - return self._response_downloaded(response, None, cb_kwargs={}, follow=True) + return self._response_downloaded(response, self.parse_start_url, cb_kwargs={}, follow=True) + + def parse_start_url(self, response): + """Overrideable callback function for processing start_urls. It must + return a list of ScrapedItems and/or Requests""" + return [] def process_results(self, results, response): """This overridable method is called for each result (item or request)