diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 606c413ae..28bc1467e 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -12,8 +12,8 @@ class Command(ScrapyCommand): requires_project = True spider = None - items = [] - requests = [] + items = {} + requests = {} first_response = None @@ -39,8 +39,63 @@ class Command(ScrapyCommand): help="use this callback for parsing, instead looking for a callback") parser.add_option("-d", "--depth", dest="depth", type="int", default=1, \ help="maximum depth for parsing requests [default: %default]") - # parser.add_option("-v", "--verbose", dest="verbose", action="store_true", \ - # help="print each depth level one by one") + parser.add_option("-v", "--verbose", dest="verbose", action="store_true", \ + help="print each depth level one by one") + + + @property + def max_level(self): + levels = self.items.keys() + self.requests.keys() + if levels: return max(levels) + else: return 0 + + def add_items(self, lvl, new_items): + old_items = self.items.get(lvl, []) + self.items[lvl] = old_items + new_items + + def add_requests(self, lvl, new_reqs): + old_reqs = self.requests.get(lvl, []) + self.requests[lvl] = old_reqs + new_reqs + + def print_items(self, lvl=None, colour=True): + if lvl is None: + items = [item for lst in self.items.values() for item in lst] + else: + items = self.items.get(lvl, []) + + print "# Scraped Items ", "-"*60 + display.pprint([dict(x) for x in items], colorize=colour) + + def print_requests(self, lvl=None, colour=True): + if lvl is None: + levels = self.requests.keys() + if levels: + requests = self.requests[max(levels)] + else: + requests = [] + else: + requests = self.requests.get(lvl, []) + + print "# Requests ", "-"*65 + display.pprint(requests, colorize=colour) + + def print_results(self, opts): + colour = not opts.nocolour + + if opts.verbose: + for level in xrange(1, self.max_level+1): + print '\n>>> DEPTH LEVEL: %s <<<' % level + if not opts.noitems: + self.print_items(level, colour) + if not opts.nolinks: + self.print_requests(level, colour) + else: + print '\n>>> STATUS DEPTH LEVEL %s <<<' % self.max_level + if not opts.noitems: + self.print_items(colour=colour) + if not opts.nolinks: + self.print_requests(colour=colour) + def run_callback(self, response, cb): items, requests = [], [] @@ -61,14 +116,6 @@ class Command(ScrapyCommand): log.msg("No CrawlSpider rules found in spider %r, please specify " "a callback to use for parsing" % self.spider.name, log.ERROR) - def print_results(self, opts): - if not opts.noitems: - print "# Scraped Items ", "-"*60 - display.pprint([dict(x) for x in self.items], colorize=not opts.nocolour) - if not opts.nolinks: - print "# Requests ", "-"*65 - display.pprint(self.requests, colorize=not opts.nocolour) - def set_spider(self, url, opts): if opts.spider: try: @@ -110,18 +157,18 @@ class Command(ScrapyCommand): log.msg('Cannot find callback %r in spider: %s' % (callback, spider.name)) # parse items and requests - items, requests = self.run_callback(response, cb) - self.items += items - depth = response.meta['_depth'] + + items, requests = self.run_callback(response, cb) + self.add_items(depth, items) + self.add_requests(depth, requests) + if depth < opts.depth: for req in requests: req.meta['_depth'] = depth + 1 req.meta['_callback'] = req.callback req.callback = callback return requests - else: - self.requests += requests request.meta['_depth'] = 1 request.meta['_callback'] = request.callback