From 3badb39ed719cc48d9713887dad75ee321286071 Mon Sep 17 00:00:00 2001 From: elpolilla Date: Thu, 31 Jul 2008 23:08:54 +0000 Subject: [PATCH] removed report feature for not being generic --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40142 --- scrapy/trunk/scrapy/command/commands/crawl.py | 5 -- scrapy/trunk/scrapy/report/__init__.py | 67 ------------------- 2 files changed, 72 deletions(-) delete mode 100644 scrapy/trunk/scrapy/report/__init__.py diff --git a/scrapy/trunk/scrapy/command/commands/crawl.py b/scrapy/trunk/scrapy/command/commands/crawl.py index a4734148d..866ce16c9 100644 --- a/scrapy/trunk/scrapy/command/commands/crawl.py +++ b/scrapy/trunk/scrapy/command/commands/crawl.py @@ -1,7 +1,6 @@ from scrapy.command import ScrapyCommand from scrapy.core.manager import scrapymanager from scrapy.replay import Replay -from scrapy.report import Report from scrapy.conf import settings @@ -19,8 +18,6 @@ class Command(ScrapyCommand): parser.add_option("--restrict", dest="restrict", action="store_true", help="restrict crawling only to the given urls") parser.add_option("--record", dest="record", help="use FILE for recording session (see replay command)", metavar="FILE") parser.add_option("--record-dir", dest="recorddir", help="use DIR for recording (instead of file)", metavar="DIR") - parser.add_option("--report", dest="doreport", action='store_true', help="generate a report of the scraped products in a text file") - parser.add_option("--report-dropped", dest="doreport_dropped", action="store_true", help="generate a report of the dropped products in a text file") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -38,8 +35,6 @@ class Command(ScrapyCommand): # disconnecting since pydispatcher uses weak references self.replay = Replay(opts.record or opts.recorddir, mode='record', usedir=bool(opts.recorddir)) self.replay.record(args=args, opts=opts.__dict__) - if opts.doreport or opts.doreport_dropped: - self.report = Report(passed=opts.doreport, dropped=opts.doreport_dropped) def run(self, args, opts): scrapymanager.runonce(*args, **opts.__dict__) diff --git a/scrapy/trunk/scrapy/report/__init__.py b/scrapy/trunk/scrapy/report/__init__.py deleted file mode 100644 index 5e65befae..000000000 --- a/scrapy/trunk/scrapy/report/__init__.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding: utf8 -*- -from datetime import datetime -from pydispatch import dispatcher -from scrapy.core import signals - -class Report(object): - def __init__(self, passed, dropped): - self.domain = '' - self.passed_file = None - self.dropped_file = None - self.passed = passed - self.dropped = dropped - self.total = { 'passed': 0, 'dropped': 0, 'variants': 0 } - - dispatcher.connect(self.domain_open, signal=signals.domain_open) - dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) - if self.passed: - dispatcher.connect(self.item_passed, signal=signals.item_passed) - if self.dropped: - dispatcher.connect(self.item_dropped, signal=signals.item_dropped) - - def get_product_attribs(self, product): - product_attribs = '' - for attrib, value in product.iteritems(): - product_attribs = '%s%s: %s\n' % (product_attribs, attrib, value) - return product_attribs - - def get_product_text(self, product, dropped=False): - product_text = '###Product\n%s' % self.get_product_attribs(product) - if product.variants: - product_text = '%s\n##Variants\n%s' % (product_text, '\n'.join([self.get_product_attribs(variant) for variant in product.variants])) - if dropped: - product_text = '%s--- Dropping reason: %s ---\n' % (product_text, dropped) - product_text = product_text + '\n\n' - return product_text - - def domain_open(self, domain, spider): - self.domain = domain - now = datetime.now() - filename = '%s_%s_%s.report' % (self.domain, now.strftime('%Y%m%d'), now.strftime('%H%M')) - if self.passed: - self.passed_file = open(filename, 'w') - self.passed_file.write('Scraping results for domain "%s"\n\n%s%s%s' % (self.domain, '##################################\n', - '### Products scraped correctly ###\n', '##################################\n')) - if self.dropped: - self.dropped_file = open(filename + '.dropped', 'w') - self.dropped_file.write('Scraping results for domain "%s"\n\n%s%s%s' % (self.domain, - '########################\n', '### Dropped products ###\n', '########################\n')) - - def item_passed(self, item, spider): - self.total['passed'] += 1 - self.total['variants'] += len(item.variants) - self.passed_file.write(self.get_product_text(item)) - - def item_dropped(self, item, spider, response, exception): - self.total['dropped'] += 1 - self.dropped_file.write(self.get_product_text(item, exception)) - - def engine_stopped(self): - if self.passed: - if self.passed_file: - self.passed_file.write('\n--- Total scraped: %d products + %d variants\n' % (self.total['passed'], self.total['variants'])) - self.passed_file.close() - if self.dropped: - if self.dropped_file: - self.dropped_file.write('\n--- Total dropped products: %d\n' % self.total['dropped']) - self.dropped_file.close()