--report option added to crawl command

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40130

scrapy/trunk/scrapy/command/commands/crawl.py

@@ -1,6 +1,7 @@
 from scrapy.command import ScrapyCommand
 from scrapy.core.manager import scrapymanager
 from scrapy.replay import Replay
+from scrapy.report import Report
 from scrapy.conf import settings
 
 
@@ -18,6 +19,8 @@ class Command(ScrapyCommand):
         parser.add_option("--restrict", dest="restrict", action="store_true", help="restrict crawling only to the given urls")
         parser.add_option("--record", dest="record", help="use FILE for recording session (see replay command)", metavar="FILE")
         parser.add_option("--record-dir", dest="recorddir", help="use DIR for recording (instead of file)", metavar="DIR")
+        parser.add_option("--report", dest="doreport", action='store_true', help="generate a report of the scraped products in a text file")
+        parser.add_option("--report-dropped", dest="doreport_dropped", action="store_true", help="choose whether to report dropped products or not")
 
     def process_options(self, args, opts):
         ScrapyCommand.process_options(self, args, opts)
@@ -35,6 +38,8 @@ class Command(ScrapyCommand):
             # disconnecting since pydispatcher uses weak references 
             self.replay = Replay(opts.record or opts.recorddir, mode='record', usedir=bool(opts.recorddir))
             self.replay.record(args=args, opts=opts.__dict__)
+        if opts.doreport:
+            self.report = Report(dropped=opts.doreport_dropped)
 
     def run(self, args, opts):
         scrapymanager.runonce(*args, **opts.__dict__)

scrapy/trunk/scrapy/report/__init__.py

@@ -0,0 +1,54 @@
+# -*- coding: utf8 -*-
+from datetime import datetime
+from pydispatch import dispatcher
+from scrapy.core import signals
+
+class Report(object):
+    def __init__(self, dropped):
+        self.domain = ''
+        self.passed_file = None
+        self.dropped_file = None
+        self.dropped = dropped
+        self.total = { 'passed': 0, 'dropped': 0 }
+
+        dispatcher.connect(self.domain_open, signal=signals.domain_open)
+        dispatcher.connect(self.item_passed, signal=signals.item_passed)
+        dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped)
+        if self.dropped:
+            dispatcher.connect(self.item_dropped, signal=signals.item_dropped)
+
+    def get_product_text(self, product, dropped=False):
+        product_text = ''
+        for attrib, value in product.iteritems():
+            product_text = '%s%s: %s\n' % (product_text, attrib, value)
+        if dropped:
+            product_text = '%sdropping reason: %s\n' % (product_text, dropped)
+        product_text = product_text + '\n\n'
+        return product_text
+
+    def domain_open(self, domain, spider):
+        self.domain = domain
+        now = datetime.now()
+        filename = '%s_%s_%s.report' % (self.domain, now.strftime('%Y%m%d'), now.strftime('%H%M'))
+        self.passed_file = open(filename, 'w')
+        self.passed_file.write('Scraping results for domain "%s"\n\n%s%s%s' % (self.domain, '##################################\n',
+          '### Products scraped correctly ###\n', '##################################\n'))
+        if self.dropped:
+            self.dropped_file = open(filename + '.dropped', 'w')
+            self.dropped_file.write('Scraping results for domain "%s"\n\n%s%s%s' % (self.domain,
+              '########################\n', '### Dropped products ###\n', '########################\n'))
+
+    def item_passed(self, item, spider):
+        self.total['passed'] += 1
+        self.passed_file.write(self.get_product_text(item))
+
+    def item_dropped(self, item, spider, response, exception):
+        self.total['dropped'] += 1
+        self.dropped_file.write(self.get_product_text(item, exception))
+
+    def engine_stopped(self):
+        self.passed_file.write('\n--- Total scraped products: %d\n' % self.total['passed'])
+        self.passed_file.close()
+        if self.dropped:
+            self.dropped_file.write('\n--- Total dropped products: %d\n' % self.total['dropped'])
+            self.dropped_file.close()