RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%4093
This commit is contained in:
olveyra 2008-07-22 17:56:04 +00:00
parent 5e2db86cde
commit a27b085c04
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py
View File

@ -162,7 +162,7 @@ class ClusterMaster(pb.Root):
self.pending = []
self.loading = []
self.nodes = {}
self.statistics = {"domains": {}, "scraped_total": 0, "start_time": datetime.datetime.utcnow(), "pending": self.print_pending() }
self.statistics = {"domains": {}, "scraped_total": 0, "start_time": datetime.datetime.utcnow()}
self.global_settings = {}
#load cluster global settings
for sname in settings.getlist('GLOBAL_CLUSTER_SETTINGS'):
@ -210,6 +210,7 @@ class ClusterMaster(pb.Root):
self.nodes[name].update_status()
self.nodes[name].update_statistics()
self.statistics["scraped_total"] += self.nodes[name].statistics["scraped_total"]
self.statistics["timestamp"] = datetime.datetime.utcnow()
for domain in self.nodes[name].statistics["domains"]:
if not domain in self.statistics["domains"] or self.nodes[name].statistics["domains"][domain]["last_start_time"] >= self.statistics["domains"][domain]["last_start_time"]:
self.statistics["domains"][domain] = self.nodes[name].statistics["domains"][domain]