diff --git a/scrapy/trunk/scrapy/contrib_exp/cluster/worker/manager.py b/scrapy/trunk/scrapy/contrib_exp/cluster/worker/manager.py index 7f448f626..6053a1147 100644 --- a/scrapy/trunk/scrapy/contrib_exp/cluster/worker/manager.py +++ b/scrapy/trunk/scrapy/contrib_exp/cluster/worker/manager.py @@ -1,5 +1,5 @@ -import sys import os +import sys import time import datetime import cPickle as pickle @@ -14,6 +14,7 @@ from scrapy.core.exceptions import NotConfigured from scrapy.utils.misc import load_object from scrapy.conf import settings + class ScrapyProcessProtocol(protocol.ProcessProtocol): def __init__(self, worker, domain, logfile=None, spider_settings=None): @@ -73,6 +74,12 @@ class ScrapyProcessProtocol(protocol.ProcessProtocol): self.worker.update_master(self.domain, "running") def processEnded(self, status): + if settings.getbool('CLUSTER_WORKER_GZIP_LOGS'): + try: + self.logfile = gzip_file(self.logfile) + except Exception, e: + log.msg("failed to compress %s exception=%s (domain=%s, pid=%s)" % (self.logfile, e, self.domain, self.pid)) + if isinstance(status.value, ProcessDone): st = "done" er = "" diff --git a/scrapy/trunk/scrapy/utils/misc.py b/scrapy/trunk/scrapy/utils/misc.py index 7538961cc..6d747fd2e 100644 --- a/scrapy/trunk/scrapy/utils/misc.py +++ b/scrapy/trunk/scrapy/utils/misc.py @@ -2,8 +2,11 @@ Auxiliary functions which doesn't fit anywhere else """ from __future__ import with_statement +from contextlib import closing +import os import re +import gzip import string import hashlib import csv @@ -202,3 +205,24 @@ def md5sum(buffer): return m.hexdigest() +def gzip_file(logfile): + """Gzip a file in place, just like gzip unix command + + >>> import gzip + >>> import tempfile + >>> logfile = tempfile.mktemp() + >>> handle = open(logfile, 'wb') + >>> handle.write('something to compress') + >>> handle.close() + >>> logfile_gz = gzip_file(logfile) + >>> gzip.open(logfile_gz).read() + 'something to compress' + """ + logfile_gz = '%s.gz' % logfile + with closing(gzip.open(logfile_gz, 'wb')) as f_out: + with open(logfile) as f_in: + f_out.writelines(f_in) + os.remove(logfile) + return logfile_gz + +