cluster: add log gzipping support

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401026

scrapy/trunk/scrapy/contrib_exp/cluster/worker/manager.py

@@ -1,5 +1,5 @@
-import sys
 import os
+import sys
 import time
 import datetime
 import cPickle as pickle
@@ -14,6 +14,7 @@ from scrapy.core.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
 from scrapy.conf import settings
 
+
 class ScrapyProcessProtocol(protocol.ProcessProtocol):
 
     def __init__(self, worker, domain, logfile=None, spider_settings=None):
@@ -73,6 +74,12 @@ class ScrapyProcessProtocol(protocol.ProcessProtocol):
         self.worker.update_master(self.domain, "running")
 
     def processEnded(self, status):
+        if settings.getbool('CLUSTER_WORKER_GZIP_LOGS'):
+            try:
+                self.logfile = gzip_file(self.logfile)
+            except Exception, e:
+                log.msg("failed to compress %s exception=%s (domain=%s, pid=%s)" % (self.logfile, e, self.domain, self.pid))
+
         if isinstance(status.value, ProcessDone):
             st = "done"
             er = ""

scrapy/trunk/scrapy/utils/misc.py

@@ -2,8 +2,11 @@
 Auxiliary functions which doesn't fit anywhere else
 """
 from __future__ import with_statement
+from contextlib import closing
 
+import os
 import re
+import gzip
 import string
 import hashlib
 import csv
@@ -202,3 +205,24 @@ def md5sum(buffer):
     return m.hexdigest()
 
 
+def gzip_file(logfile):
+    """Gzip a file in place, just like gzip unix command
+
+    >>> import gzip
+    >>> import tempfile
+    >>> logfile = tempfile.mktemp()
+    >>> handle = open(logfile, 'wb')
+    >>> handle.write('something to compress')
+    >>> handle.close()
+    >>> logfile_gz = gzip_file(logfile)
+    >>> gzip.open(logfile_gz).read()
+    'something to compress'
+    """
+    logfile_gz = '%s.gz' % logfile
+    with closing(gzip.open(logfile_gz, 'wb')) as f_out:
+        with open(logfile) as f_in:
+            f_out.writelines(f_in)
+    os.remove(logfile)
+    return logfile_gz
+
+