Added an histogram plot to simpages group to the report. Added quantities of html elements to symbol of the simhash. Now is more effective.

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40209

scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/__init__.py

@@ -5,6 +5,7 @@ SimpageMiddleware is a middleware for detecting similar page layouts
 import sys
 import datetime
 import pprint
+import pickle
 
 from pydispatch import dispatcher
 
@@ -13,7 +14,10 @@ from scrapy.http import Response
 from scrapy.core.exceptions import NotConfigured
 from scrapy.conf import settings
 
-from .metrics import tagdepth
+from scrapy.contrib.rulengine.responseWrapper import ResponseWrapper
+from scrapy.contrib.rulengine.pipeline import RulesPipeline
+
+from .metrics import tagdepth, histogram
 
 class SimpagesMiddleware(object):
     
@@ -24,42 +28,54 @@ class SimpagesMiddleware(object):
         if not settings.getbool('SIMPAGES_ENABLED'):
             raise NotConfigured
         repfilename = settings.get('SIMPAGES_REPORT_FILE')
-        self.reportfile = open(repfilename, "a") if repfilename else None
+        self.reportfile = open(repfilename, "a") if repfilename else None        
+        persistence_filename = open(repfilename + '.pickle', 'w') if repfilename else None
+        self.persistent_simgroup = pickle.Pickler(persistence_filename) if persistence_filename else None
         self.sim_groups = {}
         self.last_group = 0
         dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped)
+        #Rules 
+        RulesPipeline.loadRules()
 
     def process_response(self, request, response, spider):
         if isinstance(response, Response):
-            group, simrate = self.get_similarity_group(response)
+            group, simrate, simhash = self.get_similarity_group(response)
             if group:
-                self.sim_groups[group]['similar_urls'].append((response.url, simrate))
+                wres = ResponseWrapper(response)
+                rp = RulesPipeline(wres)
+                rules_info = rp.execute()
+                self.sim_groups[group]['similar_urls'].append((response.url, simrate, simhash, rules_info))
             else:
                 self.create_similarity_group(response)
         return response
 
     def get_similarity_group(self, response):
-        sh = self.metric.simhash(response)
+        sh = self.metric.simhash(response, symnumbers=True)
         for group, data in self.sim_groups.iteritems():
             simrate = self.metric.compare(sh, data['simhash'])
             if simrate > self.threshold:
-                return (group, simrate)
-        return (None, 0)
+                return (group, simrate, data['simhash'])
+        return (None, 0, set())
 
     def create_similarity_group(self, response):
         self.last_group += 1
         data = {}
-        data['simhash'] = self.metric.simhash(response)
+        data['simhash'] = self.metric.simhash(response, symnumbers=True)
         data['first_url'] = response.url
+        wres = ResponseWrapper(response)
+        rp = RulesPipeline(wres)
+        data['rules_info'] = rp.execute()
         data['similar_urls'] = []
         self.sim_groups[self.last_group] = data
 
     def get_report(self):
+        data_hist = dict( [(k, len(v['similar_urls'])) for k, v in self.sim_groups.items()] )
         r =  "Page similarity results\n"
         r += "=======================\n\n"
         r += "Datetime : %s\n" % datetime.datetime.now()
         r += "Metric   : %s\n" % self.metric.__name__
         r += "Threshold: %s\n" % self.threshold
+        r += "Distribution per group:\n%s\n" % histogram.plot(data_hist)
         r += "Results  :\n"
         r += pprint.pformat(self.sim_groups)
         r += "\n\n"
@@ -71,3 +87,5 @@ class SimpagesMiddleware(object):
             self.reportfile.write(rep)
         else:
             print rep
+        if self.persistent_simgroup:
+            self.persistent_simgroup.dump(self.sim_groups)

scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/metrics/__init__.py

@@ -4,7 +4,7 @@ SimpagesMiddleware.
 
 A metric must implement two functions:
 
-1. simhash(response)
+1. simhash(response, *args)
 
 Receives a response and returns a simhash of that response. A simhash can be
 an object of any type and its only purpose is to provide a fast way for

scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/metrics/histogram.py

@@ -0,0 +1,39 @@
+import math
+
+def plot(data):
+    """
+    data is a dict of tuples of the form: {key: quantity, ...}.
+    Make the histogram of the data dict.
+    """
+    maxv = max(data.values())
+    minv = min(data.values())
+    step = (maxv - minv) * 0.1 if (maxv - minv) != 0 else 1
+    s = []
+    for key, q in data.items():
+        s1 = "%s%s: " % (key, blanks(6 - len(str(key))))
+        for i in xrange(1, int(math.ceil(q/step)+1)):
+            s1+= "="
+        if s1[len(s1)-1] == '=':
+            s1+= " " 
+        s1 += str(q)
+        s.append(s1)
+    maxl = len(max(s, key=lambda x:len(x)))
+    s2 = ''
+    for i in xrange(1,maxl+1):
+        s2 += '-'
+
+    r = "\tgroup | quantities\n"
+    r += "\t%s" % s2
+    for x in s:
+        r += "\n\t%s" % x
+    return r
+
+def blanks(n):
+    return ''.join([' ' for x in range(1,n+1)])
+
+def print_plot(data):
+    print plot(data)
+
+
+
+

scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/metrics/tagdepth.py

@@ -11,7 +11,7 @@ from __future__ import division
 
 from BeautifulSoup import Tag
 
-relevant_tags = set(['div', 'table', 'td', 'tr', 'h1'])
+relevant_tags = set(['div', 'table', 'td', 'tr', 'h1','p'])
 
 def get_symbol_dict(node, tags=(), depth=1):
     symdict = {}
@@ -22,10 +22,18 @@ def get_symbol_dict(node, tags=(), depth=1):
             symdict.update(get_symbol_dict(tag, tags, depth+1))
     return symdict
 
-def simhash(response):
+def simhash(response, symnumbers=False):
     soup = response.soup
-    symdict = get_symbol_dict(response.soup.find('body'), relevant_tags)
-    return set(symdict.keys())
+    symdict = get_symbol_dict(soup.find('body'), relevant_tags)
+    if symnumbers:
+        s = set([k+str(v) for k,v in symdict.items()])
+    else:
+        s = set(symdict.keys())
+    return s
 
 def compare(sh1, sh2):
-    return len(sh1 & sh2) / len(sh1 | sh2)
+    if sh1 == sh2:
+        return 1.0
+    else:
+        return len(sh1 & sh2) / len(sh1 | sh2)
+