cleaned up simpages code a bit, added some documentation

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40181

scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/__init__.py

@@ -39,9 +39,9 @@ class SimpagesMiddleware(object):
         return response
 
     def get_similarity_group(self, response):
-        fp = self.metric.simhash(response)
+        sh = self.metric.simhash(response)
         for group, data in self.sim_groups.iteritems():
-            simrate = self.metric.compare(fp, data['simhash'])
+            simrate = self.metric.compare(sh, data['simhash'])
             if simrate > self.threshold:
                 return (group, simrate)
         return (None, 0)

scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/metrics/__init__.py

@@ -0,0 +1,19 @@
+"""
+This module contains several metrics that can be used with the
+SimpagesMiddleware.
+
+A metric must implement two functions:
+
+1. simhash(response)
+
+Receives a response and returns a simhash of that response. A simhash can be
+an object of any type and its only purpose is to provide a fast way for
+comparing the [simhashed] response with another responses (that will also be
+simhashed).
+
+2. compare(simhash1, simhash2)
+
+Receives two simhashes and must return a (float) value between 0 and 1,
+depending on how similar the two simhashes (and, thus, the responses they
+represent) are. 0 means completely different, 1 means identical.
+"""

scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/metrics/tagdepth.py

@@ -1,23 +1,31 @@
-#!/usr/bin/env python
+"""
+tagdepth metric
+
+Compares pages analyzing a predefined set of (relevant)
+tags and the depth where they appear in the page markup document. 
+
+Requires ResponseSoup extension enabled.
+"""
+
 from __future__ import division
 
-from BeautifulSoup import BeautifulSoup, Tag
+from BeautifulSoup import Tag
 
-relevant_tags = ['div', 'table', 'td', 'tr', 'h1']
+relevant_tags = set(['div', 'table', 'td', 'tr', 'h1'])
 
 def get_symbol_dict(node, tags=(), depth=1):
     symdict = {}
     for tag in node:
         if isinstance(tag, Tag) and tag.name in tags:
-            symbol = str("%d%s" % (depth, tag.name))
+            symbol = "%d%s" % (depth, str(tag.name))
             symdict[symbol] = symdict.setdefault(symbol, 0) + 1
             symdict.update(get_symbol_dict(tag, tags, depth+1))
     return symdict
 
 def simhash(response):
-    soup = BeautifulSoup(response.body.to_string())
-    symdict = get_symbol_dict(soup.find('body'), relevant_tags)
+    soup = response.soup
+    symdict = get_symbol_dict(response.soup.find('body'), relevant_tags)
     return set(symdict.keys())
 
-def compare(fp1, fp2):
-    return len(fp1 & fp2) / len(fp1 | fp2)
+def compare(sh1, sh2):
+    return len(sh1 & sh2) / len(sh1 | sh2)