added prototype page similarity code, to detect different layouts

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40180

scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/__init__.py

@@ -0,0 +1,73 @@
+"""
+SimpageMiddleware is a middleware for detecting similar page layouts
+"""
+
+import sys
+import datetime
+import pprint
+
+from pydispatch import dispatcher
+
+from scrapy.core import signals
+from scrapy.http import Response
+from scrapy.core.exceptions import NotConfigured
+from scrapy.conf import settings
+
+from .metrics import tagdepth
+
+class SimpagesMiddleware(object):
+    
+    metric = tagdepth
+    threshold = 0.8
+
+    def __init__(self):
+        if not settings.getbool('SIMPAGES_ENABLED'):
+            raise NotConfigured
+        repfilename = settings.get('SIMPAGES_REPORT_FILE')
+        self.reportfile = open(repfilename, "a") if repfilename else None
+        self.sim_groups = {}
+        self.last_group = 0
+        dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped)
+
+    def process_response(self, request, response, spider):
+        if isinstance(response, Response):
+            group, simrate = self.get_similarity_group(response)
+            if group:
+                self.sim_groups[group]['similar_urls'].append((response.url, simrate))
+            else:
+                self.create_similarity_group(response)
+        return response
+
+    def get_similarity_group(self, response):
+        fp = self.metric.simhash(response)
+        for group, data in self.sim_groups.iteritems():
+            simrate = self.metric.compare(fp, data['simhash'])
+            if simrate > self.threshold:
+                return (group, simrate)
+        return (None, 0)
+
+    def create_similarity_group(self, response):
+        self.last_group += 1
+        data = {}
+        data['simhash'] = self.metric.simhash(response)
+        data['first_url'] = response.url
+        data['similar_urls'] = []
+        self.sim_groups[self.last_group] = data
+
+    def get_report(self):
+        r =  "Page similarity results\n"
+        r += "=======================\n\n"
+        r += "Datetime : %s\n" % datetime.datetime.now()
+        r += "Metric   : %s\n" % self.metric.__name__
+        r += "Threshold: %s\n" % self.threshold
+        r += "Results  :\n"
+        r += pprint.pformat(self.sim_groups)
+        r += "\n\n"
+        return r
+
+    def engine_stopped(self):
+        rep = self.get_report()
+        if self.reportfile:
+            self.reportfile.write(rep)
+        else:
+            print rep

scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/metrics/tagdepth.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+from __future__ import division
+
+from BeautifulSoup import BeautifulSoup, Tag
+
+relevant_tags = ['div', 'table', 'td', 'tr', 'h1']
+
+def get_symbol_dict(node, tags=(), depth=1):
+    symdict = {}
+    for tag in node:
+        if isinstance(tag, Tag) and tag.name in tags:
+            symbol = str("%d%s" % (depth, tag.name))
+            symdict[symbol] = symdict.setdefault(symbol, 0) + 1
+            symdict.update(get_symbol_dict(tag, tags, depth+1))
+    return symdict
+
+def simhash(response):
+    soup = BeautifulSoup(response.body.to_string())
+    symdict = get_symbol_dict(soup.find('body'), relevant_tags)
+    return set(symdict.keys())
+
+def compare(fp1, fp2):
+    return len(fp1 & fp2) / len(fp1 | fp2)