diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/__init__.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/__init__.py index cee3e942f..e33ab14b7 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/__init__.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/__init__.py @@ -39,9 +39,9 @@ class SimpagesMiddleware(object): return response def get_similarity_group(self, response): - fp = self.metric.simhash(response) + sh = self.metric.simhash(response) for group, data in self.sim_groups.iteritems(): - simrate = self.metric.compare(fp, data['simhash']) + simrate = self.metric.compare(sh, data['simhash']) if simrate > self.threshold: return (group, simrate) return (None, 0) diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/metrics/__init__.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/metrics/__init__.py index e69de29bb..524624980 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/metrics/__init__.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/metrics/__init__.py @@ -0,0 +1,19 @@ +""" +This module contains several metrics that can be used with the +SimpagesMiddleware. + +A metric must implement two functions: + +1. simhash(response) + +Receives a response and returns a simhash of that response. A simhash can be +an object of any type and its only purpose is to provide a fast way for +comparing the [simhashed] response with another responses (that will also be +simhashed). + +2. compare(simhash1, simhash2) + +Receives two simhashes and must return a (float) value between 0 and 1, +depending on how similar the two simhashes (and, thus, the responses they +represent) are. 0 means completely different, 1 means identical. +""" diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/metrics/tagdepth.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/metrics/tagdepth.py index 8b4e6e3b1..d4e04a406 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/metrics/tagdepth.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/simpages/metrics/tagdepth.py @@ -1,23 +1,31 @@ -#!/usr/bin/env python +""" +tagdepth metric + +Compares pages analyzing a predefined set of (relevant) +tags and the depth where they appear in the page markup document. + +Requires ResponseSoup extension enabled. +""" + from __future__ import division -from BeautifulSoup import BeautifulSoup, Tag +from BeautifulSoup import Tag -relevant_tags = ['div', 'table', 'td', 'tr', 'h1'] +relevant_tags = set(['div', 'table', 'td', 'tr', 'h1']) def get_symbol_dict(node, tags=(), depth=1): symdict = {} for tag in node: if isinstance(tag, Tag) and tag.name in tags: - symbol = str("%d%s" % (depth, tag.name)) + symbol = "%d%s" % (depth, str(tag.name)) symdict[symbol] = symdict.setdefault(symbol, 0) + 1 symdict.update(get_symbol_dict(tag, tags, depth+1)) return symdict def simhash(response): - soup = BeautifulSoup(response.body.to_string()) - symdict = get_symbol_dict(soup.find('body'), relevant_tags) + soup = response.soup + symdict = get_symbol_dict(response.soup.find('body'), relevant_tags) return set(symdict.keys()) -def compare(fp1, fp2): - return len(fp1 & fp2) / len(fp1 | fp2) +def compare(sh1, sh2): + return len(sh1 & sh2) / len(sh1 | sh2)