diff --git a/scrapy/trunk/scrapy/conf/default_settings.py b/scrapy/trunk/scrapy/conf/default_settings.py
index a78d86020..334f6ff60 100644
--- a/scrapy/trunk/scrapy/conf/default_settings.py
+++ b/scrapy/trunk/scrapy/conf/default_settings.py
@@ -55,6 +55,8 @@ DEPTH_STATS = True
 DOWNLOAD_DELAY = 0
 DOWNLOAD_TIMEOUT = 180      # 3mins
 
+DOWNLOADER_DEBUG = False
+
 DOWNLOADER_MIDDLEWARES = [
     # Engine side
     'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware',
diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py
index 946983361..2f401a0cf 100644
--- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py
+++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py
@@ -1,20 +1,18 @@
 from __future__ import with_statement
 
 import os
-import sys
 import hashlib
 import datetime
 import urlparse
 import cPickle as pickle
 from pydispatch import dispatcher
-from twisted.internet import defer
 
 from scrapy.core import signals
 from scrapy import log
-from scrapy.core.engine import scrapyengine
 from scrapy.http import Response, Headers
 from scrapy.http.headers import headers_dict_to_raw
-from scrapy.core.exceptions import UsageError, NotConfigured, HttpException, IgnoreRequest
+from scrapy.core.exceptions import NotConfigured, HttpException, IgnoreRequest
+from scrapy.utils.request import request_fingerprint
 from scrapy.conf import settings
 
 
@@ -33,7 +31,7 @@ class CacheMiddleware(object):
         if not is_cacheable(request):
             return
 
-        key = request.fingerprint()
+        key = request_fingerprint(request)
         domain = spider.domain_name
         try:
             response = self.cache.retrieve_response(domain, key)
@@ -53,7 +51,7 @@ class CacheMiddleware(object):
             return response
 
         if isinstance(response, Response) and not response.cached:
-            key = request.fingerprint()
+            key = request_fingerprint(request)
             domain = spider.domain_name
             self.cache.store(domain, key, request, response)
 
@@ -64,9 +62,9 @@ class CacheMiddleware(object):
             return
 
         if isinstance(exception, HttpException) and isinstance(exception.response, Response):
-            key = request.fingerprint()
+            key = request_fingerprint(request)
             domain = spider.domain_name
-            self.cache.store(domain,key, request, exception.response)
+            self.cache.store(domain, key, request, exception.response)
 
 def is_cacheable(request):
     scheme, _, _, _, _ = urlparse.urlsplit(request.url)
diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py
index 803c1eb9e..1ea1018a7 100644
--- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py
+++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py
@@ -30,6 +30,7 @@ from twisted.internet.defer import TimeoutError as UserTimeoutError
 
 from scrapy import log
 from scrapy.core.exceptions import HttpException
+from scrapy.utils.request import request_fingerprint
 from scrapy.conf import settings
 
 class RetryMiddleware(object):
@@ -44,7 +45,7 @@ class RetryMiddleware(object):
 
     def process_exception(self, request, exception, spider):
         if isinstance(exception, self.EXCEPTIONS_TO_RETRY) or (isinstance(exception, HttpException) and (int(exception.status) in self.retry_http_codes)):
-            fp = request.fingerprint()
+            fp = request_fingerprint(request)
             self.failed_count[fp] = self.failed_count.get(fp, 0) + 1
 
             if self.failed_count[fp] <= self.retry_times:
diff --git a/scrapy/trunk/scrapy/contrib/history/scheduler.py b/scrapy/trunk/scrapy/contrib/history/scheduler.py
index e8f20f032..7dd517154 100644
--- a/scrapy/trunk/scrapy/contrib/history/scheduler.py
+++ b/scrapy/trunk/scrapy/contrib/history/scheduler.py
@@ -5,6 +5,7 @@ from twisted.internet import defer
 from scrapy import log
 from scrapy.core.scheduler import Scheduler
 from scrapy.core.exceptions import IgnoreRequest
+from scrapy.utils.request import request_fingerprint
 
 class RulesScheduler(Scheduler):
     """Scheduler that uses rules to determine if we should follow links
@@ -33,7 +34,7 @@ class RulesScheduler(Scheduler):
         The url will only be added if we have not checked it already within
         a specified time period.
         """
-        requestid = request.fingerprint()
+        requestid = request_fingerprint(request)
         added = self.groupfilter.add(domain, requestid)
 
         if request.dont_filter or added:
diff --git a/scrapy/trunk/scrapy/contrib/pipeline/media.py b/scrapy/trunk/scrapy/contrib/pipeline/media.py
index f4332b8a3..5f19de77d 100644
--- a/scrapy/trunk/scrapy/contrib/pipeline/media.py
+++ b/scrapy/trunk/scrapy/contrib/pipeline/media.py
@@ -3,6 +3,7 @@ from twisted.internet import defer
 from scrapy.utils.defer import mustbe_deferred, defer_result
 from scrapy import log
 from scrapy.core.engine import scrapyengine
+from scrapy.utils.request import request_fingerprint
 from scrapy.spider import spiders
 
 
@@ -53,7 +54,7 @@ class MediaPipeline(object):
 
     def _enqueue(self, request, info):
         wad = request.deferred or defer.Deferred()
-        fp = request.fingerprint()
+        fp = request_fingerprint(request)
 
         # if already downloaded, return cached result.
         if fp in info.downloaded:
diff --git a/scrapy/trunk/scrapy/core/downloader/manager.py b/scrapy/trunk/scrapy/core/downloader/manager.py
index b6cd2068e..abac5fe37 100644
--- a/scrapy/trunk/scrapy/core/downloader/manager.py
+++ b/scrapy/trunk/scrapy/core/downloader/manager.py
@@ -13,6 +13,7 @@ from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy import log
 from scrapy.conf import settings
 from scrapy.utils.defer import chain_deferred, mustbe_deferred
+from scrapy.utils.request import request_info
 
 
 class SiteDetails(object):
@@ -43,7 +44,7 @@ class SiteDetails(object):
 
 
 class Downloader(object):
-    """Maintain many concurrent downloads and provide an HTTP abstraction.
+    """Mantain many concurrent downloads and provide an HTTP abstraction.
     It supports a limited number of connections per domain and many domains in
     parallel.
     """
@@ -59,6 +60,8 @@ class Downloader(object):
         self.middleware = DownloaderMiddlewareManager()
         self.middleware.download_function = self.enqueue
         self.download_function = download_any
+        self.concurrent_domains = settings.getint('CONCURRENT_DOMAINS')
+        self.debug_mode = settings.getbool('DOWNLOADER_DEBUG')
 
     def fetch(self, request, spider):
         """ Main method to use to request a download
@@ -120,13 +123,15 @@ class Downloader(object):
             self.engine.closed_domain(domain)
 
     def _download(self, request, spider, deferred):
-        log.msg('Activating %s' % request.traceinfo(), log.TRACE)
+        if self.debug_mode:
+            log.msg('Activating %s' % request_info(request), log.DEBUG)
         domain = spider.domain_name
         site = self.sites.get(domain)
         site.downloading.add(request)
 
         def _remove(result):
-            log.msg('Deactivating %s' % request.traceinfo(), log.TRACE)
+            if self.debug_mode:
+                log.msg('Deactivating %s' % request_info(request), log.DEBUG)
             site.downloading.remove(request)
             return result
 
@@ -153,14 +158,16 @@ class Downloader(object):
 
     def close_domain(self, domain):
         """Free any resources associated with the given domain"""
-        log.msg("Downloader closing domain %s" % domain, log.TRACE, domain=domain)
+        if self.debug_mode:
+            log.msg("Downloader closing domain %s" % domain, log.DEBUG, domain=domain)
         site = self.sites.get(domain)
         if site:
             site.closed = True
             spider = spiders.fromdomain(domain)
             self.process_queue(spider)
         else:
-            log.msg('Domain %s already closed' % domain, log.TRACE, domain=domain)
+            if self.debug_mode:
+                log.msg('Domain %s already closed' % domain, log.DEBUG, domain=domain)
 
     def needs_backout(self, domain):
         site = self.sites.get(domain)
@@ -195,13 +202,8 @@ class Downloader(object):
 
     def has_capacity(self):
         """Does the downloader have capacity to handle more domains"""
-        return len(self.sites) < settings.getint('CONCURRENT_DOMAINS')
+        return len(self.sites) < self.concurrent_domains
 
     def is_idle(self):
         return not self.sites
 
-    # deprecated
-    def clear_requests(self, domain):
-        log.msg("Downloader clearing request for domain %s" % domain, log.TRACE, domain=domain)
-        self.sites[domain].queue = []
-
diff --git a/scrapy/trunk/scrapy/core/engine.py b/scrapy/trunk/scrapy/core/engine.py
index 52f682461..4cd5e5c7f 100644
--- a/scrapy/trunk/scrapy/core/engine.py
+++ b/scrapy/trunk/scrapy/core/engine.py
@@ -1,6 +1,8 @@
 """
-Core execution engine for the web crawling framework. This controls the 
-scheduler, downloader and spiders.
+This is the Scrapy engine which controls the Scheduler, Downloader and Spiders.
+
+For more information see docs/topics/architecture.rst
+
 """
 from datetime import datetime
 
@@ -21,6 +23,7 @@ from scrapy.item.pipeline import ItemPipelineManager
 from scrapy.spider import spiders
 from scrapy.spider.middleware import SpiderMiddlewareManager
 from scrapy.utils.defer import chain_deferred, defer_succeed, mustbe_deferred, deferred_degenerate
+from scrapy.utils.request import request_info
 
 class ExecutionEngine(object):
     """
@@ -247,7 +250,7 @@ class ExecutionEngine(object):
                     signals.send_catch_log(signal=signals.request_received, sender=self.__class__, request=item, spider=spider, response=response)
                     self.crawl(request=item, spider=spider, priority=priority)
                 else:
-                    log.msg('Garbage found in spider output while processing %s, got type %s' % (request, type(item)), log.TRACE, domain=domain)
+                    log.msg('Garbage found in spider output while processing %s, got type %s' % (request, type(item)), log.DEBUG, domain=domain)
 
             class _ResultContainer(object):
                 def append(self, item):
@@ -292,10 +295,10 @@ class ExecutionEngine(object):
         domain = spider.domain_name
         if not self.scheduler.domain_is_open(domain):
             if self.debug_mode: 
-                log.msg('Scheduling %s (delayed)' % request.traceinfo(), log.TRACE)
+                log.msg('Scheduling %s (delayed)' % request_info(request), log.DEBUG)
             return self._add_starter(request, spider, domain_priority)
         if self.debug_mode: 
-            log.msg('Scheduling %s (now)' % request.traceinfo(), log.TRACE)
+            log.msg('Scheduling %s (now)' % request_info(request), log.DEBUG)
         schd = self.scheduler.enqueue_request(domain, request, priority)
         self.next_request(spider)
         return schd
@@ -338,13 +341,15 @@ class ExecutionEngine(object):
         del self.starters[domain]
 
     def download(self, request, spider):
-        log.msg('Downloading %s' % request.traceinfo(), log.TRACE)
+        if self.debug_mode:
+            log.msg('Downloading %s' % request_info(request), log.DEBUG)
         domain = spider.domain_name
 
         def _on_success(response):
             """handle the result of a page download"""
             assert isinstance(response, (Response, Request))
-            log.msg("Requested %s" % request.traceinfo(), level=log.TRACE, domain=domain)
+            if self.debug_mode:
+                log.msg("Requested %s" % request_info(request), level=log.DEBUG, domain=domain)
             if isinstance(response, Response):
                 response.request = request # tie request to obtained response
                 cached = 'cached' if response.cached else 'live'
diff --git a/scrapy/trunk/scrapy/core/scheduler/schedulers.py b/scrapy/trunk/scrapy/core/scheduler/schedulers.py
index 5abf936b7..015e9d5fe 100644
--- a/scrapy/trunk/scrapy/core/scheduler/schedulers.py
+++ b/scrapy/trunk/scrapy/core/scheduler/schedulers.py
@@ -8,6 +8,7 @@ from scrapy.core.scheduler.filter import GroupFilter
 from scrapy import log
 from scrapy.core.exceptions import IgnoreRequest
 from scrapy.utils.datatypes import PriorityQueue, PriorityStack
+from scrapy.utils.request import request_fingerprint
 from scrapy.utils.defer import defer_fail
 from scrapy.conf import settings
 
@@ -108,7 +109,7 @@ class Scheduler(object) :
         """
         Add a page to be scraped for a domain that is currently being scraped.
         """
-        requestid = request.fingerprint()
+        requestid = request_fingerprint(request)
         added = self.groupfilter.add(domain, requestid)
 
         if request.dont_filter or added:
@@ -123,7 +124,7 @@ class Scheduler(object) :
         Returns True if the given Request was scheduled before for the given
         domain
         """
-        return self.groupfilter.has(domain, request.fingerprint())
+        return self.groupfilter.has(domain, request_fingerprint(request))
 
     def next_request(self, domain):
         """
diff --git a/scrapy/trunk/scrapy/http/request.py b/scrapy/trunk/scrapy/http/request.py
index d64f610d4..1716af270 100644
--- a/scrapy/trunk/scrapy/http/request.py
+++ b/scrapy/trunk/scrapy/http/request.py
@@ -1,5 +1,4 @@
 import urllib
-import hashlib
 from copy import copy
 from base64 import urlsafe_b64encode
 
@@ -14,7 +13,7 @@ from scrapy.utils.defer import chain_deferred
 class Request(object):
     def __init__(self, url, callback=None, context=None, method=None, body=None, headers=None, cookies=None,
             referer=None, url_encoding='utf-8', link_text='', http_user='', http_pass='', dont_filter=None, 
-            fingerprint_params=None, domain=None):
+            domain=None):
 
         self.encoding = url_encoding  # this one has to be set first
         self.set_url(url)
@@ -43,9 +42,6 @@ class Request(object):
         self.context = context or {}
         # dont_filter be filtered by scheduler
         self.dont_filter = dont_filter
-        # fingerprint parameters
-        self.fingerprint_params = fingerprint_params or {}
-        self._fingerprint = None
         # shortcut for setting referer
         if referer is not None:
             self.headers['referer'] = referer
@@ -56,6 +52,9 @@ class Request(object):
         self.link_text = link_text
         #allows to directly specify the spider for the request
         self.domain = domain
+
+        # bucket to store cached data such as fingerprint and others
+        self._cache = {}
         
     def append_callback(self, callback, *args, **kwargs):
         if isinstance(callback, defer.Deferred):
@@ -73,7 +72,6 @@ class Request(object):
         assert isinstance(url, basestring), 'Request url argument must be str or unicode, got %s:' % type(url).__name__
         decoded_url = url if isinstance(url, unicode) else url.decode(self.encoding)
         self._url = Url(safe_url_string(decoded_url, self.encoding))
-        self._fingerprint = None # invalidate cached fingerprint
     url = property(lambda x: x._url, set_url)
 
     def httpauth(self, http_user, http_pass):
@@ -92,11 +90,6 @@ class Request(object):
         """Return raw HTTP request size"""
         return len(self.to_string())
 
-    def traceinfo(self):
-        fp = self.fingerprint()
-        version = '%s..%s' % (fp[:4], fp[-4:])
-        return "<Request: %s %s (%s)>" % (self.method, self.url, version)
-
     def __repr__(self):
         d = {
             'method': self.method,
@@ -111,61 +104,15 @@ class Request(object):
     def copy(self):
         """Clone request except `context` attribute"""
         new = copy(self)
+        new._cache = {}
         for att in self.__dict__:
-            if att not in ['context', 'url', 'deferred', '_fingerprint']:
+            if att not in ['_cache', 'context', 'url', 'deferred']:
                 value = getattr(self, att)
                 setattr(new, att, copy(value))
         new.deferred = defer.Deferred()
         new.context = self.context # requests shares same context dictionary
-        new._fingerprint = None # reset fingerprint
         return new
 
-    def fingerprint(self):
-        """Returns unique resource fingerprint with caching support"""
-        if not self._fingerprint or self.fingerprint_params:
-            self.update_fingerprint()
-        return self._fingerprint
-
-    def update_fingerprint(self):
-        """Update request fingerprint, based on its current data. A request
-        fingerprint is a hash which uniquely identifies the HTTP resource"""
-
-        headers = {}
-        if self.fingerprint_params:
-            if 'tamperfunc' in self.fingerprint_params:
-                tamperfunc = self.fingerprint_params['tamperfunc']
-                assert callable(tamperfunc)
-                req = tamperfunc(self.copy())
-                assert isinstance(req, Request)
-                try:
-                    del req.fingerprint_params['tamperfunc']
-                except KeyError:
-                    pass
-                return req.fingerprint()
-
-            if self.headers:
-                if 'include_headers' in self.fingerprint_params:
-                    keys = [k.lower() for k in self.fingerprint_params['include_headers']]
-                    headers = dict([(k, v) for k, v in self.headers.items() if k.lower() in keys])
-                elif 'exclude_headers' in self.fingerprint_params:
-                    keys = [k.lower() for k in self.fingerprint_params['exclude_headers']]
-                    headers = dict([(k, v) for k, v in self.headers.items() if k.lower() not in keys])
-
-        # fingerprint generation
-        fp = hashlib.sha1()
-        fp.update(canonicalize(self.url))
-        fp.update(self.method)
-
-        if self.body and self.method in ['POST', 'PUT']:
-            fp.update(self.body)
-
-        if headers:
-            for k, v in sorted([(k.lower(), v) for k, v in headers.items()]):
-                fp.update(k)
-                fp.update(v)
-
-        self._fingerprint = fp.hexdigest()
-
     def to_string(self):
         """ Return raw HTTP request representation (as string). This is
         provided only for reference since it's not the actual stream of bytes
diff --git a/scrapy/trunk/scrapy/replay/__init__.py b/scrapy/trunk/scrapy/replay/__init__.py
index 5026914a5..507c999dd 100644
--- a/scrapy/trunk/scrapy/replay/__init__.py
+++ b/scrapy/trunk/scrapy/replay/__init__.py
@@ -132,7 +132,6 @@ class Replay(object):
             self.passed_new[str(item.guid)] = item
 
     def response_received(self, response, spider):
-        #key = response.request.fingerprint()
         key = response.version()
         if (self.recording or self.updating) and key:
             self.responses_old[key] = response.copy()
diff --git a/scrapy/trunk/scrapy/tests/test_http_request.py b/scrapy/trunk/scrapy/tests/test_http_request.py
index 511ca9797..c036c4196 100644
--- a/scrapy/trunk/scrapy/tests/test_http_request.py
+++ b/scrapy/trunk/scrapy/tests/test_http_request.py
@@ -3,12 +3,10 @@ from scrapy.http import Request, Headers
 from scrapy.core.scheduler import GroupFilter
 
 class RequestTest(unittest.TestCase):
-    """ c14n comparison functions """
 
     def test_groupfilter(self):
-        k1 = Request(url="http://www.scrapy.org/path?key=val").fingerprint()
-        k2 = Request(url="http://www.scrapy.org/path?key=val&key=val").fingerprint()
-        self.assertEqual(k1, k2)
+        k1 = "id1"
+        k2 = "id1"
 
         f = GroupFilter()
         f.open("mygroup")
@@ -40,7 +38,7 @@ class RequestTest(unittest.TestCase):
         # headers must not be unicode
         h = Headers({'key1': u'val1', u'key2': 'val2'})
         h[u'newkey'] = u'newval'
-        for k,v in h.iteritems():
+        for k, v in h.iteritems():
             self.assert_(isinstance(k, str))
             self.assert_(isinstance(v, str))
 
@@ -58,146 +56,6 @@ class RequestTest(unittest.TestCase):
         set_.add(r2)
         self.assertEqual(len(set_), 2)
 
-    def test_fingerprint(self):
-        url = 'http://www.scrapy.org'
-        r = Request(url=url)
-        urlhash = r.fingerprint()
-
-        # fingerprint including all initials headers
-        r.fingerprint_params['exclude_headers'] = []
-        fullhash = r.fingerprint()
-        del r.fingerprint_params['exclude_headers']
-
-        # all headers are excluded from fingerprint by default
-        r.headers['Accept'] = 'application/json'
-        accepthash = r.fingerprint()
-        r.headers['User-Agent'] = 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.8) Gecko/20071004 Iceweasel/2.0.0.8 (Debian-2.0.0.8-1)'
-        useragenthash = r.fingerprint()
-        self.assertEqual(urlhash, accepthash)
-        self.assertEqual(urlhash, useragenthash)
-        self.assertEqual(accepthash, useragenthash)
-
-        # take User-Agent header in count as specified in `include_headers`
-        r.fingerprint_params['include_headers'] = ['user-agent']
-        useragenthash = r.fingerprint()
-        self.assertNotEqual(useragenthash, urlhash)
-
-        # include_headers has precedence over exclude_headers
-        r.fingerprint_params['exclude_headers'] = ['user-agent', 'Accept']
-        self.assertEqual(useragenthash, r.fingerprint())
-
-        del r.fingerprint_params['include_headers']
-        self.assertNotEqual(useragenthash, r.fingerprint()) # exclude_headers is excluding 'User-Agent' from hash
-        self.assertEqual(fullhash, r.fingerprint()) # all headers previously seted was excluded
-
-        # set more headers
-        r.headers['Accept-Language'] = 'en-us,en;q=0.5'
-        r.headers['SESSIONID'] = 'an ugly session id header'
-        self.assertNotEqual(urlhash, r.fingerprint())
-
-        # force emtpy include_headers (ignore exclude_headers)
-        r.fingerprint_params['include_headers'] = []
-        self.assertEqual(urlhash, r.fingerprint())
-
-        # Tamper Function
-        r.fingerprint_params['tamperfunc'] = lambda req: Request(url=req.url)
-        self.assertEqual(urlhash, r.fingerprint())
-
-        # Compare request with None vs {} header
-        r = Request(url=url, headers=None)
-        o = Request(url=url, headers={})
-        self.assertEqual(o.fingerprint(), r.fingerprint())
-        self.assertNotEqual(r, o)
-
-        # Different ways of setting headers attribute
-        headers = {'Accept':'gzip', 'Custom-Header':'nothing to tell you'}
-        r = Request(url=url, headers=headers)
-        p = Request(url=url, headers=r.headers)
-        o = Request(url=url)
-        o.headers = headers
-        self.assertEqual(r.fingerprint(), o.fingerprint())
-        self.assertEqual(r.fingerprint(), p.fingerprint())
-        self.assertEqual(o.fingerprint(), p.fingerprint())
-        self.assertNotEqual(r, o)
-        self.assertNotEqual(r, p)
-        self.assertNotEqual(o, p)
-
-        # same url, implicit method
-        r1 = Request(url=url)
-        r2 = Request(url=url, method='GET')
-        self.assertEqual(r1.fingerprint(), r2.fingerprint())
-
-        # same url, different method
-        r3 = Request(url=url, method='POST')
-        self.assertNotEqual(r1.fingerprint(), r3.fingerprint())
-
-        # implicit POST method
-        r3.body = ''
-        r4 = Request(url=url, body='')
-        self.assertEqual(r3.fingerprint(), r4.fingerprint())
-
-        # body is not important in GET or DELETE
-        r1 = Request(url=url, method='get', body='data')
-        r2 = Request(url=url, method='get')
-        self.assertEqual(r1.fingerprint(), r2.fingerprint())
-
-        r1 = Request(url=url, method='delete', body='data')
-        r2 = Request(url=url, method='delete')
-        self.assertEqual(r1.fingerprint(), r2.fingerprint())
-
-        # no body by default
-        r1 = Request(url=url, method='POST')
-        r2 = Request(url=url, method='POST', body=None)
-        self.assertEqual(r1.fingerprint(), r2.fingerprint())
-
-        # empty body is equal to None body
-        r3 = Request(url=url, method='POST', body='')
-        self.assertEqual(r1.fingerprint(), r3.fingerprint())
-
-    def test_insensitive_request_fingerprints(self):
-        url = 'http://www.scrapy.org'
-        fp = {'include_headers':['Accept','Content-Type']}
-        r1a = Request(url=url.lower())
-        r1b = Request(url=url.upper())
-        self.assertEqual(r1a.fingerprint(), r1b.fingerprint())
-
-        r1a = Request(url=url.lower(), method='get')
-        r1b = Request(url=url.upper(), method='GET')
-        self.assertEqual(r1a.fingerprint(), r1b.fingerprint())
-
-        r1c = Request(url=url.upper(), method='GET', body='this is not important')
-        self.assertEqual(r1b.fingerprint(), r1c.fingerprint())
-
-        r1a = Request(url=url.lower(), method='get', fingerprint_params=fp)
-        r1b = Request(url=url.upper(), method='GET', fingerprint_params=fp)
-        self.assertEqual(r1a.fingerprint(), r1b.fingerprint())
-
-        r1a = Request(url=url.lower(), method='get', headers={'ACCEPT':'Black'})
-        r1b = Request(url=url.upper(), method='GET', headers={'ACCEPT':'Black'})
-        self.assertEqual(r1a.fingerprint(), r1b.fingerprint())
-
-        r1a = Request(url=url.lower(), method='get', fingerprint_params=fp, headers={'ACCEPT':'Black'})
-        r1b = Request(url=url.upper(), method='GET', fingerprint_params=fp, headers={'ACCEPT':'Black'})
-        self.assertEqual(r1a.fingerprint(), r1b.fingerprint())
-
-        r1a = Request(url=url.lower(), method='get', fingerprint_params=fp, headers={'ACCEPT':'Black'})
-        r1b = Request(url=url.upper(), method='GET', fingerprint_params=fp, headers={'Accept':'Black'})
-        r1c = Request(url=url.upper(), method='gEt', fingerprint_params=fp, headers={'accept':'Black'})
-        self.assertEqual(r1a.fingerprint(), r1b.fingerprint())
-        self.assertEqual(r1a.fingerprint(), r1c.fingerprint())
-
-        r1a = Request(url=url.lower(), method='get', fingerprint_params=fp, headers={'ACCEPT':'Black', 'content-type':'application/json'})
-        r1b = Request(url=url.upper(), method='GET', fingerprint_params=fp, headers={'Accept':'Black', 'Content-Type':'application/json'})
-        r1c = Request(url=url.upper(), method='Get', fingerprint_params=fp, headers={'accepT':'Black', 'CONTENT-TYPE':'application/json'})
-        self.assertEqual(r1a.fingerprint(), r1b.fingerprint())
-        self.assertEqual(r1a.fingerprint(), r1c.fingerprint())
-
-        r1a = Request(url=url.lower(), method='Post', fingerprint_params=fp, headers={'ACCEPT':'Black', 'content-type':'application/json'})
-        r1b = Request(url=url.upper(), method='POST', fingerprint_params=fp, headers={'Accept':'Black', 'Content-Type':'application/json'})
-        r1c = Request(url=url.upper(), method='posT', fingerprint_params=fp, headers={'accepT':'Black', 'CONTENT-TYPE':'application/json'})
-        self.assertEqual(r1a.fingerprint(), r1b.fingerprint())
-        self.assertEqual(r1a.fingerprint(), r1c.fingerprint())
-
     def test_url(self):
         """Request url tests"""
         r = Request(url="http://www.scrapy.org/path")
@@ -226,5 +84,3 @@ class RequestTest(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
-
-
diff --git a/scrapy/trunk/scrapy/tests/test_utils_request.py b/scrapy/trunk/scrapy/tests/test_utils_request.py
new file mode 100644
index 000000000..60b4bdac5
--- /dev/null
+++ b/scrapy/trunk/scrapy/tests/test_utils_request.py
@@ -0,0 +1,61 @@
+import unittest
+from scrapy.http import Request
+from scrapy.utils.request import request_fingerprint
+
+class UtilsRequestTest(unittest.TestCase):
+
+    def test_request_fingerprint(self):
+        url = 'http://www.scrapy.org'
+        r = Request(url=url)
+        urlhash = request_fingerprint(r)
+
+        r1 = Request("http://www.example.com/query?id=111&cat=222")
+        r2 = Request("http://www.example.com/query?cat=222&id=111")
+        self.assertEqual(request_fingerprint(r1), request_fingerprint(r1))
+        self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
+
+        # make sure caching is working
+        self.assertEqual(request_fingerprint(r1), r1._cache['fingerprint'])
+
+        r1 = Request("http://www.example.com/members/offers.html")
+        r2 = Request("http://www.example.com/members/offers.html")
+        r2.headers['SESSIONID'] = "somehash"
+        self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
+
+        r1 = Request("http://www.example.com/")
+        r2 = Request("http://www.example.com/")
+        r2.headers['Accept-Language'] = 'en'
+        r3 = Request("http://www.example.com/")
+        r3.headers['Accept-Language'] = 'en'
+        r3.headers['SESSIONID'] = "somehash"
+
+        self.assertEqual(request_fingerprint(r1), request_fingerprint(r2), request_fingerprint(r3))
+
+        self.assertEqual(request_fingerprint(r1),
+                         request_fingerprint(r1, include_headers=['Accept-Language']))
+
+        self.assertNotEqual(request_fingerprint(r1),
+                         request_fingerprint(r2, include_headers=['Accept-Language']))
+
+        self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']),
+                         request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language']))
+
+        r1 = Request("http://www.example.com")
+        r2 = Request("http://www.example.com", method='POST')
+        r3 = Request("http://www.example.com", method='POST', body='request body')
+
+        self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
+        self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r3))
+
+        # cached fingerprint must be cleared on request copy
+        r1 = Request("http://www.example.com")
+        fp1 = request_fingerprint(r1)
+        r2 = r1.copy()
+        r2.url = "http://www.example.com/other"
+        fp2 = request_fingerprint(r2)
+        self.assertNotEqual(fp1, fp2)
+
+if __name__ == "__main__":
+    unittest.main()
+
+
diff --git a/scrapy/trunk/scrapy/utils/request.py b/scrapy/trunk/scrapy/utils/request.py
new file mode 100644
index 000000000..41d49c282
--- /dev/null
+++ b/scrapy/trunk/scrapy/utils/request.py
@@ -0,0 +1,65 @@
+"""
+This module provides some useful functions for working with
+scrapy.http.Request objects
+"""
+
+import hashlib
+
+from scrapy.utils.url import canonicalize_url
+
+def request_fingerprint(request, include_headers=()):
+    """
+    Return the request fingerprint.
+    
+    The request fingerprint is a hash that uniquely identifies the resource the
+    request points to. For example, take the following two urls:
+    
+    http://www.example.com/query?id=111&cat=222
+    http://www.example.com/query?cat=222&id=111
+
+    Even though those are two different URLs both point to the same resource
+    and are equivalent (ie. they should return the same response).
+
+    Another example are cookies used to store session ids. Suppose the
+    following page is only accesible to authenticated users:
+    
+    http://www.example.com/members/offers.html
+
+    Lot of sites use a cookie to store the session id, which adds a random
+    component to the HTTP Request and thus should be ignored when calculating
+    the fingerprint. 
+    
+    For this reason, request headers are ignored by default when calculating
+    the fingeprint. If you want to include specific headers use the
+    include_headers argument, which is a list of Request headers to include.
+
+    """
+
+    if include_headers:
+        include_headers = [h.lower() for h in sorted(include_headers)]
+        cachekey = 'fingerprint' + '_'.join(include_headers)
+    else:
+        cachekey = 'fingerprint'
+
+    try:
+        return request._cache[cachekey]
+    except KeyError:
+        fp = hashlib.sha1()
+        fp.update(request.method)
+        fp.update(canonicalize_url(request.url))
+        fp.update(request.body or '')
+        for hdr in include_headers:
+            if hdr in request.headers:
+                fp.update(hdr)
+                fp.update(request.headers.get(hdr, ''))
+        fphash = fp.hexdigest()
+        request._cache[cachekey] = fphash
+        return fphash
+
+def request_info(request):
+    """Return a short string with request info including method, url and
+    fingeprint. Mainly used for debugging
+    """
+    fp = request_fingerprint(request)
+    return "<Request: %s %s (%s..)>" % (request.method, request.url, fp[:8])
+