RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

* moved request fingerprinting from Request class to scrapy.utils.request - closes #50 

* cleaned up fingerprint tests suite (only left relevant tests)

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40722
This commit is contained in:
Pablo Hoffman 2009-01-14 01:17:40 +00:00
parent 519458bdae
commit 0ea37f51db
13 changed files with 177 additions and 238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/trunk/scrapy/conf/default_settings.py
View File

@ -55,6 +55,8 @@ DEPTH_STATS = True
DOWNLOAD_DELAY = 0
DOWNLOAD_TIMEOUT = 180 # 3mins
DOWNLOADER_DEBUG = False
DOWNLOADER_MIDDLEWARES = [
# Engine side
'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware',

14
scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py
View File

@ -1,20 +1,18 @@
from __future__ import with_statement
import os
import sys
import hashlib
import datetime
import urlparse
import cPickle as pickle
from pydispatch import dispatcher
from twisted.internet import defer
from scrapy.core import signals
from scrapy import log
from scrapy.core.engine import scrapyengine
from scrapy.http import Response, Headers
from scrapy.http.headers import headers_dict_to_raw
from scrapy.core.exceptions import UsageError, NotConfigured, HttpException, IgnoreRequest
from scrapy.core.exceptions import NotConfigured, HttpException, IgnoreRequest
from scrapy.utils.request import request_fingerprint
from scrapy.conf import settings
@ -33,7 +31,7 @@ class CacheMiddleware(object):
if not is_cacheable(request):
return
key = request.fingerprint()
key = request_fingerprint(request)
domain = spider.domain_name
try:
response = self.cache.retrieve_response(domain, key)
@ -53,7 +51,7 @@ class CacheMiddleware(object):
return response
if isinstance(response, Response) and not response.cached:
key = request.fingerprint()
key = request_fingerprint(request)
domain = spider.domain_name
self.cache.store(domain, key, request, response)
@ -64,9 +62,9 @@ class CacheMiddleware(object):
return
if isinstance(exception, HttpException) and isinstance(exception.response, Response):
key = request.fingerprint()
key = request_fingerprint(request)
domain = spider.domain_name
self.cache.store(domain,key, request, exception.response)
self.cache.store(domain, key, request, exception.response)
def is_cacheable(request):
scheme, _, _, _, _ = urlparse.urlsplit(request.url)

3
scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py
View File

@ -30,6 +30,7 @@ from twisted.internet.defer import TimeoutError as UserTimeoutError
from scrapy import log
from scrapy.core.exceptions import HttpException
from scrapy.utils.request import request_fingerprint
from scrapy.conf import settings
class RetryMiddleware(object):
@ -44,7 +45,7 @@ class RetryMiddleware(object):
def process_exception(self, request, exception, spider):
if isinstance(exception, self.EXCEPTIONS_TO_RETRY) or (isinstance(exception, HttpException) and (int(exception.status) in self.retry_http_codes)):
fp = request.fingerprint()
fp = request_fingerprint(request)
self.failed_count[fp] = self.failed_count.get(fp, 0) + 1
if self.failed_count[fp] <= self.retry_times:

3
scrapy/trunk/scrapy/contrib/history/scheduler.py
View File

@ -5,6 +5,7 @@ from twisted.internet import defer
from scrapy import log
from scrapy.core.scheduler import Scheduler
from scrapy.core.exceptions import IgnoreRequest
from scrapy.utils.request import request_fingerprint
class RulesScheduler(Scheduler):
"""Scheduler that uses rules to determine if we should follow links
@ -33,7 +34,7 @@ class RulesScheduler(Scheduler):
The url will only be added if we have not checked it already within
a specified time period.
"""
requestid = request.fingerprint()
requestid = request_fingerprint(request)
added = self.groupfilter.add(domain, requestid)
if request.dont_filter or added:

3
scrapy/trunk/scrapy/contrib/pipeline/media.py
View File

@ -3,6 +3,7 @@ from twisted.internet import defer
from scrapy.utils.defer import mustbe_deferred, defer_result
from scrapy import log
from scrapy.core.engine import scrapyengine
from scrapy.utils.request import request_fingerprint
from scrapy.spider import spiders
@ -53,7 +54,7 @@ class MediaPipeline(object):
def _enqueue(self, request, info):
wad = request.deferred or defer.Deferred()
fp = request.fingerprint()
fp = request_fingerprint(request)
# if already downloaded, return cached result.
if fp in info.downloaded:

24
scrapy/trunk/scrapy/core/downloader/manager.py
View File

@ -13,6 +13,7 @@ from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy import log
from scrapy.conf import settings
from scrapy.utils.defer import chain_deferred, mustbe_deferred
from scrapy.utils.request import request_info
class SiteDetails(object):
@ -43,7 +44,7 @@ class SiteDetails(object):
class Downloader(object):
"""Maintain many concurrent downloads and provide an HTTP abstraction.
"""Mantain many concurrent downloads and provide an HTTP abstraction.
It supports a limited number of connections per domain and many domains in
parallel.
"""
@ -59,6 +60,8 @@ class Downloader(object):
self.middleware = DownloaderMiddlewareManager()
self.middleware.download_function = self.enqueue
self.download_function = download_any
self.concurrent_domains = settings.getint('CONCURRENT_DOMAINS')
self.debug_mode = settings.getbool('DOWNLOADER_DEBUG')
def fetch(self, request, spider):
""" Main method to use to request a download
@ -120,13 +123,15 @@ class Downloader(object):
self.engine.closed_domain(domain)
def _download(self, request, spider, deferred):
log.msg('Activating %s' % request.traceinfo(), log.TRACE)
if self.debug_mode:
log.msg('Activating %s' % request_info(request), log.DEBUG)
domain = spider.domain_name
site = self.sites.get(domain)
site.downloading.add(request)
def _remove(result):
log.msg('Deactivating %s' % request.traceinfo(), log.TRACE)
if self.debug_mode:
log.msg('Deactivating %s' % request_info(request), log.DEBUG)
site.downloading.remove(request)
return result
@ -153,14 +158,16 @@ class Downloader(object):
def close_domain(self, domain):
"""Free any resources associated with the given domain"""
log.msg("Downloader closing domain %s" % domain, log.TRACE, domain=domain)
if self.debug_mode:
log.msg("Downloader closing domain %s" % domain, log.DEBUG, domain=domain)
site = self.sites.get(domain)
if site:
site.closed = True
spider = spiders.fromdomain(domain)
self.process_queue(spider)
else:
log.msg('Domain %s already closed' % domain, log.TRACE, domain=domain)
if self.debug_mode:
log.msg('Domain %s already closed' % domain, log.DEBUG, domain=domain)
def needs_backout(self, domain):
site = self.sites.get(domain)
@ -195,13 +202,8 @@ class Downloader(object):
def has_capacity(self):
"""Does the downloader have capacity to handle more domains"""
return len(self.sites) < settings.getint('CONCURRENT_DOMAINS')
return len(self.sites) < self.concurrent_domains
def is_idle(self):
return not self.sites
# deprecated
def clear_requests(self, domain):
log.msg("Downloader clearing request for domain %s" % domain, log.TRACE, domain=domain)
self.sites[domain].queue = []

19
scrapy/trunk/scrapy/core/engine.py
View File

@ -1,6 +1,8 @@
"""
Core execution engine for the web crawling framework. This controls the
scheduler, downloader and spiders.
This is the Scrapy engine which controls the Scheduler, Downloader and Spiders.
For more information see docs/topics/architecture.rst
"""
from datetime import datetime
@ -21,6 +23,7 @@ from scrapy.item.pipeline import ItemPipelineManager
from scrapy.spider import spiders
from scrapy.spider.middleware import SpiderMiddlewareManager
from scrapy.utils.defer import chain_deferred, defer_succeed, mustbe_deferred, deferred_degenerate
from scrapy.utils.request import request_info
class ExecutionEngine(object):
"""
@ -247,7 +250,7 @@ class ExecutionEngine(object):
signals.send_catch_log(signal=signals.request_received, sender=self.__class__, request=item, spider=spider, response=response)
self.crawl(request=item, spider=spider, priority=priority)
else:
log.msg('Garbage found in spider output while processing %s, got type %s' % (request, type(item)), log.TRACE, domain=domain)
log.msg('Garbage found in spider output while processing %s, got type %s' % (request, type(item)), log.DEBUG, domain=domain)
class _ResultContainer(object):
def append(self, item):
@ -292,10 +295,10 @@ class ExecutionEngine(object):
domain = spider.domain_name
if not self.scheduler.domain_is_open(domain):
if self.debug_mode:
log.msg('Scheduling %s (delayed)' % request.traceinfo(), log.TRACE)
log.msg('Scheduling %s (delayed)' % request_info(request), log.DEBUG)
return self._add_starter(request, spider, domain_priority)
if self.debug_mode:
log.msg('Scheduling %s (now)' % request.traceinfo(), log.TRACE)
log.msg('Scheduling %s (now)' % request_info(request), log.DEBUG)
schd = self.scheduler.enqueue_request(domain, request, priority)
self.next_request(spider)
return schd
@ -338,13 +341,15 @@ class ExecutionEngine(object):
del self.starters[domain]
def download(self, request, spider):
log.msg('Downloading %s' % request.traceinfo(), log.TRACE)
if self.debug_mode:
log.msg('Downloading %s' % request_info(request), log.DEBUG)
domain = spider.domain_name
def _on_success(response):
"""handle the result of a page download"""
assert isinstance(response, (Response, Request))
log.msg("Requested %s" % request.traceinfo(), level=log.TRACE, domain=domain)
if self.debug_mode:
log.msg("Requested %s" % request_info(request), level=log.DEBUG, domain=domain)
if isinstance(response, Response):
response.request = request # tie request to obtained response
cached = 'cached' if response.cached else 'live'

5
scrapy/trunk/scrapy/core/scheduler/schedulers.py
View File

@ -8,6 +8,7 @@ from scrapy.core.scheduler.filter import GroupFilter
from scrapy import log
from scrapy.core.exceptions import IgnoreRequest
from scrapy.utils.datatypes import PriorityQueue, PriorityStack
from scrapy.utils.request import request_fingerprint
from scrapy.utils.defer import defer_fail
from scrapy.conf import settings
@ -108,7 +109,7 @@ class Scheduler(object) :
"""
Add a page to be scraped for a domain that is currently being scraped.
"""
requestid = request.fingerprint()
requestid = request_fingerprint(request)
added = self.groupfilter.add(domain, requestid)
if request.dont_filter or added:
@ -123,7 +124,7 @@ class Scheduler(object) :
Returns True if the given Request was scheduled before for the given
domain
"""
return self.groupfilter.has(domain, request.fingerprint())
return self.groupfilter.has(domain, request_fingerprint(request))
def next_request(self, domain):
"""

65
scrapy/trunk/scrapy/http/request.py
View File

@ -1,5 +1,4 @@
import urllib
import hashlib
from copy import copy
from base64 import urlsafe_b64encode
@ -14,7 +13,7 @@ from scrapy.utils.defer import chain_deferred
class Request(object):
def __init__(self, url, callback=None, context=None, method=None, body=None, headers=None, cookies=None,
referer=None, url_encoding='utf-8', link_text='', http_user='', http_pass='', dont_filter=None,
fingerprint_params=None, domain=None):
domain=None):
self.encoding = url_encoding # this one has to be set first
self.set_url(url)
@ -43,9 +42,6 @@ class Request(object):
self.context = context or {}
# dont_filter be filtered by scheduler
self.dont_filter = dont_filter
# fingerprint parameters
self.fingerprint_params = fingerprint_params or {}
self._fingerprint = None
# shortcut for setting referer
if referer is not None:
self.headers['referer'] = referer
@ -56,6 +52,9 @@ class Request(object):
self.link_text = link_text
#allows to directly specify the spider for the request
self.domain = domain
# bucket to store cached data such as fingerprint and others
self._cache = {}
def append_callback(self, callback, *args, **kwargs):
if isinstance(callback, defer.Deferred):
@ -73,7 +72,6 @@ class Request(object):
assert isinstance(url, basestring), 'Request url argument must be str or unicode, got %s:' % type(url).__name__
decoded_url = url if isinstance(url, unicode) else url.decode(self.encoding)
self._url = Url(safe_url_string(decoded_url, self.encoding))
self._fingerprint = None # invalidate cached fingerprint
url = property(lambda x: x._url, set_url)
def httpauth(self, http_user, http_pass):
@ -92,11 +90,6 @@ class Request(object):
"""Return raw HTTP request size"""
return len(self.to_string())
def traceinfo(self):
fp = self.fingerprint()
version = '%s..%s' % (fp[:4], fp[-4:])
return "<Request: %s %s (%s)>" % (self.method, self.url, version)
def __repr__(self):
d = {
'method': self.method,
@ -111,61 +104,15 @@ class Request(object):
def copy(self):
"""Clone request except `context` attribute"""
new = copy(self)
new._cache = {}
for att in self.__dict__:
if att not in ['context', 'url', 'deferred', '_fingerprint']:
if att not in ['_cache', 'context', 'url', 'deferred']:
value = getattr(self, att)
setattr(new, att, copy(value))
new.deferred = defer.Deferred()
new.context = self.context # requests shares same context dictionary
new._fingerprint = None # reset fingerprint
return new
def fingerprint(self):
"""Returns unique resource fingerprint with caching support"""
if not self._fingerprint or self.fingerprint_params:
self.update_fingerprint()
return self._fingerprint
def update_fingerprint(self):
"""Update request fingerprint, based on its current data. A request
fingerprint is a hash which uniquely identifies the HTTP resource"""
headers = {}
if self.fingerprint_params:
if 'tamperfunc' in self.fingerprint_params:
tamperfunc = self.fingerprint_params['tamperfunc']
assert callable(tamperfunc)
req = tamperfunc(self.copy())
assert isinstance(req, Request)
try:
del req.fingerprint_params['tamperfunc']
except KeyError:
pass
return req.fingerprint()
if self.headers:
if 'include_headers' in self.fingerprint_params:
keys = [k.lower() for k in self.fingerprint_params['include_headers']]
headers = dict([(k, v) for k, v in self.headers.items() if k.lower() in keys])
elif 'exclude_headers' in self.fingerprint_params:
keys = [k.lower() for k in self.fingerprint_params['exclude_headers']]
headers = dict([(k, v) for k, v in self.headers.items() if k.lower() not in keys])
# fingerprint generation
fp = hashlib.sha1()
fp.update(canonicalize(self.url))
fp.update(self.method)
if self.body and self.method in ['POST', 'PUT']:
fp.update(self.body)
if headers:
for k, v in sorted([(k.lower(), v) for k, v in headers.items()]):
fp.update(k)
fp.update(v)
self._fingerprint = fp.hexdigest()
def to_string(self):
""" Return raw HTTP request representation (as string). This is
provided only for reference since it's not the actual stream of bytes

1
scrapy/trunk/scrapy/replay/__init__.py
View File

@ -132,7 +132,6 @@ class Replay(object):
self.passed_new[str(item.guid)] = item
def response_received(self, response, spider):
#key = response.request.fingerprint()
key = response.version()
if (self.recording or self.updating) and key:
self.responses_old[key] = response.copy()

150
scrapy/trunk/scrapy/tests/test_http_request.py
View File

@ -3,12 +3,10 @@ from scrapy.http import Request, Headers
from scrapy.core.scheduler import GroupFilter
class RequestTest(unittest.TestCase):
""" c14n comparison functions """
def test_groupfilter(self):
k1 = Request(url="http://www.scrapy.org/path?key=val").fingerprint()
k2 = Request(url="http://www.scrapy.org/path?key=val&key=val").fingerprint()
self.assertEqual(k1, k2)
k1 = "id1"
k2 = "id1"
f = GroupFilter()
f.open("mygroup")
@ -40,7 +38,7 @@ class RequestTest(unittest.TestCase):
# headers must not be unicode
h = Headers({'key1': u'val1', u'key2': 'val2'})
h[u'newkey'] = u'newval'
for k,v in h.iteritems():
for k, v in h.iteritems():
self.assert_(isinstance(k, str))
self.assert_(isinstance(v, str))
@ -58,146 +56,6 @@ class RequestTest(unittest.TestCase):
set_.add(r2)
self.assertEqual(len(set_), 2)
def test_fingerprint(self):
url = 'http://www.scrapy.org'
r = Request(url=url)
urlhash = r.fingerprint()
# fingerprint including all initials headers
r.fingerprint_params['exclude_headers'] = []
fullhash = r.fingerprint()
del r.fingerprint_params['exclude_headers']
# all headers are excluded from fingerprint by default
r.headers['Accept'] = 'application/json'
accepthash = r.fingerprint()
r.headers['User-Agent'] = 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.8) Gecko/20071004 Iceweasel/2.0.0.8 (Debian-2.0.0.8-1)'
useragenthash = r.fingerprint()
self.assertEqual(urlhash, accepthash)
self.assertEqual(urlhash, useragenthash)
self.assertEqual(accepthash, useragenthash)
# take User-Agent header in count as specified in `include_headers`
r.fingerprint_params['include_headers'] = ['user-agent']
useragenthash = r.fingerprint()
self.assertNotEqual(useragenthash, urlhash)
# include_headers has precedence over exclude_headers
r.fingerprint_params['exclude_headers'] = ['user-agent', 'Accept']
self.assertEqual(useragenthash, r.fingerprint())
del r.fingerprint_params['include_headers']
self.assertNotEqual(useragenthash, r.fingerprint()) # exclude_headers is excluding 'User-Agent' from hash
self.assertEqual(fullhash, r.fingerprint()) # all headers previously seted was excluded
# set more headers
r.headers['Accept-Language'] = 'en-us,en;q=0.5'
r.headers['SESSIONID'] = 'an ugly session id header'
self.assertNotEqual(urlhash, r.fingerprint())
# force emtpy include_headers (ignore exclude_headers)
r.fingerprint_params['include_headers'] = []
self.assertEqual(urlhash, r.fingerprint())
# Tamper Function
r.fingerprint_params['tamperfunc'] = lambda req: Request(url=req.url)
self.assertEqual(urlhash, r.fingerprint())
# Compare request with None vs {} header
r = Request(url=url, headers=None)
o = Request(url=url, headers={})
self.assertEqual(o.fingerprint(), r.fingerprint())
self.assertNotEqual(r, o)
# Different ways of setting headers attribute
headers = {'Accept':'gzip', 'Custom-Header':'nothing to tell you'}
r = Request(url=url, headers=headers)
p = Request(url=url, headers=r.headers)
o = Request(url=url)
o.headers = headers
self.assertEqual(r.fingerprint(), o.fingerprint())
self.assertEqual(r.fingerprint(), p.fingerprint())
self.assertEqual(o.fingerprint(), p.fingerprint())
self.assertNotEqual(r, o)
self.assertNotEqual(r, p)
self.assertNotEqual(o, p)
# same url, implicit method
r1 = Request(url=url)
r2 = Request(url=url, method='GET')
self.assertEqual(r1.fingerprint(), r2.fingerprint())
# same url, different method
r3 = Request(url=url, method='POST')
self.assertNotEqual(r1.fingerprint(), r3.fingerprint())
# implicit POST method
r3.body = ''
r4 = Request(url=url, body='')
self.assertEqual(r3.fingerprint(), r4.fingerprint())
# body is not important in GET or DELETE
r1 = Request(url=url, method='get', body='data')
r2 = Request(url=url, method='get')
self.assertEqual(r1.fingerprint(), r2.fingerprint())
r1 = Request(url=url, method='delete', body='data')
r2 = Request(url=url, method='delete')
self.assertEqual(r1.fingerprint(), r2.fingerprint())
# no body by default
r1 = Request(url=url, method='POST')
r2 = Request(url=url, method='POST', body=None)
self.assertEqual(r1.fingerprint(), r2.fingerprint())
# empty body is equal to None body
r3 = Request(url=url, method='POST', body='')
self.assertEqual(r1.fingerprint(), r3.fingerprint())
def test_insensitive_request_fingerprints(self):
url = 'http://www.scrapy.org'
fp = {'include_headers':['Accept','Content-Type']}
r1a = Request(url=url.lower())
r1b = Request(url=url.upper())
self.assertEqual(r1a.fingerprint(), r1b.fingerprint())
r1a = Request(url=url.lower(), method='get')
r1b = Request(url=url.upper(), method='GET')
self.assertEqual(r1a.fingerprint(), r1b.fingerprint())
r1c = Request(url=url.upper(), method='GET', body='this is not important')
self.assertEqual(r1b.fingerprint(), r1c.fingerprint())
r1a = Request(url=url.lower(), method='get', fingerprint_params=fp)
r1b = Request(url=url.upper(), method='GET', fingerprint_params=fp)
self.assertEqual(r1a.fingerprint(), r1b.fingerprint())
r1a = Request(url=url.lower(), method='get', headers={'ACCEPT':'Black'})
r1b = Request(url=url.upper(), method='GET', headers={'ACCEPT':'Black'})
self.assertEqual(r1a.fingerprint(), r1b.fingerprint())
r1a = Request(url=url.lower(), method='get', fingerprint_params=fp, headers={'ACCEPT':'Black'})
r1b = Request(url=url.upper(), method='GET', fingerprint_params=fp, headers={'ACCEPT':'Black'})
self.assertEqual(r1a.fingerprint(), r1b.fingerprint())
r1a = Request(url=url.lower(), method='get', fingerprint_params=fp, headers={'ACCEPT':'Black'})
r1b = Request(url=url.upper(), method='GET', fingerprint_params=fp, headers={'Accept':'Black'})
r1c = Request(url=url.upper(), method='gEt', fingerprint_params=fp, headers={'accept':'Black'})
self.assertEqual(r1a.fingerprint(), r1b.fingerprint())
self.assertEqual(r1a.fingerprint(), r1c.fingerprint())
r1a = Request(url=url.lower(), method='get', fingerprint_params=fp, headers={'ACCEPT':'Black', 'content-type':'application/json'})
r1b = Request(url=url.upper(), method='GET', fingerprint_params=fp, headers={'Accept':'Black', 'Content-Type':'application/json'})
r1c = Request(url=url.upper(), method='Get', fingerprint_params=fp, headers={'accepT':'Black', 'CONTENT-TYPE':'application/json'})
self.assertEqual(r1a.fingerprint(), r1b.fingerprint())
self.assertEqual(r1a.fingerprint(), r1c.fingerprint())
r1a = Request(url=url.lower(), method='Post', fingerprint_params=fp, headers={'ACCEPT':'Black', 'content-type':'application/json'})
r1b = Request(url=url.upper(), method='POST', fingerprint_params=fp, headers={'Accept':'Black', 'Content-Type':'application/json'})
r1c = Request(url=url.upper(), method='posT', fingerprint_params=fp, headers={'accepT':'Black', 'CONTENT-TYPE':'application/json'})
self.assertEqual(r1a.fingerprint(), r1b.fingerprint())
self.assertEqual(r1a.fingerprint(), r1c.fingerprint())
def test_url(self):
"""Request url tests"""
r = Request(url="http://www.scrapy.org/path")
@ -226,5 +84,3 @@ class RequestTest(unittest.TestCase):
if __name__ == "__main__":
unittest.main()

61
scrapy/trunk/scrapy/tests/test_utils_request.py Normal file
View File

@ -0,0 +1,61 @@
import unittest
from scrapy.http import Request
from scrapy.utils.request import request_fingerprint
class UtilsRequestTest(unittest.TestCase):
def test_request_fingerprint(self):
url = 'http://www.scrapy.org'
r = Request(url=url)
urlhash = request_fingerprint(r)
r1 = Request("http://www.example.com/query?id=111&cat=222")
r2 = Request("http://www.example.com/query?cat=222&id=111")
self.assertEqual(request_fingerprint(r1), request_fingerprint(r1))
self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
# make sure caching is working
self.assertEqual(request_fingerprint(r1), r1._cache['fingerprint'])
r1 = Request("http://www.example.com/members/offers.html")
r2 = Request("http://www.example.com/members/offers.html")
r2.headers['SESSIONID'] = "somehash"
self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
r1 = Request("http://www.example.com/")
r2 = Request("http://www.example.com/")
r2.headers['Accept-Language'] = 'en'
r3 = Request("http://www.example.com/")
r3.headers['Accept-Language'] = 'en'
r3.headers['SESSIONID'] = "somehash"
self.assertEqual(request_fingerprint(r1), request_fingerprint(r2), request_fingerprint(r3))
self.assertEqual(request_fingerprint(r1),
request_fingerprint(r1, include_headers=['Accept-Language']))
self.assertNotEqual(request_fingerprint(r1),
request_fingerprint(r2, include_headers=['Accept-Language']))
self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']),
request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language']))
r1 = Request("http://www.example.com")
r2 = Request("http://www.example.com", method='POST')
r3 = Request("http://www.example.com", method='POST', body='request body')
self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r3))
# cached fingerprint must be cleared on request copy
r1 = Request("http://www.example.com")
fp1 = request_fingerprint(r1)
r2 = r1.copy()
r2.url = "http://www.example.com/other"
fp2 = request_fingerprint(r2)
self.assertNotEqual(fp1, fp2)
if __name__ == "__main__":
unittest.main()

65
scrapy/trunk/scrapy/utils/request.py Normal file
View File

@ -0,0 +1,65 @@
"""
This module provides some useful functions for working with
scrapy.http.Request objects
"""
import hashlib
from scrapy.utils.url import canonicalize_url
def request_fingerprint(request, include_headers=()):
"""
Return the request fingerprint.
The request fingerprint is a hash that uniquely identifies the resource the
request points to. For example, take the following two urls:
http://www.example.com/query?id=111&cat=222
http://www.example.com/query?cat=222&id=111
Even though those are two different URLs both point to the same resource
and are equivalent (ie. they should return the same response).
Another example are cookies used to store session ids. Suppose the
following page is only accesible to authenticated users:
http://www.example.com/members/offers.html
Lot of sites use a cookie to store the session id, which adds a random
component to the HTTP Request and thus should be ignored when calculating
the fingerprint.
For this reason, request headers are ignored by default when calculating
the fingeprint. If you want to include specific headers use the
include_headers argument, which is a list of Request headers to include.
"""
if include_headers:
include_headers = [h.lower() for h in sorted(include_headers)]
cachekey = 'fingerprint' + '_'.join(include_headers)
else:
cachekey = 'fingerprint'
try:
return request._cache[cachekey]
except KeyError:
fp = hashlib.sha1()
fp.update(request.method)
fp.update(canonicalize_url(request.url))
fp.update(request.body or '')
for hdr in include_headers:
if hdr in request.headers:
fp.update(hdr)
fp.update(request.headers.get(hdr, ''))
fphash = fp.hexdigest()
request._cache[cachekey] = fphash
return fphash
def request_info(request):
"""Return a short string with request info including method, url and
fingeprint. Mainly used for debugging
"""
fp = request_fingerprint(request)
return "<Request: %s %s (%s..)>" % (request.method, request.url, fp[:8])