From 90d408b04fa1471cee0b9ea03a5423cd3011fdc2 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sun, 24 May 2009 19:13:06 -0300 Subject: [PATCH] Some changes to HTTP cache middleware: * documented * moved from scrapy.contrib.downloadermiddleware.cache.CacheMiddleware to scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware * settings prefix changed from CACHE2_ to HTTPCACHE_ --HG-- rename : scrapy/contrib/downloadermiddleware/cache.py => scrapy/contrib/downloadermiddleware/httpcache.py --- docs/ref/downloader-middleware.rst | 29 +++ docs/ref/settings.rst | 36 ++-- docs/topics/extensions.rst | 6 +- .../googledir/googledir/settings.py | 6 +- scrapy/command/commands/crawl.py | 4 - scrapy/conf/default_settings.py | 12 +- scrapy/contrib/downloadermiddleware/cache.py | 195 +----------------- .../contrib/downloadermiddleware/httpcache.py | 190 +++++++++++++++++ 8 files changed, 258 insertions(+), 220 deletions(-) create mode 100644 scrapy/contrib/downloadermiddleware/httpcache.py diff --git a/docs/ref/downloader-middleware.rst b/docs/ref/downloader-middleware.rst index 8665e78be..ed9d017f7 100644 --- a/docs/ref/downloader-middleware.rst +++ b/docs/ref/downloader-middleware.rst @@ -40,3 +40,32 @@ thus it's recommended to leave it always enabled. Those tasks are: .. _default Form content type: http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4.1 +HttpCacheMiddleware +------------------- + +.. module:: scrapy.contrib.downloadermiddleware.httpcache + :synopsis: HTTP Cache downloader middleware + +.. class:: HttpCacheMiddleware + + This middleware provides low-level cache to all HTTP requests and responses. + Every request and its corresponding response are cached and then, when that + same request is seen again, the response is returned without transferring + anything from the Internet. + + The HTTP cache is useful for testing spiders faster (without having to wait for + downloads every time) and for trying your spider off-line when you don't have + an Internet connection. + + The :class:`HttpCacheMiddleware` can be configured through the following + settings (see the settings documentation for more info): + + * :setting:`HTTPCACHE_DIR` - this one actually enables the cache besides + settings the cache dir + * :setting:`HTTPCACHE_IGNORE_MISSING` - ignoring missing requests instead + of downloading them + * :setting:`HTTPCACHE_SECTORIZE` - split HTTP cache in several directories + (for performance reasons) + * :setting:`HTTPCACHE_EXPIRATION_SECS` - how many secs until the cache is + considered out of date + diff --git a/docs/ref/settings.rst b/docs/ref/settings.rst index 107e3a4b8..31085b3bd 100644 --- a/docs/ref/settings.rst +++ b/docs/ref/settings.rst @@ -42,44 +42,44 @@ Default: ``1.0`` The version of the bot implemented by this Scrapy project. This will be used to construct the User-Agent by default. -.. setting:: CACHE2_DIR +.. setting:: HTTPCACHE_DIR -CACHE2_DIR ----------- +HTTPCACHE_DIR +------------- Default: ``''`` (empty string) -The directory to use for storing the low-level HTTP cache. If empty the HTTP +The directory to use for storing the (low-level) HTTP cache. If empty the HTTP cache will be disabled. -.. setting:: CACHE2_EXPIRATION_SECS +.. setting:: HTTPCACHE_EXPIRATION_SECS -CACHE2_EXPIRATION_SECS ----------------------- +HTTPCACHE_EXPIRATION_SECS +------------------------- Default: ``0`` -Number of seconds to use for cache expiration. Requests that were cached before -this time will be re-downloaded. If zero, cached requests will always expire. -Negative numbers means requests will never expire. +Number of seconds to use for HTTP cache expiration. Requests that were cached +before this time will be re-downloaded. If zero, cached requests will always +expire. Negative numbers means requests will never expire. -.. setting:: CACHE2_IGNORE_MISSING +.. setting:: HTTPCACHE_IGNORE_MISSING -CACHE2_IGNORE_MISSING ---------------------- +HTTPCACHE_IGNORE_MISSING +------------------------ Default: ``False`` If enabled, requests not found in the cache will be ignored instead of downloaded. -.. setting:: CACHE2_SECTORIZE +.. setting:: HTTPCACHE_SECTORIZE -CACHE2_SECTORIZE ----------------- +HTTPCACHE_SECTORIZE +------------------- Default: ``True`` -Whether to split HTTP cache storage in several dirs for performance improvements. +Whether to split HTTP cache storage in several dirs for performance. .. setting:: CLOSEDOMAIN_NOTIFY @@ -320,7 +320,7 @@ Default:: 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800, 'scrapy.contrib.downloadermiddleware.debug.CrawlDebug': 840, 'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850, - 'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware': 900, + 'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900, } A dict containing the downloader middlewares enabled by default in Scrapy. You diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 1a05075d6..f7b189944 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -48,9 +48,9 @@ Available, enabled and disabled extensions ========================================== Not all available extensions will be enabled. Some of them usually depend on a -particular setting. For example, the Cache extension is typically available but -disabled by default unless you the :setting:`CACHE2_DIR` setting is set. Both -enabled and disabled extension can be accessed through the +particular setting. For example, the HTTP Cache extension is available by default +but disabled unless the :setting:`HTTPCACHE_DIR` setting is set. Both enabled +and disabled extension can be accessed through the :ref:`ref-extension-manager`. Accessing enabled extensions diff --git a/examples/experimental/googledir/googledir/settings.py b/examples/experimental/googledir/googledir/settings.py index a5c324741..0828373b8 100644 --- a/examples/experimental/googledir/googledir/settings.py +++ b/examples/experimental/googledir/googledir/settings.py @@ -39,9 +39,9 @@ MYSQL_CONNECTION_PING_PERIOD = 600 SCHEDULER = 'scrapy.core.scheduler.Scheduler' SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO -#CACHE2_DIR = '/tmp/cache2' # if set, enables HTTP cache -#CACHE2_IGNORE_MISSING = 0 # ignore requests not in cache -#CACHE2_SECTORIZE = 1 # sectorize domains to distribute storage among servers +#HTTPCACHE_DIR = '/tmp/cache2' # if set, enables HTTP cache +#HTTPCACHE_IGNORE_MISSING = 0 # ignore requests not in cache +#HTTPCACHE_SECTORIZE = 1 # sectorize domains to distribute storage among servers #STATS_ENABLED = 1 # enable stats #STATS_CLEANUP = 0 # cleanup domain stats when a domain is closed (saves memory) diff --git a/scrapy/command/commands/crawl.py b/scrapy/command/commands/crawl.py index 276aa8858..0b2017681 100644 --- a/scrapy/command/commands/crawl.py +++ b/scrapy/command/commands/crawl.py @@ -16,7 +16,6 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--nocache", dest="nocache", action="store_true", help="disable HTTP cache") parser.add_option("--nopipeline", dest="nopipeline", action="store_true", help="disable scraped item pipeline") parser.add_option("--restrict", dest="restrict", action="store_true", help="restrict crawling only to the given urls") parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", help="don't follow links (for use with URLs only)") @@ -27,9 +26,6 @@ class Command(ScrapyCommand): if opts.nopipeline: settings.overrides['ITEM_PIPELINES'] = [] - if opts.nocache: - settings.overrides['CACHE2_DIR'] = None - if opts.restrict: settings.overrides['RESTRICT_TO_URLS'] = args diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 898d975cb..b7e742cdc 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -20,11 +20,6 @@ ADAPTORS_DEBUG = False BOT_NAME = 'scrapybot' BOT_VERSION = '1.0' -CACHE2_DIR = '' -CACHE2_IGNORE_MISSING = False -CACHE2_SECTORIZE = True -CACHE2_EXPIRATION_SECS = 0 - CLOSEDOMAIN_TIMEOUT = 0 CLOSEDOMAIN_NOTIFY = [] @@ -74,7 +69,7 @@ DOWNLOADER_MIDDLEWARES_BASE = { 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800, 'scrapy.contrib.downloadermiddleware.debug.CrawlDebug': 840, 'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850, - 'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware': 900, + 'scrapy.contrib.downloadermiddleware.cache.HttpCacheMiddleware': 900, # Downloader side } @@ -108,6 +103,11 @@ EXTENSIONS = [ GROUPSETTINGS_ENABLED = False GROUPSETTINGS_MODULE = '' +HTTPCACHE_DIR = '' +HTTPCACHE_IGNORE_MISSING = False +HTTPCACHE_SECTORIZE = True +HTTPCACHE_EXPIRATION_SECS = 0 + # Item pipelines are typically set in specific commands settings ITEM_PIPELINES = [] diff --git a/scrapy/contrib/downloadermiddleware/cache.py b/scrapy/contrib/downloadermiddleware/cache.py index d63c81d32..f848afcd1 100644 --- a/scrapy/contrib/downloadermiddleware/cache.py +++ b/scrapy/contrib/downloadermiddleware/cache.py @@ -1,191 +1,14 @@ -from __future__ import with_statement +# FIXME: code below is for backwards compatibility and should be removed before +# the 0.7 release -import errno -import os -import hashlib -import datetime -import cPickle as pickle -from pydispatch import dispatcher +import warnings -from scrapy.core import signals -from scrapy import log -from scrapy.http import Response, Headers -from scrapy.core.exceptions import NotConfigured, HttpException, IgnoreRequest -from scrapy.core.downloader.responsetypes import responsetypes -from scrapy.utils.request import request_fingerprint -from scrapy.utils.http import headers_dict_to_raw, headers_raw_to_dict -from scrapy.conf import settings +from scrapy.contrib.downloadermiddleware.httpcache import HttpCacheMiddleware -class CacheMiddleware(object): - def __init__(self): - if not settings['CACHE2_DIR']: - raise NotConfigured - self.cache = Cache(settings['CACHE2_DIR'], sectorize=settings.getbool('CACHE2_SECTORIZE')) - self.ignore_missing = settings.getbool('CACHE2_IGNORE_MISSING') - dispatcher.connect(self.open_domain, signal=signals.domain_open) +class CacheMiddleware(HttpCacheMiddleware): - def open_domain(self, domain): - self.cache.open_domain(domain) - - def process_request(self, request, spider): - if not is_cacheable(request): - return - - key = request_fingerprint(request) - domain = spider.domain_name - try: - response = self.cache.retrieve_response(domain, key) - except: - log.msg("Corrupt cache for %s" % request.url, log.WARNING) - response = False - if response: - if not 200 <= int(response.status) < 300: - raise HttpException(response.status, None, response) - return response - elif self.ignore_missing: - raise IgnoreRequest("Ignored request not in cache: %s" % request) - - def process_response(self, request, response, spider): - if not is_cacheable(request): - return response - - if isinstance(response, Response) and not response.meta.get('cached'): - key = request_fingerprint(request) - domain = spider.domain_name - self.cache.store(domain, key, request, response) - - return response - - def process_exception(self, request, exception, spider): - if not is_cacheable(request): - return - - if isinstance(exception, HttpException) and isinstance(exception.response, Response): - key = request_fingerprint(request) - domain = spider.domain_name - self.cache.store(domain, key, request, exception.response) - -def is_cacheable(request): - return request.url.scheme in ['http', 'https'] - - -class Cache(object): - DOMAIN_SECTORDIR = 'data' - DOMAIN_LINKDIR = 'domains' - - def __init__(self, cachedir, sectorize=False): - self.cachedir = cachedir - self.sectorize = sectorize - - self.baselinkpath = os.path.join(self.cachedir, self.DOMAIN_LINKDIR) - if not os.path.exists(self.baselinkpath): - os.makedirs(self.baselinkpath) - - self.basesectorpath = os.path.join(self.cachedir, self.DOMAIN_SECTORDIR) - if not os.path.exists(self.basesectorpath): - os.makedirs(self.basesectorpath) - - def domainsectorpath(self, domain): - sector = hashlib.sha1(domain).hexdigest()[0] - return os.path.join(self.basesectorpath, sector, domain) - - def domainlinkpath(self, domain): - return os.path.join(self.baselinkpath, domain) - - def requestpath(self, domain, key): - linkpath = self.domainlinkpath(domain) - return os.path.join(linkpath, key[0:2], key) - - def open_domain(self, domain): - if domain: - linkpath = self.domainlinkpath(domain) - if self.sectorize: - sectorpath = self.domainsectorpath(domain) - if not os.path.exists(sectorpath): - os.makedirs(sectorpath) - if not os.path.exists(linkpath): - try: - os.symlink(sectorpath, linkpath) - except: - os.makedirs(linkpath) # windows filesystem - else: - if not os.path.exists(linkpath): - os.makedirs(linkpath) - - def read_meta(self, domain, key): - """Return the metadata dictionary (possibly empty) if the entry is - cached, None otherwise. - """ - requestpath = self.requestpath(domain, key) - try: - with open(os.path.join(requestpath, 'pickled_meta'), 'r') as f: - metadata = pickle.load(f) - except IOError, e: - if e.errno != errno.ENOENT: - raise - return None - expiration_secs = settings.getint('CACHE2_EXPIRATION_SECS') - if expiration_secs >= 0: - expiration_date = metadata['timestamp'] + datetime.timedelta(seconds=expiration_secs) - if datetime.datetime.utcnow() > expiration_date: - log.msg('dropping old cached response from %s' % metadata['timestamp'], level=log.DEBUG) - return None - return metadata - - def retrieve_response(self, domain, key): - """ - Return response dictionary if request has correspondent cache record; - return None if not. - """ - metadata = self.read_meta(domain, key) - if metadata is None: - return None # not cached - - requestpath = self.requestpath(domain, key) - responsebody = responseheaders = None - with open(os.path.join(requestpath, 'response_body')) as f: - responsebody = f.read() - with open(os.path.join(requestpath, 'response_headers')) as f: - responseheaders = f.read() - - url = metadata['url'] - headers = Headers(headers_raw_to_dict(responseheaders)) - status = metadata['status'] - - respcls = responsetypes.from_args(headers=headers, url=url) - response = respcls(url=url, headers=headers, status=status, body=responsebody) - response.meta['cached'] = True - response.flags.append('cached') - return response - - def store(self, domain, key, request, response): - requestpath = self.requestpath(domain, key) - if not os.path.exists(requestpath): - os.makedirs(requestpath) - - metadata = { - 'url':request.url, - 'method': request.method, - 'status': response.status, - 'domain': domain, - 'timestamp': datetime.datetime.utcnow(), - } - - # metadata - with open(os.path.join(requestpath, 'meta_data'), 'w') as f: - f.write(repr(metadata)) - # pickled metadata (to recover without using eval) - with open(os.path.join(requestpath, 'pickled_meta'), 'w') as f: - pickle.dump(metadata, f) - # response - with open(os.path.join(requestpath, 'response_headers'), 'w') as f: - f.write(headers_dict_to_raw(response.headers)) - with open(os.path.join(requestpath, 'response_body'), 'w') as f: - f.write(response.body) - # request - with open(os.path.join(requestpath, 'request_headers'), 'w') as f: - f.write(headers_dict_to_raw(request.headers)) - if request.body: - with open(os.path.join(requestpath, 'request_body'), 'w') as f: - f.write(request.body) + def __init__(self, *args, **kwargs): + warnings.warn("scrapy.contrib.downloadermiddleware.cache.CacheMiddleware was moved to scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware", + DeprecationWarning, stacklevel=2) + HttpCacheMiddleware.__init__(self, *args, **kwargs) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py new file mode 100644 index 000000000..9c45bce8a --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -0,0 +1,190 @@ +from __future__ import with_statement + +import errno +import os +import hashlib +import datetime +import cPickle as pickle +from pydispatch import dispatcher + +from scrapy.core import signals +from scrapy import log +from scrapy.http import Response, Headers +from scrapy.core.exceptions import NotConfigured, HttpException, IgnoreRequest +from scrapy.core.downloader.responsetypes import responsetypes +from scrapy.utils.request import request_fingerprint +from scrapy.utils.http import headers_dict_to_raw, headers_raw_to_dict +from scrapy.conf import settings + +class HttpCacheMiddleware(object): + def __init__(self): + if not settings['HTTPCACHE_DIR']: + raise NotConfigured + self.cache = Cache(settings['HTTPCACHE_DIR'], sectorize=settings.getbool('HTTPCACHE_SECTORIZE')) + self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') + dispatcher.connect(self.open_domain, signal=signals.domain_open) + + def open_domain(self, domain): + self.cache.open_domain(domain) + + def process_request(self, request, spider): + if not is_cacheable(request): + return + + key = request_fingerprint(request) + domain = spider.domain_name + try: + response = self.cache.retrieve_response(domain, key) + except: + log.msg("Corrupt cache for %s" % request.url, log.WARNING) + response = False + if response: + if not 200 <= int(response.status) < 300: + raise HttpException(response.status, None, response) + return response + elif self.ignore_missing: + raise IgnoreRequest("Ignored request not in cache: %s" % request) + + def process_response(self, request, response, spider): + if not is_cacheable(request): + return response + + if isinstance(response, Response) and not response.meta.get('cached'): + key = request_fingerprint(request) + domain = spider.domain_name + self.cache.store(domain, key, request, response) + + return response + + def process_exception(self, request, exception, spider): + if not is_cacheable(request): + return + + if isinstance(exception, HttpException) and isinstance(exception.response, Response): + key = request_fingerprint(request) + domain = spider.domain_name + self.cache.store(domain, key, request, exception.response) + +def is_cacheable(request): + return request.url.scheme in ['http', 'https'] + + +class Cache(object): + DOMAIN_SECTORDIR = 'data' + DOMAIN_LINKDIR = 'domains' + + def __init__(self, cachedir, sectorize=False): + self.cachedir = cachedir + self.sectorize = sectorize + + self.baselinkpath = os.path.join(self.cachedir, self.DOMAIN_LINKDIR) + if not os.path.exists(self.baselinkpath): + os.makedirs(self.baselinkpath) + + self.basesectorpath = os.path.join(self.cachedir, self.DOMAIN_SECTORDIR) + if not os.path.exists(self.basesectorpath): + os.makedirs(self.basesectorpath) + + def domainsectorpath(self, domain): + sector = hashlib.sha1(domain).hexdigest()[0] + return os.path.join(self.basesectorpath, sector, domain) + + def domainlinkpath(self, domain): + return os.path.join(self.baselinkpath, domain) + + def requestpath(self, domain, key): + linkpath = self.domainlinkpath(domain) + return os.path.join(linkpath, key[0:2], key) + + def open_domain(self, domain): + if domain: + linkpath = self.domainlinkpath(domain) + if self.sectorize: + sectorpath = self.domainsectorpath(domain) + if not os.path.exists(sectorpath): + os.makedirs(sectorpath) + if not os.path.exists(linkpath): + try: + os.symlink(sectorpath, linkpath) + except: + os.makedirs(linkpath) # windows filesystem + else: + if not os.path.exists(linkpath): + os.makedirs(linkpath) + + def read_meta(self, domain, key): + """Return the metadata dictionary (possibly empty) if the entry is + cached, None otherwise. + """ + requestpath = self.requestpath(domain, key) + try: + with open(os.path.join(requestpath, 'pickled_meta'), 'r') as f: + metadata = pickle.load(f) + except IOError, e: + if e.errno != errno.ENOENT: + raise + return None + expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') + if expiration_secs >= 0: + expiration_date = metadata['timestamp'] + datetime.timedelta(seconds=expiration_secs) + if datetime.datetime.utcnow() > expiration_date: + log.msg('dropping old cached response from %s' % metadata['timestamp'], level=log.DEBUG) + return None + return metadata + + def retrieve_response(self, domain, key): + """ + Return response dictionary if request has correspondent cache record; + return None if not. + """ + metadata = self.read_meta(domain, key) + if metadata is None: + return None # not cached + + requestpath = self.requestpath(domain, key) + responsebody = responseheaders = None + with open(os.path.join(requestpath, 'response_body')) as f: + responsebody = f.read() + with open(os.path.join(requestpath, 'response_headers')) as f: + responseheaders = f.read() + + url = metadata['url'] + headers = Headers(headers_raw_to_dict(responseheaders)) + status = metadata['status'] + + respcls = responsetypes.from_args(headers=headers, url=url) + response = respcls(url=url, headers=headers, status=status, body=responsebody) + response.meta['cached'] = True + response.flags.append('cached') + return response + + def store(self, domain, key, request, response): + requestpath = self.requestpath(domain, key) + if not os.path.exists(requestpath): + os.makedirs(requestpath) + + metadata = { + 'url':request.url, + 'method': request.method, + 'status': response.status, + 'domain': domain, + 'timestamp': datetime.datetime.utcnow(), + } + + # metadata + with open(os.path.join(requestpath, 'meta_data'), 'w') as f: + f.write(repr(metadata)) + # pickled metadata (to recover without using eval) + with open(os.path.join(requestpath, 'pickled_meta'), 'w') as f: + pickle.dump(metadata, f) + # response + with open(os.path.join(requestpath, 'response_headers'), 'w') as f: + f.write(headers_dict_to_raw(response.headers)) + with open(os.path.join(requestpath, 'response_body'), 'w') as f: + f.write(response.body) + # request + with open(os.path.join(requestpath, 'request_headers'), 'w') as f: + f.write(headers_dict_to_raw(request.headers)) + if request.body: + with open(os.path.join(requestpath, 'request_body'), 'w') as f: + f.write(request.body)