diff --git a/panda/src/putil/bamCache.I b/panda/src/putil/bamCache.I index dbb6e37cfa..164b74ec5c 100644 --- a/panda/src/putil/bamCache.I +++ b/panda/src/putil/bamCache.I @@ -17,6 +17,19 @@ //////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +// Function: BamCache::set_active +// Access: Published +// Description: Changes the state of the active flag. "active" means +// that the cache should be consulted automatically on +// loads, "not active" means that objects should be +// loaded directly without consulting the cache. +//////////////////////////////////////////////////////////////////// +INLINE void BamCache:: +set_active(bool active) { + _active = active; +} + //////////////////////////////////////////////////////////////////// // Function: BamCache::get_active // Access: Published @@ -31,19 +44,6 @@ get_active() const { return _active; } -//////////////////////////////////////////////////////////////////// -// Function: BamCache::set_active -// Access: Published -// Description: Changes the state of the active flag. "active" means -// that the cache should be consulted automatically on -// loads, "not active" means that objects should be -// loaded directly without consulting the cache. -//////////////////////////////////////////////////////////////////// -INLINE void BamCache:: -set_active(bool active) { - _active = active; -} - //////////////////////////////////////////////////////////////////// // Function: BamCache::get_root // Access: Published @@ -55,6 +55,60 @@ get_root() const { return _root; } +//////////////////////////////////////////////////////////////////// +// Function: BamCache::set_flush_time +// Access: Published +// Description: Specifies the time in seconds between automatic +// flushes of the cache index. +//////////////////////////////////////////////////////////////////// +INLINE void BamCache:: +set_flush_time(int flush_time) { + _flush_time = flush_time; +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCache::get_flush_time +// Access: Published +// Description: Returns the time in seconds between automatic +// flushes of the cache index. +//////////////////////////////////////////////////////////////////// +INLINE int BamCache:: +get_flush_time() const { + return _flush_time; +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCache::set_cache_max_kbytes +// Access: Published +// Description: Specifies the maximum size, in kilobytes, which the +// cache is allowed to grow to. If a newly cached file +// would exceed this size, an older file is removed from +// the cache. +// +// Note that in the case of multiple different processes +// simultaneously operating on the same cache directory, +// the actual cache size may slightlyexceed this value +// from time to time due to latency in checking between +// the processes. +//////////////////////////////////////////////////////////////////// +INLINE void BamCache:: +set_cache_max_kbytes(int max_kbytes) { + _max_kbytes = max_kbytes; + check_cache_size(); +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCache::get_cache_max_kbytes +// Access: Published +// Description: Returns the maximum size, in kilobytes, which the +// cache is allowed to grow to. See +// set_cache_max_kbytes(). +//////////////////////////////////////////////////////////////////// +INLINE int BamCache:: +get_cache_max_kbytes() const { + return _max_kbytes; +} + //////////////////////////////////////////////////////////////////// // Function: BamCache::get_global_ptr // Access: Published, Static diff --git a/panda/src/putil/bamCache.cxx b/panda/src/putil/bamCache.cxx index cb12680422..89022226dd 100644 --- a/panda/src/putil/bamCache.cxx +++ b/panda/src/putil/bamCache.cxx @@ -25,6 +25,8 @@ #include "bam.h" #include "typeRegistry.h" #include "string_utils.h" +#include "configVariableInt.h" +#include "configVariableString.h" BamCache *BamCache::_global_ptr = NULL; @@ -39,6 +41,28 @@ BamCache() : _index(new BamCacheIndex), _index_stale_since(0) { + ConfigVariableFilename model_cache_dir + ("model-cache-dir", Filename(), + PRC_DESC("The full path to a directory, local to this computer, in which " + "model and texture files will be cached on load. If a directory " + "name is specified here, files may be loaded from the cache " + "instead of from their actual pathnames, which may save load time, " + "especially if you are loading egg files instead of bam files, " + "or if you are loading models from a shared network drive. " + "If this is the empty string, no cache will be used.")); + + ConfigVariableInt model_cache_flush + ("model-cache-flush", 30, + PRC_DESC("This is the amount of time, in seconds, between automatic " + "flushes of the model-cache index.")); + + ConfigVariableInt model_cache_max_kbytes + ("model-cache-max-kbytes", 1048576, + PRC_DESC("This is the maximum size of the model cache, in kilobytes.")); + + _flush_time = model_cache_flush; + _max_kbytes = model_cache_max_kbytes; + set_root(model_cache_dir); } //////////////////////////////////////////////////////////////////// @@ -85,6 +109,7 @@ set_root(const Filename &root) { _index = new BamCacheIndex; _index_stale_since = 0; read_index(); + check_cache_size(); } //////////////////////////////////////////////////////////////////// @@ -242,7 +267,7 @@ void BamCache:: consider_flush_index() { if (_index_stale_since != 0) { int elapsed = (int)time(NULL) - (int)_index_stale_since; - if (elapsed > model_cache_flush) { + if (elapsed > _flush_time) { flush_index(); } } @@ -292,6 +317,7 @@ flush_index() { _index_ref_contents = orig_index; read_index(); } + check_cache_size(); } //////////////////////////////////////////////////////////////////// @@ -380,6 +406,8 @@ merge_index(BamCacheIndex *new_index) { } BamCacheIndex *old_index = _index; + old_index->release_records(); + new_index->release_records(); _index = new BamCacheIndex; BamCacheIndex::Records::const_iterator ai = old_index->_records.begin(); @@ -394,7 +422,6 @@ merge_index(BamCacheIndex *new_index) { Filename cache_pathname(_root, record->get_cache_filename()); if (cache_pathname.exists()) { // The file exists; keep it. - _index->_cache_size += record->_record_size; _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); } ++ai; @@ -405,7 +432,6 @@ merge_index(BamCacheIndex *new_index) { Filename cache_pathname(_root, record->get_cache_filename()); if (cache_pathname.exists()) { // The file exists; keep it. - _index->_cache_size += record->_record_size; _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); } ++bi; @@ -417,7 +443,6 @@ merge_index(BamCacheIndex *new_index) { if (*a_record == *b_record) { // They're the same entry. It doesn't really matter which one // we keep. - _index->_cache_size += a_record->_record_size; _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(a_record->get_source_pathname(), a_record)); } else { @@ -429,7 +454,6 @@ merge_index(BamCacheIndex *new_index) { if (cache_pathname.exists()) { PT(BamCacheRecord) record = do_read_record(cache_pathname, false); if (record != (BamCacheRecord *)NULL) { - _index->_cache_size += record->_record_size; _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); } } @@ -447,7 +471,6 @@ merge_index(BamCacheIndex *new_index) { Filename cache_pathname(_root, record->get_cache_filename()); if (cache_pathname.exists()) { // The file exists; keep it. - _index->_cache_size += record->_record_size; _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); } ++ai; @@ -459,11 +482,12 @@ merge_index(BamCacheIndex *new_index) { Filename cache_pathname(_root, record->get_cache_filename()); if (cache_pathname.exists()) { // The file exists; keep it. - _index->_cache_size += record->_record_size; _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); } ++bi; } + + _index->process_new_records(); } //////////////////////////////////////////////////////////////////// @@ -505,13 +529,14 @@ rebuild_index() { util_cat.info() << "Multiple cache files defining " << record->get_source_pathname() << "\n"; pathname.unlink(); - } else { - _index->_cache_size += record->_record_size; } } } } + _index->process_new_records(); + _index_stale_since = time(NULL); + check_cache_size(); flush_index(); } @@ -519,27 +544,16 @@ rebuild_index() { // Function: BamCache::add_to_index // Access: Private // Description: Updates the index entry for the indicated record. +// Note that a copy of the record is made first. //////////////////////////////////////////////////////////////////// void BamCache:: add_to_index(const BamCacheRecord *record) { PT(BamCacheRecord) new_record = record->make_copy(); - pair result = - _index->_records.insert(BamCacheIndex::Records::value_type(new_record->get_source_pathname(), new_record)); - if (!result.second) { - // We already had a record for this filename; it gets replaced. - PT(BamCacheRecord) orig_record = (*result.first).second; - if (*orig_record == *record) { - // Well, never mind. The record hasn't changed. - return; - } - - _index->_cache_size -= orig_record->_record_size; - (*result.first).second = new_record; + if (_index->add_record(new_record)) { + mark_index_stale(); + check_cache_size(); } - - _index->_cache_size += new_record->_record_size; - mark_index_stale(); } //////////////////////////////////////////////////////////////////// @@ -550,16 +564,33 @@ add_to_index(const BamCacheRecord *record) { //////////////////////////////////////////////////////////////////// void BamCache:: remove_from_index(const Filename &source_pathname) { - BamCacheIndex::Records::iterator ri = _index->_records.find(source_pathname); - if (ri == _index->_records.end()) { - // No entry for this record; no problem. + if (_index->remove_record(source_pathname)) { + mark_index_stale(); + } +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCache::check_cache_size +// Access: Private +// Description: If the cache size has exceeded its specified size +// limit, removes an old file. +//////////////////////////////////////////////////////////////////// +void BamCache:: +check_cache_size() { + if (_index->_cache_size == 0) { + // 0 means no limit. return; } - PT(BamCacheRecord) record = (*ri).second; - _index->_cache_size -= record->_record_size; - _index->_records.erase(ri); - mark_index_stale(); + if (_index->_cache_size > _max_kbytes * 1024) { + while (_index->_cache_size > _max_kbytes * 1024) { + PT(BamCacheRecord) record = _index->evict_old_file(); + nassertv(record != (BamCacheRecord *)NULL); + Filename cache_pathname(_root, record->get_cache_filename()); + cache_pathname.unlink(); + } + mark_index_stale(); + } } //////////////////////////////////////////////////////////////////// @@ -894,20 +925,7 @@ void BamCache:: make_global() { _global_ptr = new BamCache; - ConfigVariableFilename model_cache_dir - ("model-cache-dir", Filename(), - PRC_DESC("The full path to a directory, local to this computer, in which " - "model and texture files will be cached on load. If a directory " - "name is specified here, files may be loaded from the cache " - "instead of from their actual pathnames, which may save load time, " - "especially if you are loading egg files instead of bam files, " - "or if you are loading models from a shared network drive. " - "If this is the empty string, no cache will be used.")); - if (model_cache_dir.empty()) { + if (_global_ptr->get_root().empty()) { _global_ptr->set_active(false); - - } else { - _global_ptr->set_active(true); - _global_ptr->set_root(model_cache_dir); } } diff --git a/panda/src/putil/bamCache.h b/panda/src/putil/bamCache.h index 7c06d869e3..5a3190fbef 100644 --- a/panda/src/putil/bamCache.h +++ b/panda/src/putil/bamCache.h @@ -49,11 +49,17 @@ PUBLISHED: BamCache(); ~BamCache(); - INLINE bool get_active() const; INLINE void set_active(bool flag); + INLINE bool get_active() const; - INLINE const Filename &get_root() const; void set_root(const Filename &root); + INLINE const Filename &get_root() const; + + INLINE void set_flush_time(int flush_time); + INLINE int get_flush_time() const; + + INLINE void set_cache_max_kbytes(int max_kbytes); + INLINE int get_cache_max_kbytes() const; PT(BamCacheRecord) lookup(const Filename &source_filename, const string &cache_extension); @@ -75,6 +81,8 @@ private: void add_to_index(const BamCacheRecord *record); void remove_from_index(const Filename &source_filename); + void check_cache_size(); + static BamCacheIndex *do_read_index(Filename &index_pathname); static bool do_write_index(Filename &index_pathname, const BamCacheIndex *index); @@ -90,6 +98,8 @@ private: bool _active; Filename _root; + int _flush_time; + int _max_kbytes; static BamCache *_global_ptr; BamCacheIndex *_index; diff --git a/panda/src/putil/bamCacheIndex.I b/panda/src/putil/bamCacheIndex.I index 5d63f78112..854de2cf6b 100644 --- a/panda/src/putil/bamCacheIndex.I +++ b/panda/src/putil/bamCacheIndex.I @@ -23,7 +23,8 @@ // Description: //////////////////////////////////////////////////////////////////// INLINE BamCacheIndex:: -BamCacheIndex() : +BamCacheIndex() : + LinkedListNode(true), _cache_size(0) { } diff --git a/panda/src/putil/bamCacheIndex.cxx b/panda/src/putil/bamCacheIndex.cxx index c7fac2597b..6ba2e2c891 100644 --- a/panda/src/putil/bamCacheIndex.cxx +++ b/panda/src/putil/bamCacheIndex.cxx @@ -18,9 +18,25 @@ #include "bamCacheIndex.h" #include "indent.h" +#include TypeHandle BamCacheIndex::_type_handle; + +//////////////////////////////////////////////////////////////////// +// Function: BamCacheIndex::Destructor +// Access: Published +// Description: +//////////////////////////////////////////////////////////////////// +BamCacheIndex:: +~BamCacheIndex() { +#ifndef NDEBUG + // We need to "empty" the linked list to make the LinkedListNode + // destructors happy. + release_records(); +#endif +} + //////////////////////////////////////////////////////////////////// // Function: BamCacheIndex::write // Access: Public @@ -33,7 +49,7 @@ write(ostream &out, int indent_level) const { Records::const_iterator ri; for (ri = _records.begin(); ri != _records.end(); ++ri) { - PT(BamCacheRecord) record = (*ri).second; + BamCacheRecord *record = (*ri).second; indent(out, indent_level + 2) << setw(10) << record->_record_size << " " << record->get_cache_filename() << " " @@ -44,6 +60,133 @@ write(ostream &out, int indent_level) const { << setw(12) << _cache_size << " bytes total\n"; } +//////////////////////////////////////////////////////////////////// +// Function: BamCacheIndex::process_new_records +// Access: Private +// Description: Should be called after the _records index has been +// filled externally, this will sort the records by +// access time and calculate _cache_size. +//////////////////////////////////////////////////////////////////// +void BamCacheIndex:: +process_new_records() { + nassertv(_cache_size == 0); + + // Fill up a vector so we can sort the records into order by access + // time. + RecordVector rv; + rv.reserve(_records.size()); + + Records::const_iterator ri; + for (ri = _records.begin(); ri != _records.end(); ++ri) { + BamCacheRecord *record = (*ri).second; + _cache_size += record->_record_size; + rv.push_back(record); + } + + sort(rv.begin(), rv.end(), BamCacheRecord::SortByAccessTime()); + + // Now put them into the linked list. + RecordVector::const_iterator rvi; + for (rvi = rv.begin(); rvi != rv.end(); ++rvi) { + BamCacheRecord *record = *rvi; + record->insert_before(this); + } +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCacheIndex::release_records +// Access: Private +// Description: This is the inverse of process_new_records: it +// releases the records from the linked list, so that +// they may be added to another index or whatever. +// Calling this, of course, invalidates the index until +// process_new_records() is called again. +//////////////////////////////////////////////////////////////////// +void BamCacheIndex:: +release_records() { + Records::const_iterator ri; + for (ri = _records.begin(); ri != _records.end(); ++ri) { + BamCacheRecord *record = (*ri).second; + record->_next = NULL; + record->_prev = NULL; + } + _next = this; + _prev = this; + _cache_size = 0; +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCacheIndex::evict_old_file +// Access: Private +// Description: Evicts an old file from the cache. Records the record. +//////////////////////////////////////////////////////////////////// +PT(BamCacheRecord) BamCacheIndex:: +evict_old_file() { + // The first record in the linked list is the least-recently-used + // one. + nassertr(_next != this, NULL); + PT(BamCacheRecord) record = (BamCacheRecord *)_next; + bool removed = remove_record(record->get_source_pathname()); + nassertr(removed, NULL); + + return record; +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCacheIndex::add_record +// Access: Private +// Description: Adds a newly-created BamCacheRecord into the index. +// If a matching record is already in the index, it is +// replaced with the new record. Returns true if the +// record was added, or false if the equivalent record +// was already there and the index is unchanged. +//////////////////////////////////////////////////////////////////// +bool BamCacheIndex:: +add_record(BamCacheRecord *record) { + pair result = + _records.insert(Records::value_type(record->get_source_pathname(), record)); + if (!result.second) { + // We already had a record for this filename; it gets replaced. + BamCacheRecord *orig_record = (*result.first).second; + orig_record->remove_from_list(); + if (*orig_record == *record) { + // Well, never mind. The record hasn't changed. + orig_record->insert_before(this); + return false; + } + + _cache_size -= orig_record->_record_size; + (*result.first).second = record; + } + record->insert_before(this); + + _cache_size += record->_record_size; + return true; +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCacheIndex::remove_record +// Access: Private +// Description: Searches for the matching record in the index and +// removes it if it is found. Returns true if the +// record was found and removed, or false if there was +// no such record and the index is unchanged. +//////////////////////////////////////////////////////////////////// +bool BamCacheIndex:: +remove_record(const Filename &source_pathname) { + Records::iterator ri = _records.find(source_pathname); + if (ri == _records.end()) { + // No entry for this record; no problem. + return false; + } + + BamCacheRecord *record = (*ri).second; + record->remove_from_list(); + _cache_size -= record->_record_size; + _records.erase(ri); + return true; +} + //////////////////////////////////////////////////////////////////// // Function: BamCacheIndex::register_with_read_factory // Access: Public, Static @@ -113,13 +256,13 @@ complete_pointers(TypedWritable **p_list, BamReader *manager) { util_cat.info() << "Multiple cache files defining " << record->get_source_pathname() << " in index.\n"; - } else { - _cache_size += record->_record_size; } } _record_vector.clear(); + process_new_records(); + return pi; } diff --git a/panda/src/putil/bamCacheIndex.h b/panda/src/putil/bamCacheIndex.h index 29d0cf2f18..41b3f92906 100644 --- a/panda/src/putil/bamCacheIndex.h +++ b/panda/src/putil/bamCacheIndex.h @@ -24,6 +24,7 @@ #include "pointerTo.h" #include "filename.h" #include "typedWritable.h" +#include "linkedListNode.h" #include "pmap.h" #include "pvector.h" @@ -37,13 +38,22 @@ // For the most part, this class is used only by the // BamCache class. //////////////////////////////////////////////////////////////////// -class EXPCL_PANDA BamCacheIndex : public TypedWritable { +class EXPCL_PANDA BamCacheIndex : public TypedWritable, public LinkedListNode { private: INLINE BamCacheIndex(); + ~BamCacheIndex(); public: void write(ostream &out, int indent_level = 0) const; +private: + void process_new_records(); + void release_records(); + PT(BamCacheRecord) evict_old_file(); + + bool add_record(BamCacheRecord *record); + bool remove_record(const Filename &source_pathname); + private: typedef pmap Records; diff --git a/panda/src/putil/bamCacheRecord.I b/panda/src/putil/bamCacheRecord.I index 3d5d5a0f35..ce73e928fc 100644 --- a/panda/src/putil/bamCacheRecord.I +++ b/panda/src/putil/bamCacheRecord.I @@ -180,3 +180,13 @@ set_data(TypedWritable *data, bool owns_pointer) { _data = data; _owns_pointer = owns_pointer; } + +//////////////////////////////////////////////////////////////////// +// Function: BamCacheRecord::SortByAccessTime::operator () +// Access: Public +// Description: Returns true if a sorts before b, false otherwise. +//////////////////////////////////////////////////////////////////// +INLINE bool BamCacheRecord::SortByAccessTime:: +operator () (const BamCacheRecord *a, const BamCacheRecord *b) const { + return (a->_record_access_time < b->_record_access_time); +} diff --git a/panda/src/putil/bamCacheRecord.h b/panda/src/putil/bamCacheRecord.h index cdbdcfc07a..b8c53eb8ec 100644 --- a/panda/src/putil/bamCacheRecord.h +++ b/panda/src/putil/bamCacheRecord.h @@ -21,6 +21,7 @@ #include "pandabase.h" #include "typedWritableReferenceCount.h" +#include "linkedListNode.h" class BamWriter; class BamReader; @@ -36,7 +37,8 @@ class FactoryParams; // contains information needed to test the validity of // the cache. //////////////////////////////////////////////////////////////////// -class EXPCL_PANDA BamCacheRecord : public TypedWritableReferenceCount { +class EXPCL_PANDA BamCacheRecord : public TypedWritableReferenceCount, + public LinkedListNode { private: BamCacheRecord(); BamCacheRecord(const Filename &source_pathname, @@ -71,6 +73,12 @@ PUBLISHED: void write(ostream &out, int indent_level = 0) const; private: + // This class is used to sort BamCacheRecords by access time. + class SortByAccessTime { + public: + INLINE bool operator () (const BamCacheRecord *a, const BamCacheRecord *b) const; + }; + static string format_timestamp(time_t timestamp); Filename _source_pathname; @@ -126,6 +134,7 @@ private: friend class BamCache; friend class BamCacheIndex; + friend class BamCacheRecord::SortByAccessTime; }; INLINE ostream &operator << (ostream &out, const BamCacheRecord &record) { diff --git a/panda/src/putil/config_util.cxx b/panda/src/putil/config_util.cxx index 5d82b75120..0f949e43f3 100644 --- a/panda/src/putil/config_util.cxx +++ b/panda/src/putil/config_util.cxx @@ -158,9 +158,3 @@ ConfigVariableDouble sleep_precision ConfigVariableDouble average_frame_rate_interval ("average-frame-rate-interval", 1.0); - -ConfigVariableInt model_cache_flush -("model-cache-flush", 30, - PRC_DESC("This is the amount of time, in seconds, between automatic " - "flushes of the model-cache index, if model-cache-dir is " - "in effect.")); diff --git a/panda/src/putil/config_util.h b/panda/src/putil/config_util.h index e52e8676ee..68242eeda1 100644 --- a/panda/src/putil/config_util.h +++ b/panda/src/putil/config_util.h @@ -24,7 +24,6 @@ #include "configVariableSearchPath.h" #include "configVariableEnum.h" #include "configVariableDouble.h" -#include "configVariableInt.h" #include "bamEndian.h" #include "bamTextureMode.h" #include "dconfig.h" @@ -62,6 +61,4 @@ extern ConfigVariableDouble max_dt; extern ConfigVariableDouble sleep_precision; extern ConfigVariableDouble average_frame_rate_interval; -extern ConfigVariableInt model_cache_flush; - #endif /* __CONFIG_UTIL_H__ */