From 7d43e513582d05e513ae8cfdbefbd81fa27c9359 Mon Sep 17 00:00:00 2001 From: David Rose Date: Tue, 20 Jun 2006 00:23:24 +0000 Subject: [PATCH] add BamCacheIndex --- panda/src/display/graphicsEngine.cxx | 14 + panda/src/egg/eggData.cxx | 19 +- panda/src/egg/eggData.h | 3 + panda/src/egg/eggGroupNode.cxx | 12 +- panda/src/egg/eggGroupNode.h | 4 +- panda/src/egg2pg/eggLoader.cxx | 11 + panda/src/egg2pg/eggLoader.h | 2 + panda/src/egg2pg/load_egg_file.cxx | 11 +- panda/src/egg2pg/load_egg_file.h | 5 +- panda/src/egg2pg/loaderFileTypeEgg.cxx | 5 +- panda/src/egg2pg/loaderFileTypeEgg.h | 3 +- panda/src/gobj/texture.cxx | 2 +- panda/src/pgraph/loader.cxx | 2 +- panda/src/pgraph/loaderFileType.cxx | 3 +- panda/src/pgraph/loaderFileType.h | 4 +- panda/src/pgraph/loaderFileTypeBam.cxx | 7 +- panda/src/pgraph/loaderFileTypeBam.h | 3 +- panda/src/putil/Sources.pp | 3 + panda/src/putil/bamCache.I | 13 + panda/src/putil/bamCache.cxx | 609 +++++++++++++++++++++++-- panda/src/putil/bamCache.h | 41 +- panda/src/putil/bamCacheIndex.I | 29 ++ panda/src/putil/bamCacheIndex.cxx | 143 ++++++ panda/src/putil/bamCacheIndex.h | 89 ++++ panda/src/putil/bamCacheRecord.I | 28 ++ panda/src/putil/bamCacheRecord.cxx | 33 +- panda/src/putil/bamCacheRecord.h | 12 + panda/src/putil/config_util.cxx | 9 + panda/src/putil/config_util.h | 4 + panda/src/putil/putil_composite1.cxx | 1 + panda/src/testbed/pview.cxx | 6 + 31 files changed, 1064 insertions(+), 66 deletions(-) create mode 100644 panda/src/putil/bamCacheIndex.I create mode 100644 panda/src/putil/bamCacheIndex.cxx create mode 100644 panda/src/putil/bamCacheIndex.h diff --git a/panda/src/display/graphicsEngine.cxx b/panda/src/display/graphicsEngine.cxx index f8bafd7b3f..e9e7529faf 100644 --- a/panda/src/display/graphicsEngine.cxx +++ b/panda/src/display/graphicsEngine.cxx @@ -40,6 +40,7 @@ #include "pipeline.h" #include "throw_event.h" #include "objectDeletor.h" +#include "bamCache.h" #if defined(WIN32) #define WINDOWS_LEAN_AND_MEAN @@ -458,6 +459,14 @@ remove_all_windows() { _app.do_close(this, current_thread); _app.do_pending(this, current_thread); terminate_threads(current_thread); + + // It seems a safe assumption that we're about to exit the + // application or otherwise shut down Panda. Although it's a bit of + // a hack, since it's not really related to removing windows, this + // would nevertheless be a fine time to ensure the model cache (if + // any) has been flushed to disk. + BamCache *cache = BamCache::get_global_ptr(); + cache->flush_index(); } //////////////////////////////////////////////////////////////////// @@ -527,6 +536,11 @@ void GraphicsEngine:: render_frame() { Thread *current_thread = Thread::get_current_thread(); + // Since this gets called every frame, we should take advantage of + // the opportunity to flush the cache if necessary. + BamCache *cache = BamCache::get_global_ptr(); + cache->consider_flush_index(); + // Anything that happens outside of GraphicsEngine::render_frame() // is deemed to be App. #ifdef DO_PSTATS diff --git a/panda/src/egg/eggData.cxx b/panda/src/egg/eggData.cxx index 1013bd5a2b..22b4615f48 100644 --- a/panda/src/egg/eggData.cxx +++ b/panda/src/egg/eggData.cxx @@ -170,7 +170,24 @@ merge(EggData &other) { bool EggData:: load_externals(const DSearchPath &searchpath) { return - r_load_externals(searchpath, get_coordinate_system()); + r_load_externals(searchpath, get_coordinate_system(), NULL); +} + +//////////////////////////////////////////////////////////////////// +// Function: EggData::load_externals +// Access: Public +// Description: Loads up all the egg files referenced by +// entries within the egg structure, and inserts their +// contents in place of the entries. Searches +// for files in the searchpath, if not found directly, +// and writes error messages to the indicated output +// stream. Returns true if all externals were loaded +// successfully, false otherwise. +//////////////////////////////////////////////////////////////////// +bool EggData:: +load_externals(const DSearchPath &searchpath, BamCacheRecord *record) { + return + r_load_externals(searchpath, get_coordinate_system(), record); } //////////////////////////////////////////////////////////////////// diff --git a/panda/src/egg/eggData.h b/panda/src/egg/eggData.h index 47f3a58261..5e080a7f06 100644 --- a/panda/src/egg/eggData.h +++ b/panda/src/egg/eggData.h @@ -27,6 +27,8 @@ #include "pnotify.h" #include "dSearchPath.h" +class BamCacheRecord; + //////////////////////////////////////////////////////////////////// // Class : EggData // Description : This is the primary interface into all the egg data, @@ -54,6 +56,7 @@ PUBLISHED: void merge(EggData &other); bool load_externals(const DSearchPath &searchpath = DSearchPath()); + bool load_externals(const DSearchPath &searchpath, BamCacheRecord *record); int collapse_equivalent_textures(); int collapse_equivalent_materials(); diff --git a/panda/src/egg/eggGroupNode.cxx b/panda/src/egg/eggGroupNode.cxx index c4ded208fc..84657c7bd2 100644 --- a/panda/src/egg/eggGroupNode.cxx +++ b/panda/src/egg/eggGroupNode.cxx @@ -36,6 +36,7 @@ #include "dSearchPath.h" #include "deg_2_rad.h" #include "dcast.h" +#include "bamCacheRecord.h" #include @@ -1545,7 +1546,8 @@ find_materials(EggMaterialCollection *collection) { // EggData::load_externals(). //////////////////////////////////////////////////////////////////// bool EggGroupNode:: -r_load_externals(const DSearchPath &searchpath, CoordinateSystem coordsys) { +r_load_externals(const DSearchPath &searchpath, CoordinateSystem coordsys, + BamCacheRecord *record) { bool success = true; Children::iterator ci; @@ -1576,8 +1578,12 @@ r_load_externals(const DSearchPath &searchpath, CoordinateSystem coordsys) { if (ext_data.read(filename)) { // The external file was read correctly. Add its contents // into the tree at this point. + if (record != (BamCacheRecord *)NULL) { + record->add_dependent_file(filename); + } + success = - ext_data.load_externals(searchpath) + ext_data.load_externals(searchpath, record) && success; new_node->steal_children(ext_data); } @@ -1586,7 +1592,7 @@ r_load_externals(const DSearchPath &searchpath, CoordinateSystem coordsys) { } else if (child->is_of_type(EggGroupNode::get_class_type())) { EggGroupNode *group_child = DCAST(EggGroupNode, child); success = - group_child->r_load_externals(searchpath, coordsys) + group_child->r_load_externals(searchpath, coordsys, record) && success; } } diff --git a/panda/src/egg/eggGroupNode.h b/panda/src/egg/eggGroupNode.h index 772e36087f..b46f917a8c 100644 --- a/panda/src/egg/eggGroupNode.h +++ b/panda/src/egg/eggGroupNode.h @@ -36,6 +36,7 @@ class EggPolygon; class EggVertex; class EggVertexPool; class DSearchPath; +class BamCacheRecord; //////////////////////////////////////////////////////////////////// // Class : EggGroupNode @@ -174,7 +175,8 @@ protected: int find_textures(EggTextureCollection *collection); int find_materials(EggMaterialCollection *collection); bool r_load_externals(const DSearchPath &searchpath, - CoordinateSystem coordsys); + CoordinateSystem coordsys, + BamCacheRecord *record); private: INLINE static bool is_right(const LVector2d &v1, const LVector2d &v2); diff --git a/panda/src/egg2pg/eggLoader.cxx b/panda/src/egg2pg/eggLoader.cxx index 8feb597b06..ffe77975cf 100644 --- a/panda/src/egg2pg/eggLoader.cxx +++ b/panda/src/egg2pg/eggLoader.cxx @@ -877,6 +877,17 @@ load_texture(TextureDef &def, const EggTexture *egg_tex) { wanted_alpha = egg_tex->has_alpha_filename(); } + // Since some properties of the textures are inferred from the + // texture files themselves (if the properties are not explicitly + // specified in the egg file), then we add the textures as + // dependents for the egg file. + if (_record != (BamCacheRecord *)NULL) { + _record->add_dependent_file(egg_tex->get_fullpath()); + if (egg_tex->has_alpha_filename() && wanted_alpha) { + _record->add_dependent_file(egg_tex->get_alpha_fullpath()); + } + } + PT(Texture) tex; switch (egg_tex->get_texture_type()) { case EggTexture::TT_unspecified: diff --git a/panda/src/egg2pg/eggLoader.h b/panda/src/egg2pg/eggLoader.h index 6c67de6b12..4b16233b04 100644 --- a/panda/src/egg2pg/eggLoader.h +++ b/panda/src/egg2pg/eggLoader.h @@ -41,6 +41,7 @@ #include "eggTransform.h" #include "geomVertexData.h" #include "geomPrimitive.h" +#include "bamCacheRecord.h" class EggNode; class EggBin; @@ -237,6 +238,7 @@ private: public: PT(PandaNode) _root; PT(EggData) _data; + PT(BamCacheRecord) _record; bool _error; friend class EggRenderState; diff --git a/panda/src/egg2pg/load_egg_file.cxx b/panda/src/egg2pg/load_egg_file.cxx index 66bdbd4797..aaf0f3b431 100644 --- a/panda/src/egg2pg/load_egg_file.cxx +++ b/panda/src/egg2pg/load_egg_file.cxx @@ -22,10 +22,11 @@ #include "sceneGraphReducer.h" #include "virtualFileSystem.h" #include "config_util.h" +#include "bamCacheRecord.h" static PT(PandaNode) load_from_loader(EggLoader &loader) { - loader._data->load_externals(); + loader._data->load_externals(DSearchPath(), loader._record); loader.build_graph(); @@ -71,7 +72,8 @@ load_from_loader(EggLoader &loader) { // required. //////////////////////////////////////////////////////////////////// PT(PandaNode) -load_egg_file(const string &filename, CoordinateSystem cs) { +load_egg_file(const string &filename, CoordinateSystem cs, + BamCacheRecord *record) { Filename egg_filename = Filename::text_filename(filename); VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); if (!vfs->exists(egg_filename)) { @@ -83,10 +85,15 @@ load_egg_file(const string &filename, CoordinateSystem cs) { egg2pg_cat.info() << "Reading " << egg_filename << "\n"; + if (record != (BamCacheRecord *)NULL) { + record->add_dependent_file(egg_filename); + } + EggLoader loader; loader._data->set_egg_filename(egg_filename); loader._data->set_auto_resolve_externals(true); loader._data->set_coordinate_system(cs); + loader._record = record; bool okflag; istream *istr = vfs->open_read_file(egg_filename, true); diff --git a/panda/src/egg2pg/load_egg_file.h b/panda/src/egg2pg/load_egg_file.h index 24fb761ad7..58a39de7db 100644 --- a/panda/src/egg2pg/load_egg_file.h +++ b/panda/src/egg2pg/load_egg_file.h @@ -25,6 +25,8 @@ #include "coordinateSystem.h" #include "eggData.h" +class BamCacheRecord; + BEGIN_PUBLISH //////////////////////////////////////////////////////////////////// // Function: load_egg_file @@ -37,7 +39,8 @@ BEGIN_PUBLISH // bit more manual control over the loading process. //////////////////////////////////////////////////////////////////// EXPCL_PANDAEGG PT(PandaNode) -load_egg_file(const string &filename, CoordinateSystem cs = CS_default); +load_egg_file(const string &filename, CoordinateSystem cs = CS_default, + BamCacheRecord *record = NULL); //////////////////////////////////////////////////////////////////// // Function: load_egg_data diff --git a/panda/src/egg2pg/loaderFileTypeEgg.cxx b/panda/src/egg2pg/loaderFileTypeEgg.cxx index b4d6062a21..c78c15bec5 100644 --- a/panda/src/egg2pg/loaderFileTypeEgg.cxx +++ b/panda/src/egg2pg/loaderFileTypeEgg.cxx @@ -70,7 +70,8 @@ supports_compressed() const { // Description: //////////////////////////////////////////////////////////////////// PT(PandaNode) LoaderFileTypeEgg:: -load_file(const Filename &path, const LoaderOptions &) const { - PT(PandaNode) result = load_egg_file(path); +load_file(const Filename &path, const LoaderOptions &, + BamCacheRecord *record) const { + PT(PandaNode) result = load_egg_file(path, CS_default, record); return result; } diff --git a/panda/src/egg2pg/loaderFileTypeEgg.h b/panda/src/egg2pg/loaderFileTypeEgg.h index 1917d8386c..4fc3299943 100644 --- a/panda/src/egg2pg/loaderFileTypeEgg.h +++ b/panda/src/egg2pg/loaderFileTypeEgg.h @@ -35,7 +35,8 @@ public: virtual string get_extension() const; virtual bool supports_compressed() const; - virtual PT(PandaNode) load_file(const Filename &path, const LoaderOptions &optoins) const; + virtual PT(PandaNode) load_file(const Filename &path, const LoaderOptions &optoins, + BamCacheRecord *record) const; public: static TypeHandle get_class_type() { diff --git a/panda/src/gobj/texture.cxx b/panda/src/gobj/texture.cxx index 8eec1ce49b..daa1552318 100644 --- a/panda/src/gobj/texture.cxx +++ b/panda/src/gobj/texture.cxx @@ -470,7 +470,7 @@ read_txo(istream &in, const string &filename) { } else if (!object->is_of_type(Texture::get_class_type())) { gobj_cat.error() - << "Texture object " << filename << "contains a " + << "Texture object " << filename << " contains a " << object->get_type() << ", not a Texture.\n"; return false; } diff --git a/panda/src/pgraph/loader.cxx b/panda/src/pgraph/loader.cxx index 359af93481..439d7f5d28 100644 --- a/panda/src/pgraph/loader.cxx +++ b/panda/src/pgraph/loader.cxx @@ -485,7 +485,7 @@ load_file(const Filename &filename, const LoaderOptions &options) const { } LoaderFileType *type = results.get_file_type(i); - PT(PandaNode) result = type->load_file(path, options); + PT(PandaNode) result = type->load_file(path, options, record); if (result != (PandaNode *)NULL){ if (record != (BamCacheRecord *)NULL) { record->set_data(result, false); diff --git a/panda/src/pgraph/loaderFileType.cxx b/panda/src/pgraph/loaderFileType.cxx index 2593321ea7..7f8026d3d8 100644 --- a/panda/src/pgraph/loaderFileType.cxx +++ b/panda/src/pgraph/loaderFileType.cxx @@ -69,7 +69,8 @@ supports_compressed() const { // Description: //////////////////////////////////////////////////////////////////// PT(PandaNode) LoaderFileType:: -load_file(const Filename &path, const LoaderOptions &options) const { +load_file(const Filename &path, const LoaderOptions &options, + BamCacheRecord *record) const { loader_cat.error() << get_type() << " cannot read PandaNode objects.\n"; return NULL; diff --git a/panda/src/pgraph/loaderFileType.h b/panda/src/pgraph/loaderFileType.h index f4675eacb2..c61554d667 100644 --- a/panda/src/pgraph/loaderFileType.h +++ b/panda/src/pgraph/loaderFileType.h @@ -28,6 +28,7 @@ #include "dSearchPath.h" class LoaderOptions; +class BamCacheRecord; //////////////////////////////////////////////////////////////////// // Class : LoaderFileType @@ -50,7 +51,8 @@ PUBLISHED: virtual bool supports_compressed() const; public: - virtual PT(PandaNode) load_file(const Filename &path, const LoaderOptions &options) const; + virtual PT(PandaNode) load_file(const Filename &path, const LoaderOptions &options, + BamCacheRecord *record) const; public: static TypeHandle get_class_type() { diff --git a/panda/src/pgraph/loaderFileTypeBam.cxx b/panda/src/pgraph/loaderFileTypeBam.cxx index 21c3339f43..bfd5c52aec 100644 --- a/panda/src/pgraph/loaderFileTypeBam.cxx +++ b/panda/src/pgraph/loaderFileTypeBam.cxx @@ -72,7 +72,12 @@ supports_compressed() const { // Description: //////////////////////////////////////////////////////////////////// PT(PandaNode) LoaderFileTypeBam:: -load_file(const Filename &path, const LoaderOptions &options) const { +load_file(const Filename &path, const LoaderOptions &options, + BamCacheRecord *record) const { + if (record != (BamCacheRecord *)NULL) { + record->add_dependent_file(path); + } + bool report_errors = (options.get_flags() & LoaderOptions::LF_report_errors) != 0; BamFile bam_file; if (!bam_file.open_read(path, report_errors)) { diff --git a/panda/src/pgraph/loaderFileTypeBam.h b/panda/src/pgraph/loaderFileTypeBam.h index a3061fcfca..2af7ea3352 100644 --- a/panda/src/pgraph/loaderFileTypeBam.h +++ b/panda/src/pgraph/loaderFileTypeBam.h @@ -35,7 +35,8 @@ public: virtual string get_extension() const; virtual bool supports_compressed() const; - virtual PT(PandaNode) load_file(const Filename &path, const LoaderOptions &options) const; + virtual PT(PandaNode) load_file(const Filename &path, const LoaderOptions &options, + BamCacheRecord *record) const; public: static TypeHandle get_class_type() { diff --git a/panda/src/putil/Sources.pp b/panda/src/putil/Sources.pp index aa4178e5dd..da67424f17 100644 --- a/panda/src/putil/Sources.pp +++ b/panda/src/putil/Sources.pp @@ -12,6 +12,7 @@ animInterface.h animInterface.I \ bam.h \ bamCache.h bamCache.I \ + bamCacheIndex.h bamCacheIndex.I \ bamCacheRecord.h bamCacheRecord.I \ bamEndian.h \ bamReader.I bamReader.N bamReader.h bamReaderParam.I \ @@ -65,6 +66,7 @@ #define INCLUDED_SOURCES \ animInterface.cxx \ bamCache.cxx \ + bamCacheIndex.cxx \ bamCacheRecord.cxx \ bamEndian.cxx \ bamReader.cxx bamReaderParam.cxx \ @@ -104,6 +106,7 @@ animInterface.h animInterface.I \ bam.h \ bamCache.h bamCache.I \ + bamCacheIndex.h bamCacheIndex.I \ bamCacheRecord.h bamCacheRecord.I \ bamEndian.h \ bamReader.I bamReader.h bamReaderParam.I bamReaderParam.h \ diff --git a/panda/src/putil/bamCache.I b/panda/src/putil/bamCache.I index 1f49e68e3b..dbb6e37cfa 100644 --- a/panda/src/putil/bamCache.I +++ b/panda/src/putil/bamCache.I @@ -69,3 +69,16 @@ get_global_ptr() { } return _global_ptr; } + +//////////////////////////////////////////////////////////////////// +// Function: BamCache::mark_index_stale +// Access: Private +// Description: Indicates that the index has been modified and will +// need to be written to disk eventually. +//////////////////////////////////////////////////////////////////// +INLINE void BamCache:: +mark_index_stale() { + if (_index_stale_since == 0) { + _index_stale_since = time(NULL); + } +} diff --git a/panda/src/putil/bamCache.cxx b/panda/src/putil/bamCache.cxx index 5939db9a63..cb12680422 100644 --- a/panda/src/putil/bamCache.cxx +++ b/panda/src/putil/bamCache.cxx @@ -17,12 +17,14 @@ //////////////////////////////////////////////////////////////////// #include "bamCache.h" +#include "bamCacheIndex.h" #include "hashVal.h" #include "datagramInputFile.h" #include "datagramOutputFile.h" #include "config_util.h" #include "bam.h" #include "typeRegistry.h" +#include "string_utils.h" BamCache *BamCache::_global_ptr = NULL; @@ -33,7 +35,9 @@ BamCache *BamCache::_global_ptr = NULL; //////////////////////////////////////////////////////////////////// BamCache:: BamCache() : - _active(true) + _active(true), + _index(new BamCacheIndex), + _index_stale_since(0) { } @@ -44,6 +48,9 @@ BamCache() : //////////////////////////////////////////////////////////////////// BamCache:: ~BamCache() { + flush_index(); + delete _index; + _index = NULL; } //////////////////////////////////////////////////////////////////// @@ -60,6 +67,7 @@ BamCache:: //////////////////////////////////////////////////////////////////// void BamCache:: set_root(const Filename &root) { + flush_index(); _root = root; // For now, the filename must be a directory. Maybe eventually we @@ -72,6 +80,11 @@ set_root(const Filename &root) { dirname.make_dir(); } nassertv(root.is_directory()); + + delete _index; + _index = new BamCacheIndex; + _index_stale_since = 0; + read_index(); } //////////////////////////////////////////////////////////////////// @@ -98,6 +111,8 @@ set_root(const Filename &root) { //////////////////////////////////////////////////////////////////// PT(BamCacheRecord) BamCache:: lookup(const Filename &source_filename, const string &cache_extension) { + consider_flush_index(); + Filename source_pathname(source_filename); source_pathname.make_absolute(); @@ -129,6 +144,8 @@ store(BamCacheRecord *record) { nassertr(!record->_cache_pathname.empty(), false); nassertr(record->has_data(), false); + consider_flush_index(); + #ifndef NDEBUG // Ensure that the cache_pathname is within the _root directory tree. Filename rel_pathname(record->_cache_pathname); @@ -138,30 +155,40 @@ store(BamCacheRecord *record) { record->_recorded_time = time(NULL); - ofstream cache_file; + Filename cache_pathname = Filename::binary_filename(record->_cache_pathname); - Filename filename = Filename::binary_filename(record->_cache_pathname); - if (!filename.open_write(cache_file)) { + // We actually do the write to a temporary filename first, and then + // move it into place, so that no one attempts to read the file + // while it is in the process of being written. + Filename temp_pathname = cache_pathname; + temp_pathname.set_extension("tmp"); + temp_pathname.set_binary(); + + ofstream temp_file; + if (!temp_pathname.open_write(temp_file)) { util_cat.error() - << "Could not open cache file: " << filename << "\n"; + << "Could not open cache file: " << temp_pathname << "\n"; return false; } DatagramOutputFile dout; - if (!dout.open(cache_file)) { + if (!dout.open(temp_file)) { util_cat.error() - << "Could not write cache file: " << filename << "\n"; + << "Could not write cache file: " << temp_pathname << "\n"; + temp_pathname.unlink(); return false; } if (!dout.write_header(_bam_header)) { util_cat.error() - << "Unable to write to " << filename << "\n"; + << "Unable to write to " << temp_pathname << "\n"; + temp_pathname.unlink(); return false; } - BamWriter writer(&dout, filename); + BamWriter writer(&dout, temp_pathname); if (!writer.init()) { + temp_pathname.unlink(); return false; } @@ -176,13 +203,478 @@ store(BamCacheRecord *record) { } if (!writer.write_object(record)) { + temp_pathname.unlink(); return false; } if (!writer.write_object(record->get_data())) { + temp_pathname.unlink(); return false; } + record->_record_size = temp_file.tellp(); + temp_file.close(); + + // Now move the file into place. + if (!temp_pathname.rename_to(cache_pathname)) { + cache_pathname.unlink(); + if (!temp_pathname.rename_to(cache_pathname)) { + util_cat.error() + << "Unable to rename " << temp_pathname << " to " + << cache_pathname << "\n"; + temp_pathname.unlink(); + return false; + } + } + + add_to_index(record); + + return true; +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCache::consider_flush_index +// Access: Published +// Description: Flushes the index if enough time has elapsed since +// the index was last flushed. +//////////////////////////////////////////////////////////////////// +void BamCache:: +consider_flush_index() { + if (_index_stale_since != 0) { + int elapsed = (int)time(NULL) - (int)_index_stale_since; + if (elapsed > model_cache_flush) { + flush_index(); + } + } +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCache::flush_index +// Access: Published +// Description: Ensures the index is written to disk. +//////////////////////////////////////////////////////////////////// +void BamCache:: +flush_index() { + if (_index_stale_since == 0) { + // Never mind. + return; + } + + while (true) { + Filename temp_pathname = Filename::temporary(_root, "index-", ".boo"); + + if (!do_write_index(temp_pathname, _index)) { + return; + } + + // Now atomically write the name of this index file to the index + // reference file. + Filename index_ref_pathname(_root, Filename("index_name.txt")); + string old_index = _index_ref_contents; + string new_index = temp_pathname.get_basename() + "\n"; + string orig_index; + if (index_ref_pathname.atomic_compare_and_exchange_contents(orig_index, old_index, new_index)) { + // We successfully wrote our version of the index, and no other + // process beat us to it. Our index is now the official one. + // Remove the old index. + _index_pathname.unlink(); + _index_pathname = temp_pathname; + _index_ref_contents = new_index; + _index_stale_since = 0; + return; + } + + // Shoot, some other process updated the index while we were + // trying to update it, and they beat us to it. We have to merge, + // and try again. + temp_pathname.unlink(); + _index_pathname = Filename(_root, Filename(trim(orig_index))); + _index_ref_contents = orig_index; + read_index(); + } +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCache::read_index +// Access: Private +// Description: Reads, or re-reads the index file from disk. If +// _index_stale_since is nonzero, the index file is read +// and then merged with our current index. +//////////////////////////////////////////////////////////////////// +void BamCache:: +read_index() { + if (!read_index_pathname(_index_pathname, _index_ref_contents)) { + // Couldn't read the index ref; rebuild the index. + rebuild_index(); + return; + } + + while (true) { + BamCacheIndex *new_index = do_read_index(_index_pathname); + if (new_index != (BamCacheIndex *)NULL) { + merge_index(new_index); + return; + } + + // We couldn't read the index. Maybe it's been removed already. + // See if the index_pathname has changed. + Filename old_index_pathname = _index_pathname; + if (!read_index_pathname(_index_pathname, _index_ref_contents)) { + // Couldn't read the index ref; rebuild the index. + rebuild_index(); + return; + } + + if (old_index_pathname == _index_pathname) { + // Nope, we just couldn't read it. Delete it and build a new + // one. + _index_pathname.unlink(); + rebuild_index(); + flush_index(); + return; + } + } +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCache::read_index_pathname +// Access: Private +// Description: Atomically reads the current index filename from the +// index reference file. The index filename moves +// around as different processes update the index. +//////////////////////////////////////////////////////////////////// +bool BamCache:: +read_index_pathname(Filename &index_pathname, string &index_ref_contents) const { + index_ref_contents.clear(); + Filename index_ref_pathname(_root, Filename("index_name.txt")); + if (!index_ref_pathname.atomic_read_contents(index_ref_contents)) { + return false; + } + + string trimmed = trim(index_ref_contents); + if (trimmed.empty()) { + index_pathname = Filename(); + } else { + index_pathname = Filename(_root, Filename(trimmed)); + } + return true; +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCache::merge_index +// Access: Private +// Description: The supplied index file has been updated by some other +// process. Merge it with our current index. +// +// Ownership of the pointer is transferred with this +// call. The caller should assume that new_index will +// be deleted by this method. +//////////////////////////////////////////////////////////////////// +void BamCache:: +merge_index(BamCacheIndex *new_index) { + if (_index_stale_since == 0) { + // If our index isn't stale, just replace it. + delete _index; + _index = new_index; + return; + } + + BamCacheIndex *old_index = _index; + _index = new BamCacheIndex; + + BamCacheIndex::Records::const_iterator ai = old_index->_records.begin(); + BamCacheIndex::Records::const_iterator bi = new_index->_records.begin(); + + while (ai != old_index->_records.end() && + bi != new_index->_records.end()) { + if ((*ai).first < (*bi).first) { + // Here is an entry we have in our index, not present in the new + // index. + PT(BamCacheRecord) record = (*ai).second; + Filename cache_pathname(_root, record->get_cache_filename()); + if (cache_pathname.exists()) { + // The file exists; keep it. + _index->_cache_size += record->_record_size; + _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); + } + ++ai; + + } else if ((*bi).first < (*ai).first) { + // Here is an entry in the new index, not present in our index. + PT(BamCacheRecord) record = (*bi).second; + Filename cache_pathname(_root, record->get_cache_filename()); + if (cache_pathname.exists()) { + // The file exists; keep it. + _index->_cache_size += record->_record_size; + _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); + } + ++bi; + + } else { + // Here is an entry we have in both. + PT(BamCacheRecord) a_record = (*ai).second; + PT(BamCacheRecord) b_record = (*bi).second; + if (*a_record == *b_record) { + // They're the same entry. It doesn't really matter which one + // we keep. + _index->_cache_size += a_record->_record_size; + _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(a_record->get_source_pathname(), a_record)); + + } else { + // They're different. Just throw them both away, and re-read + // the current data from the cache file. + + Filename cache_pathname(_root, a_record->get_cache_filename()); + + if (cache_pathname.exists()) { + PT(BamCacheRecord) record = do_read_record(cache_pathname, false); + if (record != (BamCacheRecord *)NULL) { + _index->_cache_size += record->_record_size; + _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); + } + } + } + + ++ai; + ++bi; + } + } + + while (ai != old_index->_records.end()) { + // Here is an entry we have in our index, not present in the new + // index. + PT(BamCacheRecord) record = (*ai).second; + Filename cache_pathname(_root, record->get_cache_filename()); + if (cache_pathname.exists()) { + // The file exists; keep it. + _index->_cache_size += record->_record_size; + _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); + } + ++ai; + } + + while (bi != new_index->_records.end()) { + // Here is an entry in the new index, not present in our index. + PT(BamCacheRecord) record = (*bi).second; + Filename cache_pathname(_root, record->get_cache_filename()); + if (cache_pathname.exists()) { + // The file exists; keep it. + _index->_cache_size += record->_record_size; + _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); + } + ++bi; + } +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCache::rebuild_index +// Access: Private +// Description: Regenerates the index from scratch by scanning the +// directory. +//////////////////////////////////////////////////////////////////// +void BamCache:: +rebuild_index() { + vector_string contents; + if (!_root.scan_directory(contents)) { + util_cat.error() + << "Unable to read directory " << _root << ", caching disabled.\n"; + set_active(false); + return; + } + + delete _index; + _index = new BamCacheIndex; + + vector_string::const_iterator ci; + for (ci = contents.begin(); ci != contents.end(); ++ci) { + Filename filename(*ci); + if (filename.get_extension() == "bam" || + filename.get_extension() == "txo") { + Filename pathname(_root, filename); + + PT(BamCacheRecord) record = do_read_record(pathname, false); + if (record == (BamCacheRecord *)NULL) { + // Well, it was invalid, so blow it away. + pathname.unlink(); + + } else { + record->_record_access_time = record->_recorded_time; + + bool inserted = _index->_records.insert(BamCacheIndex::Records::value_type(record->get_source_pathname(), record)).second; + if (!inserted) { + util_cat.info() + << "Multiple cache files defining " << record->get_source_pathname() << "\n"; + pathname.unlink(); + } else { + _index->_cache_size += record->_record_size; + } + } + } + } + _index_stale_since = time(NULL); + flush_index(); +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCache::add_to_index +// Access: Private +// Description: Updates the index entry for the indicated record. +//////////////////////////////////////////////////////////////////// +void BamCache:: +add_to_index(const BamCacheRecord *record) { + PT(BamCacheRecord) new_record = record->make_copy(); + + pair result = + _index->_records.insert(BamCacheIndex::Records::value_type(new_record->get_source_pathname(), new_record)); + if (!result.second) { + // We already had a record for this filename; it gets replaced. + PT(BamCacheRecord) orig_record = (*result.first).second; + if (*orig_record == *record) { + // Well, never mind. The record hasn't changed. + return; + } + + _index->_cache_size -= orig_record->_record_size; + (*result.first).second = new_record; + } + + _index->_cache_size += new_record->_record_size; + mark_index_stale(); +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCache::remove_from_index +// Access: Private +// Description: Removes the index entry for the indicated record, if +// there is one. +//////////////////////////////////////////////////////////////////// +void BamCache:: +remove_from_index(const Filename &source_pathname) { + BamCacheIndex::Records::iterator ri = _index->_records.find(source_pathname); + if (ri == _index->_records.end()) { + // No entry for this record; no problem. + return; + } + + PT(BamCacheRecord) record = (*ri).second; + _index->_cache_size -= record->_record_size; + _index->_records.erase(ri); + mark_index_stale(); +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCache::do_read_index +// Access: Private, Static +// Description: Reads the index data from the specified filename. +// Returns a newly-allocated BamCacheIndex object on +// success, or NULL on failure. +//////////////////////////////////////////////////////////////////// +BamCacheIndex *BamCache:: +do_read_index(Filename &index_pathname) { + if (index_pathname.empty()) { + return NULL; + } + + index_pathname.set_binary(); + ifstream index_file; + if (!index_pathname.open_read(index_file)) { + util_cat.error() + << "Could not open index file: " << index_pathname << "\n"; + return NULL; + } + + DatagramInputFile din; + + if (!din.open(index_file)) { + util_cat.debug() + << "Could not read index file: " << index_pathname << "\n"; + return NULL; + } + + string head; + if (!din.read_header(head, _bam_header.size())) { + util_cat.debug() + << index_pathname << " is not an index file.\n"; + return NULL; + } + + if (head != _bam_header) { + util_cat.debug() + << index_pathname << " is not an index file.\n"; + return NULL; + } + + BamReader reader(&din, index_pathname); + if (!reader.init()) { + return NULL; + } + + TypedWritable *object = reader.read_object(); + + if (object == (TypedWritable *)NULL) { + util_cat.error() + << "Cache index " << index_pathname << " is empty.\n"; + return NULL; + + } else if (!object->is_of_type(BamCacheIndex::get_class_type())) { + util_cat.error() + << "Cache index " << index_pathname << " contains a " + << object->get_type() << ", not a BamCacheIndex.\n"; + return NULL; + } + + BamCacheIndex *index = DCAST(BamCacheIndex, object); + if (!reader.resolve()) { + util_cat.error() + << "Unable to fully resolve cache index file.\n"; + return NULL; + } + + return index; +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCache::do_write_index +// Access: Private, Static +// Description: Writes the given index data to the specified filename. +//////////////////////////////////////////////////////////////////// +bool BamCache:: +do_write_index(Filename &index_pathname, const BamCacheIndex *index) { + index_pathname.set_binary(); + ofstream index_file; + if (!index_pathname.open_write(index_file)) { + util_cat.error() + << "Could not open index file: " << index_pathname << "\n"; + return false; + } + + DatagramOutputFile dout; + if (!dout.open(index_file)) { + util_cat.error() + << "Could not write index file: " << index_pathname << "\n"; + index_pathname.unlink(); + return false; + } + + if (!dout.write_header(_bam_header)) { + util_cat.error() + << "Unable to write to " << index_pathname << "\n"; + index_pathname.unlink(); + return false; + } + + BamWriter writer(&dout, index_pathname); + if (!writer.init()) { + index_pathname.unlink(); + return false; + } + + if (!writer.write_object(index)) { + index_pathname.unlink(); + return false; + } + + index_file.close(); return true; } @@ -197,12 +689,13 @@ store(BamCacheRecord *record) { //////////////////////////////////////////////////////////////////// PT(BamCacheRecord) BamCache:: find_and_read_record(const Filename &source_pathname, - const Filename &cache_filename) const { + const Filename &cache_filename) { int pass = 0; while (true) { PT(BamCacheRecord) record = read_record(source_pathname, cache_filename, pass); if (record != (BamCacheRecord *)NULL) { + add_to_index(record); return record; } ++pass; @@ -219,28 +712,64 @@ find_and_read_record(const Filename &source_pathname, PT(BamCacheRecord) BamCache:: read_record(const Filename &source_pathname, const Filename &cache_filename, - int pass) const { - Filename filename(_root, cache_filename); + int pass) { + Filename cache_pathname(_root, cache_filename); if (pass != 0) { ostringstream strm; - strm << filename.get_basename_wo_extension() << "_" << pass; - filename.set_basename_wo_extension(strm.str()); + strm << cache_pathname.get_basename_wo_extension() << "_" << pass; + cache_pathname.set_basename_wo_extension(strm.str()); } - if (!filename.exists()) { + if (!cache_pathname.exists()) { // There is no such cache file already. Declare it. - filename.touch(); PT(BamCacheRecord) record = new BamCacheRecord(source_pathname, cache_filename); - record->_cache_pathname = filename; + record->_cache_pathname = cache_pathname; return record; } - filename.set_binary(); - ifstream cache_file; - if (!filename.open_read(cache_file)) { + PT(BamCacheRecord) record = do_read_record(cache_pathname, true); + if (record == (BamCacheRecord *)NULL) { + // Well, it was invalid, so blow it away, and make a new one. + cache_pathname.unlink(); + remove_from_index(source_pathname); + + PT(BamCacheRecord) record = + new BamCacheRecord(source_pathname, cache_filename); + record->_cache_pathname = cache_pathname; + return record; + } + + if (record->get_source_pathname() != source_pathname) { + // This might be just a hash conflict. util_cat.debug() - << "Could not open cache file: " << filename << "\n"; + << "Cache file " << cache_pathname << " references " + << record->get_source_pathname() << ", not " + << source_pathname << "\n"; + return NULL; + } + + if (!record->has_data()) { + // If we didn't find any data, the caller will have to reload it. + record->clear_dependent_files(); + } + + record->_cache_pathname = cache_pathname; + return record; +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCache::do_read_record +// Access: Private, Static +// Description: Actually reads a record from the file. +//////////////////////////////////////////////////////////////////// +PT(BamCacheRecord) BamCache:: +do_read_record(Filename &cache_pathname, bool read_data) { + cache_pathname.set_binary(); + ifstream cache_file; + if (!cache_pathname.open_read(cache_file)) { + util_cat.debug() + << "Could not open cache file: " << cache_pathname << "\n"; return NULL; } @@ -248,24 +777,24 @@ read_record(const Filename &source_pathname, if (!din.open(cache_file)) { util_cat.debug() - << "Could not read cache file: " << filename << "\n"; + << "Could not read cache file: " << cache_pathname << "\n"; return NULL; } string head; if (!din.read_header(head, _bam_header.size())) { util_cat.debug() - << filename << " is not a cache file.\n"; + << cache_pathname << " is not a cache file.\n"; return NULL; } if (head != _bam_header) { util_cat.debug() - << filename << " is not a cache file.\n"; + << cache_pathname << " is not a cache file.\n"; return NULL; } - BamReader reader(&din, filename); + BamReader reader(&din, cache_pathname); if (!reader.init()) { return NULL; } @@ -273,12 +802,12 @@ read_record(const Filename &source_pathname, TypedWritable *object = reader.read_object(); if (object == (TypedWritable *)NULL) { util_cat.debug() - << filename << " is empty.\n"; + << cache_pathname << " is empty.\n"; return NULL; } else if (!object->is_of_type(BamCacheRecord::get_class_type())) { util_cat.debug() - << "Cache file " << filename << "contains a " + << "Cache file " << cache_pathname << " contains a " << object->get_type() << ", not a BamCacheRecord.\n"; return NULL; } @@ -286,15 +815,7 @@ read_record(const Filename &source_pathname, PT(BamCacheRecord) record = DCAST(BamCacheRecord, object); if (!reader.resolve()) { util_cat.debug() - << "Unable to fully resolve cache record in " << filename << "\n"; - return NULL; - } - - if (record->get_source_pathname() != source_pathname) { - util_cat.debug() - << "Cache file " << filename << " references " - << record->get_source_pathname() << ", not " - << source_pathname << "\n"; + << "Unable to fully resolve cache record in " << cache_pathname << "\n"; return NULL; } @@ -303,7 +824,7 @@ read_record(const Filename &source_pathname, // and therefore the cache record will be returned. // We still need to decide whether the cache record is stale. - if (record->dependents_unchanged()) { + if (read_data && record->dependents_unchanged()) { // The cache record doesn't appear to be stale. Load the cached // object. object = reader.read_object(); @@ -311,7 +832,7 @@ read_record(const Filename &source_pathname, if (object != (TypedWritable *)NULL) { if (!reader.resolve()) { util_cat.debug() - << "Unable to fully resolve cached object in " << filename << "\n"; + << "Unable to fully resolve cached object in " << cache_pathname << "\n"; delete object; } else { // The object is valid. Store it in the record. @@ -319,13 +840,15 @@ read_record(const Filename &source_pathname, } } } + + // Also get the file size. + cache_file.clear(); + cache_file.seekg(0, ios::end); + record->_record_size = cache_file.tellg(); - if (!record->has_data()) { - // If we didn't find any data, the caller will have to reload it. - record->clear_dependent_files(); - } + // And the last access time is now, duh. + record->_record_access_time = time(NULL); - record->_cache_pathname = filename; return record; } diff --git a/panda/src/putil/bamCache.h b/panda/src/putil/bamCache.h index 32a2afd971..7c06d869e3 100644 --- a/panda/src/putil/bamCache.h +++ b/panda/src/putil/bamCache.h @@ -22,6 +22,11 @@ #include "pandabase.h" #include "bamCacheRecord.h" #include "pointerTo.h" +#include "filename.h" +#include "pmap.h" +#include "pvector.h" + +class BamCacheIndex; //////////////////////////////////////////////////////////////////// // Class : BamCache @@ -29,6 +34,15 @@ // objects generated from model files and texture images // (as well as possibly other kinds of loadable objects // that can be stored in bam file format). +// +// This class also maintains a persistent index that +// lists all of the cached objects (see BamCacheIndex). +// We go through some considerable effort to make sure +// this index gets saved correctly to disk, even in the +// presence of multiple different processes writing to +// the same index, and without relying too heavily on +// low-level os-provided file locks (which work poorly +// with C++ iostreams). //////////////////////////////////////////////////////////////////// class EXPCL_PANDA BamCache { PUBLISHED: @@ -45,14 +59,31 @@ PUBLISHED: const string &cache_extension); bool store(BamCacheRecord *record); + void consider_flush_index(); + void flush_index(); + INLINE static BamCache *get_global_ptr(); private: + void read_index(); + bool read_index_pathname(Filename &index_pathname, + string &index_ref_contents) const; + void merge_index(BamCacheIndex *new_index); + void rebuild_index(); + INLINE void mark_index_stale(); + + void add_to_index(const BamCacheRecord *record); + void remove_from_index(const Filename &source_filename); + + static BamCacheIndex *do_read_index(Filename &index_pathname); + static bool do_write_index(Filename &index_pathname, const BamCacheIndex *index); + PT(BamCacheRecord) find_and_read_record(const Filename &source_pathname, - const Filename &cache_filename) const; + const Filename &cache_filename); PT(BamCacheRecord) read_record(const Filename &source_pathname, const Filename &cache_filename, - int pass) const; + int pass); + static PT(BamCacheRecord) do_read_record(Filename &cache_pathname, bool read_data); static string hash_filename(const string &filename); static void make_global(); @@ -60,6 +91,12 @@ private: bool _active; Filename _root; static BamCache *_global_ptr; + + BamCacheIndex *_index; + time_t _index_stale_since; + + Filename _index_pathname; + string _index_ref_contents; }; #include "bamCache.I" diff --git a/panda/src/putil/bamCacheIndex.I b/panda/src/putil/bamCacheIndex.I new file mode 100644 index 0000000000..5d63f78112 --- /dev/null +++ b/panda/src/putil/bamCacheIndex.I @@ -0,0 +1,29 @@ +// Filename: bamCacheIndex.I +// Created by: drose (19Jun06) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) 2001 - 2004, Disney Enterprises, Inc. All rights reserved +// +// All use of this software is subject to the terms of the Panda 3d +// Software license. You should have received a copy of this license +// along with this source code; you will also find a current copy of +// the license at http://etc.cmu.edu/panda3d/docs/license/ . +// +// To contact the maintainers of this program write to +// panda3d-general@lists.sourceforge.net . +// +//////////////////////////////////////////////////////////////////// + + +//////////////////////////////////////////////////////////////////// +// Function: BamCacheIndex::Constructor +// Access: Published +// Description: +//////////////////////////////////////////////////////////////////// +INLINE BamCacheIndex:: +BamCacheIndex() : + _cache_size(0) +{ +} diff --git a/panda/src/putil/bamCacheIndex.cxx b/panda/src/putil/bamCacheIndex.cxx new file mode 100644 index 0000000000..c7fac2597b --- /dev/null +++ b/panda/src/putil/bamCacheIndex.cxx @@ -0,0 +1,143 @@ +// Filename: bamCacheIndex.cxx +// Created by: drose (19Jun06) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) 2001 - 2004, Disney Enterprises, Inc. All rights reserved +// +// All use of this software is subject to the terms of the Panda 3d +// Software license. You should have received a copy of this license +// along with this source code; you will also find a current copy of +// the license at http://etc.cmu.edu/panda3d/docs/license/ . +// +// To contact the maintainers of this program write to +// panda3d-general@lists.sourceforge.net . +// +//////////////////////////////////////////////////////////////////// + +#include "bamCacheIndex.h" +#include "indent.h" + +TypeHandle BamCacheIndex::_type_handle; + +//////////////////////////////////////////////////////////////////// +// Function: BamCacheIndex::write +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +void BamCacheIndex:: +write(ostream &out, int indent_level) const { + indent(out, indent_level) + << "BamCacheIndex, " << _records.size() << " records:\n"; + + Records::const_iterator ri; + for (ri = _records.begin(); ri != _records.end(); ++ri) { + PT(BamCacheRecord) record = (*ri).second; + indent(out, indent_level + 2) + << setw(10) << record->_record_size << " " + << record->get_cache_filename() << " " + << record->get_source_pathname() << "\n"; + } + out << "\n"; + indent(out, indent_level) + << setw(12) << _cache_size << " bytes total\n"; +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCacheIndex::register_with_read_factory +// Access: Public, Static +// Description: Tells the BamReader how to create objects of type +// BamCacheRecord. +//////////////////////////////////////////////////////////////////// +void BamCacheIndex:: +register_with_read_factory() { + BamReader::get_factory()->register_factory(get_class_type(), make_from_bam); +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCacheIndex::write_datagram +// Access: Public, Virtual +// Description: Writes the contents of this object to the datagram +// for shipping out to a Bam file. +//////////////////////////////////////////////////////////////////// +void BamCacheIndex:: +write_datagram(BamWriter *manager, Datagram &dg) { + TypedWritable::write_datagram(manager, dg); + + dg.add_uint32(_records.size()); + Records::const_iterator ri; + for (ri = _records.begin(); ri != _records.end(); ++ri) { + manager->write_pointer(dg, (*ri).second); + } +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCacheIndex::make_from_bam +// Access: Protected, Static +// Description: This function is called by the BamReader's factory +// when a new object of type BamCacheIndex is encountered +// in the Bam file. It should create the BamCacheIndex +// and extract its information from the file. +//////////////////////////////////////////////////////////////////// +TypedWritable *BamCacheIndex:: +make_from_bam(const FactoryParams ¶ms) { + BamCacheIndex *object = new BamCacheIndex; + DatagramIterator scan; + BamReader *manager; + + parse_params(params, scan, manager); + object->fillin(scan, manager); + + return object; +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCacheIndex::complete_pointers +// Access: Public, Virtual +// Description: Receives an array of pointers, one for each time +// manager->read_pointer() was called in fillin(). +// Returns the number of pointers processed. +//////////////////////////////////////////////////////////////////// +int BamCacheIndex:: +complete_pointers(TypedWritable **p_list, BamReader *manager) { + int pi = TypedWritable::complete_pointers(p_list, manager); + + RecordVector::iterator vi; + for (vi = _record_vector.begin(); vi != _record_vector.end(); ++vi) { + PT(BamCacheRecord) record = DCAST(BamCacheRecord, p_list[pi++]); + (*vi) = record; + + bool inserted = _records.insert(Records::value_type(record->get_source_pathname(), record)).second; + if (!inserted) { + util_cat.info() + << "Multiple cache files defining " << record->get_source_pathname() + << " in index.\n"; + } else { + _cache_size += record->_record_size; + } + } + + _record_vector.clear(); + + return pi; +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCacheIndex::fillin +// Access: Protected +// Description: This internal function is called by make_from_bam to +// read in all of the relevant data from the BamFile for +// the new BamCacheIndex. +//////////////////////////////////////////////////////////////////// +void BamCacheIndex:: +fillin(DatagramIterator &scan, BamReader *manager) { + TypedWritable::fillin(scan, manager); + + int num_records = scan.get_uint32(); + _record_vector.reserve(num_records); + for (int i = 0; i < num_records; ++i) { + _record_vector.push_back(NULL); + manager->read_pointer(scan); + } +} diff --git a/panda/src/putil/bamCacheIndex.h b/panda/src/putil/bamCacheIndex.h new file mode 100644 index 0000000000..29d0cf2f18 --- /dev/null +++ b/panda/src/putil/bamCacheIndex.h @@ -0,0 +1,89 @@ +// Filename: bamCacheIndex.h +// Created by: drose (19Jun06) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) 2001 - 2004, Disney Enterprises, Inc. All rights reserved +// +// All use of this software is subject to the terms of the Panda 3d +// Software license. You should have received a copy of this license +// along with this source code; you will also find a current copy of +// the license at http://etc.cmu.edu/panda3d/docs/license/ . +// +// To contact the maintainers of this program write to +// panda3d-general@lists.sourceforge.net . +// +//////////////////////////////////////////////////////////////////// + +#ifndef BAMCACHEINDEX_H +#define BAMCACHEINDEX_H + +#include "pandabase.h" +#include "bamCacheRecord.h" +#include "pointerTo.h" +#include "filename.h" +#include "typedWritable.h" +#include "pmap.h" +#include "pvector.h" + +//////////////////////////////////////////////////////////////////// +// Class : BamCacheIndex +// Description : This represents the in-memory index that records the +// list of files stored in the BamCache. Since the +// memory is also flushed to disk from time to time, +// this class is a TypedWritable object. +// +// For the most part, this class is used only by the +// BamCache class. +//////////////////////////////////////////////////////////////////// +class EXPCL_PANDA BamCacheIndex : public TypedWritable { +private: + INLINE BamCacheIndex(); + +public: + void write(ostream &out, int indent_level = 0) const; + +private: + typedef pmap Records; + + Records _records; + off_t _cache_size; + + // This structure is a temporary container. It is only filled in + // while reading from a bam file. + typedef pvector< PT(BamCacheRecord) > RecordVector; + RecordVector _record_vector; + +public: + static void register_with_read_factory(); + virtual void write_datagram(BamWriter *manager, Datagram &dg); + +protected: + static TypedWritable *make_from_bam(const FactoryParams ¶ms); + virtual int complete_pointers(TypedWritable **plist, BamReader *manager); + void fillin(DatagramIterator &scan, BamReader *manager); + +public: + static TypeHandle get_class_type() { + return _type_handle; + } + static void init_type() { + TypedWritable::init_type(); + register_type(_type_handle, "BamCacheIndex", + TypedWritable::get_class_type()); + } + virtual TypeHandle get_type() const { + return get_class_type(); + } + virtual TypeHandle force_init_type() {init_type(); return get_class_type();} + +private: + static TypeHandle _type_handle; + + friend class BamCache; +}; + +#include "bamCacheIndex.I" + +#endif diff --git a/panda/src/putil/bamCacheRecord.I b/panda/src/putil/bamCacheRecord.I index 2a567ce728..3d5d5a0f35 100644 --- a/panda/src/putil/bamCacheRecord.I +++ b/panda/src/putil/bamCacheRecord.I @@ -17,6 +17,34 @@ //////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +// Function: BamCacheRecord::make_copy +// Access: Published +// Description: Returns a duplicate of the BamCacheRecord. The +// duplicate will not have a data pointer set, even +// though one may have been assigned to the original via +// set_data(). +//////////////////////////////////////////////////////////////////// +INLINE PT(BamCacheRecord) BamCacheRecord:: +make_copy() const { + return new BamCacheRecord(*this); +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCacheRecord::operator == +// Access: Published +// Description: Returns true if the record matches the other record +// in those attributes which get written to disk. Does +// not compare the data pointer. +//////////////////////////////////////////////////////////////////// +INLINE bool BamCacheRecord:: +operator == (const BamCacheRecord &other) const { + return (_source_pathname == other._source_pathname && + _cache_filename == other._cache_filename && + _recorded_time == other._recorded_time && + _record_size == other._record_size); +} + //////////////////////////////////////////////////////////////////// // Function: BamCacheRecord::get_source_pathname // Access: Published diff --git a/panda/src/putil/bamCacheRecord.cxx b/panda/src/putil/bamCacheRecord.cxx index 9091da81cb..9d29801010 100644 --- a/panda/src/putil/bamCacheRecord.cxx +++ b/panda/src/putil/bamCacheRecord.cxx @@ -30,8 +30,11 @@ TypeHandle BamCacheRecord::_type_handle; //////////////////////////////////////////////////////////////////// BamCacheRecord:: BamCacheRecord() : + _recorded_time(0), + _record_size(0), _data(NULL), - _owns_pointer(false) + _owns_pointer(false), + _record_access_time(0) { } @@ -45,8 +48,29 @@ BamCacheRecord(const Filename &source_pathname, const Filename &cache_filename) : _source_pathname(source_pathname), _cache_filename(cache_filename), + _recorded_time(0), + _record_size(0), _data(NULL), - _owns_pointer(false) + _owns_pointer(false), + _record_access_time(0) +{ +} + +//////////////////////////////////////////////////////////////////// +// Function: BamCacheRecord::Copy Constructor +// Access: Private +// Description: Use make_copy() to make a copy. The copy does not +// share the data pointer. +//////////////////////////////////////////////////////////////////// +BamCacheRecord:: +BamCacheRecord(const BamCacheRecord ©) : + _source_pathname(copy._source_pathname), + _cache_filename(copy._cache_filename), + _recorded_time(copy._recorded_time), + _record_size(copy._record_size), + _data(NULL), + _owns_pointer(false), + _record_access_time(copy._record_access_time) { } @@ -160,8 +184,7 @@ write(ostream &out, int indent_level) const { DependentFiles::const_iterator fi; for (fi = _files.begin(); fi != _files.end(); ++fi) { const DependentFile &dfile = (*fi); - indent(out, indent_level) - << " " + indent(out, indent_level + 2) << setw(10) << dfile._size << " " << format_timestamp(dfile._timestamp) << " " << dfile._pathname << "\n"; @@ -222,6 +245,7 @@ write_datagram(BamWriter *manager, Datagram &dg) { dg.add_string(_source_pathname); dg.add_string(_cache_filename); dg.add_uint32(_recorded_time); + dg.add_uint64(_record_size); dg.add_uint16(_files.size()); DependentFiles::const_iterator fi; @@ -267,6 +291,7 @@ fillin(DatagramIterator &scan, BamReader *manager) { _source_pathname = scan.get_string(); _cache_filename = scan.get_string(); _recorded_time = scan.get_uint32(); + _record_size = scan.get_uint64(); unsigned int num_files = scan.get_uint16(); _files.reserve(num_files); diff --git a/panda/src/putil/bamCacheRecord.h b/panda/src/putil/bamCacheRecord.h index 5493d1ccdd..cdbdcfc07a 100644 --- a/panda/src/putil/bamCacheRecord.h +++ b/panda/src/putil/bamCacheRecord.h @@ -41,10 +41,15 @@ private: BamCacheRecord(); BamCacheRecord(const Filename &source_pathname, const Filename &cache_filename); + BamCacheRecord(const BamCacheRecord ©); PUBLISHED: virtual ~BamCacheRecord(); + INLINE PT(BamCacheRecord) make_copy() const; + + INLINE bool operator == (const BamCacheRecord &other) const; + INLINE const Filename &get_source_pathname() const; INLINE const Filename &get_cache_filename() const; INLINE time_t get_recorded_time() const; @@ -71,6 +76,7 @@ private: Filename _source_pathname; Filename _cache_filename; time_t _recorded_time; + off_t _record_size; // this is accurate only in the index file. class DependentFile { public: @@ -88,6 +94,11 @@ private: TypedWritable *_data; bool _owns_pointer; + // The following are not recorded to disk, nor even returned by the + // BamCache interface. They are strictly meaningful to the + // BamCacheRecords stored internally within the BamCache object. + time_t _record_access_time; + public: static void register_with_read_factory(); virtual void write_datagram(BamWriter *manager, Datagram &dg); @@ -114,6 +125,7 @@ private: static TypeHandle _type_handle; friend class BamCache; + friend class BamCacheIndex; }; INLINE ostream &operator << (ostream &out, const BamCacheRecord &record) { diff --git a/panda/src/putil/config_util.cxx b/panda/src/putil/config_util.cxx index 4b4c3f39bc..5d82b75120 100644 --- a/panda/src/putil/config_util.cxx +++ b/panda/src/putil/config_util.cxx @@ -18,6 +18,7 @@ #include "config_util.h" #include "animInterface.h" +#include "bamCacheIndex.h" #include "bamCacheRecord.h" #include "bamReader.h" #include "bamReaderParam.h" @@ -80,6 +81,7 @@ ConfigVariableSearchPath sound_path ConfigureFn(config_util) { AnimInterface::init_type(); + BamCacheIndex::init_type(); BamCacheRecord::init_type(); BamReaderParam::init_type(); BitArray::init_type(); @@ -107,6 +109,7 @@ ConfigureFn(config_util) { register_type(BamReader::_remove_flag, "remove"); + BamCacheIndex::register_with_read_factory(); BamCacheRecord::register_with_read_factory(); } @@ -155,3 +158,9 @@ ConfigVariableDouble sleep_precision ConfigVariableDouble average_frame_rate_interval ("average-frame-rate-interval", 1.0); + +ConfigVariableInt model_cache_flush +("model-cache-flush", 30, + PRC_DESC("This is the amount of time, in seconds, between automatic " + "flushes of the model-cache index, if model-cache-dir is " + "in effect.")); diff --git a/panda/src/putil/config_util.h b/panda/src/putil/config_util.h index f7f9e7c0f5..e52e8676ee 100644 --- a/panda/src/putil/config_util.h +++ b/panda/src/putil/config_util.h @@ -23,6 +23,8 @@ #include "notifyCategoryProxy.h" #include "configVariableSearchPath.h" #include "configVariableEnum.h" +#include "configVariableDouble.h" +#include "configVariableInt.h" #include "bamEndian.h" #include "bamTextureMode.h" #include "dconfig.h" @@ -60,4 +62,6 @@ extern ConfigVariableDouble max_dt; extern ConfigVariableDouble sleep_precision; extern ConfigVariableDouble average_frame_rate_interval; +extern ConfigVariableInt model_cache_flush; + #endif /* __CONFIG_UTIL_H__ */ diff --git a/panda/src/putil/putil_composite1.cxx b/panda/src/putil/putil_composite1.cxx index dd84199266..4432df8e59 100644 --- a/panda/src/putil/putil_composite1.cxx +++ b/panda/src/putil/putil_composite1.cxx @@ -1,5 +1,6 @@ #include "animInterface.cxx" #include "bamCache.cxx" +#include "bamCacheIndex.cxx" #include "bamCacheRecord.cxx" #include "bamEndian.cxx" #include "bamReader.cxx" diff --git a/panda/src/testbed/pview.cxx b/panda/src/testbed/pview.cxx index 8aef135abc..46d29d2833 100644 --- a/panda/src/testbed/pview.cxx +++ b/panda/src/testbed/pview.cxx @@ -25,6 +25,7 @@ #include "sceneGraphReducer.h" #include "partGroup.h" #include "cardMaker.h" +#include "bamCache.h" // By including checkPandaVersion.h, we guarantee that runtime // attempts to run pview will fail if it inadvertently links with the @@ -143,6 +144,11 @@ void event_0(CPT_Event event, void *) { // 0: run hacky test. + BamCache *cache = BamCache::get_global_ptr(); + cache->flush_index(); + + return; + EventParameter param = event->get_parameter(0); WindowFramework *wf; DCAST_INTO_V(wf, param.get_ptr());