From 25159b2efa3e366d1c71ce174072a41f32493fee Mon Sep 17 00:00:00 2001 From: David Rose Date: Tue, 22 May 2007 21:03:41 +0000 Subject: [PATCH] vertex-data-allow-reread --- panda/src/express/datagramGenerator.cxx | 31 +++++++++- panda/src/express/datagramGenerator.h | 5 ++ panda/src/gobj/config_gobj.cxx | 10 ++++ panda/src/gobj/config_gobj.h | 1 + panda/src/gobj/geomVertexArrayData.I | 2 +- panda/src/gobj/geomVertexArrayData.cxx | 8 +++ panda/src/gobj/vertexDataBuffer.I | 31 +++++++++- panda/src/gobj/vertexDataBuffer.cxx | 77 +++++++++++++++++++------ panda/src/gobj/vertexDataBuffer.h | 8 +++ panda/src/pgraph/config_pgraph.cxx | 8 +++ panda/src/pgraph/config_pgraph.h | 1 + panda/src/pgraph/loader.cxx | 14 +++-- panda/src/putil/bamReader.I | 44 ++++++++++++++ panda/src/putil/bamReader.cxx | 15 ----- panda/src/putil/bamReader.h | 5 +- panda/src/putil/datagramInputFile.cxx | 43 ++++++++++++-- panda/src/putil/datagramInputFile.h | 5 ++ 17 files changed, 260 insertions(+), 48 deletions(-) diff --git a/panda/src/express/datagramGenerator.cxx b/panda/src/express/datagramGenerator.cxx index 0adf1c46fd..8caf4e821a 100644 --- a/panda/src/express/datagramGenerator.cxx +++ b/panda/src/express/datagramGenerator.cxx @@ -23,10 +23,39 @@ //////////////////////////////////////////////////////////////////// // Function: DatagramGenerator::Destructor -// Access: Public, Virtual> +// Access: Public, Virtual // Description: Does nothing since this is class is just // the definition of an interface //////////////////////////////////////////////////////////////////// DatagramGenerator:: ~DatagramGenerator() { } + +//////////////////////////////////////////////////////////////////// +// Function: DatagramGenerator::get_file +// Access: Public, Virtual +// Description: Returns the VirtualFile that provides the source for +// these datagrams, if any, or NULL if the datagrams do +// not originate from a VirtualFile. +//////////////////////////////////////////////////////////////////// +VirtualFile *DatagramGenerator:: +get_file() { + return NULL; +} + +//////////////////////////////////////////////////////////////////// +// Function: DatagramGenerator::get_file_pos +// Access: Public, Virtual +// Description: Returns the current file position within the data +// stream, if any, or 0 if the file position is not +// meaningful or cannot be determined. +// +// For DatagramGenerators that return a meaningful file +// position, this will be pointing to the first byte +// following the datagram returned after a call to +// get_datagram(). +//////////////////////////////////////////////////////////////////// +streampos DatagramGenerator:: +get_file_pos() { + return 0; +} diff --git a/panda/src/express/datagramGenerator.h b/panda/src/express/datagramGenerator.h index 599d0f1de4..a3b8d91de4 100644 --- a/panda/src/express/datagramGenerator.h +++ b/panda/src/express/datagramGenerator.h @@ -23,6 +23,8 @@ #include "datagram.h" +class VirtualFile; + //////////////////////////////////////////////////////////////////// // Class : DatagramGenerator // Description : This class defines the abstract interace to any @@ -37,6 +39,9 @@ public: virtual bool get_datagram(Datagram &data) = 0; virtual bool is_eof() = 0; virtual bool is_error() = 0; + + virtual VirtualFile *get_file(); + virtual streampos get_file_pos(); }; #include "datagramGenerator.I" diff --git a/panda/src/gobj/config_gobj.cxx b/panda/src/gobj/config_gobj.cxx index 5f55bb39ef..467495a984 100644 --- a/panda/src/gobj/config_gobj.cxx +++ b/panda/src/gobj/config_gobj.cxx @@ -300,6 +300,16 @@ ConfigVariableInt vertex_data_small_size "is deemed too small to pay the overhead of paging it in and out, " "and it is permanently retained resident.")); +ConfigVariableBool vertex_data_allow_reread +("vertex-data-allow-reread", false, + PRC_DESC("Set this true to allow a GeomVertexArrayData to re-read itself " + "from the original bam file it was read from, if it is evicted " + "and needs to be reloaded. This avoids the need to write the " + "data to the page file, but it requires that bam files are not " + "modified during program execution. Set it false to prevent " + "this, so that the vertex data will need to be written to a " + "page file when it is evicted.")); + ConfigureFn(config_gobj) { diff --git a/panda/src/gobj/config_gobj.h b/panda/src/gobj/config_gobj.h index 41fe0c0649..a209ee2465 100644 --- a/panda/src/gobj/config_gobj.h +++ b/panda/src/gobj/config_gobj.h @@ -74,6 +74,7 @@ extern EXPCL_PANDA ConfigVariableDouble default_keystone; extern EXPCL_PANDA ConfigVariableFilename vertex_save_file_directory; extern EXPCL_PANDA ConfigVariableString vertex_save_file_prefix; extern EXPCL_PANDA ConfigVariableInt vertex_data_small_size; +extern EXPCL_PANDA ConfigVariableBool vertex_data_allow_reread; #endif diff --git a/panda/src/gobj/geomVertexArrayData.I b/panda/src/gobj/geomVertexArrayData.I index d4ca70f1d2..1a6c35c9b4 100644 --- a/panda/src/gobj/geomVertexArrayData.I +++ b/panda/src/gobj/geomVertexArrayData.I @@ -488,7 +488,7 @@ get_subdata(size_t start, size_t size) const { //////////////////////////////////////////////////////////////////// void GeomVertexArrayDataHandle:: check_resident() const { - _object->mark_used_lru(); + _object->set_lru_size(_object->get_lru_size()); } INLINE ostream & diff --git a/panda/src/gobj/geomVertexArrayData.cxx b/panda/src/gobj/geomVertexArrayData.cxx index c9731a615c..e25cba9da2 100644 --- a/panda/src/gobj/geomVertexArrayData.cxx +++ b/panda/src/gobj/geomVertexArrayData.cxx @@ -600,7 +600,13 @@ fillin(DatagramIterator &scan, BamReader *manager, void *extra_data) { } else { // Now, the array data is just stored directly. size_t size = scan.get_uint32(); + _buffer.unclean_realloc(size); + if (vertex_data_allow_reread) { + streampos source_pos = manager->get_file_pos() + (streampos)scan.get_current_index() - (streampos)scan.get_datagram().get_length(); + _buffer.set_file(manager->get_file(), source_pos); + } + const unsigned char *source_data = (const unsigned char *)scan.get_datagram().get_data(); memcpy(_buffer.get_write_pointer(), source_data + scan.get_current_index(), size); @@ -609,6 +615,8 @@ fillin(DatagramIterator &scan, BamReader *manager, void *extra_data) { if (manager->get_file_endian() != BE_native) { // For non-native endian files, we have to convert the data. + _buffer.set_file(NULL, 0); + if (array_data->_array_format == (GeomVertexArrayFormat *)NULL) { // But we can't do that until we've completed the _array_format // pointer, which tells us how to convert it. diff --git a/panda/src/gobj/vertexDataBuffer.I b/panda/src/gobj/vertexDataBuffer.I index 319b891c5d..c43ea5369b 100644 --- a/panda/src/gobj/vertexDataBuffer.I +++ b/panda/src/gobj/vertexDataBuffer.I @@ -64,6 +64,8 @@ INLINE void VertexDataBuffer:: operator = (const VertexDataBuffer ©) { unclean_realloc(copy.get_size()); memcpy(_resident_data, copy.get_read_pointer(), _size); + _source_file = copy._source_file; + _source_pos = copy._source_pos; } //////////////////////////////////////////////////////////////////// @@ -89,6 +91,11 @@ get_read_pointer() const { // pointer, which will force its page to resident status. return _block->get_pointer(); } + if (_resident_data == (unsigned char *)NULL && !_source_file.is_null()) { + // If we need to re-read the original source, do so. + ((VertexDataBuffer *)this)->page_in(); + } + return _resident_data; } @@ -99,9 +106,11 @@ get_read_pointer() const { //////////////////////////////////////////////////////////////////// INLINE unsigned char *VertexDataBuffer:: get_write_pointer() { - if (_block != (VertexDataBlock *)NULL) { + if (_block != (VertexDataBlock *)NULL || + _resident_data == (unsigned char *)NULL) { page_in(); } + _source_file.clear(); return _resident_data; } @@ -159,3 +168,23 @@ swap(VertexDataBuffer &other) { other._size = size; other._block = block; } + +//////////////////////////////////////////////////////////////////// +// Function: VertexDataBuffer::set_file +// Access: Public +// Description: Stores a reference to a byte location on an on-disk +// file from which a copy of the data can be retrieved +// if necessary. +// +// If this is non-NULL, the data will not be written to +// the page file; instead, the specified file will be +// re-opened and re-read if necessary. +// +// If this is set to NULL, the data will be paged out +// and in normally. +//////////////////////////////////////////////////////////////////// +INLINE void VertexDataBuffer:: +set_file(VirtualFile *source_file, streampos source_pos) { + _source_file = source_file; + _source_pos = source_pos; +} diff --git a/panda/src/gobj/vertexDataBuffer.cxx b/panda/src/gobj/vertexDataBuffer.cxx index dbca0f8d7f..963c34135f 100644 --- a/panda/src/gobj/vertexDataBuffer.cxx +++ b/panda/src/gobj/vertexDataBuffer.cxx @@ -17,7 +17,9 @@ //////////////////////////////////////////////////////////////////// #include "vertexDataBuffer.h" +#include "pStatTimer.h" +PStatCollector VertexDataBuffer::_vdata_reread_pcollector("*:Vertex Data:Reread"); TypeHandle VertexDataBuffer::_type_handle; //////////////////////////////////////////////////////////////////// @@ -31,36 +33,34 @@ TypeHandle VertexDataBuffer::_type_handle; void VertexDataBuffer:: clean_realloc(size_t size) { if (size != _size) { + _source_file.clear(); + if (size == 0) { // If we're going to size 0, we don't necessarily need to page - // in first. - if (_block != (VertexDataBlock *)NULL) { - // We're currently paged out. Discard the page. - _block = NULL; - } else { - // We're currently paged in. Decrement the global total. - get_class_type().dec_memory_usage(TypeHandle::MC_array, _size); - } + // in first. But if we're paged out, discard the page. + _block = NULL; if (_resident_data != (unsigned char *)NULL) { free(_resident_data); _resident_data = NULL; + get_class_type().dec_memory_usage(TypeHandle::MC_array, _size); } _block = NULL; } else { - // Page if if we're currently paged out. - if (_block != (VertexDataBlock *)NULL) { + // Page in if we're currently paged out. + if (_block != (VertexDataBlock *)NULL || + _resident_data == (unsigned char *)NULL) { page_in(); } - get_class_type().dec_memory_usage(TypeHandle::MC_array, _size); - get_class_type().inc_memory_usage(TypeHandle::MC_array, size); - if (_resident_data == (unsigned char *)NULL) { _resident_data = (unsigned char *)malloc(size); + get_class_type().inc_memory_usage(TypeHandle::MC_array, size); } else { _resident_data = (unsigned char *)::realloc(_resident_data, size); + get_class_type().dec_memory_usage(TypeHandle::MC_array, _size); + get_class_type().inc_memory_usage(TypeHandle::MC_array, size); } nassertv(_resident_data != (unsigned char *)NULL); } @@ -87,11 +87,15 @@ page_out(VertexDataBook &book) { } nassertv(_resident_data != (unsigned char *)NULL); - _block = book.alloc(_size); - memcpy(_block->get_pointer(), _resident_data, _size); + if (_source_file == (VirtualFile *)NULL) { + // We only need to allocate a block if we don't have a source + // file. + _block = book.alloc(_size); + memcpy(_block->get_pointer(), _resident_data, _size); + } + free(_resident_data); _resident_data = NULL; - get_class_type().dec_memory_usage(TypeHandle::MC_array, _size); } @@ -104,6 +108,43 @@ page_out(VertexDataBook &book) { //////////////////////////////////////////////////////////////////// void VertexDataBuffer:: page_in() { + if (_source_file != (VirtualFile *)NULL && _resident_data == (unsigned char *)NULL) { + // Re-read the data from its original source. + PStatTimer timer(_vdata_reread_pcollector); + + _resident_data = (unsigned char *)malloc(_size); + nassertv(_resident_data != (unsigned char *)NULL); + get_class_type().inc_memory_usage(TypeHandle::MC_array, _size); + + istream *in = _source_file->open_read_file(true); + if (in == (istream *)NULL) { + gobj_cat.error() + << "Error reopening " << _source_file->get_filename() + << " to reread vertex data.\n"; + } else { + if (gobj_cat.is_debug()) { + gobj_cat.debug() + << "rereading " << _size << " bytes from " + << _source_file->get_filename() << ", position " + << _source_pos << "\n"; + } + + in->seekg(_source_pos); + + in->read((char *)_resident_data, _size); + if (in->fail() || in->eof()) { + gobj_cat.error() + << "Error rereading " << _size << " bytes from " + << _source_file->get_filename() << ", position " + << _source_pos << "\n"; + } + + VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); + vfs->close_read_file(in); + } + return; + } + if (_block == (VertexDataBlock *)NULL) { // We're already paged in. return; @@ -113,8 +154,8 @@ page_in() { _resident_data = (unsigned char *)malloc(_size); nassertv(_resident_data != (unsigned char *)NULL); + get_class_type().inc_memory_usage(TypeHandle::MC_array, _size); + memcpy(_resident_data, _block->get_pointer(), _size); _block = NULL; - - get_class_type().inc_memory_usage(TypeHandle::MC_array, _size); } diff --git a/panda/src/gobj/vertexDataBuffer.h b/panda/src/gobj/vertexDataBuffer.h index 0c1018e20a..6ec20639a3 100644 --- a/panda/src/gobj/vertexDataBuffer.h +++ b/panda/src/gobj/vertexDataBuffer.h @@ -22,6 +22,8 @@ #include "pandabase.h" #include "vertexDataBook.h" #include "pointerTo.h" +#include "virtualFile.h" +#include "pStatCollector.h" //////////////////////////////////////////////////////////////////// // Class : VertexDataBuffer @@ -49,10 +51,16 @@ public: void page_out(VertexDataBook &book); void page_in(); + INLINE void set_file(VirtualFile *source_file, streampos source_pos); + private: unsigned char *_resident_data; size_t _size; PT(VertexDataBlock) _block; + PT(VirtualFile) _source_file; + streampos _source_pos; + + static PStatCollector _vdata_reread_pcollector; public: static TypeHandle get_class_type() { diff --git a/panda/src/pgraph/config_pgraph.cxx b/panda/src/pgraph/config_pgraph.cxx index 7a63ed3c42..05170fd574 100644 --- a/panda/src/pgraph/config_pgraph.cxx +++ b/panda/src/pgraph/config_pgraph.cxx @@ -232,6 +232,14 @@ ConfigVariableInt max_collect_indices "imposing a limit on the original size of any one " "GeomPrimitive.")); +ConfigVariableBool premunge_data +("premunge-data", true, + PRC_DESC("Set this true to preconvert vertex data at model load time to " + "match the data requirements of the current GSG. For instance, " + "color columns are pre-converted to match OpenGL or DirectX " + "encoding requirements, as appropriate. When this is false, the " + "data will be munged at render time instead.")); + ConfigVariableBool polylight_info ("polylight-info", false, PRC_DESC("Set this true to view some info statements regarding the polylight. " diff --git a/panda/src/pgraph/config_pgraph.h b/panda/src/pgraph/config_pgraph.h index 5b8623011a..71e78fdfca 100644 --- a/panda/src/pgraph/config_pgraph.h +++ b/panda/src/pgraph/config_pgraph.h @@ -51,6 +51,7 @@ extern ConfigVariableBool retransform_sprites; extern ConfigVariableBool support_fade_lod; extern ConfigVariableInt max_collect_vertices; extern ConfigVariableInt max_collect_indices; +extern ConfigVariableBool premunge_data; extern ConfigVariableBool polylight_info; extern ConfigVariableDouble lod_fade_time; diff --git a/panda/src/pgraph/loader.cxx b/panda/src/pgraph/loader.cxx index bdcdc1f349..ee5774ce36 100644 --- a/panda/src/pgraph/loader.cxx +++ b/panda/src/pgraph/loader.cxx @@ -218,8 +218,10 @@ load_file(const Filename &filename, const LoaderOptions &options) const { << "Model " << path << " found in disk cache.\n"; } PT(PandaNode) result = DCAST(PandaNode, record->extract_data()); - SceneGraphReducer sgr; - sgr.premunge(result, RenderState::make_empty()); + if (premunge_data) { + SceneGraphReducer sgr; + sgr.premunge(result, RenderState::make_empty()); + } return result; } } @@ -232,9 +234,11 @@ load_file(const Filename &filename, const LoaderOptions &options) const { record->set_data(result, false); cache->store(record); } - - SceneGraphReducer sgr; - sgr.premunge(result, RenderState::make_empty()); + + if (premunge_data) { + SceneGraphReducer sgr; + sgr.premunge(result, RenderState::make_empty()); + } return result; } } diff --git a/panda/src/putil/bamReader.I b/panda/src/putil/bamReader.I index 1d8e24e31d..44365e4853 100644 --- a/panda/src/putil/bamReader.I +++ b/panda/src/putil/bamReader.I @@ -104,6 +104,35 @@ get_current_minor_ver() const { return _cur_minor; } +//////////////////////////////////////////////////////////////////// +// Function: BamReader::get_file +// Access: Public +// Description: Returns the VirtualFile that provides the source for +// these datagrams, if any, or NULL if the datagrams do +// not originate from a VirtualFile. +//////////////////////////////////////////////////////////////////// +INLINE VirtualFile *BamReader:: +get_file() { + return _source->get_file(); +} + +//////////////////////////////////////////////////////////////////// +// Function: BamReader::get_file_pos +// Access: Public +// Description: Returns the current file position within the data +// stream, if any, or 0 if the file position is not +// meaningful or cannot be determined. +// +// For BamReaders that return a meaningful file +// position, this will be pointing to the first byte +// following the datagram returned after a call to +// get_datagram(). +//////////////////////////////////////////////////////////////////// +INLINE streampos BamReader:: +get_file_pos() { + return _source->get_file_pos(); +} + //////////////////////////////////////////////////////////////////// // Function: BamReader::get_factory // Access: Public, Static @@ -129,6 +158,21 @@ create_factory() { _factory = new WritableFactory; } +//////////////////////////////////////////////////////////////////// +// Function: BamReader::get_datagram +// Access: Private +// Description: Reads a single datagram from the stream. Returns +// true on success, false on failure. +//////////////////////////////////////////////////////////////////// +INLINE bool BamReader:: +get_datagram(Datagram &datagram) { + if (_source->is_error()) { + return false; + } + + return _source->get_datagram(datagram); +} + //////////////////////////////////////////////////////////////////// // Function: parse_params // Access: Private, Static diff --git a/panda/src/putil/bamReader.cxx b/panda/src/putil/bamReader.cxx index a533b204f7..a1dd33341e 100644 --- a/panda/src/putil/bamReader.cxx +++ b/panda/src/putil/bamReader.cxx @@ -1152,18 +1152,3 @@ finalize() { fi = _finalize_list.begin(); } } - -//////////////////////////////////////////////////////////////////// -// Function: BamReader::get_datagram -// Access: Private -// Description: Reads a single datagram from the stream. Returns -// true on success, false on failure. -//////////////////////////////////////////////////////////////////// -bool BamReader:: -get_datagram(Datagram &datagram) { - if (_source->is_error()) { - return false; - } - - return _source->get_datagram(datagram); -} diff --git a/panda/src/putil/bamReader.h b/panda/src/putil/bamReader.h index 6802bcac3b..5dbd0090e5 100644 --- a/panda/src/putil/bamReader.h +++ b/panda/src/putil/bamReader.h @@ -142,6 +142,9 @@ public: TypeHandle read_handle(DatagramIterator &scan); + INLINE VirtualFile *get_file(); + INLINE streampos get_file_pos(); + public: INLINE static WritableFactory *get_factory(); private: @@ -156,7 +159,7 @@ private: bool resolve_cycler_pointers(PipelineCyclerBase *cycler, const vector_int &pointer_ids); void finalize(); - bool get_datagram(Datagram &datagram); + INLINE bool get_datagram(Datagram &datagram); private: static WritableFactory *_factory; diff --git a/panda/src/putil/datagramInputFile.cxx b/panda/src/putil/datagramInputFile.cxx index a2c1b696f9..b32999178d 100644 --- a/panda/src/putil/datagramInputFile.cxx +++ b/panda/src/putil/datagramInputFile.cxx @@ -39,12 +39,12 @@ open(Filename filename) { filename.set_binary(); VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); - PT(VirtualFile) file = vfs->get_file(filename); - if (file == (VirtualFile *)NULL) { + _vfile = vfs->get_file(filename); + if (_vfile == (VirtualFile *)NULL) { // No such file. return false; } - _in = file->open_read_file(true); + _in = _vfile->open_read_file(true); _owns_in = (_in != (istream *)NULL); return _owns_in && !_in->fail(); } @@ -76,6 +76,7 @@ open(istream &in) { //////////////////////////////////////////////////////////////////// void DatagramInputFile:: close() { + _vfile.clear(); _in_file.close(); if (_owns_in) { VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); @@ -141,18 +142,16 @@ get_datagram(Datagram &data) { } // Now, read the datagram itself. - char *buffer = new char[num_bytes]; + char *buffer = (char *)alloca(num_bytes); nassertr(buffer != (char *)NULL, false); _in->read(buffer, num_bytes); if (_in->fail() || _in->eof()) { _error = true; - delete[] buffer; return false; } data = Datagram(buffer, num_bytes); - delete[] buffer; return true; } @@ -186,3 +185,35 @@ is_error() { } return _error; } + +//////////////////////////////////////////////////////////////////// +// Function: DatagramInputFile::get_file +// Access: Public, Virtual +// Description: Returns the VirtualFile that provides the source for +// these datagrams, if any, or NULL if the datagrams do +// not originate from a VirtualFile. +//////////////////////////////////////////////////////////////////// +VirtualFile *DatagramInputFile:: +get_file() { + return _vfile; +} + +//////////////////////////////////////////////////////////////////// +// Function: DatagramInputFile::get_file_pos +// Access: Public, Virtual +// Description: Returns the current file position within the data +// stream, if any, or 0 if the file position is not +// meaningful or cannot be determined. +// +// For DatagramInputFiles that return a meaningful file +// position, this will be pointing to the first byte +// following the datagram returned after a call to +// get_datagram(). +//////////////////////////////////////////////////////////////////// +streampos DatagramInputFile:: +get_file_pos() { + if (_in == (istream *)NULL) { + return 0; + } + return _in->tellg(); +} diff --git a/panda/src/putil/datagramInputFile.h b/panda/src/putil/datagramInputFile.h index bc429cd418..920550f045 100644 --- a/panda/src/putil/datagramInputFile.h +++ b/panda/src/putil/datagramInputFile.h @@ -23,6 +23,7 @@ #include "datagramGenerator.h" #include "filename.h" +#include "virtualFile.h" //////////////////////////////////////////////////////////////////// // Class : DatagramInputFile @@ -44,9 +45,13 @@ public: virtual bool is_eof(); virtual bool is_error(); + virtual VirtualFile *get_file(); + virtual streampos get_file_pos(); + private: bool _read_first_datagram; bool _error; + PT(VirtualFile) _vfile; ifstream _in_file; istream *_in; bool _owns_in;