vertex-data-allow-reread

panda/src/express/datagramGenerator.cxx

@@ -23,10 +23,39 @@
 
 ////////////////////////////////////////////////////////////////////
 //     Function: DatagramGenerator::Destructor
-//       Access: Public, Virtual>
+//       Access: Public, Virtual
 //  Description: Does nothing since this is class is just
 //               the definition of an interface
 ////////////////////////////////////////////////////////////////////
 DatagramGenerator::
 ~DatagramGenerator() {
 }
+
+////////////////////////////////////////////////////////////////////
+//     Function: DatagramGenerator::get_file
+//       Access: Public, Virtual
+//  Description: Returns the VirtualFile that provides the source for
+//               these datagrams, if any, or NULL if the datagrams do
+//               not originate from a VirtualFile.
+////////////////////////////////////////////////////////////////////
+VirtualFile *DatagramGenerator::
+get_file() {
+  return NULL;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: DatagramGenerator::get_file_pos
+//       Access: Public, Virtual
+//  Description: Returns the current file position within the data
+//               stream, if any, or 0 if the file position is not
+//               meaningful or cannot be determined.
+//
+//               For DatagramGenerators that return a meaningful file
+//               position, this will be pointing to the first byte
+//               following the datagram returned after a call to
+//               get_datagram().
+////////////////////////////////////////////////////////////////////
+streampos DatagramGenerator::
+get_file_pos() {
+  return 0;
+}

panda/src/express/datagramGenerator.h

@@ -23,6 +23,8 @@
 
 #include "datagram.h"
 
+class VirtualFile;
+
 ////////////////////////////////////////////////////////////////////
 //       Class : DatagramGenerator
 // Description : This class defines the abstract interace to any
@@ -37,6 +39,9 @@ public:
   virtual bool get_datagram(Datagram &data) = 0;
   virtual bool is_eof() = 0;
   virtual bool is_error() = 0;
+
+  virtual VirtualFile *get_file();
+  virtual streampos get_file_pos();
 };
 
 #include "datagramGenerator.I"

panda/src/gobj/config_gobj.cxx

@@ -300,6 +300,16 @@ ConfigVariableInt vertex_data_small_size
           "is deemed too small to pay the overhead of paging it in and out, "
           "and it is permanently retained resident."));
 
+ConfigVariableBool vertex_data_allow_reread
+("vertex-data-allow-reread", false,
+ PRC_DESC("Set this true to allow a GeomVertexArrayData to re-read itself "
+          "from the original bam file it was read from, if it is evicted "
+          "and needs to be reloaded.  This avoids the need to write the "
+          "data to the page file, but it requires that bam files are not "
+          "modified during program execution.  Set it false to prevent "
+          "this, so that the vertex data will need to be written to a "
+          "page file when it is evicted."));
+
 
 
 ConfigureFn(config_gobj) {

panda/src/gobj/config_gobj.h

@@ -74,6 +74,7 @@ extern EXPCL_PANDA ConfigVariableDouble default_keystone;
 extern EXPCL_PANDA ConfigVariableFilename vertex_save_file_directory;
 extern EXPCL_PANDA ConfigVariableString vertex_save_file_prefix;
 extern EXPCL_PANDA ConfigVariableInt vertex_data_small_size;
+extern EXPCL_PANDA ConfigVariableBool vertex_data_allow_reread;
 
 #endif
 

panda/src/gobj/geomVertexArrayData.I

@@ -488,7 +488,7 @@ get_subdata(size_t start, size_t size) const {
 ////////////////////////////////////////////////////////////////////
 void GeomVertexArrayDataHandle::
 check_resident() const {
-  _object->mark_used_lru();
+  _object->set_lru_size(_object->get_lru_size());
 }
 
 INLINE ostream &

panda/src/gobj/geomVertexArrayData.cxx

@@ -600,7 +600,13 @@ fillin(DatagramIterator &scan, BamReader *manager, void *extra_data) {
   } else {
     // Now, the array data is just stored directly.
     size_t size = scan.get_uint32();
+
     _buffer.unclean_realloc(size);
+    if (vertex_data_allow_reread) {
+      streampos source_pos = manager->get_file_pos() + (streampos)scan.get_current_index() - (streampos)scan.get_datagram().get_length();
+      _buffer.set_file(manager->get_file(), source_pos);
+    }
+
     const unsigned char *source_data = 
       (const unsigned char *)scan.get_datagram().get_data();
     memcpy(_buffer.get_write_pointer(), source_data + scan.get_current_index(), size);
@@ -609,6 +615,8 @@ fillin(DatagramIterator &scan, BamReader *manager, void *extra_data) {
 
   if (manager->get_file_endian() != BE_native) {
     // For non-native endian files, we have to convert the data.  
+    _buffer.set_file(NULL, 0);
+
     if (array_data->_array_format == (GeomVertexArrayFormat *)NULL) {
       // But we can't do that until we've completed the _array_format
       // pointer, which tells us how to convert it.

panda/src/gobj/vertexDataBuffer.I

@@ -64,6 +64,8 @@ INLINE void VertexDataBuffer::
 operator = (const VertexDataBuffer &copy) {
   unclean_realloc(copy.get_size());
   memcpy(_resident_data, copy.get_read_pointer(), _size);
+  _source_file = copy._source_file;
+  _source_pos = copy._source_pos;
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -89,6 +91,11 @@ get_read_pointer() const {
     // pointer, which will force its page to resident status.
     return _block->get_pointer();
   }
+  if (_resident_data == (unsigned char *)NULL && !_source_file.is_null()) {
+    // If we need to re-read the original source, do so.
+    ((VertexDataBuffer *)this)->page_in();
+  }
+
   return _resident_data;
 }
 
@@ -99,9 +106,11 @@ get_read_pointer() const {
 ////////////////////////////////////////////////////////////////////
 INLINE unsigned char *VertexDataBuffer::
 get_write_pointer() {
-  if (_block != (VertexDataBlock *)NULL) {
+  if (_block != (VertexDataBlock *)NULL || 
+      _resident_data == (unsigned char *)NULL) {
     page_in();
   }
+  _source_file.clear();
   return _resident_data;
 }
 
@@ -159,3 +168,23 @@ swap(VertexDataBuffer &other) {
   other._size = size;
   other._block = block;
 }
+
+////////////////////////////////////////////////////////////////////
+//     Function: VertexDataBuffer::set_file
+//       Access: Public
+//  Description: Stores a reference to a byte location on an on-disk
+//               file from which a copy of the data can be retrieved
+//               if necessary.  
+//
+//               If this is non-NULL, the data will not be written to
+//               the page file; instead, the specified file will be
+//               re-opened and re-read if necessary.
+//
+//               If this is set to NULL, the data will be paged out
+//               and in normally.
+////////////////////////////////////////////////////////////////////
+INLINE void VertexDataBuffer::
+set_file(VirtualFile *source_file, streampos source_pos) {
+  _source_file = source_file;
+  _source_pos = source_pos;
+}

panda/src/gobj/vertexDataBuffer.cxx

@@ -17,7 +17,9 @@
 ////////////////////////////////////////////////////////////////////
 
 #include "vertexDataBuffer.h"
+#include "pStatTimer.h"
 
+PStatCollector VertexDataBuffer::_vdata_reread_pcollector("*:Vertex Data:Reread");
 TypeHandle VertexDataBuffer::_type_handle;
 
 ////////////////////////////////////////////////////////////////////
@@ -31,36 +33,34 @@ TypeHandle VertexDataBuffer::_type_handle;
 void VertexDataBuffer::
 clean_realloc(size_t size) {
   if (size != _size) {
+    _source_file.clear();
+
     if (size == 0) {
       // If we're going to size 0, we don't necessarily need to page
-      // in first.
-      if (_block != (VertexDataBlock *)NULL) {
-        // We're currently paged out.  Discard the page.
-        _block = NULL;
-      } else {
-        // We're currently paged in.  Decrement the global total.
-        get_class_type().dec_memory_usage(TypeHandle::MC_array, _size);
-      }
+      // in first.  But if we're paged out, discard the page.
+      _block = NULL;
         
       if (_resident_data != (unsigned char *)NULL) {
         free(_resident_data);
         _resident_data = NULL;
+        get_class_type().dec_memory_usage(TypeHandle::MC_array, _size);
       }
       _block = NULL;
       
     } else {
-      // Page if if we're currently paged out.
-      if (_block != (VertexDataBlock *)NULL) {
+      // Page in if we're currently paged out.
+      if (_block != (VertexDataBlock *)NULL || 
+          _resident_data == (unsigned char *)NULL) {
         page_in();
       }
       
-      get_class_type().dec_memory_usage(TypeHandle::MC_array, _size);
-      get_class_type().inc_memory_usage(TypeHandle::MC_array, size);
-    
       if (_resident_data == (unsigned char *)NULL) {
         _resident_data = (unsigned char *)malloc(size);
+        get_class_type().inc_memory_usage(TypeHandle::MC_array, size);
       } else {
         _resident_data = (unsigned char *)::realloc(_resident_data, size);
+        get_class_type().dec_memory_usage(TypeHandle::MC_array, _size);
+        get_class_type().inc_memory_usage(TypeHandle::MC_array, size);
       }
       nassertv(_resident_data != (unsigned char *)NULL);
     }
@@ -87,11 +87,15 @@ page_out(VertexDataBook &book) {
   }
   nassertv(_resident_data != (unsigned char *)NULL);
 
-  _block = book.alloc(_size);
-  memcpy(_block->get_pointer(), _resident_data, _size);
+  if (_source_file == (VirtualFile *)NULL) {
+    // We only need to allocate a block if we don't have a source
+    // file.
+    _block = book.alloc(_size);
+    memcpy(_block->get_pointer(), _resident_data, _size);
+  }
+
   free(_resident_data);
   _resident_data = NULL;
-
   get_class_type().dec_memory_usage(TypeHandle::MC_array, _size);
 }
 
@@ -104,6 +108,43 @@ page_out(VertexDataBook &book) {
 ////////////////////////////////////////////////////////////////////
 void VertexDataBuffer::
 page_in() {
+  if (_source_file != (VirtualFile *)NULL && _resident_data == (unsigned char *)NULL) {
+    // Re-read the data from its original source.
+    PStatTimer timer(_vdata_reread_pcollector);
+
+    _resident_data = (unsigned char *)malloc(_size);
+    nassertv(_resident_data != (unsigned char *)NULL);
+    get_class_type().inc_memory_usage(TypeHandle::MC_array, _size);
+
+    istream *in = _source_file->open_read_file(true);
+    if (in == (istream *)NULL) {
+      gobj_cat.error()
+        << "Error reopening " << _source_file->get_filename()
+        << " to reread vertex data.\n";
+    } else {
+      if (gobj_cat.is_debug()) {
+        gobj_cat.debug()
+          << "rereading " << _size << " bytes from " 
+          << _source_file->get_filename() << ", position " 
+          << _source_pos << "\n";
+      }
+
+      in->seekg(_source_pos);
+
+      in->read((char *)_resident_data, _size);
+      if (in->fail() || in->eof()) {
+        gobj_cat.error()
+          << "Error rereading " << _size << " bytes from " 
+          << _source_file->get_filename() << ", position " 
+          << _source_pos << "\n";
+      }
+
+      VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
+      vfs->close_read_file(in);
+    }
+    return;
+  }
+
   if (_block == (VertexDataBlock *)NULL) {
     // We're already paged in.
     return;
@@ -113,8 +154,8 @@ page_in() {
 
   _resident_data = (unsigned char *)malloc(_size);
   nassertv(_resident_data != (unsigned char *)NULL);
+  get_class_type().inc_memory_usage(TypeHandle::MC_array, _size);
+
   memcpy(_resident_data, _block->get_pointer(), _size);
   _block = NULL;
-  
-  get_class_type().inc_memory_usage(TypeHandle::MC_array, _size);
 }

panda/src/gobj/vertexDataBuffer.h

@@ -22,6 +22,8 @@
 #include "pandabase.h"
 #include "vertexDataBook.h"
 #include "pointerTo.h"
+#include "virtualFile.h"
+#include "pStatCollector.h"
 
 ////////////////////////////////////////////////////////////////////
 //       Class : VertexDataBuffer
@@ -49,10 +51,16 @@ public:
   void page_out(VertexDataBook &book);
   void page_in();
 
+  INLINE void set_file(VirtualFile *source_file, streampos source_pos);
+
 private:
   unsigned char *_resident_data;
   size_t _size;
   PT(VertexDataBlock) _block;
+  PT(VirtualFile) _source_file;
+  streampos _source_pos;
+
+  static PStatCollector _vdata_reread_pcollector;
 
 public:
   static TypeHandle get_class_type() {

panda/src/pgraph/config_pgraph.cxx

@@ -232,6 +232,14 @@ ConfigVariableInt max_collect_indices
           "imposing a limit on the original size of any one "
           "GeomPrimitive."));
 
+ConfigVariableBool premunge_data
+("premunge-data", true,
+ PRC_DESC("Set this true to preconvert vertex data at model load time to "
+          "match the data requirements of the current GSG.  For instance, "
+          "color columns are pre-converted to match OpenGL or DirectX "
+          "encoding requirements, as appropriate.  When this is false, the "
+          "data will be munged at render time instead."));
+
 ConfigVariableBool polylight_info
 ("polylight-info", false,
  PRC_DESC("Set this true to view some info statements regarding the polylight. "

panda/src/pgraph/config_pgraph.h

@@ -51,6 +51,7 @@ extern ConfigVariableBool retransform_sprites;
 extern ConfigVariableBool support_fade_lod;
 extern ConfigVariableInt max_collect_vertices;
 extern ConfigVariableInt max_collect_indices;
+extern ConfigVariableBool premunge_data;
 
 extern ConfigVariableBool polylight_info;
 extern ConfigVariableDouble lod_fade_time;

panda/src/pgraph/loader.cxx

@@ -218,8 +218,10 @@ load_file(const Filename &filename, const LoaderOptions &options) const {
               << "Model " << path << " found in disk cache.\n";
           }
           PT(PandaNode) result = DCAST(PandaNode, record->extract_data());
-          SceneGraphReducer sgr;
-          sgr.premunge(result, RenderState::make_empty());
+          if (premunge_data) {
+            SceneGraphReducer sgr;
+            sgr.premunge(result, RenderState::make_empty());
+          }
           return result;
         }
       }
@@ -232,9 +234,11 @@ load_file(const Filename &filename, const LoaderOptions &options) const {
           record->set_data(result, false);
           cache->store(record);
         }
-        
-        SceneGraphReducer sgr;
-        sgr.premunge(result, RenderState::make_empty());
+
+        if (premunge_data) {
+          SceneGraphReducer sgr;
+          sgr.premunge(result, RenderState::make_empty());
+        }
         return result;
       }
     }

panda/src/putil/bamReader.I

@@ -104,6 +104,35 @@ get_current_minor_ver() const {
   return _cur_minor;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: BamReader::get_file
+//       Access: Public
+//  Description: Returns the VirtualFile that provides the source for
+//               these datagrams, if any, or NULL if the datagrams do
+//               not originate from a VirtualFile.
+////////////////////////////////////////////////////////////////////
+INLINE VirtualFile *BamReader::
+get_file() {
+  return _source->get_file();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: BamReader::get_file_pos
+//       Access: Public
+//  Description: Returns the current file position within the data
+//               stream, if any, or 0 if the file position is not
+//               meaningful or cannot be determined.
+//
+//               For BamReaders that return a meaningful file
+//               position, this will be pointing to the first byte
+//               following the datagram returned after a call to
+//               get_datagram().
+////////////////////////////////////////////////////////////////////
+INLINE streampos BamReader::
+get_file_pos() {
+  return _source->get_file_pos();
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: BamReader::get_factory
 //       Access: Public, Static
@@ -129,6 +158,21 @@ create_factory() {
   _factory = new WritableFactory;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: BamReader::get_datagram
+//       Access: Private
+//  Description: Reads a single datagram from the stream.  Returns
+//               true on success, false on failure.
+////////////////////////////////////////////////////////////////////
+INLINE bool BamReader::
+get_datagram(Datagram &datagram) {
+  if (_source->is_error()) {
+    return false;
+  }
+
+  return _source->get_datagram(datagram);
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: parse_params
 //       Access: Private, Static

panda/src/putil/bamReader.cxx

@@ -1152,18 +1152,3 @@ finalize() {
     fi = _finalize_list.begin();
   }
 }
-
-////////////////////////////////////////////////////////////////////
-//     Function: BamReader::get_datagram
-//       Access: Private
-//  Description: Reads a single datagram from the stream.  Returns
-//               true on success, false on failure.
-////////////////////////////////////////////////////////////////////
-bool BamReader::
-get_datagram(Datagram &datagram) {
-  if (_source->is_error()) {
-    return false;
-  }
-
-  return _source->get_datagram(datagram);
-}

panda/src/putil/bamReader.h

@@ -142,6 +142,9 @@ public:
 
   TypeHandle read_handle(DatagramIterator &scan);
 
+  INLINE VirtualFile *get_file();
+  INLINE streampos get_file_pos();
+
 public:
   INLINE static WritableFactory *get_factory();
 private:
@@ -156,7 +159,7 @@ private:
   bool resolve_cycler_pointers(PipelineCyclerBase *cycler, const vector_int &pointer_ids);
   void finalize();
 
-  bool get_datagram(Datagram &datagram);
+  INLINE bool get_datagram(Datagram &datagram);
 
 private:
   static WritableFactory *_factory;

panda/src/putil/datagramInputFile.cxx

@@ -39,12 +39,12 @@ open(Filename filename) {
   filename.set_binary();
 
   VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
-  PT(VirtualFile) file = vfs->get_file(filename);
-  if (file == (VirtualFile *)NULL) {
+  _vfile = vfs->get_file(filename);
+  if (_vfile == (VirtualFile *)NULL) {
     // No such file.
     return false;
   }
-  _in = file->open_read_file(true);
+  _in = _vfile->open_read_file(true);
   _owns_in = (_in != (istream *)NULL);
   return _owns_in && !_in->fail();
 }
@@ -76,6 +76,7 @@ open(istream &in) {
 ////////////////////////////////////////////////////////////////////
 void DatagramInputFile::
 close() {
+  _vfile.clear();
   _in_file.close();
   if (_owns_in) {
     VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
@@ -141,18 +142,16 @@ get_datagram(Datagram &data) {
   }
 
   // Now, read the datagram itself.
-  char *buffer = new char[num_bytes];
+  char *buffer = (char *)alloca(num_bytes);
   nassertr(buffer != (char *)NULL, false);
 
   _in->read(buffer, num_bytes);
   if (_in->fail() || _in->eof()) {
     _error = true;
-    delete[] buffer;
     return false;
   }
 
   data = Datagram(buffer, num_bytes);
-  delete[] buffer;
 
   return true;
 }
@@ -186,3 +185,35 @@ is_error() {
   }
   return _error;
 }
+
+////////////////////////////////////////////////////////////////////
+//     Function: DatagramInputFile::get_file
+//       Access: Public, Virtual
+//  Description: Returns the VirtualFile that provides the source for
+//               these datagrams, if any, or NULL if the datagrams do
+//               not originate from a VirtualFile.
+////////////////////////////////////////////////////////////////////
+VirtualFile *DatagramInputFile::
+get_file() {
+  return _vfile;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: DatagramInputFile::get_file_pos
+//       Access: Public, Virtual
+//  Description: Returns the current file position within the data
+//               stream, if any, or 0 if the file position is not
+//               meaningful or cannot be determined.
+//
+//               For DatagramInputFiles that return a meaningful file
+//               position, this will be pointing to the first byte
+//               following the datagram returned after a call to
+//               get_datagram().
+////////////////////////////////////////////////////////////////////
+streampos DatagramInputFile::
+get_file_pos() {
+  if (_in == (istream *)NULL) {
+    return 0;
+  }
+  return _in->tellg();
+}

panda/src/putil/datagramInputFile.h

@@ -23,6 +23,7 @@
 
 #include "datagramGenerator.h"
 #include "filename.h"
+#include "virtualFile.h"
 
 ////////////////////////////////////////////////////////////////////
 //       Class : DatagramInputFile
@@ -44,9 +45,13 @@ public:
   virtual bool is_eof();
   virtual bool is_error();
 
+  virtual VirtualFile *get_file();
+  virtual streampos get_file_pos();
+
 private:
   bool _read_first_datagram;
   bool _error;
+  PT(VirtualFile) _vfile;
   ifstream _in_file;
   istream *_in;
   bool _owns_in;